diff --git a/.gitea/workflows/_deprecated-feedser-ci.yml.disabled b/.gitea/workflows/_deprecated-concelier-ci.yml.disabled similarity index 100% rename from .gitea/workflows/_deprecated-feedser-ci.yml.disabled rename to .gitea/workflows/_deprecated-concelier-ci.yml.disabled diff --git a/.gitea/workflows/_deprecated-feedser-tests.yml.disabled b/.gitea/workflows/_deprecated-concelier-tests.yml.disabled similarity index 100% rename from .gitea/workflows/_deprecated-feedser-tests.yml.disabled rename to .gitea/workflows/_deprecated-concelier-tests.yml.disabled diff --git a/.gitea/workflows/build-test-deploy.yml b/.gitea/workflows/build-test-deploy.yml index 09b4287a..e5b90aff 100644 --- a/.gitea/workflows/build-test-deploy.yml +++ b/.gitea/workflows/build-test-deploy.yml @@ -35,7 +35,22 @@ env: CI_CACHE_ROOT: /data/.cache/stella-ops/feedser RUNNER_TOOL_CACHE: /toolcache -jobs: +jobs: + profile-validation: + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Helm + run: | + curl -fsSL https://get.helm.sh/helm-v3.16.0-linux-amd64.tar.gz -o /tmp/helm.tgz + tar -xzf /tmp/helm.tgz -C /tmp + sudo install -m 0755 /tmp/linux-amd64/helm /usr/local/bin/helm + + - name: Validate deployment profiles + run: ./deploy/tools/validate-profiles.sh + build-test: runs-on: ubuntu-22.04 environment: ${{ github.event_name == 'pull_request' && 'preview' || 'staging' }} @@ -61,15 +76,124 @@ jobs: - name: Build solution (warnings as errors) run: dotnet build src/StellaOps.Feedser.sln --configuration $BUILD_CONFIGURATION --no-restore -warnaserror - - name: Run unit and integration tests - run: | - mkdir -p "$TEST_RESULTS_DIR" - dotnet test src/StellaOps.Feedser.sln \ - --configuration $BUILD_CONFIGURATION \ - --no-build \ - --logger "trx;LogFileName=stellaops-feedser-tests.trx" \ - --results-directory "$TEST_RESULTS_DIR" - + - name: Run unit and integration tests + run: | + mkdir -p "$TEST_RESULTS_DIR" + dotnet test src/StellaOps.Feedser.sln \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --logger "trx;LogFileName=stellaops-feedser-tests.trx" \ + --results-directory "$TEST_RESULTS_DIR" + + - name: Build scanner language analyzer projects + run: | + dotnet restore src/StellaOps.sln + for project in \ + src/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.Java/StellaOps.Scanner.Analyzers.Lang.Java.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.Go/StellaOps.Scanner.Analyzers.Lang.Go.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.DotNet/StellaOps.Scanner.Analyzers.Lang.DotNet.csproj \ + src/StellaOps.Scanner.Analyzers.Lang.Rust/StellaOps.Scanner.Analyzers.Lang.Rust.csproj + do + dotnet build "$project" --configuration $BUILD_CONFIGURATION --no-restore -warnaserror + done + + - name: Run scanner language analyzer tests + run: | + dotnet test src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj \ + --configuration $BUILD_CONFIGURATION \ + --no-build \ + --logger "trx;LogFileName=stellaops-scanner-lang-tests.trx" \ + --results-directory "$TEST_RESULTS_DIR" + + - name: Publish BuildX SBOM generator + run: | + dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj \ + --configuration $BUILD_CONFIGURATION \ + --output out/buildx + + - name: Verify BuildX descriptor determinism + run: | + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll handshake \ + --manifest out/buildx \ + --cas out/cas + + cat <<'JSON' > out/buildx-sbom.cdx.json +{"bomFormat":"CycloneDX","specVersion":"1.5"} +JSON + + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image sha256:5c2c5bfe0d4d77f1a0f9866fd415dd8da5b62af05d7c3d4b53f28de3ebef0101 \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + --artifact-type application/vnd.stellaops.sbom.layer+json \ + --sbom-format cyclonedx-json \ + --sbom-kind inventory \ + --repository ${{ github.repository }} \ + --build-ref ${{ github.sha }} \ + > out/buildx-descriptor.json + + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image sha256:5c2c5bfe0d4d77f1a0f9866fd415dd8da5b62af05d7c3d4b53f28de3ebef0101 \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + --artifact-type application/vnd.stellaops.sbom.layer+json \ + --sbom-format cyclonedx-json \ + --sbom-kind inventory \ + --repository ${{ github.repository }} \ + --build-ref ${{ github.sha }} \ + > out/buildx-descriptor-repeat.json + + python - <<'PY' +import json, sys +from pathlib import Path + +def normalize(path: str) -> dict: + data = json.loads(Path(path).read_text(encoding='utf-8')) + data.pop('generatedAt', None) + return data + +baseline = normalize('out/buildx-descriptor.json') +repeat = normalize('out/buildx-descriptor-repeat.json') + +if baseline != repeat: + sys.exit('BuildX descriptor output changed between runs.') +PY + + - name: Upload BuildX determinism artifacts + uses: actions/upload-artifact@v4 + with: + name: buildx-determinism + path: | + out/buildx-descriptor.json + out/buildx-descriptor-repeat.json + out/buildx-sbom.cdx.json + if-no-files-found: error + retention-days: 7 + + - name: Package OS analyzer plug-ins + run: | + if [ ! -d "plugins/scanner/analyzers/os" ]; then + echo "OS analyzer plug-in directory not found" >&2 + exit 1 + fi + + mkdir -p artifacts/plugins/os + tar -czf artifacts/plugins/os/stellaops-scanner-os-analyzers.tar.gz -C plugins/scanner/analyzers/os . + sha256sum artifacts/plugins/os/stellaops-scanner-os-analyzers.tar.gz > artifacts/plugins/os/stellaops-scanner-os-analyzers.tar.gz.sha256 + + - name: Upload OS analyzer plug-ins + uses: actions/upload-artifact@v4 + with: + name: scanner-os-analyzers + path: artifacts/plugins/os + if-no-files-found: error + retention-days: 7 + - name: Publish Feedser web service run: | mkdir -p "$PUBLISH_DIR" @@ -142,7 +266,7 @@ jobs: runs-on: ubuntu-22.04 env: DOCS_OUTPUT_DIR: ${{ github.workspace }}/artifacts/docs-site - steps: + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -164,18 +288,100 @@ jobs: uses: actions/upload-artifact@v4 with: name: feedser-docs-site - path: ${{ env.DOCS_OUTPUT_DIR }} - if-no-files-found: error - retention-days: 7 - - deploy: - runs-on: ubuntu-22.04 - needs: [build-test, docs] - if: >- - needs.build-test.result == 'success' && - needs.docs.result == 'success' && - ( - (github.event_name == 'push' && github.ref == 'refs/heads/main') || + path: ${{ env.DOCS_OUTPUT_DIR }} + if-no-files-found: error + retention-days: 7 + + scanner-perf: + runs-on: ubuntu-22.04 + needs: build-test + env: + BENCH_DIR: bench/Scanner.Analyzers + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Run analyzer microbench + working-directory: ${{ env.BENCH_DIR }} + run: | + node run-bench.js \ + --repo-root "${{ github.workspace }}" \ + --out latest.csv \ + --threshold-ms 5000 + + - name: Compare against baseline + working-directory: ${{ env.BENCH_DIR }} + run: | + node - <<'NODE' + const fs = require('fs'); + const path = require('path'); + + function parseCsv(file) { + const rows = fs.readFileSync(file, 'utf8').trim().split(/\r?\n/); + rows.shift(); + const data = {}; + for (const row of rows) { + const [id, iterations, sampleCount, mean, p95, max] = row.split(','); + data[id] = { + iterations: Number(iterations), + sampleCount: Number(sampleCount), + mean: Number(mean), + p95: Number(p95), + max: Number(max), + }; + } + return data; + } + + const baseline = parseCsv('baseline.csv'); + const latest = parseCsv('latest.csv'); + const allowedMultiplier = 1.20; + const regressions = []; + + for (const [id, baseMetrics] of Object.entries(baseline)) { + const current = latest[id]; + if (!current) { + regressions.push(`Scenario ${id} missing from latest run`); + continue; + } + if (current.mean > baseMetrics.mean * allowedMultiplier) { + regressions.push(`Scenario ${id} mean ${current.mean.toFixed(2)}ms exceeded baseline ${baseMetrics.mean.toFixed(2)}ms by >20%`); + } + if (current.max > baseMetrics.max * allowedMultiplier) { + regressions.push(`Scenario ${id} max ${current.max.toFixed(2)}ms exceeded baseline ${baseMetrics.max.toFixed(2)}ms by >20%`); + } + } + + if (regressions.length > 0) { + console.error('Performance regression detected:'); + for (const msg of regressions) { + console.error(` - ${msg}`); + } + process.exit(1); + } + NODE + + - name: Upload bench report + uses: actions/upload-artifact@v4 + with: + name: scanner-analyzers-bench + path: ${{ env.BENCH_DIR }}/latest.csv + retention-days: 7 + + deploy: + runs-on: ubuntu-22.04 + needs: [build-test, docs, scanner-perf] + if: >- + needs.build-test.result == 'success' && + needs.docs.result == 'success' && + needs.scanner-perf.result == 'success' && + ( + (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch' ) environment: staging diff --git a/.gitea/workflows/docs.yml b/.gitea/workflows/docs.yml index c7e19ad1..3812c428 100755 --- a/.gitea/workflows/docs.yml +++ b/.gitea/workflows/docs.yml @@ -34,18 +34,34 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} - - name: Install markdown linters - run: | - npm install markdown-link-check remark-cli remark-preset-lint-recommended + - name: Install documentation toolchain + run: | + npm install --no-save markdown-link-check remark-cli remark-preset-lint-recommended ajv ajv-cli ajv-formats - name: Link check run: | find docs -name '*.md' -print0 | \ xargs -0 -n1 -I{} npx markdown-link-check --quiet '{}' - - name: Remark lint - run: | - npx remark docs -qf + - name: Remark lint + run: | + npx remark docs -qf + + - name: Validate event schemas + run: | + set -euo pipefail + for schema in docs/events/*.json; do + npx ajv compile -c ajv-formats -s "$schema" + done + for sample in docs/events/samples/*.json; do + schema_name=$(basename "$sample" .sample.json) + schema_path="docs/events/${schema_name}.json" + if [ ! -f "$schema_path" ]; then + echo "Missing schema for sample ${sample}" >&2 + exit 1 + fi + npx ajv validate -c ajv-formats -s "$schema_path" -d "$sample" + done - name: Setup Python uses: actions/setup-python@v5 diff --git a/AGENTS.md b/AGENTS.md index 1b530c01..e61f0e77 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Licence audit identifies potential conflicts, especially copyleft obligations. Misconfiguration checks detect unsafe Dockerfile patterns (root user, latest tags, permissive modes). Provenance features include in-toto/SLSA attestations signed with cosign for supply-chain trust. -| Guiding principle | What it means for Feedser | +| Guiding principle | What it means for Concelier | |-------------------|---------------------------| | **SBOM-first ingest** | Prefer signed SBOMs or reproducible layer diffs before falling back to raw scraping; connectors treat source docs as provenance, never as mutable truth. | | **Deterministic outputs** | Same inputs yield identical canonical advisories and exported JSON/Trivy DB artefacts; merge hashes and export manifests are reproducible across machines. | @@ -41,31 +41,14 @@ All modules are contained by one or more projects. Each project goes in its dedi # 4) Modules StellaOps is contained by different modules installable via docker containers -- Feedser. Responsible for aggregation and delivery of vulnerability database +- Concelier. Responsible for aggregation and delivery of vulnerability database - Cli. Command line tool to unlock full potential - request database operations, install scanner, request scan, configure backend - Backend. Configures and Manages scans - UI. UI to access the backend (and scanners) - Agent. Installable daemon that does the scanning - Zastava. Realtime monitor for allowed (verified) installations. -## 4.1) Feedser -It is webservice based module that is responsible for aggregating vulnerabilities information from various sources, parsing and normalizing them into a canonical shape, merging and deduplicating the results in one place, with export capabilities to Json and TrivyDb. It supports init and resume for all of the sources, parse/normalize and merge/deduplication operations, plus export. Export supports delta exports—similarly to full and incremential database backups. - -### 4.1.1) Usage -It supports operations to be started by cmd line: -# stella db [fetch|merge|export] [init|resume ] -or -api available on https://db.stella-ops.org - -### 4.1.2) Data flow (end‑to‑end) - -1. **Fetch**: connectors request source windows with retries/backoff, persist raw documents with SHA256/ETag metadata. -2. **Parse & Normalize**: validate to DTOs (schema-checked), quarantine failures, normalize to canonical advisories (aliases, affected ranges with NEVRA/EVR/SemVer, references, provenance). -3. **Merge & Deduplicate**: enforce precedence, build/maintain alias graphs, compute deterministic hashes, and eliminate duplicates before persisting to MongoDB. -4. **Export**: JSON tree and/or Trivy DB; package and (optionally) push; write export state. - -### 4.1.3) Architecture -For more information of the architecture see `./docs/ARCHITECTURE_FEEDSER.md`. +For more information of the architecture see `./docs/*ARCHITECTURE*.md` files. --- @@ -118,11 +101,11 @@ You main characteristics: - **Directory ownership**: Each agent works **only inside its module directory**. Cross‑module edits require a brief handshake in issues/PR description. - **Scoping**: Use each module’s `AGENTS.md` and `TASKS.md` to plan; autonomous agents must read `src/AGENTS.md` and the module docs before acting. - **Determinism**: Sort keys, normalize timestamps to UTC ISO‑8601, avoid non‑deterministic data in exports and tests. -- **Status tracking**: Update your module’s `TASKS.md` as you progress (TODO → DOING → DONE/BLOCKED). Before starting of actual work - ensure you have set the task to DOING. When complete or stop update the status in corresponding TASKS.md or in ./SPRINTS.md file. +- **Status tracking**: Update your module’s `TASKS.md` as you progress (TODO → DOING → DONE/BLOCKED). Before starting of actual work - ensure you have set the task to DOING. When complete or stop update the status in corresponding TASKS.md and in ./SPRINTS.md and ./EXECPLAN.md file. - **Coordination**: In case task is discovered as blocked on other team or task, according TASKS.md files that dependency is on needs to be changed by adding new tasks describing the requirement. the current task must be updated as completed. In case task changes, scope or requirements or rules - other documentations needs be updated accordingly. -- **Sprint synchronization**: When given task seek for relevant directory to work on from SPRINTS.md. Confirm its state on both SPRINTS.md and the relevant TODOS.md file. Always check the AGENTS.md in the relevant TODOS.md directory. +- **Sprint synchronization**: When given task seek for relevant directory to work on from SPRINTS.md. Confirm its state on both SPRINTS.md and EXECPLAN.md and the relevant TASKS.md file. Always check the AGENTS.md in the relevant TASKS.md directory. - **Tests**: Add/extend fixtures and unit tests per change; never regress determinism or precedence. -- **Test layout**: Use module-specific projects in `StellaOps.Feedser..Tests`; shared fixtures/harnesses live in `StellaOps.Feedser.Testing`. +- **Test layout**: Use module-specific projects in `StellaOps.Concelier..Tests`; shared fixtures/harnesses live in `StellaOps.Concelier.Testing`. - **Execution autonomous**: In case you need to continue with more than one options just continue sequentially, unless the continue requires design decision. --- diff --git a/EXECPLAN.md b/EXECPLAN.md new file mode 100644 index 00000000..62cfea29 --- /dev/null +++ b/EXECPLAN.md @@ -0,0 +1,1270 @@ +# Execution Tree for Open Backlog +Generated from SPRINTS.md and module TASKS.md files on 2025-10-19. Waves cluster tasks by dependency depth; Wave 0 has no unresolved blockers and later waves depend on earlier ones. + +## Wave Instructions +### Wave 0 +- Team Attestor Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Attestor/TASKS.md`. Focus on ATTESTOR-API-11-201 (TODO), ATTESTOR-VERIFY-11-202 (TODO), ATTESTOR-OBS-11-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Authority Core & Security Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTH-DPOP-11-001 (DOING 2025-10-19), AUTH-MTLS-11-002 (DOING 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Authority Core & Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/TASKS.md`. Focus on AUTHSTORAGE-MONGO-08-001 (DONE 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team DevEx/CLI: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on EXCITITOR-CLI-01-002 (TODO), CLI-RUNTIME-13-005 (TODO). Confirm prerequisites (external: EXCITITOR-CLI-01-001, EXCITITOR-EXPORT-01-001) before starting and report status in module TASKS.md. +- Team DevOps Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SEC-10-301 (DOING 2025-10-19); Wave 0A prerequisites reconfirmed so remediation work may proceed. Keep module TASKS.md/Sprints in sync as patches land. +- Team Diff Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Diff/TASKS.md`. Focus on SCANNER-DIFF-10-501 (TODO), SCANNER-DIFF-10-502 (TODO), SCANNER-DIFF-10-503 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Docs Guild, Plugin Team: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on DOC4.AUTH-PDG (REVIEW). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Docs/CLI: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on EXCITITOR-CLI-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-CLI-01-001) before starting and report status in module TASKS.md. +- Team Emit Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. Focus on SCANNER-EMIT-10-601 (TODO), SCANNER-EMIT-10-602 (TODO), SCANNER-EMIT-10-603 (TODO), SCANNER-EMIT-10-604 (TODO), SCANNER-EMIT-10-605 (TODO), SCANNER-EMIT-10-606 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team EntryTrace Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.EntryTrace/TASKS.md`. Focus on SCANNER-ENTRYTRACE-10-401 (TODO), SCANNER-ENTRYTRACE-10-402 (TODO), SCANNER-ENTRYTRACE-10-403 (TODO), SCANNER-ENTRYTRACE-10-404 (TODO), SCANNER-ENTRYTRACE-10-405 (TODO), SCANNER-ENTRYTRACE-10-406 (TODO), SCANNER-ENTRYTRACE-10-407 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Language Analyzer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md`, `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-301 (TODO), SCANNER-ANALYZERS-LANG-10-307 (TODO), SCANNER-ANALYZERS-LANG-10-308 (TODO), SCANNER-ANALYZERS-LANG-10-302..309 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Notify Models Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.Models/TASKS.md`. Focus on NOTIFY-MODELS-15-101 (TODO), NOTIFY-MODELS-15-102 (TODO), NOTIFY-MODELS-15-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Notify Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.Storage.Mongo/TASKS.md`. Focus on NOTIFY-STORAGE-15-201 (TODO), NOTIFY-STORAGE-15-202 (TODO), NOTIFY-STORAGE-15-203 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Notify WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-101 (TODO), NOTIFY-WEB-15-102 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Platform Events Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on PLATFORM-EVENTS-09-401 (TODO). Confirm prerequisites (external: DOCS-EVENTS-09-003) before starting and report status in module TASKS.md. +- Team Plugin Platform Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Focus on PLUGIN-DI-08-001 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Plugin Platform Guild, Authority Core: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Plugin/TASKS.md`. Focus on PLUGIN-DI-08-002 (TODO); coordination session booked for 2025-10-20 to unblock implementation. Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Policy Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Policy/TASKS.md`. Focus on POLICY-CORE-09-004 (TODO), POLICY-CORE-09-005 (TODO), POLICY-CORE-09-006 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Runtime Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `docs/TASKS.md`. Focus on RUNTIME-GUILD-09-402 (TODO). Confirm prerequisites (external: SCANNER-POLICY-09-107) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-EVENTS-15-201 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-300 (DOING). Confirm prerequisites (external: SAMPLES-10-001) before starting and report status in module TASKS.md. +- Team Scheduler Models Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Models/TASKS.md`. Focus on SCHED-MODELS-16-103 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. +- Team Scheduler Queue Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. Focus on SCHED-QUEUE-16-401 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. +- Team Scheduler Storage Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md`. Focus on SCHED-STORAGE-16-201 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. +- Team Scheduler WebService Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-101 (TODO). Confirm prerequisites (external: SCHED-MODELS-16-101) before starting and report status in module TASKS.md. +- Team Signer Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Signer/TASKS.md`. Focus on SIGNER-API-11-101 (TODO), SIGNER-REF-11-102 (TODO), SIGNER-QUOTA-11-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-302C (TODO). Confirm prerequisites (external: SCANNER-ANALYZERS-LANG-10-302B) before starting and report status in module TASKS.md. +- Team Team Connector Resumption – CERT/RedHat: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md`. Focus on FEEDCONN-REDHAT-02-001 (DOING). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Team Excititor Attestation: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Attestation/TASKS.md`. Focus on EXCITITOR-ATTEST-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-ATTEST-01-002) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Cisco: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-CISCO-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-CONN-CISCO-01-002, EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – MSRC: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-MS-01-002 (TODO). Confirm prerequisites (external: EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Oracle: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-ORACLE-01-001 (DOING). Confirm prerequisites (external: EXCITITOR-CONN-ABS-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – SUSE: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md`. Focus on EXCITITOR-CONN-SUSE-01-002 (TODO). Confirm prerequisites (external: EXCITITOR-CONN-SUSE-01-001, EXCITITOR-STORAGE-01-003) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Ubuntu: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-UBUNTU-01-002 (TODO). Confirm prerequisites (external: EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003) before starting and report status in module TASKS.md. +- Team Team Excititor Export: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Export/TASKS.md`. Focus on EXCITITOR-EXPORT-01-005 (TODO). Confirm prerequisites (external: EXCITITOR-CORE-02-001, EXCITITOR-EXPORT-01-004) before starting and report status in module TASKS.md. +- Team Team Excititor Formats: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Formats.CSAF/TASKS.md`, `src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md`, `src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md`. Focus on EXCITITOR-FMT-CSAF-01-002 (TODO), EXCITITOR-FMT-CSAF-01-003 (TODO), EXCITITOR-FMT-CYCLONE-01-002 (TODO), EXCITITOR-FMT-CYCLONE-01-003 (TODO), EXCITITOR-FMT-OPENVEX-01-002 (TODO), EXCITITOR-FMT-OPENVEX-01-003 (TODO). Confirm prerequisites (external: EXCITITOR-EXPORT-01-001, EXCITITOR-FMT-CSAF-01-001, EXCITITOR-FMT-CYCLONE-01-001, EXCITITOR-FMT-OPENVEX-01-001, EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Storage: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Storage.Mongo/TASKS.md`. Focus on EXCITITOR-STORAGE-MONGO-08-001 (DONE 2025-10-19), EXCITITOR-STORAGE-03-001 (TODO). Confirm prerequisites (external: EXCITITOR-STORAGE-01-003, EXCITITOR-STORAGE-02-001) before starting and report status in module TASKS.md. +- Team Team Excititor WebService: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.WebService/TASKS.md`. Focus on EXCITITOR-WEB-01-002 (TODO), EXCITITOR-WEB-01-003 (TODO), EXCITITOR-WEB-01-004 (TODO). Confirm prerequisites (external: EXCITITOR-ATTEST-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-WEB-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Worker: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Excititor.Worker/TASKS.md`. Focus on EXCITITOR-WORKER-01-002 (TODO), EXCITITOR-WORKER-01-004 (TODO), EXCITITOR-WORKER-02-001 (TODO). Confirm prerequisites (external: EXCITITOR-CORE-02-001, EXCITITOR-WORKER-01-001) before starting and report status in module TASKS.md. +- Team Team Merge & QA Enforcement: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Merge/TASKS.md`. Focus on FEEDMERGE-COORD-02-900 (DOING). Confirm prerequisites (none) before starting and report status in module TASKS.md. **2025-10-19:** Coordination refreshed; connector owners notified and TASKS.md entries updated. +- Team Team Normalization & Storage Backbone: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Storage.Mongo/TASKS.md`. Focus on FEEDSTORAGE-MONGO-08-001 (DONE 2025-10-19). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Team WebService & Authority: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md`, `src/StellaOps.Concelier.WebService/TASKS.md`. Focus on SEC2.PLG (DOING), SEC3.PLG (DOING), SEC5.PLG (DOING), PLG4-6.CAPABILITIES (BLOCKED), PLG6.DIAGRAM (TODO), PLG7.RFC (REVIEW), FEEDWEB-DOCS-01-001 (DOING), FEEDWEB-OPS-01-006 (TODO), FEEDWEB-OPS-01-007 (BLOCKED). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Tools Guild, BE-Conn-MSRC: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.Common/TASKS.md`. Focus on FEEDCONN-SHARED-STATE-003 (**TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team UX Specialist, Angular Eng: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Web/TASKS.md`. Focus on WEB1.TRIVY-SETTINGS (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Zastava Core Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Zastava.Core/TASKS.md`. Focus on ZASTAVA-CORE-12-201 (TODO), ZASTAVA-CORE-12-202 (TODO), ZASTAVA-CORE-12-203 (TODO), ZASTAVA-OPS-12-204 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. +- Team Zastava Webhook Guild: read EXECPLAN.md Wave 0 and SPRINTS.md rows for `src/StellaOps.Zastava.Webhook/TASKS.md`. Focus on ZASTAVA-WEBHOOK-12-101 (TODO), ZASTAVA-WEBHOOK-12-102 (TODO), ZASTAVA-WEBHOOK-12-103 (TODO). Confirm prerequisites (none) before starting and report status in module TASKS.md. + +### Wave 1 +- Team Bench Guild, Language Analyzer Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `bench/TASKS.md`. Focus on BENCH-SCANNER-10-002 (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-301 (Wave 0)) before starting and report status in module TASKS.md. +- Team DevEx/CLI, QA Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-RUNTIME-13-009 (TODO). Confirm prerequisites (internal: CLI-RUNTIME-13-005 (Wave 0)) before starting and report status in module TASKS.md. +- Team DevOps Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-REL-14-001 (TODO). Confirm prerequisites (internal: ATTESTOR-API-11-201 (Wave 0), SIGNER-API-11-101 (Wave 0)) before starting and report status in module TASKS.md. +- Team DevOps Guild, Scanner WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SCANNER-09-204 (TODO). Confirm prerequisites (internal: SCANNER-EVENTS-15-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Emit Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Emit/TASKS.md`. Focus on SCANNER-EMIT-10-607 (TODO), SCANNER-EMIT-17-701 (TODO). Confirm prerequisites (internal: POLICY-CORE-09-005 (Wave 0), SCANNER-EMIT-10-602 (Wave 0), SCANNER-EMIT-10-604 (Wave 0)) before starting and report status in module TASKS.md. +- Team Language Analyzer Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-309 (DOING), SCANNER-ANALYZERS-LANG-10-306 (TODO), SCANNER-ANALYZERS-LANG-10-302 (DOING), SCANNER-ANALYZERS-LANG-10-304 (TODO), SCANNER-ANALYZERS-LANG-10-305 (TODO), SCANNER-ANALYZERS-LANG-10-303 (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-301 (Wave 0), SCANNER-ANALYZERS-LANG-10-307 (Wave 0)) before starting and report status in module TASKS.md. +- Team Licensing Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `ops/licensing/TASKS.md`. Focus on DEVOPS-LIC-14-004 (TODO). Confirm prerequisites (internal: AUTH-MTLS-11-002 (Wave 0)) before starting and report status in module TASKS.md. +- Team Notify Engine Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-301 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. +- Team Notify Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.Queue/TASKS.md`. Focus on NOTIFY-QUEUE-15-401 (TODO). Confirm prerequisites (internal: NOTIFY-MODELS-15-101 (Wave 0)) before starting and report status in module TASKS.md. +- Team Notify WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-103 (DONE). Confirm prerequisites (internal: NOTIFY-WEB-15-102 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-12-301 (TODO). Confirm prerequisites (internal: ZASTAVA-CORE-12-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-301 (TODO). Confirm prerequisites (internal: SCANNER-EMIT-10-605 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler Queue Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Queue/TASKS.md`. Focus on SCHED-QUEUE-16-402 (TODO), SCHED-QUEUE-16-403 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler Storage Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md`. Focus on SCHED-STORAGE-16-203 (TODO), SCHED-STORAGE-16-202 (TODO). Confirm prerequisites (internal: SCHED-STORAGE-16-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler WebService Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-104 (TODO), SCHED-WEB-16-102 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0), SCHED-STORAGE-16-201 (Wave 0), SCHED-WEB-16-101 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler Worker Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-201 (TODO). Confirm prerequisites (internal: SCHED-QUEUE-16-401 (Wave 0)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305A (TODO), SCANNER-ANALYZERS-LANG-10-304A (TODO), SCANNER-ANALYZERS-LANG-10-307N (TODO), SCANNER-ANALYZERS-LANG-10-303A (TODO), SCANNER-ANALYZERS-LANG-10-306A (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-302C (Wave 0), SCANNER-ANALYZERS-LANG-10-307 (Wave 0)) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – MSRC: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-MS-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-MS-01-002 (Wave 0); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Oracle: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-ORACLE-01-002 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-ORACLE-01-001 (Wave 0); external: EXCITITOR-STORAGE-01-003) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – SUSE: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md`. Focus on EXCITITOR-CONN-SUSE-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-SUSE-01-002 (Wave 0); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Ubuntu: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-UBUNTU-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-UBUNTU-01-002 (Wave 0); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Export: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Export/TASKS.md`. Focus on EXCITITOR-EXPORT-01-006 (TODO). Confirm prerequisites (internal: EXCITITOR-EXPORT-01-005 (Wave 0), POLICY-CORE-09-005 (Wave 0)) before starting and report status in module TASKS.md. +- Team Team Excititor Worker: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Excititor.Worker/TASKS.md`. Focus on EXCITITOR-WORKER-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-ATTEST-01-003 (Wave 0); external: EXCITITOR-EXPORT-01-002, EXCITITOR-WORKER-01-001) before starting and report status in module TASKS.md. +- Team UI Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.UI/TASKS.md`. Focus on UI-ATTEST-11-005 (TODO), UI-VEX-13-003 (TODO), UI-POLICY-13-007 (TODO), UI-ADMIN-13-004 (TODO), UI-AUTH-13-001 (TODO), UI-SCANS-13-002 (TODO), UI-NOTIFY-13-006 (DOING), UI-SCHED-13-005 (TODO). Confirm prerequisites (internal: ATTESTOR-API-11-201 (Wave 0), AUTH-DPOP-11-001 (Wave 0), AUTH-MTLS-11-002 (Wave 0), EXCITITOR-EXPORT-01-005 (Wave 0), NOTIFY-WEB-15-101 (Wave 0), POLICY-CORE-09-006 (Wave 0), SCHED-WEB-16-101 (Wave 0), SIGNER-API-11-101 (Wave 0); external: EXCITITOR-CORE-02-001, SCANNER-WEB-09-102, SCANNER-WEB-09-103) before starting and report status in module TASKS.md. +- Team Zastava Observer Guild: read EXECPLAN.md Wave 1 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-001 (TODO). Confirm prerequisites (internal: ZASTAVA-CORE-12-201 (Wave 0)) before starting and report status in module TASKS.md. + +### Wave 2 +- Team Bench Guild, Notify Team: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `bench/TASKS.md`. Focus on BENCH-NOTIFY-15-001 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1)) before starting and report status in module TASKS.md. +- Team Bench Guild, Scheduler Team: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `bench/TASKS.md`. Focus on BENCH-IMPACT-16-001 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1)) before starting and report status in module TASKS.md. +- Team Deployment Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/deployment/TASKS.md`. Focus on DEVOPS-OPS-14-003 (TODO). Confirm prerequisites (internal: DEVOPS-REL-14-001 (Wave 1)) before starting and report status in module TASKS.md. +- Team DevOps Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-MIRROR-08-001 (DONE 2025-10-19), DEVOPS-PERF-10-002 (TODO), DEVOPS-REL-17-002 (TODO). Confirm prerequisites (internal: BENCH-SCANNER-10-002 (Wave 1), DEVOPS-REL-14-001 (Wave 1), SCANNER-EMIT-17-701 (Wave 1)) before starting and report status in module TASKS.md. +- Team DevOps Guild, Notify Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/devops/TASKS.md`. Focus on DEVOPS-SCANNER-09-205 (TODO). Confirm prerequisites (internal: DEVOPS-SCANNER-09-204 (Wave 1)) before starting and report status in module TASKS.md. +- Team Notify Engine Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-302 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1)) before starting and report status in module TASKS.md. +- Team Notify Queue Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Queue/TASKS.md`. Focus on NOTIFY-QUEUE-15-403 (TODO), NOTIFY-QUEUE-15-402 (TODO). Confirm prerequisites (internal: NOTIFY-QUEUE-15-401 (Wave 1)) before starting and report status in module TASKS.md. +- Team Notify WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.WebService/TASKS.md`. Focus on NOTIFY-WEB-15-104 (TODO). Confirm prerequisites (internal: NOTIFY-QUEUE-15-401 (Wave 1), NOTIFY-STORAGE-15-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Notify Worker Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-201 (TODO), NOTIFY-WORKER-15-202 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-301 (Wave 1), NOTIFY-QUEUE-15-401 (Wave 1)) before starting and report status in module TASKS.md. +- Team Offline Kit Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `ops/offline-kit/TASKS.md`. Focus on DEVOPS-OFFLINE-14-002 (TODO). Confirm prerequisites (internal: DEVOPS-REL-14-001 (Wave 1)) before starting and report status in module TASKS.md. +- Team Samples Guild, Policy Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `samples/TASKS.md`. Focus on SAMPLES-13-004 (TODO). Confirm prerequisites (internal: POLICY-CORE-09-006 (Wave 0), UI-POLICY-13-007 (Wave 1)) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-12-302 (TODO). Confirm prerequisites (internal: SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-CORE-12-201 (Wave 0)) before starting and report status in module TASKS.md. +- Team Scheduler ImpactIndex Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.ImpactIndex/TASKS.md`. Focus on SCHED-IMPACT-16-303 (TODO), SCHED-IMPACT-16-302 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1)) before starting and report status in module TASKS.md. +- Team Scheduler WebService Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.WebService/TASKS.md`. Focus on SCHED-WEB-16-103 (TODO). Confirm prerequisites (internal: SCHED-WEB-16-102 (Wave 1)) before starting and report status in module TASKS.md. +- Team Scheduler Worker Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-202 (TODO), SCHED-WORKER-16-205 (TODO). Confirm prerequisites (internal: SCHED-IMPACT-16-301 (Wave 1), SCHED-WORKER-16-201 (Wave 1)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305B (TODO), SCANNER-ANALYZERS-LANG-10-304B (TODO), SCANNER-ANALYZERS-LANG-10-308N (TODO), SCANNER-ANALYZERS-LANG-10-303B (TODO), SCANNER-ANALYZERS-LANG-10-306B (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303A (Wave 1), SCANNER-ANALYZERS-LANG-10-304A (Wave 1), SCANNER-ANALYZERS-LANG-10-305A (Wave 1), SCANNER-ANALYZERS-LANG-10-306A (Wave 1), SCANNER-ANALYZERS-LANG-10-307N (Wave 1)) before starting and report status in module TASKS.md. +- Team Team Excititor Connectors – Oracle: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md`. Focus on EXCITITOR-CONN-ORACLE-01-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-ORACLE-01-002 (Wave 1); external: EXCITITOR-POLICY-01-001) before starting and report status in module TASKS.md. +- Team Team Excititor Export: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Excititor.Export/TASKS.md`. Focus on EXCITITOR-EXPORT-01-007 (TODO). Confirm prerequisites (internal: EXCITITOR-EXPORT-01-006 (Wave 1)) before starting and report status in module TASKS.md. +- Team Zastava Observer Guild: read EXECPLAN.md Wave 2 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-002 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-12-001 (Wave 1)) before starting and report status in module TASKS.md. + +### Wave 3 +- Team DevEx/CLI: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-OFFLINE-13-006 (TODO). Confirm prerequisites (internal: DEVOPS-OFFLINE-14-002 (Wave 2)) before starting and report status in module TASKS.md. +- Team DevEx/CLI, Scanner WebService Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-RUNTIME-13-008 (TODO). Confirm prerequisites (internal: SCANNER-RUNTIME-12-302 (Wave 2)) before starting and report status in module TASKS.md. +- Team Excititor Connectors – Stella: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-001 (TODO). Confirm prerequisites (internal: EXCITITOR-EXPORT-01-007 (Wave 2)) before starting and report status in module TASKS.md. +- Team Notify Engine Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-303 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-302 (Wave 2)) before starting and report status in module TASKS.md. +- Team Notify Worker Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-203 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-302 (Wave 2)) before starting and report status in module TASKS.md. +- Team Scheduler Worker Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-203 (TODO). Confirm prerequisites (internal: SCHED-WORKER-16-202 (Wave 2)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-305C (TODO), SCANNER-ANALYZERS-LANG-10-304C (TODO), SCANNER-ANALYZERS-LANG-10-309N (TODO), SCANNER-ANALYZERS-LANG-10-303C (TODO), SCANNER-ANALYZERS-LANG-10-306C (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303B (Wave 2), SCANNER-ANALYZERS-LANG-10-304B (Wave 2), SCANNER-ANALYZERS-LANG-10-305B (Wave 2), SCANNER-ANALYZERS-LANG-10-306B (Wave 2), SCANNER-ANALYZERS-LANG-10-308N (Wave 2)) before starting and report status in module TASKS.md. +- Team Zastava Observer Guild: read EXECPLAN.md Wave 3 and SPRINTS.md rows for `src/StellaOps.Zastava.Observer/TASKS.md`. Focus on ZASTAVA-OBS-12-003 (TODO), ZASTAVA-OBS-12-004 (TODO), ZASTAVA-OBS-17-005 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-12-002 (Wave 2)) before starting and report status in module TASKS.md. + +### Wave 4 +- Team DevEx/CLI: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Cli/TASKS.md`. Focus on CLI-PLUGIN-13-007 (TODO). Confirm prerequisites (internal: CLI-OFFLINE-13-006 (Wave 3), CLI-RUNTIME-13-005 (Wave 0)) before starting and report status in module TASKS.md. +- Team Docs Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `docs/TASKS.md`. Focus on DOCS-RUNTIME-17-004 (TODO). Confirm prerequisites (internal: DEVOPS-REL-17-002 (Wave 2), SCANNER-EMIT-17-701 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. +- Team Excititor Connectors – Stella: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-002 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-STELLA-07-001 (Wave 3)) before starting and report status in module TASKS.md. +- Team Notify Connectors Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-501 (TODO), NOTIFY-CONN-TEAMS-15-601 (TODO), NOTIFY-CONN-EMAIL-15-701 (TODO), NOTIFY-CONN-WEBHOOK-15-801 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-303 (Wave 3)) before starting and report status in module TASKS.md. +- Team Notify Engine Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Notify.Engine/TASKS.md`. Focus on NOTIFY-ENGINE-15-304 (TODO). Confirm prerequisites (internal: NOTIFY-ENGINE-15-303 (Wave 3)) before starting and report status in module TASKS.md. +- Team Notify Worker Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Notify.Worker/TASKS.md`. Focus on NOTIFY-WORKER-15-204 (TODO). Confirm prerequisites (internal: NOTIFY-WORKER-15-203 (Wave 3)) before starting and report status in module TASKS.md. +- Team Policy Guild, Scanner WebService Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Policy/TASKS.md`. Focus on POLICY-RUNTIME-17-201 (TODO). Confirm prerequisites (internal: ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. +- Team Scheduler Worker Guild: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Scheduler.Worker/TASKS.md`. Focus on SCHED-WORKER-16-204 (TODO). Confirm prerequisites (internal: SCHED-WORKER-16-203 (Wave 3)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 4 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-307D (TODO), SCANNER-ANALYZERS-LANG-10-307G (TODO), SCANNER-ANALYZERS-LANG-10-307P (TODO), SCANNER-ANALYZERS-LANG-10-307R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-303C (Wave 3), SCANNER-ANALYZERS-LANG-10-304C (Wave 3), SCANNER-ANALYZERS-LANG-10-305C (Wave 3), SCANNER-ANALYZERS-LANG-10-306C (Wave 3)) before starting and report status in module TASKS.md. + +### Wave 5 +- Team Excititor Connectors – Stella: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md`. Focus on EXCITITOR-CONN-STELLA-07-003 (TODO). Confirm prerequisites (internal: EXCITITOR-CONN-STELLA-07-002 (Wave 4)) before starting and report status in module TASKS.md. +- Team Notify Connectors Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-502 (DOING), NOTIFY-CONN-TEAMS-15-602 (DOING), NOTIFY-CONN-EMAIL-15-702 (DOING), NOTIFY-CONN-WEBHOOK-15-802 (DOING). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-701 (Wave 4), NOTIFY-CONN-SLACK-15-501 (Wave 4), NOTIFY-CONN-TEAMS-15-601 (Wave 4), NOTIFY-CONN-WEBHOOK-15-801 (Wave 4)) before starting and report status in module TASKS.md. +- Team Scanner WebService Guild: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.WebService/TASKS.md`. Focus on SCANNER-RUNTIME-17-401 (TODO). Confirm prerequisites (internal: POLICY-RUNTIME-17-201 (Wave 4), SCANNER-EMIT-17-701 (Wave 1), SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 5 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-308D (TODO), SCANNER-ANALYZERS-LANG-10-308G (TODO), SCANNER-ANALYZERS-LANG-10-308P (TODO), SCANNER-ANALYZERS-LANG-10-308R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-307D (Wave 4), SCANNER-ANALYZERS-LANG-10-307G (Wave 4), SCANNER-ANALYZERS-LANG-10-307P (Wave 4), SCANNER-ANALYZERS-LANG-10-307R (Wave 4)) before starting and report status in module TASKS.md. + +### Wave 6 +- Team Notify Connectors Guild: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Notify.Connectors.Email/TASKS.md`, `src/StellaOps.Notify.Connectors.Slack/TASKS.md`, `src/StellaOps.Notify.Connectors.Teams/TASKS.md`, `src/StellaOps.Notify.Connectors.Webhook/TASKS.md`. Focus on NOTIFY-CONN-SLACK-15-503 (TODO), NOTIFY-CONN-TEAMS-15-603 (TODO), NOTIFY-CONN-EMAIL-15-703 (TODO), NOTIFY-CONN-WEBHOOK-15-803 (TODO). Confirm prerequisites (internal: NOTIFY-CONN-EMAIL-15-702 (Wave 5), NOTIFY-CONN-SLACK-15-502 (Wave 5), NOTIFY-CONN-TEAMS-15-602 (Wave 5), NOTIFY-CONN-WEBHOOK-15-802 (Wave 5)) before starting and report status in module TASKS.md. +- Team TBD: read EXECPLAN.md Wave 6 and SPRINTS.md rows for `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md`, `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md`. Focus on SCANNER-ANALYZERS-LANG-10-309D (TODO), SCANNER-ANALYZERS-LANG-10-309G (TODO), SCANNER-ANALYZERS-LANG-10-309P (TODO), SCANNER-ANALYZERS-LANG-10-309R (TODO). Confirm prerequisites (internal: SCANNER-ANALYZERS-LANG-10-308D (Wave 5), SCANNER-ANALYZERS-LANG-10-308G (Wave 5), SCANNER-ANALYZERS-LANG-10-308P (Wave 5), SCANNER-ANALYZERS-LANG-10-308R (Wave 5)) before starting and report status in module TASKS.md. + +### Wave 7 +- Team Team Core Engine & Storage Analytics: read EXECPLAN.md Wave 7 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-001 (DONE 2025-10-19). Confirm prerequisites (internal: FEEDSTORAGE-DATA-07-001 (Wave 10)) before starting and report status in module TASKS.md. + +### Wave 8 +- Team Team Core Engine & Data Science: read EXECPLAN.md Wave 8 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-002 (TODO). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. + +### Wave 9 +- Team Team Core Engine & Storage Analytics: read EXECPLAN.md Wave 9 and SPRINTS.md rows for `src/StellaOps.Concelier.Core/TASKS.md`. Focus on FEEDCORE-ENGINE-07-003 (TODO). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. + +### Wave 10 +- Team Team Normalization & Storage Backbone: read EXECPLAN.md Wave 10 and SPRINTS.md rows for `src/StellaOps.Concelier.Storage.Mongo/TASKS.md`. Focus on FEEDSTORAGE-DATA-07-001 (TODO). Confirm prerequisites (internal: FEEDMERGE-ENGINE-07-001 (Wave 11)) before starting and report status in module TASKS.md. + +### Wave 11 +- Team BE-Merge: read EXECPLAN.md Wave 11 and SPRINTS.md rows for `src/StellaOps.Concelier.Merge/TASKS.md`. Focus on FEEDMERGE-ENGINE-07-001 (TODO). Confirm prerequisites (internal: FEEDSTORAGE-DATA-07-001 (Wave 10)) before starting and report status in module TASKS.md. + +### Wave 12 +- Team Concelier Export Guild: read EXECPLAN.md Wave 12 and SPRINTS.md rows for `src/StellaOps.Concelier.Exporter.Json/TASKS.md`. Focus on CONCELIER-EXPORT-08-201 (TODO). Confirm prerequisites (internal: FEEDCORE-ENGINE-07-001 (Wave 7)) before starting and report status in module TASKS.md. + +### Wave 13 +- Team Concelier Export Guild: read EXECPLAN.md Wave 13 and SPRINTS.md rows for `src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md`. Focus on CONCELIER-EXPORT-08-202 (DONE 2025-10-19). Confirm prerequisites (internal: CONCELIER-EXPORT-08-201 (Wave 12)) before starting and report status in module TASKS.md. + +### Wave 14 +- Team Concelier WebService Guild: read EXECPLAN.md Wave 14 and SPRINTS.md rows for `src/StellaOps.Concelier.WebService/TASKS.md`. Focus on CONCELIER-WEB-08-201 (TODO). Confirm prerequisites (internal: CONCELIER-EXPORT-08-201 (Wave 12), DEVOPS-MIRROR-08-001 (Wave 2)) before starting and report status in module TASKS.md. + +### Wave 15 +- Team BE-Conn-Stella: read EXECPLAN.md Wave 15 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-001 (TODO). Confirm prerequisites (internal: CONCELIER-EXPORT-08-201 (Wave 12)) before starting and report status in module TASKS.md. + +### Wave 16 +- Team BE-Conn-Stella: read EXECPLAN.md Wave 16 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-002 (TODO). Confirm prerequisites (internal: FEEDCONN-STELLA-08-001 (Wave 15)) before starting and report status in module TASKS.md. + +### Wave 17 +- Team BE-Conn-Stella: read EXECPLAN.md Wave 17 and SPRINTS.md rows for `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md`. Focus on FEEDCONN-STELLA-08-003 (TODO). Confirm prerequisites (internal: FEEDCONN-STELLA-08-002 (Wave 16)) before starting and report status in module TASKS.md. + +## Wave 0 — 98 task(s) ready now +- **Sprint 1** · Backlog + - Team: UX Specialist, Angular Eng + - Path: `src/StellaOps.Web/TASKS.md` + 1. [TODO] WEB1.TRIVY-SETTINGS — Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. + • Prereqs: — + • Current: TODO +- **Sprint 1** · Developer Tooling + - Team: DevEx/CLI + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] EXCITITOR-CLI-01-002 — EXCITITOR-CLI-01-002 – Export download & attestation UX + • Prereqs: EXCITITOR-CLI-01-001 (external/completed), EXCITITOR-EXPORT-01-001 (external/completed) + • Current: TODO – Display export metadata (sha256, size, Rekor link), support optional artifact download path, and handle cache hits gracefully. + - Team: Docs/CLI + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] EXCITITOR-CLI-01-003 — EXCITITOR-CLI-01-003 – CLI docs & examples for Excititor + • Prereqs: EXCITITOR-CLI-01-001 (external/completed) + • Current: TODO – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Excititor verbs, offline guidance, and attestation verification workflow. +- **Sprint 1** · Stabilize In-Progress Foundations + - Team: Team Connector Resumption – CERT/RedHat + - Path: `src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md` + 1. [DOING] FEEDCONN-REDHAT-02-001 — Fixture validation sweep — Instructions to work: — Regenerating RHSA fixtures awaits remaining range provenance patches; review snapshot diffs and update docs once upstream helpers land. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. + • Prereqs: — + • Current: DOING (2025-10-10) + - Team: Team WebService & Authority + - Path: `src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md` + 1. [DOING] SEC2.PLG — Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`; Serilog enrichment complete, storage durability tests in flight. + • Prereqs: — + • Current: DOING (2025-10-14) + 2. [DOING] SEC3.PLG — Ensure lockout responses carry rate-limit metadata through plugin logs/events; retry-after propagation and limiter tests underway. + • Prereqs: — + • Current: DOING (2025-10-14) + 3. [DOING] SEC5.PLG — Address plugin-specific mitigations in threat model backlog; mitigation items tracked, docs updates pending. + • Prereqs: — + • Current: DOING (2025-10-14) + 4. [BLOCKED] PLG4-6.CAPABILITIES — Finalise capability metadata exposure and docs once Authority rate-limiter stream (CORE8/SEC3) is stable; awaiting dependency unblock. + • Prereqs: — + • Current: BLOCKED (2025-10-12) + 5. [TODO] PLG6.DIAGRAM — Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. + • Prereqs: — + • Current: TODO + 6. [REVIEW] PLG7.RFC — Socialize LDAP plugin RFC and capture guild feedback; awaiting final review sign-off and follow-up issue tracking. + • Prereqs: — + • Current: REVIEW (2025-10-13) + - Path: `src/StellaOps.Concelier.WebService/TASKS.md` + 1. [DOING] FEEDWEB-DOCS-01-001 — Document authority toggle & scope requirements — Quickstart updates are staged; awaiting Docs guild review before publishing operator guide refresh. + • Prereqs: — + • Current: DOING (2025-10-10) + 2. [DONE] FEEDWEB-OPS-01-006 — Rename plugin drop directory to namespaced path — Build outputs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`, plugin host defaults updated, and docs/tests refreshed (see `dotnet test src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj --no-restore`). + • Prereqs: — + • Current: TODO + 3. [BLOCKED] FEEDWEB-OPS-01-007 — Authority resilience adoption — Roll out retry/offline knobs to deployment docs and align CLI parity once LIB5 resilience options land; unblock when library release is available and docs review completes. + • Prereqs: — + • Current: BLOCKED (2025-10-10) +- **Sprint 2** · Connector & Data Implementation Wave + - Team: Docs Guild, Plugin Team + - Path: `docs/TASKS.md` + 1. [REVIEW] DOC4.AUTH-PDG — Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. + • Prereqs: — + • Current: REVIEW + - Team: Team Merge & QA Enforcement + - Path: `src/StellaOps.Concelier.Merge/TASKS.md` + 1. [DOING] FEEDMERGE-COORD-02-900 — Range primitives rollout coordination — Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical range primitives with provenance tags; fixtures tracked in `RANGE_PRIMITIVES_COORDINATION.md`. + • Prereqs: — + • Current: DOING (2025-10-12) +- **Sprint 3** · Backlog + - Team: Tools Guild, BE-Conn-MSRC + - Path: `src/StellaOps.Concelier.Connector.Common/TASKS.md` + 1. [**TODO] FEEDCONN-SHARED-STATE-003 — FEEDCONN-SHARED-STATE-003 Source state seeding helper + • Prereqs: — + • Current: **TODO (2025-10-15)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands. +- **Sprint 5** · Excititor Core Foundations + - Team: Team Excititor Attestation + - Path: `src/StellaOps.Excititor.Attestation/TASKS.md` + 1. [TODO] EXCITITOR-ATTEST-01-003 — EXCITITOR-ATTEST-01-003 – Verification suite & observability + • Prereqs: EXCITITOR-ATTEST-01-002 (external/completed) + • Current: TODO – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests. + - Team: Team Excititor WebService + - Path: `src/StellaOps.Excititor.WebService/TASKS.md` + 1. [TODO] EXCITITOR-WEB-01-002 — EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints + • Prereqs: EXCITITOR-WEB-01-001 (external/completed) + • Current: TODO – Implement `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with token scope enforcement and structured run telemetry. + 2. [TODO] EXCITITOR-WEB-01-003 — EXCITITOR-WEB-01-003 – Export & verify endpoints + • Prereqs: EXCITITOR-WEB-01-001 (external/completed), EXCITITOR-EXPORT-01-001 (external/completed), EXCITITOR-ATTEST-01-001 (external/completed) + • Current: TODO – Add `/excititor/export`, `/excititor/export/{id}`, `/excititor/export/{id}/download`, `/excititor/verify`, returning artifact + attestation metadata with cache awareness. +- **Sprint 6** · Excititor Ingest & Formats + - Team: Team Excititor Connectors – Cisco + - Path: `src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-CISCO-01-003 — EXCITITOR-CONN-CISCO-01-003 – Provider trust metadata + • Prereqs: EXCITITOR-CONN-CISCO-01-002 (external/completed), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Emit cosign/PGP trust metadata and advisory provenance hints for policy weighting. + - Team: Team Excititor Connectors – MSRC + - Path: `src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-MS-01-002 — EXCITITOR-CONN-MS-01-002 – CSAF download pipeline + • Prereqs: EXCITITOR-CONN-MS-01-001 (external/completed), EXCITITOR-STORAGE-01-003 (external/completed) + • Current: TODO – Fetch CSAF packages with retry/backoff, checksum verification, and raw document persistence plus quarantine for schema failures. + - Team: Team Excititor Connectors – Oracle + - Path: `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md` + 1. [DOING] EXCITITOR-CONN-ORACLE-01-001 — EXCITITOR-CONN-ORACLE-01-001 – Oracle CSAF catalogue discovery + • Prereqs: EXCITITOR-CONN-ABS-01-001 (external/completed) + • Current: DOING (2025-10-17) – Implement catalogue discovery, CPU calendar awareness, and offline snapshot import for Oracle CSAF feeds. + - Team: Team Excititor Connectors – SUSE + - Path: `src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md` + 1. [TODO] EXCITITOR-CONN-SUSE-01-002 — EXCITITOR-CONN-SUSE-01-002 – Checkpointed event ingestion + • Prereqs: EXCITITOR-CONN-SUSE-01-001 (external/completed), EXCITITOR-STORAGE-01-003 (external/completed) + • Current: TODO – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads. + - Team: Team Excititor Connectors – Ubuntu + - Path: `src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-UBUNTU-01-002 — EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication + • Prereqs: EXCITITOR-CONN-UBUNTU-01-001 (external/completed), EXCITITOR-STORAGE-01-003 (external/completed) + • Current: TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence. + - Team: Team Excititor Formats + - Path: `src/StellaOps.Excititor.Formats.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-FMT-CSAF-01-002 — EXCITITOR-FMT-CSAF-01-002 – Status/justification mapping + • Prereqs: EXCITITOR-FMT-CSAF-01-001 (external/completed), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Normalize CSAF `product_status` + `justification` values into policy-aware enums with audit diagnostics for unsupported codes. + 2. [TODO] EXCITITOR-FMT-CSAF-01-003 — EXCITITOR-FMT-CSAF-01-003 – CSAF export adapter + • Prereqs: EXCITITOR-EXPORT-01-001 (external/completed), EXCITITOR-FMT-CSAF-01-001 (external/completed) + • Current: TODO – Provide CSAF export writer producing deterministic documents (per vuln/product) and manifest metadata for attestation. + - Path: `src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md` + 1. [TODO] EXCITITOR-FMT-CYCLONE-01-002 — EXCITITOR-FMT-CYCLONE-01-002 – Component reference reconciliation + • Prereqs: EXCITITOR-FMT-CYCLONE-01-001 (external/completed) + • Current: TODO – Implement helpers to reconcile component/service references against policy expectations and emit diagnostics for missing SBOM links. + 2. [TODO] EXCITITOR-FMT-CYCLONE-01-003 — EXCITITOR-FMT-CYCLONE-01-003 – CycloneDX export serializer + • Prereqs: EXCITITOR-EXPORT-01-001 (external/completed), EXCITITOR-FMT-CYCLONE-01-001 (external/completed) + • Current: TODO – Provide exporters producing CycloneDX VEX output with canonical ordering and hash-stable manifests. + - Path: `src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md` + 1. [TODO] EXCITITOR-FMT-OPENVEX-01-002 — EXCITITOR-FMT-OPENVEX-01-002 – Statement merge utilities + • Prereqs: EXCITITOR-FMT-OPENVEX-01-001 (external/completed) + • Current: TODO – Add reducers merging multiple OpenVEX statements, resolving conflicts deterministically, and emitting policy diagnostics. + 2. [TODO] EXCITITOR-FMT-OPENVEX-01-003 — EXCITITOR-FMT-OPENVEX-01-003 – OpenVEX export writer + • Prereqs: EXCITITOR-EXPORT-01-001 (external/completed), EXCITITOR-FMT-OPENVEX-01-001 (external/completed) + • Current: TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering. + - Team: Team Excititor Worker + - Path: `src/StellaOps.Excititor.Worker/TASKS.md` + 1. [TODO] EXCITITOR-WORKER-01-002 — EXCITITOR-WORKER-01-002 – Resume tokens & retry policy + • Prereqs: EXCITITOR-WORKER-01-001 (external/completed) + • Current: TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec. +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Excititor Export + - Path: `src/StellaOps.Excititor.Export/TASKS.md` + 1. [TODO] EXCITITOR-EXPORT-01-005 — EXCITITOR-EXPORT-01-005 – Score & resolve envelope surfaces + • Prereqs: EXCITITOR-EXPORT-01-004 (external/completed), EXCITITOR-CORE-02-001 (external/completed) + • Current: TODO – Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses. + - Team: Team Excititor WebService + - Path: `src/StellaOps.Excititor.WebService/TASKS.md` + 1. [TODO] EXCITITOR-WEB-01-004 — Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. + • Prereqs: — + • Current: TODO + - Team: Team Excititor Worker + - Path: `src/StellaOps.Excititor.Worker/TASKS.md` + 1. [TODO] EXCITITOR-WORKER-01-004 — EXCITITOR-WORKER-01-004 – TTL refresh & stability damper + • Prereqs: EXCITITOR-WORKER-01-001 (external/completed), EXCITITOR-CORE-02-001 (external/completed) + • Current: TODO – Monitor consensus/VEX TTLs, apply 24–48h dampers before flipping published status/score, and trigger re-resolve when base image or kernel fingerprints change. +- **Sprint 8** · Mongo strengthening + - Team: Authority Core & Storage Guild + - Path: `src/StellaOps.Authority/TASKS.md` + 1. [DONE] AUTHSTORAGE-MONGO-08-001 — Harden Authority Mongo usage — Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. + • Prereqs: — + • Current: BLOCKED (2025-10-19) + - Team: Team Excititor Storage + - Path: `src/StellaOps.Excititor.Storage.Mongo/TASKS.md` + 1. [DONE 2025-10-19] EXCITITOR-STORAGE-MONGO-08-001 — Session + causal consistency hardening shipped with scoped session provider, repository updates, and replica-set consistency tests (`dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`) + • Prereqs: EXCITITOR-STORAGE-01-003 (external/completed) + • Current: DONE – Scoped sessions with causal consistency in place; repositories/tests updated for deterministic read-your-write semantics. + - Team: Team Normalization & Storage Backbone + - Path: `src/StellaOps.Concelier.Storage.Mongo/TASKS.md` + 1. [DONE] FEEDSTORAGE-MONGO-08-001 — Causal-consistent Concelier storage sessions — Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. + • Prereqs: — + • Current: TODO +- **Sprint 8** · Platform Maintenance + - Team: Team Excititor Storage + - Path: `src/StellaOps.Excititor.Storage.Mongo/TASKS.md` + 1. [DONE 2025-10-19] EXCITITOR-STORAGE-03-001 — Statement backfill tooling + • Prereqs: EXCITITOR-STORAGE-02-001 (external/completed) + • Current: DONE – Admin backfill endpoint, CLI command (`stellaops excititor backfill-statements`), integration coverage, and operator runbook published; further automation tracked separately if needed. + - Team: Team Excititor Worker + - Path: `src/StellaOps.Excititor.Worker/TASKS.md` + 1. [TODO] EXCITITOR-WORKER-02-001 — EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory + • Prereqs: EXCITITOR-WORKER-01-001 (external/completed) + • Current: TODO – Bump `Microsoft.Extensions.Caching.Memory` (and related packages) to the latest .NET 10 preview, regenerate lockfiles, and re-run worker/webservice tests to clear NU1903 high severity warning. +- **Sprint 8** · Plugin Infrastructure + - Team: Plugin Platform Guild + - Path: `src/StellaOps.Plugin/TASKS.md` + 1. [TODO] PLUGIN-DI-08-001 — Scoped service support in plugin bootstrap — Teach the plugin loader/registrar to surface services with scoped lifetimes, honour `StellaOps.DependencyInjection` metadata, and document the new contract. + • Prereqs: — + • Current: TODO + - Team: Plugin Platform Guild, Authority Core + - Path: `src/StellaOps.Plugin/TASKS.md` + 1. [TODO] PLUGIN-DI-08-002 — Update Authority plugin integration — Flow scoped services through identity-provider registrars, bootstrap flows, and background jobs; add regression coverage around scoped lifetimes. (Coordination session set for 2025-10-20 15:00–16:00 UTC; document outcomes before implementation.) + • Prereqs: — + • Current: TODO +- **Sprint 9** · Docs & Governance + - Team: Platform Events Guild + - Path: `docs/TASKS.md` + 1. [TODO] PLATFORM-EVENTS-09-401 — Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. + • Prereqs: DOCS-EVENTS-09-003 (external/completed) + • Current: TODO + - Team: Runtime Guild + - Path: `docs/TASKS.md` + 1. [TODO] RUNTIME-GUILD-09-402 — Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. + • Prereqs: SCANNER-POLICY-09-107 (external/completed) + • Current: TODO +- **Sprint 9** · Policy Foundations + - Team: Policy Guild + - Path: `src/StellaOps.Policy/TASKS.md` + 1. [TODO] POLICY-CORE-09-004 — Versioned scoring config with schema validation, trust table, and golden fixtures. + • Prereqs: — + • Current: TODO + 2. [TODO] POLICY-CORE-09-005 — Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. + • Prereqs: — + • Current: TODO + 3. [TODO] POLICY-CORE-09-006 — Unknown state & confidence decay – deterministic bands surfaced in policy outputs. + • Prereqs: — + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-302C — Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. + • Prereqs: SCANNER-ANALYZERS-LANG-10-302B (external/completed) + • Current: TODO +- **Sprint 10** · DevOps Perf + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [DOING] DEVOPS-SEC-10-301 — Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs (Wave 0A prerequisites cleared; remediation in progress). + • Prereqs: — + • Current: TODO +- **Sprint 10** · Scanner Analyzers & SBOM + - Team: Diff Guild + - Path: `src/StellaOps.Scanner.Diff/TASKS.md` + 1. [TODO] SCANNER-DIFF-10-501 — Build component differ tracking add/remove/version changes with deterministic ordering. + • Prereqs: — + • Current: TODO + 2. [TODO] SCANNER-DIFF-10-502 — Attribute diffs to introducing/removing layers including provenance evidence. + • Prereqs: — + • Current: TODO + 3. [TODO] SCANNER-DIFF-10-503 — Produce JSON diff output for inventory vs usage views aligned with API contract. + • Prereqs: — + • Current: TODO + - Team: Emit Guild + - Path: `src/StellaOps.Scanner.Emit/TASKS.md` + 1. [TODO] SCANNER-EMIT-10-601 — Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. + • Prereqs: — + • Current: TODO + 2. [TODO] SCANNER-EMIT-10-602 — Compose usage SBOM leveraging EntryTrace to flag actual usage. + • Prereqs: — + • Current: TODO + 3. [TODO] SCANNER-EMIT-10-603 — Generate BOM index sidecar (purl table + roaring bitmap + usage flag). + • Prereqs: — + • Current: TODO + 4. [TODO] SCANNER-EMIT-10-604 — Package artifacts for export + attestation with deterministic manifests. + • Prereqs: — + • Current: TODO + 5. [TODO] SCANNER-EMIT-10-605 — Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). + • Prereqs: — + • Current: TODO + 6. [TODO] SCANNER-EMIT-10-606 — Usage view bit flags integrated with EntryTrace. + • Prereqs: — + • Current: TODO + - Team: EntryTrace Guild + - Path: `src/StellaOps.Scanner.EntryTrace/TASKS.md` + 1. [TODO] SCANNER-ENTRYTRACE-10-401 — POSIX shell AST parser with deterministic output. + • Prereqs: — + • Current: TODO + 2. [TODO] SCANNER-ENTRYTRACE-10-402 — Command resolution across layered rootfs with evidence attribution. + • Prereqs: — + • Current: TODO + 3. [TODO] SCANNER-ENTRYTRACE-10-403 — Interpreter tracing for shell wrappers to Python/Node/Java launchers. + • Prereqs: — + • Current: TODO + 4. [TODO] SCANNER-ENTRYTRACE-10-404 — Python entry analyzer (venv shebang, module invocation, usage flag). + • Prereqs: — + • Current: TODO + 5. [TODO] SCANNER-ENTRYTRACE-10-405 — Node/Java launcher analyzer capturing script/jar targets. + • Prereqs: — + • Current: TODO + 6. [TODO] SCANNER-ENTRYTRACE-10-406 — Explainability + diagnostics for unresolved constructs with metrics. + • Prereqs: — + • Current: TODO + 7. [TODO] SCANNER-ENTRYTRACE-10-407 — Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). + • Prereqs: — + • Current: TODO + - Team: Language Analyzer Guild + - Path: `src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-302..309 — Detailed per-language sprint plan (Node, Python, Go, .NET, Rust) with gates and benchmarks. + • Prereqs: — + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-301 — Java analyzer emitting `pkg:maven` with provenance. + • Prereqs: — + • Current: TODO + 2. [TODO] SCANNER-ANALYZERS-LANG-10-307 — Shared language evidence helpers + usage flag propagation. + • Prereqs: — + • Current: TODO + 3. [TODO] SCANNER-ANALYZERS-LANG-10-308 — Determinism + fixture harness for language analyzers. + • Prereqs: — + • Current: TODO +- **Sprint 11** · Signing Chain Bring-up + - Team: Attestor Guild + - Path: `src/StellaOps.Attestor/TASKS.md` + 1. [TODO] ATTESTOR-API-11-201 — `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. + • Prereqs: — + • Current: TODO + 2. [TODO] ATTESTOR-VERIFY-11-202 — `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. + • Prereqs: — + • Current: TODO + 3. [TODO] ATTESTOR-OBS-11-203 — Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. + • Prereqs: — + • Current: TODO + - Team: Authority Core & Security Guild + - Path: `src/StellaOps.Authority/TASKS.md` + 1. [DOING] AUTH-DPOP-11-001 — Implement DPoP proof validation + nonce handling for high-value audiences per architecture. + • Prereqs: — + • Current: DOING (2025-10-19) + 2. [DOING] AUTH-MTLS-11-002 — Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. + • Prereqs: — + • Current: DOING (2025-10-19) + - Team: Signer Guild + - Path: `src/StellaOps.Signer/TASKS.md` + 1. [TODO] SIGNER-API-11-101 — `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. + • Prereqs: — + • Current: TODO + 2. [TODO] SIGNER-REF-11-102 — `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. + • Prereqs: — + • Current: TODO + 3. [TODO] SIGNER-QUOTA-11-103 — Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. + • Prereqs: — + • Current: TODO +- **Sprint 12** · Runtime Guardrails + - Team: Zastava Core Guild + - Path: `src/StellaOps.Zastava.Core/TASKS.md` + 1. [TODO] ZASTAVA-CORE-12-201 — Define runtime event/admission DTOs, hashing helpers, and versioning strategy. + • Prereqs: — + • Current: TODO + 2. [TODO] ZASTAVA-CORE-12-202 — Provide configuration/logging/metrics utilities shared by Observer/Webhook. + • Prereqs: — + • Current: TODO + 3. [TODO] ZASTAVA-CORE-12-203 — Authority client helpers, OpTok caching, and security guardrails for runtime services. + • Prereqs: — + • Current: TODO + 4. [TODO] ZASTAVA-OPS-12-204 — Operational runbooks, alert rules, and dashboard exports for runtime plane. + • Prereqs: — + • Current: TODO + - Team: Zastava Webhook Guild + - Path: `src/StellaOps.Zastava.Webhook/TASKS.md` + 1. [TODO] ZASTAVA-WEBHOOK-12-101 — Admission controller host with TLS bootstrap and Authority auth. + • Prereqs: — + • Current: TODO + 2. [TODO] ZASTAVA-WEBHOOK-12-102 — Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. + • Prereqs: — + • Current: TODO + 3. [TODO] ZASTAVA-WEBHOOK-12-103 — Caching, fail-open/closed toggles, metrics/logging for admission decisions. + • Prereqs: — + • Current: TODO +- **Sprint 13** · UX & CLI Experience + - Team: DevEx/CLI + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] CLI-RUNTIME-13-005 — Add runtime policy test verbs that consume `/policy/runtime` and display verdicts. + • Prereqs: — + • Current: TODO +- **Sprint 15** · Notify Foundations + - Team: Notify Models Guild + - Path: `src/StellaOps.Notify.Models/TASKS.md` + 1. [TODO] NOTIFY-MODELS-15-101 — Define core Notify DTOs, validation helpers, canonical serialization. + • Prereqs: — + • Current: TODO + 2. [TODO] NOTIFY-MODELS-15-102 — Publish schema docs and sample payloads for Notify. + • Prereqs: — + • Current: TODO + 3. [TODO] NOTIFY-MODELS-15-103 — Versioning/migration helpers for rules/templates/deliveries. + • Prereqs: — + • Current: TODO + - Team: Notify Storage Guild + - Path: `src/StellaOps.Notify.Storage.Mongo/TASKS.md` + 1. [TODO] NOTIFY-STORAGE-15-201 — Mongo schemas/indexes for rules, channels, deliveries, digests, locks, audit. + • Prereqs: — + • Current: TODO + 2. [TODO] NOTIFY-STORAGE-15-202 — Repositories with tenant scoping, soft delete, TTL, causal consistency options. + • Prereqs: — + • Current: TODO + 3. [TODO] NOTIFY-STORAGE-15-203 — Delivery history retention and query APIs. + • Prereqs: — + • Current: TODO + - Team: Notify WebService Guild + - Path: `src/StellaOps.Notify.WebService/TASKS.md` + 1. [TODO] NOTIFY-WEB-15-101 — Minimal API host with Authority enforcement and plug-in loading. + • Prereqs: — + • Current: TODO + 2. [TODO] NOTIFY-WEB-15-102 — Rules/channel/template CRUD with audit logging. + • Prereqs: — + • Current: TODO + - Team: Scanner WebService Guild + - Path: `src/StellaOps.Scanner.WebService/TASKS.md` + 1. [TODO] SCANNER-EVENTS-15-201 — Emit `scanner.report.ready` + `scanner.scan.completed` events. + • Prereqs: — + • Current: TODO +- **Sprint 16** · Scheduler Intelligence + - Team: Scheduler ImpactIndex Guild + - Path: `src/StellaOps.Scheduler.ImpactIndex/TASKS.md` + 1. [DOING] SCHED-IMPACT-16-300 — **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). + • Prereqs: SAMPLES-10-001 (external/completed) + • Current: DOING + - Team: Scheduler Models Guild + - Path: `src/StellaOps.Scheduler.Models/TASKS.md` + 1. [TODO] SCHED-MODELS-16-103 — Versioning/migration helpers (schedule evolution, run state transitions). + • Prereqs: SCHED-MODELS-16-101 (external/completed) + • Current: TODO + - Team: Scheduler Queue Guild + - Path: `src/StellaOps.Scheduler.Queue/TASKS.md` + 1. [TODO] SCHED-QUEUE-16-401 — Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. + • Prereqs: SCHED-MODELS-16-101 (external/completed) + • Current: TODO + - Team: Scheduler Storage Guild + - Path: `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md` + 1. [TODO] SCHED-STORAGE-16-201 — Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. + • Prereqs: SCHED-MODELS-16-101 (external/completed) + • Current: TODO + - Team: Scheduler WebService Guild + - Path: `src/StellaOps.Scheduler.WebService/TASKS.md` + 1. [TODO] SCHED-WEB-16-101 — Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. + • Prereqs: SCHED-MODELS-16-101 (external/completed) + • Current: TODO + +## Wave 1 — 45 task(s) ready after Wave 0 +- **Sprint 6** · Excititor Ingest & Formats + - Team: Team Excititor Connectors – MSRC + - Path: `src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-MS-01-003 — EXCITITOR-CONN-MS-01-003 – Trust metadata & provenance hints + • Prereqs: EXCITITOR-CONN-MS-01-002 (Wave 0), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration. + - Team: Team Excititor Connectors – Oracle + - Path: `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-ORACLE-01-002 — EXCITITOR-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline + • Prereqs: EXCITITOR-CONN-ORACLE-01-001 (Wave 0), EXCITITOR-STORAGE-01-003 (external/completed) + • Current: TODO – Fetch CSAF documents with retry/backoff, checksum validation, revision deduplication, and raw persistence. + - Team: Team Excititor Connectors – SUSE + - Path: `src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md` + 1. [TODO] EXCITITOR-CONN-SUSE-01-003 — EXCITITOR-CONN-SUSE-01-003 – Trust metadata & policy hints + • Prereqs: EXCITITOR-CONN-SUSE-01-002 (Wave 0), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine. + - Team: Team Excititor Connectors – Ubuntu + - Path: `src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-UBUNTU-01-003 — EXCITITOR-CONN-UBUNTU-01-003 – Trust metadata & provenance + • Prereqs: EXCITITOR-CONN-UBUNTU-01-002 (Wave 0), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics. + - Team: Team Excititor Worker + - Path: `src/StellaOps.Excititor.Worker/TASKS.md` + 1. [TODO] EXCITITOR-WORKER-01-003 — EXCITITOR-WORKER-01-003 – Verification & cache GC loops + • Prereqs: EXCITITOR-WORKER-01-001 (external/completed), EXCITITOR-ATTEST-01-003 (Wave 0), EXCITITOR-EXPORT-01-002 (external/completed) + • Current: TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios. +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Excititor Export + - Path: `src/StellaOps.Excititor.Export/TASKS.md` + 1. [TODO] EXCITITOR-EXPORT-01-006 — EXCITITOR-EXPORT-01-006 – Quiet provenance packaging + • Prereqs: EXCITITOR-EXPORT-01-005 (Wave 0), POLICY-CORE-09-005 (Wave 0) + • Current: TODO – Attach `quietedBy` statement IDs, signers, and justification codes to exports/offline bundles, mirror metadata into attested manifest, and add regression fixtures. +- **Sprint 9** · DevOps Foundations + - Team: DevOps Guild, Scanner WebService Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-SCANNER-09-204 — Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. + • Prereqs: SCANNER-EVENTS-15-201 (Wave 0) + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-305A — Parse `*.deps.json` + `runtimeconfig.json`, build RID graph, and normalize to `pkg:nuget` components. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-304A — Parse Go build info blob (`runtime/debug` format) and `.note.go.buildid`; map to module/version and evidence. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-307N — Integrate shared helpers for license/licence evidence, canonical JSON serialization, and usage flag propagation. + • Prereqs: SCANNER-ANALYZERS-LANG-10-302C (Wave 0) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-303A — STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-306A — Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO +- **Sprint 10** · Benchmarks + - Team: Bench Guild, Language Analyzer Guild + - Path: `bench/TASKS.md` + 1. [TODO] BENCH-SCANNER-10-002 — Wire real language analyzers into bench harness & refresh baselines post-implementation. + • Prereqs: SCANNER-ANALYZERS-LANG-10-301 (Wave 0) + • Current: TODO +- **Sprint 10** · Scanner Analyzers & SBOM + - Team: Emit Guild + - Path: `src/StellaOps.Scanner.Emit/TASKS.md` + 1. [TODO] SCANNER-EMIT-10-607 — Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. + • Prereqs: SCANNER-EMIT-10-604 (Wave 0), POLICY-CORE-09-005 (Wave 0) + • Current: TODO + - Team: Language Analyzer Guild + - Path: `src/StellaOps.Scanner.Analyzers.Lang/TASKS.md` + 1. [DOING] SCANNER-ANALYZERS-LANG-10-309 — Package language analyzers as restart-time plug-ins (manifest + host registration). + • Prereqs: SCANNER-ANALYZERS-LANG-10-301 (Wave 0) + • Current: DOING (2025-10-19) + 2. [TODO] SCANNER-ANALYZERS-LANG-10-306 — Rust analyzer detecting crate provenance or falling back to `bin:{sha256}`. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + 3. [DOING] SCANNER-ANALYZERS-LANG-10-302 — Node analyzer resolving workspaces/symlinks into `pkg:npm` identities. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: DOING (2025-10-19) + 4. [TODO] SCANNER-ANALYZERS-LANG-10-304 — Go analyzer leveraging buildinfo for `pkg:golang` components. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + 5. [TODO] SCANNER-ANALYZERS-LANG-10-305 — .NET analyzer parsing `*.deps.json`, assembly metadata, and RID variants. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO + 6. [TODO] SCANNER-ANALYZERS-LANG-10-303 — Python analyzer consuming `*.dist-info` metadata and RECORD hashes. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307 (Wave 0) + • Current: TODO +- **Sprint 11** · UI Integration + - Team: UI Guild + - Path: `src/StellaOps.UI/TASKS.md` + 1. [TODO] UI-ATTEST-11-005 — Attestation visibility (Rekor id, status) on Scan Detail. + • Prereqs: SIGNER-API-11-101 (Wave 0), ATTESTOR-API-11-201 (Wave 0) + • Current: TODO +- **Sprint 12** · Runtime Guardrails + - Team: Scanner WebService Guild + - Path: `src/StellaOps.Scanner.WebService/TASKS.md` + 1. [TODO] SCANNER-RUNTIME-12-301 — Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. + • Prereqs: ZASTAVA-CORE-12-201 (Wave 0) + • Current: TODO + - Team: Zastava Observer Guild + - Path: `src/StellaOps.Zastava.Observer/TASKS.md` + 1. [TODO] ZASTAVA-OBS-12-001 — Build container lifecycle watcher that tails CRI (containerd/cri-o/docker) events and emits deterministic runtime records with buffering + backoff. + • Prereqs: ZASTAVA-CORE-12-201 (Wave 0) + • Current: TODO +- **Sprint 13** · UX & CLI Experience + - Team: DevEx/CLI, QA Guild + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] CLI-RUNTIME-13-009 — CLI-RUNTIME-13-009 – Runtime policy smoke fixture + • Prereqs: CLI-RUNTIME-13-005 (Wave 0) + • Current: TODO – Build Spectre test harness exercising `runtime policy test` against a stubbed backend to lock output shape (table + `--json`) and guard regressions. Integrate into `dotnet test` suite. + - Team: UI Guild + - Path: `src/StellaOps.UI/TASKS.md` + 1. [TODO] UI-VEX-13-003 — Implement VEX explorer + policy editor with preview integration. + • Prereqs: EXCITITOR-CORE-02-001 (external/completed), EXCITITOR-EXPORT-01-005 (Wave 0) + • Current: TODO + 2. [TODO] UI-POLICY-13-007 — Surface policy confidence metadata (band, age, quiet provenance) on preview and report views. + • Prereqs: POLICY-CORE-09-006 (Wave 0), SCANNER-WEB-09-103 (external/completed) + • Current: TODO + 3. [TODO] UI-ADMIN-13-004 — Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. + • Prereqs: AUTH-MTLS-11-002 (Wave 0) + • Current: TODO + 4. [TODO] UI-AUTH-13-001 — Integrate Authority OIDC + DPoP flows with session management. + • Prereqs: AUTH-DPOP-11-001 (Wave 0), AUTH-MTLS-11-002 (Wave 0) + • Current: TODO + 5. [TODO] UI-SCANS-13-002 — Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. + • Prereqs: SCANNER-WEB-09-102 (external/completed), SIGNER-API-11-101 (Wave 0) + • Current: TODO + 6. [DOING] UI-NOTIFY-13-006 — Notify panel: channels/rules CRUD, deliveries view, test send integration. + • Prereqs: NOTIFY-WEB-15-101 (Wave 0) + • Current: TODO + 7. [TODO] UI-SCHED-13-005 — Scheduler panel: schedules CRUD, run history, dry-run preview using API/mocks. + • Prereqs: SCHED-WEB-16-101 (Wave 0) + • Current: TODO +- **Sprint 14** · Release & Offline Ops + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-REL-14-001 — Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. + • Prereqs: SIGNER-API-11-101 (Wave 0), ATTESTOR-API-11-201 (Wave 0) + • Current: TODO + - Team: Licensing Guild + - Path: `ops/licensing/TASKS.md` + 1. [TODO] DEVOPS-LIC-14-004 — Implement registry token service tied to Authority (DPoP/mTLS), plan gating, revocation handling, and monitoring per architecture. + • Prereqs: AUTH-MTLS-11-002 (Wave 0) + • Current: TODO +- **Sprint 15** · Notify Foundations + - Team: Notify Engine Guild + - Path: `src/StellaOps.Notify.Engine/TASKS.md` + 1. [TODO] NOTIFY-ENGINE-15-301 — Rules evaluation core: tenant/kind filters, severity/delta gates, VEX gating, throttling, idempotency key generation. + • Prereqs: NOTIFY-MODELS-15-101 (Wave 0) + • Current: TODO + - Team: Notify Queue Guild + - Path: `src/StellaOps.Notify.Queue/TASKS.md` + 1. [TODO] NOTIFY-QUEUE-15-401 — Build queue abstraction + Redis Streams adapter with ack/claim APIs, idempotency tokens, serialization contracts. + • Prereqs: NOTIFY-MODELS-15-101 (Wave 0) + • Current: TODO + - Team: Notify WebService Guild + - Path: `src/StellaOps.Notify.WebService/TASKS.md` + 1. [DONE] NOTIFY-WEB-15-103 — Delivery history + test-send endpoints with rate limits. + • Prereqs: NOTIFY-WEB-15-102 (Wave 0) + • Current: TODO +- **Sprint 16** · Scheduler Intelligence + - Team: Scheduler ImpactIndex Guild + - Path: `src/StellaOps.Scheduler.ImpactIndex/TASKS.md` + 1. [TODO] SCHED-IMPACT-16-301 — Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). + • Prereqs: SCANNER-EMIT-10-605 (Wave 0) + • Current: TODO + - Team: Scheduler Queue Guild + - Path: `src/StellaOps.Scheduler.Queue/TASKS.md` + 1. [TODO] SCHED-QUEUE-16-402 — Add NATS JetStream adapter with configuration binding, health probes, failover. + • Prereqs: SCHED-QUEUE-16-401 (Wave 0) + • Current: TODO + 2. [TODO] SCHED-QUEUE-16-403 — Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. + • Prereqs: SCHED-QUEUE-16-401 (Wave 0) + • Current: TODO + - Team: Scheduler Storage Guild + - Path: `src/StellaOps.Scheduler.Storage.Mongo/TASKS.md` + 1. [TODO] SCHED-STORAGE-16-203 — Audit/logging pipeline + run stats materialized views for UI. + • Prereqs: SCHED-STORAGE-16-201 (Wave 0) + • Current: TODO + 2. [TODO] SCHED-STORAGE-16-202 — Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. + • Prereqs: SCHED-STORAGE-16-201 (Wave 0) + • Current: TODO + - Team: Scheduler WebService Guild + - Path: `src/StellaOps.Scheduler.WebService/TASKS.md` + 1. [TODO] SCHED-WEB-16-104 — Webhook endpoints for Feedser/Vexer exports with mTLS/HMAC validation and rate limiting. + • Prereqs: SCHED-QUEUE-16-401 (Wave 0), SCHED-STORAGE-16-201 (Wave 0) + • Current: TODO + 2. [TODO] SCHED-WEB-16-102 — Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. + • Prereqs: SCHED-WEB-16-101 (Wave 0) + • Current: TODO + - Team: Scheduler Worker Guild + - Path: `src/StellaOps.Scheduler.Worker/TASKS.md` + 1. [TODO] SCHED-WORKER-16-201 — Planner loop (cron + event triggers) with lease management, fairness, and rate limiting (§6). + • Prereqs: SCHED-QUEUE-16-401 (Wave 0) + • Current: TODO +- **Sprint 17** · Symbol Intelligence & Forensics + - Team: Emit Guild + - Path: `src/StellaOps.Scanner.Emit/TASKS.md` + 1. [TODO] SCANNER-EMIT-17-701 — Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. + • Prereqs: SCANNER-EMIT-10-602 (Wave 0) + • Current: TODO + +## Wave 2 — 29 task(s) ready after Wave 1 +- **Sprint 6** · Excititor Ingest & Formats + - Team: Team Excititor Connectors – Oracle + - Path: `src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md` + 1. [TODO] EXCITITOR-CONN-ORACLE-01-003 — EXCITITOR-CONN-ORACLE-01-003 – Trust metadata + provenance + • Prereqs: EXCITITOR-CONN-ORACLE-01-002 (Wave 1), EXCITITOR-POLICY-01-001 (external/completed) + • Current: TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting. +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Excititor Export + - Path: `src/StellaOps.Excititor.Export/TASKS.md` + 1. [TODO] EXCITITOR-EXPORT-01-007 — EXCITITOR-EXPORT-01-007 – Mirror bundle + domain manifest + • Prereqs: EXCITITOR-EXPORT-01-006 (Wave 1) + • Current: TODO – Create per-domain mirror bundles with consensus/score artifacts, publish signed index for downstream Excititor sync, and ensure deterministic digests + fixtures. +- **Sprint 8** · Mirror Distribution + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [DONE] DEVOPS-MIRROR-08-001 — Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. + • Prereqs: DEVOPS-REL-14-001 (Wave 1) + • Current: DONE (2025-10-19) +- **Sprint 9** · DevOps Foundations + - Team: DevOps Guild, Notify Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-SCANNER-09-205 — Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. + • Prereqs: DEVOPS-SCANNER-09-204 (Wave 1) + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-305B — Extract assembly metadata (strong name, file/product info) and optional Authenticode details when offline cert bundle provided. + • Prereqs: SCANNER-ANALYZERS-LANG-10-305A (Wave 1) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-304B — Implement DWARF-lite reader for VCS metadata + dirty flag; add cache to avoid re-reading identical binaries. + • Prereqs: SCANNER-ANALYZERS-LANG-10-304A (Wave 1) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-308N — Author determinism harness + fixtures for Node analyzer; add benchmark suite. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307N (Wave 1) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-303B — RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. + • Prereqs: SCANNER-ANALYZERS-LANG-10-303A (Wave 1) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-306B — Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. + • Prereqs: SCANNER-ANALYZERS-LANG-10-306A (Wave 1) + • Current: TODO +- **Sprint 10** · DevOps Perf + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-PERF-10-002 — Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. + • Prereqs: BENCH-SCANNER-10-002 (Wave 1) + • Current: TODO +- **Sprint 10** · Samples + - Team: Samples Guild, Policy Guild + - Path: `samples/TASKS.md` + 1. [TODO] SAMPLES-13-004 — Add policy preview/report fixtures showing confidence bands and unknown-age tags. + • Prereqs: POLICY-CORE-09-006 (Wave 0), UI-POLICY-13-007 (Wave 1) + • Current: TODO +- **Sprint 12** · Runtime Guardrails + - Team: Scanner WebService Guild + - Path: `src/StellaOps.Scanner.WebService/TASKS.md` + 1. [TODO] SCANNER-RUNTIME-12-302 — Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. + • Prereqs: SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-CORE-12-201 (Wave 0) + • Current: TODO + - Team: Zastava Observer Guild + - Path: `src/StellaOps.Zastava.Observer/TASKS.md` + 1. [TODO] ZASTAVA-OBS-12-002 — Capture entrypoint traces and loaded libraries, hashing binaries and correlating to SBOM baseline per architecture sections 2.1 and 10. + • Prereqs: ZASTAVA-OBS-12-001 (Wave 1) + • Current: TODO +- **Sprint 14** · Release & Offline Ops + - Team: Deployment Guild + - Path: `ops/deployment/TASKS.md` + 1. [TODO] DEVOPS-OPS-14-003 — Document and script upgrade/rollback flows, channel management, and compatibility matrices per architecture. + • Prereqs: DEVOPS-REL-14-001 (Wave 1) + • Current: TODO + - Team: Offline Kit Guild + - Path: `ops/offline-kit/TASKS.md` + 1. [TODO] DEVOPS-OFFLINE-14-002 — Build offline kit packaging workflow (artifact bundling, manifest generation, signature verification). + • Prereqs: DEVOPS-REL-14-001 (Wave 1) + • Current: TODO +- **Sprint 15** · Benchmarks + - Team: Bench Guild, Notify Team + - Path: `bench/TASKS.md` + 1. [TODO] BENCH-NOTIFY-15-001 — Notify dispatch throughput bench (vary rule density) with results CSV. + • Prereqs: NOTIFY-ENGINE-15-301 (Wave 1) + • Current: TODO +- **Sprint 15** · Notify Foundations + - Team: Notify Engine Guild + - Path: `src/StellaOps.Notify.Engine/TASKS.md` + 1. [TODO] NOTIFY-ENGINE-15-302 — Action planner + digest coalescer with window management and dedupe per architecture §4. + • Prereqs: NOTIFY-ENGINE-15-301 (Wave 1) + • Current: TODO + - Team: Notify Queue Guild + - Path: `src/StellaOps.Notify.Queue/TASKS.md` + 1. [TODO] NOTIFY-QUEUE-15-403 — Delivery queue for channel actions with retry schedules, poison queues, and metrics instrumentation. + • Prereqs: NOTIFY-QUEUE-15-401 (Wave 1) + • Current: TODO + 2. [TODO] NOTIFY-QUEUE-15-402 — Add NATS JetStream adapter with configuration binding, health probes, failover. + • Prereqs: NOTIFY-QUEUE-15-401 (Wave 1) + • Current: TODO + - Team: Notify WebService Guild + - Path: `src/StellaOps.Notify.WebService/TASKS.md` + 1. [TODO] NOTIFY-WEB-15-104 — Configuration binding for Mongo/queue/secrets; startup diagnostics. + • Prereqs: NOTIFY-STORAGE-15-201 (Wave 0), NOTIFY-QUEUE-15-401 (Wave 1) + • Current: TODO + - Team: Notify Worker Guild + - Path: `src/StellaOps.Notify.Worker/TASKS.md` + 1. [TODO] NOTIFY-WORKER-15-201 — Implement bus subscription + leasing loop with correlation IDs, backoff, dead-letter handling (§1–§5). + • Prereqs: NOTIFY-QUEUE-15-401 (Wave 1) + • Current: TODO + 2. [TODO] NOTIFY-WORKER-15-202 — Wire rules evaluation pipeline (tenant scoping, filters, throttles, digests, idempotency) with deterministic decisions. + • Prereqs: NOTIFY-ENGINE-15-301 (Wave 1) + • Current: TODO +- **Sprint 16** · Benchmarks + - Team: Bench Guild, Scheduler Team + - Path: `bench/TASKS.md` + 1. [TODO] BENCH-IMPACT-16-001 — ImpactIndex throughput bench (resolve 10k productKeys) + RAM profile. + • Prereqs: SCHED-IMPACT-16-301 (Wave 1) + • Current: TODO +- **Sprint 16** · Scheduler Intelligence + - Team: Scheduler ImpactIndex Guild + - Path: `src/StellaOps.Scheduler.ImpactIndex/TASKS.md` + 1. [TODO] SCHED-IMPACT-16-303 — Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. + • Prereqs: SCHED-IMPACT-16-301 (Wave 1) + • Current: TODO + 2. [TODO] SCHED-IMPACT-16-302 — Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. + • Prereqs: SCHED-IMPACT-16-301 (Wave 1) + • Current: TODO + - Team: Scheduler WebService Guild + - Path: `src/StellaOps.Scheduler.WebService/TASKS.md` + 1. [TODO] SCHED-WEB-16-103 — Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. + • Prereqs: SCHED-WEB-16-102 (Wave 1) + • Current: TODO + - Team: Scheduler Worker Guild + - Path: `src/StellaOps.Scheduler.Worker/TASKS.md` + 1. [TODO] SCHED-WORKER-16-202 — Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. + • Prereqs: SCHED-IMPACT-16-301 (Wave 1) + • Current: TODO + 2. [TODO] SCHED-WORKER-16-205 — Metrics/telemetry: run stats, queue depth, planner latency, delta counts. + • Prereqs: SCHED-WORKER-16-201 (Wave 1) + • Current: TODO +- **Sprint 17** · Symbol Intelligence & Forensics + - Team: DevOps Guild + - Path: `ops/devops/TASKS.md` + 1. [TODO] DEVOPS-REL-17-002 — Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. + • Prereqs: DEVOPS-REL-14-001 (Wave 1), SCANNER-EMIT-17-701 (Wave 1) + • Current: TODO + +## Wave 3 — 14 task(s) ready after Wave 2 +- **Sprint 7** · Contextual Truth Foundations + - Team: Excititor Connectors – Stella + - Path: `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md` + 1. [TODO] EXCITITOR-CONN-STELLA-07-001 — Implement mirror fetch client consuming `https://.stella-ops.org/excititor/exports/index.json`, validating signatures/digests, storing raw consensus bundles with provenance. + • Prereqs: EXCITITOR-EXPORT-01-007 (Wave 2) + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-305C — Handle self-contained apps and native assets; merge with EntryTrace usage hints. + • Prereqs: SCANNER-ANALYZERS-LANG-10-305B (Wave 2) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-304C — Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. + • Prereqs: SCANNER-ANALYZERS-LANG-10-304B (Wave 2) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-309N — Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). + • Prereqs: SCANNER-ANALYZERS-LANG-10-308N (Wave 2) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-303C — Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. + • Prereqs: SCANNER-ANALYZERS-LANG-10-303B (Wave 2) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-306C — Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. + • Prereqs: SCANNER-ANALYZERS-LANG-10-306B (Wave 2) + • Current: TODO +- **Sprint 12** · Runtime Guardrails + - Team: Zastava Observer Guild + - Path: `src/StellaOps.Zastava.Observer/TASKS.md` + 1. [TODO] ZASTAVA-OBS-12-003 — Implement runtime posture checks (signature/SBOM/attestation presence) with offline caching and warning surfaces. + • Prereqs: ZASTAVA-OBS-12-002 (Wave 2) + • Current: TODO + 2. [TODO] ZASTAVA-OBS-12-004 — Batch `/runtime/events` submissions with disk-backed buffer, rate limits, and deterministic envelopes. + • Prereqs: ZASTAVA-OBS-12-002 (Wave 2) + • Current: TODO +- **Sprint 13** · UX & CLI Experience + - Team: DevEx/CLI + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] CLI-OFFLINE-13-006 — CLI-OFFLINE-13-006 – Offline kit workflows + • Prereqs: DEVOPS-OFFLINE-14-002 (Wave 2) + • Current: TODO – Implement `offline kit pull/import/status` commands with integrity checks, resumable downloads, and doc updates. + - Team: DevEx/CLI, Scanner WebService Guild + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] CLI-RUNTIME-13-008 — CLI-RUNTIME-13-008 – Runtime policy contract sync + • Prereqs: SCANNER-RUNTIME-12-302 (Wave 2) + • Current: TODO – Once `/api/v1/scanner/policy/runtime` exits TODO, verify CLI output against final schema (field names, metadata) and update formatter/tests if the contract moves. Capture joint review notes in docs/09 and link Scanner task sign-off. +- **Sprint 15** · Notify Foundations + - Team: Notify Engine Guild + - Path: `src/StellaOps.Notify.Engine/TASKS.md` + 1. [TODO] NOTIFY-ENGINE-15-303 — Template rendering engine (Slack, Teams, Email, Webhook) with helpers and i18n support. + • Prereqs: NOTIFY-ENGINE-15-302 (Wave 2) + • Current: TODO + - Team: Notify Worker Guild + - Path: `src/StellaOps.Notify.Worker/TASKS.md` + 1. [TODO] NOTIFY-WORKER-15-203 — Channel dispatch orchestration: invoke connectors, manage retries/jitter, record delivery outcomes. + • Prereqs: NOTIFY-ENGINE-15-302 (Wave 2) + • Current: TODO +- **Sprint 16** · Scheduler Intelligence + - Team: Scheduler Worker Guild + - Path: `src/StellaOps.Scheduler.Worker/TASKS.md` + 1. [TODO] SCHED-WORKER-16-203 — Runner execution: call Scanner `/reports` (analysis-only) or `/scans` when configured; collect deltas; handle retries. + • Prereqs: SCHED-WORKER-16-202 (Wave 2) + • Current: TODO +- **Sprint 17** · Symbol Intelligence & Forensics + - Team: Zastava Observer Guild + - Path: `src/StellaOps.Zastava.Observer/TASKS.md` + 1. [TODO] ZASTAVA-OBS-17-005 — Collect GNU build-id for ELF processes and attach it to emitted runtime events to enable symbol lookup + debug-store correlation. + • Prereqs: ZASTAVA-OBS-12-002 (Wave 2) + • Current: TODO + +## Wave 4 — 15 task(s) ready after Wave 3 +- **Sprint 7** · Contextual Truth Foundations + - Team: Excititor Connectors – Stella + - Path: `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md` + 1. [TODO] EXCITITOR-CONN-STELLA-07-002 — Normalize mirror bundles into VexClaim sets referencing original provider metadata and mirror provenance. + • Prereqs: EXCITITOR-CONN-STELLA-07-001 (Wave 3) + • Current: TODO +- **Sprint 9** · Policy Foundations + - Team: Policy Guild, Scanner WebService Guild + - Path: `src/StellaOps.Policy/TASKS.md` + 1. [TODO] POLICY-RUNTIME-17-201 — Define runtime reachability feed contract and alignment plan for `SCANNER-RUNTIME-17-401` once Zastava endpoints land; document policy expectations for reachability tags. + • Prereqs: ZASTAVA-OBS-17-005 (Wave 3) + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-307D — Integrate shared helpers (license mapping, quiet provenance) and concurrency-safe caches. + • Prereqs: SCANNER-ANALYZERS-LANG-10-305C (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-307G — Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. + • Prereqs: SCANNER-ANALYZERS-LANG-10-304C (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-307P — Shared helper integration (license metadata, quiet provenance, component merging). + • Prereqs: SCANNER-ANALYZERS-LANG-10-303C (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-307R — Finalize shared helper usage (license, usage flags) and concurrency-safe caches. + • Prereqs: SCANNER-ANALYZERS-LANG-10-306C (Wave 3) + • Current: TODO +- **Sprint 13** · UX & CLI Experience + - Team: DevEx/CLI + - Path: `src/StellaOps.Cli/TASKS.md` + 1. [TODO] CLI-PLUGIN-13-007 — CLI-PLUGIN-13-007 – Plugin packaging + • Prereqs: CLI-RUNTIME-13-005 (Wave 0), CLI-OFFLINE-13-006 (Wave 3) + • Current: TODO – Package non-core verbs as restart-time plug-ins (manifest + loader updates, tests ensuring no hot reload). +- **Sprint 15** · Notify Foundations + - Team: Notify Connectors Guild + - Path: `src/StellaOps.Notify.Connectors.Email/TASKS.md` + 1. [TODO] NOTIFY-CONN-EMAIL-15-701 — Implement SMTP connector with STARTTLS/implicit TLS support, HTML+text rendering, attachment policy enforcement. + • Prereqs: NOTIFY-ENGINE-15-303 (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Slack/TASKS.md` + 1. [TODO] NOTIFY-CONN-SLACK-15-501 — Implement Slack connector with bot token auth, message rendering (blocks), rate limit handling, retries/backoff. + • Prereqs: NOTIFY-ENGINE-15-303 (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Teams/TASKS.md` + 1. [TODO] NOTIFY-CONN-TEAMS-15-601 — Implement Teams connector using Adaptive Cards 1.5, handle webhook auth, size limits, retries. + • Prereqs: NOTIFY-ENGINE-15-303 (Wave 3) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Webhook/TASKS.md` + 1. [TODO] NOTIFY-CONN-WEBHOOK-15-801 — Implement webhook connector: JSON payload, signature (HMAC/Ed25519), retries/backoff, status code handling. + • Prereqs: NOTIFY-ENGINE-15-303 (Wave 3) + • Current: TODO + - Team: Notify Engine Guild + - Path: `src/StellaOps.Notify.Engine/TASKS.md` + 1. [TODO] NOTIFY-ENGINE-15-304 — Test-send sandbox + preview utilities for WebService. + • Prereqs: NOTIFY-ENGINE-15-303 (Wave 3) + • Current: TODO + - Team: Notify Worker Guild + - Path: `src/StellaOps.Notify.Worker/TASKS.md` + 1. [TODO] NOTIFY-WORKER-15-204 — Metrics/telemetry: `notify.sent_total`, `notify.dropped_total`, latency histograms, tracing integration. + • Prereqs: NOTIFY-WORKER-15-203 (Wave 3) + • Current: TODO +- **Sprint 16** · Scheduler Intelligence + - Team: Scheduler Worker Guild + - Path: `src/StellaOps.Scheduler.Worker/TASKS.md` + 1. [TODO] SCHED-WORKER-16-204 — Emit events (`scheduler.rescan.delta`, `scanner.report.ready`) for Notify/UI with summaries. + • Prereqs: SCHED-WORKER-16-203 (Wave 3) + • Current: TODO +- **Sprint 17** · Symbol Intelligence & Forensics + - Team: Docs Guild + - Path: `docs/TASKS.md` + 1. [TODO] DOCS-RUNTIME-17-004 — Document build-id workflows: SBOM exposure, runtime event payloads, debug-store layout, and operator guidance for symbol retrieval. + • Prereqs: SCANNER-EMIT-17-701 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3), DEVOPS-REL-17-002 (Wave 2) + • Current: TODO + +## Wave 5 — 10 task(s) ready after Wave 4 +- **Sprint 7** · Contextual Truth Foundations + - Team: Excititor Connectors – Stella + - Path: `src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md` + 1. [TODO] EXCITITOR-CONN-STELLA-07-003 — Implement incremental cursor handling per-export digest, support resume, and document configuration for downstream Excititor mirrors. + • Prereqs: EXCITITOR-CONN-STELLA-07-002 (Wave 4) + • Current: TODO +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-308D — Determinism fixtures + benchmark harness; compare to competitor scanners for accuracy/perf. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307D (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-308G — Determinism fixtures + benchmark harness (Vs competitor). + • Prereqs: SCANNER-ANALYZERS-LANG-10-307G (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-308P — Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307P (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-308R — Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. + • Prereqs: SCANNER-ANALYZERS-LANG-10-307R (Wave 4) + • Current: TODO +- **Sprint 15** · Notify Foundations + - Team: Notify Connectors Guild + - Path: `src/StellaOps.Notify.Connectors.Email/TASKS.md` + 1. [DOING] NOTIFY-CONN-EMAIL-15-702 — Add DKIM signing optional support and health/test-send flows. + • Prereqs: NOTIFY-CONN-EMAIL-15-701 (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Slack/TASKS.md` + 1. [DOING] NOTIFY-CONN-SLACK-15-502 — Health check & test-send support with minimal scopes and redacted tokens. + • Prereqs: NOTIFY-CONN-SLACK-15-501 (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Teams/TASKS.md` + 1. [DOING] NOTIFY-CONN-TEAMS-15-602 — Provide health/test-send support with fallback text for legacy clients. + • Prereqs: NOTIFY-CONN-TEAMS-15-601 (Wave 4) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Webhook/TASKS.md` + 1. [DOING] NOTIFY-CONN-WEBHOOK-15-802 — Health/test-send support with signature validation hints and secret management. + • Prereqs: NOTIFY-CONN-WEBHOOK-15-801 (Wave 4) + • Current: TODO +- **Sprint 17** · Symbol Intelligence & Forensics + - Team: Scanner WebService Guild + - Path: `src/StellaOps.Scanner.WebService/TASKS.md` + 1. [TODO] SCANNER-RUNTIME-17-401 — Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. + • Prereqs: SCANNER-RUNTIME-12-301 (Wave 1), ZASTAVA-OBS-17-005 (Wave 3), SCANNER-EMIT-17-701 (Wave 1), POLICY-RUNTIME-17-201 (Wave 4) + • Current: TODO + +## Wave 6 — 8 task(s) ready after Wave 5 +- **Sprint 10** · Backlog + - Team: TBD + - Path: `src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-309D — Package plug-in (manifest, DI registration) and update Offline Kit instructions. + • Prereqs: SCANNER-ANALYZERS-LANG-10-308D (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-309G — Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. + • Prereqs: SCANNER-ANALYZERS-LANG-10-308G (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-309P — Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. + • Prereqs: SCANNER-ANALYZERS-LANG-10-308P (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md` + 1. [TODO] SCANNER-ANALYZERS-LANG-10-309R — Package plug-in manifest + Offline Kit documentation; ensure Worker integration. + • Prereqs: SCANNER-ANALYZERS-LANG-10-308R (Wave 5) + • Current: TODO +- **Sprint 15** · Notify Foundations + - Team: Notify Connectors Guild + - Path: `src/StellaOps.Notify.Connectors.Email/TASKS.md` + 1. [TODO] NOTIFY-CONN-EMAIL-15-703 — Package Email connector as restart-time plug-in (manifest + host registration). + • Prereqs: NOTIFY-CONN-EMAIL-15-702 (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Slack/TASKS.md` + 1. [TODO] NOTIFY-CONN-SLACK-15-503 — Package Slack connector as restart-time plug-in (manifest + host registration). + • Prereqs: NOTIFY-CONN-SLACK-15-502 (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Teams/TASKS.md` + 1. [TODO] NOTIFY-CONN-TEAMS-15-603 — Package Teams connector as restart-time plug-in (manifest + host registration). + • Prereqs: NOTIFY-CONN-TEAMS-15-602 (Wave 5) + • Current: TODO + - Path: `src/StellaOps.Notify.Connectors.Webhook/TASKS.md` + 1. [TODO] NOTIFY-CONN-WEBHOOK-15-803 — Package Webhook connector as restart-time plug-in (manifest + host registration). + • Prereqs: NOTIFY-CONN-WEBHOOK-15-802 (Wave 5) + • Current: TODO + +## Wave 7 — 1 task(s) ready after Wave 6 +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Core Engine & Storage Analytics + - Path: `src/StellaOps.Concelier.Core/TASKS.md` + 1. [DONE] FEEDCORE-ENGINE-07-001 — FEEDCORE-ENGINE-07-001 – Advisory event log & asOf queries + • Prereqs: FEEDSTORAGE-DATA-07-001 (Wave 10) + • Current: DONE (2025-10-19) – `AdvisoryEventLog` service and repository abstractions landed with canonical hashing, lower-cased keys, replay API, and doc updates. Tests: `dotnet test src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj`. + +## Wave 8 — 1 task(s) ready after Wave 7 +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Core Engine & Data Science + - Path: `src/StellaOps.Concelier.Core/TASKS.md` + 1. [TODO] FEEDCORE-ENGINE-07-002 — FEEDCORE-ENGINE-07-002 – Noise prior computation service + • Prereqs: FEEDCORE-ENGINE-07-001 (Wave 7) + • Current: TODO – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Excititor/scan suppressors with reproducible statistics. + +## Wave 9 — 1 task(s) ready after Wave 8 +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Core Engine & Storage Analytics + - Path: `src/StellaOps.Concelier.Core/TASKS.md` + 1. [TODO] FEEDCORE-ENGINE-07-003 — FEEDCORE-ENGINE-07-003 – Unknown state ledger & confidence seeding + • Prereqs: FEEDCORE-ENGINE-07-001 (Wave 7) + • Current: TODO – Persist `unknown_vuln_range/unknown_origin/ambiguous_fix` markers with initial confidence bands, expose query surface for Policy, and add fixtures validating canonical serialization. + +## Wave 10 — 1 task(s) ready after Wave 9 +- **Sprint 7** · Contextual Truth Foundations + - Team: Team Normalization & Storage Backbone + - Path: `src/StellaOps.Concelier.Storage.Mongo/TASKS.md` + 1. [TODO] FEEDSTORAGE-DATA-07-001 — FEEDSTORAGE-DATA-07-001 Advisory statement & conflict collections + • Prereqs: FEEDMERGE-ENGINE-07-001 (Wave 11) + • Current: TODO – Create `advisory_statements` (immutable) and `advisory_conflicts` collections, define `asOf`/`vulnerabilityKey` indexes, and document migration/rollback steps for event-sourced merge. + +## Wave 11 — 1 task(s) ready after Wave 10 +- **Sprint 7** · Contextual Truth Foundations + - Team: BE-Merge + - Path: `src/StellaOps.Concelier.Merge/TASKS.md` + 1. [TODO] FEEDMERGE-ENGINE-07-001 — FEEDMERGE-ENGINE-07-001 Conflict sets & explainers + • Prereqs: FEEDSTORAGE-DATA-07-001 (Wave 10) + • Current: TODO – Persist conflict sets referencing advisory statements, output rule/explainer payloads with replay hashes, and add integration tests covering deterministic `asOf` evaluations. + +## Wave 12 — 1 task(s) ready after Wave 11 +- **Sprint 8** · Mirror Distribution + - Team: Concelier Export Guild + - Path: `src/StellaOps.Concelier.Exporter.Json/TASKS.md` + 1. [DONE] CONCELIER-EXPORT-08-201 — CONCELIER-EXPORT-08-201 – Mirror bundle + domain manifest + • Prereqs: FEEDCORE-ENGINE-07-001 (Wave 7) + • Current: DONE (2025-10-19) – Mirror bundles + manifests + signed index shipped; regression coverage via `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj` (2025-10-19). + +## Wave 13 — 1 task(s) ready after Wave 12 +- **Sprint 8** · Mirror Distribution + - Team: Concelier Export Guild + - Path: `src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md` + 1. [DONE] CONCELIER-EXPORT-08-202 — CONCELIER-EXPORT-08-202 – Mirror-ready Trivy DB bundles + • Prereqs: CONCELIER-EXPORT-08-201 (Wave 12) + • Current: DONE (2025-10-19) – Trivy exporter mirror options produce `mirror/index.json` plus per-domain manifest/metadata/db files with reproducible SHA-256 digests; validated via `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj`. + +## Wave 14 — 1 task(s) ready after Wave 13 +- **Sprint 8** · Mirror Distribution + - Team: Concelier WebService Guild + - Path: `src/StellaOps.Concelier.WebService/TASKS.md` + 1. [DOING] CONCELIER-WEB-08-201 — CONCELIER-WEB-08-201 – Mirror distribution endpoints + • Prereqs: CONCELIER-EXPORT-08-201 (Wave 12), DEVOPS-MIRROR-08-001 (Wave 2) + • Current: DOING (2025-10-19) – Wiring API surface against exporter-delivered `mirror/index.json` + signed bundles, layering quota/auth and updating docs/test fixtures for downstream sync. + +## Wave 15 — 1 task(s) ready after Wave 14 +- **Sprint 8** · Mirror Distribution + - Team: BE-Conn-Stella + - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` + 1. [DOING] FEEDCONN-STELLA-08-001 — Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. + • Prereqs: CONCELIER-EXPORT-08-201 (Wave 12) + • Current: DOING (2025-10-19) – Client consuming new signed mirror bundles/index, standing up verification + storage plumbing ahead of DTO mapping. + +## Wave 16 — 1 task(s) ready after Wave 15 +- **Sprint 8** · Mirror Distribution + - Team: BE-Conn-Stella + - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` + 1. [TODO] FEEDCONN-STELLA-08-002 — Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. + • Prereqs: FEEDCONN-STELLA-08-001 (Wave 15) + • Current: TODO + +## Wave 17 — 1 task(s) ready after Wave 16 +- **Sprint 8** · Mirror Distribution + - Team: BE-Conn-Stella + - Path: `src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md` + 1. [TODO] FEEDCONN-STELLA-08-003 — Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. + • Prereqs: FEEDCONN-STELLA-08-002 (Wave 16) + • Current: TODO diff --git a/Mongo2Go-4.1.0.tar.gz b/Mongo2Go-4.1.0.tar.gz new file mode 100644 index 00000000..6f154723 Binary files /dev/null and b/Mongo2Go-4.1.0.tar.gz differ diff --git a/Mongo2Go-4.1.0/.gitattributes b/Mongo2Go-4.1.0/.gitattributes new file mode 100644 index 00000000..adea2ff2 --- /dev/null +++ b/Mongo2Go-4.1.0/.gitattributes @@ -0,0 +1 @@ +*.nuspec text eol=lf \ No newline at end of file diff --git a/Mongo2Go-4.1.0/.github/workflows/continuous-integration.yml b/Mongo2Go-4.1.0/.github/workflows/continuous-integration.yml new file mode 100644 index 00000000..aaa4d8e4 --- /dev/null +++ b/Mongo2Go-4.1.0/.github/workflows/continuous-integration.yml @@ -0,0 +1,76 @@ +name: Continuous Integration + +on: + push: + branches: + - '**' # Trigger on all branches for commits + tags: + - 'v*' # Trigger only on version tags for deployments + +env: + Configuration: Release + ContinuousIntegrationBuild: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_NOLOGO: true + +jobs: + build: + strategy: + matrix: + os: [macos-latest, ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + name: Build and Test + steps: + - name: Install libssl1.1 (restores libcrypto.so.1.1 which is required by MongoDB binaries v4.4.4) + if: runner.os == 'Linux' + run: | + echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list + sudo apt update + sudo apt install -y libssl1.1 + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Retrieve cached NuGet packages + uses: actions/cache@v4 + with: + path: ~/.nuget/packages + key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }} + - name: Restore NuGet packages + run: dotnet restore --locked-mode --verbosity normal + - name: Build solution + run: dotnet build --configuration ${{ env.Configuration }} --verbosity normal + - name: Run tests + run: dotnet test --configuration ${{ env.Configuration }} --no-build --verbosity normal + + publish: + runs-on: macos-latest + needs: build + if: startsWith(github.ref, 'refs/tags/') + name: Deploy NuGet and GitHub Release + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Restore NuGet packages + run: dotnet restore --locked-mode --verbosity normal + - name: Build solution + run: dotnet build --configuration ${{ env.Configuration }} --verbosity normal + - name: Create NuGet package + run: dotnet pack --output ./artifacts --configuration ${{ env.Configuration }} --verbosity normal + - name: Upload NuGet package artifact + uses: actions/upload-artifact@v4 + with: + name: mongo2go-nuget-package + path: ./artifacts/*.nupkg + - name: Publish NuGet package + run: dotnet nuget push ./artifacts/*.nupkg --source https://api.nuget.org/v3/index.json --api-key "${{ secrets.NUGET_API_KEY }}" --skip-duplicate + - name: Create GitHub Release + run: | + gh release create ${{ github.ref_name }} ./artifacts/*.nupkg \ + --title "${{ github.ref_name }}" \ + --notes "A new release has been created. Please update the release notes manually with details about changes and improvements." \ + --draft + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/.gitignore b/Mongo2Go-4.1.0/.gitignore new file mode 100644 index 00000000..709ae0f9 --- /dev/null +++ b/Mongo2Go-4.1.0/.gitignore @@ -0,0 +1,14 @@ +src/Mongo2Go/bin/ +src/Mongo2GoTests/bin/ +src/MongoDownloader/bin/ +src/packages/ +obj/ +*ReSharper* +*.suo +*.dotCover +*.user +~$* +*/StyleCop.Cache +*.nupkg +**/.vs +.idea/ diff --git a/Mongo2Go-4.1.0/LICENSE b/Mongo2Go-4.1.0/LICENSE new file mode 100644 index 00000000..5f8753cb --- /dev/null +++ b/Mongo2Go-4.1.0/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2012-2025 Johannes Hoppe and many ❤️ contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Mongo2Go-4.1.0/Mongo2Go.sln b/Mongo2Go-4.1.0/Mongo2Go.sln new file mode 100644 index 00000000..f6aa5d6c --- /dev/null +++ b/Mongo2Go-4.1.0/Mongo2Go.sln @@ -0,0 +1,48 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27004.2005 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{0B557702-3C09-4514-BDD5-55A44F22113F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mongo2Go", "src\Mongo2Go\Mongo2Go.csproj", "{040A1626-1D04-40D6-BCCF-2D207AE648FC}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mongo2GoTests", "src\Mongo2GoTests\Mongo2GoTests.csproj", "{ADE5A672-6A00-4561-BCC1-E5497016DE24}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MongoDownloader", "src\MongoDownloader\MongoDownloader.csproj", "{7E10E0DE-8092-4ECB-B05A-0A15472AB8D2}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{56AB91A3-555C-4D59-BB92-570465DC2CA0}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {040A1626-1D04-40D6-BCCF-2D207AE648FC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {040A1626-1D04-40D6-BCCF-2D207AE648FC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {040A1626-1D04-40D6-BCCF-2D207AE648FC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {040A1626-1D04-40D6-BCCF-2D207AE648FC}.Release|Any CPU.Build.0 = Release|Any CPU + {ADE5A672-6A00-4561-BCC1-E5497016DE24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ADE5A672-6A00-4561-BCC1-E5497016DE24}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ADE5A672-6A00-4561-BCC1-E5497016DE24}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ADE5A672-6A00-4561-BCC1-E5497016DE24}.Release|Any CPU.Build.0 = Release|Any CPU + {7E10E0DE-8092-4ECB-B05A-0A15472AB8D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7E10E0DE-8092-4ECB-B05A-0A15472AB8D2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7E10E0DE-8092-4ECB-B05A-0A15472AB8D2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7E10E0DE-8092-4ECB-B05A-0A15472AB8D2}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {040A1626-1D04-40D6-BCCF-2D207AE648FC} = {0B557702-3C09-4514-BDD5-55A44F22113F} + {ADE5A672-6A00-4561-BCC1-E5497016DE24} = {0B557702-3C09-4514-BDD5-55A44F22113F} + {7E10E0DE-8092-4ECB-B05A-0A15472AB8D2} = {0B557702-3C09-4514-BDD5-55A44F22113F} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {08364BFD-1801-4718-83F5-F6F99950B05E} + EndGlobalSection +EndGlobal diff --git a/Mongo2Go-4.1.0/README.md b/Mongo2Go-4.1.0/README.md new file mode 100644 index 00000000..b32b3b31 --- /dev/null +++ b/Mongo2Go-4.1.0/README.md @@ -0,0 +1,565 @@ +Mongo2Go - MongoDB for integration tests & local debugging +======== + +![Logo](src/mongo2go_200_200.png) + +[![NuGet](https://img.shields.io/nuget/v/Mongo2Go.svg?label=NuGet&logo=NuGet)](https://www.nuget.org/packages/Mongo2Go/) + + +Mongo2Go is a managed wrapper around MongoDB binaries. +It targets **.NET Framework 4.7.2** and **.NET Standard 2.1.** and works with Windows, Linux and macOS. +This Nuget package contains the executables of _mongod_, _mongoimport_ and _mongoexport_ **for Windows, Linux and macOS** . + +__Brought to you by [Johannes Hoppe](https://twitter.com/johanneshoppe) with the help of many ❤️ contributors!__ + +Mongo2Go has two use cases: + +1. Providing multiple, temporary and isolated MongoDB databases for integration tests +2. Providing a quick to set up MongoDB database for a local developer environment + + +Integration test +------------------------------------- +With each call of the static method **MongoDbRunner.Start()** a new MongoDB instance will be set up. +A free port will be used (starting with port 27018) and a corresponding data directory will be created. +The method returns an instance of MongoDbRunner, which implements IDisposable. +As soon as the MongoDbRunner is disposed (or if the Finalizer is called by the GC), +the wrapped MongoDB process will be killed and all data in the data directory will be deleted. + + +Local debugging +------------------------ +In this mode a single MongoDB instance will be started on the default port (27017). +No data will be deleted and the MongoDB instance won’t be killed automatically. +Multiple calls to **MongoDbRunner.StartForDebugging()** will return an instance with the State “AlreadyRunning”. +You can ignore the IDisposable interface, as it won’t have any effect. +**I highly recommend to not use this mode on productive machines!** +Here you should set up a MongoDB as it is described in the manual. +For you convenience the MongoDbRunner also exposes _mongoexport_ and _mongoimport_ +which allow you to quickly set up a working environment. + + +Single server replica set mode to enable transactions +------------------------- +`MongoDbRunner.Start()` can be set up to take in an optional boolean parameter called `singleNodeReplSet`. +When passed in with the value `true` - (**`MongoDbRunner.Start(singleNodeReplSet: true)`**) +- a single node mongod instance will be started as a replica set with the name `singleNodeReplSet`. +Replica set mode is required for transactions to work in MongoDB 4.0 or greater + +Replica set initialization requires the use of a short delay to allow for the replica set to stabilize. This delay is linked to a timeout value of 5 seconds. + +If the timeout expires before the replica set has stabilized a `TimeoutException` will be thrown. + +The default timeout can be changed through the optional parameter `singleNodeReplSetWaitTimeout`, which allows values between 0 and 65535 seconds: **`MongoDbRunner.Start(singleNodeReplSet: true, singleNodeReplSetWaitTimeout: 10)`** + +Additional mongod arguments +--------------------------- +`MongoDbRunner.Start()` can be set up to consume additional `mongod` arguments. This can be done using the string parameter called `additionalMongodArguments`. + +The list of additional arguments cannot contain arguments already defined internally by Mongo2Go. An `ArgumentException` will be thrown in this case, specifying which additional arguments are required to be discarded. + +Example of usage of the additional `mongod` arguments: **`MongoDbRunner.Start(additionalMongodArguments: "--quiet")`** + +Installation +-------------- +The Mongo2Go Nuget package can be found at [https://nuget.org/packages/Mongo2Go/](https://nuget.org/packages/Mongo2Go/) +To install it via the .NET CLI, simply enter: + +```sh +dotnet add package Mongo2Go +``` + +* The new 4.x branch targets __.NET Framework 4.7.2__ and __.NET Standard 2.1__. Please use this version if possible. +* The old 3.x branch targets __.NET Standard 2.0__. No new features will be added, only bugfixes might be made. +* The old 2.x branch targets __.NET Standard 1.6__. No new features will be added, only bugfixes might be made. +* The old 1.x branch targets good-old classic __.NET 4.6.1__. This is for legacy environments only. No changes will be made. + + +Examples +-------- + +**Example: Integration Test (here: Machine.Specifications & Fluent Assertions)** + +```c# +[Subject("Runner Integration Test")] +public class when_using_the_inbuild_serialization : MongoIntegrationTest +{ + static TestDocument findResult; + + Establish context = () => + { + CreateConnection(); + _collection.Insert(TestDocument.DummyData1()); + }; + + Because of = () => findResult = _collection.FindOneAs(); + + It should_return_a_result = () => findResult.ShouldNotBeNull(); + It should_hava_expected_data = () => findResult.ShouldHave().AllPropertiesBut(d => d.Id).EqualTo(TestDocument.DummyData1()); + + Cleanup stuff = () => _runner.Dispose(); +} + +public class MongoIntegrationTest +{ + internal static MongoDbRunner _runner; + internal static MongoCollection _collection; + + internal static void CreateConnection() + { + _runner = MongoDbRunner.Start(); + + MongoClient client = new MongoClient(_runner.ConnectionString); + MongoDatabase database = client.GetDatabase("IntegrationTest"); + _collection = database.GetCollection("TestCollection"); + } +} +``` + +More tests can be found at https://github.com/Mongo2Go/Mongo2Go/tree/master/src/Mongo2GoTests/Runner + +**Example: Exporting seed data** + +```c# +using (MongoDbRunner runner = MongoDbRunner.StartForDebugging()) { + + runner.Export("TestDatabase", "TestCollection", @"..\..\App_Data\test.json"); +} +``` + +**Example: Importing for local debugging (compatible with ASP.NET MVC 4 Web API as well as ASP.NET Core)** + +```c# +public class WebApiApplication : System.Web.HttpApplication +{ + private MongoDbRunner _runner; + + protected void Application_Start() + { + _runner = MongoDbRunner.StartForDebugging(); + _runner.Import("TestDatabase", "TestCollection", @"..\..\App_Data\test.json", true); + + MongoClient client = new MongoClient(_runner.ConnectionString); + MongoDatabase database = client.GetDatabase("TestDatabase"); + MongoCollection collection = database.GetCollection("TestCollection"); + + /* happy coding! */ + } + + protected void Application_End() + { + _runner.Dispose(); + } +} +``` + +**Example: Transactions (New feature since v2.2.8)** + +
+ Full integration test with transaction handling (click to show) + + +```c# + public class when_transaction_completes : MongoTransactionTest + { + private static TestDocument mainDocument; + private static TestDocument dependentDocument; + Establish context = () => + + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + client = new MongoClient(_runner.ConnectionString); + database = client.GetDatabase(_databaseName); + _mainCollection = database.GetCollection(_mainCollectionName); + _dependentCollection = database.GetCollection(_dependentCollectionName); + _mainCollection.InsertOne(TestDocument.DummyData2()); + _dependentCollection.InsertOne(TestDocument.DummyData2()); + }; + + private Because of = () => + { + var filter = Builders.Filter.Where(x => x.IntTest == 23); + var update = Builders.Update.Inc(i => i.IntTest, 10); + + using (var sessionHandle = client.StartSession()) + { + try + { + var i = 0; + while (i < 10) + { + try + { + i++; + sessionHandle.StartTransaction(new TransactionOptions( + readConcern: ReadConcern.Local, + writeConcern: WriteConcern.W1)); + try + { + var first = _mainCollection.UpdateOne(sessionHandle, filter, update); + var second = _dependentCollection.UpdateOne(sessionHandle, filter, update); + } + catch (Exception e) + { + sessionHandle.AbortTransaction(); + throw; + } + + var j = 0; + while (j < 10) + { + try + { + j++; + sessionHandle.CommitTransaction(); + break; + } + catch (MongoException e) + { + if (e.HasErrorLabel("UnknownTransactionCommitResult")) + continue; + throw; + } + } + break; + } + catch (MongoException e) + { + if (e.HasErrorLabel("TransientTransactionError")) + continue; + throw; + } + } + } + catch (Exception e) + { + //failed after multiple attempts so log and do what is appropriate in your case + } + } + + mainDocument = _mainCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + dependentDocument = _dependentCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + }; + + It main_should_be_33 = () => mainDocument.IntTest.Should().Be(33); + It dependent_should_be_33 = () => dependentDocument.IntTest.Should().Be(33); + Cleanup cleanup = () => _runner.Dispose(); + } + +``` +
+ +**Example: Logging with `ILogger`** +
+ Wire mongod's logs at info and above levels to a custom `ILogger` (click to show) + +```c# +public class MongoIntegrationTest +{ + internal static MongoDbRunner _runner; + + internal static void CreateConnection() + { + // Create a custom logger. + // Replace this code with your own configuration of an ILogger. + var provider = new ServiceCollection() + .AddLogging(config => + { + // Log to a simple console and to event logs. + config.AddSimpleConsole(); + config.AddEventLog(); + }) + .BuildServiceProvider(); + var logger = provider.GetSerivce().CreateLogger("Mongo2Go"); + + _runner = MongoDbRunner.Start(logger: logger); + } +} +``` +
+ +
+ Wire mongod's logs at debug levels to a custom `ILogger` (click to show) + +```c# +public class MongoIntegrationTest +{ + internal static MongoDbRunner _runner; + + internal static void CreateConnection() + { + // Create a custom logger. + // Replace this code with your own configuration of an ILogger. + var provider = new ServiceCollection() + .AddLogging(config => + { + // Mongod's D1-D2 levels are logged with Debug level. + // D3-D5 levels are logged with Trace level. + config.SetMinimumLevel(LogLevel.Trace); + + // Log to System.Diagnostics.Debug and to the event source. + config.AddDebug(); + config.AddEventSourceLogger(); + }) + .BuildServiceProvider(); + var logger = provider.GetSerivce().CreateLogger("Mongo2Go"); + + _runner = MongoDbRunner.Start( + additionalMongodArguments: "vvvvv", // Tell mongod to output its D5 level logs + logger: logger); + } +} +``` +
+ +Changelog +------------------------------------- + +### Mongo2Go 4.1.0, January 30 2025 + +- Updated **MongoDB.Driver** to version **3.1.0**, ensuring compatibility with the latest MongoDB client features (PR [#156](https://github.com/Mongo2Go/Mongo2Go/pull/156), fixes [#154](https://github.com/Mongo2Go/Mongo2Go/issues/154) - many thanks to [Teneko](https://github.com/teneko)) +- Please note that the bundled version of MongoDB included with this package remains **v4.4.4**. +- **Note for Ubuntu users**: MongoDB 4.4.4 requires **libcrypto.so.1.1**, which is no longer included in Ubuntu 22.04 and newer. If you encounter an error like: + ``` + error while loading shared libraries: libcrypto.so.1.1: cannot open shared object file: No such file or directory + ``` + You can fix this by installing OpenSSL 1.1 manually: + ```bash + echo "deb http://security.ubuntu.com/ubuntu focal-security main" | sudo tee /etc/apt/sources.list.d/focal-security.list + sudo apt update + sudo apt install -y libssl1.1 + ``` + This restores `libcrypto.so.1.1` and allows Mongo2Go/MongoDB to run properly. + + +### Mongo2Go 4.0.0, November 19 2024 + +- A big thank you to [DrewM-Hax0r](https://github.com/DrewM-Hax0r) for championing this release! (PR [#153](https://github.com/Mongo2Go/Mongo2Go/pull/153), fixes [#152](https://github.com/Mongo2Go/Mongo2Go/issues/152)) +- This is a new major version for Mongo2Go (4.x), driven by: + - Dropping support for old .NET Framework versions earlier than 4.7.2 due to updated framework targets. + - MongoDB driver switching to strong-named assemblies (see [.NET Driver Version 2.28.0 Release Notes](https://www.mongodb.com/community/forums/t/net-driver-2-28-0-released/289745)). +- Updated **MongoDB driver to version 3** and re-targeted the project to meet the requirements of the new driver version. +- Fixed an issue with the single-node replica set option, caused by outdated connection strings that were incompatible with the latest MongoDB driver. +- Replaced deprecated dependent packages with updated, supported versions, and patched vulnerabilities by upgrading vulnerable dependencies. +- Please note that the bundled version of MongoDB included with this package has not changed and is still **v4.4.4**. This version of MongoDB is still compatible with the latest version of the driver, so there was no need to update at this time. +- **Bugfix:** Corrected binary search path on Linux when `NUGET_PACKAGES` is specified (PR [#140](https://github.com/Mongo2Go/Mongo2Go/pull/140), fixes [#134](https://github.com/Mongo2Go/Mongo2Go/issues/134) - many thanks to [Ove Andersen](https://github.com/azzlack)) +- **Bugfix**: Stops extra empty temporary data being generated (PR [#138](https://github.com/Mongo2Go/Mongo2Go/pull/138), fixes [#136](https://github.com/Mongo2Go/Mongo2Go/issues/136) - many thanks to [Alex Wardle](https://github.com/awardle)) + +
+ Changelog v3.0.0 to 3.1.3 (click to show) + +### Mongo2Go 3.1.3, April 30 2021 + +* targeting .NET Standard 2.0 instead of 2.1, this makes Mongo2Go compatible with .NET Framework (version 4.7.1 and later) (PR [#118](https://github.com/Mongo2Go/Mongo2Go/pull/118) - many thanks to [Cédric Luthi](https://github.com/0xced)) +* fixes handling of the path search for the NUGET_PACKAGE environment variable (PR [#119](https://github.com/Mongo2Go/Mongo2Go/pull/119) - many thanks to [Timm Hoffmeister](https://github.com/vader1986)) +* internal: `dotnet pack` is now used to create the nupkg file for a release (PR [#121](https://github.com/Mongo2Go/Mongo2Go/pull/121) - many thanks to [Cédric Luthi](https://github.com/0xced)) + +### Mongo2Go 3.1.1, April 08 2021 + +* internal: Better algorithm for determining a free port. This allows parallel execution of tests and increases compatibility with Raider and other test runners. (PR [#116](https://github.com/Mongo2Go/Mongo2Go/pull/116), fixes [#115](https://github.com/Mongo2Go/Mongo2Go/issues/115) and [#106](https://github.com/Mongo2Go/Mongo2Go/issues/106) - many thanks to [liangshiwei](https://github.com/realLiangshiwei)) + +### Mongo2Go 3.1.0, April 07 2021 + +* **NEW: Configurable logging!** adds the option to inject a `Microsoft.Extensions.Logging.ILogger` to `MongoDbRunner.Start(logger)` arguments. Now you can adjust or disable the console output to avoid noise in CI environments. Please note the two examples shown above. (PR [#113](https://github.com/Mongo2Go/Mongo2Go/pull/113), fixes [#94](https://github.com/Mongo2Go/Mongo2Go/issues/94), [#95](https://github.com/Mongo2Go/Mongo2Go/issues/95) and [#113](https://github.com/Mongo2Go/Mongo2Go/issues/113) - many thanks to [Corentin Altepe](https://github.com/corentinaltepe)) +* internal: replaces `--sslMode disabled` (deprecated) with `--tlsMode disabled` in command line arguments to mongod. + +### Mongo2Go 3.0.0, March 26 2021 + +* includes MongoDB binaries of **version 4.4.4** with support for Windows, Linux and macOS +* targets **.NET Standard 2.1** (can be used with .NET Core 3.0 and .NET 5.0) + +* adds new MongoDownloader tool (PR [#109](https://github.com/Mongo2Go/Mongo2Go/pull/109), fixes [#82](https://github.com/Mongo2Go/Mongo2Go/issues/82) and [#112](https://github.com/Mongo2Go/Mongo2Go/issues/112) - many thanks to [Cédric Luthi](https://github.com/0xced)) +* adds support for `NUGET_PACKAGES` environment variable (PR [#110](https://github.com/Mongo2Go/Mongo2Go/pull/110) - many thanks to [Bastian Eicher](https://github.com/bastianeicher)) + +
+ +
+ Changelog v2.0.0-alpha1 to v2.2.16 (click to show) + +### Mongo2Go 2.2.16, December 13 2020 + +* fix for non existing starting path for binary search (PR [#107](https://github.com/Mongo2Go/Mongo2Go/pull/107), fixes [#105](https://github.com/Mongo2Go/Mongo2Go/issues/105) - many thanks to [Gurov Yury](https://github.com/kenoma)) + +### Mongo2Go 2.2.15, December 12 2020 + +* throw exception if cluster is not ready for transactions after `singleNodeReplSetWaitTimeout` (PR [#103](https://github.com/Mongo2Go/Mongo2Go/pull/103) - many thanks for the continued support by [José Mira](https://github.com/zmira)) +s +### Mongo2Go 2.2.14, October 17 2020 + +* fixes a bug with pulling mongo binaries from wrong version (PR [#87](https://github.com/Mongo2Go/Mongo2Go/pull/87), fixes [#86](https://github.com/Mongo2Go/Mongo2Go/issues/86) - many thanks to [mihevc](https://github.com/mihevc)) +* ensures transaction is ready (solves error message: `System.NotSupportedException : StartTransaction cannot determine if transactions are supported because there are no connected servers.`) (PR [#101](https://github.com/Mongo2Go/Mongo2Go/pull/101), fixes [#89](https://github.com/Mongo2Go/Mongo2Go/issues/89), [#91](https://github.com/Mongo2Go/Mongo2Go/issues/91) and [#100](https://github.com/Mongo2Go/Mongo2Go/issues/100) - many thanks to [liangshiwei](https://github.com/realLiangshiwei)) + +### Mongo2Go 2.2.12, September 07 2019 +* performance: waits for replica set ready log message, or throws if timeout expires, instead of using `Thread.Sleep(5000)` (PR [#83](https://github.com/Mongo2Go/Mongo2Go/pull/83), fixes [#80](https://github.com/Mongo2Go/Mongo2Go/issues/80) - many thanks again to [José Mira](https://github.com/zmira)) + +### Mongo2Go 2.2.11, May 10 2019 +* allows additional custom MongoDB arguments (PR [#69](https://github.com/Mongo2Go/Mongo2Go/pull/69), fixes [#68](https://github.com/Mongo2Go/Mongo2Go/issues/68) - many thanks to [José Mira](https://github.com/zmira)) +* adds option to set port for `StartForDebugging()` (PR [#72](https://github.com/Mongo2Go/Mongo2Go/pull/72), fixes [#71](https://github.com/Mongo2Go/Mongo2Go/issues/71) - many thanks to [Danny Bies](https://github.com/dannyBies)) + +### Mongo2Go 2.2.9, February 04 2019 +* fixes a file path issue on Linux if you run on an SDK version beyond .NET Standard 1.6 (PR [#63](https://github.com/Mongo2Go/Mongo2Go/pull/63), fixes [#62](https://github.com/Mongo2Go/Mongo2Go/issues/62) and [#61](https://github.com/Mongo2Go/Mongo2Go/issues/61)) - many thanks to [Jeroen Vannevel](https://github.com/Vannevelj)) +* continuous integration runs on Linux (Travis CI) and Windows (AppVeyor) now + +### Mongo2Go 2.2.8, October 12 2018 +* updated MongoDB binaries to 4.0.2 to support tests leveraging transaction across different collections and databases +* updated MongoDB C# driver to 2.7.0 to be compatible with MongoDB 4.0 +* adds `singleNodeReplSet` paramter to `MongoDbRunner.Start` which allows mongod instance to be started as a replica set to enable transaction support (PR [#57](https://github.com/Mongo2Go/Mongo2Go/pull/57) - many thanks to [Mahi Satyanarayana](https://github.com/gbackmania)) +* fixes port lookup for UnixPortWatcher (PR [#58](https://github.com/Mongo2Go/Mongo2Go/pull/58) - many thanks to [Viktor Kolybaba](https://github.com/VikKol)) + +### Mongo2Go 2.2.7, August 13 2018 +* updates the `MongoBinaryLocator` to look for binaries in the nuget cache if they are not found in the project directory. + * this will make Mongo2Go compatible with projects using the nuget `PackageReference` option. (PR [#56](https://github.com/Mongo2Go/Mongo2Go/pull/56), fixes [#39](https://github.com/Mongo2Go/Mongo2Go/issues/39) and [#55](https://github.com/Mongo2Go/Mongo2Go/issues/55)) +* adds the `binariesSearchDirectory` parameter to `MongoDbRunner.Start` which allows an additional binaries search directory to be provided. + * this will make the db runner more flexible if someone decides to use it in some unpredictable way. +* many thanks to [Nicholas Markkula](https://github.com/nickmkk) + +### Mongo2Go 2.2.6, July 20 2018 +* fixes broken linux support (fixes [#47](https://github.com/Mongo2Go/Mongo2Go/issues/47)) + +### Mongo2Go 2.2.5, July 19 2018 +* fixes unresponsive process issue (PR [#52](https://github.com/Mongo2Go/Mongo2Go/pull/52), fixes [#49](https://github.com/Mongo2Go/Mongo2Go/issues/49)) +* many thanks to [narendrachava](https://github.com/narendrachava) + +### Mongo2Go 2.2.4, June 06 2018 +* better support for TeamCity: removed MaxLevelOfRecursion limitation when searching for MongoDb binaries (PR [#50](https://github.com/Mongo2Go/Mongo2Go/pull/50), fixes [#39](https://github.com/Mongo2Go/Mongo2Go/issues/39)) +* many thanks to [Stanko Culaja](https://github.com/culaja) + +### Mongo2Go 2.2.2, June 05 2018 +* includes mongod, mongoimport and mongoexport v3.6.1 for Windows, Linux and macOS via PR [#46](https://github.com/Mongo2Go/Mongo2Go/pull/46), which fixes [#45](https://github.com/Mongo2Go/Mongo2Go/issues/45) +* many thanks to [Joe Chan](https://github.com/joehmchan) + +### Mongo2Go 2.2.1, November 23 2017 +* no MongoDB binaries changed, still .NET Standard 1.6 +* feature: uses temporary directory instead of good-old windows style `C:\data\db` by default (PR [#42](https://github.com/Mongo2Go/Mongo2Go/pull/42)) - `MongoDbRunner.Start()` and `MongoDbRunner.StartForDebugging()` will now work without any extra parameters for Linux/macOS +* bugfix: runs again on Linux/macOS, by making the binaries executable (PR [#42](https://github.com/Mongo2Go/Mongo2Go/pull/42), which fixes [#37](https://github.com/Mongo2Go/Mongo2Go/issues/37) and might also fix [#43](https://github.com/Mongo2Go/Mongo2Go/issues/43)) +* internal: Unit Tests are running again (PR [#44](https://github.com/Mongo2Go/Mongo2Go/pull/44), which fixes [#31](https://github.com/Mongo2Go/Mongo2Go/issues/31), [#40](https://github.com/Mongo2Go/Mongo2Go/issues/40)) +* internal: No hardcoded path passed to MongoDbRunner constructor (fixes [41](https://github.com/Mongo2Go/Mongo2Go/issues/41)) +* many thanks to [Per Liedman](https://github.com/perliedman) + +### Mongo2Go 2.2.0, August 17 2017 +* includes mongod, mongoimport and mongoexport v3.4.7 for Windows, Linux and macOS +* targets .NET Standard 1.6 (can be used with .NET Core 1.0 / 1.1 / 2.0) +* many thanks to [Aviram Fireberger](https://github.com/avrum) + +### Mongo2Go 2.1.0, March 10 2017 +* skips v2.0 to have same numbers as v1.x. +* no MongoDB binaries changed since 2.0.0-alpha1 (still MongoDB v3.2.7 for Windows, Linux and macOS) +* targets .NET Standard 1.6 (can be used with .NET Core 1.0 / 1.1) +* bugfix: prevent windows firewall popup (PR [#30](https://github.com/Mongo2Go/Mongo2Go/pull/30), which fixes [#21](https://github.com/Mongo2Go/Mongo2Go/pull/21)) +* many thanks to [kubal5003](https://github.com/kubal5003) + +### Mongo2Go 1.1.0, March 10 2017 _(legacy branch!)_ +* no MongoDB binaries changed since v1.0 (still MongoDB v3.2.7 for Windows, Linux and macOS) +* targets .NET 4.6.1 +* bugfix: prevent windows firewall popup (PR [#29](https://github.com/Mongo2Go/Mongo2Go/pull/29), which fixes [#21](https://github.com/Mongo2Go/Mongo2Go/pull/21)) +* many thanks to [kubal5003](https://github.com/kubal5003) + + +### Mongo2Go 2.0.0-alpha1, December 19 2016 +* this version has no support for .NET Framework 4.6, please continue to use the stable package v.1.0.0 +* NEW: first support of .NET Standard 1.6 ([#25](https://github.com/Mongo2Go/Mongo2Go/pull/25)) + * many thanks to [Hassaan Ahmed](https://github.com/bannerflow-hassaan) + * see the [Wiki](https://github.com/Mongo2Go/Mongo2Go/wiki/NetStandard) for more information about .NET Core 1.0 / .NET Standard 1.6 + +
+ +
+ Changelog v0.1.0 to v1.0.0 (click to show) + +### Mongo2Go 1.0.0, November 14 2016 +* v1.0 finally marked as stable +* no changes to 1.0.0-beta4 +* changes since last stable version (0.2): + * includes mongod, mongoimport and mongoexport v3.2.7 for Windows, Linux and macOS + * support for Windows, Linux and macOS + * uses MongoDB.Driver 2.3.0 + * **requires .NET 4.6** + * various small bugfixes and improvements + +### Mongo2Go 1.0.0-beta4, October 24 2016 +* update to MongoDB.Driver 2.3.0 ([#23](https://github.com/Mongo2Go/Mongo2Go/pull/23)) +* upgraded to __.NET 4.6__ +* internal change: update MSpec as well and add MSTest Adapter for MSpec (ReSharper console runner doesn't support 4.6) +* many thanks to [Alexander Zeitler](https://github.com/AlexZeitler) +* please report any kind of [issues here on github](https://github.com/Mongo2Go/Mongo2Go/issues) so that we can mark 1.0.0 as stable! + +### Mongo2Go 1.0.0-beta3, August 22 2016 +* feature: process windows are hidden now ([#20](https://github.com/Mongo2Go/Mongo2Go/pull/20)) +* bugfix: random folders are used for storing databases ([#18](https://github.com/Mongo2Go/Mongo2Go/pull/18)) +* many thanks to [Matt Kocaj](https://github.com/cottsak) +* please report any kind of [issues here on github](https://github.com/Mongo2Go/Mongo2Go/issues) so that we can mark 1.0.0 as stable! + +### Mongo2Go 1.0.0-beta2, July 29 2016 +* fixes for bugs that were introduced by the big rewrite for cross-platform support +* changes from pull request [#14](https://github.com/Mongo2Go/Mongo2Go/pull/14), which fixes [#12](https://github.com/Mongo2Go/Mongo2Go/issues/12), [#13](https://github.com/Mongo2Go/Mongo2Go/issues/13) and [#15](https://github.com/Mongo2Go/Mongo2Go/issues/15), many thanks to [Mitch Ferrer](https://github.com/G3N7) +* please report any kind of [issues here on github](https://github.com/Mongo2Go/Mongo2Go/issues) so that we can mark 1.0.0 as stable! + + +### Mongo2Go 1.0.0-beta, July 24 2016 +* **:tada: NEW: support for Linux and macOS :tada:** +* many thanks to [Kristofer Linnestjerna](https://github.com/krippz) from [netclean.com](http://www.netclean.com/) for the new cross-platform support +* includes mongod, mongoimport and mongoexport v3.2.7 for Windows, Linux and macOS +* changes from pull request [#8](https://github.com/Mongo2Go/Mongo2Go/pull/8), [#10](https://github.com/Mongo2Go/Mongo2Go/pull/10), [#11](https://github.com/Mongo2Go/Mongo2Go/pull/11) which fixes [#9](https://github.com/Mongo2Go/Mongo2Go/issues/9) +* please report any kind of [issues here on github](https://github.com/Mongo2Go/Mongo2Go/issues) so that we can mark 1.0.0 as stable! + +### Mongo2Go 0.2, May 30 2016 +* includes mongod, mongoimport and mongoexport v3.2.6, + (**64bit** from [win32/mongodb-win32-x86_64-2008plus-3.2.6.zip](http://downloads.mongodb.org/win32/mongodb-win32-x86_64-2008plus-3.2.6.zip?_ga=1.190428203.1815541971.1457905247) since 32bit builds are deprecated now) +* removes outmoded Strong-Name signing from assemblies (please open an issue if you really need this, see also [mspec#190](https://github.com/machine/machine.specifications/issues/190)) +* changes from pull request [#7](https://github.com/Mongo2Go/Mongo2Go/pull/7), thanks to [Mitch Ferrer](https://github.com/G3N7) + +### Mongo2Go 0.1.8, March 13 2016 +* includes mongod, mongoimport and mongoexport v3.0.10 (32bit) +* changes from pull request [#5](https://github.com/Mongo2Go/Mongo2Go/pull/5), thanks to [Aristarkh Zagorodnikov](https://github.com/onyxmaster) + +### Mongo2Go 0.1.6, July 21 2015 +* includes mongod, mongoimport and mongoexport v3.0.4 (32bit) +* bug fix [#4](https://github.com/Mongo2Go/Mongo2Go/issues/4): +Sometimes the runner tries to delete the database directory before the mongod process has been stopped, this throws an IOException. +Now the runner waits until the mongod process has been stopped before the database directory will be deleted. +* Thanks [Sergey Zwezdin](https://github.com/sergun) + +### Mongo2Go 0.1.5, July 08 2015 +* includes mongod, mongoimport and mongoexport v2.6.6 (32bit) +* changes from pull request [#3](https://github.com/Mongo2Go/Mongo2Go/pull/3) +* new: `Start` and `StartForDebugging` methods accept an optional parameter to specify a different data directory (default is "C:\data\db") +* many thanks to [Marc](https://github.com/Silv3rcircl3) + +### Mongo2Go 0.1.4, January 26 2015 +* includes mongod, mongoimport and mongoexport v2.6.6 (32bit) +* changes from pull request [#2](https://github.com/Mongo2Go/Mongo2Go/pull/2) +* internal updates for testing the package (not part of the release) + * updated MSpec package so that it would work with the latest VS and R# test runner + * updated Mongo C# Driver, Fluent Assertions, and Moq packages to latest versions + * fixed date handling for mongoimport and mongoexport to pass tests +* many thanks to [Jesse Sweetland](https://github.com/sweetlandj) + +### Mongo2Go 0.1.3, September 20 2012 +* includes mongod, mongoimport and mongoexport v2.2.0 (32bit) + +### Mongo2Go 0.1.2, August 20 2012 +* stable version +* includes mongod, mongoimport and mongoexport v2.2.0-rc1 (32bit) + +### Mongo2Go 0.1.1, August 16 2012 +* second alpha version +* includes mongod, mongoimport and mongoexport v2.2.0-rc1 (32bit) + + +### Mongo2Go 0.1.0, August 15 2012 +* first alpha version +* includes mongod, mongoimport and mongoexport v2.2.0-rc1 (32bit) + +
+ +How to contribute +------------------------------------- + +Just fork the project, make your changes send us a PR. + +In the root folder, just run: +``` +dotnet restore +dotnet build +dotnet test src/Mongo2GoTests +``` diff --git a/Mongo2Go-4.1.0/README_INTERNAL.md b/Mongo2Go-4.1.0/README_INTERNAL.md new file mode 100644 index 00000000..4d7681ec --- /dev/null +++ b/Mongo2Go-4.1.0/README_INTERNAL.md @@ -0,0 +1,66 @@ +# Mongo2Go - Knowledge for Maintainers + +## Creating a Release + +Mongo2Go uses [MinVer](https://github.com/adamralph/minver) for versioning. +Releases are fully automated via GitHub Actions and triggered by tagging a commit with the desired semantic version number. +This process involves two steps to ensure reliable deployments. + +### Steps to Create a Release + +1. **Push Your Changes** + - Commit and push your changes to the main branch. This will trigger a CI build to validate the changes. + ```bash + git commit -m "Your commit message" + git push + ``` + +2. **Wait for the CI Build** + - Ensure that the GitHub Actions workflow completes successfully. This confirms your changes are valid. + +3. **Tag the Commit** + - Once the CI build passes, create a lightweight tag with the desired version number + - Use an **annotated tag** to ensure the release is properly versioned and auditable (`-a` flag): + ```bash + git tag -a v4.0.0 + ``` + - Push the tag to trigger the deployment workflow: + ```bash + git push --tags + ``` + +4. **Draft Release Created** + - The workflow will: + 1. Create a multi-target NuGet package. + 2. Publish the package to nuget.org. + 3. Create a **draft release** on GitHub with a placeholder note. + +5. **Review and Finalize the Release** + - Visit the [Releases page](https://github.com/Mongo2Go/Mongo2Go/releases). + - Open the draft release, update the release notes with details about the changes (e.g., changelog, features, fixes), and publish the release manually. + + +## Workflow Details + +- **Two-Step Process**: + 1. The first push (commit) triggers a CI build to validate the changes. + 2. The second push (tag) triggers the deployment workflow. + +- **Triggers**: + - Commits are validated for all branches. + - Tags starting with `v` trigger deployment. + +- **Draft Releases**: + - Releases are created as drafts, allowing maintainers to review and add release notes before publishing. + +- **Automation**: + - The workflow automates building, testing, publishing to nuget.org, and creating a draft GitHub release. + + +## Best Practices for Maintainers + +- **Semantic Versioning**: Ensure that tags follow the [semantic versioning](https://semver.org/) format (`vMAJOR.MINOR.PATCH`). +- **Pre-Releases**: Use pre-release tags for non-final versions (e.g., `v4.0.0-rc.1`). +- **Detailed Release Notes**: Always add detailed information to the GitHub release, highlighting major changes, fixes, and improvements. +- **Final Review**: Review the draft release to ensure all details are correct before publishing. + diff --git a/Mongo2Go-4.1.0/global.json b/Mongo2Go-4.1.0/global.json new file mode 100644 index 00000000..d4b092c1 --- /dev/null +++ b/Mongo2Go-4.1.0/global.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://json.schemastore.org/global", + "sdk": { + "allowPrerelease": false, + "rollForward": "latestMinor", + "version": "8.0.110" + } +} diff --git a/Mongo2Go-4.1.0/package_create.sh b/Mongo2Go-4.1.0/package_create.sh new file mode 100644 index 00000000..812d1e1c --- /dev/null +++ b/Mongo2Go-4.1.0/package_create.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# just to be sure +#git clean -fdx + +echo +echo "*** Your dotnet version:" +dotnet --version + +echo +echo "*** Creating package:" +dotnet pack --configuration Release src/Mongo2Go/Mongo2Go.csproj -p:ContinuousIntegrationBuild=true + +echo +echo "*** Package content:" +zipinfo src/Mongo2Go/bin/Release/Mongo2Go.*.nupkg \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FileSystem.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FileSystem.cs new file mode 100644 index 00000000..a2985009 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FileSystem.cs @@ -0,0 +1,44 @@ +using System.Diagnostics; +using System.IO; + +namespace Mongo2Go.Helper +{ + public class FileSystem : IFileSystem + { + public void CreateFolder(string path) + { + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + } + + public void DeleteFolder(string path) + { + if (Directory.Exists(path)) + { + Directory.Delete(path, true); + } + } + + public void DeleteFile(string fullFileName) + { + if (File.Exists(fullFileName)) + { + File.Delete(fullFileName); + } + } + + public void MakeFileExecutable (string path) + { + //when on linux or osx we must set the executeble flag on mongo binarys + var p = Process.Start("chmod", $"+x {path}"); + p.WaitForExit(); + + if (p.ExitCode != 0) + { + throw new IOException($"Could not set executable bit for {path}"); + } + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FolderSearch.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FolderSearch.cs new file mode 100644 index 00000000..6ea417b1 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/FolderSearch.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; + +namespace Mongo2Go.Helper +{ + public static class FolderSearch + { + private static readonly char[] _separators = { Path.DirectorySeparatorChar }; + + public static string CurrentExecutingDirectory() + { + string filePath = new Uri(typeof(FolderSearch).GetTypeInfo().Assembly.CodeBase).LocalPath; + return Path.GetDirectoryName(filePath); + } + + public static string FindFolder(this string startPath, string searchPattern) + { + if (startPath == null || searchPattern == null) + { + return null; + } + + string currentPath = startPath; + + foreach (var part in searchPattern.Split(_separators, StringSplitOptions.None)) + { + if (!Directory.Exists(currentPath)) + { + return null; + } + + string[] matchesDirectory = Directory.GetDirectories(currentPath, part); + if (!matchesDirectory.Any()) + { + return null; + } + + if (matchesDirectory.Length > 1) + { + currentPath = MatchVersionToAssemblyVersion(matchesDirectory) + ?? matchesDirectory.OrderBy(x => x).Last(); + } + else + { + currentPath = matchesDirectory.First(); + } + } + + return currentPath; + } + + public static string FindFolderUpwards(this string startPath, string searchPattern) + { + if (string.IsNullOrEmpty(startPath)) + { + return null; + } + + string matchingFolder = startPath.FindFolder(searchPattern); + return matchingFolder ?? startPath.RemoveLastPart().FindFolderUpwards(searchPattern); + } + + internal static string RemoveLastPart(this string path) + { + if (!path.Contains(Path.DirectorySeparatorChar)) + { + return null; + } + + List parts = path.Split(new[] { Path.DirectorySeparatorChar }, StringSplitOptions.None).ToList(); + parts.RemoveAt(parts.Count() - 1); + return string.Join(Path.DirectorySeparatorChar.ToString(), parts.ToArray()); + } + + /// + /// Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + /// + public static string FinalizePath(string fileName) + { + string finalPath; + + if (Path.IsPathRooted(fileName)) + { + finalPath = fileName; + } + else + { + finalPath = Path.Combine(CurrentExecutingDirectory(), fileName); + finalPath = Path.GetFullPath(finalPath); + } + + return finalPath; + } + + private static string MatchVersionToAssemblyVersion(string[] folders) + { + var version = typeof(FolderSearch).GetTypeInfo().Assembly.GetCustomAttribute().InformationalVersion; + + foreach (var folder in folders) + { + var lastFolder = new DirectoryInfo(folder).Name; + if (lastFolder == version) + return folder; + } + + return null; + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IFileSystem.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IFileSystem.cs new file mode 100644 index 00000000..bb827a88 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IFileSystem.cs @@ -0,0 +1,10 @@ +namespace Mongo2Go.Helper +{ + public interface IFileSystem + { + void CreateFolder(string path); + void DeleteFolder(string path); + void DeleteFile(string fullFileName); + void MakeFileExecutable (string path ); + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoBinaryLocator.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoBinaryLocator.cs new file mode 100644 index 00000000..8efde0ee --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoBinaryLocator.cs @@ -0,0 +1,7 @@ +namespace Mongo2Go.Helper +{ + public interface IMongoBinaryLocator + { + string Directory { get; } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcess.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcess.cs new file mode 100644 index 00000000..901aa74c --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcess.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace Mongo2Go.Helper +{ + public interface IMongoDbProcess : IDisposable + { + IEnumerable StandardOutput { get; } + IEnumerable ErrorOutput { get; } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcessStarter.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcessStarter.cs new file mode 100644 index 00000000..8fe9c3d3 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IMongoDbProcessStarter.cs @@ -0,0 +1,11 @@ +using Microsoft.Extensions.Logging; + +namespace Mongo2Go.Helper +{ + public interface IMongoDbProcessStarter + { + IMongoDbProcess Start(string binariesDirectory, string dataDirectory, int port, bool singleNodeReplSet, string additionalMongodArguments, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null); + + IMongoDbProcess Start(string binariesDirectory, string dataDirectory, int port, bool doNotKill, bool singleNodeReplSet, string additionalMongodArguments, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null); + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortPool.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortPool.cs new file mode 100644 index 00000000..13d1bc5b --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortPool.cs @@ -0,0 +1,10 @@ +namespace Mongo2Go.Helper +{ + public interface IPortPool + { + /// + /// Returns and reserves a new port + /// + int GetNextOpenPort(); + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortWatcher.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortWatcher.cs new file mode 100644 index 00000000..8a5e0404 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IPortWatcher.cs @@ -0,0 +1,8 @@ +namespace Mongo2Go.Helper +{ + public interface IPortWatcher + { + int FindOpenPort(); + bool IsPortAvailable(int portNumber); + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IProcessWatcher.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IProcessWatcher.cs new file mode 100644 index 00000000..98045ec3 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/IProcessWatcher.cs @@ -0,0 +1,7 @@ +namespace Mongo2Go.Helper +{ + public interface IProcessWatcher + { + bool IsProcessRunning(string processName); + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoBinaryLocator.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoBinaryLocator.cs new file mode 100644 index 00000000..96322ba0 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoBinaryLocator.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Mongo2Go.Helper +{ + + public class MongoBinaryLocator : IMongoBinaryLocator + { + private readonly string _nugetPrefix = Path.Combine("packages", "Mongo2Go*"); + private readonly string _nugetCachePrefix = Path.Combine("packages", "mongo2go", "*"); + private readonly string _nugetCacheBasePrefix = Path.Combine("mongo2go", "*"); + public const string DefaultWindowsSearchPattern = @"tools\mongodb-windows*\bin"; + public const string DefaultLinuxSearchPattern = "tools/mongodb-linux*/bin"; + public const string DefaultOsxSearchPattern = "tools/mongodb-macos*/bin"; + public const string WindowsNugetCacheLocation = @"%USERPROFILE%\.nuget\packages"; + public static readonly string OsxAndLinuxNugetCacheLocation = Environment.GetEnvironmentVariable("HOME") + "/.nuget/packages"; + private string _binFolder = string.Empty; + private readonly string _searchPattern; + private readonly string _nugetCacheDirectory; + private readonly string _additionalSearchDirectory; + + public MongoBinaryLocator(string searchPatternOverride, string additionalSearchDirectory) + { + _additionalSearchDirectory = additionalSearchDirectory; + _nugetCacheDirectory = Environment.GetEnvironmentVariable("NUGET_PACKAGES"); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + _searchPattern = DefaultOsxSearchPattern; + _nugetCacheDirectory = _nugetCacheDirectory ?? OsxAndLinuxNugetCacheLocation; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + _searchPattern = DefaultLinuxSearchPattern; + _nugetCacheDirectory = _nugetCacheDirectory ?? OsxAndLinuxNugetCacheLocation; + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + _searchPattern = DefaultWindowsSearchPattern; + _nugetCacheDirectory = _nugetCacheDirectory ?? Environment.ExpandEnvironmentVariables(WindowsNugetCacheLocation); + } + else + { + throw new MonogDbBinariesNotFoundException($"Unknown OS: {RuntimeInformation.OSDescription}"); + } + + if (!string.IsNullOrEmpty(searchPatternOverride)) + { + _searchPattern = searchPatternOverride; + } + } + + public string Directory { + get { + if (string.IsNullOrEmpty(_binFolder)){ + return _binFolder = ResolveBinariesDirectory (); + } else { + return _binFolder; + } + } + } + + private string ResolveBinariesDirectory() + { + var searchDirectories = new[] + { + // First search from the additional search directory, if provided + _additionalSearchDirectory, + // Then search from the project directory + FolderSearch.CurrentExecutingDirectory(), + // Finally search from the nuget cache directory + _nugetCacheDirectory + }; + return FindBinariesDirectory(searchDirectories.Where(x => !string.IsNullOrWhiteSpace(x)).ToList()); + } + + private string FindBinariesDirectory(IList searchDirectories) + { + foreach (var directory in searchDirectories) + { + var binaryFolder = + // First try just the search pattern + directory.FindFolderUpwards(_searchPattern) ?? + // Next try the search pattern with nuget installation prefix + directory.FindFolderUpwards(Path.Combine(_nugetPrefix, _searchPattern)) ?? + // Finally try the search pattern with the nuget cache prefix + directory.FindFolderUpwards(Path.Combine(_nugetCachePrefix, _searchPattern)) ?? + // Finally try the search pattern with the basic nuget cache prefix + directory.FindFolderUpwards(Path.Combine(_nugetCacheBasePrefix, _searchPattern)); + if (binaryFolder != null) return binaryFolder; + } + throw new MonogDbBinariesNotFoundException( + $"Could not find Mongo binaries using the search patterns \"{_searchPattern}\", \"{Path.Combine(_nugetPrefix, _searchPattern)}\", \"{Path.Combine(_nugetCachePrefix, _searchPattern)}\", and \"{Path.Combine(_nugetCacheBasePrefix, _searchPattern)}\". " + + $"You can override the search pattern and directory when calling MongoDbRunner.Start. We have detected the OS as {RuntimeInformation.OSDescription}.\n" + + $"We walked up to root directory from the following locations.\n {string.Join("\n", searchDirectories)}"); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.IDisposable.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.IDisposable.cs new file mode 100644 index 00000000..f8443bc8 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.IDisposable.cs @@ -0,0 +1,55 @@ +using System; + +namespace Mongo2Go.Helper +{ + // IDisposable and friends + public partial class MongoDbProcess + { + ~MongoDbProcess() + { + Dispose(false); + } + + public bool Disposed { get; private set; } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (Disposed) + { + return; + } + + if (disposing) + { + // we have no "managed resources" - but we leave this switch to avoid an FxCop CA1801 warnig + } + + if (_process == null) + { + return; + } + + if (_process.DoNotKill) + { + return; + } + + if (!_process.HasExited) + { + _process.Kill(); + _process.WaitForExit(); + } + + _process.Dispose(); + _process = null; + + Disposed = true; + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.cs new file mode 100644 index 00000000..68affad2 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcess.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +namespace Mongo2Go.Helper +{ + public partial class MongoDbProcess : IMongoDbProcess + { + + private WrappedProcess _process; + + public IEnumerable ErrorOutput { get; set; } + public IEnumerable StandardOutput { get; set; } + + internal MongoDbProcess(WrappedProcess process) + { + _process = process; + } + + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcessStarter.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcessStarter.cs new file mode 100644 index 00000000..275d346f --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoDbProcessStarter.cs @@ -0,0 +1,92 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.Core.Servers; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Mongo2Go.Helper +{ + public class MongoDbProcessStarter : IMongoDbProcessStarter + { + private const string ProcessReadyIdentifier = "waiting for connections"; + private const string Space = " "; + private const string ReplicaSetName = "singleNodeReplSet"; + private const string ReplicaSetReadyIdentifier = "transition to primary complete; database writes are now permitted"; + + /// + /// Starts a new process. Process can be killed + /// + public IMongoDbProcess Start(string binariesDirectory, string dataDirectory, int port, bool singleNodeReplSet, string additionalMongodArguments, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null) + { + return Start(binariesDirectory, dataDirectory, port, false, singleNodeReplSet, additionalMongodArguments, singleNodeReplSetWaitTimeout, logger); + } + + /// + /// Starts a new process. + /// + public IMongoDbProcess Start(string binariesDirectory, string dataDirectory, int port, bool doNotKill, bool singleNodeReplSet, string additionalMongodArguments, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null) + { + string fileName = @"{0}{1}{2}".Formatted(binariesDirectory, System.IO.Path.DirectorySeparatorChar.ToString(), MongoDbDefaults.MongodExecutable); + + string arguments = (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) ? + @"--dbpath ""{0}"" --port {1} --bind_ip 127.0.0.1".Formatted(dataDirectory, port) : + @"--tlsMode disabled --dbpath ""{0}"" --port {1} --bind_ip 127.0.0.1".Formatted(dataDirectory, port); + + arguments = singleNodeReplSet ? arguments + Space + "--replSet" + Space + ReplicaSetName : arguments; + arguments += MongodArguments.GetValidAdditionalArguments(arguments, additionalMongodArguments); + + WrappedProcess wrappedProcess = ProcessControl.ProcessFactory(fileName, arguments); + wrappedProcess.DoNotKill = doNotKill; + + ProcessOutput output = ProcessControl.StartAndWaitForReady(wrappedProcess, 5, ProcessReadyIdentifier, logger); + if (singleNodeReplSet) + { + var replicaSetReady = false; + + // subscribe to output from mongod process and check for replica set ready message + wrappedProcess.OutputDataReceived += (_, args) => replicaSetReady |= !string.IsNullOrWhiteSpace(args.Data) && args.Data.IndexOf(ReplicaSetReadyIdentifier, StringComparison.OrdinalIgnoreCase) >= 0; + + MongoClient client = new MongoClient("mongodb://127.0.0.1:{0}/?directConnection=true&replicaSet={1}".Formatted(port, ReplicaSetName)); + var admin = client.GetDatabase("admin"); + var replConfig = new BsonDocument(new List() + { + new BsonElement("_id", ReplicaSetName), + new BsonElement("members", + new BsonArray {new BsonDocument {{"_id", 0}, {"host", "127.0.0.1:{0}".Formatted(port)}}}) + }); + var command = new BsonDocument("replSetInitiate", replConfig); + admin.RunCommand(command); + + // wait until replica set is ready or until the timeout is reached + SpinWait.SpinUntil(() => replicaSetReady, TimeSpan.FromSeconds(singleNodeReplSetWaitTimeout)); + + if (!replicaSetReady) + { + throw new TimeoutException($"Replica set initialization took longer than the specified timeout of {singleNodeReplSetWaitTimeout} seconds. Please consider increasing the value of {nameof(singleNodeReplSetWaitTimeout)}."); + } + + // wait until transaction is ready or until the timeout is reached + SpinWait.SpinUntil(() => + client.Cluster.Description.Servers.Any(s => s.State == ServerState.Connected && s.IsDataBearing), + TimeSpan.FromSeconds(singleNodeReplSetWaitTimeout)); + + if (!client.Cluster.Description.Servers.Any(s => s.State == ServerState.Connected && s.IsDataBearing)) + { + throw new TimeoutException($"Cluster readiness for transactions took longer than the specified timeout of {singleNodeReplSetWaitTimeout} seconds. Please consider increasing the value of {nameof(singleNodeReplSetWaitTimeout)}."); + } + } + + MongoDbProcess mongoDbProcess = new MongoDbProcess(wrappedProcess) + { + ErrorOutput = output.ErrorOutput, + StandardOutput = output.StandardOutput + }; + + return mongoDbProcess; + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoImportExport.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoImportExport.cs new file mode 100644 index 00000000..c17dc20a --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoImportExport.cs @@ -0,0 +1,46 @@ +using System.Diagnostics; +using System.IO; + +namespace Mongo2Go.Helper +{ + public static class MongoImportExport + { + /// + /// Input File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + /// + public static ProcessOutput Import(string binariesDirectory, int port, string database, string collection, string inputFile, bool drop, string additionalMongodArguments = null) + { + string finalPath = FolderSearch.FinalizePath(inputFile); + + if (!File.Exists(finalPath)) + { + throw new FileNotFoundException("File not found", finalPath); + } + + string fileName = Path.Combine("{0}", "{1}").Formatted(binariesDirectory, MongoDbDefaults.MongoImportExecutable); + string arguments = @"--host localhost --port {0} --db {1} --collection {2} --file ""{3}""".Formatted(port, database, collection, finalPath); + if (drop) { arguments += " --drop"; } + arguments += MongodArguments.GetValidAdditionalArguments(arguments, additionalMongodArguments); + + Process process = ProcessControl.ProcessFactory(fileName, arguments); + + return ProcessControl.StartAndWaitForExit(process); + } + + /// + /// Output File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + /// + public static ProcessOutput Export(string binariesDirectory, int port, string database, string collection, string outputFile, string additionalMongodArguments = null) + { + string finalPath = FolderSearch.FinalizePath(outputFile); + + string fileName = Path.Combine("{0}", "{1}").Formatted(binariesDirectory, MongoDbDefaults.MongoExportExecutable); + string arguments = @"--host localhost --port {0} --db {1} --collection {2} --out ""{3}""".Formatted(port, database, collection, finalPath); + arguments += MongodArguments.GetValidAdditionalArguments(arguments, additionalMongodArguments); + + Process process = ProcessControl.ProcessFactory(fileName, arguments); + + return ProcessControl.StartAndWaitForExit(process); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoLogStatement.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoLogStatement.cs new file mode 100644 index 00000000..7abd08d3 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongoLogStatement.cs @@ -0,0 +1,77 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Mongo2Go.Helper +{ + /// + /// Structure of a log generated by mongod. Used to deserialize the logs + /// and pass them to an ILogger. + /// See: https://docs.mongodb.com/manual/reference/log-messages/#json-log-output-format + /// Note: "truncated" and "size" are not parsed as we're unsure how to + /// properly parse and use them. + /// + class MongoLogStatement + { + [JsonPropertyName("t")] + public MongoDate MongoDate { get; set; } + + /// + /// Severity of the logs as defined by MongoDB. Mapped to LogLevel + /// as defined by Microsoft. + /// D1-D2 mapped to Debug level. D3-D5 mapped Trace level. + /// + [JsonPropertyName("s")] + public string Severity { get; set; } + + public LogLevel Level + { + get + { + if (string.IsNullOrEmpty(Severity)) + return LogLevel.None; + switch (Severity) + { + case "F": return LogLevel.Critical; + case "E": return LogLevel.Error; + case "W": return LogLevel.Warning; + case "I": return LogLevel.Information; + case "D": + case "D1": + case "D2": + return LogLevel.Debug; + case "D3": + case "D4": + case "D5": + default: + return LogLevel.Trace; + } + } + } + + [JsonPropertyName("c")] + public string Component { get; set; } + + [JsonPropertyName("ctx")] + public string Context { get; set; } + + [JsonPropertyName("id")] + public int? Id { get; set; } + + [JsonPropertyName("msg")] + public string Message { get; set; } + + [JsonPropertyName("tags")] + public IEnumerable Tags { get; set; } + + [JsonPropertyName("attr")] + public IDictionary Attributes { get; set; } + } + class MongoDate + { + [JsonPropertyName("$date")] + public DateTime DateTime { get; set; } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongodArguments.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongodArguments.cs new file mode 100644 index 00000000..3d1edb47 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/MongodArguments.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; + +namespace Mongo2Go.Helper +{ + public static class MongodArguments + { + private const string ArgumentSeparator = "--"; + private const string Space = " "; + + /// + /// Returns the if it is verified that it does not contain any mongod argument already defined by Mongo2Go. + /// + /// mongod arguments defined by Mongo2Go + /// Additional mongod arguments + /// contains at least one mongod argument already defined by Mongo2Go + /// A string with the additional mongod arguments + public static string GetValidAdditionalArguments(string existingMongodArguments, string additionalMongodArguments) + { + if (string.IsNullOrWhiteSpace(additionalMongodArguments)) + { + return string.Empty; + } + + var existingMongodArgumentArray = existingMongodArguments.Trim().Split(new[] { ArgumentSeparator }, StringSplitOptions.RemoveEmptyEntries); + + var existingMongodArgumentOptions = new List(); + for (var i = 0; i < existingMongodArgumentArray.Length; i++) + { + var argumentOptionSplit = existingMongodArgumentArray[i].Split(' '); + + if (argumentOptionSplit.Length == 0 + || string.IsNullOrWhiteSpace(argumentOptionSplit[0].Trim())) + { + continue; + } + + existingMongodArgumentOptions.Add(argumentOptionSplit[0].Trim()); + } + + var additionalMongodArgumentArray = additionalMongodArguments.Trim().Split(new[] { ArgumentSeparator }, StringSplitOptions.RemoveEmptyEntries); + + var validAdditionalMongodArguments = new List(); + var duplicateMongodArguments = new List(); + for (var i = 0; i < additionalMongodArgumentArray.Length; i++) + { + var additionalArgument = additionalMongodArgumentArray[i].Trim(); + var argumentOptionSplit = additionalArgument.Split(' '); + + if (argumentOptionSplit.Length == 0 + || string.IsNullOrWhiteSpace(argumentOptionSplit[0].Trim())) + { + continue; + } + + if (existingMongodArgumentOptions.Contains(argumentOptionSplit[0].Trim())) + { + duplicateMongodArguments.Add(argumentOptionSplit[0].Trim()); + } + + validAdditionalMongodArguments.Add(ArgumentSeparator + additionalArgument); + } + + if (duplicateMongodArguments.Count != 0) + { + throw new ArgumentException($"mongod arguments defined by Mongo2Go ({string.Join(", ", existingMongodArgumentOptions)}) cannot be overriden. Please remove the following additional argument(s): {string.Join(", ", duplicateMongodArguments)}."); + } + + return validAdditionalMongodArguments.Count == 0 + ? string.Empty + : Space + string.Join(" ", validAdditionalMongodArguments); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NetStandard21Compatibility.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NetStandard21Compatibility.cs new file mode 100644 index 00000000..31e6fbbf --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NetStandard21Compatibility.cs @@ -0,0 +1,24 @@ +#if NETSTANDARD2_0 +using System; + +namespace Mongo2Go.Helper +{ + public static class NetStandard21Compatibility + { + /// + /// Returns a value indicating whether a specified string occurs within this , using the specified comparison rules. + /// + /// The string to operate on. + /// The string to seek. + /// One of the enumeration values that specifies the rules to use in the comparison. + /// if the parameter occurs within this string, or if is the empty string (""); otherwise, . + /// is + public static bool Contains(this string @string, string value, StringComparison comparisonType) + { + if (@string == null) throw new ArgumentNullException(nameof(@string)); + + return @string.IndexOf(value, comparisonType) >= 0; + } + } +} +#endif diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NoFreePortFoundException.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NoFreePortFoundException.cs new file mode 100644 index 00000000..04443d6a --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/NoFreePortFoundException.cs @@ -0,0 +1,11 @@ +using System; + +namespace Mongo2Go.Helper +{ + public class NoFreePortFoundException : Exception + { + public NoFreePortFoundException() { } + public NoFreePortFoundException(string message) : base(message) { } + public NoFreePortFoundException(string message, Exception inner) : base(message, inner) { } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortPool.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortPool.cs new file mode 100644 index 00000000..1b26a47b --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortPool.cs @@ -0,0 +1,37 @@ +using System; + +namespace Mongo2Go.Helper +{ + /// + /// Intention: port numbers won't be assigned twice to avoid connection problems with integration tests + /// + public sealed class PortPool : IPortPool + { + private static readonly PortPool Instance = new PortPool(); + + // Explicit static constructor to tell C# compiler + // not to mark type as beforefieldinit + static PortPool() + { + } + + // Singleton + private PortPool() + { + } + + public static PortPool GetInstance + { + get { return Instance; } + } + + /// + /// Returns and reserves a new port + /// + public int GetNextOpenPort() + { + IPortWatcher portWatcher = PortWatcherFactory.CreatePortWatcher(); + return portWatcher.FindOpenPort(); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcher.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcher.cs new file mode 100644 index 00000000..3dccdf8d --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcher.cs @@ -0,0 +1,38 @@ +using System.Linq; +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; + +namespace Mongo2Go.Helper +{ + public class PortWatcher : IPortWatcher + { + public int FindOpenPort() + { + // Locate a free port on the local machine by binding a socket to + // an IPEndPoint using IPAddress.Any and port 0. The socket will + // select a free port. + int listeningPort = 0; + Socket portSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + try + { + IPEndPoint socketEndPoint = new IPEndPoint(IPAddress.Any, 0); + portSocket.Bind(socketEndPoint); + socketEndPoint = (IPEndPoint)portSocket.LocalEndPoint; + listeningPort = socketEndPoint.Port; + } + finally + { + portSocket.Close(); + } + + return listeningPort; + } + + public bool IsPortAvailable(int portNumber) + { + IPEndPoint[] tcpConnInfoArray = IPGlobalProperties.GetIPGlobalProperties().GetActiveTcpListeners(); + return tcpConnInfoArray.All(endpoint => endpoint.Port != portNumber); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcherFactory.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcherFactory.cs new file mode 100644 index 00000000..18ab1821 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/PortWatcherFactory.cs @@ -0,0 +1,14 @@ +using System.Runtime.InteropServices; + +namespace Mongo2Go.Helper +{ + public class PortWatcherFactory + { + public static IPortWatcher CreatePortWatcher() + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Linux) + ? (IPortWatcher) new UnixPortWatcher() + : new PortWatcher(); + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessControl.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessControl.cs new file mode 100644 index 00000000..72a455f7 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessControl.cs @@ -0,0 +1,163 @@ +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text.Json; +using System.Threading; + +namespace Mongo2Go.Helper +{ + public static class ProcessControl + { + public static WrappedProcess ProcessFactory(string fileName, string arguments) + { + ProcessStartInfo startInfo = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + CreateNoWindow = true, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + WrappedProcess process = new WrappedProcess { StartInfo = startInfo }; + return process; + } + + public static ProcessOutput StartAndWaitForExit(Process process) + { + List errorOutput = new List(); + List standardOutput = new List(); + + process.ErrorDataReceived += (sender, args) => errorOutput.Add(args.Data); + process.OutputDataReceived += (sender, args) => standardOutput.Add(args.Data); + + process.Start(); + + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + + process.WaitForExit(); + + process.CancelErrorRead(); + process.CancelOutputRead(); + + return new ProcessOutput(errorOutput, standardOutput); + } + + /// + /// Reads from Output stream to determine if process is ready + /// + public static ProcessOutput StartAndWaitForReady(Process process, int timeoutInSeconds, string processReadyIdentifier, ILogger logger = null) + { + if (timeoutInSeconds < 1 || + timeoutInSeconds > 10) + { + throw new ArgumentOutOfRangeException("timeoutInSeconds", "The amount in seconds should have a value between 1 and 10."); + } + + // Determine when the process is ready, and store the error and standard outputs + // to eventually return them. + List errorOutput = new List(); + List standardOutput = new List(); + bool processReady = false; + + void OnProcessOnErrorDataReceived(object sender, DataReceivedEventArgs args) => errorOutput.Add(args.Data); + void OnProcessOnOutputDataReceived(object sender, DataReceivedEventArgs args) + { + standardOutput.Add(args.Data); + + if (!string.IsNullOrEmpty(args.Data) && args.Data.IndexOf(processReadyIdentifier, StringComparison.OrdinalIgnoreCase) >= 0) + { + processReady = true; + } + } + + process.ErrorDataReceived += OnProcessOnErrorDataReceived; + process.OutputDataReceived += OnProcessOnOutputDataReceived; + + if (logger == null) + WireLogsToConsoleAndDebugOutput(process); + else + WireLogsToLogger(process, logger); + + process.Start(); + + process.BeginErrorReadLine(); + process.BeginOutputReadLine(); + + int lastResortCounter = 0; + int timeOut = timeoutInSeconds * 10; + while (!processReady) + { + Thread.Sleep(100); + if (++lastResortCounter > timeOut) + { + // we waited X seconds. + // for any reason the detection did not worked, eg. the identifier changed + // lets assume everything is still ok + break; + } + } + + //unsubscribing writing to list - to prevent memory overflow. + process.ErrorDataReceived -= OnProcessOnErrorDataReceived; + process.OutputDataReceived -= OnProcessOnOutputDataReceived; + + return new ProcessOutput(errorOutput, standardOutput); + } + + /// + /// Send the mongod process logs to .NET's console and debug outputs. + /// + /// + private static void WireLogsToConsoleAndDebugOutput(Process process) + { + void DebugOutputHandler(object sender, DataReceivedEventArgs args) => Debug.WriteLine(args.Data); + void ConsoleOutputHandler(object sender, DataReceivedEventArgs args) => Console.WriteLine(args.Data); + + //Writing to debug trace & console to enable test runners to capture the output + process.ErrorDataReceived += DebugOutputHandler; + process.ErrorDataReceived += ConsoleOutputHandler; + process.OutputDataReceived += DebugOutputHandler; + process.OutputDataReceived += ConsoleOutputHandler; + } + + /// + /// Parses and redirects mongod logs to ILogger. + /// + /// + /// + private static void WireLogsToLogger(Process process, ILogger logger) + { + // Parse the structured log and wire it to logger + void OnReceivingLogFromMongod(object sender, DataReceivedEventArgs args) + { + if (string.IsNullOrWhiteSpace(args.Data)) + return; + try + { + var log = JsonSerializer.Deserialize(args.Data); + logger.Log(log.Level, + "{message} - {attributes} - {date} - {component} - {context} - {id} - {tags}", + log.Message, log.Attributes, log.MongoDate.DateTime, log.Component, log.Context, log.Id, log.Tags); + } + catch (Exception ex) when (ex is JsonException || ex is NotSupportedException) + { + logger.LogWarning(ex, + "Failed parsing the mongod logs {log}. It could be that the format has changed. " + + "See: https://docs.mongodb.com/manual/reference/log-messages/#std-label-log-message-json-output-format", + args.Data); + } + catch (Exception) + { + // Nothing else to do. Swallow the exception and do not wire the logs. + } + }; + process.ErrorDataReceived += OnReceivingLogFromMongod; + process.OutputDataReceived += OnReceivingLogFromMongod; + } + + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessOutput.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessOutput.cs new file mode 100644 index 00000000..925af15a --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessOutput.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; + +namespace Mongo2Go.Helper +{ + public class ProcessOutput + { + public ProcessOutput(IEnumerable errorOutput, IEnumerable standardOutput) + { + StandardOutput = standardOutput; + ErrorOutput = errorOutput; + } + + public IEnumerable StandardOutput { get; private set; } + public IEnumerable ErrorOutput { get; private set; } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessWatcher.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessWatcher.cs new file mode 100644 index 00000000..287ad4c9 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/ProcessWatcher.cs @@ -0,0 +1,13 @@ +using System.Diagnostics; +using System.Linq; + +namespace Mongo2Go.Helper +{ + public class ProcessWatcher : IProcessWatcher + { + public bool IsProcessRunning(string processName) + { + return Process.GetProcessesByName(processName).Any(); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/StringFormatExtension.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/StringFormatExtension.cs new file mode 100644 index 00000000..dbe6b5a3 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/StringFormatExtension.cs @@ -0,0 +1,27 @@ +using System; +using System.Globalization; + +namespace Mongo2Go.Helper +{ + /// + /// saves about 40 keystrokes + /// + public static class StringFormatExtension + { + /// + /// Populates the template using the provided arguments and the invariant culture + /// + public static string Formatted(this string template, params object[] args) + { + return template.Formatted(CultureInfo.InvariantCulture, args); + } + + /// + /// Populates the template using the provided arguments using the provided formatter + /// + public static string Formatted(this string template, IFormatProvider formatter, params object[] args) + { + return string.IsNullOrEmpty(template) ? string.Empty : string.Format(formatter, template, args); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/UnixPortWatcher.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/UnixPortWatcher.cs new file mode 100644 index 00000000..ffec7894 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/UnixPortWatcher.cs @@ -0,0 +1,46 @@ +using System.Net; +using System.Net.Sockets; + +namespace Mongo2Go.Helper +{ + + public class UnixPortWatcher : IPortWatcher + { + public int FindOpenPort () + { + // Locate a free port on the local machine by binding a socket to + // an IPEndPoint using IPAddress.Any and port 0. The socket will + // select a free port. + int listeningPort = 0; + Socket portSocket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + try + { + IPEndPoint socketEndPoint = new IPEndPoint(IPAddress.Any, 0); + portSocket.Bind(socketEndPoint); + socketEndPoint = (IPEndPoint)portSocket.LocalEndPoint; + listeningPort = socketEndPoint.Port; + } + finally + { + portSocket.Close(); + } + + return listeningPort; + } + + public bool IsPortAvailable (int portNumber) + { + TcpListener tcpListener = new TcpListener (IPAddress.Loopback, portNumber); + try { + tcpListener.Start (); + return true; + } + catch (SocketException) { + return false; + } finally + { + tcpListener.Stop (); + } + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Helper/WrappedProcess.cs b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/WrappedProcess.cs new file mode 100644 index 00000000..d0093738 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Helper/WrappedProcess.cs @@ -0,0 +1,9 @@ +using System.Diagnostics; + +namespace Mongo2Go.Helper +{ + public class WrappedProcess : Process + { + public bool DoNotKill { get; set; } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj b/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj new file mode 100644 index 00000000..51c40c9c --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/Mongo2Go.csproj @@ -0,0 +1,93 @@ + + + + net472;netstandard2.1 + Johannes Hoppe and many contributors + Mongo2Go is a managed wrapper around MongoDB binaries. It targets .NET Framework 4.7.2 and .NET Standard 2.1. +This Nuget package contains the executables of mongod, mongoimport and mongoexport v4.4.4 for Windows, Linux and macOS. + + +Mongo2Go has two use cases: + +1. Providing multiple, temporary and isolated MongoDB databases for integration tests +2. Providing a quick to set up MongoDB database for a local developer environment + HAUS HOPPE - ITS + Copyright © 2012-2025 Johannes Hoppe and many ❤️ contributors + true + icon.png + MIT + https://github.com/Mongo2Go/Mongo2Go + https://github.com/Mongo2Go/Mongo2Go/releases + MongoDB Mongo unit test integration runner + https://github.com/Mongo2Go/Mongo2Go + git + Mongo2Go + Mongo2Go is a managed wrapper around MongoDB binaries. + + + + 4 + 1701;1702;1591;1573 + + + + 4 + 1701;1702;1591;1573 + + + + 1701;1702;1591;1573 + + + + 1701;1702;1591;1573 + + + + true + true + true + + + + embedded + true + true + + + + v + + + + + + true + icon.png + + + true + tools + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbDefaults.cs b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbDefaults.cs new file mode 100644 index 00000000..ef112211 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbDefaults.cs @@ -0,0 +1,22 @@ +namespace Mongo2Go +{ + public static class MongoDbDefaults + { + public const string ProcessName = "mongod"; + + public const string MongodExecutable = "mongod"; + + public const string MongoExportExecutable = "mongoexport"; + + public const string MongoImportExecutable = "mongoimport"; + + public const int DefaultPort = 27017; + + // but we don't want to get in trouble with productive systems + public const int TestStartPort = 27018; + + public const string Lockfile = "mongod.lock"; + + public const int SingleNodeReplicaSetWaitTimeout = 10; + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbPortAlreadyTakenException.cs b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbPortAlreadyTakenException.cs new file mode 100644 index 00000000..d272bd84 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbPortAlreadyTakenException.cs @@ -0,0 +1,11 @@ +using System; + +namespace Mongo2Go +{ + public class MongoDbPortAlreadyTakenException : Exception + { + public MongoDbPortAlreadyTakenException() { } + public MongoDbPortAlreadyTakenException(string message) : base(message) { } + public MongoDbPortAlreadyTakenException(string message, Exception inner) : base(message, inner) { } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.IDisposable.cs b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.IDisposable.cs new file mode 100644 index 00000000..bd894bbe --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.IDisposable.cs @@ -0,0 +1,54 @@ +using System; + +namespace Mongo2Go +{ + // IDisposable and friends + public partial class MongoDbRunner + { + ~MongoDbRunner() + { + Dispose(false); + } + + public bool Disposed { get; private set; } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + private void Dispose(bool disposing) + { + if (Disposed) + { + return; + } + + if (State != State.Running) + { + return; + } + + if (disposing) + { + // we have no "managed resources" - but we leave this switch to avoid an FxCop CA1801 warnig + } + + if (_mongoDbProcess != null) + { + _mongoDbProcess.Dispose(); + } + + // will be null if we are working in debugging mode (single instance) + if (_dataDirectoryWithPort != null) + { + // finally clean up the data directory we created previously + _fileSystem.DeleteFolder(_dataDirectoryWithPort); + } + + Disposed = true; + State = State.Stopped; + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.cs b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.cs new file mode 100644 index 00000000..6818c151 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/MongoDbRunner.cs @@ -0,0 +1,221 @@ +using Microsoft.Extensions.Logging; +using Mongo2Go.Helper; +using System; +using System.IO; +using System.Runtime.InteropServices; + +namespace Mongo2Go +{ + /// + /// Mongo2Go main entry point + /// + public partial class MongoDbRunner : IDisposable + { + private readonly IMongoDbProcess _mongoDbProcess; + private readonly IFileSystem _fileSystem; + private readonly string _dataDirectoryWithPort; + private readonly int _port; + private readonly IMongoBinaryLocator _mongoBin; + + /// + /// State of the current MongoDB instance + /// + public State State { get; private set; } + + /// + /// Connections string that should be used to establish a connection the MongoDB instance + /// + public string ConnectionString { get; private set; } + + /// + /// Starts Multiple MongoDB instances with each call + /// On dispose: kills them and deletes their data directory + /// + /// (Optional) If null, mongod logs are wired to .NET's Console and Debug output (provided you haven't added the --quiet additional argument). + /// If not null, mongod logs are parsed and wired to the provided logger. + /// Should be used for integration tests + public static MongoDbRunner Start(string dataDirectory = null, string binariesSearchPatternOverride = null, string binariesSearchDirectory = null, bool singleNodeReplSet = false, string additionalMongodArguments = null, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null) + { + if (dataDirectory == null) { + dataDirectory = GetTemporaryDataDirectory(); + } + + // this is required to support multiple instances to run in parallel + dataDirectory += Guid.NewGuid().ToString().Replace("-", "").Substring(0, 20); + + return new MongoDbRunner( + PortPool.GetInstance, + new FileSystem(), + new MongoDbProcessStarter(), + new MongoBinaryLocator(binariesSearchPatternOverride, binariesSearchDirectory), + dataDirectory, + singleNodeReplSet, + additionalMongodArguments, + singleNodeReplSetWaitTimeout, + logger); + } + + /// + /// !!! + /// This method is only used for an internal unit test. Use MongoDbRunner.Start() instead. + /// But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + /// + /// see https://github.com/Mongo2Go/Mongo2Go/issues/41 + [Obsolete("Use MongoDbRunner.Start() if possible.")] + public static MongoDbRunner StartUnitTest( + IPortPool portPool, + IFileSystem fileSystem, + IMongoDbProcessStarter processStarter, + IMongoBinaryLocator mongoBin, + string dataDirectory = null, + string additionalMongodArguments = null) + { + return new MongoDbRunner( + portPool, + fileSystem, + processStarter, + mongoBin, + dataDirectory, + additionalMongodArguments: additionalMongodArguments); + } + + /// + /// Only starts one single MongoDB instance (even on multiple calls), does not kill it, does not delete data + /// + /// + /// Should be used for local debugging only + /// WARNING: one single instance on one single machine is not a suitable setup for productive environments!!! + /// + public static MongoDbRunner StartForDebugging(string dataDirectory = null, string binariesSearchPatternOverride = null, string binariesSearchDirectory = null, bool singleNodeReplSet = false, int port = MongoDbDefaults.DefaultPort, string additionalMongodArguments = null, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout) + { + return new MongoDbRunner( + new ProcessWatcher(), + new PortWatcher(), + new FileSystem(), + new MongoDbProcessStarter(), + new MongoBinaryLocator(binariesSearchPatternOverride, binariesSearchDirectory), port, dataDirectory, singleNodeReplSet, additionalMongodArguments, singleNodeReplSetWaitTimeout); + } + + /// + /// !!! + /// This method is only used for an internal unit test. Use MongoDbRunner.StartForDebugging() instead. + /// But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + /// + /// see https://github.com/Mongo2Go/Mongo2Go/issues/41 + [Obsolete("Use MongoDbRunner.StartForDebugging() if possible.")] + public static MongoDbRunner StartForDebuggingUnitTest( + IProcessWatcher processWatcher, + IPortWatcher portWatcher, + IFileSystem fileSystem, + IMongoDbProcessStarter processStarter, + IMongoBinaryLocator mongoBin, + string dataDirectory = null, + string additionalMongodArguments = null) + { + return new MongoDbRunner( + processWatcher, + portWatcher, + fileSystem, + processStarter, + mongoBin, + MongoDbDefaults.DefaultPort, + dataDirectory, + additionalMongodArguments: additionalMongodArguments); + } + + /// + /// Executes Mongoimport on the associated MongoDB Instace + /// + public void Import(string database, string collection, string inputFile, bool drop, string additionalMongodArguments = null) + { + MongoImportExport.Import(_mongoBin.Directory, _port, database, collection, inputFile, drop, additionalMongodArguments); + } + + /// + /// Executes Mongoexport on the associated MongoDB Instace + /// + public void Export(string database, string collection, string outputFile, string additionalMongodArguments = null) + { + MongoImportExport.Export(_mongoBin.Directory, _port, database, collection, outputFile, additionalMongodArguments); + } + + /// + /// usage: local debugging + /// + private MongoDbRunner(IProcessWatcher processWatcher, IPortWatcher portWatcher, IFileSystem fileSystem, IMongoDbProcessStarter processStarter, IMongoBinaryLocator mongoBin, int port, string dataDirectory = null, bool singleNodeReplSet = false, string additionalMongodArguments = null, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout) + { + _fileSystem = fileSystem; + _mongoBin = mongoBin; + _port = port; + + MakeMongoBinarysExecutable(); + + ConnectionString = singleNodeReplSet + ? "mongodb://127.0.0.1:{0}/?directConnection=true&replicaSet=singleNodeReplSet&readPreference=primary".Formatted(_port) + : "mongodb://127.0.0.1:{0}/".Formatted(_port); + + if (processWatcher.IsProcessRunning(MongoDbDefaults.ProcessName) && !portWatcher.IsPortAvailable(_port)) + { + State = State.AlreadyRunning; + return; + } + + if (!portWatcher.IsPortAvailable(_port)) + { + throw new MongoDbPortAlreadyTakenException("MongoDB can't be started. The TCP port {0} is already taken.".Formatted(_port)); + } + + if (dataDirectory == null) { + dataDirectory = GetTemporaryDataDirectory(); + } + + _fileSystem.CreateFolder(dataDirectory); + _fileSystem.DeleteFile("{0}{1}{2}".Formatted(dataDirectory, Path.DirectorySeparatorChar.ToString(), MongoDbDefaults.Lockfile)); + _mongoDbProcess = processStarter.Start(_mongoBin.Directory, dataDirectory, _port, true, singleNodeReplSet, additionalMongodArguments, singleNodeReplSetWaitTimeout); + + State = State.Running; + } + + /// + /// usage: integration tests + /// + private MongoDbRunner(IPortPool portPool, IFileSystem fileSystem, IMongoDbProcessStarter processStarter, IMongoBinaryLocator mongoBin, string dataDirectory = null, bool singleNodeReplSet = false, string additionalMongodArguments = null, ushort singleNodeReplSetWaitTimeout = MongoDbDefaults.SingleNodeReplicaSetWaitTimeout, ILogger logger = null) + { + _fileSystem = fileSystem; + _port = portPool.GetNextOpenPort(); + _mongoBin = mongoBin; + + if (dataDirectory == null) { + dataDirectory = GetTemporaryDataDirectory(); + } + + MakeMongoBinarysExecutable(); + + ConnectionString = singleNodeReplSet + ? "mongodb://127.0.0.1:{0}/?directConnection=true&replicaSet=singleNodeReplSet&readPreference=primary".Formatted(_port) + : "mongodb://127.0.0.1:{0}/".Formatted(_port); + + _dataDirectoryWithPort = "{0}_{1}".Formatted(dataDirectory, _port); + _fileSystem.CreateFolder(_dataDirectoryWithPort); + _fileSystem.DeleteFile("{0}{1}{2}".Formatted(_dataDirectoryWithPort, Path.DirectorySeparatorChar.ToString(), MongoDbDefaults.Lockfile)); + + _mongoDbProcess = processStarter.Start(_mongoBin.Directory, _dataDirectoryWithPort, _port, singleNodeReplSet, additionalMongodArguments, singleNodeReplSetWaitTimeout, logger); + + State = State.Running; + } + + private void MakeMongoBinarysExecutable() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || + RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + _fileSystem.MakeFileExecutable(Path.Combine(_mongoBin.Directory, MongoDbDefaults.MongodExecutable)); + _fileSystem.MakeFileExecutable(Path.Combine(_mongoBin.Directory, MongoDbDefaults.MongoExportExecutable)); + _fileSystem.MakeFileExecutable(Path.Combine(_mongoBin.Directory, MongoDbDefaults.MongoImportExecutable)); + } + } + + + private static string GetTemporaryDataDirectory() => Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/MonogDbBinariesNotFoundException.cs b/Mongo2Go-4.1.0/src/Mongo2Go/MonogDbBinariesNotFoundException.cs new file mode 100644 index 00000000..5845cf18 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/MonogDbBinariesNotFoundException.cs @@ -0,0 +1,11 @@ +using System; + +namespace Mongo2Go +{ + public class MonogDbBinariesNotFoundException : Exception + { + public MonogDbBinariesNotFoundException() { } + public MonogDbBinariesNotFoundException(string message) : base(message) { } + public MonogDbBinariesNotFoundException(string message, Exception inner) : base(message, inner) { } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/State.cs b/Mongo2Go-4.1.0/src/Mongo2Go/State.cs new file mode 100644 index 00000000..69e00969 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/State.cs @@ -0,0 +1,9 @@ +namespace Mongo2Go +{ + public enum State + { + Stopped, + Running, + AlreadyRunning + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2Go/packages.lock.json b/Mongo2Go-4.1.0/src/Mongo2Go/packages.lock.json new file mode 100644 index 00000000..e58dba7d --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2Go/packages.lock.json @@ -0,0 +1,490 @@ +{ + "version": 1, + "dependencies": { + ".NETFramework,Version=v4.7.2": { + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==", + "dependencies": { + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4" + } + }, + "Microsoft.NETFramework.ReferenceAssemblies": { + "type": "Direct", + "requested": "[1.0.3, )", + "resolved": "1.0.3", + "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==", + "dependencies": { + "Microsoft.NETFramework.ReferenceAssemblies.net472": "1.0.3" + } + }, + "Microsoft.SourceLink.GitHub": { + "type": "Direct", + "requested": "[1.0.0, )", + "resolved": "1.0.0", + "contentHash": "aZyGyGg2nFSxix+xMkPmlmZSsnGQ3w+mIG23LTxJZHN+GPwTQ5FpPgDo7RMOq+Kcf5D4hFWfXkGhoGstawX13Q==", + "dependencies": { + "Microsoft.Build.Tasks.Git": "1.0.0", + "Microsoft.SourceLink.Common": "1.0.0" + } + }, + "MinVer": { + "type": "Direct", + "requested": "[2.5.0, )", + "resolved": "2.5.0", + "contentHash": "+vgY+COxnu93nZEVYScloRuboNRIYkElokxTdtKLt6isr/f6GllPt0oLfrHj7fzxgj7SC5xMZg5c2qvd6qyHDQ==" + }, + "MongoDB.Driver": { + "type": "Direct", + "requested": "[3.5.0, )", + "resolved": "3.5.0", + "contentHash": "ST90u7psyMkNNOWFgSkexsrB3kPn7Ynl2DlMFj2rJyYuc6SIxjmzu4ufy51yzM+cPVE1SvVcdb5UFobrRw6cMg==", + "dependencies": { + "DnsClient": "1.6.1", + "Microsoft.Extensions.Logging.Abstractions": "2.0.0", + "MongoDB.Bson": "3.5.0", + "SharpCompress": "0.30.1", + "Snappier": "1.0.0", + "System.Buffers": "4.5.1", + "System.Net.Http": "4.3.4", + "System.Runtime.InteropServices.RuntimeInformation": "4.3.0", + "ZstdSharp.Port": "0.7.3" + } + }, + "SharpCompress": { + "type": "Direct", + "requested": "[0.41.0, )", + "resolved": "0.41.0", + "contentHash": "z04dBVdTIAFTRKi38f0LkajaKA++bR+M8kYCbasXePILD2H+qs7CkLpyiippB24CSbTrWIgpBKm6BenZqkUwvw==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "8.0.0", + "System.Buffers": "4.6.0", + "System.Memory": "4.6.0", + "System.Text.Encoding.CodePages": "8.0.0", + "ZstdSharp.Port": "0.8.6" + } + }, + "System.Text.Json": { + "type": "Direct", + "requested": "[6.0.10, )", + "resolved": "6.0.10", + "contentHash": "NSB0kDipxn2ychp88NXWfFRFlmi1bst/xynOutbnpEfRCT9JZkZ7KOmF/I/hNKo2dILiMGnqblm+j1sggdLB9g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Numerics.Vectors": "4.5.0", + "System.Runtime.CompilerServices.Unsafe": "6.0.0", + "System.Text.Encodings.Web": "6.0.0", + "System.Threading.Tasks.Extensions": "4.5.4", + "System.ValueTuple": "4.5.0" + } + }, + "DnsClient": { + "type": "Transitive", + "resolved": "1.6.1", + "contentHash": "4H/f2uYJOZ+YObZjpY9ABrKZI+JNw3uizp6oMzTXwDw6F+2qIPhpRl/1t68O/6e98+vqNiYGu+lswmwdYUy3gg==", + "dependencies": { + "Microsoft.Win32.Registry": "5.0.0", + "System.Buffers": "4.5.1" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==", + "dependencies": { + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "Microsoft.Build.Tasks.Git": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "z2fpmmt+1Jfl+ZnBki9nSP08S1/tbEOxFdsK1rSR+LBehIJz1Xv9/6qOOoGNqlwnAGGVGis1Oj6S8Kt9COEYlQ==" + }, + "Microsoft.NETFramework.ReferenceAssemblies.net472": { + "type": "Transitive", + "resolved": "1.0.3", + "contentHash": "0E7evZXHXaDYYiLRfpyXvCh+yzM2rNTyuZDI+ZO7UUqSc6GfjePiXTdqJGtgIKUwdI81tzQKmaWprnUiPj9hAw==" + }, + "Microsoft.SourceLink.Common": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "G8DuQY8/DK5NN+3jm5wcMcd9QYD90UV7MiLmdljSJixi3U/vNaeBKmmXUqI4DJCOeWizIUEh4ALhSt58mR+5eg==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "MongoDB.Bson": { + "type": "Transitive", + "resolved": "3.5.0", + "contentHash": "JGNK6BanLDEifgkvPLqVFCPus5EDCy416pxf1dxUBRSVd3D9+NB3AvMVX190eXlk5/UXuCxpsQv7jWfNKvppBQ==", + "dependencies": { + "System.Memory": "4.5.5", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "Snappier": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "rFtK2KEI9hIe8gtx3a0YDXdHOpedIf9wYCEYtBEmtlyiWVX3XlCNV03JrmmAi/Cdfn7dxK+k0sjjcLv4fpHnqA==", + "dependencies": { + "System.Memory": "4.5.4", + "System.Runtime.CompilerServices.Unsafe": "4.7.1", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA==" + }, + "System.IO": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "3qjaHvxQPDpSOYICjUoTsmoq5u6QJAFRUITgeT/4gqkF1bajbSmb1kwSxEA8AHlofqgcKJcM8udgieRNhaJ5Cg==" + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "OEkbBQoklHngJ8UD8ez2AERSk2g+/qpAaSWWCBFbpH727HxDq5ydVkuncBaKcKfwRqXGWx64dS6G1SUScMsitg==", + "dependencies": { + "System.Buffers": "4.6.0", + "System.Numerics.Vectors": "4.6.0", + "System.Runtime.CompilerServices.Unsafe": "6.1.0" + } + }, + "System.Net.Http": { + "type": "Transitive", + "resolved": "4.3.4", + "contentHash": "aOa2d51SEbmM+H+Csw7yJOuNZoHkrP2XnAurye5HWYgGVVU54YZDvsLUYRv6h18X3sPnjNCANmN7ZhIPiqMcjA==", + "dependencies": { + "System.Security.Cryptography.X509Certificates": "4.3.0" + } + }, + "System.Numerics.Vectors": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "t+SoieZsRuEyiw/J+qXUbolyO219tKQQI0+2/YI+Qv7YdGValA6WiuokrNKqjrTNsy5ABWU11bdKOzUdheteXg==" + }, + "System.Runtime": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "JufQi0vPQ0xGnAczR13AUFglDyVYt4Kqnz1AZaiKZ5+GICq0/1MH/mO/eAJHt/mHW1zjKBJd7kV26SrxddAhiw==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.1.0", + "contentHash": "5o/HZxx6RVqYlhKSq8/zronDkALJZUT2Vz0hx43f0gwe8mwlM0y2nYlqdBwLMzr262Bwvpikeb/yEwkAa5PADg==" + }, + "System.Runtime.InteropServices.RuntimeInformation": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "cbz4YJMqRDR7oLeMRbdYv7mYzc++17lNhScCX0goO2XpGWdvAt60CGN+FHdePUEHCe/Jy9jUlvNAiNdM+7jsOw==" + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Cryptography.Algorithms": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "W1kd2Y8mYSCgc3ULTAZ0hOP2dSdG5YauTb1089T0/kRcN2MpSAW1izOFROrJgxSlMn3ArsgHXagigyi+ibhevg==", + "dependencies": { + "System.IO": "4.3.0", + "System.Runtime": "4.3.0", + "System.Security.Cryptography.Encoding": "4.3.0", + "System.Security.Cryptography.Primitives": "4.3.0" + } + }, + "System.Security.Cryptography.Encoding": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "1DEWjZZly9ae9C79vFwqaO5kaOlI5q+3/55ohmq/7dpDyDfc8lYe7YVxJUZ5MF/NtbkRjwFRo14yM4OEo9EmDw==" + }, + "System.Security.Cryptography.Primitives": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "7bDIyVFNL/xKeFHjhobUAQqSpJq9YTOpbEs6mR233Et01STBMXNAc/V+BM6dwYGc95gVh/Zf+iVXWzj3mE8DWg==" + }, + "System.Security.Cryptography.X509Certificates": { + "type": "Transitive", + "resolved": "4.3.0", + "contentHash": "t2Tmu6Y2NtJ2um0RtcuhP7ZdNNxXEgUm2JeoA/0NvlMjAhKCnM1NX07TDl3244mVp3QU6LPEhT3HTtH1uF7IYw==", + "dependencies": { + "System.Security.Cryptography.Algorithms": "4.3.0", + "System.Security.Cryptography.Encoding": "4.3.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Text.Encoding.CodePages": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "OZIsVplFGaVY90G2SbpgU7EnCoOO5pw1t4ic21dBF3/1omrJFpAGoNAVpPyMVOC90/hvgkGG3VFqR13YgZMQfg==", + "dependencies": { + "System.Memory": "4.5.5", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "Vg8eB5Tawm1IFqj4TVK1czJX89rhFxJo9ELqc/Eiq0eXy13RK00eubyU6TJE6y+GQXjyV5gSfiewDUZjQgSE0w==", + "dependencies": { + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "4.5.3" + } + }, + "System.ValueTuple": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "okurQJO6NRE/apDIP23ajJ0hpiNmJ+f0BwOlB/cSqTLQlw5upkf+5+96+iG2Jw40G1fCVCyPz/FhIABUjMR+RQ==" + }, + "ZstdSharp.Port": { + "type": "Transitive", + "resolved": "0.8.6", + "contentHash": "iP4jVLQoQmUjMU88g1WObiNr6YKZGvh4aOXn3yOJsHqZsflwRsxZPcIBvNXgjXO3vQKSLctXGLTpcBPLnWPS8A==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "5.0.0", + "System.Memory": "4.5.5", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + } + }, + ".NETStandard,Version=v2.1": { + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Direct", + "requested": "[6.0.0, )", + "resolved": "6.0.0", + "contentHash": "/HggWBbTwy8TgebGSX5DBZ24ndhzi93sHUBDvP1IxbZD7FDokYzdAr6+vbWGjw2XAfR2EJ1sfKUotpjHnFWPxA==", + "dependencies": { + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4" + } + }, + "Microsoft.NETFramework.ReferenceAssemblies": { + "type": "Direct", + "requested": "[1.0.3, )", + "resolved": "1.0.3", + "contentHash": "vUc9Npcs14QsyOD01tnv/m8sQUnGTGOw1BCmKcv77LBJY7OxhJ+zJF7UD/sCL3lYNFuqmQEVlkfS4Quif6FyYg==", + "dependencies": { + "Microsoft.NETFramework.ReferenceAssemblies.net461": "1.0.3" + } + }, + "Microsoft.SourceLink.GitHub": { + "type": "Direct", + "requested": "[1.0.0, )", + "resolved": "1.0.0", + "contentHash": "aZyGyGg2nFSxix+xMkPmlmZSsnGQ3w+mIG23LTxJZHN+GPwTQ5FpPgDo7RMOq+Kcf5D4hFWfXkGhoGstawX13Q==", + "dependencies": { + "Microsoft.Build.Tasks.Git": "1.0.0", + "Microsoft.SourceLink.Common": "1.0.0" + } + }, + "MinVer": { + "type": "Direct", + "requested": "[2.5.0, )", + "resolved": "2.5.0", + "contentHash": "+vgY+COxnu93nZEVYScloRuboNRIYkElokxTdtKLt6isr/f6GllPt0oLfrHj7fzxgj7SC5xMZg5c2qvd6qyHDQ==" + }, + "MongoDB.Driver": { + "type": "Direct", + "requested": "[3.5.0, )", + "resolved": "3.5.0", + "contentHash": "ST90u7psyMkNNOWFgSkexsrB3kPn7Ynl2DlMFj2rJyYuc6SIxjmzu4ufy51yzM+cPVE1SvVcdb5UFobrRw6cMg==", + "dependencies": { + "DnsClient": "1.6.1", + "Microsoft.Extensions.Logging.Abstractions": "2.0.0", + "MongoDB.Bson": "3.5.0", + "SharpCompress": "0.30.1", + "Snappier": "1.0.0", + "System.Buffers": "4.5.1", + "ZstdSharp.Port": "0.7.3" + } + }, + "SharpCompress": { + "type": "Direct", + "requested": "[0.41.0, )", + "resolved": "0.41.0", + "contentHash": "z04dBVdTIAFTRKi38f0LkajaKA++bR+M8kYCbasXePILD2H+qs7CkLpyiippB24CSbTrWIgpBKm6BenZqkUwvw==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "8.0.0", + "System.Buffers": "4.6.0", + "System.Memory": "4.6.0", + "System.Text.Encoding.CodePages": "8.0.0", + "ZstdSharp.Port": "0.8.6" + } + }, + "System.Text.Json": { + "type": "Direct", + "requested": "[6.0.10, )", + "resolved": "6.0.10", + "contentHash": "NSB0kDipxn2ychp88NXWfFRFlmi1bst/xynOutbnpEfRCT9JZkZ7KOmF/I/hNKo2dILiMGnqblm+j1sggdLB9g==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Numerics.Vectors": "4.5.0", + "System.Runtime.CompilerServices.Unsafe": "6.0.0", + "System.Text.Encodings.Web": "6.0.0", + "System.Threading.Tasks.Extensions": "4.5.4" + } + }, + "DnsClient": { + "type": "Transitive", + "resolved": "1.6.1", + "contentHash": "4H/f2uYJOZ+YObZjpY9ABrKZI+JNw3uizp6oMzTXwDw6F+2qIPhpRl/1t68O/6e98+vqNiYGu+lswmwdYUy3gg==", + "dependencies": { + "Microsoft.Win32.Registry": "5.0.0" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==" + }, + "Microsoft.Build.Tasks.Git": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "z2fpmmt+1Jfl+ZnBki9nSP08S1/tbEOxFdsK1rSR+LBehIJz1Xv9/6qOOoGNqlwnAGGVGis1Oj6S8Kt9COEYlQ==" + }, + "Microsoft.NETFramework.ReferenceAssemblies.net461": { + "type": "Transitive", + "resolved": "1.0.3", + "contentHash": "AmOJZwCqnOCNp6PPcf9joyogScWLtwy0M1WkqfEQ0M9nYwyDD7EX9ZjscKS5iYnyvteX7kzSKFCKt9I9dXA6mA==" + }, + "Microsoft.SourceLink.Common": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "G8DuQY8/DK5NN+3jm5wcMcd9QYD90UV7MiLmdljSJixi3U/vNaeBKmmXUqI4DJCOeWizIUEh4ALhSt58mR+5eg==" + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==", + "dependencies": { + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Security.AccessControl": "5.0.0", + "System.Security.Principal.Windows": "5.0.0" + } + }, + "MongoDB.Bson": { + "type": "Transitive", + "resolved": "3.5.0", + "contentHash": "JGNK6BanLDEifgkvPLqVFCPus5EDCy416pxf1dxUBRSVd3D9+NB3AvMVX190eXlk5/UXuCxpsQv7jWfNKvppBQ==", + "dependencies": { + "System.Memory": "4.5.5", + "System.Runtime.CompilerServices.Unsafe": "5.0.0" + } + }, + "Snappier": { + "type": "Transitive", + "resolved": "1.0.0", + "contentHash": "rFtK2KEI9hIe8gtx3a0YDXdHOpedIf9wYCEYtBEmtlyiWVX3XlCNV03JrmmAi/Cdfn7dxK+k0sjjcLv4fpHnqA==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "4.7.1" + } + }, + "System.Buffers": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "lN6tZi7Q46zFzAbRYXTIvfXcyvQQgxnY7Xm6C6xQ9784dEL1amjM6S6Iw4ZpsvesAKnRVsM4scrDQaDqSClkjA==" + }, + "System.Memory": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "OEkbBQoklHngJ8UD8ez2AERSk2g+/qpAaSWWCBFbpH727HxDq5ydVkuncBaKcKfwRqXGWx64dS6G1SUScMsitg==", + "dependencies": { + "System.Buffers": "4.6.0", + "System.Numerics.Vectors": "4.6.0", + "System.Runtime.CompilerServices.Unsafe": "6.1.0" + } + }, + "System.Numerics.Vectors": { + "type": "Transitive", + "resolved": "4.6.0", + "contentHash": "t+SoieZsRuEyiw/J+qXUbolyO219tKQQI0+2/YI+Qv7YdGValA6WiuokrNKqjrTNsy5ABWU11bdKOzUdheteXg==" + }, + "System.Runtime.CompilerServices.Unsafe": { + "type": "Transitive", + "resolved": "6.1.0", + "contentHash": "5o/HZxx6RVqYlhKSq8/zronDkALJZUT2Vz0hx43f0gwe8mwlM0y2nYlqdBwLMzr262Bwvpikeb/yEwkAa5PADg==" + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dagJ1mHZO3Ani8GH0PHpPEe/oYO+rVdbQjvjJkBRNQkX4t0r1iaeGn8+/ybkSLEan3/slM0t59SVdHzuHf2jmw==", + "dependencies": { + "System.Security.Principal.Windows": "5.0.0" + } + }, + "System.Security.Principal.Windows": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "t0MGLukB5WAVU9bO3MGzvlGnyJPgUlcwerXn1kzBRjwLKixT96XV0Uza41W49gVd8zEMFu9vQEFlv0IOrytICA==" + }, + "System.Text.Encoding.CodePages": { + "type": "Transitive", + "resolved": "8.0.0", + "contentHash": "OZIsVplFGaVY90G2SbpgU7EnCoOO5pw1t4ic21dBF3/1omrJFpAGoNAVpPyMVOC90/hvgkGG3VFqR13YgZMQfg==", + "dependencies": { + "System.Memory": "4.5.5", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.Text.Encodings.Web": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "Vg8eB5Tawm1IFqj4TVK1czJX89rhFxJo9ELqc/Eiq0eXy13RK00eubyU6TJE6y+GQXjyV5gSfiewDUZjQgSE0w==", + "dependencies": { + "System.Buffers": "4.5.1", + "System.Memory": "4.5.4", + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + }, + "System.Threading.Tasks.Extensions": { + "type": "Transitive", + "resolved": "4.5.4", + "contentHash": "zteT+G8xuGu6mS+mzDzYXbzS7rd3K6Fjb9RiZlYlJPam2/hU7JCBZBVEcywNuR+oZ1ncTvc/cq0faRr3P01OVg==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "4.5.3" + } + }, + "ZstdSharp.Port": { + "type": "Transitive", + "resolved": "0.8.6", + "contentHash": "iP4jVLQoQmUjMU88g1WObiNr6YKZGvh4aOXn3yOJsHqZsflwRsxZPcIBvNXgjXO3vQKSLctXGLTpcBPLnWPS8A==", + "dependencies": { + "System.Runtime.CompilerServices.Unsafe": "6.0.0" + } + } + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/FolderSearchTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/FolderSearchTests.cs new file mode 100644 index 00000000..f3a2d9c4 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/FolderSearchTests.cs @@ -0,0 +1,146 @@ +using System; +using System.IO; +using System.Reflection; +using FluentAssertions; +using Machine.Specifications; +using Mongo2Go.Helper; + +// ReSharper disable InconsistentNaming +// ReSharper disable UnusedMember.Local +namespace Mongo2GoTests +{ + [Subject("FolderSearch")] + public class when_requesting_current_executing_directory + { + public static string directory; + + Because of = () => directory = FolderSearch.CurrentExecutingDirectory(); + It should_contain_correct_path = () => directory.Should().Contain(Path.Combine("Mongo2GoTests", "bin")); + } + + [Subject("FolderSearch")] + public class when_searching_for_folder : FolderSearchSpec + { + static string startDirectory = Path.Combine(BaseDir, "test1", "test2"); + static string searchPattern = Path.Combine("packages", "Mongo2Go*", "tools", "mongodb-win32-i386*", "bin"); + static string directory; + + Because of = () => directory = startDirectory.FindFolder(searchPattern); + It should_find_the_path_with_the_highest_version_number = () => directory.Should().Be(MongoBinaries); + } + + + [Subject("FolderSearch")] + public class when_searching_for_not_existing_folder : FolderSearchSpec + { + static string startDirectory = Path.Combine(BaseDir, "test1", "test2"); + static string searchPattern = Path.Combine("packages", "Mongo2Go*", "XXX", "mongodb-win32-i386*", "bin"); + static string directory; + + Because of = () => directory = startDirectory.FindFolder(searchPattern); + It should_return_null = () => directory.Should().BeNull(); + } + + [Subject("FolderSearch")] + public class when_searching_for_not_existing_start_dir : FolderSearchSpec + { + static string startDirectory = Path.Combine(Path.GetRandomFileName()); + static string searchPattern = Path.Combine("packages", "Mongo2Go*", "XXX", "mongodb-win32-i386*", "bin"); + static string directory; + + Because of = () => directory = startDirectory.FindFolder(searchPattern); + It should_return_null = () => directory.Should().BeNull(); + } + + [Subject("FolderSearch")] + public class when_searching_for_folder_upwards : FolderSearchSpec + { + static string searchPattern = Path.Combine("packages", "Mongo2Go*", "tools", "mongodb-win32-i386*", "bin"); + static string directory; + + Because of = () => directory = LocationOfAssembly.FindFolderUpwards(searchPattern); + It should_find_the_path_with_the_highest_version_number = () => directory.Should().Be(MongoBinaries); + } + + [Subject("FolderSearch")] + public class when_searching_for_not_existing_folder_upwards : FolderSearchSpec + { + static string searchPattern = Path.Combine("packages", "Mongo2Go*", "XXX", "mongodb-win32-i386*", "bin"); + static string directory; + + Because of = () => directory = LocationOfAssembly.FindFolderUpwards(searchPattern); + It should_return_null = () => directory.Should().BeNull(); + } + + [Subject("FolderSearch")] + public class when_remove_last_part_of_path + { + static string directory; + + Because of = () => directory = Path.Combine("test1", "test2", "test3").RemoveLastPart(); + It should_remove_the_element = () => directory.Should().Be(Path.Combine("test1", "test2")); + } + + [Subject("FolderSearch")] + public class when_remove_last_part_of_single_element_path + { + static string directory; + + Because of = () => directory = "test1".RemoveLastPart(); + It should_return_null = () => directory.Should().BeNull(); + } + + [Subject("FolderSearch")] + public class when_directory_contains_multiple_versions_mongo2go + { + private readonly string[] directories; + + private static string getAssemblyVersion() + { + // ReSharper disable once PossibleNullReferenceException + return typeof(FolderSearch).GetTypeInfo().Assembly.GetCustomAttribute().InformationalVersion; + } + + public when_directory_contains_multiple_versions_mongo2go() + { + + // setup some directories + directories = new[] + { + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, getAssemblyVersion() + "a"), + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "2.2.9"), + Path.Combine(AppDomain.CurrentDomain.BaseDirectory, getAssemblyVersion()) + }; + + foreach (var d in directories) + Directory.CreateDirectory(d); + } + + private static string path; + + private Because of = () => path = FolderSearch.FindFolder(AppDomain.CurrentDomain.BaseDirectory, "*"); + + private It should_return_the_one_that_matches_our_own_assembly_version = + () => path.Should().Be(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, getAssemblyVersion())); + } + + public class FolderSearchSpec + { + public static string BaseDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + public static string MongoBinaries = Path.Combine(BaseDir, "test1", "test2", "packages", "Mongo2Go.1.2.3", "tools", "mongodb-win32-i386-2.0.7-rc0", "bin"); + public static string MongoOlderBinaries = Path.Combine(BaseDir, "test1", "test2", "packages", "Mongo2Go.1.1.1", "tools", "mongodb-win32-i386-2.0.7-rc0", "bin"); + public static string LocationOfAssembly = Path.Combine(BaseDir, "test1", "test2", "Project", "bin"); + + Establish context = () => + { + if (!Directory.Exists(BaseDir)) { Directory.CreateDirectory(BaseDir); } + if (!Directory.Exists(MongoBinaries)) { Directory.CreateDirectory(MongoBinaries); } + if (!Directory.Exists(MongoOlderBinaries)) { Directory.CreateDirectory(MongoOlderBinaries); } + if (!Directory.Exists(LocationOfAssembly)) { Directory.CreateDirectory(LocationOfAssembly); } + }; + + Cleanup stuff = () => { if (Directory.Exists(BaseDir)) { Directory.Delete(BaseDir, true); }}; + } +} +// ReSharper restore UnusedMember.Local +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj b/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj new file mode 100644 index 00000000..7c596c21 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Mongo2GoTests.csproj @@ -0,0 +1,21 @@ + + + net8.0 + false + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/MongoDbAdditionalArgumentsTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/MongoDbAdditionalArgumentsTests.cs new file mode 100644 index 00000000..1ec07138 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/MongoDbAdditionalArgumentsTests.cs @@ -0,0 +1,76 @@ +using FluentAssertions; +using Machine.Specifications; +using Mongo2Go.Helper; +using System; + +// ReSharper disable InconsistentNaming +// ReSharper disable UnusedMember.Local +namespace Mongo2GoTests +{ + [Subject(typeof(MongodArguments))] + public class when_null_additional_arguments_return_empty_string + { + private static string validAdditionalArguments; + + Because of = () => validAdditionalArguments = MongodArguments.GetValidAdditionalArguments(string.Empty, null); + It should_be_empty_string = () => validAdditionalArguments.Should().BeEmpty(); + } + + [Subject(typeof(MongodArguments))] + public class when_no_additional_arguments_return_empty_string + { + private static string validAdditionalArguments; + + Because of = () => validAdditionalArguments = MongodArguments.GetValidAdditionalArguments(string.Empty, string.Empty); + It should_be_empty_string = () => validAdditionalArguments.Should().BeEmpty(); + } + + [Subject(typeof(MongodArguments))] + public class when_additional_arguments_start_with_argument_separator_return_additional_arguments + { + private static string validAdditionalArguments; + private const string additionalArgumentsUnderTest = " --argument_1 under_test --argument_2 under test"; + private const string expectedAdditionalArguments = " --argument_1 under_test --argument_2 under test"; + + Because of = () => validAdditionalArguments = MongodArguments.GetValidAdditionalArguments(string.Empty, additionalArgumentsUnderTest); + It should_be_expected_additional_arguments = () => validAdditionalArguments.Should().Be(expectedAdditionalArguments); + } + + [Subject(typeof(MongodArguments))] + public class when_additional_arguments_does_not_start_with_argument_separator_return_additional_arguments + { + private static string validAdditionalArguments; + private const string additionalArgumentsUnderTest = "argument_1 under_test --argument_2 under test"; + private const string expectedAdditionalArguments = " --argument_1 under_test --argument_2 under test"; + + Because of = () => validAdditionalArguments = MongodArguments.GetValidAdditionalArguments(string.Empty, additionalArgumentsUnderTest); + It should_be_expected_additional_arguments = () => validAdditionalArguments.Should().Be(expectedAdditionalArguments); + } + + [Subject(typeof(MongodArguments))] + public class when_existing_arguments_and_additional_arguments_do_not_have_shared_options_return_additional_arguments + { + private static string validAdditionalArguments; + private const string existingArguments = "--existing_argument1 --existing_argument2"; + private const string additionalArgumentsUnderTest = " --argument_1 under_test --argument_2 under test"; + private const string expectedAdditionalArguments = " --argument_1 under_test --argument_2 under test"; + + Because of = () => validAdditionalArguments = MongodArguments.GetValidAdditionalArguments(existingArguments, additionalArgumentsUnderTest); + It should_be_expected_additional_arguments = () => validAdditionalArguments.Should().Be(expectedAdditionalArguments); + } + + [Subject(typeof(MongodArguments))] + public class when_existing_arguments_and_additional_arguments_have_shared_options_throw_argument_exception + { + private static Exception exception; + private const string duplicateArgument = "existing_argument2"; + private static readonly string existingArguments = $"--existing_argument1 --{duplicateArgument}"; + private static readonly string additionalArgumentsUnderTest = $" --argument_1 under_test --{duplicateArgument} argument2_new_value --argument_2 under test"; + + Because of = () => exception = Catch.Exception(() => MongodArguments.GetValidAdditionalArguments(existingArguments, additionalArgumentsUnderTest)); + It should_throw_argument_exception = () => exception.Should().BeOfType(); + It should_contain_more_than_instance_of_the_duplicate_argument = () => exception.Message.IndexOf(duplicateArgument, StringComparison.InvariantCultureIgnoreCase).Should().NotBe(exception.Message.LastIndexOf(duplicateArgument, StringComparison.InvariantCultureIgnoreCase)); + } +} +// ReSharper restore UnusedMember.Local +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoDebuggingTest.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoDebuggingTest.cs new file mode 100644 index 00000000..0cbe12a7 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoDebuggingTest.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; + +namespace Mongo2GoTests.Runner +{ + public class MongoDebuggingTest + { + internal static MongoDbRunner _runner; + internal static IMongoCollection _collection; + internal static string _databaseName = "IntegrationTest"; + internal static string _collectionName = "TestCollection"; + internal static IMongoDatabase _database; + + internal static void CreateConnection() + { + _runner = MongoDbRunner.StartForDebugging(singleNodeReplSet: false); + + MongoClient client = new MongoClient(_runner.ConnectionString); + _database = client.GetDatabase(_databaseName); + _collection = _database.GetCollection(_collectionName); + } + + public static IList ReadBsonFile(string fileName) + { + string[] content = File.ReadAllLines(fileName); + return content.Select(s => BsonSerializer.Deserialize(s)).ToList(); + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoIntegrationTest.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoIntegrationTest.cs new file mode 100644 index 00000000..3ff56ff4 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoIntegrationTest.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Logging; +using Mongo2Go; +using MongoDB.Driver; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace Mongo2GoTests.Runner +{ + public class MongoIntegrationTest + { + internal static MongoDbRunner _runner; + internal static IMongoCollection _collection; + internal static string _databaseName = "IntegrationTest"; + internal static string _collectionName = "TestCollection"; + + internal static void CreateConnection(ILogger logger = null) + { + _runner = MongoDbRunner.Start(singleNodeReplSet: false, logger: logger); + + MongoClient client = new MongoClient(_runner.ConnectionString); + IMongoDatabase database = client.GetDatabase(_databaseName); + _collection = database.GetCollection(_collectionName); + } + } + + public static class TaskExtensions + { + public static async Task WithTimeout(this Task task, TimeSpan timeout) + { + using (var cancellationTokenSource = new CancellationTokenSource()) + { + + var completedTask = await Task.WhenAny(task, Task.Delay(timeout, cancellationTokenSource.Token)); + if (completedTask == task) + { + cancellationTokenSource.Cancel(); + await task; + } + else + { + throw new TimeoutException("The operation has timed out."); + } + } + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoTransactionTest.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoTransactionTest.cs new file mode 100644 index 00000000..9caa6dc9 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/MongoTransactionTest.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Mongo2Go; +using MongoDB.Driver; + +namespace Mongo2GoTests.Runner +{ + public class MongoTransactionTest + { + internal static MongoDbRunner _runner; + internal static IMongoCollection _mainCollection; + internal static IMongoCollection _dependentCollection; + internal static string _databaseName = "TransactionTest"; + internal static string _mainCollectionName = "MainCollection"; + internal static string _dependentCollectionName = "DependentCollection"; + internal static IMongoDatabase database; + internal static IMongoClient client; + internal static void CreateConnection(ushort? singleNodeReplSetWaitTimeout = null) + { + if (singleNodeReplSetWaitTimeout.HasValue) + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true, singleNodeReplSetWaitTimeout: singleNodeReplSetWaitTimeout.Value); + } + else + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + } + + client = new MongoClient(_runner.ConnectionString); + database = client.GetDatabase(_databaseName); + _mainCollection = database.GetCollection(_mainCollectionName); + _dependentCollection = database.GetCollection(_dependentCollectionName); + } + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerImportExportTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerImportExportTests.cs new file mode 100644 index 00000000..3a219669 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerImportExportTests.cs @@ -0,0 +1,89 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using FluentAssertions; +using Machine.Specifications; +using Mongo2Go.Helper; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using It = Machine.Specifications.It; + +// ReSharper disable InconsistentNaming +// ReSharper disable UnusedMember.Local +namespace Mongo2GoTests.Runner +{ + [Subject("Runner Integration Test")] + public class when_using_monogoexport : MongoDebuggingTest + { + static readonly string _testFile = Path.GetTempPath() + "testExport.json"; + static IList parsedContent; + + Establish context = () => + + { + CreateConnection(); + _database.DropCollection(_collectionName); + + _collection.InsertOne(TestDocument.DummyData1()); + _collection.InsertOne(TestDocument.DummyData2()); + _collection.InsertOne(TestDocument.DummyData3()); + }; + + Because of = () => + { + _runner.Export(_databaseName, _collectionName, _testFile); + Thread.Sleep(500); + parsedContent = ReadBsonFile(_testFile); + }; + + It should_preserve_all_values1 = () => parsedContent[0].Should().BeEquivalentTo(TestDocument.DummyData1(), cfg => cfg.Excluding(d => d.Id)); + It should_preserve_all_values2 = () => parsedContent[1].Should().BeEquivalentTo(TestDocument.DummyData2(), cfg => cfg.Excluding(d => d.Id)); + It should_preserve_all_values3 = () => parsedContent[2].Should().BeEquivalentTo(TestDocument.DummyData3(), cfg => cfg.Excluding(d => d.Id)); + + Cleanup stuff = () => + { + new FileSystem().DeleteFile(_testFile); + _runner.Dispose(); + }; + } + + [Subject("Runner Integration Test")] + public class when_using_monogoimport : MongoDebuggingTest + { + static IQueryable query; + static readonly string _testFile = Path.GetTempPath() + "testImport.json"; + + const string _filecontent = + @"{ ""_id"" : { ""$oid"" : ""50227b375dff9218248eadc4"" }, ""StringTest"" : ""Hello World"", ""IntTest"" : 42, ""DateTest"" : { ""$date"" : ""1984-09-30T06:06:06.171Z"" }, ""ListTest"" : [ ""I"", ""am"", ""a"", ""list"", ""of"", ""strings"" ] }" + "\r\n" + + @"{ ""_id"" : { ""$oid"" : ""50227b375dff9218248eadc5"" }, ""StringTest"" : ""Foo"", ""IntTest"" : 23, ""DateTest"" : null, ""ListTest"" : null }" + "\r\n" + + @"{ ""_id"" : { ""$oid"" : ""50227b375dff9218248eadc6"" }, ""StringTest"" : ""Bar"", ""IntTest"" : 77, ""DateTest"" : null, ""ListTest"" : null }" + "\r\n"; + + Establish context = () => + { + CreateConnection(); + _database.DropCollection(_collectionName); + File.WriteAllText(_testFile, _filecontent); + }; + + Because of = () => + { + _runner.Import(_databaseName, _collectionName, _testFile, true); + Thread.Sleep(500); + query = _collection.AsQueryable().Select(c => c).OrderBy(c => c.Id); ; + + }; + + It should_return_document1 = () => query.ToList().ElementAt(0).Should().BeEquivalentTo(TestDocument.DummyData1(), cfg => cfg.Excluding(d => d.Id)); + It should_return_document2 = () => query.ToList().ElementAt(1).Should().BeEquivalentTo(TestDocument.DummyData2(), cfg => cfg.Excluding(d => d.Id)); + It should_return_document3 = () => query.ToList().ElementAt(2).Should().BeEquivalentTo(TestDocument.DummyData3(), cfg => cfg.Excluding(d => d.Id)); + + Cleanup stuff = () => + { + new FileSystem().DeleteFile(_testFile); + _runner.Dispose(); + }; + } +} +// ReSharper restore UnusedMember.Local +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerIntegrationTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerIntegrationTests.cs new file mode 100644 index 00000000..3af3a881 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerIntegrationTests.cs @@ -0,0 +1,118 @@ +using FluentAssertions; +using Machine.Specifications; +using MELT; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using It = Machine.Specifications.It; + +// ReSharper disable InconsistentNaming +// ReSharper disable UnusedMember.Local +namespace Mongo2GoTests.Runner +{ + [Subject("Runner Integration Test")] + public class when_using_the_inbuild_serialization : MongoIntegrationTest + { + static TestDocument findResult; + + + Establish context = () => + { + CreateConnection(); + _collection.InsertOne(TestDocument.DummyData1()); + }; + + Because of = () => findResult = _collection.FindSync(_ => true).First(); + + It should_return_a_result = () => findResult.Should().NotBeNull(); + It should_hava_expected_data = () => findResult.Should().BeEquivalentTo(TestDocument.DummyData1(), cfg => cfg.Excluding(d => d.Id)); + + Cleanup stuff = () => _runner.Dispose(); + } + + [Subject("Runner Integration Test")] + public class when_using_the_new_linq_support : MongoIntegrationTest + { + static List queryResult; + + Establish context = () => + { + CreateConnection(); + _collection.InsertOne(TestDocument.DummyData1()); + _collection.InsertOne(TestDocument.DummyData2()); + _collection.InsertOne(TestDocument.DummyData3()); + }; + + Because of = () => + { + queryResult = (from c in _collection.AsQueryable() + where c.StringTest == TestDocument.DummyData2().StringTest || c.StringTest == TestDocument.DummyData3().StringTest + select c).ToList(); + }; + + It should_return_two_documents = () => queryResult.Count().Should().Be(2); + It should_return_document2 = () => queryResult.ElementAt(0).IntTest = TestDocument.DummyData2().IntTest; + It should_return_document3 = () => queryResult.ElementAt(1).IntTest = TestDocument.DummyData3().IntTest; + + Cleanup stuff = () => _runner.Dispose(); + } + + [Subject("Runner Integration Test")] + public class when_using_commands_that_create_console_output : MongoIntegrationTest + { + static List taskList = new List(); + + private Establish context = () => + { + CreateConnection(); + }; + + private Because of = () => + { + var createIndexModel = new CreateIndexModel(Builders.IndexKeys.Ascending(x => x.IntTest)); + taskList.Add(_collection.Indexes.CreateOneAsync(createIndexModel).WithTimeout(TimeSpan.FromMilliseconds(5000))); + taskList.Add(_collection.Indexes.DropAllAsync().WithTimeout(TimeSpan.FromMilliseconds(5000))); + }; + + It should_not_timeout = () => Task.WaitAll(taskList.ToArray()); + + Cleanup stuff = () => _runner.Dispose(); + } + + + [Subject("Runner Integration Test")] + public class when_using_microsoft_ilogger : MongoIntegrationTest + { + static List taskList = new List(); + static ITestLoggerFactory loggerFactory; + + private Establish context = () => + { + loggerFactory = TestLoggerFactory.Create(); + var logger = loggerFactory.CreateLogger("MyTestLogger"); + CreateConnection(logger); + }; + + private Because of = () => + { + var createIndexModel = new CreateIndexModel(Builders.IndexKeys.Ascending(x => x.IntTest)); + taskList.Add(_collection.Indexes.CreateOneAsync(createIndexModel).WithTimeout(TimeSpan.FromMilliseconds(5000))); + taskList.Add(_collection.Indexes.DropAllAsync().WithTimeout(TimeSpan.FromMilliseconds(5000))); + }; + + It should_not_timeout = () => Task.WaitAll(taskList.ToArray()); + It should_have_received_many_logs = () => + loggerFactory.Sink.LogEntries.Count(l => l.LogLevel == Microsoft.Extensions.Logging.LogLevel.Information) + .Should().BeGreaterThan(10); + It should_have_created_collection_statement = () => loggerFactory.Sink.LogEntries + .Count(l => l.Properties.Any(p => p.Key == "message" && (string)p.Value == "createCollection")) + .Should().BeGreaterOrEqualTo(1); + + Cleanup stuff = () => _runner.Dispose(); + } +} +// ReSharper restore UnusedMember.Local +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTests.cs new file mode 100644 index 00000000..9d3be52c --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTests.cs @@ -0,0 +1,108 @@ +using FluentAssertions; +using Machine.Specifications; +using Mongo2Go; +using Mongo2Go.Helper; +using Moq; +using System.IO; +using It = Machine.Specifications.It; + +#pragma warning disable CS0618 // Type or member is obsolete + +// ReSharper disable InconsistentNaming +// ReSharper disable UnusedMember.Local + +namespace Mongo2GoTests.Runner +{ + [Subject("Runner")] + public class when_instantiating_the_runner_for_integration_test + { + static MongoDbRunner runner; + static Mock portPoolMock; + static Mock fileSystemMock; + static Mock processStarterMock; + static Mock binaryLocatorMock; + + static string exptectedDataDirectory; + static string exptectedLogfile; + static readonly string exptectedConnectString = "mongodb://127.0.0.1:{0}/".Formatted(MongoDbDefaults.TestStartPort + 1); + + Establish context = () => + { + portPoolMock = new Mock(); + portPoolMock.Setup(m => m.GetNextOpenPort()).Returns(MongoDbDefaults.TestStartPort + 1); + + fileSystemMock = new Mock(); + fileSystemMock.Setup(m => m.CreateFolder(Moq.It.IsAny())).Callback(s => + { + exptectedDataDirectory = s; + exptectedLogfile = Path.Combine(exptectedDataDirectory, MongoDbDefaults.Lockfile); + }); + + var processMock = new Mock(); + + processStarterMock = new Mock(); + processStarterMock.Setup(m => m.Start(Moq.It.IsAny(), Moq.It.IsAny(), Moq.It.IsAny(), false, Moq.It.IsAny(), Moq.It.IsAny(), null)).Returns(processMock.Object); + + binaryLocatorMock = new Mock (); + binaryLocatorMock.Setup(m => m.Directory).Returns(string.Empty); + }; + + Because of = () => runner = MongoDbRunner.StartUnitTest(portPoolMock.Object, fileSystemMock.Object, processStarterMock.Object, binaryLocatorMock.Object); + + It should_create_the_data_directory = () => fileSystemMock.Verify(x => x.CreateFolder(Moq.It.Is(s => s.StartsWith(Path.GetTempPath()))), Times.Exactly(1)); + It should_delete_old_lock_file = () => fileSystemMock.Verify(x => x.DeleteFile(exptectedLogfile), Times.Exactly(1)); + + It should_start_the_process = () => processStarterMock.Verify(x => x.Start(Moq.It.IsAny(), Moq.It.IsAny(), Moq.It.IsAny(), false, Moq.It.IsAny(), Moq.It.IsAny(), null), Times.Exactly(1)); + + It should_have_expected_connection_string = () => runner.ConnectionString.Should().Be(exptectedConnectString); + It should_return_an_instance_with_state_running = () => runner.State.Should().Be(State.Running); + } + + [Subject("Runner")] + public class when_instantiating_the_runner_for_local_debugging + { + static MongoDbRunner runner; + static Mock portWatcherMock; + static Mock processWatcherMock; + static Mock fileSystemMock; + static Mock processStarterMock; + static Mock binaryLocatorMock; + + static string exptectedDataDirectory; + static string exptectedLogfile; + + Establish context = () => + { + processWatcherMock = new Mock(); + processWatcherMock.Setup(m => m.IsProcessRunning(Moq.It.IsAny())).Returns(false); + + portWatcherMock = new Mock(); + portWatcherMock.Setup(m => m.IsPortAvailable(Moq.It.IsAny())).Returns(true); + + fileSystemMock = new Mock(); + fileSystemMock.Setup(m => m.CreateFolder(Moq.It.IsAny())).Callback(s => + { + exptectedDataDirectory = s; + exptectedLogfile = Path.Combine(exptectedDataDirectory, MongoDbDefaults.Lockfile); + }); + + var processMock = new Mock(); + processStarterMock = new Mock(); + processStarterMock.Setup(m => m.Start(Moq.It.IsAny(), exptectedDataDirectory, MongoDbDefaults.DefaultPort, true, false, Moq.It.IsAny(), Moq.It.IsAny(), null)).Returns(processMock.Object); + + binaryLocatorMock = new Mock (); + binaryLocatorMock.Setup(m => m.Directory).Returns(string.Empty); + }; + + Because of = () => runner = MongoDbRunner.StartForDebuggingUnitTest(processWatcherMock.Object, portWatcherMock.Object, fileSystemMock.Object, processStarterMock.Object, binaryLocatorMock.Object); + + It should_check_for_already_running_process = () => processWatcherMock.Verify(x => x.IsProcessRunning(MongoDbDefaults.ProcessName), Times.Exactly(1)); + It should_check_the_default_port = () => portWatcherMock.Verify(x => x.IsPortAvailable(MongoDbDefaults.DefaultPort), Times.Exactly(1)); + It should_create_the_data_directory = () => fileSystemMock.Verify(x => x.CreateFolder(Moq.It.Is(s => s.StartsWith(Path.GetTempPath()))), Times.Exactly(1)); + It should_delete_old_lock_file = () => fileSystemMock.Verify(x => x.DeleteFile(exptectedLogfile), Times.Exactly(1)); + It should_return_an_instance_with_state_running = () => runner.State.Should().Be(State.Running); + It should_start_the_process_without_kill = () => processStarterMock.Verify(x => x.Start(Moq.It.IsAny(), exptectedDataDirectory, MongoDbDefaults.DefaultPort, true, false, Moq.It.IsAny(), Moq.It.IsAny(), null), Times.Exactly(1)); + } +} +// ReSharper restore UnusedMember.Local +// ReSharper restore InconsistentNaming \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTransactionTests.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTransactionTests.cs new file mode 100644 index 00000000..612794c1 --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/RunnerTransactionTests.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using FluentAssertions; +using Machine.Specifications; +using MongoDB.Driver; + +namespace Mongo2GoTests.Runner +{ + [Subject("Runner Transaction Test")] + public class when_transaction_completes : MongoTransactionTest + { + private static TestDocument mainDocument; + private static TestDocument dependentDocument; + + Establish context = () => + + { + CreateConnection(); + database.DropCollection(_mainCollectionName); + database.DropCollection(_dependentCollectionName); + _mainCollection.InsertOne(TestDocument.DummyData2()); + _dependentCollection.InsertOne(TestDocument.DummyData2()); + }; + + private Because of = () => + { + var filter = Builders.Filter.Where(x => x.IntTest == 23); + var update = Builders.Update.Inc(i => i.IntTest, 10); + + using (var sessionHandle = client.StartSession()) + { + try + { + var i = 0; + while (i < 10) + { + try + { + i++; + sessionHandle.StartTransaction(new TransactionOptions( + readConcern: ReadConcern.Local, + writeConcern: WriteConcern.W1)); + try + { + var first = _mainCollection.UpdateOne(sessionHandle, filter, update); + var second = _dependentCollection.UpdateOne(sessionHandle, filter, update); + } + catch (Exception) + { + sessionHandle.AbortTransaction(); + throw; + } + + var j = 0; + while (j < 10) + { + try + { + j++; + sessionHandle.CommitTransaction(); + break; + } + catch (MongoException e) + { + if (e.HasErrorLabel("UnknownTransactionCommitResult")) + continue; + throw; + } + } + break; + } + catch (MongoException e) + { + if (e.HasErrorLabel("TransientTransactionError")) + continue; + throw; + } + } + } + catch (Exception) + { + + } + } + + mainDocument = _mainCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + dependentDocument = _dependentCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + }; + + It main_should_be_33 = () => mainDocument.IntTest.Should().Be(33); + It dependent_should_be_33 = () => dependentDocument.IntTest.Should().Be(33); + Cleanup cleanup = () => _runner.Dispose(); + } + + + [Subject("Runner Transaction Test")] + public class when_transaction_is_aborted_before_commit : MongoTransactionTest + { + private static TestDocument mainDocument; + private static TestDocument dependentDocument; + private static TestDocument mainDocument_before_commit; + private static TestDocument dependentDocument_before_commit; + Establish context = () => + + { + CreateConnection(); + database.DropCollection(_mainCollectionName); + database.DropCollection(_dependentCollectionName); + _mainCollection.InsertOne(TestDocument.DummyData2()); + _dependentCollection.InsertOne(TestDocument.DummyData2()); + + }; + + private Because of = () => + { + var filter = Builders.Filter.Where(x => x.IntTest == 23); + var update = Builders.Update.Inc(i => i.IntTest, 10); + + using (var sessionHandle = client.StartSession()) + { + try + { + var i = 0; + while (i < 2) + { + try + { + i++; + sessionHandle.StartTransaction(new TransactionOptions( + readConcern: ReadConcern.Local, + writeConcern: WriteConcern.W1)); + try + { + var first = _mainCollection.UpdateOne(sessionHandle, filter, update); + var second = _dependentCollection.UpdateOne(sessionHandle, filter, update); + mainDocument_before_commit = _mainCollection.FindSync(sessionHandle, Builders.Filter.Empty).ToList().FirstOrDefault(); + dependentDocument_before_commit = _dependentCollection.FindSync(sessionHandle, Builders.Filter.Empty).ToList().FirstOrDefault(); + } + catch (Exception) + { + sessionHandle.AbortTransaction(); + throw; + } + + //Throw exception and do not commit + throw new ApplicationException(); + } + catch (MongoException e) + { + if (e.HasErrorLabel("TransientTransactionError")) + continue; + throw; + } + + } + } + catch (Exception) + { + + } + } + + mainDocument = _mainCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + dependentDocument = _dependentCollection.FindSync(Builders.Filter.Empty).FirstOrDefault(); + }; + + It main_should_be_still_23_after_aborting = () => mainDocument.IntTest.Should().Be(23); + It dependent_should_be_still_23_after_aborting = () => dependentDocument.IntTest.Should().Be(23); + It main_should_be_33_before_aborting = () => mainDocument_before_commit.IntTest.Should().Be(33); + It dependent_should_be_33_before_aborting = () => dependentDocument_before_commit.IntTest.Should().Be(33); + Cleanup cleanup = () => _runner.Dispose(); + } + + [Subject("Runner Transaction Test")] + public class when_replica_set_not_ready_before_timeout_expires : MongoTransactionTest + { + private static Exception exception; + + Because of = () => exception = Catch.Exception(() => CreateConnection(0)); + + // this passes on Windows (TimeoutException as expected) + // but breaks on my Mac (MongoDB.Driver.MongoCommandException: Command replSetInitiate failed: already initialized.) + It should_throw_timeout_exception = () => { + Console.WriteLine(exception.ToString()); + exception.Should().BeOfType(); + }; + } +} diff --git a/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/TestDocument.cs b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/TestDocument.cs new file mode 100644 index 00000000..34b54eee --- /dev/null +++ b/Mongo2Go-4.1.0/src/Mongo2GoTests/Runner/TestDocument.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Mongo2GoTests.Runner +{ + public class TestDocument + { + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } + + public string StringTest { get; set; } + + public int IntTest { get; set; } + + [BsonDateTimeOptions(Kind = DateTimeKind.Local)] + public DateTime? DateTest { get; set; } + + public List ListTest { get; set; } + + public static TestDocument DummyData1() + { + return new TestDocument + { + StringTest = "Hello World", + IntTest = 42, + DateTest = new DateTime(1984, 09, 30, 6, 6, 6, 171, DateTimeKind.Utc).ToLocalTime(), + ListTest = new List {"I", "am", "a", "list", "of", "strings"} + }; + } + + public static TestDocument DummyData2() + { + return new TestDocument + { + StringTest = "Foo", + IntTest = 23, + }; + } + + public static TestDocument DummyData3() + { + return new TestDocument + { + StringTest = "Bar", + IntTest = 77, + }; + } + + } +} diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveExtractor.cs b/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveExtractor.cs new file mode 100644 index 00000000..ee0ed755 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveExtractor.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ByteSizeLib; +using Espresso3389.HttpStream; +using HttpProgress; +using ICSharpCode.SharpZipLib.GZip; +using ICSharpCode.SharpZipLib.Tar; +using ICSharpCode.SharpZipLib.Zip; + +namespace MongoDownloader +{ + internal class ArchiveExtractor + { + private static readonly int CachePageSize = Convert.ToInt32(ByteSize.FromMebiBytes(4).Bytes); + + private readonly Options _options; + private readonly BinaryStripper? _binaryStripper; + + public ArchiveExtractor(Options options, BinaryStripper? binaryStripper) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _binaryStripper = binaryStripper; + } + + public async Task>> DownloadExtractZipArchiveAsync(Download download, DirectoryInfo extractDirectory, ArchiveProgress progress, CancellationToken cancellationToken) + { + var bytesTransferred = 0L; + using var headResponse = await _options.HttpClient.SendAsync(new HttpRequestMessage(HttpMethod.Head, download.Archive.Url), cancellationToken); + var contentLength = headResponse.Content.Headers.ContentLength ?? 0; + var cacheFile = new FileInfo(Path.Combine(_options.CacheDirectory.FullName, download.Archive.Url.Segments.Last())); + await using var cacheStream = new FileStream(cacheFile.FullName, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + var stopwatch = Stopwatch.StartNew(); + await using var httpStream = new HttpStream(download.Archive.Url, cacheStream, ownStream: false, CachePageSize, cached: null); + httpStream.RangeDownloaded += (_, args) => + { + bytesTransferred += args.Length; + progress.Report(new CopyProgress(stopwatch.Elapsed, 0, bytesTransferred, contentLength)); + }; + using var zipFile = new ZipFile(httpStream); + var binaryRegex = _options.Binaries[(download.Product, download.Platform)]; + var licenseRegex = _options.Licenses[(download.Product, download.Platform)]; + var stripTasks = new List>(); + foreach (var entry in zipFile.Cast().Where(e => e.IsFile)) + { + var nameParts = entry.Name.Split('\\', '/').Skip(1).ToList(); + var zipEntryPath = string.Join('/', nameParts); + var isBinaryFile = binaryRegex.IsMatch(zipEntryPath); + var isLicenseFile = licenseRegex.IsMatch(zipEntryPath); + if (isBinaryFile || isLicenseFile) + { + var destinationPathParts = isLicenseFile ? nameParts.Prepend(ProductDirectoryName(download.Product)) : nameParts; + var destinationFile = new FileInfo(Path.Combine(destinationPathParts.Prepend(extractDirectory.FullName).ToArray())); + destinationFile.Directory?.Create(); + await using var destinationStream = destinationFile.OpenWrite(); + await using var inputStream = zipFile.GetInputStream(entry); + await inputStream.CopyToAsync(destinationStream, cancellationToken); + if (isBinaryFile && _binaryStripper is not null) + { + stripTasks.Add(_binaryStripper.StripAsync(destinationFile, cancellationToken)); + } + } + } + progress.Report(new CopyProgress(stopwatch.Elapsed, 0, bytesTransferred, bytesTransferred)); + return stripTasks; + } + + public IEnumerable> ExtractArchive(Download download, FileInfo archive, DirectoryInfo extractDirectory, CancellationToken cancellationToken) + { + switch (Path.GetExtension(archive.Name)) + { + case ".tgz": + return ExtractTarGzipArchive(download, archive, extractDirectory, cancellationToken); + default: + throw new NotSupportedException($"Only .tgz archives are currently supported. \"{archive.FullName}\" can not be extracted."); + } + } + + private IEnumerable> ExtractTarGzipArchive(Download download, FileInfo archive, DirectoryInfo extractDirectory, CancellationToken cancellationToken) + { + // See https://github.com/icsharpcode/SharpZipLib/wiki/GZip-and-Tar-Samples#-simple-full-extract-from-a-tgz-targz + using var archiveStream = archive.OpenRead(); + using var gzipStream = new GZipInputStream(archiveStream); + using var tarArchive = TarArchive.CreateInputTarArchive(gzipStream, Encoding.UTF8); + var extractedFileNames = new List(); + tarArchive.ProgressMessageEvent += (_, entry, _) => + { + cancellationToken.ThrowIfCancellationRequested(); + extractedFileNames.Add(entry.Name); + }; + tarArchive.ExtractContents(extractDirectory.FullName); + return CleanupExtractedFiles(download, extractDirectory, extractedFileNames); + } + + private IEnumerable> CleanupExtractedFiles(Download download, DirectoryInfo extractDirectory, IEnumerable extractedFileNames) + { + var rootDirectoryToDelete = new HashSet(); + var binaryRegex = _options.Binaries[(download.Product, download.Platform)]; + var licenseRegex = _options.Licenses[(download.Product, download.Platform)]; + var stripTasks = new List>(); + foreach (var extractedFileName in extractedFileNames.Select(e => e.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar))) + { + var extractedFile = new FileInfo(Path.Combine(extractDirectory.FullName, extractedFileName)); + var parts = extractedFileName.Split(Path.DirectorySeparatorChar); + var entryFileName = string.Join("/", parts.Skip(1)); + rootDirectoryToDelete.Add(parts[0]); + var isBinaryFile = binaryRegex.IsMatch(entryFileName); + var isLicenseFile = licenseRegex.IsMatch(entryFileName); + if (!(isBinaryFile || isLicenseFile)) + { + extractedFile.Delete(); + } + else + { + var destinationPathParts = parts.Skip(1); + if (isLicenseFile) + { + destinationPathParts = destinationPathParts.Prepend(ProductDirectoryName(download.Product)); + } + var destinationFile = new FileInfo(Path.Combine(destinationPathParts.Prepend(extractDirectory.FullName).ToArray())); + destinationFile.Directory?.Create(); + extractedFile.MoveTo(destinationFile.FullName); + if (isBinaryFile && _binaryStripper is not null) + { + stripTasks.Add(_binaryStripper.StripAsync(destinationFile)); + } + } + } + var rootArchiveDirectory = new DirectoryInfo(Path.Combine(extractDirectory.FullName, rootDirectoryToDelete.Single())); + var binDirectory = new DirectoryInfo(Path.Combine(rootArchiveDirectory.FullName, "bin")); + binDirectory.Delete(recursive: false); + rootArchiveDirectory.Delete(recursive: false); + return stripTasks; + } + + private static string ProductDirectoryName(Product product) + { + return product switch + { + Product.CommunityServer => "community-server", + Product.DatabaseTools => "database-tools", + _ => throw new ArgumentOutOfRangeException(nameof(product), product, null) + }; + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveProgress.cs b/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveProgress.cs new file mode 100644 index 00000000..df127bab --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/ArchiveProgress.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using ByteSizeLib; +using HttpProgress; +using Spectre.Console; + +namespace MongoDownloader +{ + public class ArchiveProgress : IProgress + { + private readonly ProgressTask _archiveProgress; + private readonly ProgressTask _globalProgress; + private readonly IEnumerable _allArchiveProgresses; + private readonly Download _download; + private readonly string _completedDescription; + + public ArchiveProgress(ProgressTask archiveProgress, ProgressTask globalProgress, IEnumerable allArchiveProgresses, Download download, string completedDescription) + { + _archiveProgress = archiveProgress ?? throw new ArgumentNullException(nameof(archiveProgress)); + _globalProgress = globalProgress ?? throw new ArgumentNullException(nameof(globalProgress)); + _allArchiveProgresses = allArchiveProgresses ?? throw new ArgumentNullException(nameof(allArchiveProgresses)); + _download = download ?? throw new ArgumentNullException(nameof(download)); + _completedDescription = completedDescription ?? throw new ArgumentNullException(nameof(completedDescription)); + } + + public void Report(ICopyProgress progress) + { + _archiveProgress.Value = progress.BytesTransferred; + _archiveProgress.MaxValue = progress.ExpectedBytes; + + string text; + bool isIndeterminate; + if (progress.BytesTransferred < progress.ExpectedBytes) + { + var speed = ByteSize.FromBytes(progress.BytesTransferred / progress.TransferTime.TotalSeconds); + text = $"Downloading {_download} from {_download.Archive.Url} at {speed:0.0}/s"; + isIndeterminate = false; + } + else + { + text = $"Downloaded {_download}"; + isIndeterminate = true; + // Cheat by subtracting 1 so that the progress stays at 99% in indeterminate mode for + // remaining tasks (stripping) to complete with an indeterminate progress bar + _archiveProgress.Value = progress.BytesTransferred - 1; + } + Report(text, isIndeterminate); + + lock (_globalProgress) + { + _globalProgress.Value = _allArchiveProgresses.Sum(e => e.Value); + _globalProgress.MaxValue = _allArchiveProgresses.Sum(e => e.MaxValue); + } + } + + public void Report(string action) + { + Report(action, isIndeterminate: true); + } + + public void ReportCompleted(ByteSize strippedSize) + { + _archiveProgress.Value = _archiveProgress.MaxValue; + + lock (_globalProgress) + { + if (_allArchiveProgresses.All(e => e.IsFinished)) + { + _globalProgress.Description = _completedDescription; + _globalProgress.Value = _globalProgress.MaxValue; + } + } + + var saved = strippedSize.Bytes > 0 ? $" (saved {strippedSize:#.#} by stripping)" : ""; + Report($"Extracted {_download}{saved}", isIndeterminate: false); + } + + private void Report(string description, bool isIndeterminate) + { + _archiveProgress.Description = description; + _archiveProgress.IsIndeterminate = isIndeterminate; + lock (_globalProgress) + { + _globalProgress.IsIndeterminate = _allArchiveProgresses.All(e => e.IsFinished || e.IsIndeterminate); + } + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/BinaryStripper.cs b/Mongo2Go-4.1.0/src/MongoDownloader/BinaryStripper.cs new file mode 100644 index 00000000..a6ccb618 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/BinaryStripper.cs @@ -0,0 +1,92 @@ +using System; +using System.ComponentModel; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using ByteSizeLib; +using CliWrap; + +namespace MongoDownloader +{ + public class BinaryStripper + { + private const string LlvmStripToolName = "llvm-strip"; + + private readonly string _llvmStripPath; + + private BinaryStripper(string llvmStripPath) + { + _llvmStripPath = llvmStripPath ?? throw new ArgumentNullException(nameof(llvmStripPath)); + } + + public static async Task CreateAsync(CancellationToken cancellationToken) + { + var llvmStripPath = await GetLlvmStripPathAsync(cancellationToken); + return new BinaryStripper(llvmStripPath); + } + + public async Task StripAsync(FileInfo executable, CancellationToken cancellationToken = default) + { + var sizeBefore = ByteSize.FromBytes(executable.Length); + await Cli.Wrap(_llvmStripPath).WithArguments(executable.FullName).ExecuteAsync(cancellationToken); + executable.Refresh(); + var sizeAfter = ByteSize.FromBytes(executable.Length); + return sizeBefore - sizeAfter; + } + + private static async Task GetLlvmStripPathAsync(CancellationToken cancellationToken) + { + try + { + await Cli.Wrap(LlvmStripToolName).WithArguments("--version").ExecuteAsync(cancellationToken); + // llvm-strip is on the PATH + return LlvmStripToolName; + } + catch (Win32Exception exception) when (exception.NativeErrorCode == 2) + { + // llvm-strip is NOT in the PATH, let's search with homebrew + var llvmStripToolPath = await TryGetLlvmStripPathWithHomebrew(); + + if (llvmStripToolPath != null) + { + return llvmStripToolPath; + } + + throw new FileNotFoundException($"The \"{LlvmStripToolName}\" tool was not found."); + } + } + + private static async Task TryGetLlvmStripPathWithHomebrew() + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return null; + } + + string? llvmStripToolPath = null; + try + { + await Cli.Wrap("brew") + // don't validate exit code, if `brew list llvm` fails it's because the llvm formula is not installed + .WithValidation(CommandResultValidation.None) + .WithArguments(new[] {"list", "llvm"}) + .WithStandardOutputPipe(PipeTarget.ToDelegate(line => + { + if (llvmStripToolPath == null && line.EndsWith(LlvmStripToolName)) + { + llvmStripToolPath = line; + } + })) + .ExecuteAsync(); + } + catch (Win32Exception exception) when (exception.NativeErrorCode == 2) + { + // brew is not installed + return null; + } + + return llvmStripToolPath; + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/DataModel.cs b/Mongo2Go-4.1.0/src/MongoDownloader/DataModel.cs new file mode 100644 index 00000000..26366c38 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/DataModel.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Text.Json.Serialization; + +// ReSharper disable AutoPropertyCanBeMadeGetOnly.Global +// ReSharper disable ClassNeverInstantiated.Global +// ReSharper disable CollectionNeverUpdated.Global + +namespace MongoDownloader +{ + public enum Platform + { + Linux, + // ReSharper disable once InconsistentNaming + macOS, + Windows, + } + + public enum Product + { + CommunityServer, + DatabaseTools, + } + + /// + /// The root object of the JSON describing the available releases. + /// + public class Release + { + [JsonPropertyName("versions")] + public List Versions { get; set; } = new(); + } + + public class Version + { + [JsonPropertyName("version")] + public string Number { get; set; } = ""; + + [JsonPropertyName("production_release")] + public bool Production { get; set; } = false; + + [JsonPropertyName("downloads")] + public List Downloads { get; set; } = new(); + } + + public class Download + { + /// + /// Used to identify the platform for the Community Server archives + /// + [JsonPropertyName("target")] + public string Target { get; set; } = ""; + + /// + /// Used to identify the platform for the Database Tools archives + /// + [JsonPropertyName("name")] + public string Name { get; set; } = ""; + + [JsonPropertyName("arch")] + public string Arch { get; set; } = ""; + + [JsonPropertyName("edition")] + public string Edition { get; set; } = ""; + + [JsonPropertyName("archive")] + public Archive Archive { get; set; } = new(); + + public Product Product { get; set; } + + public Platform Platform { get; set; } + + public Architecture Architecture { get; set; } + + public override string ToString() => $"{Product} for {Platform}/{Architecture.ToString().ToLowerInvariant()}"; + } + + public class Archive + { + [JsonPropertyName("url")] + public Uri Url { get; set; } = default!; + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/MongoDbDownloader.cs b/Mongo2Go-4.1.0/src/MongoDownloader/MongoDbDownloader.cs new file mode 100644 index 00000000..a126b912 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/MongoDbDownloader.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using ByteSizeLib; +using HttpProgress; +using Spectre.Console; + +namespace MongoDownloader +{ + internal class MongoDbDownloader + { + private readonly ArchiveExtractor _extractor; + private readonly Options _options; + + public MongoDbDownloader(ArchiveExtractor extractor, Options options) + { + _extractor = extractor ?? throw new ArgumentNullException(nameof(extractor)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public async Task RunAsync(DirectoryInfo toolsDirectory, CancellationToken cancellationToken) + { + var strippedSize = await AnsiConsole + .Progress() + .Columns( + new ProgressBarColumn(), + new PercentageColumn(), + new RemainingTimeColumn(), + new DownloadedColumn(), + new TaskDescriptionColumn { Alignment = Justify.Left } + ) + .StartAsync(async context => await RunAsync(context, toolsDirectory, cancellationToken)); + + return strippedSize; + } + + private async Task RunAsync(ProgressContext context, DirectoryInfo toolsDirectory, CancellationToken cancellationToken) + { + const double initialMaxValue = double.Epsilon; + var globalProgress = context.AddTask("Downloading MongoDB", maxValue: initialMaxValue); + + var (communityServerVersion, communityServerDownloads) = await GetCommunityServerDownloadsAsync(cancellationToken); + globalProgress.Description = $"Downloading MongoDB Community Server {communityServerVersion.Number}"; + + var (databaseToolsVersion, databaseToolsDownloads) = await GetDatabaseToolsDownloadsAsync(cancellationToken); + globalProgress.Description = $"Downloading MongoDB Community Server {communityServerVersion.Number} and Database Tools {databaseToolsVersion.Number}"; + + var tasks = new List>(); + var allArchiveProgresses = new List(); + foreach (var download in communityServerDownloads.Concat(databaseToolsDownloads)) + { + var archiveProgress = context.AddTask($"Downloading {download} from {download.Archive.Url}", maxValue: initialMaxValue); + var directoryName = $"mongodb-{download.Platform.ToString().ToLowerInvariant()}-{download.Architecture.ToString().ToLowerInvariant()}-{communityServerVersion.Number}-database-tools-{databaseToolsVersion.Number}"; + var extractDirectory = new DirectoryInfo(Path.Combine(toolsDirectory.FullName, directoryName)); + allArchiveProgresses.Add(archiveProgress); + var progress = new ArchiveProgress(archiveProgress, globalProgress, allArchiveProgresses, download, $"✅ Downloaded and extracted MongoDB Community Server {communityServerVersion.Number} and Database Tools {databaseToolsVersion.Number} into {new Uri(toolsDirectory.FullName).AbsoluteUri}"); + tasks.Add(ProcessArchiveAsync(download, extractDirectory, progress, cancellationToken)); + } + var strippedSizes = await Task.WhenAll(tasks); + return strippedSizes.Aggregate(new ByteSize(0), (current, strippedSize) => current + strippedSize); + } + + private async Task ProcessArchiveAsync(Download download, DirectoryInfo extractDirectory, ArchiveProgress progress, CancellationToken cancellationToken) + { + IEnumerable> stripTasks; + var archiveExtension = Path.GetExtension(download.Archive.Url.AbsolutePath); + if (archiveExtension == ".zip") + { + stripTasks = await _extractor.DownloadExtractZipArchiveAsync(download, extractDirectory, progress, cancellationToken); + } + else + { + var archiveFileInfo = await DownloadArchiveAsync(download.Archive, progress, cancellationToken); + stripTasks = _extractor.ExtractArchive(download, archiveFileInfo, extractDirectory, cancellationToken); + } + progress.Report("Stripping binaries"); + var completedStripTasks = await Task.WhenAll(stripTasks); + var totalStrippedSize = completedStripTasks.Aggregate(new ByteSize(0), (current, strippedSize) => current + strippedSize); + progress.ReportCompleted(totalStrippedSize); + return totalStrippedSize; + } + + private async Task DownloadArchiveAsync(Archive archive, IProgress progress, CancellationToken cancellationToken) + { + _options.CacheDirectory.Create(); + var destinationFile = new FileInfo(Path.Combine(_options.CacheDirectory.FullName, archive.Url.Segments.Last())); + var useCache = bool.TryParse(Environment.GetEnvironmentVariable("MONGO2GO_DOWNLOADER_USE_CACHED_FILE") ?? "", out var useCachedFile) && useCachedFile; + if (useCache && destinationFile.Exists) + { + progress.Report(new CopyProgress(TimeSpan.Zero, 0, 1, 1)); + return destinationFile; + } + await using var destinationStream = destinationFile.OpenWrite(); + await _options.HttpClient.GetAsync(archive.Url.AbsoluteUri, destinationStream, progress, cancellationToken); + return destinationFile; + } + + private async Task<(Version version, IEnumerable downloads)> GetCommunityServerDownloadsAsync(CancellationToken cancellationToken) + { + var release = await _options.HttpClient.GetFromJsonAsync(_options.CommunityServerUrl, cancellationToken) ?? throw new InvalidOperationException($"Failed to deserialize {nameof(Release)}"); + var version = release.Versions.FirstOrDefault(e => e.Production) ?? throw new InvalidOperationException("No Community Server production version was found"); + var downloads = Enum.GetValues().SelectMany(platform => GetDownloads(platform, Product.CommunityServer, version, _options, _options.Edition)); + return (version, downloads); + } + + private async Task<(Version version, IEnumerable downloads)> GetDatabaseToolsDownloadsAsync(CancellationToken cancellationToken) + { + var release = await _options.HttpClient.GetFromJsonAsync(_options.DatabaseToolsUrl, cancellationToken) ?? throw new InvalidOperationException($"Failed to deserialize {nameof(Release)}"); + var version = release.Versions.FirstOrDefault() ?? throw new InvalidOperationException("No Database Tools version was found"); + var downloads = Enum.GetValues().SelectMany(platform => GetDownloads(platform, Product.DatabaseTools, version, _options)); + return (version, downloads); + } + + private static IEnumerable GetDownloads(Platform platform, Product product, Version version, Options options, Regex? editionRegex = null) + { + var platformRegex = options.PlatformIdentifiers[platform]; + Func platformPredicate = product switch + { + Product.CommunityServer => download => platformRegex.IsMatch(download.Target), + Product.DatabaseTools => download => platformRegex.IsMatch(download.Name), + _ => throw new ArgumentOutOfRangeException(nameof(product), product, $"The value of argument '{nameof(product)}' ({product}) is invalid for enum type '{nameof(Product)}'.") + }; + + foreach (var architecture in options.Architectures[platform]) + { + var architectureRegex = options.ArchitectureIdentifiers[architecture]; + var matchingDownloads = version.Downloads + .Where(platformPredicate) + .Where(e => architectureRegex.IsMatch(e.Arch)) + .Where(e => editionRegex?.IsMatch(e.Edition) ?? true) + .ToList(); + + if (matchingDownloads.Count == 0) + { + var downloads = version.Downloads.OrderBy(e => e.Target).ThenBy(e => e.Arch); + var messages = Enumerable.Empty() + .Append($"Download not found for {platform}/{architecture}.") + .Append($" Available downloads for {product} {version.Number}:") + .Concat(downloads.Select(e => $" - {e.Target}/{e.Arch} ({e.Edition})")); + throw new InvalidOperationException(string.Join(Environment.NewLine, messages)); + } + + if (matchingDownloads.Count > 1) + { + throw new InvalidOperationException($"Found {matchingDownloads.Count} downloads for {platform}/{architecture} but expected to find only one."); + } + + var download = matchingDownloads[0]; + download.Platform = platform; + download.Architecture = architecture; + download.Product = product; + + yield return download; + } + } + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/MongoDownloader.csproj b/Mongo2Go-4.1.0/src/MongoDownloader/MongoDownloader.csproj new file mode 100644 index 00000000..cb8a14ed --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/MongoDownloader.csproj @@ -0,0 +1,21 @@ + + + + Exe + net8.0 + enable + false + + + + + + + + + + + + + + diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/Options.cs b/Mongo2Go-4.1.0/src/MongoDownloader/Options.cs new file mode 100644 index 00000000..ab1d42b1 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/Options.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Runtime.InteropServices; +using System.Text.RegularExpressions; + +namespace MongoDownloader +{ + internal class Options + { + /// + /// The instance used to fetch data over HTTP. + /// + public HttpClient HttpClient { get; init; } = new(); + + /// + /// The URL of the MongoDB Community Server download information JSON. + /// + public string CommunityServerUrl { get; init; } = "https://s3.amazonaws.com/downloads.mongodb.org/current.json"; + + /// + /// The URL of the MongoDB Database Tools download information JSON. + /// + public string DatabaseToolsUrl { get; init; } = "https://s3.amazonaws.com/downloads.mongodb.org/tools/db/release.json"; + + /// + /// The directory to store the downloaded archive files. + /// + public DirectoryInfo CacheDirectory { get; init; } = new(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.InternetCache), nameof(MongoDownloader))); + + /// + /// The architectures to download for a given platform. + /// + public IReadOnlyDictionary> Architectures { get; init; } = new Dictionary> + { + [Platform.Linux] = new[] { Architecture.Arm64, Architecture.X64 }, + [Platform.macOS] = new[] { Architecture.X64 }, + [Platform.Windows] = new[] { Architecture.X64 }, + }; + + /// + /// The edition of the archive to download. + /// + /// macOS and Windows use base and Linux uses targeted for the community edition + public Regex Edition { get; init; } = new(@"base|targeted"); + + /// + /// The regular expressions used to identify platform-specific archives to download. + /// + public IReadOnlyDictionary PlatformIdentifiers { get; init; } = new Dictionary + { + [Platform.Linux] = new(@"ubuntu2004", RegexOptions.IgnoreCase), + [Platform.macOS] = new(@"macOS", RegexOptions.IgnoreCase), + [Platform.Windows] = new(@"windows", RegexOptions.IgnoreCase), + }; + + /// + /// The regular expressions used to identify architectures to download. + /// + public IReadOnlyDictionary ArchitectureIdentifiers { get; init; } = new Dictionary + { + [Architecture.Arm64] = new("arm64|aarch64", RegexOptions.IgnoreCase), + [Architecture.X64] = new("x86_64", RegexOptions.IgnoreCase), + }; + + /// + /// A dictionary describing how to match MongoDB binaries inside the zip archives. + /// + /// The key is a tuple with the / and the + /// value is a regular expressions to match against the zip file name entry. + /// + public IReadOnlyDictionary<(Product, Platform), Regex> Binaries { get; init; } = new Dictionary<(Product, Platform), Regex> + { + [(Product.CommunityServer, Platform.Linux)] = new(@"bin/mongod"), + [(Product.CommunityServer, Platform.macOS)] = new(@"bin/mongod"), + [(Product.CommunityServer, Platform.Windows)] = new(@"bin/mongod\.exe"), + [(Product.DatabaseTools, Platform.Linux)] = new(@"bin/(mongoexport|mongoimport)"), + [(Product.DatabaseTools, Platform.macOS)] = new(@"bin/(mongoexport|mongoimport)"), + [(Product.DatabaseTools, Platform.Windows)] = new(@"bin/(mongoexport|mongoimport)\.exe"), + }; + + /// + /// A dictionary describing how to match licence files inside the zip archives. + /// + /// The key is a tuple with the / and the + /// value is a regular expressions to match against the zip file name entry. + /// + public IReadOnlyDictionary<(Product, Platform), Regex> Licenses { get; init; } = new Dictionary<(Product, Platform), Regex> + { + // The regular expression matches anything at the zip top level, i.e. does not contain any slash (/) character + [(Product.CommunityServer, Platform.Linux)] = new(@"^[^/]+$"), + [(Product.CommunityServer, Platform.macOS)] = new(@"^[^/]+$"), + [(Product.CommunityServer, Platform.Windows)] = new(@"^[^/]+$"), + [(Product.DatabaseTools, Platform.Linux)] = new(@"^[^/]+$"), + [(Product.DatabaseTools, Platform.macOS)] = new(@"^[^/]+$"), + [(Product.DatabaseTools, Platform.Windows)] = new(@"^[^/]+$"), + }; + } +} \ No newline at end of file diff --git a/Mongo2Go-4.1.0/src/MongoDownloader/Program.cs b/Mongo2Go-4.1.0/src/MongoDownloader/Program.cs new file mode 100644 index 00000000..6fcb3a23 --- /dev/null +++ b/Mongo2Go-4.1.0/src/MongoDownloader/Program.cs @@ -0,0 +1,86 @@ +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Spectre.Console; + +namespace MongoDownloader +{ + internal static class Program + { + private static async Task Main(string[] args) + { + try + { + var toolsDirectory = GetToolsDirectory(); + + foreach (DirectoryInfo dir in toolsDirectory.EnumerateDirectories()) + { + dir.Delete(true); + } + + var cancellationTokenSource = new CancellationTokenSource(); + Console.CancelKeyPress += (_, eventArgs) => + { + // Try to cancel gracefully the first time, then abort the process the second time Ctrl+C is pressed + eventArgs.Cancel = !cancellationTokenSource.IsCancellationRequested; + cancellationTokenSource.Cancel(); + }; + var options = new Options(); + var performStrip = args.All(e => e != "--no-strip"); + var binaryStripper = performStrip ? await GetBinaryStripperAsync(cancellationTokenSource.Token) : null; + var archiveExtractor = new ArchiveExtractor(options, binaryStripper); + var downloader = new MongoDbDownloader(archiveExtractor, options); + var strippedSize = await downloader.RunAsync(toolsDirectory, cancellationTokenSource.Token); + if (performStrip) + { + AnsiConsole.WriteLine($"Saved {strippedSize:#.#} by stripping executables"); + } + return 0; + } + catch (Exception exception) + { + if (exception is not OperationCanceledException) + { + AnsiConsole.WriteException(exception, ExceptionFormats.ShortenPaths); + } + return 1; + } + } + + private static DirectoryInfo GetToolsDirectory() + { + for (var directory = new DirectoryInfo("."); directory != null; directory = directory.Parent) + { + var toolsDirectory = directory.GetDirectories("tools", SearchOption.TopDirectoryOnly).SingleOrDefault(); + if (toolsDirectory?.Exists ?? false) + { + return toolsDirectory; + } + } + throw new InvalidOperationException("The tools directory was not found"); + } + + private static async Task GetBinaryStripperAsync(CancellationToken cancellationToken) + { + try + { + return await BinaryStripper.CreateAsync(cancellationToken); + } + catch (FileNotFoundException exception) + { + string installCommand; + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + installCommand = "brew install llvm"; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + installCommand = "scoop install llvm"; + else + installCommand = "apt-get install llvm"; + + throw new Exception($"{exception.Message} Either install llvm with `{installCommand}` or run MongoDownloader with the --no-strip option to skip binary stripping.", exception); + } + } + } +} diff --git a/Mongo2Go-4.1.0/src/mongo2go_200_200.png b/Mongo2Go-4.1.0/src/mongo2go_200_200.png new file mode 100644 index 00000000..23750b60 Binary files /dev/null and b/Mongo2Go-4.1.0/src/mongo2go_200_200.png differ diff --git a/Mongo2Go-4.1.0/src/mongo2go_big.png b/Mongo2Go-4.1.0/src/mongo2go_big.png new file mode 100644 index 00000000..368d6452 Binary files /dev/null and b/Mongo2Go-4.1.0/src/mongo2go_big.png differ diff --git a/Mongo2Go-4.1.0/tools/README.md b/Mongo2Go-4.1.0/tools/README.md new file mode 100644 index 00000000..a90616a8 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/README.md @@ -0,0 +1,11 @@ +The binaries in this directory are automatically downloaded with the `MongoDownloader` tool. + +In order to download the latest binary: + +1. Go into the `Mongo2Go/src/MongoDownloader` directory +2. Run the downloader with `dotnet run` + +* The _MongoDB Community Server_ binaries are fetched from [https://s3.amazonaws.com/downloads.mongodb.org/current.json](https://s3.amazonaws.com/downloads.mongodb.org/current.json) + The latest production version is downloaded and extracted. +* The _MongoDB Database Tools_ archives are fetched from [https://s3.amazonaws.com/downloads.mongodb.org/tools/db/release.json](https://s3.amazonaws.com/downloads.mongodb.org/tools/db/release.json) + The latest version is downloaded and extracted. \ No newline at end of file diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt new file mode 100644 index 00000000..b01add13 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt @@ -0,0 +1,557 @@ + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 new file mode 100644 index 00000000..197b2ffd --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README new file mode 100644 index 00000000..97f1dc72 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README @@ -0,0 +1,87 @@ +MongoDB README + +Welcome to MongoDB! + +COMPONENTS + + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + install_compass - Installs MongoDB Compass for your platform. + +BUILDING + + See docs/building.md. + +RUNNING + + For command line options invoke: + + $ ./mongod --help + + To run a single server database: + + $ sudo mkdir -p /data/db + $ ./mongod + $ + $ # The mongo javascript shell connects to localhost and test database by default: + $ ./mongo + > help + +INSTALLING COMPASS + + You can install compass using the install_compass script packaged with MongoDB: + + $ ./install_compass + + This will download the appropriate MongoDB Compass package for your platform + and install it. + +DRIVERS + + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. + +DOCUMENTATION + + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +FORUMS + + https://community.mongodb.com + + A forum for technical questions about using MongoDB. + + https://community.mongodb.com/c/server-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ + +LICENSE + + MongoDB is free and open-source. Versions released prior to October 16, + 2018 are published under the AGPL. All versions released after October + 16, 2018, including patch fixes for prior versions, are published under + the Server Side Public License (SSPL) v1. See individual files for + details. + diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..8c9e17e3 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES @@ -0,0 +1,1568 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through any of the ways detailed here : + + mongodb-dev@googlegroups.com + +The attached notices are provided for information only. + +For any licenses that require disclosure of source, sources are available at +https://github.com/mongodb/mongo. + + +1) License Notice for Boost +--------------------------- + +http://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +3) License Notice for PCRE +-------------------------- + +http://www.pcre.org/licence.txt + +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 7 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2008 University of Cambridge +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2008, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +4) License notice for Aladdin MD5 +--------------------------------- + +Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +L. Peter Deutsch +ghost@aladdin.com + +5) License notice for Snappy - http://code.google.com/p/snappy/ +--------------------------------- + Copyright 2005 and onwards Google Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + A light-weight compression algorithm. It is designed for speed of + compression and decompression, rather than for the utmost in space + savings. + + For getting better compression ratios when you are compressing data + with long repeated sequences or compressing data that is similar to + other data, while still compressing fast, you might look at first + using BMDiff and then compressing the output of BMDiff with + Snappy. + +6) License notice for Google Perftools (TCMalloc utility) +--------------------------------- +New BSD License + +Copyright (c) 1998-2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +7) License notice for Linenoise +------------------------------- + + Copyright (c) 2010, Salvatore Sanfilippo + Copyright (c) 2010, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +8) License notice for S2 Geometry Library +----------------------------------------- + Copyright 2005 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +9) License notice for MurmurHash +-------------------------------- + + Copyright (c) 2010-2012 Austin Appleby + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +10) License notice for Snowball + Copyright (c) 2001, Dr Martin Porter + All rights reserved. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +11) License notice for yaml-cpp +------------------------------- + +Copyright (c) 2008 Jesse Beder. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +12) License notice for zlib +--------------------------- + +http://www.zlib.net/zlib_license.html + +zlib.h -- interface of the 'zlib' general purpose compression library +version 1.2.8, April 28th, 2013 + +Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu + + +13) License notice for 3rd party software included in the WiredTiger library +---------------------------------------------------------------------------- + +http://source.wiredtiger.com/license.html + +WiredTiger Distribution Files | Copyright Holder | License +----------------------------- | ----------------------------------- | ---------------------- +src/include/bitstring.i | University of California, Berkeley | BSD-3-Clause License +src/include/queue.h | University of California, Berkeley | BSD-3-Clause License +src/os_posix/os_getopt.c | University of California, Berkeley | BSD-3-Clause License +src/support/hash_city.c | Google, Inc. | The MIT License +src/support/hash_fnv.c | Authors | Public Domain + + +Other optional 3rd party software included in the WiredTiger distribution is removed by MongoDB. + + +BSD-3-CLAUSE LICENSE +-------------------- + +http://www.opensource.org/licenses/BSD-3-Clause + +Copyright (c) 1987, 1989, 1991, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + +THE MIT LICENSE +--------------- + +http://www.opensource.org/licenses/MIT + +Copyright (c) 2011 Google, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +14) License Notice for SpiderMonkey +----------------------------------- + +|------------------------------------------------|------------------|---------------| +| SpiderMonkey Distribution Files | Copyright Holder | License | +|------------------------------------------------|------------------|---------------| +| js/src/jit/shared/AssemblerBuffer-x86-shared.h | Apple, Inc | BSD-2-Clause | +| js/src/jit/shared/BaseAssembler-x86-shared.h | | | +|------------------------------------------------|------------------|---------------| +| js/src/builtin/ | Google, Inc | BSD-3-Clause | +| js/src/irregexp/ | | | +| js/src/jit/arm/ | | | +| js/src/jit/mips/ | | | +| mfbt/double-conversion/ | | | +|------------------------------------------------|------------------|---------------| +| intl/icu/source/common/unicode/ | IBM, Inc | ICU | +|------------------------------------------------|------------------|---------------| +| js/src/asmjs/ | Mozilla, Inc | Apache2 | +|------------------------------------------------|------------------|---------------| +| js/public/ | Mozilla, Inc | MPL2 | +| js/src/ | | | +| mfbt | | | +|------------------------------------------------|------------------|---------------| +| js/src/vm/Unicode.cpp | None | Public Domain | +|------------------------------------------------|------------------|---------------| +| mfbt/lz4.c | Yann Collet | BSD-2-Clause | +| mfbt/lz4.h | | | +|------------------------------------------------|------------------|---------------| + +Other optional 3rd party software included in the SpiderMonkey distribution is removed by MongoDB. + + +Apple, Inc: BSD-2-Clause +------------------------ + +Copyright (C) 2008 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Google, Inc: BSD-3-Clause +------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ICU License - ICU 1.8.1 and later +--------------------------------- + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2012 International Business Machines Corporation and +others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, provided that the above copyright notice(s) and this +permission notice appear in all copies of the Software and that both the +above copyright notice(s) and this permission notice appear in supporting +documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in this Software without prior written authorization of the +copyright holder. + +All trademarks and registered trademarks mentioned herein are the property +of their respective owners. + + +Mozilla, Inc: Apache 2 +---------------------- + +Copyright 2014 Mozilla Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Mozilla, Inc: MPL 2 +------------------- + +Copyright 2014 Mozilla Foundation + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +Public Domain +------------- + +Any copyright is dedicated to the Public Domain. +http://creativecommons.org/licenses/publicdomain/ + + +LZ4: BSD-2-Clause +----------------- + +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + +15) License Notice for Intel DFP Math Library +--------------------------------------------- + +Copyright (c) 2011, Intel Corp. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + his list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +16) License Notice for Unicode Data +----------------------------------- + +Copyright © 1991-2015 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +17 ) License Notice for Valgrind.h +---------------------------------- + +---------------------------------------------------------------- + +Notice that the following BSD-style license applies to this one +file (valgrind.h) only. The rest of Valgrind is licensed under the +terms of the GNU General Public License, version 2, unless +otherwise indicated. See the COPYING file in the source +distribution for details. + +---------------------------------------------------------------- + +This file is part of Valgrind, a dynamic binary instrumentation +framework. + +Copyright (C) 2000-2015 Julian Seward. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------- + +Notice that the above BSD-style license applies to this one file +(valgrind.h) only. The entire rest of Valgrind is licensed under +the terms of the GNU General Public License, version 2. See the +COPYING file in the source distribution for details. + +---------------------------------------------------------------- + +18) License notice for ICU4C +---------------------------- + +ICU License - ICU 1.8.1 and later + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. Unicode Data Files and Software + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyrighy (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +19) License notice for timelib +------------------------------ + +The MIT License (MIT) + +Copyright (c) 2015-2017 Derick Rethans +Copyright (c) 2017 MongoDB, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +20) License notice for windows dirent implementation +---------------------------------------------------- + + * Dirent interface for Microsoft Visual Studio + * Version 1.21 + * + * Copyright (C) 2006-2012 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + + + 21) License notice for abseil-cpp +---------------------------- + + Copyright (c) Google Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + 22) License notice for Zstandard +---------------------------- + + BSD License + + For Zstandard software + + Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 23) License notice for ASIO +---------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 24) License notice for MPark.Variant +------------------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 25) License notice for fmt +--------------------------- + +Copyright (c) 2012 - present, Victor Zverovich +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or other + materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 26) License notice for SafeInt +--------------------------- + +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the MIT License. + +MIT License + +Copyright (c) 2018 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + 27) License Notice for Raft TLA+ Specification +----------------------------------------------- + +https://github.com/ongardie/dissertation/blob/master/LICENSE + +Copyright 2014 Diego Ongaro. + +Some of our TLA+ specifications are based on the Raft TLA+ specification by Diego Ongaro. + +End diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md new file mode 100644 index 00000000..550ae4ed --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2014 MongoDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md new file mode 100644 index 00000000..e60e0cd0 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md @@ -0,0 +1,72 @@ +MongoDB Tools +=================================== + + - **bsondump** - _display BSON files in a human-readable format_ + - **mongoimport** - _Convert data from JSON, TSV or CSV and insert them into a collection_ + - **mongoexport** - _Write an existing collection to CSV or JSON format_ + - **mongodump/mongorestore** - _Dump MongoDB backups to disk in .BSON format, or restore them to a live database_ + - **mongostat** - _Monitor live MongoDB servers, replica sets, or sharded clusters_ + - **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_ + - **mongotop** - _Monitor read/write activity on a mongo server_ + + +Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS + +Building Tools +--------------- + +We currently build the tools with Go version 1.15. Other Go versions may work but they are untested. + +Using `go get` to directly build the tools will not work. To build them, it's recommended to first clone this repository: + +``` +git clone https://github.com/mongodb/mongo-tools +cd mongo-tools +``` + +Then run `./make build` to build all the tools, placing them in the `bin` directory inside the repository. + +You can also build a subset of the tools using the `-tools` option. For example, `./make build -tools=mongodump,mongorestore` builds only `mongodump` and `mongorestore`. + +To use the build/test scripts in this repository, you **_must_** set GOROOT to your Go root directory. This may depend on how you installed Go. + +``` +export GOROOT=/usr/local/go +``` + +Updating Dependencies +--------------- +Starting with version 100.3.1, the tools use `go mod` to manage dependencies. All dependencies are listed in the `go.mod` file and are directly vendored in the `vendor` directory. + +In order to make changes to dependencies, you first need to change the `go.mod` file. You can manually edit that file to add/update/remove entries, or you can run the following in the repository directory: + +``` +go mod edit -require=@ # for adding or updating a dependency +go mod edit -droprequire= # for removing a dependency +``` + +Then run `go mod vendor -v` to reconstruct the `vendor` directory to match the changed `go.mod` file. + +Optionally, run `go mod tidy -v` to ensure that the `go.mod` file matches the `mongo-tools` source code. + +Contributing +--------------- +See our [Contributor's Guide](CONTRIBUTING.md). + +Documentation +--------------- +See the MongoDB packages [documentation](https://docs.mongodb.org/database-tools/). + +For documentation on older versions of the MongoDB, reference that version of the [MongoDB Server Manual](docs.mongodb.com/manual): + +- [MongoDB 4.2 Tools](https://docs.mongodb.org/v4.2/reference/program) +- [MongoDB 4.0 Tools](https://docs.mongodb.org/v4.0/reference/program) +- [MongoDB 3.6 Tools](https://docs.mongodb.org/v3.6/reference/program) + +Adding New Platforms Support +--------------- +See our [Adding New Platform Support Guide](PLATFORMSUPPORT.md). + +Vendoring the Change into Server Repo +--------------- +See our [Vendor the Change into Server Repo](SERVERVENDORING.md). diff --git a/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..3d75e64b --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES @@ -0,0 +1,3319 @@ +--------------------------------------------------------------------- +License notice for hashicorp/go-rootcerts +--------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +--------------------------------------------------------------------- +License notice for JSON and CSV code from github.com/golang/go +--------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/escaper +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Lucas Morales + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo/bson +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/openssl +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/3rf/mongo-lint +---------------------------------------------------------------------- + +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/go-stack/stack +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/golang/snappy +---------------------------------------------------------------------- + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/google/gopacket +---------------------------------------------------------------------- + +Copyright (c) 2012 Google, Inc. All rights reserved. +Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Andreas Krennmair, Google, nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/gopherjs/gopherjs +---------------------------------------------------------------------- + +Copyright (c) 2013 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/howeyc/gopass +---------------------------------------------------------------------- + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/jessevdk/go-flags +---------------------------------------------------------------------- + +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/jtolds/gls +---------------------------------------------------------------------- + +Copyright (c) 2013, Space Monkey, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mattn/go-runewidth +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mongodb/mongo-go-driver +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/nsf/termbox-go +---------------------------------------------------------------------- + +Copyright (C) 2012 termbox-go authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/patrickmn/go-cache +---------------------------------------------------------------------- + +Copyright (c) 2012-2015 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions +---------------------------------------------------------------------- + +Copyright (c) 2015 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/go-render +---------------------------------------------------------------------- + +// Copyright (c) 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglematchers +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglemock +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/ogletest +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/reqtrace +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey +---------------------------------------------------------------------- + +Copyright (c) 2014 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey/web/client/resources/fonts/Open_Sans +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/spacemonkeygo/spacelog +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/xdg/scram +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/xdg/stringprep +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/youmark/pkcs8 +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for golang.org/x/crypto +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/sync +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/text +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for gopkg.in/tomb.v2 +---------------------------------------------------------------------- + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt new file mode 100644 index 00000000..b01add13 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt @@ -0,0 +1,557 @@ + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 new file mode 100644 index 00000000..197b2ffd --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README new file mode 100644 index 00000000..97f1dc72 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README @@ -0,0 +1,87 @@ +MongoDB README + +Welcome to MongoDB! + +COMPONENTS + + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + install_compass - Installs MongoDB Compass for your platform. + +BUILDING + + See docs/building.md. + +RUNNING + + For command line options invoke: + + $ ./mongod --help + + To run a single server database: + + $ sudo mkdir -p /data/db + $ ./mongod + $ + $ # The mongo javascript shell connects to localhost and test database by default: + $ ./mongo + > help + +INSTALLING COMPASS + + You can install compass using the install_compass script packaged with MongoDB: + + $ ./install_compass + + This will download the appropriate MongoDB Compass package for your platform + and install it. + +DRIVERS + + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. + +DOCUMENTATION + + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +FORUMS + + https://community.mongodb.com + + A forum for technical questions about using MongoDB. + + https://community.mongodb.com/c/server-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ + +LICENSE + + MongoDB is free and open-source. Versions released prior to October 16, + 2018 are published under the AGPL. All versions released after October + 16, 2018, including patch fixes for prior versions, are published under + the Server Side Public License (SSPL) v1. See individual files for + details. + diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..8c9e17e3 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES @@ -0,0 +1,1568 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through any of the ways detailed here : + + mongodb-dev@googlegroups.com + +The attached notices are provided for information only. + +For any licenses that require disclosure of source, sources are available at +https://github.com/mongodb/mongo. + + +1) License Notice for Boost +--------------------------- + +http://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +3) License Notice for PCRE +-------------------------- + +http://www.pcre.org/licence.txt + +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 7 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2008 University of Cambridge +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2008, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +4) License notice for Aladdin MD5 +--------------------------------- + +Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +L. Peter Deutsch +ghost@aladdin.com + +5) License notice for Snappy - http://code.google.com/p/snappy/ +--------------------------------- + Copyright 2005 and onwards Google Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + A light-weight compression algorithm. It is designed for speed of + compression and decompression, rather than for the utmost in space + savings. + + For getting better compression ratios when you are compressing data + with long repeated sequences or compressing data that is similar to + other data, while still compressing fast, you might look at first + using BMDiff and then compressing the output of BMDiff with + Snappy. + +6) License notice for Google Perftools (TCMalloc utility) +--------------------------------- +New BSD License + +Copyright (c) 1998-2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +7) License notice for Linenoise +------------------------------- + + Copyright (c) 2010, Salvatore Sanfilippo + Copyright (c) 2010, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +8) License notice for S2 Geometry Library +----------------------------------------- + Copyright 2005 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +9) License notice for MurmurHash +-------------------------------- + + Copyright (c) 2010-2012 Austin Appleby + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +10) License notice for Snowball + Copyright (c) 2001, Dr Martin Porter + All rights reserved. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +11) License notice for yaml-cpp +------------------------------- + +Copyright (c) 2008 Jesse Beder. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +12) License notice for zlib +--------------------------- + +http://www.zlib.net/zlib_license.html + +zlib.h -- interface of the 'zlib' general purpose compression library +version 1.2.8, April 28th, 2013 + +Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu + + +13) License notice for 3rd party software included in the WiredTiger library +---------------------------------------------------------------------------- + +http://source.wiredtiger.com/license.html + +WiredTiger Distribution Files | Copyright Holder | License +----------------------------- | ----------------------------------- | ---------------------- +src/include/bitstring.i | University of California, Berkeley | BSD-3-Clause License +src/include/queue.h | University of California, Berkeley | BSD-3-Clause License +src/os_posix/os_getopt.c | University of California, Berkeley | BSD-3-Clause License +src/support/hash_city.c | Google, Inc. | The MIT License +src/support/hash_fnv.c | Authors | Public Domain + + +Other optional 3rd party software included in the WiredTiger distribution is removed by MongoDB. + + +BSD-3-CLAUSE LICENSE +-------------------- + +http://www.opensource.org/licenses/BSD-3-Clause + +Copyright (c) 1987, 1989, 1991, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + +THE MIT LICENSE +--------------- + +http://www.opensource.org/licenses/MIT + +Copyright (c) 2011 Google, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +14) License Notice for SpiderMonkey +----------------------------------- + +|------------------------------------------------|------------------|---------------| +| SpiderMonkey Distribution Files | Copyright Holder | License | +|------------------------------------------------|------------------|---------------| +| js/src/jit/shared/AssemblerBuffer-x86-shared.h | Apple, Inc | BSD-2-Clause | +| js/src/jit/shared/BaseAssembler-x86-shared.h | | | +|------------------------------------------------|------------------|---------------| +| js/src/builtin/ | Google, Inc | BSD-3-Clause | +| js/src/irregexp/ | | | +| js/src/jit/arm/ | | | +| js/src/jit/mips/ | | | +| mfbt/double-conversion/ | | | +|------------------------------------------------|------------------|---------------| +| intl/icu/source/common/unicode/ | IBM, Inc | ICU | +|------------------------------------------------|------------------|---------------| +| js/src/asmjs/ | Mozilla, Inc | Apache2 | +|------------------------------------------------|------------------|---------------| +| js/public/ | Mozilla, Inc | MPL2 | +| js/src/ | | | +| mfbt | | | +|------------------------------------------------|------------------|---------------| +| js/src/vm/Unicode.cpp | None | Public Domain | +|------------------------------------------------|------------------|---------------| +| mfbt/lz4.c | Yann Collet | BSD-2-Clause | +| mfbt/lz4.h | | | +|------------------------------------------------|------------------|---------------| + +Other optional 3rd party software included in the SpiderMonkey distribution is removed by MongoDB. + + +Apple, Inc: BSD-2-Clause +------------------------ + +Copyright (C) 2008 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Google, Inc: BSD-3-Clause +------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ICU License - ICU 1.8.1 and later +--------------------------------- + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2012 International Business Machines Corporation and +others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, provided that the above copyright notice(s) and this +permission notice appear in all copies of the Software and that both the +above copyright notice(s) and this permission notice appear in supporting +documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in this Software without prior written authorization of the +copyright holder. + +All trademarks and registered trademarks mentioned herein are the property +of their respective owners. + + +Mozilla, Inc: Apache 2 +---------------------- + +Copyright 2014 Mozilla Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Mozilla, Inc: MPL 2 +------------------- + +Copyright 2014 Mozilla Foundation + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +Public Domain +------------- + +Any copyright is dedicated to the Public Domain. +http://creativecommons.org/licenses/publicdomain/ + + +LZ4: BSD-2-Clause +----------------- + +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + +15) License Notice for Intel DFP Math Library +--------------------------------------------- + +Copyright (c) 2011, Intel Corp. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + his list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +16) License Notice for Unicode Data +----------------------------------- + +Copyright © 1991-2015 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +17 ) License Notice for Valgrind.h +---------------------------------- + +---------------------------------------------------------------- + +Notice that the following BSD-style license applies to this one +file (valgrind.h) only. The rest of Valgrind is licensed under the +terms of the GNU General Public License, version 2, unless +otherwise indicated. See the COPYING file in the source +distribution for details. + +---------------------------------------------------------------- + +This file is part of Valgrind, a dynamic binary instrumentation +framework. + +Copyright (C) 2000-2015 Julian Seward. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------- + +Notice that the above BSD-style license applies to this one file +(valgrind.h) only. The entire rest of Valgrind is licensed under +the terms of the GNU General Public License, version 2. See the +COPYING file in the source distribution for details. + +---------------------------------------------------------------- + +18) License notice for ICU4C +---------------------------- + +ICU License - ICU 1.8.1 and later + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. Unicode Data Files and Software + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyrighy (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +19) License notice for timelib +------------------------------ + +The MIT License (MIT) + +Copyright (c) 2015-2017 Derick Rethans +Copyright (c) 2017 MongoDB, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +20) License notice for windows dirent implementation +---------------------------------------------------- + + * Dirent interface for Microsoft Visual Studio + * Version 1.21 + * + * Copyright (C) 2006-2012 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + + + 21) License notice for abseil-cpp +---------------------------- + + Copyright (c) Google Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + 22) License notice for Zstandard +---------------------------- + + BSD License + + For Zstandard software + + Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 23) License notice for ASIO +---------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 24) License notice for MPark.Variant +------------------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 25) License notice for fmt +--------------------------- + +Copyright (c) 2012 - present, Victor Zverovich +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or other + materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 26) License notice for SafeInt +--------------------------- + +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the MIT License. + +MIT License + +Copyright (c) 2018 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + 27) License Notice for Raft TLA+ Specification +----------------------------------------------- + +https://github.com/ongardie/dissertation/blob/master/LICENSE + +Copyright 2014 Diego Ongaro. + +Some of our TLA+ specifications are based on the Raft TLA+ specification by Diego Ongaro. + +End diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md new file mode 100644 index 00000000..550ae4ed --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2014 MongoDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md new file mode 100644 index 00000000..e60e0cd0 --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md @@ -0,0 +1,72 @@ +MongoDB Tools +=================================== + + - **bsondump** - _display BSON files in a human-readable format_ + - **mongoimport** - _Convert data from JSON, TSV or CSV and insert them into a collection_ + - **mongoexport** - _Write an existing collection to CSV or JSON format_ + - **mongodump/mongorestore** - _Dump MongoDB backups to disk in .BSON format, or restore them to a live database_ + - **mongostat** - _Monitor live MongoDB servers, replica sets, or sharded clusters_ + - **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_ + - **mongotop** - _Monitor read/write activity on a mongo server_ + + +Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS + +Building Tools +--------------- + +We currently build the tools with Go version 1.15. Other Go versions may work but they are untested. + +Using `go get` to directly build the tools will not work. To build them, it's recommended to first clone this repository: + +``` +git clone https://github.com/mongodb/mongo-tools +cd mongo-tools +``` + +Then run `./make build` to build all the tools, placing them in the `bin` directory inside the repository. + +You can also build a subset of the tools using the `-tools` option. For example, `./make build -tools=mongodump,mongorestore` builds only `mongodump` and `mongorestore`. + +To use the build/test scripts in this repository, you **_must_** set GOROOT to your Go root directory. This may depend on how you installed Go. + +``` +export GOROOT=/usr/local/go +``` + +Updating Dependencies +--------------- +Starting with version 100.3.1, the tools use `go mod` to manage dependencies. All dependencies are listed in the `go.mod` file and are directly vendored in the `vendor` directory. + +In order to make changes to dependencies, you first need to change the `go.mod` file. You can manually edit that file to add/update/remove entries, or you can run the following in the repository directory: + +``` +go mod edit -require=@ # for adding or updating a dependency +go mod edit -droprequire= # for removing a dependency +``` + +Then run `go mod vendor -v` to reconstruct the `vendor` directory to match the changed `go.mod` file. + +Optionally, run `go mod tidy -v` to ensure that the `go.mod` file matches the `mongo-tools` source code. + +Contributing +--------------- +See our [Contributor's Guide](CONTRIBUTING.md). + +Documentation +--------------- +See the MongoDB packages [documentation](https://docs.mongodb.org/database-tools/). + +For documentation on older versions of the MongoDB, reference that version of the [MongoDB Server Manual](docs.mongodb.com/manual): + +- [MongoDB 4.2 Tools](https://docs.mongodb.org/v4.2/reference/program) +- [MongoDB 4.0 Tools](https://docs.mongodb.org/v4.0/reference/program) +- [MongoDB 3.6 Tools](https://docs.mongodb.org/v3.6/reference/program) + +Adding New Platforms Support +--------------- +See our [Adding New Platform Support Guide](PLATFORMSUPPORT.md). + +Vendoring the Change into Server Repo +--------------- +See our [Vendor the Change into Server Repo](SERVERVENDORING.md). diff --git a/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..3d75e64b --- /dev/null +++ b/Mongo2Go-4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES @@ -0,0 +1,3319 @@ +--------------------------------------------------------------------- +License notice for hashicorp/go-rootcerts +--------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +--------------------------------------------------------------------- +License notice for JSON and CSV code from github.com/golang/go +--------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/escaper +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Lucas Morales + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo/bson +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/openssl +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/3rf/mongo-lint +---------------------------------------------------------------------- + +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/go-stack/stack +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/golang/snappy +---------------------------------------------------------------------- + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/google/gopacket +---------------------------------------------------------------------- + +Copyright (c) 2012 Google, Inc. All rights reserved. +Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Andreas Krennmair, Google, nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/gopherjs/gopherjs +---------------------------------------------------------------------- + +Copyright (c) 2013 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/howeyc/gopass +---------------------------------------------------------------------- + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/jessevdk/go-flags +---------------------------------------------------------------------- + +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/jtolds/gls +---------------------------------------------------------------------- + +Copyright (c) 2013, Space Monkey, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mattn/go-runewidth +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mongodb/mongo-go-driver +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/nsf/termbox-go +---------------------------------------------------------------------- + +Copyright (C) 2012 termbox-go authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/patrickmn/go-cache +---------------------------------------------------------------------- + +Copyright (c) 2012-2015 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions +---------------------------------------------------------------------- + +Copyright (c) 2015 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/go-render +---------------------------------------------------------------------- + +// Copyright (c) 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglematchers +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglemock +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/ogletest +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/reqtrace +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey +---------------------------------------------------------------------- + +Copyright (c) 2014 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey/web/client/resources/fonts/Open_Sans +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/spacemonkeygo/spacelog +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/xdg/scram +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/xdg/stringprep +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/youmark/pkcs8 +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for golang.org/x/crypto +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/sync +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/text +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for gopkg.in/tomb.v2 +---------------------------------------------------------------------- + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Mongo2Go.4.1.0.nupkg b/Mongo2Go.4.1.0.nupkg new file mode 100644 index 00000000..a9378128 Binary files /dev/null and b/Mongo2Go.4.1.0.nupkg differ diff --git a/Mongo2Go.4.1.0/.signature.p7s b/Mongo2Go.4.1.0/.signature.p7s new file mode 100644 index 00000000..39597809 Binary files /dev/null and b/Mongo2Go.4.1.0/.signature.p7s differ diff --git a/Mongo2Go.4.1.0/Mongo2Go.nuspec b/Mongo2Go.4.1.0/Mongo2Go.nuspec new file mode 100644 index 00000000..f6e628d4 --- /dev/null +++ b/Mongo2Go.4.1.0/Mongo2Go.nuspec @@ -0,0 +1,46 @@ + + + + Mongo2Go + 4.1.0 + Johannes Hoppe and many contributors + MIT + https://licenses.nuget.org/MIT + icon.png + https://github.com/Mongo2Go/Mongo2Go + Mongo2Go is a managed wrapper around MongoDB binaries. It targets .NET Framework 4.7.2 and .NET Standard 2.1. +This Nuget package contains the executables of mongod, mongoimport and mongoexport v4.4.4 for Windows, Linux and macOS. + + +Mongo2Go has two use cases: + +1. Providing multiple, temporary and isolated MongoDB databases for integration tests +2. Providing a quick to set up MongoDB database for a local developer environment + https://github.com/Mongo2Go/Mongo2Go/releases + Copyright © 2012-2025 Johannes Hoppe and many ❤️ contributors + MongoDB Mongo unit test integration runner + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Mongo2Go.4.1.0/[Content_Types].xml b/Mongo2Go.4.1.0/[Content_Types].xml new file mode 100644 index 00000000..eac33e7f --- /dev/null +++ b/Mongo2Go.4.1.0/[Content_Types].xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Mongo2Go.4.1.0/_rels/.rels b/Mongo2Go.4.1.0/_rels/.rels new file mode 100644 index 00000000..1a91feb8 --- /dev/null +++ b/Mongo2Go.4.1.0/_rels/.rels @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/Mongo2Go.4.1.0/icon.png b/Mongo2Go.4.1.0/icon.png new file mode 100644 index 00000000..23750b60 Binary files /dev/null and b/Mongo2Go.4.1.0/icon.png differ diff --git a/Mongo2Go.4.1.0/lib/net472/Mongo2Go.xml b/Mongo2Go.4.1.0/lib/net472/Mongo2Go.xml new file mode 100644 index 00000000..67d0b4d0 --- /dev/null +++ b/Mongo2Go.4.1.0/lib/net472/Mongo2Go.xml @@ -0,0 +1,175 @@ + + + + Mongo2Go + + + + + Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Returns and reserves a new port + + + + + Returns the if it is verified that it does not contain any mongod argument already defined by Mongo2Go. + + mongod arguments defined by Mongo2Go + Additional mongod arguments + contains at least one mongod argument already defined by Mongo2Go + A string with the additional mongod arguments + + + + Starts a new process. Process can be killed + + + + + Starts a new process. + + + + + Input File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Output File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Structure of a log generated by mongod. Used to deserialize the logs + and pass them to an ILogger. + See: https://docs.mongodb.com/manual/reference/log-messages/#json-log-output-format + Note: "truncated" and "size" are not parsed as we're unsure how to + properly parse and use them. + + + + + Severity of the logs as defined by MongoDB. Mapped to LogLevel + as defined by Microsoft. + D1-D2 mapped to Debug level. D3-D5 mapped Trace level. + + + + + Intention: port numbers won't be assigned twice to avoid connection problems with integration tests + + + + + Returns and reserves a new port + + + + + Reads from Output stream to determine if process is ready + + + + + Send the mongod process logs to .NET's console and debug outputs. + + + + + + Parses and redirects mongod logs to ILogger. + + + + + + + saves about 40 keystrokes + + + + + Populates the template using the provided arguments and the invariant culture + + + + + Populates the template using the provided arguments using the provided formatter + + + + + Mongo2Go main entry point + + + + + State of the current MongoDB instance + + + + + Connections string that should be used to establish a connection the MongoDB instance + + + + + Starts Multiple MongoDB instances with each call + On dispose: kills them and deletes their data directory + + (Optional) If null, mongod logs are wired to .NET's Console and Debug output (provided you haven't added the --quiet additional argument). + If not null, mongod logs are parsed and wired to the provided logger. + Should be used for integration tests + + + + !!! + This method is only used for an internal unit test. Use MongoDbRunner.Start() instead. + But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + + see https://github.com/Mongo2Go/Mongo2Go/issues/41 + + + + Only starts one single MongoDB instance (even on multiple calls), does not kill it, does not delete data + + + Should be used for local debugging only + WARNING: one single instance on one single machine is not a suitable setup for productive environments!!! + + + + + !!! + This method is only used for an internal unit test. Use MongoDbRunner.StartForDebugging() instead. + But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + + see https://github.com/Mongo2Go/Mongo2Go/issues/41 + + + + Executes Mongoimport on the associated MongoDB Instace + + + + + Executes Mongoexport on the associated MongoDB Instace + + + + + usage: local debugging + + + + + usage: integration tests + + + + diff --git a/Mongo2Go.4.1.0/lib/netstandard2.1/Mongo2Go.xml b/Mongo2Go.4.1.0/lib/netstandard2.1/Mongo2Go.xml new file mode 100644 index 00000000..67d0b4d0 --- /dev/null +++ b/Mongo2Go.4.1.0/lib/netstandard2.1/Mongo2Go.xml @@ -0,0 +1,175 @@ + + + + Mongo2Go + + + + + Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Returns and reserves a new port + + + + + Returns the if it is verified that it does not contain any mongod argument already defined by Mongo2Go. + + mongod arguments defined by Mongo2Go + Additional mongod arguments + contains at least one mongod argument already defined by Mongo2Go + A string with the additional mongod arguments + + + + Starts a new process. Process can be killed + + + + + Starts a new process. + + + + + Input File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Output File: Absolute path stays unchanged, relative path will be relative to current executing directory (usually the /bin folder) + + + + + Structure of a log generated by mongod. Used to deserialize the logs + and pass them to an ILogger. + See: https://docs.mongodb.com/manual/reference/log-messages/#json-log-output-format + Note: "truncated" and "size" are not parsed as we're unsure how to + properly parse and use them. + + + + + Severity of the logs as defined by MongoDB. Mapped to LogLevel + as defined by Microsoft. + D1-D2 mapped to Debug level. D3-D5 mapped Trace level. + + + + + Intention: port numbers won't be assigned twice to avoid connection problems with integration tests + + + + + Returns and reserves a new port + + + + + Reads from Output stream to determine if process is ready + + + + + Send the mongod process logs to .NET's console and debug outputs. + + + + + + Parses and redirects mongod logs to ILogger. + + + + + + + saves about 40 keystrokes + + + + + Populates the template using the provided arguments and the invariant culture + + + + + Populates the template using the provided arguments using the provided formatter + + + + + Mongo2Go main entry point + + + + + State of the current MongoDB instance + + + + + Connections string that should be used to establish a connection the MongoDB instance + + + + + Starts Multiple MongoDB instances with each call + On dispose: kills them and deletes their data directory + + (Optional) If null, mongod logs are wired to .NET's Console and Debug output (provided you haven't added the --quiet additional argument). + If not null, mongod logs are parsed and wired to the provided logger. + Should be used for integration tests + + + + !!! + This method is only used for an internal unit test. Use MongoDbRunner.Start() instead. + But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + + see https://github.com/Mongo2Go/Mongo2Go/issues/41 + + + + Only starts one single MongoDB instance (even on multiple calls), does not kill it, does not delete data + + + Should be used for local debugging only + WARNING: one single instance on one single machine is not a suitable setup for productive environments!!! + + + + + !!! + This method is only used for an internal unit test. Use MongoDbRunner.StartForDebugging() instead. + But if you find it to be useful (eg. to change every aspect on your own) feel free to implement the interfaces on your own! + + see https://github.com/Mongo2Go/Mongo2Go/issues/41 + + + + Executes Mongoimport on the associated MongoDB Instace + + + + + Executes Mongoexport on the associated MongoDB Instace + + + + + usage: local debugging + + + + + usage: integration tests + + + + diff --git a/Mongo2Go.4.1.0/package/services/metadata/core-properties/002438390f9b42fb9cd0a8f5b12ea55f.psmdcp b/Mongo2Go.4.1.0/package/services/metadata/core-properties/002438390f9b42fb9cd0a8f5b12ea55f.psmdcp new file mode 100644 index 00000000..62e65722 --- /dev/null +++ b/Mongo2Go.4.1.0/package/services/metadata/core-properties/002438390f9b42fb9cd0a8f5b12ea55f.psmdcp @@ -0,0 +1,16 @@ + + + Johannes Hoppe and many contributors + Mongo2Go is a managed wrapper around MongoDB binaries. It targets .NET Framework 4.7.2 and .NET Standard 2.1. +This Nuget package contains the executables of mongod, mongoimport and mongoexport v4.4.4 for Windows, Linux and macOS. + + +Mongo2Go has two use cases: + +1. Providing multiple, temporary and isolated MongoDB databases for integration tests +2. Providing a quick to set up MongoDB database for a local developer environment + Mongo2Go + 4.1.0 + MongoDB Mongo unit test integration runner + NuGet.Build.Tasks.Pack, Version=6.11.1.2, Culture=neutral, PublicKeyToken=31bf3856ad364e35;.NET Standard 2.0 + \ No newline at end of file diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt new file mode 100644 index 00000000..b01add13 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt @@ -0,0 +1,557 @@ + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 new file mode 100644 index 00000000..197b2ffd --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/MPL-2 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README new file mode 100644 index 00000000..97f1dc72 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/README @@ -0,0 +1,87 @@ +MongoDB README + +Welcome to MongoDB! + +COMPONENTS + + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + install_compass - Installs MongoDB Compass for your platform. + +BUILDING + + See docs/building.md. + +RUNNING + + For command line options invoke: + + $ ./mongod --help + + To run a single server database: + + $ sudo mkdir -p /data/db + $ ./mongod + $ + $ # The mongo javascript shell connects to localhost and test database by default: + $ ./mongo + > help + +INSTALLING COMPASS + + You can install compass using the install_compass script packaged with MongoDB: + + $ ./install_compass + + This will download the appropriate MongoDB Compass package for your platform + and install it. + +DRIVERS + + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. + +DOCUMENTATION + + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +FORUMS + + https://community.mongodb.com + + A forum for technical questions about using MongoDB. + + https://community.mongodb.com/c/server-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ + +LICENSE + + MongoDB is free and open-source. Versions released prior to October 16, + 2018 are published under the AGPL. All versions released after October + 16, 2018, including patch fixes for prior versions, are published under + the Server Side Public License (SSPL) v1. See individual files for + details. + diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..8c9e17e3 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES @@ -0,0 +1,1568 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through any of the ways detailed here : + + mongodb-dev@googlegroups.com + +The attached notices are provided for information only. + +For any licenses that require disclosure of source, sources are available at +https://github.com/mongodb/mongo. + + +1) License Notice for Boost +--------------------------- + +http://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +3) License Notice for PCRE +-------------------------- + +http://www.pcre.org/licence.txt + +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 7 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2008 University of Cambridge +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2008, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +4) License notice for Aladdin MD5 +--------------------------------- + +Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +L. Peter Deutsch +ghost@aladdin.com + +5) License notice for Snappy - http://code.google.com/p/snappy/ +--------------------------------- + Copyright 2005 and onwards Google Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + A light-weight compression algorithm. It is designed for speed of + compression and decompression, rather than for the utmost in space + savings. + + For getting better compression ratios when you are compressing data + with long repeated sequences or compressing data that is similar to + other data, while still compressing fast, you might look at first + using BMDiff and then compressing the output of BMDiff with + Snappy. + +6) License notice for Google Perftools (TCMalloc utility) +--------------------------------- +New BSD License + +Copyright (c) 1998-2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +7) License notice for Linenoise +------------------------------- + + Copyright (c) 2010, Salvatore Sanfilippo + Copyright (c) 2010, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +8) License notice for S2 Geometry Library +----------------------------------------- + Copyright 2005 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +9) License notice for MurmurHash +-------------------------------- + + Copyright (c) 2010-2012 Austin Appleby + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +10) License notice for Snowball + Copyright (c) 2001, Dr Martin Porter + All rights reserved. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +11) License notice for yaml-cpp +------------------------------- + +Copyright (c) 2008 Jesse Beder. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +12) License notice for zlib +--------------------------- + +http://www.zlib.net/zlib_license.html + +zlib.h -- interface of the 'zlib' general purpose compression library +version 1.2.8, April 28th, 2013 + +Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu + + +13) License notice for 3rd party software included in the WiredTiger library +---------------------------------------------------------------------------- + +http://source.wiredtiger.com/license.html + +WiredTiger Distribution Files | Copyright Holder | License +----------------------------- | ----------------------------------- | ---------------------- +src/include/bitstring.i | University of California, Berkeley | BSD-3-Clause License +src/include/queue.h | University of California, Berkeley | BSD-3-Clause License +src/os_posix/os_getopt.c | University of California, Berkeley | BSD-3-Clause License +src/support/hash_city.c | Google, Inc. | The MIT License +src/support/hash_fnv.c | Authors | Public Domain + + +Other optional 3rd party software included in the WiredTiger distribution is removed by MongoDB. + + +BSD-3-CLAUSE LICENSE +-------------------- + +http://www.opensource.org/licenses/BSD-3-Clause + +Copyright (c) 1987, 1989, 1991, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + +THE MIT LICENSE +--------------- + +http://www.opensource.org/licenses/MIT + +Copyright (c) 2011 Google, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +14) License Notice for SpiderMonkey +----------------------------------- + +|------------------------------------------------|------------------|---------------| +| SpiderMonkey Distribution Files | Copyright Holder | License | +|------------------------------------------------|------------------|---------------| +| js/src/jit/shared/AssemblerBuffer-x86-shared.h | Apple, Inc | BSD-2-Clause | +| js/src/jit/shared/BaseAssembler-x86-shared.h | | | +|------------------------------------------------|------------------|---------------| +| js/src/builtin/ | Google, Inc | BSD-3-Clause | +| js/src/irregexp/ | | | +| js/src/jit/arm/ | | | +| js/src/jit/mips/ | | | +| mfbt/double-conversion/ | | | +|------------------------------------------------|------------------|---------------| +| intl/icu/source/common/unicode/ | IBM, Inc | ICU | +|------------------------------------------------|------------------|---------------| +| js/src/asmjs/ | Mozilla, Inc | Apache2 | +|------------------------------------------------|------------------|---------------| +| js/public/ | Mozilla, Inc | MPL2 | +| js/src/ | | | +| mfbt | | | +|------------------------------------------------|------------------|---------------| +| js/src/vm/Unicode.cpp | None | Public Domain | +|------------------------------------------------|------------------|---------------| +| mfbt/lz4.c | Yann Collet | BSD-2-Clause | +| mfbt/lz4.h | | | +|------------------------------------------------|------------------|---------------| + +Other optional 3rd party software included in the SpiderMonkey distribution is removed by MongoDB. + + +Apple, Inc: BSD-2-Clause +------------------------ + +Copyright (C) 2008 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Google, Inc: BSD-3-Clause +------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ICU License - ICU 1.8.1 and later +--------------------------------- + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2012 International Business Machines Corporation and +others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, provided that the above copyright notice(s) and this +permission notice appear in all copies of the Software and that both the +above copyright notice(s) and this permission notice appear in supporting +documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in this Software without prior written authorization of the +copyright holder. + +All trademarks and registered trademarks mentioned herein are the property +of their respective owners. + + +Mozilla, Inc: Apache 2 +---------------------- + +Copyright 2014 Mozilla Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Mozilla, Inc: MPL 2 +------------------- + +Copyright 2014 Mozilla Foundation + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +Public Domain +------------- + +Any copyright is dedicated to the Public Domain. +http://creativecommons.org/licenses/publicdomain/ + + +LZ4: BSD-2-Clause +----------------- + +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + +15) License Notice for Intel DFP Math Library +--------------------------------------------- + +Copyright (c) 2011, Intel Corp. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + his list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +16) License Notice for Unicode Data +----------------------------------- + +Copyright © 1991-2015 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +17 ) License Notice for Valgrind.h +---------------------------------- + +---------------------------------------------------------------- + +Notice that the following BSD-style license applies to this one +file (valgrind.h) only. The rest of Valgrind is licensed under the +terms of the GNU General Public License, version 2, unless +otherwise indicated. See the COPYING file in the source +distribution for details. + +---------------------------------------------------------------- + +This file is part of Valgrind, a dynamic binary instrumentation +framework. + +Copyright (C) 2000-2015 Julian Seward. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------- + +Notice that the above BSD-style license applies to this one file +(valgrind.h) only. The entire rest of Valgrind is licensed under +the terms of the GNU General Public License, version 2. See the +COPYING file in the source distribution for details. + +---------------------------------------------------------------- + +18) License notice for ICU4C +---------------------------- + +ICU License - ICU 1.8.1 and later + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. Unicode Data Files and Software + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyrighy (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +19) License notice for timelib +------------------------------ + +The MIT License (MIT) + +Copyright (c) 2015-2017 Derick Rethans +Copyright (c) 2017 MongoDB, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +20) License notice for windows dirent implementation +---------------------------------------------------- + + * Dirent interface for Microsoft Visual Studio + * Version 1.21 + * + * Copyright (C) 2006-2012 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + + + 21) License notice for abseil-cpp +---------------------------- + + Copyright (c) Google Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + 22) License notice for Zstandard +---------------------------- + + BSD License + + For Zstandard software + + Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 23) License notice for ASIO +---------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 24) License notice for MPark.Variant +------------------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 25) License notice for fmt +--------------------------- + +Copyright (c) 2012 - present, Victor Zverovich +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or other + materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 26) License notice for SafeInt +--------------------------- + +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the MIT License. + +MIT License + +Copyright (c) 2018 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + 27) License Notice for Raft TLA+ Specification +----------------------------------------------- + +https://github.com/ongardie/dissertation/blob/master/LICENSE + +Copyright 2014 Diego Ongaro. + +Some of our TLA+ specifications are based on the Raft TLA+ specification by Diego Ongaro. + +End diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md new file mode 100644 index 00000000..550ae4ed --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2014 MongoDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md new file mode 100644 index 00000000..e60e0cd0 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/README.md @@ -0,0 +1,72 @@ +MongoDB Tools +=================================== + + - **bsondump** - _display BSON files in a human-readable format_ + - **mongoimport** - _Convert data from JSON, TSV or CSV and insert them into a collection_ + - **mongoexport** - _Write an existing collection to CSV or JSON format_ + - **mongodump/mongorestore** - _Dump MongoDB backups to disk in .BSON format, or restore them to a live database_ + - **mongostat** - _Monitor live MongoDB servers, replica sets, or sharded clusters_ + - **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_ + - **mongotop** - _Monitor read/write activity on a mongo server_ + + +Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS + +Building Tools +--------------- + +We currently build the tools with Go version 1.15. Other Go versions may work but they are untested. + +Using `go get` to directly build the tools will not work. To build them, it's recommended to first clone this repository: + +``` +git clone https://github.com/mongodb/mongo-tools +cd mongo-tools +``` + +Then run `./make build` to build all the tools, placing them in the `bin` directory inside the repository. + +You can also build a subset of the tools using the `-tools` option. For example, `./make build -tools=mongodump,mongorestore` builds only `mongodump` and `mongorestore`. + +To use the build/test scripts in this repository, you **_must_** set GOROOT to your Go root directory. This may depend on how you installed Go. + +``` +export GOROOT=/usr/local/go +``` + +Updating Dependencies +--------------- +Starting with version 100.3.1, the tools use `go mod` to manage dependencies. All dependencies are listed in the `go.mod` file and are directly vendored in the `vendor` directory. + +In order to make changes to dependencies, you first need to change the `go.mod` file. You can manually edit that file to add/update/remove entries, or you can run the following in the repository directory: + +``` +go mod edit -require=@ # for adding or updating a dependency +go mod edit -droprequire= # for removing a dependency +``` + +Then run `go mod vendor -v` to reconstruct the `vendor` directory to match the changed `go.mod` file. + +Optionally, run `go mod tidy -v` to ensure that the `go.mod` file matches the `mongo-tools` source code. + +Contributing +--------------- +See our [Contributor's Guide](CONTRIBUTING.md). + +Documentation +--------------- +See the MongoDB packages [documentation](https://docs.mongodb.org/database-tools/). + +For documentation on older versions of the MongoDB, reference that version of the [MongoDB Server Manual](docs.mongodb.com/manual): + +- [MongoDB 4.2 Tools](https://docs.mongodb.org/v4.2/reference/program) +- [MongoDB 4.0 Tools](https://docs.mongodb.org/v4.0/reference/program) +- [MongoDB 3.6 Tools](https://docs.mongodb.org/v3.6/reference/program) + +Adding New Platforms Support +--------------- +See our [Adding New Platform Support Guide](PLATFORMSUPPORT.md). + +Vendoring the Change into Server Repo +--------------- +See our [Vendor the Change into Server Repo](SERVERVENDORING.md). diff --git a/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..3d75e64b --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-linux-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES @@ -0,0 +1,3319 @@ +--------------------------------------------------------------------- +License notice for hashicorp/go-rootcerts +--------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +--------------------------------------------------------------------- +License notice for JSON and CSV code from github.com/golang/go +--------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/escaper +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Lucas Morales + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo/bson +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/openssl +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/3rf/mongo-lint +---------------------------------------------------------------------- + +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/go-stack/stack +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/golang/snappy +---------------------------------------------------------------------- + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/google/gopacket +---------------------------------------------------------------------- + +Copyright (c) 2012 Google, Inc. All rights reserved. +Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Andreas Krennmair, Google, nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/gopherjs/gopherjs +---------------------------------------------------------------------- + +Copyright (c) 2013 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/howeyc/gopass +---------------------------------------------------------------------- + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/jessevdk/go-flags +---------------------------------------------------------------------- + +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/jtolds/gls +---------------------------------------------------------------------- + +Copyright (c) 2013, Space Monkey, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mattn/go-runewidth +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mongodb/mongo-go-driver +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/nsf/termbox-go +---------------------------------------------------------------------- + +Copyright (C) 2012 termbox-go authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/patrickmn/go-cache +---------------------------------------------------------------------- + +Copyright (c) 2012-2015 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions +---------------------------------------------------------------------- + +Copyright (c) 2015 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/go-render +---------------------------------------------------------------------- + +// Copyright (c) 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglematchers +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglemock +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/ogletest +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/reqtrace +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey +---------------------------------------------------------------------- + +Copyright (c) 2014 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey/web/client/resources/fonts/Open_Sans +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/spacemonkeygo/spacelog +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/xdg/scram +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/xdg/stringprep +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/youmark/pkcs8 +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for golang.org/x/crypto +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/sync +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/text +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for gopkg.in/tomb.v2 +---------------------------------------------------------------------- + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt new file mode 100644 index 00000000..b01add13 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt @@ -0,0 +1,557 @@ + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 new file mode 100644 index 00000000..197b2ffd --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/MPL-2 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README new file mode 100644 index 00000000..97f1dc72 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/README @@ -0,0 +1,87 @@ +MongoDB README + +Welcome to MongoDB! + +COMPONENTS + + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + install_compass - Installs MongoDB Compass for your platform. + +BUILDING + + See docs/building.md. + +RUNNING + + For command line options invoke: + + $ ./mongod --help + + To run a single server database: + + $ sudo mkdir -p /data/db + $ ./mongod + $ + $ # The mongo javascript shell connects to localhost and test database by default: + $ ./mongo + > help + +INSTALLING COMPASS + + You can install compass using the install_compass script packaged with MongoDB: + + $ ./install_compass + + This will download the appropriate MongoDB Compass package for your platform + and install it. + +DRIVERS + + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. + +DOCUMENTATION + + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +FORUMS + + https://community.mongodb.com + + A forum for technical questions about using MongoDB. + + https://community.mongodb.com/c/server-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ + +LICENSE + + MongoDB is free and open-source. Versions released prior to October 16, + 2018 are published under the AGPL. All versions released after October + 16, 2018, including patch fixes for prior versions, are published under + the Server Side Public License (SSPL) v1. See individual files for + details. + diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..8c9e17e3 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES @@ -0,0 +1,1568 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through any of the ways detailed here : + + mongodb-dev@googlegroups.com + +The attached notices are provided for information only. + +For any licenses that require disclosure of source, sources are available at +https://github.com/mongodb/mongo. + + +1) License Notice for Boost +--------------------------- + +http://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +3) License Notice for PCRE +-------------------------- + +http://www.pcre.org/licence.txt + +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 7 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2008 University of Cambridge +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2008, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +4) License notice for Aladdin MD5 +--------------------------------- + +Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +L. Peter Deutsch +ghost@aladdin.com + +5) License notice for Snappy - http://code.google.com/p/snappy/ +--------------------------------- + Copyright 2005 and onwards Google Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + A light-weight compression algorithm. It is designed for speed of + compression and decompression, rather than for the utmost in space + savings. + + For getting better compression ratios when you are compressing data + with long repeated sequences or compressing data that is similar to + other data, while still compressing fast, you might look at first + using BMDiff and then compressing the output of BMDiff with + Snappy. + +6) License notice for Google Perftools (TCMalloc utility) +--------------------------------- +New BSD License + +Copyright (c) 1998-2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +7) License notice for Linenoise +------------------------------- + + Copyright (c) 2010, Salvatore Sanfilippo + Copyright (c) 2010, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +8) License notice for S2 Geometry Library +----------------------------------------- + Copyright 2005 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +9) License notice for MurmurHash +-------------------------------- + + Copyright (c) 2010-2012 Austin Appleby + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +10) License notice for Snowball + Copyright (c) 2001, Dr Martin Porter + All rights reserved. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +11) License notice for yaml-cpp +------------------------------- + +Copyright (c) 2008 Jesse Beder. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +12) License notice for zlib +--------------------------- + +http://www.zlib.net/zlib_license.html + +zlib.h -- interface of the 'zlib' general purpose compression library +version 1.2.8, April 28th, 2013 + +Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu + + +13) License notice for 3rd party software included in the WiredTiger library +---------------------------------------------------------------------------- + +http://source.wiredtiger.com/license.html + +WiredTiger Distribution Files | Copyright Holder | License +----------------------------- | ----------------------------------- | ---------------------- +src/include/bitstring.i | University of California, Berkeley | BSD-3-Clause License +src/include/queue.h | University of California, Berkeley | BSD-3-Clause License +src/os_posix/os_getopt.c | University of California, Berkeley | BSD-3-Clause License +src/support/hash_city.c | Google, Inc. | The MIT License +src/support/hash_fnv.c | Authors | Public Domain + + +Other optional 3rd party software included in the WiredTiger distribution is removed by MongoDB. + + +BSD-3-CLAUSE LICENSE +-------------------- + +http://www.opensource.org/licenses/BSD-3-Clause + +Copyright (c) 1987, 1989, 1991, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + +THE MIT LICENSE +--------------- + +http://www.opensource.org/licenses/MIT + +Copyright (c) 2011 Google, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +14) License Notice for SpiderMonkey +----------------------------------- + +|------------------------------------------------|------------------|---------------| +| SpiderMonkey Distribution Files | Copyright Holder | License | +|------------------------------------------------|------------------|---------------| +| js/src/jit/shared/AssemblerBuffer-x86-shared.h | Apple, Inc | BSD-2-Clause | +| js/src/jit/shared/BaseAssembler-x86-shared.h | | | +|------------------------------------------------|------------------|---------------| +| js/src/builtin/ | Google, Inc | BSD-3-Clause | +| js/src/irregexp/ | | | +| js/src/jit/arm/ | | | +| js/src/jit/mips/ | | | +| mfbt/double-conversion/ | | | +|------------------------------------------------|------------------|---------------| +| intl/icu/source/common/unicode/ | IBM, Inc | ICU | +|------------------------------------------------|------------------|---------------| +| js/src/asmjs/ | Mozilla, Inc | Apache2 | +|------------------------------------------------|------------------|---------------| +| js/public/ | Mozilla, Inc | MPL2 | +| js/src/ | | | +| mfbt | | | +|------------------------------------------------|------------------|---------------| +| js/src/vm/Unicode.cpp | None | Public Domain | +|------------------------------------------------|------------------|---------------| +| mfbt/lz4.c | Yann Collet | BSD-2-Clause | +| mfbt/lz4.h | | | +|------------------------------------------------|------------------|---------------| + +Other optional 3rd party software included in the SpiderMonkey distribution is removed by MongoDB. + + +Apple, Inc: BSD-2-Clause +------------------------ + +Copyright (C) 2008 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Google, Inc: BSD-3-Clause +------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ICU License - ICU 1.8.1 and later +--------------------------------- + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2012 International Business Machines Corporation and +others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, provided that the above copyright notice(s) and this +permission notice appear in all copies of the Software and that both the +above copyright notice(s) and this permission notice appear in supporting +documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in this Software without prior written authorization of the +copyright holder. + +All trademarks and registered trademarks mentioned herein are the property +of their respective owners. + + +Mozilla, Inc: Apache 2 +---------------------- + +Copyright 2014 Mozilla Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Mozilla, Inc: MPL 2 +------------------- + +Copyright 2014 Mozilla Foundation + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +Public Domain +------------- + +Any copyright is dedicated to the Public Domain. +http://creativecommons.org/licenses/publicdomain/ + + +LZ4: BSD-2-Clause +----------------- + +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + +15) License Notice for Intel DFP Math Library +--------------------------------------------- + +Copyright (c) 2011, Intel Corp. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + his list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +16) License Notice for Unicode Data +----------------------------------- + +Copyright © 1991-2015 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +17 ) License Notice for Valgrind.h +---------------------------------- + +---------------------------------------------------------------- + +Notice that the following BSD-style license applies to this one +file (valgrind.h) only. The rest of Valgrind is licensed under the +terms of the GNU General Public License, version 2, unless +otherwise indicated. See the COPYING file in the source +distribution for details. + +---------------------------------------------------------------- + +This file is part of Valgrind, a dynamic binary instrumentation +framework. + +Copyright (C) 2000-2015 Julian Seward. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------- + +Notice that the above BSD-style license applies to this one file +(valgrind.h) only. The entire rest of Valgrind is licensed under +the terms of the GNU General Public License, version 2. See the +COPYING file in the source distribution for details. + +---------------------------------------------------------------- + +18) License notice for ICU4C +---------------------------- + +ICU License - ICU 1.8.1 and later + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. Unicode Data Files and Software + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyrighy (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +19) License notice for timelib +------------------------------ + +The MIT License (MIT) + +Copyright (c) 2015-2017 Derick Rethans +Copyright (c) 2017 MongoDB, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +20) License notice for windows dirent implementation +---------------------------------------------------- + + * Dirent interface for Microsoft Visual Studio + * Version 1.21 + * + * Copyright (C) 2006-2012 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + + + 21) License notice for abseil-cpp +---------------------------- + + Copyright (c) Google Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + 22) License notice for Zstandard +---------------------------- + + BSD License + + For Zstandard software + + Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 23) License notice for ASIO +---------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 24) License notice for MPark.Variant +------------------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 25) License notice for fmt +--------------------------- + +Copyright (c) 2012 - present, Victor Zverovich +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or other + materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 26) License notice for SafeInt +--------------------------- + +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the MIT License. + +MIT License + +Copyright (c) 2018 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + 27) License Notice for Raft TLA+ Specification +----------------------------------------------- + +https://github.com/ongardie/dissertation/blob/master/LICENSE + +Copyright 2014 Diego Ongaro. + +Some of our TLA+ specifications are based on the Raft TLA+ specification by Diego Ongaro. + +End diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md new file mode 100644 index 00000000..550ae4ed --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2014 MongoDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md new file mode 100644 index 00000000..e60e0cd0 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/README.md @@ -0,0 +1,72 @@ +MongoDB Tools +=================================== + + - **bsondump** - _display BSON files in a human-readable format_ + - **mongoimport** - _Convert data from JSON, TSV or CSV and insert them into a collection_ + - **mongoexport** - _Write an existing collection to CSV or JSON format_ + - **mongodump/mongorestore** - _Dump MongoDB backups to disk in .BSON format, or restore them to a live database_ + - **mongostat** - _Monitor live MongoDB servers, replica sets, or sharded clusters_ + - **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_ + - **mongotop** - _Monitor read/write activity on a mongo server_ + + +Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS + +Building Tools +--------------- + +We currently build the tools with Go version 1.15. Other Go versions may work but they are untested. + +Using `go get` to directly build the tools will not work. To build them, it's recommended to first clone this repository: + +``` +git clone https://github.com/mongodb/mongo-tools +cd mongo-tools +``` + +Then run `./make build` to build all the tools, placing them in the `bin` directory inside the repository. + +You can also build a subset of the tools using the `-tools` option. For example, `./make build -tools=mongodump,mongorestore` builds only `mongodump` and `mongorestore`. + +To use the build/test scripts in this repository, you **_must_** set GOROOT to your Go root directory. This may depend on how you installed Go. + +``` +export GOROOT=/usr/local/go +``` + +Updating Dependencies +--------------- +Starting with version 100.3.1, the tools use `go mod` to manage dependencies. All dependencies are listed in the `go.mod` file and are directly vendored in the `vendor` directory. + +In order to make changes to dependencies, you first need to change the `go.mod` file. You can manually edit that file to add/update/remove entries, or you can run the following in the repository directory: + +``` +go mod edit -require=@ # for adding or updating a dependency +go mod edit -droprequire= # for removing a dependency +``` + +Then run `go mod vendor -v` to reconstruct the `vendor` directory to match the changed `go.mod` file. + +Optionally, run `go mod tidy -v` to ensure that the `go.mod` file matches the `mongo-tools` source code. + +Contributing +--------------- +See our [Contributor's Guide](CONTRIBUTING.md). + +Documentation +--------------- +See the MongoDB packages [documentation](https://docs.mongodb.org/database-tools/). + +For documentation on older versions of the MongoDB, reference that version of the [MongoDB Server Manual](docs.mongodb.com/manual): + +- [MongoDB 4.2 Tools](https://docs.mongodb.org/v4.2/reference/program) +- [MongoDB 4.0 Tools](https://docs.mongodb.org/v4.0/reference/program) +- [MongoDB 3.6 Tools](https://docs.mongodb.org/v3.6/reference/program) + +Adding New Platforms Support +--------------- +See our [Adding New Platform Support Guide](PLATFORMSUPPORT.md). + +Vendoring the Change into Server Repo +--------------- +See our [Vendor the Change into Server Repo](SERVERVENDORING.md). diff --git a/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..3d75e64b --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-macos-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES @@ -0,0 +1,3319 @@ +--------------------------------------------------------------------- +License notice for hashicorp/go-rootcerts +--------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +--------------------------------------------------------------------- +License notice for JSON and CSV code from github.com/golang/go +--------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/escaper +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Lucas Morales + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo/bson +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/openssl +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/3rf/mongo-lint +---------------------------------------------------------------------- + +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/go-stack/stack +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/golang/snappy +---------------------------------------------------------------------- + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/google/gopacket +---------------------------------------------------------------------- + +Copyright (c) 2012 Google, Inc. All rights reserved. +Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Andreas Krennmair, Google, nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/gopherjs/gopherjs +---------------------------------------------------------------------- + +Copyright (c) 2013 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/howeyc/gopass +---------------------------------------------------------------------- + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/jessevdk/go-flags +---------------------------------------------------------------------- + +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/jtolds/gls +---------------------------------------------------------------------- + +Copyright (c) 2013, Space Monkey, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mattn/go-runewidth +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mongodb/mongo-go-driver +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/nsf/termbox-go +---------------------------------------------------------------------- + +Copyright (C) 2012 termbox-go authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/patrickmn/go-cache +---------------------------------------------------------------------- + +Copyright (c) 2012-2015 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions +---------------------------------------------------------------------- + +Copyright (c) 2015 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/go-render +---------------------------------------------------------------------- + +// Copyright (c) 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglematchers +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglemock +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/ogletest +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/reqtrace +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey +---------------------------------------------------------------------- + +Copyright (c) 2014 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey/web/client/resources/fonts/Open_Sans +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/spacemonkeygo/spacelog +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/xdg/scram +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/xdg/stringprep +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/youmark/pkcs8 +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for golang.org/x/crypto +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/sync +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/text +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for gopkg.in/tomb.v2 +---------------------------------------------------------------------- + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt new file mode 100644 index 00000000..b01add13 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/LICENSE-Community.txt @@ -0,0 +1,557 @@ + Server Side Public License + VERSION 1, OCTOBER 16, 2018 + + Copyright © 2018 MongoDB, Inc. + + Everyone is permitted to copy and distribute verbatim copies of this + license document, but changing it is not allowed. + + TERMS AND CONDITIONS + + 0. Definitions. + + “This License” refers to Server Side Public License. + + “Copyright” also means copyright-like laws that apply to other kinds of + works, such as semiconductor masks. + + “The Program” refers to any copyrightable work licensed under this + License. Each licensee is addressed as “you”. “Licensees” and + “recipients” may be individuals or organizations. + + To “modify” a work means to copy from or adapt all or part of the work in + a fashion requiring copyright permission, other than the making of an + exact copy. The resulting work is called a “modified version” of the + earlier work or a work “based on” the earlier work. + + A “covered work” means either the unmodified Program or a work based on + the Program. + + To “propagate” a work means to do anything with it that, without + permission, would make you directly or secondarily liable for + infringement under applicable copyright law, except executing it on a + computer or modifying a private copy. Propagation includes copying, + distribution (with or without modification), making available to the + public, and in some countries other activities as well. + + To “convey” a work means any kind of propagation that enables other + parties to make or receive copies. Mere interaction with a user through a + computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays “Appropriate Legal Notices” to the + extent that it includes a convenient and prominently visible feature that + (1) displays an appropriate copyright notice, and (2) tells the user that + there is no warranty for the work (except to the extent that warranties + are provided), that licensees may convey the work under this License, and + how to view a copy of this License. If the interface presents a list of + user commands or options, such as a menu, a prominent item in the list + meets this criterion. + + 1. Source Code. + + The “source code” for a work means the preferred form of the work for + making modifications to it. “Object code” means any non-source form of a + work. + + A “Standard Interface” means an interface that either is an official + standard defined by a recognized standards body, or, in the case of + interfaces specified for a particular programming language, one that is + widely used among developers working in that language. The “System + Libraries” of an executable work include anything, other than the work as + a whole, that (a) is included in the normal form of packaging a Major + Component, but which is not part of that Major Component, and (b) serves + only to enable use of the work with that Major Component, or to implement + a Standard Interface for which an implementation is available to the + public in source code form. A “Major Component”, in this context, means a + major essential component (kernel, window system, and so on) of the + specific operating system (if any) on which the executable work runs, or + a compiler used to produce the work, or an object code interpreter used + to run it. + + The “Corresponding Source” for a work in object code form means all the + source code needed to generate, install, and (for an executable work) run + the object code and to modify the work, including scripts to control + those activities. However, it does not include the work's System + Libraries, or general-purpose tools or generally available free programs + which are used unmodified in performing those activities but which are + not part of the work. For example, Corresponding Source includes + interface definition files associated with source files for the work, and + the source code for shared libraries and dynamically linked subprograms + that the work is specifically designed to require, such as by intimate + data communication or control flow between those subprograms and other + parts of the work. + + The Corresponding Source need not include anything that users can + regenerate automatically from other parts of the Corresponding Source. + + The Corresponding Source for a work in source code form is that same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of + copyright on the Program, and are irrevocable provided the stated + conditions are met. This License explicitly affirms your unlimited + permission to run the unmodified Program, subject to section 13. The + output from running a covered work is covered by this License only if the + output, given its content, constitutes a covered work. This License + acknowledges your rights of fair use or other equivalent, as provided by + copyright law. Subject to section 13, you may make, run and propagate + covered works that you do not convey, without conditions so long as your + license otherwise remains in force. You may convey covered works to + others for the sole purpose of having them make modifications exclusively + for you, or provide you with facilities for running those works, provided + that you comply with the terms of this License in conveying all + material for which you do not control copyright. Those thus making or + running the covered works for you must do so exclusively on your + behalf, under your direction and control, on terms that prohibit them + from making any copies of your copyrighted material outside their + relationship with you. + + Conveying under any other circumstances is permitted solely under the + conditions stated below. Sublicensing is not allowed; section 10 makes it + unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological + measure under any applicable law fulfilling obligations under article 11 + of the WIPO copyright treaty adopted on 20 December 1996, or similar laws + prohibiting or restricting circumvention of such measures. + + When you convey a covered work, you waive any legal power to forbid + circumvention of technological measures to the extent such circumvention is + effected by exercising rights under this License with respect to the + covered work, and you disclaim any intention to limit operation or + modification of the work as a means of enforcing, against the work's users, + your or third parties' legal rights to forbid circumvention of + technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you + receive it, in any medium, provided that you conspicuously and + appropriately publish on each copy an appropriate copyright notice; keep + intact all notices stating that this License and any non-permissive terms + added in accord with section 7 apply to the code; keep intact all notices + of the absence of any warranty; and give all recipients a copy of this + License along with the Program. You may charge any price or no price for + each copy that you convey, and you may offer support or warranty + protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to + produce it from the Program, in the form of source code under the terms + of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, + and giving a relevant date. + + b) The work must carry prominent notices stating that it is released + under this License and any conditions added under section 7. This + requirement modifies the requirement in section 4 to “keep intact all + notices”. + + c) You must license the entire work, as a whole, under this License to + anyone who comes into possession of a copy. This License will therefore + apply, along with any applicable section 7 additional terms, to the + whole of the work, and all its parts, regardless of how they are + packaged. This License gives no permission to license the work in any + other way, but it does not invalidate such permission if you have + separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your work + need not make them do so. + + A compilation of a covered work with other separate and independent + works, which are not by their nature extensions of the covered work, and + which are not combined with it such as to form a larger program, in or on + a volume of a storage or distribution medium, is called an “aggregate” if + the compilation and its resulting copyright are not used to limit the + access or legal rights of the compilation's users beyond what the + individual works permit. Inclusion of a covered work in an aggregate does + not cause this License to apply to the other parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms of + sections 4 and 5, provided that you also convey the machine-readable + Corresponding Source under the terms of this License, in one of these + ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium customarily + used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a written + offer, valid for at least three years and valid for as long as you + offer spare parts or customer support for that product model, to give + anyone who possesses the object code either (1) a copy of the + Corresponding Source for all the software in the product that is + covered by this License, on a durable physical medium customarily used + for software interchange, for a price no more than your reasonable cost + of physically performing this conveying of source, or (2) access to + copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This alternative is + allowed only occasionally and noncommercially, and only if you received + the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place + (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to copy + the object code is a network server, the Corresponding Source may be on + a different server (operated by you or a third party) that supports + equivalent copying facilities, provided you maintain clear directions + next to the object code saying where to find the Corresponding Source. + Regardless of what server hosts the Corresponding Source, you remain + obligated to ensure that it is available for as long as needed to + satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you + inform other peers where the object code and Corresponding Source of + the work are being offered to the general public at no charge under + subsection 6d. + + A separable portion of the object code, whose source code is excluded + from the Corresponding Source as a System Library, need not be included + in conveying the object code work. + + A “User Product” is either (1) a “consumer product”, which means any + tangible personal property which is normally used for personal, family, + or household purposes, or (2) anything designed or sold for incorporation + into a dwelling. In determining whether a product is a consumer product, + doubtful cases shall be resolved in favor of coverage. For a particular + product received by a particular user, “normally used” refers to a + typical or common use of that class of product, regardless of the status + of the particular user or of the way in which the particular user + actually uses, or expects or is expected to use, the product. A product + is a consumer product regardless of whether the product has substantial + commercial, industrial or non-consumer uses, unless such uses represent + the only significant mode of use of the product. + + “Installation Information” for a User Product means any methods, + procedures, authorization keys, or other information required to install + and execute modified versions of a covered work in that User Product from + a modified version of its Corresponding Source. The information must + suffice to ensure that the continued functioning of the modified object + code is in no case prevented or interfered with solely because + modification has been made. + + If you convey an object code work under this section in, or with, or + specifically for use in, a User Product, and the conveying occurs as part + of a transaction in which the right of possession and use of the User + Product is transferred to the recipient in perpetuity or for a fixed term + (regardless of how the transaction is characterized), the Corresponding + Source conveyed under this section must be accompanied by the + Installation Information. But this requirement does not apply if neither + you nor any third party retains the ability to install modified object + code on the User Product (for example, the work has been installed in + ROM). + + The requirement to provide Installation Information does not include a + requirement to continue to provide support service, warranty, or updates + for a work that has been modified or installed by the recipient, or for + the User Product in which it has been modified or installed. Access + to a network may be denied when the modification itself materially + and adversely affects the operation of the network or violates the + rules and protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, in + accord with this section must be in a format that is publicly documented + (and with an implementation available to the public in source code form), + and must require no special password or key for unpacking, reading or + copying. + + 7. Additional Terms. + + “Additional permissions” are terms that supplement the terms of this + License by making exceptions from one or more of its conditions. + Additional permissions that are applicable to the entire Program shall be + treated as though they were included in this License, to the extent that + they are valid under applicable law. If additional permissions apply only + to part of the Program, that part may be used separately under those + permissions, but the entire Program remains governed by this License + without regard to the additional permissions. When you convey a copy of + a covered work, you may at your option remove any additional permissions + from that copy, or from any part of it. (Additional permissions may be + written to require their own removal in certain cases when you modify the + work.) You may place additional permissions on material, added by you to + a covered work, for which you have or can give appropriate copyright + permission. + + Notwithstanding any other provision of this License, for material you add + to a covered work, you may (if authorized by the copyright holders of + that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade + names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material + by anyone who conveys the material (or modified versions of it) with + contractual assumptions of liability to the recipient, for any + liability that these contractual assumptions directly impose on those + licensors and authors. + + All other non-permissive additional terms are considered “further + restrictions” within the meaning of section 10. If the Program as you + received it, or any part of it, contains a notice stating that it is + governed by this License along with a term that is a further restriction, + you may remove that term. If a license document contains a further + restriction but permits relicensing or conveying under this License, you + may add to a covered work material governed by the terms of that license + document, provided that the further restriction does not survive such + relicensing or conveying. + + If you add terms to a covered work in accord with this section, you must + place, in the relevant source files, a statement of the additional terms + that apply to those files, or a notice indicating where to find the + applicable terms. Additional terms, permissive or non-permissive, may be + stated in the form of a separately written license, or stated as + exceptions; the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly + provided under this License. Any attempt otherwise to propagate or modify + it is void, and will automatically terminate your rights under this + License (including any patent licenses granted under the third paragraph + of section 11). + + However, if you cease all violation of this License, then your license + from a particular copyright holder is reinstated (a) provisionally, + unless and until the copyright holder explicitly and finally terminates + your license, and (b) permanently, if the copyright holder fails to + notify you of the violation by some reasonable means prior to 60 days + after the cessation. + + Moreover, your license from a particular copyright holder is reinstated + permanently if the copyright holder notifies you of the violation by some + reasonable means, this is the first time you have received notice of + violation of this License (for any work) from that copyright holder, and + you cure the violation prior to 30 days after your receipt of the notice. + + Termination of your rights under this section does not terminate the + licenses of parties who have received copies or rights from you under + this License. If your rights have been terminated and not permanently + reinstated, you do not qualify to receive new licenses for the same + material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or run a + copy of the Program. Ancillary propagation of a covered work occurring + solely as a consequence of using peer-to-peer transmission to receive a + copy likewise does not require acceptance. However, nothing other than + this License grants you permission to propagate or modify any covered + work. These actions infringe copyright if you do not accept this License. + Therefore, by modifying or propagating a covered work, you indicate your + acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically receives + a license from the original licensors, to run, modify and propagate that + work, subject to this License. You are not responsible for enforcing + compliance by third parties with this License. + + An “entity transaction” is a transaction transferring control of an + organization, or substantially all assets of one, or subdividing an + organization, or merging organizations. If propagation of a covered work + results from an entity transaction, each party to that transaction who + receives a copy of the work also receives whatever licenses to the work + the party's predecessor in interest had or could give under the previous + paragraph, plus a right to possession of the Corresponding Source of the + work from the predecessor in interest, if the predecessor has it or can + get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the rights + granted or affirmed under this License. For example, you may not impose a + license fee, royalty, or other charge for exercise of rights granted + under this License, and you may not initiate litigation (including a + cross-claim or counterclaim in a lawsuit) alleging that any patent claim + is infringed by making, using, selling, offering for sale, or importing + the Program or any portion of it. + + 11. Patents. + + A “contributor” is a copyright holder who authorizes use under this + License of the Program or a work on which the Program is based. The work + thus licensed is called the contributor's “contributor version”. + + A contributor's “essential patent claims” are all patent claims owned or + controlled by the contributor, whether already acquired or hereafter + acquired, that would be infringed by some manner, permitted by this + License, of making, using, or selling its contributor version, but do not + include claims that would be infringed only as a consequence of further + modification of the contributor version. For purposes of this definition, + “control” includes the right to grant patent sublicenses in a manner + consistent with the requirements of this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free + patent license under the contributor's essential patent claims, to make, + use, sell, offer for sale, import and otherwise run, modify and propagate + the contents of its contributor version. + + In the following three paragraphs, a “patent license” is any express + agreement or commitment, however denominated, not to enforce a patent + (such as an express permission to practice a patent or covenant not to + sue for patent infringement). To “grant” such a patent license to a party + means to make such an agreement or commitment not to enforce a patent + against the party. + + If you convey a covered work, knowingly relying on a patent license, and + the Corresponding Source of the work is not available for anyone to copy, + free of charge and under the terms of this License, through a publicly + available network server or other readily accessible means, then you must + either (1) cause the Corresponding Source to be so available, or (2) + arrange to deprive yourself of the benefit of the patent license for this + particular work, or (3) arrange, in a manner consistent with the + requirements of this License, to extend the patent license to downstream + recipients. “Knowingly relying” means you have actual knowledge that, but + for the patent license, your conveying the covered work in a country, or + your recipient's use of the covered work in a country, would infringe + one or more identifiable patents in that country that you have reason + to believe are valid. + + If, pursuant to or in connection with a single transaction or + arrangement, you convey, or propagate by procuring conveyance of, a + covered work, and grant a patent license to some of the parties receiving + the covered work authorizing them to use, propagate, modify or convey a + specific copy of the covered work, then the patent license you grant is + automatically extended to all recipients of the covered work and works + based on it. + + A patent license is “discriminatory” if it does not include within the + scope of its coverage, prohibits the exercise of, or is conditioned on + the non-exercise of one or more of the rights that are specifically + granted under this License. You may not convey a covered work if you are + a party to an arrangement with a third party that is in the business of + distributing software, under which you make payment to the third party + based on the extent of your activity of conveying the work, and under + which the third party grants, to any of the parties who would receive the + covered work from you, a discriminatory patent license (a) in connection + with copies of the covered work conveyed by you (or copies made from + those copies), or (b) primarily for and in connection with specific + products or compilations that contain the covered work, unless you + entered into that arrangement, or that patent license was granted, prior + to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting any + implied license or other defenses to infringement that may otherwise be + available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or + otherwise) that contradict the conditions of this License, they do not + excuse you from the conditions of this License. If you cannot use, + propagate or convey a covered work so as to satisfy simultaneously your + obligations under this License and any other pertinent obligations, then + as a consequence you may not use, propagate or convey it at all. For + example, if you agree to terms that obligate you to collect a royalty for + further conveying from those to whom you convey the Program, the only way + you could satisfy both those terms and this License would be to refrain + entirely from conveying the Program. + + 13. Offering the Program as a Service. + + If you make the functionality of the Program or a modified version + available to third parties as a service, you must make the Service Source + Code available via network download to everyone at no charge, under the + terms of this License. Making the functionality of the Program or + modified version available to third parties as a service includes, + without limitation, enabling third parties to interact with the + functionality of the Program or modified version remotely through a + computer network, offering a service the value of which entirely or + primarily derives from the value of the Program or modified version, or + offering a service that accomplishes for users the primary purpose of the + Program or modified version. + + “Service Source Code” means the Corresponding Source for the Program or + the modified version, and the Corresponding Source for all programs that + you use to make the Program or modified version available as a service, + including, without limitation, management software, user interfaces, + application program interfaces, automation software, monitoring software, + backup software, storage software and hosting software, all such that a + user could run an instance of the service using the Service Source Code + you make available. + + 14. Revised Versions of this License. + + MongoDB, Inc. may publish revised and/or new versions of the Server Side + Public License from time to time. Such new versions will be similar in + spirit to the present version, but may differ in detail to address new + problems or concerns. + + Each version is given a distinguishing version number. If the Program + specifies that a certain numbered version of the Server Side Public + License “or any later version” applies to it, you have the option of + following the terms and conditions either of that numbered version or of + any later version published by MongoDB, Inc. If the Program does not + specify a version number of the Server Side Public License, you may + choose any version ever published by MongoDB, Inc. + + If the Program specifies that a proxy can decide which future versions of + the Server Side Public License can be used, that proxy's public statement + of acceptance of a version permanently authorizes you to choose that + version for the Program. + + Later license versions may give you additional or different permissions. + However, no additional obligations are imposed on any author or copyright + holder as a result of your choosing to follow a later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY + APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT + HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY + OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM + IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF + ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING + WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS + THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING + ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF + THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO + LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU + OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER + PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE + POSSIBILITY OF SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided above + cannot be given local legal effect according to their terms, reviewing + courts shall apply local law that most closely approximates an absolute + waiver of all civil liability in connection with the Program, unless a + warranty or assumption of liability accompanies a copy of the Program in + return for a fee. + + END OF TERMS AND CONDITIONS diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/MPL-2 b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/MPL-2 new file mode 100644 index 00000000..197b2ffd --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/MPL-2 @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/README b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/README new file mode 100644 index 00000000..97f1dc72 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/README @@ -0,0 +1,87 @@ +MongoDB README + +Welcome to MongoDB! + +COMPONENTS + + mongod - The database server. + mongos - Sharding router. + mongo - The database shell (uses interactive javascript). + +UTILITIES + + install_compass - Installs MongoDB Compass for your platform. + +BUILDING + + See docs/building.md. + +RUNNING + + For command line options invoke: + + $ ./mongod --help + + To run a single server database: + + $ sudo mkdir -p /data/db + $ ./mongod + $ + $ # The mongo javascript shell connects to localhost and test database by default: + $ ./mongo + > help + +INSTALLING COMPASS + + You can install compass using the install_compass script packaged with MongoDB: + + $ ./install_compass + + This will download the appropriate MongoDB Compass package for your platform + and install it. + +DRIVERS + + Client drivers for most programming languages are available at + https://docs.mongodb.com/manual/applications/drivers/. Use the shell + ("mongo") for administrative tasks. + +BUG REPORTS + + See https://github.com/mongodb/mongo/wiki/Submit-Bug-Reports. + +PACKAGING + + Packages are created dynamically by the package.py script located in the + buildscripts directory. This will generate RPM and Debian packages. + +DOCUMENTATION + + https://docs.mongodb.com/manual/ + +CLOUD HOSTED MONGODB + + https://www.mongodb.com/cloud/atlas + +FORUMS + + https://community.mongodb.com + + A forum for technical questions about using MongoDB. + + https://community.mongodb.com/c/server-dev + + A forum for technical questions about building and developing MongoDB. + +LEARN MONGODB + + https://university.mongodb.com/ + +LICENSE + + MongoDB is free and open-source. Versions released prior to October 16, + 2018 are published under the AGPL. All versions released after October + 16, 2018, including patch fixes for prior versions, are published under + the Server Side Public License (SSPL) v1. See individual files for + details. + diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..8c9e17e3 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/community-server/THIRD-PARTY-NOTICES @@ -0,0 +1,1568 @@ +MongoDB uses third-party libraries or other resources that may +be distributed under licenses different than the MongoDB software. + +In the event that we accidentally failed to list a required notice, +please bring it to our attention through any of the ways detailed here : + + mongodb-dev@googlegroups.com + +The attached notices are provided for information only. + +For any licenses that require disclosure of source, sources are available at +https://github.com/mongodb/mongo. + + +1) License Notice for Boost +--------------------------- + +http://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +3) License Notice for PCRE +-------------------------- + +http://www.pcre.org/licence.txt + +PCRE LICENCE +------------ + +PCRE is a library of functions to support regular expressions whose syntax +and semantics are as close as possible to those of the Perl 5 language. + +Release 7 of PCRE is distributed under the terms of the "BSD" licence, as +specified below. The documentation for PCRE, supplied in the "doc" +directory, is distributed under the same terms as the software itself. + +The basic library functions are written in C and are freestanding. Also +included in the distribution is a set of C++ wrapper functions. + + +THE BASIC LIBRARY FUNCTIONS +--------------------------- + +Written by: Philip Hazel +Email local part: ph10 +Email domain: cam.ac.uk + +University of Cambridge Computing Service, +Cambridge, England. + +Copyright (c) 1997-2008 University of Cambridge +All rights reserved. + + +THE C++ WRAPPER FUNCTIONS +------------------------- + +Contributed by: Google Inc. + +Copyright (c) 2007-2008, Google Inc. +All rights reserved. + + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +4) License notice for Aladdin MD5 +--------------------------------- + +Copyright (C) 1999, 2002 Aladdin Enterprises. All rights reserved. + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +L. Peter Deutsch +ghost@aladdin.com + +5) License notice for Snappy - http://code.google.com/p/snappy/ +--------------------------------- + Copyright 2005 and onwards Google Inc. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + A light-weight compression algorithm. It is designed for speed of + compression and decompression, rather than for the utmost in space + savings. + + For getting better compression ratios when you are compressing data + with long repeated sequences or compressing data that is similar to + other data, while still compressing fast, you might look at first + using BMDiff and then compressing the output of BMDiff with + Snappy. + +6) License notice for Google Perftools (TCMalloc utility) +--------------------------------- +New BSD License + +Copyright (c) 1998-2006, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the following + disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +7) License notice for Linenoise +------------------------------- + + Copyright (c) 2010, Salvatore Sanfilippo + Copyright (c) 2010, Pieter Noordhuis + + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of Redis nor the names of its contributors may be used + to endorse or promote products derived from this software without + specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +8) License notice for S2 Geometry Library +----------------------------------------- + Copyright 2005 Google Inc. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +9) License notice for MurmurHash +-------------------------------- + + Copyright (c) 2010-2012 Austin Appleby + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + +10) License notice for Snowball + Copyright (c) 2001, Dr Martin Porter + All rights reserved. + +THE "BSD" LICENCE +----------------- + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +11) License notice for yaml-cpp +------------------------------- + +Copyright (c) 2008 Jesse Beder. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +12) License notice for zlib +--------------------------- + +http://www.zlib.net/zlib_license.html + +zlib.h -- interface of the 'zlib' general purpose compression library +version 1.2.8, April 28th, 2013 + +Copyright (C) 1995-2013 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. + +Jean-loup Gailly Mark Adler +jloup@gzip.org madler@alumni.caltech.edu + + +13) License notice for 3rd party software included in the WiredTiger library +---------------------------------------------------------------------------- + +http://source.wiredtiger.com/license.html + +WiredTiger Distribution Files | Copyright Holder | License +----------------------------- | ----------------------------------- | ---------------------- +src/include/bitstring.i | University of California, Berkeley | BSD-3-Clause License +src/include/queue.h | University of California, Berkeley | BSD-3-Clause License +src/os_posix/os_getopt.c | University of California, Berkeley | BSD-3-Clause License +src/support/hash_city.c | Google, Inc. | The MIT License +src/support/hash_fnv.c | Authors | Public Domain + + +Other optional 3rd party software included in the WiredTiger distribution is removed by MongoDB. + + +BSD-3-CLAUSE LICENSE +-------------------- + +http://www.opensource.org/licenses/BSD-3-Clause + +Copyright (c) 1987, 1989, 1991, 1993, 1994 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + + +THE MIT LICENSE +--------------- + +http://www.opensource.org/licenses/MIT + +Copyright (c) 2011 Google, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +14) License Notice for SpiderMonkey +----------------------------------- + +|------------------------------------------------|------------------|---------------| +| SpiderMonkey Distribution Files | Copyright Holder | License | +|------------------------------------------------|------------------|---------------| +| js/src/jit/shared/AssemblerBuffer-x86-shared.h | Apple, Inc | BSD-2-Clause | +| js/src/jit/shared/BaseAssembler-x86-shared.h | | | +|------------------------------------------------|------------------|---------------| +| js/src/builtin/ | Google, Inc | BSD-3-Clause | +| js/src/irregexp/ | | | +| js/src/jit/arm/ | | | +| js/src/jit/mips/ | | | +| mfbt/double-conversion/ | | | +|------------------------------------------------|------------------|---------------| +| intl/icu/source/common/unicode/ | IBM, Inc | ICU | +|------------------------------------------------|------------------|---------------| +| js/src/asmjs/ | Mozilla, Inc | Apache2 | +|------------------------------------------------|------------------|---------------| +| js/public/ | Mozilla, Inc | MPL2 | +| js/src/ | | | +| mfbt | | | +|------------------------------------------------|------------------|---------------| +| js/src/vm/Unicode.cpp | None | Public Domain | +|------------------------------------------------|------------------|---------------| +| mfbt/lz4.c | Yann Collet | BSD-2-Clause | +| mfbt/lz4.h | | | +|------------------------------------------------|------------------|---------------| + +Other optional 3rd party software included in the SpiderMonkey distribution is removed by MongoDB. + + +Apple, Inc: BSD-2-Clause +------------------------ + +Copyright (C) 2008 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Google, Inc: BSD-3-Clause +------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ICU License - ICU 1.8.1 and later +--------------------------------- + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2012 International Business Machines Corporation and +others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, provided that the above copyright notice(s) and this +permission notice appear in all copies of the Software and that both the +above copyright notice(s) and this permission notice appear in supporting +documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS +SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in this Software without prior written authorization of the +copyright holder. + +All trademarks and registered trademarks mentioned herein are the property +of their respective owners. + + +Mozilla, Inc: Apache 2 +---------------------- + +Copyright 2014 Mozilla Foundation + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Mozilla, Inc: MPL 2 +------------------- + +Copyright 2014 Mozilla Foundation + +This Source Code Form is subject to the terms of the Mozilla Public +License, v. 2.0. If a copy of the MPL was not distributed with this +file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +Public Domain +------------- + +Any copyright is dedicated to the Public Domain. +http://creativecommons.org/licenses/publicdomain/ + + +LZ4: BSD-2-Clause +----------------- + +Copyright (C) 2011-2014, Yann Collet. +BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at : +- LZ4 source repository : http://code.google.com/p/lz4/ +- LZ4 public forum : https://groups.google.com/forum/#!forum/lz4c + +15) License Notice for Intel DFP Math Library +--------------------------------------------- + +Copyright (c) 2011, Intel Corp. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + his list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of Intel Corporation nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +16) License Notice for Unicode Data +----------------------------------- + +Copyright © 1991-2015 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +17 ) License Notice for Valgrind.h +---------------------------------- + +---------------------------------------------------------------- + +Notice that the following BSD-style license applies to this one +file (valgrind.h) only. The rest of Valgrind is licensed under the +terms of the GNU General Public License, version 2, unless +otherwise indicated. See the COPYING file in the source +distribution for details. + +---------------------------------------------------------------- + +This file is part of Valgrind, a dynamic binary instrumentation +framework. + +Copyright (C) 2000-2015 Julian Seward. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------- + +Notice that the above BSD-style license applies to this one file +(valgrind.h) only. The entire rest of Valgrind is licensed under +the terms of the GNU General Public License, version 2. See the +COPYING file in the source distribution for details. + +---------------------------------------------------------------- + +18) License notice for ICU4C +---------------------------- + +ICU License - ICU 1.8.1 and later + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +--------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +1. Unicode Data Files and Software + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2016 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in +http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Unicode data files and any associated documentation +(the "Data Files") or Unicode software and any associated documentation +(the "Software") to deal in the Data Files or Software +without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, and/or sell copies of +the Data Files or Software, and to permit persons to whom the Data Files +or Software are furnished to do so, provided that +(a) this copyright and permission notice appear with all copies +of the Data Files or Software, +(b) this copyright and permission notice appear in associated +documentation, and +(c) there is clear notice in each modified Data File or in the Software +as well as in the documentation associated with the Data File(s) or +Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT OF THIRD PARTY RIGHTS. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS +NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL +DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, +DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, +use or other dealings in these Data Files or Software without prior +written authorization of the copyright holder. + +2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyrighy (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +3. Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (c) 2013 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: http://code.google.com/p/lao-dictionary/ + # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt + # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary, with slight + # modifications. + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, + # are permitted provided that the following conditions are met: + # + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in + # binary form must reproduce the above copyright notice, this list of + # conditions and the following disclaimer in the documentation and/or + # other materials provided with the distribution. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +4. Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +5. Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +19) License notice for timelib +------------------------------ + +The MIT License (MIT) + +Copyright (c) 2015-2017 Derick Rethans +Copyright (c) 2017 MongoDB, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +20) License notice for windows dirent implementation +---------------------------------------------------- + + * Dirent interface for Microsoft Visual Studio + * Version 1.21 + * + * Copyright (C) 2006-2012 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + + + 21) License notice for abseil-cpp +---------------------------- + + Copyright (c) Google Inc. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + 22) License notice for Zstandard +---------------------------- + + BSD License + + For Zstandard software + + Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without modification, + are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 23) License notice for ASIO +---------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 24) License notice for MPark.Variant +------------------------------------- +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + 25) License notice for fmt +--------------------------- + +Copyright (c) 2012 - present, Victor Zverovich +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted +provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or other + materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + 26) License notice for SafeInt +--------------------------- + +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under the MIT License. + +MIT License + +Copyright (c) 2018 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + 27) License Notice for Raft TLA+ Specification +----------------------------------------------- + +https://github.com/ongardie/dissertation/blob/master/LICENSE + +Copyright 2014 Diego Ongaro. + +Some of our TLA+ specifications are based on the Raft TLA+ specification by Diego Ongaro. + +End diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md new file mode 100644 index 00000000..550ae4ed --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/LICENSE.md @@ -0,0 +1,13 @@ +Copyright 2014 MongoDB, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/README.md b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/README.md new file mode 100644 index 00000000..e60e0cd0 --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/README.md @@ -0,0 +1,72 @@ +MongoDB Tools +=================================== + + - **bsondump** - _display BSON files in a human-readable format_ + - **mongoimport** - _Convert data from JSON, TSV or CSV and insert them into a collection_ + - **mongoexport** - _Write an existing collection to CSV or JSON format_ + - **mongodump/mongorestore** - _Dump MongoDB backups to disk in .BSON format, or restore them to a live database_ + - **mongostat** - _Monitor live MongoDB servers, replica sets, or sharded clusters_ + - **mongofiles** - _Read, write, delete, or update files in [GridFS](http://docs.mongodb.org/manual/core/gridfs/)_ + - **mongotop** - _Monitor read/write activity on a mongo server_ + + +Report any bugs, improvements, or new feature requests at https://jira.mongodb.org/browse/TOOLS + +Building Tools +--------------- + +We currently build the tools with Go version 1.15. Other Go versions may work but they are untested. + +Using `go get` to directly build the tools will not work. To build them, it's recommended to first clone this repository: + +``` +git clone https://github.com/mongodb/mongo-tools +cd mongo-tools +``` + +Then run `./make build` to build all the tools, placing them in the `bin` directory inside the repository. + +You can also build a subset of the tools using the `-tools` option. For example, `./make build -tools=mongodump,mongorestore` builds only `mongodump` and `mongorestore`. + +To use the build/test scripts in this repository, you **_must_** set GOROOT to your Go root directory. This may depend on how you installed Go. + +``` +export GOROOT=/usr/local/go +``` + +Updating Dependencies +--------------- +Starting with version 100.3.1, the tools use `go mod` to manage dependencies. All dependencies are listed in the `go.mod` file and are directly vendored in the `vendor` directory. + +In order to make changes to dependencies, you first need to change the `go.mod` file. You can manually edit that file to add/update/remove entries, or you can run the following in the repository directory: + +``` +go mod edit -require=@ # for adding or updating a dependency +go mod edit -droprequire= # for removing a dependency +``` + +Then run `go mod vendor -v` to reconstruct the `vendor` directory to match the changed `go.mod` file. + +Optionally, run `go mod tidy -v` to ensure that the `go.mod` file matches the `mongo-tools` source code. + +Contributing +--------------- +See our [Contributor's Guide](CONTRIBUTING.md). + +Documentation +--------------- +See the MongoDB packages [documentation](https://docs.mongodb.org/database-tools/). + +For documentation on older versions of the MongoDB, reference that version of the [MongoDB Server Manual](docs.mongodb.com/manual): + +- [MongoDB 4.2 Tools](https://docs.mongodb.org/v4.2/reference/program) +- [MongoDB 4.0 Tools](https://docs.mongodb.org/v4.0/reference/program) +- [MongoDB 3.6 Tools](https://docs.mongodb.org/v3.6/reference/program) + +Adding New Platforms Support +--------------- +See our [Adding New Platform Support Guide](PLATFORMSUPPORT.md). + +Vendoring the Change into Server Repo +--------------- +See our [Vendor the Change into Server Repo](SERVERVENDORING.md). diff --git a/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES new file mode 100644 index 00000000..3d75e64b --- /dev/null +++ b/Mongo2Go.4.1.0/tools/mongodb-windows-4.4.4-database-tools-100.3.1/database-tools/THIRD-PARTY-NOTICES @@ -0,0 +1,3319 @@ +--------------------------------------------------------------------- +License notice for hashicorp/go-rootcerts +--------------------------------------------------------------------- + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +--------------------------------------------------------------------- +License notice for JSON and CSV code from github.com/golang/go +--------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/escaper +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Lucas Morales + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/llmgo/bson +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/10gen/openssl +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/3rf/mongo-lint +---------------------------------------------------------------------- + +Copyright (c) 2013 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/go-stack/stack +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/golang/snappy +---------------------------------------------------------------------- + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/google/gopacket +---------------------------------------------------------------------- + +Copyright (c) 2012 Google, Inc. All rights reserved. +Copyright (c) 2009-2011 Andreas Krennmair. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Andreas Krennmair, Google, nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/gopherjs/gopherjs +---------------------------------------------------------------------- + +Copyright (c) 2013 Richard Musiol. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/howeyc/gopass +---------------------------------------------------------------------- + +Copyright (c) 2012 Chris Howey + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/jessevdk/go-flags +---------------------------------------------------------------------- + +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/jtolds/gls +---------------------------------------------------------------------- + +Copyright (c) 2013, Space Monkey, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mattn/go-runewidth +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/mongodb/mongo-go-driver +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/nsf/termbox-go +---------------------------------------------------------------------- + +Copyright (C) 2012 termbox-go authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/patrickmn/go-cache +---------------------------------------------------------------------- + +Copyright (c) 2012-2015 Patrick Mylund Nielsen and the go-cache contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions +---------------------------------------------------------------------- + +Copyright (c) 2015 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/go-render +---------------------------------------------------------------------- + +// Copyright (c) 2015 The Chromium Authors. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglematchers +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/oglemock +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/ogletest +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/assertions/internal/reqtrace +---------------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey +---------------------------------------------------------------------- + +Copyright (c) 2014 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + +---------------------------------------------------------------------- +License notice for github.com/smartystreets/goconvey/web/client/resources/fonts/Open_Sans +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/spacemonkeygo/spacelog +---------------------------------------------------------------------- + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +---------------------------------------------------------------------- +License notice for github.com/xdg/scram +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/xdg/stringprep +---------------------------------------------------------------------- + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +---------------------------------------------------------------------- +License notice for github.com/youmark/pkcs8 +---------------------------------------------------------------------- + +The MIT License (MIT) + +Copyright (c) 2014 youmark + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- +License notice for golang.org/x/crypto +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/sync +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for golang.org/x/text +---------------------------------------------------------------------- + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- +License notice for gopkg.in/tomb.v2 +---------------------------------------------------------------------- + +tomb - support for clean goroutine termination in Go. + +Copyright (c) 2010-2011 - Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 08be7469..2a6184b8 100755 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ -# StellaOps Feedser & CLI +# StellaOps Concelier & CLI -This repository hosts the StellaOps Feedser service, its plug-in ecosystem, and the -first-party CLI (`stellaops-cli`). Feedser ingests vulnerability advisories from +This repository hosts the StellaOps Concelier service, its plug-in ecosystem, and the +first-party CLI (`stellaops-cli`). Concelier ingests vulnerability advisories from authoritative sources, stores them in MongoDB, and exports deterministic JSON and Trivy DB artefacts. The CLI drives scanner distribution, scan execution, and job -control against the Feedser API. +control against the Concelier API. ## Quickstart 1. Prepare a MongoDB instance and (optionally) install `trivy-db`/`oras`. -2. Copy `etc/feedser.yaml.sample` to `etc/feedser.yaml` and update the storage + telemetry +2. Copy `etc/concelier.yaml.sample` to `etc/concelier.yaml` and update the storage + telemetry settings. 3. Copy `etc/authority.yaml.sample` to `etc/authority.yaml`, review the issuer, token lifetimes, and plug-in descriptors, then edit the companion manifests under `etc/authority.plugins/*.yaml` to match your deployment. -4. Start the web service with `dotnet run --project src/StellaOps.Feedser.WebService`. +4. Start the web service with `dotnet run --project src/StellaOps.Concelier.WebService`. 5. Configure the CLI via environment variables (e.g. `STELLAOPS_BACKEND_URL`) and trigger jobs with `dotnet run --project src/StellaOps.Cli -- db merge`. -Detailed operator guidance is available in `docs/10_FEEDSER_CLI_QUICKSTART.md`. API and +Detailed operator guidance is available in `docs/10_CONCELIER_CLI_QUICKSTART.md`. API and command reference material lives in `docs/09_API_CLI_REFERENCE.md`. -Pipeline note: deployment workflows should template `etc/feedser.yaml` during CI/CD, +Pipeline note: deployment workflows should template `etc/concelier.yaml` during CI/CD, injecting environment-specific Mongo credentials and telemetry endpoints. Upcoming releases will add Microsoft OAuth (Entra ID) authentication support—track the quickstart for integration steps once available. @@ -29,6 +29,6 @@ for integration steps once available. ## Documentation - `docs/README.md` now consolidates the platform index and points to the updated high-level architecture. -- Module architecture dossiers live under `docs/ARCHITECTURE_*.md`; the most relevant here are `docs/ARCHITECTURE_FEEDSER.md` (service layout, merge engine, exports) and `docs/ARCHITECTURE_CLI.md` (command surface, AOT packaging, auth flows). Related services such as the Signer, Attestor, Authority, Scanner, UI, Vexer, Zastava, and DevOps pipeline each have their own dossier. -- Offline operation guidance moved to `docs/24_OFFLINE_KIT.md`, which details bundle composition, verification, and delta workflows. Feedser-specific connector operations stay in `docs/ops/feedser-certbund-operations.md` and companion runbooks under `docs/ops/`. +- Module architecture dossiers live under `docs/ARCHITECTURE_*.md`; the most relevant here are `docs/ARCHITECTURE_CONCELIER.md` (service layout, merge engine, exports) and `docs/ARCHITECTURE_CLI.md` (command surface, AOT packaging, auth flows). Related services such as the Signer, Attestor, Authority, Scanner, UI, Excititor, Zastava, and DevOps pipeline each have their own dossier. +- Offline operation guidance moved to `docs/24_OFFLINE_KIT.md`, which details bundle composition, verification, and delta workflows. Concelier-specific connector operations stay in `docs/ops/concelier-certbund-operations.md` and companion runbooks under `docs/ops/`. diff --git a/SPRINTS.md b/SPRINTS.md index 9365279e..2e91487e 100644 --- a/SPRINTS.md +++ b/SPRINTS.md @@ -1,52 +1,54 @@ +This file describe implementation of Stella Ops (docs/README.md). Implementation must respect rules from AGENTS.md (read if you have not). + | Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | | --- | --- | --- | --- | --- | --- | --- | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md for usage guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `feedser.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Feedser.Source.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Feedser.Source.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Feedser WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Feedser.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Feedser operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Feedser.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | -| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Feedser.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Feedser.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs now point at `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`; defaults/docs/tests updated to reflect the new layout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | @@ -54,103 +56,293 @@ | Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | | Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | | Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Feedser.Source.Osv.Tests`, `src/StellaOps.Feedser.Source.Ghsa.Tests`, `src/StellaOps.Feedser.Source.Nvd.Tests`, and backbone normalization/storage suites. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/feedser-cccs-operations.md` with fixtures validating EN/FR list handling. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/feedser-certbund-operations.md` captures locale guidance and offline packaging. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/feedser-msrc-operations.md`. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | -| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Feedser.Source.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | | Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | -| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Feedser-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | | Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | -| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Feedser Conflict Rules
Runbook published at `docs/ops/feedser-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | -| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/feedser-conflict-resolution.md`; task closed after connector signals verified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Feedser.Exporter.Json.Tests` validated canonical metric/CWE emission. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Feedser.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | -| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Feedser.Source.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Core/TASKS.md | DONE (2025-10-15) | Team Vexer Core & Policy | VEXER-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-15) | Team Vexer Policy | VEXER-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-15) | Team Vexer Policy | VEXER-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-16) | Team Vexer Policy | VEXER-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-16) | Team Vexer Policy | VEXER-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Policy/TASKS.md | DONE (2025-10-16) | Team Vexer Policy | VEXER-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Vexer Storage | VEXER-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Vexer Storage | VEXER-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Export/TASKS.md | DONE (2025-10-15) | Team Vexer Export | VEXER-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Export/TASKS.md | DONE (2025-10-17) | Team Vexer Export | VEXER-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | DONE (2025-10-16) | Team Vexer Attestation | VEXER-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors | VEXER-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | -| Sprint 5 | Vexer Core Foundations | src/StellaOps.Vexer.WebService/TASKS.md | DONE (2025-10-17) | Team Vexer WebService | VEXER-WEB-01-001 | Scaffold minimal API host, DI, and `/vexer/status` endpoint integrating policy, storage, export, and attestation services. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Worker/TASKS.md | DONE (2025-10-17) | Team Vexer Worker | VEXER-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Vexer Formats | VEXER-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Red Hat | VEXER-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Cisco | VEXER-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Cisco | VEXER-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – SUSE | VEXER-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – MSRC | VEXER-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Oracle | VEXER-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Vexer Connectors – Ubuntu | VEXER-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/TASKS.md | TODO | Team Vexer Connectors – OCI | VEXER-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | -| Sprint 6 | Vexer Ingest & Formats | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | VEXER-CLI-01-001 | Add `vexer` CLI verbs bridging to WebService with consistent auth and offline UX. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Core/TASKS.md | TODO | Team Vexer Core & Policy | VEXER-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Policy/TASKS.md | TODO | Team Vexer Policy | VEXER-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | TODO | Team Vexer Storage | VEXER-STORAGE-02-001 | Statement events & scoring signals – create immutable VEX statement store plus consensus extensions with indexes/migrations. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.WebService/TASKS.md | TODO | Team Vexer WebService | VEXER-WEB-01-004 | Resolve API & signed responses – expose `/vexer/resolve`, return signed consensus/score envelopes, document auth. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Attestation/TASKS.md | DONE (2025-10-16) | Team Vexer Attestation | VEXER-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Worker/TASKS.md | TODO | Team Vexer Worker | VEXER-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Vexer.Export/TASKS.md | TODO | Team Vexer Export | VEXER-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Feedser.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Feedser.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections – provision Mongo schema/indexes for event-sourced merge. | -| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Feedser.Merge/TASKS.md | TODO | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Feedser.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Feedser storage sessions
Ensure `AddMongoStorage` registers a scoped session facilitator (causal consistency + majority concerns), update repositories to accept optional session handles, and add integration coverage proving read-your-write and monotonic reads across a replica set/election scenario. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Introduce scoped MongoDB sessions with `writeConcern`/`readConcern` majority defaults, flow the session through stores used in mutations + follow-up reads, and document middleware pattern for web/API & GraphQL layers. | -| Sprint 8 | Mongo strengthening | src/StellaOps.Vexer.Storage.Mongo/TASKS.md | TODO | Team Vexer Storage | VEXER-STORAGE-MONGO-08-001 | Causal consistency for Vexer repositories
Register Mongo options with majority defaults, push session-aware overloads through raw/export/consensus/cache stores, and extend migration/tests to validate causal reads after writes (including GridFS-backed content) under replica-set failover. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals – immutable VEX statements store, consensus signal fields, and migration `20251019-consensus-signals-statements` with tests (`dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`, `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`). | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-005 | Mirror distribution endpoints – expose download APIs for downstream Excititor instances. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-006 | Quiet provenance packaging – attach quieted-by statement IDs, signers, justification codes to exports and attestations. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-007 | Mirror bundle + domain manifest – publish signed consensus bundles for mirrors. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-001 | Excititor mirror connector – ingest signed mirror bundles and map to VexClaims with resume handling. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-19) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Concelier WebService Guild | FEEDWEB-EVENTS-07-001 | Advisory event replay API – expose `/concelier/advisories/{key}/replay` with `asOf` filter, hex hashes, and conflict data. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-003 | Unknown state ledger & confidence seeding – persist unknown flags, seed confidence bands, expose query surface. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections – provision Mongo schema/indexes for event-sourced merge. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | DOING | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions
Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | DONE (2025-10-19) | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories
Session-scoped repositories shipped with new Mongo records, orchestrators/workers now share scoped sessions, and replica-set failover coverage added via `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`. | +| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-03-001 | Statement backfill tooling – shipped admin backfill endpoint, CLI hook (`stellaops excititor backfill-statements`), integration tests, and operator runbook (`docs/dev/EXCITITOR_STATEMENT_BACKFILL.md`). | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-201 | Mirror bundle + domain manifest – produce signed JSON aggregates for `*.stella-ops.org` mirrors. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-202 | Mirror-ready Trivy DB bundles – mirror options emit per-domain manifests/metadata/db archives with deterministic digests for downstream sync. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.WebService/TASKS.md | DOING (2025-10-19) | Concelier WebService Guild | CONCELIER-WEB-08-201 | Mirror distribution endpoints – expose domain-scoped index/download APIs with auth/quota. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | DOING (2025-10-19) | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Concelier mirror connector – fetch mirror manifest, verify signatures, and hydrate canonical DTOs with resume support. | +| Sprint 8 | Mirror Distribution | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-MIRROR-08-001 | Managed mirror deployments for `*.stella-ops.org` – Helm/Compose overlays, CDN, runbooks. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | DOING | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002.COORD | Authority scoped-service integration handshake
Session scheduled for 2025-10-20 15:00–16:00 UTC; agenda + attendees logged in `docs/dev/authority-plugin-di-coordination.md`. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Authority/TASKS.md | DOING | Authority Core, Plugin Platform Guild | AUTH-PLUGIN-COORD-08-002 | Coordinate scoped-service adoption for Authority plug-in registrars
Workshop locked for 2025-10-20 15:00–16:00 UTC; pre-read checklist tracked in `docs/dev/authority-plugin-di-coordination.md`. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-WEB-09-103 | Progress streaming (SSE/JSONL) with correlation IDs and ISO-8601 UTC timestamps, documented in API reference. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-105 | Policy snapshot loader + schema + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-106 | `/reports` verdict assembly (Feedser+Vexer+Policy) + signed response envelope. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Team Scanner WebService | SCANNER-POLICY-09-107 | Expose score inputs, config version, and quiet provenance in `/reports` JSON and signed payload. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-004 | Versioned scoring config with schema validation, trust table, and golden fixtures. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-005 | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-006 | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | TODO | Scanner Cache Guild | SCANNER-CACHE-10-104 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-201 | Alpine/apk analyzer emitting deterministic components with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-202 | Debian/dpkg analyzer mapping packages to purl identity with evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-203 | RPM analyzer capturing EVR, file listings, provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-204 | Shared OS evidence helpers for package identity + provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-205 | Vendor metadata enrichment (source packages, license, CVE hints). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-206 | Determinism harness + fixtures for OS analyzers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | TODO | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-207 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301 | Java analyzer emitting `pkg:maven` with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-302 | Node analyzer handling workspaces/symlinks emitting `pkg:npm`. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-303 | Python analyzer reading `*.dist-info`, RECORD hashes, entry points. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-304 | Go analyzer leveraging buildinfo for `pkg:golang` components. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-305 | .NET analyzer parsing `*.deps.json`, assembly metadata, RID variants. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-306 | Rust analyzer detecting crates or falling back to `bin:{sha256}`. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Shared language evidence helpers + usage flag propagation. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-308 | Determinism + fixture harness for language analyzers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309 | Package language analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | POSIX shell AST parser with deterministic output. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Command resolution across layered rootfs with evidence attribution. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Interpreter tracing for shell wrappers to Python/Node/Java launchers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-404 | Python entry analyzer (venv shebang, module invocation, usage flag). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-405 | Node/Java launcher analyzer capturing script/jar targets. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-406 | Explainability + diagnostics for unresolved constructs with metrics. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-502 | Attribute diffs to introducing/removing layers including provenance evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-503 | Produce JSON diff output for inventory vs usage views aligned with API contract. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-601 | Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-602 | Compose usage SBOM leveraging EntryTrace to flag actual usage. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-603 | Generate BOM index sidecar (purl table + roaring bitmap + usage flag). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-604 | Package artifacts for export + attestation with deterministic manifests. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-605 | Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-606 | Usage view bit flags integrated with EntryTrace. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-607 | Embed scoring inputs, confidence band, and quiet provenance in CycloneDX/DSSE artifacts. | +| Sprint 10 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Scanner Team | BENCH-SCANNER-10-001 | Analyzer microbench harness + baseline CSV. | +| Sprint 10 | Samples | samples/TASKS.md | TODO | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images with SBOM/BOM-Index sidecars. | +| Sprint 10 | DevOps Security | ops/devops/TASKS.md | DOING | DevOps Guild | DEVOPS-SEC-10-301 | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0; Wave 0A prerequisites confirmed complete before remediation work. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job ensuring <5 s SBOM compose. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DOING (2025-10-19) | Authority Core & Security Guild | AUTH-DPOP-11-001 | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | DOING (2025-10-19) | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-API-11-201 | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-VERIFY-11-202 | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-OBS-11-203 | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | +| Sprint 11 | UI Integration | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ATTEST-11-005 | Attestation visibility (Rekor id, status) on Scan Detail. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-201 | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-202 | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-203 | Authority client helpers, OpTok caching, and security guardrails for runtime services. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-OPS-12-204 | Operational runbooks, alert rules, and dashboard exports for runtime plane. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Container lifecycle watcher emitting deterministic runtime events with buffering. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Capture entrypoint traces + loaded libraries, hashing binaries and linking to baseline SBOM. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-003 | Posture checks for signatures/SBOM/attestation with offline caching. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-004 | Batch `/runtime/events` submissions with disk-backed buffer and rate limits. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-101 | Admission controller host with TLS bootstrap and Authority auth. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-103 | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | `/policy/runtime` endpoint joining SBOM baseline + policy verdict with TTL guidance. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AUTH-13-001 | Integrate Authority OIDC + DPoP flows with session management. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCANS-13-002 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-VEX-13-003 | Implement VEX explorer + policy editor with preview integration. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ADMIN-13-004 | Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCHED-13-005 | Scheduler panel: schedules CRUD, run history, dry-run preview. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | DOING (2025-10-19) | UI Guild | UI-NOTIFY-13-006 | Notify panel: channels/rules CRUD, deliveries view, test send. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-RUNTIME-13-005 | Add runtime policy test verbs that consume `/policy/runtime` and display verdicts. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-PLUGIN-13-007 | Package non-core CLI verbs as restart-time plug-ins (manifest + loader tests). | +| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | +| Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | +| Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | TODO | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | +| Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | TODO | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-101 | Define core Notify DTOs, validation helpers, canonical serialization. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-102 | Publish schema docs and sample payloads for Notify. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-103 | Versioning/migration helpers for rules/templates/deliveries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-201 | Mongo schemas/indexes for rules, channels, deliveries, digests, locks, audit. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-202 | Repositories with tenant scoping, soft delete, TTL, causal consistency options. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-203 | Delivery history retention and query APIs. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Bus abstraction + Redis Streams adapter with ordering/idempotency. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-402 | NATS JetStream adapter with health probes and failover. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-403 | Delivery queue with retry/dead-letter + metrics. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-301 | Rules evaluation core (filters, throttles, idempotency). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-302 | Action planner + digest coalescer. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-303 | Template rendering engine (Slack/Teams/Email/Webhook). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-304 | Test-send sandbox + preview utilities. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-101 | Minimal API host with Authority enforcement and plug-in loading. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-102 | Rules/channel/template CRUD with audit logging. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-103 | Delivery history & test-send endpoints. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-104 | Configuration binding + startup diagnostics. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-201 | Bus subscription + leasing loop with backoff. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-202 | Rules evaluation pipeline integration. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Channel dispatch orchestration with retries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-204 | Metrics/telemetry for Notify workers. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Slack connector with rate-limit aware delivery. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Slack health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Teams connector with Adaptive Cards. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Teams health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | SMTP connector with TLS + rendering. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | DKIM + health/test-send flows. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Webhook connector with signing/retries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Webhook health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-503 | Package Slack connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-603 | Package Teams connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-703 | Package Email connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DOING (2025-10-19) | Scanner WebService Guild | SCANNER-EVENTS-15-201 | Emit `scanner.report.ready` + `scanner.scan.completed` events. | +| Sprint 15 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-803 | Package Webhook connector as restart-time plug-in (manifest + host registration). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-103 | Versioning/migration helpers for schedules/runs. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit + stats materialization for UI. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Queue abstraction + Redis Streams adapter. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-402 | NATS JetStream adapter with health probes. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-403 | Dead-letter handling + metrics. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-303 | Snapshot/compaction/invalidation workflow. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DOING | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-300 | **STUB** ImpactIndex ingest/query using fixtures (to be removed by SP16 completion). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-104 | Feedser/Vexer webhook handlers with security enforcement. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Planner loop (cron/event triggers, leases, fairness). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | ImpactIndex targeting and shard planning. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Runner execution invoking Scanner analysis/content refresh. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-204 | Emit rescan/report events for Notify/UI. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-205 | Metrics/telemetry for Scheduler planners/runners. | +| Sprint 16 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Scheduler Team | BENCH-IMPACT-16-001 | ImpactIndex throughput bench + RAM profile. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-17-701 | Record GNU build-id for ELF components and surface it in SBOM/diff outputs. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-17-005 | Collect GNU build-id during runtime observation and attach it to emitted events. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-17-401 | Persist runtime build-id observations and expose them for debug-symbol correlation. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | +| Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | diff --git a/SPRINTS.updated.tmp b/SPRINTS.updated.tmp new file mode 100644 index 00000000..c01fcdd3 --- /dev/null +++ b/SPRINTS.updated.tmp @@ -0,0 +1,427 @@ +This file describe implementation of Stella Ops (docs/README.md). Implementation must respect rules from AGENTS.md (read if you have not). + +| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | +| --- | --- | --- | --- | --- | --- | --- | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC2.PLG | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`; Serilog enrichment complete, storage durability tests in flight. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC3.PLG | Ensure lockout responses carry rate-limit metadata through plugin logs/events; retry-after propagation and limiter tests underway. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DOING (2025-10-14) | Team WebService & Authority | SEC5.PLG | Address plugin-specific mitigations in threat model backlog; mitigation items tracked, docs updates pending. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | BLOCKED (2025-10-12) | Team WebService & Authority | PLG4-6.CAPABILITIES | Finalise capability metadata exposure and docs once Authority rate-limiter stream (CORE8/SEC3) is stable; awaiting dependency unblock. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | TODO | Team WebService & Authority | PLG6.DIAGRAM | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | REVIEW (2025-10-13) | Team WebService & Authority | PLG7.RFC | Socialize LDAP plugin RFC and capture guild feedback; awaiting final review sign-off and follow-up issue tracking. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DOING (2025-10-10) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Regenerating RHSA fixtures awaits remaining range provenance patches; review snapshot diffs and update docs once upstream helpers land. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DOING (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart updates are staged; awaiting Docs guild review before publishing operator guide refresh. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs now point at `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`; defaults/docs/tests updated to reflect the new layout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | BLOCKED (2025-10-10) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Roll out retry/offline knobs to deployment docs and align CLI parity once LIB5 resilience options land; unblock when library release is available and docs review completes. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | +| Sprint 9 | Sovereign Crypto Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-19) | Security Guild | SEC6.A | BouncyCastle-backed Ed25519 signing plug-in wired via `ICryptoProviderRegistry`; Scanner WebService now resolves signing through the registry; AGENTS updated to enforce plug-in rule. | +| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | +| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | +| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | TODO – Display export metadata (sha256, size, Rekor link), support optional artifact download path, and handle cache hits gracefully. | DevEx/CLI | EXCITITOR-CLI-01-002 | EXCITITOR-CLI-01-002 – Export download & attestation UX | +| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | TODO – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Excititor verbs, offline guidance, and attestation verification workflow. | Docs/CLI | EXCITITOR-CLI-01-003 | EXCITITOR-CLI-01-003 – CLI docs & examples for Excititor | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | +| Sprint 1 | Backlog | src/StellaOps.Web/TASKS.md | TODO | UX Specialist, Angular Eng | WEB1.TRIVY-SETTINGS | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/` | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DOING (2025-10-12) | Team Merge & QA Enforcement | FEEDMERGE-COORD-02-900 | Range primitives rollout coordination
Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical range primitives with provenance tags; fixtures tracked in `RANGE_PRIMITIVES_COORDINATION.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | **DONE (2025-10-11)** – Reproduced Akamai resets, drafted downgrade plan (two-stage HTTP/2 retry + relay fallback), and filed `FEEDCONN-SHARED-HTTP2-001`; module README TODO will host the per-environment knob matrix. | BE-Conn-ACSC | FEEDCONN-ACSC-02-008 | FEEDCONN-ACSC-02-008 HTTP client compatibility plan | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | REVIEW | Docs Guild, Plugin Team | DOC4.AUTH-PDG | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | +| Sprint 3 | Backlog | src/StellaOps.Concelier.Connector.Common/TASKS.md | **TODO (2025-10-15)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands. | Tools Guild, BE-Conn-MSRC | FEEDCONN-SHARED-STATE-003 | FEEDCONN-SHARED-STATE-003 Source state seeding helper | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | TODO – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests. | Team Excititor Attestation | EXCITITOR-ATTEST-01-003 | EXCITITOR-ATTEST-01-003 – Verification suite & observability | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO – Implement `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with token scope enforcement and structured run telemetry. | Team Excititor WebService | EXCITITOR-WEB-01-002 | EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO – Add `/excititor/export`, `/excititor/export/{id}`, `/excititor/export/{id}/download`, `/excititor/verify`, returning artifact + attestation metadata with cache awareness. | Team Excititor WebService | EXCITITOR-WEB-01-003 | EXCITITOR-WEB-01-003 – Export & verify endpoints | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec. | Team Excititor Worker | EXCITITOR-WORKER-01-002 | EXCITITOR-WORKER-01-002 – Resume tokens & retry policy | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios. | Team Excititor Worker | EXCITITOR-WORKER-01-003 | EXCITITOR-WORKER-01-003 – Verification & cache GC loops | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | TODO – Normalize CSAF `product_status` + `justification` values into policy-aware enums with audit diagnostics for unsupported codes. | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-002 | EXCITITOR-FMT-CSAF-01-002 – Status/justification mapping | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | TODO – Provide CSAF export writer producing deterministic documents (per vuln/product) and manifest metadata for attestation. | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-003 | EXCITITOR-FMT-CSAF-01-003 – CSAF export adapter | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | TODO – Implement helpers to reconcile component/service references against policy expectations and emit diagnostics for missing SBOM links. | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-002 | EXCITITOR-FMT-CYCLONE-01-002 – Component reference reconciliation | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | TODO – Provide exporters producing CycloneDX VEX output with canonical ordering and hash-stable manifests. | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-003 | EXCITITOR-FMT-CYCLONE-01-003 – CycloneDX export serializer | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | TODO – Add reducers merging multiple OpenVEX statements, resolving conflicts deterministically, and emitting policy diagnostics. | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-002 | EXCITITOR-FMT-OPENVEX-01-002 – Statement merge utilities | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering. | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-003 | EXCITITOR-FMT-OPENVEX-01-003 – OpenVEX export writer | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | TODO – Emit cosign/PGP trust metadata and advisory provenance hints for policy weighting. | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-003 | EXCITITOR-CONN-CISCO-01-003 – Provider trust metadata | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | TODO – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads. | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-002 | EXCITITOR-CONN-SUSE-01-002 – Checkpointed event ingestion | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine. | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-003 | EXCITITOR-CONN-SUSE-01-003 – Trust metadata & policy hints | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | TODO – Fetch CSAF packages with retry/backoff, checksum verification, and raw document persistence plus quarantine for schema failures. | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-002 | EXCITITOR-CONN-MS-01-002 – CSAF download pipeline | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration. | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-003 | EXCITITOR-CONN-MS-01-003 – Trust metadata & provenance hints | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DOING (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness and offline snapshot import; connector wiring and fixtures underway. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | TODO – Fetch CSAF documents with retry/backoff, checksum validation, revision deduplication, and raw persistence. | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-002 | EXCITITOR-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting. | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-003 | EXCITITOR-CONN-ORACLE-01-003 – Trust metadata + provenance | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence. | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-002 | EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics. | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-003 | EXCITITOR-CONN-UBUNTU-01-003 – Trust metadata & provenance | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Excititor Storage | EXCITITOR-STORAGE-02-001 | Statement events & scoring signals – create immutable VEX statement store plus consensus extensions with indexes/migrations. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | TODO | Team Excititor WebService | EXCITITOR-WEB-01-004 | Resolve API & signed responses – expose `/excititor/resolve`, return signed consensus/score envelopes, document auth. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-19) | Team Excititor WebService | EXCITITOR-WEB-01-005 | Mirror distribution endpoints – expose download APIs for downstream Excititor instances. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-01-004 | TTL refresh & stability damper – schedule re-resolve loops and guard against status flapping. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-005 | Score & resolve envelope surfaces – include signed consensus/score artifacts in exports. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-006 | Quiet provenance packaging – attach quieted-by statement IDs, signers, justification codes to exports and attestations. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Export/TASKS.md | TODO | Team Excititor Export | EXCITITOR-EXPORT-01-007 | Mirror bundle + domain manifest – publish signed consensus bundles for mirrors. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-001 | Excititor mirror connector – ingest signed mirror bundles and map to VexClaims with resume handling. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-002 | Normalize mirror bundles into VexClaim sets referencing original provider metadata and mirror provenance. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-003 | Implement incremental cursor handling per-export digest, support resume, and document configuration for downstream Excititor mirrors. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-19) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-001 | Advisory event log & asOf queries – surface immutable statements and replay capability. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-19) | Concelier WebService Guild | FEEDWEB-EVENTS-07-001 | Advisory event replay API – expose `/concelier/advisories/{key}/replay` with `asOf` filter, hex hashes, and conflict data. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Data Science | FEEDCORE-ENGINE-07-002 | Noise prior computation service – learn false-positive priors and expose deterministic summaries. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Core/TASKS.md | TODO | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-07-003 | Unknown state ledger & confidence seeding – persist unknown flags, seed confidence bands, expose query surface. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | TODO | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-07-001 | Advisory statement & conflict collections – provision Mongo schema/indexes for event-sourced merge. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Concelier.Merge/TASKS.md | TODO | BE-Merge | FEEDMERGE-ENGINE-07-001 | Conflict sets & explainers – persist conflict materialization and replay hashes for merge decisions. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | TODO | Plugin Platform Guild | PLUGIN-DI-08-001 | Scoped service support in plugin bootstrap
Teach the plugin loader/registrar to surface services with scoped lifetimes, honour `StellaOps.DependencyInjection` metadata, and document the new contract. | +| Sprint 8 | Plugin Infrastructure | src/StellaOps.Plugin/TASKS.md | TODO | Plugin Platform Guild, Authority Core | PLUGIN-DI-08-002 | Update Authority plugin integration
Flow scoped services through identity-provider registrars, bootstrap flows, and background jobs; add regression coverage around scoped lifetimes. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-19) | Team Normalization & Storage Backbone | FEEDSTORAGE-MONGO-08-001 | Causal-consistent Concelier storage sessions
Scoped session facilitator registered, repositories accept optional session handles, and replica-set failover tests verify read-your-write + monotonic reads. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Authority/TASKS.md | DONE (2025-10-19) | Authority Core & Storage Guild | AUTHSTORAGE-MONGO-08-001 | Harden Authority Mongo usage
Scoped Mongo sessions with majority read/write concerns wired through stores and GraphQL/HTTP pipelines; replica-set election regression validated. | +| Sprint 8 | Mongo strengthening | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-MONGO-08-001 | Causal consistency for Excititor repositories
Register Mongo options with majority defaults, push session-aware overloads through raw/export/consensus/cache stores, and extend migration/tests to validate causal reads after writes (including GridFS-backed content) under replica-set failover. | +| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Worker/TASKS.md | TODO | Team Excititor Worker | EXCITITOR-WORKER-02-001 | Resolve Microsoft.Extensions.Caching.Memory advisory – bump to latest .NET 10 preview, regenerate lockfiles, and rerun worker/webservice tests to clear NU1903. | +| Sprint 8 | Platform Maintenance | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | TODO | Team Excititor Storage | EXCITITOR-STORAGE-03-001 | Statement backfill tooling – provide CLI/backfill scripts that populate the `vex.statements` log via WebService ingestion and validate severity/KEV/EPSS signal replay. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.Json/TASKS.md | TODO | Concelier Export Guild | CONCELIER-EXPORT-08-201 | Mirror bundle + domain manifest – produce signed JSON aggregates for `*.stella-ops.org` mirrors. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-19) | Concelier Export Guild | CONCELIER-EXPORT-08-202 | Mirror-ready Trivy DB bundles – mirror options emit per-domain manifests/metadata/db archives with deterministic digests for downstream sync. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.WebService/TASKS.md | TODO | Concelier WebService Guild | CONCELIER-WEB-08-201 | Mirror distribution endpoints – expose domain-scoped index/download APIs with auth/quota. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Concelier mirror connector – fetch mirror manifest, verify signatures, and hydrate canonical DTOs with resume support. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | +| Sprint 8 | Mirror Distribution | src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-003 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | +| Sprint 8 | Mirror Distribution | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-MIRROR-08-001 | Managed mirror deployments for `*.stella-ops.org` – Helm/Compose overlays, CDN, runbooks. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-19) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. `docs/scanner-core-contracts.md` now carries the canonical JSON snippet + acceptance notes. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-19) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. Added `ScannerLogExtensionsPerformanceTests` to lock ≤5 µs overhead. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-103 | Progress streaming (SSE/JSONL) with correlation IDs and ISO-8601 UTC timestamps, documented in API reference. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-105 | Policy snapshot loader + schema + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-106 | `/reports` verdict assembly (Feedser+Vexer+Policy) + signed response envelope. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-POLICY-09-107 | Expose score inputs, config version, and quiet provenance in `/reports` JSON and signed payload. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-004 | Versioned scoring config with schema validation, trust table, and golden fixtures. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-005 | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild | POLICY-CORE-09-006 | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | TODO | Policy Guild, Scanner WebService Guild | POLICY-RUNTIME-17-201 | Define runtime reachability feed contract and alignment plan for `SCANNER-RUNTIME-17-401` once Zastava endpoints land; document policy expectations for reachability tags. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | TODO | DevOps Guild, Scanner WebService Guild | DEVOPS-SCANNER-09-204 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | TODO | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-205 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | TODO | Platform Events Guild | PLATFORM-EVENTS-09-401 | Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | TODO | Runtime Guild | RUNTIME-GUILD-09-402 | Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement layer cache store keyed by layer digest with metadata retention per architecture §3.3. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-102 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-103 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Cache/TASKS.md | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-104 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-201 | Alpine/apk analyzer emitting deterministic components with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-202 | Debian/dpkg analyzer mapping packages to purl identity with evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-203 | RPM analyzer capturing EVR, file listings, provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-204 | Shared OS evidence helpers for package identity + provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-205 | Vendor metadata enrichment (source packages, license, CVE hints). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | QA + OS Analyzer Guild | SCANNER-ANALYZERS-OS-10-206 | Determinism harness + fixtures for OS analyzers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.OS/TASKS.md | DONE (2025-10-19) | OS Analyzer Guild + DevOps | SCANNER-ANALYZERS-OS-10-207 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301 | Java analyzer emitting `pkg:maven` with provenance. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-302 | Node analyzer handling workspaces/symlinks emitting `pkg:npm`; workspace/symlink coverage and determinism harness in progress. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-303 | Python analyzer reading `*.dist-info`, RECORD hashes, entry points. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-304 | Go analyzer leveraging buildinfo for `pkg:golang` components. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-305 | .NET analyzer parsing `*.deps.json`, assembly metadata, RID variants. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-306 | Rust analyzer detecting crates or falling back to `bin:{sha256}`. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Shared language evidence helpers + usage flag propagation. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-308 | Determinism + fixture harness for language analyzers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/TASKS.md | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-309 | Package language analyzers as restart-time plug-ins (manifest + host registration); manifests and DI wiring under development. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-302..309 | Detailed per-language sprint plan (Node, Python, Go, .NET, Rust) with gates and benchmarks. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | POSIX shell AST parser with deterministic output. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Command resolution across layered rootfs with evidence attribution. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Interpreter tracing for shell wrappers to Python/Node/Java launchers. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-404 | Python entry analyzer (venv shebang, module invocation, usage flag). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-405 | Node/Java launcher analyzer capturing script/jar targets. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-406 | Explainability + diagnostics for unresolved constructs with metrics. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.EntryTrace/TASKS.md | TODO | EntryTrace Guild | SCANNER-ENTRYTRACE-10-407 | Package EntryTrace analyzers as restart-time plug-ins (manifest + host registration). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-502 | Attribute diffs to introducing/removing layers including provenance evidence. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Diff/TASKS.md | TODO | Diff Guild | SCANNER-DIFF-10-503 | Produce JSON diff output for inventory vs usage views aligned with API contract. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-601 | Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-602 | Compose usage SBOM leveraging EntryTrace to flag actual usage. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-603 | Generate BOM index sidecar (purl table + roaring bitmap + usage flag). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-604 | Package artifacts for export + attestation with deterministic manifests. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-605 | Emit BOM-Index sidecar schema/fixtures (CRITICAL PATH for SP16). | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-606 | Usage view bit flags integrated with EntryTrace. | +| Sprint 10 | Scanner Analyzers & SBOM | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-10-607 | Embed scoring inputs, confidence band, and quiet provenance in CycloneDX/DSSE artifacts. | +| Sprint 10 | Benchmarks | bench/TASKS.md | DONE (2025-10-19) | Bench Guild, Scanner Team | BENCH-SCANNER-10-001 | Analyzer microbench harness committed with baseline CSV + CLI hook. | +| Sprint 10 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Language Analyzer Guild | BENCH-SCANNER-10-002 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | +| Sprint 10 | Samples | samples/TASKS.md | DONE (2025-10-19) | Samples Guild, Scanner Team | SAMPLES-10-001 | Sample images, SBOMs, and BOM-Index fixtures published under `samples/`. | +| Sprint 10 | Samples | samples/TASKS.md | TODO | Samples Guild, Policy Guild | SAMPLES-13-004 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-PERF-10-001 | Perf smoke job added to CI enforcing <5 s compose budget with regression guard. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-PERF-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | +| Sprint 10 | DevOps Perf | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-SEC-10-301 | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305A | Parse `*.deps.json` + `runtimeconfig.json`, build RID graph, and normalize to `pkg:nuget` components. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305B | Extract assembly metadata (strong name, file/product info) and optional Authenticode details when offline cert bundle provided. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-305C | Handle self-contained apps and native assets; merge with EntryTrace usage hints. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307D | Integrate shared helpers (license mapping, quiet provenance) and concurrency-safe caches. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308D | Determinism fixtures + benchmark harness; compare to competitor scanners for accuracy/perf. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309D | Package plug-in (manifest, DI registration) and update Offline Kit instructions. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304A | Parse Go build info blob (`runtime/debug` format) and `.note.go.buildid`; map to module/version and evidence. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304B | Implement DWARF-lite reader for VCS metadata + dirty flag; add cache to avoid re-reading identical binaries. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-304C | Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307G | Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308G | Determinism fixtures + benchmark harness (Vs competitor). | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309G | Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-302C | Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307N | Integrate shared helpers for license/licence evidence, canonical JSON serialization, and usage flag propagation. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308N | Author determinism harness + fixtures for Node analyzer; add benchmark suite. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309N | Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303A | STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303B | RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-303C | Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307P | Shared helper integration (license metadata, quiet provenance, component merging). | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306A | Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306B | Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-306C | Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-307R | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-308R | Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. | +| Sprint 10 | Backlog | src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md | TODO | TBD | SCANNER-ANALYZERS-LANG-10-309R | Package plug-in manifest + Offline Kit documentation; ensure Worker integration. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-DPOP-11-001 | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Authority/TASKS.md | TODO | Authority Core & Security Guild | AUTH-MTLS-11-002 | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-API-11-101 | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-REF-11-102 | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Signer/TASKS.md | TODO | Signer Guild | SIGNER-QUOTA-11-103 | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-API-11-201 | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-VERIFY-11-202 | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | +| Sprint 11 | Signing Chain Bring-up | src/StellaOps.Attestor/TASKS.md | TODO | Attestor Guild | ATTESTOR-OBS-11-203 | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | +| Sprint 11 | UI Integration | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ATTEST-11-005 | Attestation visibility (Rekor id, status) on Scan Detail. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-201 | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-202 | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-CORE-12-203 | Authority client helpers, OpTok caching, and security guardrails for runtime services. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Core/TASKS.md | TODO | Zastava Core Guild | ZASTAVA-OPS-12-204 | Operational runbooks, alert rules, and dashboard exports for runtime plane. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Container lifecycle watcher emitting deterministic runtime events with buffering. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Capture entrypoint traces + loaded libraries, hashing binaries and linking to baseline SBOM. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-003 | Posture checks for signatures/SBOM/attestation with offline caching. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-004 | Batch `/runtime/events` submissions with disk-backed buffer and rate limits. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-101 | Admission controller host with TLS bootstrap and Authority auth. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-102 | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Zastava.Webhook/TASKS.md | TODO | Zastava Webhook Guild | ZASTAVA-WEBHOOK-12-103 | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301 | `/runtime/events` ingestion endpoint with validation, batching, storage hooks. | +| Sprint 12 | Runtime Guardrails | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-302 | `/policy/runtime` endpoint joining SBOM baseline + policy verdict with TTL guidance. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-AUTH-13-001 | Integrate Authority OIDC + DPoP flows with session management. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCANS-13-002 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-VEX-13-003 | Implement VEX explorer + policy editor with preview integration. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-ADMIN-13-004 | Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-SCHED-13-005 | Scheduler panel: schedules CRUD, run history, dry-run preview. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | DOING (2025-10-19) | UI Guild | UI-NOTIFY-13-006 | Notify panel: channels/rules CRUD, deliveries view, test send. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.UI/TASKS.md | TODO | UI Guild | UI-POLICY-13-007 | Surface policy confidence metadata (band, age, quiet provenance) on preview and report views. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-RUNTIME-13-005 | Add runtime policy test verbs that consume `/policy/runtime` and display verdicts. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-OFFLINE-13-006 | Implement offline kit pull/import/status commands with integrity checks. | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO | DevEx/CLI | CLI-PLUGIN-13-007 | Package non-core CLI verbs as restart-time plug-ins (manifest + loader tests). | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO – Once `/api/v1/scanner/policy/runtime` exits TODO, verify CLI output against final schema (field names, metadata) and update formatter/tests if the contract moves. Capture joint review notes in docs/09 and link Scanner task sign-off. | DevEx/CLI, Scanner WebService Guild | CLI-RUNTIME-13-008 | CLI-RUNTIME-13-008 – Runtime policy contract sync | +| Sprint 13 | UX & CLI Experience | src/StellaOps.Cli/TASKS.md | TODO – Build Spectre test harness exercising `runtime policy test` against a stubbed backend to lock output shape (table + `--json`) and guard regressions. Integrate into `dotnet test` suite. | DevEx/CLI, QA Guild | CLI-RUNTIME-13-009 | CLI-RUNTIME-13-009 – Runtime policy smoke fixture | +| Sprint 14 | Release & Offline Ops | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-14-001 | Deterministic build/release pipeline with SBOM/provenance, signing, and manifest generation. | +| Sprint 14 | Release & Offline Ops | ops/offline-kit/TASKS.md | TODO | Offline Kit Guild | DEVOPS-OFFLINE-14-002 | Offline kit packaging workflow with integrity verification and documentation. | +| Sprint 14 | Release & Offline Ops | ops/deployment/TASKS.md | TODO | Deployment Guild | DEVOPS-OPS-14-003 | Deployment/update/rollback automation and channel management documentation. | +| Sprint 14 | Release & Offline Ops | ops/licensing/TASKS.md | TODO | Licensing Guild | DEVOPS-LIC-14-004 | Registry token service tied to Authority, plan gating, revocation handling, monitoring. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-101 | Define core Notify DTOs, validation helpers, canonical serialization. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-102 | Publish schema docs and sample payloads for Notify. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Models/TASKS.md | TODO | Notify Models Guild | NOTIFY-MODELS-15-103 | Versioning/migration helpers for rules/templates/deliveries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-201 | Mongo schemas/indexes for rules, channels, deliveries, digests, locks, audit. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-202 | Repositories with tenant scoping, soft delete, TTL, causal consistency options. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Storage.Mongo/TASKS.md | TODO | Notify Storage Guild | NOTIFY-STORAGE-15-203 | Delivery history retention and query APIs. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Bus abstraction + Redis Streams adapter with ordering/idempotency. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-402 | NATS JetStream adapter with health probes and failover. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Queue/TASKS.md | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-403 | Delivery queue with retry/dead-letter + metrics. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-301 | Rules evaluation core (filters, throttles, idempotency). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-302 | Action planner + digest coalescer. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-303 | Template rendering engine (Slack/Teams/Email/Webhook). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Engine/TASKS.md | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-304 | Test-send sandbox + preview utilities. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-101 | Minimal API host with Authority enforcement and plug-in loading. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-102 | Rules/channel/template CRUD with audit logging. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-103 | Delivery history & test-send endpoints. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.WebService/TASKS.md | TODO | Notify WebService Guild | NOTIFY-WEB-15-104 | Configuration binding + startup diagnostics. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-201 | Bus subscription + leasing loop with backoff. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-202 | Rules evaluation pipeline integration. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Channel dispatch orchestration with retries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Worker/TASKS.md | TODO | Notify Worker Guild | NOTIFY-WORKER-15-204 | Metrics/telemetry for Notify workers. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Slack connector with rate-limit aware delivery. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Slack health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Teams connector with Adaptive Cards. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Teams health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | SMTP connector with TLS + rendering. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | DKIM + health/test-send flows. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Webhook connector with signing/retries. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Webhook health/test-send support. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Slack/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-503 | Package Slack connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Teams/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-603 | Package Teams connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Email/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-703 | Package Email connector as restart-time plug-in (manifest + host registration). | +| Sprint 15 | Notify Foundations | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-EVENTS-15-201 | Emit `scanner.report.ready` + `scanner.scan.completed` events. | +| Sprint 15 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Notify Team | BENCH-NOTIFY-15-001 | Notify dispatch throughput bench with results CSV. | +| Sprint 15 | Notify Foundations | src/StellaOps.Notify.Connectors.Webhook/TASKS.md | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-803 | Package Webhook connector as restart-time plug-in (manifest + host registration). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Define Scheduler DTOs & validation. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-102 | Publish schema docs/sample payloads. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Models/TASKS.md | TODO | Scheduler Models Guild | SCHED-MODELS-16-103 | Versioning/migration helpers for schedules/runs. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Mongo schemas/indexes for Scheduler state (models ready 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-202 | Repositories with tenant scoping, TTL, causal consistency (models ready 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Storage.Mongo/TASKS.md | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-203 | Audit + stats materialization for UI (models ready 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Queue abstraction + Redis Streams adapter (samples available 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-402 | NATS JetStream adapter with health probes (samples available 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Queue/TASKS.md | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-403 | Dead-letter handling + metrics (samples available 2025-10-19). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Ingest BOM-Index into roaring bitmap store. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-302 | Query APIs for ResolveByPurls/ResolveByVulns/ResolveAll. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-303 | Snapshot/compaction/invalidation workflow. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.ImpactIndex/TASKS.md | DOING | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-300 | **STUB** ImpactIndex ingest/query using fixtures (to be removed by SP16 completion). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Minimal API host with Authority enforcement. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Schedules CRUD (cron validation, pause/resume, audit). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-103 | Runs API (list/detail/cancel) + impact previews. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.WebService/TASKS.md | TODO | Scheduler WebService Guild | SCHED-WEB-16-104 | Feedser/Vexer webhook handlers with security enforcement. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Planner loop (cron/event triggers, leases, fairness). | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | ImpactIndex targeting and shard planning. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Runner execution invoking Scanner analysis/content refresh. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-204 | Emit rescan/report events for Notify/UI. | +| Sprint 16 | Scheduler Intelligence | src/StellaOps.Scheduler.Worker/TASKS.md | TODO | Scheduler Worker Guild | SCHED-WORKER-16-205 | Metrics/telemetry for Scheduler planners/runners. | +| Sprint 16 | Benchmarks | bench/TASKS.md | TODO | Bench Guild, Scheduler Team | BENCH-IMPACT-16-001 | ImpactIndex throughput bench + RAM profile. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.Emit/TASKS.md | TODO | Emit Guild | SCANNER-EMIT-17-701 | Record GNU build-id for ELF components and surface it in SBOM/diff outputs. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Zastava.Observer/TASKS.md | TODO | Zastava Observer Guild | ZASTAVA-OBS-17-005 | Collect GNU build-id during runtime observation and attach it to emitted events. | +| Sprint 17 | Symbol Intelligence & Forensics | src/StellaOps.Scanner.WebService/TASKS.md | TODO | Scanner WebService Guild | SCANNER-RUNTIME-17-401 | Persist runtime build-id observations and expose them for debug-symbol correlation. | +| Sprint 17 | Symbol Intelligence & Forensics | ops/devops/TASKS.md | TODO | DevOps Guild | DEVOPS-REL-17-002 | Ship stripped debug artifacts organised by build-id within release/offline kits. | +| Sprint 17 | Symbol Intelligence & Forensics | docs/TASKS.md | TODO | Docs Guild | DOCS-RUNTIME-17-004 | Document build-id workflows for SBOMs, runtime events, and debug-store usage. | diff --git a/SPRINTS_PRIOR_20251019.md b/SPRINTS_PRIOR_20251019.md new file mode 100644 index 00000000..c9017860 --- /dev/null +++ b/SPRINTS_PRIOR_20251019.md @@ -0,0 +1,177 @@ +Closed sprint tasks archived from SPRINTS.md on 2025-10-19. + +| Sprint | Theme | Tasks File Path | Status | Type of Specialist | Task ID | Task Description | +| --- | --- | --- | --- | --- | --- | --- | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-001 | SemVer primitive range-style metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md. This task lays the groundwork—complete the SemVer helper updates before teammates pick up FEEDMODELS-SCHEMA-01-002/003 and FEEDMODELS-SCHEMA-02-900. Use ./src/FASTER_MODELING_AND_NORMALIZATION.md for the target rule structure. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-002 | Provenance decision rationale field
Instructions to work:
AdvisoryProvenance now carries `decisionReason` and docs/tests were updated. Connectors and merge tasks should populate the field when applying precedence/freshness/tie-breaker logic; see src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md for usage guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-11) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-01-003 | Normalized version rules collection
Instructions to work:
`AffectedPackage.NormalizedVersions` and supporting comparer/docs/tests shipped. Connector owners must emit rule arrays per ./src/FASTER_MODELING_AND_NORMALIZATION.md and report progress via FEEDMERGE-COORD-02-900 so merge/storage backfills can proceed. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-12) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-02-900 | Range primitives for SemVer/EVR/NEVRA metadata
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Models/AGENTS.md before resuming this stalled effort. Confirm helpers align with the new `NormalizedVersions` representation so connectors finishing in Sprint 2 can emit consistent metadata. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
Shared `SemVerRangeRuleBuilder` now outputs primitives + normalized rules per `FASTER_MODELING_AND_NORMALIZATION.md`; CVE/GHSA connectors consuming the API have verified fixtures. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
AdvisoryStore dual-writes flattened `normalizedVersions` when `concelier.storage.enableSemVerStyle` is set; migration `20251011-semver-style-backfill` updates historical records and docs outline the rollout. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence
Storage now persists `provenance.decisionReason` for advisories and merge events; tests cover round-trips. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Bootstrapper seeds compound/sparse indexes for flattened normalized rules and `docs/dev/mongo_indices.md` documents query guidance. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Updated constructors/tests keep storage suites passing with the new feature flag defaults. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Install/runbooks document connected vs air-gapped resilience profiles and monitoring hooks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Operator guides now call out `route/status/subject/clientId/scopes/bypass/remote` audit fields and SIEM triggers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and links audit signals to the rollout checklist. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.HOST | Rate limiter policy binding
Authority host now applies configuration-driven fixed windows to `/token`, `/authorize`, and `/internal/*`; integration tests assert 429 + `Retry-After` headers; docs/config samples refreshed for Docs guild diagrams. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | SEC3.BUILD | Authority rate-limiter follow-through
`Security.RateLimiting` now fronts token/authorize/internal limiters; Authority + Configuration matrices (`dotnet test src/StellaOps.Authority/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) passed on 2025-10-11; awaiting #authority-core broadcast. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHCORE-BUILD-OPENIDDICT / AUTHCORE-STORAGE-DEVICE-TOKENS / AUTHCORE-BOOTSTRAP-INVITES | Address remaining Authority compile blockers (OpenIddict transaction shim, token device document, bootstrap invite cleanup) so `dotnet build src/StellaOps.Authority.sln` returns success. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | PLG6.DOC | Plugin developer guide polish
Section 9 now documents rate limiter metadata, config keys, and lockout interplay; YAML samples updated alongside Authority config templates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-001 | Fetch pipeline & state tracking
Summary planner now drives monthly/yearly VINCE fetches, persists pending summaries/notes, and hydrates VINCE detail queue with telemetry.
Team instructions: Read ./AGENTS.md and src/StellaOps.Concelier.Connector.CertCc/AGENTS.md. Coordinate daily with Models/Merge leads so new normalizedVersions output and provenance tags stay aligned with ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-002 | VINCE note detail fetcher
Summary planner queues VINCE note detail endpoints, persists raw JSON with SHA/ETag metadata, and records retry/backoff metrics. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-003 | DTO & parser implementation
Added VINCE DTO aggregate, Markdown→text sanitizer, vendor/status/vulnerability parsers, and parser regression fixture. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-004 | Canonical mapping & range primitives
VINCE DTO aggregate flows through `CertCcMapper`, emitting vendor range primitives + normalized version rules that persist via `_advisoryStore`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-005 | Deterministic fixtures/tests
Snapshot harness refreshed 2025-10-12; `certcc-*.snapshot.json` regenerated and regression suite green without UPDATE flag drift. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-006 | Telemetry & documentation
`CertCcDiagnostics` publishes summary/detail/parse/map metrics (meter `StellaOps.Concelier.Connector.CertCc`), README documents instruments, and log guidance captured for Ops on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-007 | Connector test harness remediation
Harness now wires `AddSourceCommon`, resets `FakeTimeProvider`, and passes canned-response regression run dated 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-008 | Snapshot coverage handoff
Fixtures regenerated with normalized ranges + provenance fields on 2025-10-11; QA handoff notes published and merge backfill unblocked. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-012 | Schema sync & snapshot regen follow-up
Fixtures regenerated with normalizedVersions + provenance decision reasons; handoff notes updated for Merge backfill 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-009 | Detail/map reintegration plan
Staged reintegration plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; coordinates enablement with FEEDCONN-CERTCC-02-004. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.CertCc/TASKS.md | DONE (2025-10-12) | Team Connector Resumption – CERT/RedHat | FEEDCONN-CERTCC-02-010 | Partial-detail graceful degradation
Detail fetch now tolerates 404/403/410 responses and regression tests cover mixed endpoint availability. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md | DONE (2025-10-11) | Team Connector Resumption – CERT/RedHat | FEEDCONN-REDHAT-02-001 | Fixture validation sweep
Instructions to work:
Fixtures regenerated post-model-helper rollout; provenance ordering and normalizedVersions scaffolding verified via tests. Conflict resolver deltas logged in src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md for Sprint 3 consumers. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-001 | Canonical mapping & range primitives
Mapper emits SemVer rules (`scheme=apple:*`); fixtures regenerated with trimmed references + new RSR coverage, update tooling finalized. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-002 | Deterministic fixtures/tests
Sanitized live fixtures + regression snapshots wired into tests; normalized rule coverage asserted. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-003 | Telemetry & documentation
Apple meter metrics wired into Concelier WebService OpenTelemetry configuration; README and fixtures document normalizedVersions coverage. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-12) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-004 | Live HTML regression sweep
Sanitised HT125326/HT125328/HT106355/HT214108/HT215500 fixtures recorded and regression tests green on 2025-10-12. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md | DONE (2025-10-11) | Team Vendor Apple Specialists | FEEDCONN-APPLE-02-005 | Fixture regeneration tooling
`UPDATE_APPLE_FIXTURES=1` flow fetches & rewrites fixtures; README documents usage.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md. Resume stalled tasks, ensuring normalizedVersions output and fixtures align with ./src/FASTER_MODELING_AND_NORMALIZATION.md before handing data to the conflict sprint. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance
Team instructions: Read ./AGENTS.md and each module's AGENTS file. Adopt the `NormalizedVersions` array emitted by the models sprint, wiring provenance `decisionReason` where merge overrides occur. Follow ./src/FASTER_MODELING_AND_NORMALIZATION.md; report via src/StellaOps.Concelier.Merge/TASKS.md (FEEDMERGE-COORD-02-900). Progress 2025-10-11: GHSA/OSV emit normalized arrays with refreshed fixtures; CVE mapper now surfaces SemVer normalized ranges; NVD/KEV adoption pending; outstanding follow-ups include FEEDSTORAGE-DATA-02-001, FEEDMERGE-ENGINE-02-002, and rolling `tools/FixtureUpdater` updates across connectors. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-CVE-02-003 | CVE normalized versions uplift | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-KEV-02-003 | KEV normalized versions propagation | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Normalized Versions Rollout | FEEDCONN-OSV-04-003 | OSV parity fixture refresh | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-10) | Team WebService & Authority | FEEDWEB-DOCS-01-001 | Document authority toggle & scope requirements
Quickstart carries toggle/scope guidance pending docs guild review (no change this sprint). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-ENGINE-01-002 | Plumb Authority client resilience options
WebService wires `authority.resilience.*` into `AddStellaOpsAuthClient` and adds binding coverage via `AuthorityClientResilienceOptionsAreBound`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-003 | Author ops guidance for resilience tuning
Operator docs now outline connected vs air-gapped resilience profiles and monitoring cues. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-004 | Document authority bypass logging patterns
Audit logging guidance highlights `route/status/subject/clientId/scopes/bypass/remote` fields and SIEM alerts. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-12) | Team WebService & Authority | FEEDWEB-DOCS-01-005 | Update Concelier operator guide for enforcement cutoff
Install guide reiterates the 2025-12-31 cutoff and ties audit signals to rollout checks. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-006 | Rename plugin drop directory to namespaced path
Build outputs, tests, and docs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Concelier.WebService/TASKS.md | DONE (2025-10-11) | Team WebService & Authority | FEEDWEB-OPS-01-007 | Authority resilience adoption
Deployment docs and CLI notes explain the LIB5 resilience knobs for rollout.
Instructions to work:
DONE Read ./AGENTS.md and src/StellaOps.Concelier.WebService/AGENTS.md. These items were mid-flight; resume implementation ensuring docs/operators receive timely updates. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCORE-ENGINE-01-001 | CORE8.RL — Rate limiter plumbing validated; integration tests green and docs handoff recorded for middleware ordering + Retry-After headers (see `docs/dev/authority-rate-limit-tuning-outline.md` for continuing guidance). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHCRYPTO-ENGINE-01-001 | SEC3.A — Shared metadata resolver confirmed via host test run; SEC3.B now unblocked for tuning guidance (outline captured in `docs/dev/authority-rate-limit-tuning-outline.md`). | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-13) | Team Authority Platform & Security Guild | AUTHSEC-DOCS-01-002 | SEC3.B — Published `docs/security/rate-limits.md` with tuning matrix, alert thresholds, and lockout interplay guidance; Docs guild can lift copy into plugin guide. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Team Authority Platform & Security Guild | AUTHSEC-CRYPTO-02-001 | SEC5.B1 — Introduce libsodium signing provider and parity tests to unblock CLI verification enhancements. | +| Sprint 1 | Bootstrap & Replay Hardening | src/StellaOps.Cryptography/TASKS.md | DONE (2025-10-14) | Security Guild | AUTHSEC-CRYPTO-02-004 | SEC5.D/E — Finish bootstrap invite lifecycle (API/store/cleanup) and token device heuristics; build currently red due to pending handler integration. | +| Sprint 1 | Developer Tooling | src/StellaOps.Cli/TASKS.md | DONE (2025-10-15) | DevEx/CLI | AUTHCLI-DIAG-01-001 | Surface password policy diagnostics in CLI startup/output so operators see weakened overrides immediately.
CLI now loads Authority plug-ins at startup, logs weakened password policies (length/complexity), and regression coverage lives in `StellaOps.Cli.Tests/Services/AuthorityDiagnosticsReporterTests`. | +| Sprint 1 | Stabilize In-Progress Foundations | src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md | DONE (2025-10-11) | Team Authority Platform & Security Guild | AUTHPLUG-DOCS-01-001 | PLG6.DOC — Developer guide copy + diagrams merged 2025-10-11; limiter guidance incorporated and handed to Docs guild for asset export. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Normalization/TASKS.md | DONE (2025-10-12) | Team Normalization & Storage Backbone | FEEDNORM-NORM-02-001 | SemVer normalized rule emitter
`SemVerRangeRuleBuilder` shipped 2025-10-12 with comparator/`||` support and fixtures aligning to `FASTER_MODELING_AND_NORMALIZATION.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-002 | Provenance decision reason persistence | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDSTORAGE-DATA-02-003 | Normalized versions indexing
Indexes seeded + docs updated 2025-10-11 to cover flattened normalized rules for connector adoption. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Normalization & Storage Backbone | FEEDMERGE-ENGINE-02-002 | Normalized versions union & dedupe
Affected package resolver unions/dedupes normalized rules, stamps merge provenance with `decisionReason`, and tests cover the rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-001 | GHSA normalized versions & provenance | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-004 | GHSA credits & ecosystem severity mapping | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-005 | GitHub quota monitoring & retries | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-006 | Production credential & scheduler rollout | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-GHSA-02-007 | Credit parity regression fixtures | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-002 | NVD normalized versions & timestamps | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-004 | NVD CVSS & CWE precedence payloads | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-NVD-02-005 | NVD merge/export parity regression | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-003 | OSV normalized versions & freshness | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-11) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-004 | OSV references & credits alignment | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-02-005 | Fixture updater workflow
Resolved 2025-10-12: OSV mapper now derives canonical PURLs for Go + scoped npm packages when raw payloads omit `purl`; conflict fixtures unchanged for invalid npm names. Verified via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`, `src/StellaOps.Concelier.Connector.Ghsa.Tests`, `src/StellaOps.Concelier.Connector.Nvd.Tests`, and backbone normalization/storage suites. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Acsc/TASKS.md | DONE (2025-10-12) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ACSC-02-001 … 02-008 | Fetch→parse→map pipeline, fixtures, diagnostics, and README finished 2025-10-12; downstream export parity captured via FEEDEXPORT-JSON-04-001 / FEEDEXPORT-TRIVY-04-001 (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cccs/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CCCS-02-001 … 02-008 | Observability meter, historical harvest plan, and DOM sanitizer refinements wrapped; ops notes live under `docs/ops/concelier-cccs-operations.md` with fixtures validating EN/FR list handling. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.CertBund/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CERTBUND-02-001 … 02-008 | Telemetry/docs (02-006) and history/locale sweep (02-007) completed alongside pipeline; runbook `docs/ops/concelier-certbund-operations.md` captures locale guidance and offline packaging. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kisa/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-KISA-02-001 … 02-007 | Connector, tests, and telemetry/docs (02-006) finalized; localisation notes in `docs/dev/kisa_connector_notes.md` complete rollout. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-RUBDU-02-001 … 02-008 | Fetch/parser/mapper refinements, regression fixtures, telemetry/docs, access options, and trusted root packaging all landed; README documents offline access strategy. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md | DONE (2025-10-13) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-NKCKI-02-001 … 02-008 | Listing fetch, parser, mapper, fixtures, telemetry/docs, and archive plan finished; Mongo2Go/libcrypto dependency resolved via bundled OpenSSL noted in ops guide. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-ICSCISA-02-001 … 02-011 | Feed parser attachment fixes, SemVer exact values, regression suites, telemetry/docs updates, and handover complete; ops runbook now details attachment verification + proxy usage. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md | DONE (2025-10-14) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-CISCO-02-001 … 02-007 | OAuth fetch pipeline, DTO/mapping, tests, and telemetry/docs shipped; monitoring/export integration follow-ups recorded in Ops docs and exporter backlog (completed). | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md | DONE (2025-10-15) | Team Connector Expansion – Regional & Vendor Feeds | FEEDCONN-MSRC-02-001 … 02-008 | Azure AD onboarding (02-008) unblocked fetch/parse/map pipeline; fixtures, telemetry/docs, and Offline Kit guidance published in `docs/ops/concelier-msrc-operations.md`. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Cve/TASKS.md | DONE (2025-10-15) | Team Connector Support & Monitoring | FEEDCONN-CVE-02-001 … 02-002 | CVE data-source selection, fetch pipeline, and docs landed 2025-10-10. 2025-10-15: smoke verified using the seeded mirror fallback; connector now logs a warning and pulls from `seed-data/cve/` until live CVE Services credentials arrive. | +| Sprint 2 | Connector & Data Implementation Wave | src/StellaOps.Concelier.Connector.Kev/TASKS.md | DONE (2025-10-12) | Team Connector Support & Monitoring | FEEDCONN-KEV-02-001 … 02-002 | KEV catalog ingestion, fixtures, telemetry, and schema validation completed 2025-10-12; ops dashboard published. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-01-001 | Canonical schema docs refresh
Updated canonical schema + provenance guides with SemVer style, normalized version rules, decision reason change log, and migration notes. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-001 | Concelier-SemVer Playbook
Published merge playbook covering mapper patterns, dedupe flow, indexes, and rollout checklist. | +| Sprint 2 | Connector & Data Implementation Wave | docs/TASKS.md | DONE (2025-10-11) | Team Docs & Knowledge Base | FEEDDOCS-DOCS-02-002 | Normalized versions query guide
Delivered Mongo index/query addendum with `$unwind` recipes, dedupe checks, and operational checklist.
Instructions to work:
DONE Read ./AGENTS.md and docs/AGENTS.md. Document every schema/index/query change produced in Sprint 1-2 leveraging ./src/FASTER_MODELING_AND_NORMALIZATION.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-001 | Canonical merger implementation
`CanonicalMerger` ships with freshness/tie-breaker logic, provenance, and unit coverage feeding Merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-03-002 | Field precedence and tie-breaker map
Field precedence tables and tie-breaker metrics wired into the canonical merge flow; docs/tests updated.
Instructions to work:
Read ./AGENTS.md and core AGENTS. Implement the conflict resolver exactly as specified in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md, coordinating with Merge and Storage teammates. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-03-001 | Merge event provenance audit prep
Merge events now persist `fieldDecisions` and analytics-ready provenance snapshots. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-DATA-02-001 | Normalized range dual-write + backfill
Dual-write/backfill flag delivered; migration + options validated in tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Storage.Mongo/TASKS.md | DONE (2025-10-11) | Team Core Engine & Storage Analytics | FEEDSTORAGE-TESTS-02-004 | Restore AdvisoryStore build after normalized versions refactor
Storage tests adjusted for normalized versions/decision reasons.
Instructions to work:
Read ./AGENTS.md and storage AGENTS. Extend merge events with decision reasons and analytics views to support the conflict rules, and deliver the dual-write/backfill for `NormalizedVersions` + `decisionReason` so connectors can roll out safely. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-001 | GHSA/NVD/OSV conflict rules
Merge pipeline consumes `CanonicalMerger` output prior to precedence merge. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-002 | Override metrics instrumentation
Merge events capture per-field decisions; counters/logs align with conflict rules. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-003 | Reference & credit union pipeline
Canonical merge preserves unions with updated tests. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-11) | Team Merge & QA Enforcement | FEEDMERGE-QA-04-001 | End-to-end conflict regression suite
Added regression tests (`AdvisoryMergeServiceTests`) covering canonical + precedence flow.
Instructions to work:
Read ./AGENTS.md and merge AGENTS. Integrate the canonical merger, instrument metrics, and deliver comprehensive regression tests following ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md. | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-002 | GHSA conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Nvd/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-NVD-04-002 | NVD conflict regression fixtures | +| Sprint 3 | Conflict Resolution Integration & Communications | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-12) | Team Connector Regression Fixtures | FEEDCONN-OSV-04-002 | OSV conflict regression fixtures
Instructions to work:
Read ./AGENTS.md and module AGENTS. Produce fixture triples supporting the precedence/tie-breaker paths defined in ./src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md and hand them to Merge QA. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-11) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-001 | Concelier Conflict Rules
Runbook published at `docs/ops/concelier-conflict-resolution.md`; metrics/log guidance aligned with Sprint 3 merge counters. | +| Sprint 3 | Conflict Resolution Integration & Communications | docs/TASKS.md | DONE (2025-10-16) | Team Documentation Guild – Conflict Guidance | FEEDDOCS-DOCS-05-002 | Conflict runbook ops rollout
Ops review completed, alert thresholds applied, and change log appended in `docs/ops/concelier-conflict-resolution.md`; task closed after connector signals verified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Models/TASKS.md | DONE (2025-10-15) | Team Models & Merge Leads | FEEDMODELS-SCHEMA-04-001 | Advisory schema parity (description/CWE/canonical metric)
Extend `Advisory` and related records with description text, CWE collection, and canonical metric pointer; refresh validation + serializer determinism tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-003 | Canonical merger parity for new fields
Teach `CanonicalMerger` to populate description, CWEResults, and canonical metric pointer with provenance + regression coverage. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Core/TASKS.md | DONE (2025-10-15) | Team Core Engine & Storage Analytics | FEEDCORE-ENGINE-04-004 | Reference normalization & freshness instrumentation cleanup
Implement URL normalization for reference dedupe, align freshness-sensitive instrumentation, and add analytics tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-004 | Merge pipeline parity for new advisory fields
Ensure merge service + merge events surface description/CWE/canonical metric decisions with updated metrics/tests. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Merge/TASKS.md | DONE (2025-10-15) | Team Merge & QA Enforcement | FEEDMERGE-ENGINE-04-005 | Connector coordination for new advisory fields
GHSA/NVD/OSV connectors now ship description, CWE, and canonical metric data with refreshed fixtures; merge coordination log updated and exporters notified. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.Json/TASKS.md | DONE (2025-10-15) | Team Exporters – JSON | FEEDEXPORT-JSON-04-001 | Surface new advisory fields in JSON exporter
Update schemas/offline bundle + fixtures once model/core parity lands.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` validated canonical metric/CWE emission. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md | DONE (2025-10-15) | Team Exporters – Trivy DB | FEEDEXPORT-TRIVY-04-001 | Propagate new advisory fields into Trivy DB package
Extend Bolt builder, metadata, and regression tests for the expanded schema.
2025-10-15: `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` confirmed canonical metric/CWE propagation. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Ghsa/TASKS.md | DONE (2025-10-16) | Team Connector Regression Fixtures | FEEDCONN-GHSA-04-004 | Harden CVSS fallback so canonical metric ids persist when GitHub omits vectors; extend fixtures and document severity precedence hand-off to Merge. | +| Sprint 4 | Schema Parity & Freshness Alignment | src/StellaOps.Concelier.Connector.Osv/TASKS.md | DONE (2025-10-16) | Team Connector Expansion – GHSA/NVD/OSV | FEEDCONN-OSV-04-005 | Map OSV advisories lacking CVSS vectors to canonical metric ids/notes and document CWE provenance quirks; schedule parity fixture updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-001 | Stand up canonical VEX claim/consensus records with deterministic serializers so Storage/Exports share a stable contract. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-002 | Implement trust-weighted consensus resolver with baseline policy weights, justification gates, telemetry output, and majority/tie handling. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-15) | Team Excititor Core & Policy | EXCITITOR-CORE-01-003 | Publish shared connector/exporter/attestation abstractions and deterministic query signature utilities for cache/attestation workflows. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-001 | Established policy options & snapshot provider covering baseline weights/overrides. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-15) | Team Excititor Policy | EXCITITOR-POLICY-01-002 | Policy evaluator now feeds consensus resolver with immutable snapshots. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-003 | Author policy diagnostics, CLI/WebService surfacing, and documentation updates. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-004 | Implement YAML/JSON schema validation and deterministic diagnostics for operator bundles. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-16) | Team Excititor Policy | EXCITITOR-POLICY-01-005 | Add policy change tracking, snapshot digests, and telemetry/logging hooks. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-15) | Team Excititor Storage | EXCITITOR-STORAGE-01-001 | Mongo mapping registry plus raw/export entities and DI extensions in place. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Storage.Mongo/TASKS.md | DONE (2025-10-16) | Team Excititor Storage | EXCITITOR-STORAGE-01-004 | Build provider/consensus/cache class maps and related collections. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-15) | Team Excititor Export | EXCITITOR-EXPORT-01-001 | Export engine delivers cache lookup, manifest creation, and policy integration. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Export/TASKS.md | DONE (2025-10-17) | Team Excititor Export | EXCITITOR-EXPORT-01-004 | Connect export engine to attestation client and persist Rekor metadata. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-001 | Implement in-toto predicate + DSSE builder providing envelopes for export attestation. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors | EXCITITOR-CONN-ABS-01-001 | Deliver shared connector context/base classes so provider plug-ins can be activated via WebService/Worker. | +| Sprint 5 | Excititor Core Foundations | src/StellaOps.Excititor.WebService/TASKS.md | DONE (2025-10-17) | Team Excititor WebService | EXCITITOR-WEB-01-001 | Scaffold minimal API host, DI, and `/excititor/status` endpoint integrating policy, storage, export, and attestation services. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Worker/TASKS.md | DONE (2025-10-17) | Team Excititor Worker | EXCITITOR-WORKER-01-001 | Create Worker host with provider scheduling and logging to drive recurring pulls/reconciliation. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CSAF-01-001 | Implement CSAF normalizer foundation translating provider documents into `VexClaim` entries. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-CYCLONE-01-001 | Implement CycloneDX VEX normalizer capturing `analysis` state and component references. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md | DONE (2025-10-17) | Team Excititor Formats | EXCITITOR-FMT-OPENVEX-01-001 | Implement OpenVEX normalizer to ingest attestations into canonical claims with provenance. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-001 | Ship Red Hat CSAF provider metadata discovery enabling incremental pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-002 | Fetch CSAF windows with ETag handling, resume tokens, quarantine on schema errors, and persist raw docs. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-003 | Populate provider trust overrides (cosign issuer, identity regex) and provenance hints for policy evaluation/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-004 | Persist resume cursors (last updated timestamp/document hashes) in storage and reload during fetch to avoid duplicates. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-005 | Register connector in Worker/WebService DI, add scheduled jobs, and document CLI triggers for Red Hat CSAF pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Red Hat | EXCITITOR-CONN-RH-01-006 | Add CSAF normalization parity fixtures ensuring RHSA-specific metadata is preserved. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-001 | Implement Cisco CSAF endpoint discovery/auth to unlock paginated pulls. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Cisco | EXCITITOR-CONN-CISCO-01-002 | Implement Cisco CSAF paginated fetch loop with dedupe and raw persistence support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – SUSE | EXCITITOR-CONN-SUSE-01-001 | Build Rancher VEX Hub discovery/subscription path with offline snapshot support. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – MSRC | EXCITITOR-CONN-MS-01-001 | Deliver AAD onboarding/token cache for MSRC CSAF ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Oracle | EXCITITOR-CONN-ORACLE-01-001 | Implement Oracle CSAF catalogue discovery with CPU calendar awareness. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md | DONE (2025-10-17) | Team Excititor Connectors – Ubuntu | EXCITITOR-CONN-UBUNTU-01-001 | Implement Ubuntu CSAF discovery and channel selection for USN ingestion. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-001 | Wire OCI discovery/auth to fetch OpenVEX attestations for configured images. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-002 | Attestation fetch & verify loop – download DSSE attestations, trigger verification, handle retries/backoff, persist raw statements. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md | DONE (2025-10-18) | Team Excititor Connectors – OCI | EXCITITOR-CONN-OCI-01-003 | Provenance metadata & policy hooks – emit image, subject digest, issuer, and trust metadata for policy weighting/logging. | +| Sprint 6 | Excititor Ingest & Formats | src/StellaOps.Cli/TASKS.md | DONE (2025-10-18) | DevEx/CLI | EXCITITOR-CLI-01-001 | Add `excititor` CLI verbs bridging to WebService with consistent auth and offline UX. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Core/TASKS.md | DONE (2025-10-19) | Team Excititor Core & Policy | EXCITITOR-CORE-02-001 | Context signal schema prep – extend consensus models with severity/KEV/EPSS fields and update canonical serializers. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Policy/TASKS.md | DONE (2025-10-19) | Team Excititor Policy | EXCITITOR-POLICY-02-001 | Scoring coefficients & weight ceilings – add α/β options, weight boosts, and validation guidance. | +| Sprint 7 | Contextual Truth Foundations | src/StellaOps.Excititor.Attestation/TASKS.md | DONE (2025-10-16) | Team Excititor Attestation | EXCITITOR-ATTEST-01-002 | Rekor v2 client integration – ship transparency log client with retries and offline queue. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-501 | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-502 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Core/TASKS.md | DONE (2025-10-18) | Team Scanner Core | SCANNER-CORE-09-503 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-001 | Buildx driver scaffold + handshake with Scanner.Emit (local CAS). | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | OCI annotations + provenance hand-off to Attestor. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-003 | CI demo: minimal SBOM push & backend report wiring. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | +| Sprint 9 | Scanner Build-time | src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-005 | Integrate determinism guard into GitHub/Gitea workflows and archive proof artifacts. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-101 | Minimal API host with Authority enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-18) | Team Scanner WebService | SCANNER-WEB-09-102 | `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation support. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.WebService/TASKS.md | DONE (2025-10-19) | Team Scanner WebService | SCANNER-WEB-09-104 | Configuration binding for Mongo, MinIO, queue, feature flags; startup diagnostics and fail-fast policy. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-201 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-202 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-203 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-204 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Worker/TASKS.md | DONE (2025-10-19) | Team Scanner Worker | SCANNER-WORKER-09-205 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests + optional live queue smoke run. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-001 | Policy schema + binder + diagnostics. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-002 | Policy snapshot store + revision digests. | +| Sprint 9 | Policy Foundations | src/StellaOps.Policy/TASKS.md | DONE | Policy Guild | POLICY-CORE-09-003 | `/policy/preview` API (image digest → projected verdict diff). | +| Sprint 9 | DevOps Foundations | ops/devops/TASKS.md | DONE (2025-10-19) | DevOps Guild | DEVOPS-HELM-09-001 | Helm/Compose environment profiles (dev/staging/airgap) with deterministic digests. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, DevEx | DOCS-ADR-09-001 | Establish ADR process and template. | +| Sprint 9 | Docs & Governance | docs/TASKS.md | DONE (2025-10-19) | Docs Guild, Platform Events | DOCS-EVENTS-09-002 | Publish event schema catalog (`docs/events/`) for critical envelopes. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-301 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-302 | MinIO layout, immutability policies, client abstraction, and configuration binding. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Storage/TASKS.md | DONE (2025-10-19) | Team Scanner Storage | SCANNER-STORAGE-09-303 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-401 | Queue abstraction + Redis Streams adapter with ack/claim APIs and idempotency tokens. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-402 | Pluggable backend support (Redis, NATS) with configuration binding, health probes, failover docs. | +| Sprint 9 | Scanner Core Foundations | src/StellaOps.Scanner.Queue/TASKS.md | DONE (2025-10-19) | Team Scanner Queue | SCANNER-QUEUE-09-403 | Retry + dead-letter strategy with structured logs/metrics for offline deployments. | diff --git a/TODOS.md b/TODOS.md index fc6f4d21..9efaf134 100644 --- a/TODOS.md +++ b/TODOS.md @@ -1,12 +1,12 @@ -# Current Focus – FEEDCONN-CERTCC -| Task | Status | Notes | -|---|---|---| -|FEEDCONN-CERTCC-02-005 Deterministic fixtures/tests|DONE (2025-10-11)|Snapshot regression for summary/detail fetch landed; fixtures regenerate via `UPDATE_CERTCC_FIXTURES`.| -|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|Fixtures + README guidance shipped; QA can rerun with `UPDATE_CERTCC_FIXTURES=1` and share recorded-request diff with Merge.| -|FEEDCONN-CERTCC-02-007 Connector test harness remediation|DONE (2025-10-11)|Harness now resets time provider, wires Source.Common, and verifies VINCE canned responses across fetch→parse→map.| -|FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.| - -# Connector Apple Status -| Task | Status | Notes | -|---|---|---| -|FEEDCONN-APPLE-02-003 Telemetry & documentation|DONE (2025-10-11)|Apple connector meter registered with WebService OpenTelemetry metrics; README and fixtures highlight normalizedVersions coverage for conflict sprint handoff.| +# Current Focus – FEEDCONN-CERTCC +| Task | Status | Notes | +|---|---|---| +|FEEDCONN-CERTCC-02-005 Deterministic fixtures/tests|DONE (2025-10-11)|Snapshot regression for summary/detail fetch landed; fixtures regenerate via `UPDATE_CERTCC_FIXTURES`.| +|FEEDCONN-CERTCC-02-008 Snapshot coverage handoff|DONE (2025-10-11)|Fixtures + README guidance shipped; QA can rerun with `UPDATE_CERTCC_FIXTURES=1` and share recorded-request diff with Merge.| +|FEEDCONN-CERTCC-02-007 Connector test harness remediation|DONE (2025-10-11)|Harness now resets time provider, wires Source.Common, and verifies VINCE canned responses across fetch→parse→map.| +|FEEDCONN-CERTCC-02-009 Detail/map reintegration plan|DONE (2025-10-11)|Plan published in `src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md`; outlines staged enablement + rollback.| + +# Connector Apple Status +| Task | Status | Notes | +|---|---|---| +|FEEDCONN-APPLE-02-003 Telemetry & documentation|DONE (2025-10-11)|Apple connector meter registered with WebService OpenTelemetry metrics; README and fixtures highlight normalizedVersions coverage for conflict sprint handoff.| diff --git a/bench/Scanner.Analyzers/README.md b/bench/Scanner.Analyzers/README.md new file mode 100644 index 00000000..e16c899a --- /dev/null +++ b/bench/Scanner.Analyzers/README.md @@ -0,0 +1,36 @@ +# Scanner Analyzer Microbench Harness + +The bench harness exercises the language analyzers against representative filesystem layouts so that regressions are caught before they ship. + +## Layout +- `run-bench.js` – Node.js script that traverses the sample `node_modules/` and `site-packages/` trees, replicating the package discovery work performed by the upcoming analyzers. +- `config.json` – Declarative list of scenarios the harness executes. Each scenario points at a directory in `samples/`. +- `baseline.csv` – Reference numbers captured on the 4 vCPU warm rig described in `docs/12_PERFORMANCE_WORKBOOK.md`. CI publishes fresh CSVs so perf trends stay visible. + +## Running locally + +```bash +cd bench/Scanner.Analyzers +node run-bench.js --out baseline.csv --samples ../.. +``` + +The harness prints a table to stdout and writes the CSV (if `--out` is specified) with the following headers: + +``` +scenario,iterations,sample_count,mean_ms,p95_ms,max_ms +``` + +Use `--iterations` to override the default (5 passes per scenario) and `--threshold-ms` to customize the failure budget. Budgets default to 5 000 ms, aligned with the SBOM compose objective. + +## Adding scenarios +1. Drop the fixture tree under `samples//...`. +2. Append a new scenario entry to `config.json` describing: + - `id` – snake_case scenario name (also used in CSV). + - `label` – human-friendly description shown in logs. + - `root` – path to the directory that will be scanned. + - `matcher` – glob describing files that will be parsed (POSIX `**` patterns). + - `parser` – `node` or `python` to choose the metadata reader. +3. Re-run `node run-bench.js --out baseline.csv`. +4. Commit both the fixture and updated baseline. + +The harness is intentionally dependency-free to remain runnable inside minimal CI runners. diff --git a/bench/Scanner.Analyzers/baseline.csv b/bench/Scanner.Analyzers/baseline.csv new file mode 100644 index 00000000..eb0770a6 --- /dev/null +++ b/bench/Scanner.Analyzers/baseline.csv @@ -0,0 +1,3 @@ +scenario,iterations,sample_count,mean_ms,p95_ms,max_ms +node_monorepo_walk,5,4,233.9428,319.8564,344.4611 +python_site_packages_walk,5,3,72.9166,74.8970,74.9884 diff --git a/bench/Scanner.Analyzers/config.json b/bench/Scanner.Analyzers/config.json new file mode 100644 index 00000000..ec86eecf --- /dev/null +++ b/bench/Scanner.Analyzers/config.json @@ -0,0 +1,20 @@ +{ + "thresholdMs": 5000, + "iterations": 5, + "scenarios": [ + { + "id": "node_monorepo_walk", + "label": "Node.js monorepo package.json harvest", + "root": "samples/runtime/npm-monorepo/node_modules", + "matcher": "**/package.json", + "parser": "node" + }, + { + "id": "python_site_packages_walk", + "label": "Python site-packages dist-info crawl", + "root": "samples/runtime/python-venv/lib/python3.11/site-packages", + "matcher": "**/*.dist-info/METADATA", + "parser": "python" + } + ] +} diff --git a/bench/Scanner.Analyzers/lang/README.md b/bench/Scanner.Analyzers/lang/README.md new file mode 100644 index 00000000..747e52f6 --- /dev/null +++ b/bench/Scanner.Analyzers/lang/README.md @@ -0,0 +1,12 @@ +# Scanner Language Analyzer Benchmarks + +This directory will capture benchmark results for language analyzers (Node, Python, Go, .NET, Rust). + +Pending tasks: +- LA1: Node analyzer microbench CSV + flamegraph. +- LA2: Python hash throughput CSV. +- LA3: Go build info extraction benchmarks. +- LA4: .NET RID dedupe performance matrix. +- LA5: Rust heuristic coverage comparisons. + +Results should be committed as deterministic CSV/JSON outputs with accompanying methodology notes. diff --git a/bench/Scanner.Analyzers/run-bench.js b/bench/Scanner.Analyzers/run-bench.js new file mode 100644 index 00000000..7ed00823 --- /dev/null +++ b/bench/Scanner.Analyzers/run-bench.js @@ -0,0 +1,249 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { performance } = require('perf_hooks'); + +function globToRegExp(pattern) { + let working = pattern + .replace(/\*\*/g, ':::DOUBLE_WILDCARD:::') + .replace(/\*/g, ':::SINGLE_WILDCARD:::'); + working = working.replace(/([.+^${}()|[\]\\])/g, '\\$1'); + working = working + .replace(/:::DOUBLE_WILDCARD:::\//g, '(?:.*/)?') + .replace(/:::DOUBLE_WILDCARD:::/g, '.*') + .replace(/:::SINGLE_WILDCARD:::/g, '[^/]*'); + return new RegExp(`^${working}$`); +} + +function walkFiles(root, matcher) { + const out = []; + const stack = [root]; + while (stack.length) { + const current = stack.pop(); + const stat = fs.statSync(current, { throwIfNoEntry: true }); + if (stat.isDirectory()) { + const entries = fs.readdirSync(current); + for (const entry of entries) { + stack.push(path.join(current, entry)); + } + } else if (stat.isFile()) { + const relativePath = path.relative(root, current).replace(/\\/g, '/'); + if (matcher.test(relativePath)) { + out.push(current); + } + } + } + return out; +} + +function parseArgs(argv) { + const args = { + config: path.join(__dirname, 'config.json'), + iterations: undefined, + thresholdMs: undefined, + out: undefined, + repoRoot: path.join(__dirname, '..', '..'), + }; + + for (let i = 2; i < argv.length; i++) { + const current = argv[i]; + switch (current) { + case '--config': + args.config = argv[++i]; + break; + case '--iterations': + args.iterations = Number(argv[++i]); + break; + case '--threshold-ms': + args.thresholdMs = Number(argv[++i]); + break; + case '--out': + args.out = argv[++i]; + break; + case '--repo-root': + case '--samples': + args.repoRoot = argv[++i]; + break; + default: + throw new Error(`Unknown argument: ${current}`); + } + } + + return args; +} + +function loadConfig(configPath) { + const json = fs.readFileSync(configPath, 'utf8'); + const cfg = JSON.parse(json); + if (!Array.isArray(cfg.scenarios) || cfg.scenarios.length === 0) { + throw new Error('config.scenarios must be a non-empty array'); + } + return cfg; +} + +function ensureWithinRepo(repoRoot, target) { + const relative = path.relative(repoRoot, target); + if (relative === '' || relative === '.') { + return true; + } + return !relative.startsWith('..') && !path.isAbsolute(relative); +} + +function parseNodePackage(contents) { + const parsed = JSON.parse(contents); + if (!parsed.name || !parsed.version) { + throw new Error('package.json missing name/version'); + } + return { name: parsed.name, version: parsed.version }; +} + +function parsePythonMetadata(contents) { + let name; + let version; + for (const line of contents.split(/\r?\n/)) { + if (!name && line.startsWith('Name:')) { + name = line.slice(5).trim(); + } else if (!version && line.startsWith('Version:')) { + version = line.slice(8).trim(); + } + if (name && version) { + break; + } + } + if (!name || !version) { + throw new Error('METADATA missing Name/Version headers'); + } + return { name, version }; +} + +function formatRow(row) { + const cols = [ + row.id.padEnd(28), + row.sampleCount.toString().padStart(5), + row.meanMs.toFixed(2).padStart(9), + row.p95Ms.toFixed(2).padStart(9), + row.maxMs.toFixed(2).padStart(9), + ]; + return cols.join(' | '); +} + +function percentile(sortedDurations, percentile) { + if (sortedDurations.length === 0) { + return 0; + } + const rank = (percentile / 100) * (sortedDurations.length - 1); + const lower = Math.floor(rank); + const upper = Math.ceil(rank); + const weight = rank - lower; + if (upper >= sortedDurations.length) { + return sortedDurations[lower]; + } + return sortedDurations[lower] + weight * (sortedDurations[upper] - sortedDurations[lower]); +} + +function main() { + const args = parseArgs(process.argv); + const cfg = loadConfig(args.config); + const iterations = args.iterations ?? cfg.iterations ?? 5; + const thresholdMs = args.thresholdMs ?? cfg.thresholdMs ?? 5000; + + const results = []; + const failures = []; + + for (const scenario of cfg.scenarios) { + const scenarioRoot = path.resolve(args.repoRoot, scenario.root); + if (!ensureWithinRepo(args.repoRoot, scenarioRoot)) { + throw new Error(`Scenario root ${scenario.root} escapes repo root ${args.repoRoot}`); + } + if (!fs.existsSync(scenarioRoot)) { + throw new Error(`Scenario root ${scenarioRoot} does not exist`); + } + + const matcher = globToRegExp(scenario.matcher.replace(/\\/g, '/')); + const durations = []; + let sampleCount = 0; + + for (let attempt = 0; attempt < iterations; attempt++) { + const start = performance.now(); + const files = walkFiles(scenarioRoot, matcher); + if (files.length === 0) { + throw new Error(`Scenario ${scenario.id} matched no files`); + } + + for (const filePath of files) { + const contents = fs.readFileSync(filePath, 'utf8'); + if (scenario.parser === 'node') { + parseNodePackage(contents); + } else if (scenario.parser === 'python') { + parsePythonMetadata(contents); + } else { + throw new Error(`Unknown parser ${scenario.parser} for scenario ${scenario.id}`); + } + } + const end = performance.now(); + durations.push(end - start); + sampleCount = files.length; + } + + durations.sort((a, b) => a - b); + const mean = durations.reduce((acc, value) => acc + value, 0) / durations.length; + const p95 = percentile(durations, 95); + const max = durations[durations.length - 1]; + + if (max > thresholdMs) { + failures.push(`${scenario.id} exceeded threshold: ${(max).toFixed(2)} ms > ${thresholdMs} ms`); + } + + results.push({ + id: scenario.id, + label: scenario.label, + sampleCount, + meanMs: mean, + p95Ms: p95, + maxMs: max, + iterations, + }); + } + + console.log('Scenario | Count | Mean(ms) | P95(ms) | Max(ms)'); + console.log('---------------------------- | ----- | --------- | --------- | ----------'); + for (const row of results) { + console.log(formatRow(row)); + } + + if (args.out) { + const header = 'scenario,iterations,sample_count,mean_ms,p95_ms,max_ms\n'; + const csvRows = results + .map((row) => + [ + row.id, + row.iterations, + row.sampleCount, + row.meanMs.toFixed(4), + row.p95Ms.toFixed(4), + row.maxMs.toFixed(4), + ].join(',') + ) + .join('\n'); + fs.writeFileSync(args.out, header + csvRows + '\n', 'utf8'); + } + + if (failures.length > 0) { + console.error('\nPerformance threshold exceeded:'); + for (const failure of failures) { + console.error(` - ${failure}`); + } + process.exitCode = 1; + } +} + +if (require.main === module) { + try { + main(); + } catch (err) { + console.error(err instanceof Error ? err.message : err); + process.exit(1); + } +} diff --git a/bench/TASKS.md b/bench/TASKS.md new file mode 100644 index 00000000..de8ef5b7 --- /dev/null +++ b/bench/TASKS.md @@ -0,0 +1,8 @@ +# Benchmarks Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| BENCH-SCANNER-10-001 | DONE | Bench Guild, Scanner Team | SCANNER-ANALYZERS-LANG-10-303 | Analyzer microbench harness (node_modules, site-packages) + baseline CSV. | Harness committed under `bench/Scanner.Analyzers`; baseline CSV recorded; CI job publishes results. | +| BENCH-SCANNER-10-002 | TODO | Bench Guild, Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301..309 | Wire real language analyzers into bench harness & refresh baselines post-implementation. | Harness executes analyzer assemblies end-to-end; updated baseline committed; CI trend doc linked. | +| BENCH-IMPACT-16-001 | TODO | Bench Guild, Scheduler Team | SCHED-IMPACT-16-301 | ImpactIndex throughput bench (resolve 10k productKeys) + RAM profile. | Benchmark script ready; baseline metrics recorded; alert thresholds defined. | +| BENCH-NOTIFY-15-001 | TODO | Bench Guild, Notify Team | NOTIFY-ENGINE-15-301 | Notify dispatch throughput bench (vary rule density) with results CSV. | Bench executed; results stored; regression alert configured. | diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 00000000..0353366c --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,20 @@ +# Deployment Profiles + +This directory contains deterministic deployment bundles for the core Stella Ops stack. All manifests reference immutable image digests and map 1:1 to the release manifests stored under `deploy/releases/`. + +## Structure + +- `releases/` – canonical release manifests (edge, stable, airgap) used to source image digests. +- `compose/` – Docker Compose bundles for dev/stage/airgap targets plus `.env` seed files. +- `compose/docker-compose.mirror.yaml` – managed mirror bundle for `*.stella-ops.org` with gateway cache and multi-tenant auth. +- `helm/stellaops/` – multi-profile Helm chart with values files for dev/stage/airgap. +- `tools/validate-profiles.sh` – helper that runs `docker compose config` and `helm lint/template` for every profile. + +## Workflow + +1. Update or add a release manifest under `releases/` with the new digests. +2. Mirror the digests into the Compose and Helm profiles that correspond to that channel. +3. Run `deploy/tools/validate-profiles.sh` (requires Docker CLI and Helm) to ensure the bundles lint and template cleanly. +4. Commit the change alongside any documentation updates (e.g. install guide cross-links). + +Maintaining the digest linkage keeps offline/air-gapped installs reproducible and avoids tag drift between environments. diff --git a/deploy/compose/README.md b/deploy/compose/README.md new file mode 100644 index 00000000..f8c2ccc3 --- /dev/null +++ b/deploy/compose/README.md @@ -0,0 +1,31 @@ +# Stella Ops Compose Profiles + +These Compose bundles ship the minimum services required to exercise the scanner pipeline plus control-plane dependencies. Every profile is pinned to immutable image digests sourced from `deploy/releases/*.yaml` and is linted via `docker compose config` in CI. + +## Layout + +| Path | Purpose | +| ---- | ------- | +| `docker-compose.dev.yaml` | Edge/nightly stack tuned for laptops and iterative work. | +| `docker-compose.stage.yaml` | Stable channel stack mirroring pre-production clusters. | +| `docker-compose.airgap.yaml` | Stable stack with air-gapped defaults (no outbound hostnames). | +| `docker-compose.mirror.yaml` | Managed mirror topology for `*.stella-ops.org` distribution (Concelier + Excititor + CDN gateway). | +| `env/*.env.example` | Seed `.env` files that document required secrets and ports per profile. | + +## Usage + +```bash +cp env/dev.env.example dev.env +docker compose --env-file dev.env -f docker-compose.dev.yaml config +docker compose --env-file dev.env -f docker-compose.dev.yaml up -d +``` + +The stage and airgap variants behave the same way—swap the file names accordingly. All profiles expose 443/8443 for the UI and REST APIs, and they share a `stellaops` Docker network scoped to the compose project. + +### Updating to a new release + +1. Import the new manifest into `deploy/releases/` (see `deploy/README.md`). +2. Update image digests in the relevant Compose file(s). +3. Re-run `docker compose config` to confirm the bundle is deterministic. + +Keep digests synchronized between Compose, Helm, and the release manifest to preserve reproducibility guarantees. `deploy/tools/validate-profiles.sh` performs a quick audit. diff --git a/deploy/compose/docker-compose.airgap.yaml b/deploy/compose/docker-compose.airgap.yaml new file mode 100644 index 00000000..fea80dba --- /dev/null +++ b/deploy/compose/docker-compose.airgap.yaml @@ -0,0 +1,204 @@ +x-release-labels: &release-labels + com.stellaops.release.version: "2025.09.2-airgap" + com.stellaops.release.channel: "airgap" + com.stellaops.profile: "airgap" + +networks: + stellaops: + driver: bridge + +volumes: + mongo-data: + minio-data: + concelier-jobs: + nats-data: + +services: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + command: ["mongod", "--bind_ip_all"] + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" + volumes: + - mongo-data:/data/db + networks: + - stellaops + labels: *release-labels + + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + volumes: + - minio-data:/data + ports: + - "${MINIO_CONSOLE_PORT:-29001}:9001" + networks: + - stellaops + labels: *release-labels + + nats: + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + command: + - "-js" + - "-sd" + - /data + restart: unless-stopped + ports: + - "${NATS_CLIENT_PORT:-24222}:4222" + volumes: + - nats-data:/data + networks: + - stellaops + labels: *release-labels + + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:5551a3269b7008cd5aceecf45df018c67459ed519557ccbe48b093b926a39bcc + restart: unless-stopped + depends_on: + - mongo + environment: + STELLAOPS_AUTHORITY__ISSUER: "${AUTHORITY_ISSUER}" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + volumes: + - ../../etc/authority.yaml:/etc/authority.yaml:ro + - ../../etc/authority.plugins:/app/etc/authority.plugins:ro + ports: + - "${AUTHORITY_PORT:-8440}:8440" + networks: + - stellaops + labels: *release-labels + + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:ddbbd664a42846cea6b40fca6465bc679b30f72851158f300d01a8571c5478fc + restart: unless-stopped + depends_on: + - authority + environment: + SIGNER__AUTHORITY__BASEURL: "https://authority:8440" + SIGNER__POE__INTROSPECTURL: "${SIGNER_POE_INTROSPECT_URL}" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${SIGNER_PORT:-8441}:8441" + networks: + - stellaops + labels: *release-labels + + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:1ff0a3124d66d3a2702d8e421df40fbd98cc75cb605d95510598ebbae1433c50 + restart: unless-stopped + depends_on: + - signer + environment: + ATTESTOR__SIGNER__BASEURL: "https://signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${ATTESTOR_PORT:-8442}:8442" + networks: + - stellaops + labels: *release-labels + + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:29e2e1a0972707e092cbd3d370701341f9fec2aa9316fb5d8100480f2a1c76b5 + restart: unless-stopped + depends_on: + - mongo + - minio + environment: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + CONCELIER__AUTHORITY__BASEURL: "https://authority:8440" + CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" + CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "${AUTHORITY_OFFLINE_CACHE_TOLERANCE:-00:30:00}" + volumes: + - concelier-jobs:/var/lib/concelier/jobs + ports: + - "${CONCELIER_PORT:-8445}:8445" + networks: + - stellaops + labels: *release-labels + + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 + restart: unless-stopped + depends_on: + - concelier + - minio + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + ports: + - "${SCANNER_WEB_PORT:-8444}:8444" + networks: + - stellaops + labels: *release-labels + + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 + restart: unless-stopped + depends_on: + - scanner-web + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + networks: + - stellaops + labels: *release-labels + + notify-web: + image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2} + restart: unless-stopped + depends_on: + - mongo + - authority + environment: + DOTNET_ENVIRONMENT: Production + volumes: + - ../../etc/notify.prod.yaml:/app/etc/notify.yaml:ro + ports: + - "${NOTIFY_WEB_PORT:-9446}:8446" + networks: + - stellaops + labels: *release-labels + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:65c0ee13f773efe920d7181512349a09d363ab3f3e177d276136bd2742325a68 + restart: unless-stopped + depends_on: + - concelier + environment: + EXCITITOR__CONCELIER__BASEURL: "https://concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + networks: + - stellaops + labels: *release-labels + + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:bee9668011ff414572131dc777faab4da24473fe12c230893f161cabee092a1d + restart: unless-stopped + depends_on: + - scanner-web + environment: + STELLAOPS_UI__BACKEND__BASEURL: "https://scanner-web:8444" + ports: + - "${UI_PORT:-9443}:8443" + networks: + - stellaops + labels: *release-labels diff --git a/deploy/compose/docker-compose.dev.yaml b/deploy/compose/docker-compose.dev.yaml new file mode 100644 index 00000000..c9f5d572 --- /dev/null +++ b/deploy/compose/docker-compose.dev.yaml @@ -0,0 +1,202 @@ +x-release-labels: &release-labels + com.stellaops.release.version: "2025.10.0-edge" + com.stellaops.release.channel: "edge" + com.stellaops.profile: "dev" + +networks: + stellaops: + driver: bridge + +volumes: + mongo-data: + minio-data: + concelier-jobs: + nats-data: + +services: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + command: ["mongod", "--bind_ip_all"] + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" + volumes: + - mongo-data:/data/db + networks: + - stellaops + labels: *release-labels + + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + volumes: + - minio-data:/data + ports: + - "${MINIO_CONSOLE_PORT:-9001}:9001" + networks: + - stellaops + labels: *release-labels + + nats: + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + command: + - "-js" + - "-sd" + - /data + restart: unless-stopped + ports: + - "${NATS_CLIENT_PORT:-4222}:4222" + volumes: + - nats-data:/data + networks: + - stellaops + labels: *release-labels + + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd + restart: unless-stopped + depends_on: + - mongo + environment: + STELLAOPS_AUTHORITY__ISSUER: "${AUTHORITY_ISSUER}" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + volumes: + - ../../etc/authority.yaml:/etc/authority.yaml:ro + - ../../etc/authority.plugins:/app/etc/authority.plugins:ro + ports: + - "${AUTHORITY_PORT:-8440}:8440" + networks: + - stellaops + labels: *release-labels + + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8bfef9a75783883d49fc18e3566553934e970b00ee090abee9cb110d2d5c3298 + restart: unless-stopped + depends_on: + - authority + environment: + SIGNER__AUTHORITY__BASEURL: "https://authority:8440" + SIGNER__POE__INTROSPECTURL: "${SIGNER_POE_INTROSPECT_URL}" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${SIGNER_PORT:-8441}:8441" + networks: + - stellaops + labels: *release-labels + + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:5cc417948c029da01dccf36e4645d961a3f6d8de7e62fe98d845f07cd2282114 + restart: unless-stopped + depends_on: + - signer + environment: + ATTESTOR__SIGNER__BASEURL: "https://signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${ATTESTOR_PORT:-8442}:8442" + networks: + - stellaops + labels: *release-labels + + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + restart: unless-stopped + depends_on: + - mongo + - minio + environment: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + CONCELIER__AUTHORITY__BASEURL: "https://authority:8440" + volumes: + - concelier-jobs:/var/lib/concelier/jobs + ports: + - "${CONCELIER_PORT:-8445}:8445" + networks: + - stellaops + labels: *release-labels + + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 + restart: unless-stopped + depends_on: + - concelier + - minio + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + ports: + - "${SCANNER_WEB_PORT:-8444}:8444" + networks: + - stellaops + labels: *release-labels + + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 + restart: unless-stopped + depends_on: + - scanner-web + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + networks: + - stellaops + labels: *release-labels + + notify-web: + image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge} + restart: unless-stopped + depends_on: + - mongo + - authority + environment: + DOTNET_ENVIRONMENT: Development + volumes: + - ../../etc/notify.dev.yaml:/app/etc/notify.yaml:ro + ports: + - "${NOTIFY_WEB_PORT:-8446}:8446" + networks: + - stellaops + labels: *release-labels + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + restart: unless-stopped + depends_on: + - concelier + environment: + EXCITITOR__CONCELIER__BASEURL: "https://concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + networks: + - stellaops + labels: *release-labels + + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf + restart: unless-stopped + depends_on: + - scanner-web + environment: + STELLAOPS_UI__BACKEND__BASEURL: "https://scanner-web:8444" + ports: + - "${UI_PORT:-8443}:8443" + networks: + - stellaops + labels: *release-labels diff --git a/deploy/compose/docker-compose.mirror.yaml b/deploy/compose/docker-compose.mirror.yaml new file mode 100644 index 00000000..3a8b5ec9 --- /dev/null +++ b/deploy/compose/docker-compose.mirror.yaml @@ -0,0 +1,152 @@ +x-release-labels: &release-labels + com.stellaops.release.version: "2025.10.0-edge" + com.stellaops.release.channel: "edge" + com.stellaops.profile: "mirror-managed" + +networks: + mirror: + driver: bridge + +volumes: + mongo-data: + minio-data: + concelier-jobs: + concelier-exports: + excititor-exports: + nginx-cache: + +services: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + command: ["mongod", "--bind_ip_all"] + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME:-stellaops_mirror}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD:-mirror-password}" + volumes: + - mongo-data:/data/db + networks: + - mirror + labels: *release-labels + + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER:-stellaops-mirror}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD:-mirror-minio-secret}" + volumes: + - minio-data:/data + networks: + - mirror + labels: *release-labels + + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + restart: unless-stopped + depends_on: + - mongo + - minio + environment: + ASPNETCORE_URLS: "http://+:8445" + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME:-stellaops_mirror}:${MONGO_INITDB_ROOT_PASSWORD:-mirror-password}@mongo:27017/concelier?authSource=admin" + CONCELIER__STORAGE__S3__ENDPOINT: "http://minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER:-stellaops-mirror}" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD:-mirror-minio-secret}" + CONCELIER__TELEMETRY__SERVICENAME: "stellaops-concelier-mirror" + CONCELIER__MIRROR__ENABLED: "true" + CONCELIER__MIRROR__EXPORTROOT: "/exports/json" + CONCELIER__MIRROR__LATESTDIRECTORYNAME: "${CONCELIER_MIRROR_LATEST_SEGMENT:-latest}" + CONCELIER__MIRROR__MIRRORDIRECTORYNAME: "${CONCELIER_MIRROR_DIRECTORY_SEGMENT:-mirror}" + CONCELIER__MIRROR__REQUIREAUTHENTICATION: "${CONCELIER_MIRROR_REQUIRE_AUTH:-true}" + CONCELIER__MIRROR__MAXINDEXREQUESTSPERHOUR: "${CONCELIER_MIRROR_INDEX_BUDGET:-600}" + CONCELIER__MIRROR__DOMAINS__0__ID: "${CONCELIER_MIRROR_DOMAIN_PRIMARY_ID:-primary}" + CONCELIER__MIRROR__DOMAINS__0__DISPLAYNAME: "${CONCELIER_MIRROR_DOMAIN_PRIMARY_NAME:-Primary Mirror}" + CONCELIER__MIRROR__DOMAINS__0__REQUIREAUTHENTICATION: "${CONCELIER_MIRROR_DOMAIN_PRIMARY_AUTH:-true}" + CONCELIER__MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR: "${CONCELIER_MIRROR_DOMAIN_PRIMARY_DOWNLOAD_BUDGET:-3600}" + CONCELIER__MIRROR__DOMAINS__1__ID: "${CONCELIER_MIRROR_DOMAIN_SECONDARY_ID:-community}" + CONCELIER__MIRROR__DOMAINS__1__DISPLAYNAME: "${CONCELIER_MIRROR_DOMAIN_SECONDARY_NAME:-Community Mirror}" + CONCELIER__MIRROR__DOMAINS__1__REQUIREAUTHENTICATION: "${CONCELIER_MIRROR_DOMAIN_SECONDARY_AUTH:-false}" + CONCELIER__MIRROR__DOMAINS__1__MAXDOWNLOADREQUESTSPERHOUR: "${CONCELIER_MIRROR_DOMAIN_SECONDARY_DOWNLOAD_BUDGET:-1800}" + CONCELIER__AUTHORITY__ENABLED: "${CONCELIER_AUTHORITY_ENABLED:-true}" + CONCELIER__AUTHORITY__ALLOWANONYMOUSFALLBACK: "${CONCELIER_AUTHORITY_ALLOW_ANON:-false}" + CONCELIER__AUTHORITY__ISSUER: "${CONCELIER_AUTHORITY_ISSUER:-https://authority.stella-ops.org}" + CONCELIER__AUTHORITY__METADATAADDRESS: "${CONCELIER_AUTHORITY_METADATA:-}" + CONCELIER__AUTHORITY__CLIENTID: "${CONCELIER_AUTHORITY_CLIENT_ID:-stellaops-concelier-mirror}" + CONCELIER__AUTHORITY__CLIENTSECRETFILE: "/run/secrets/concelier-authority-client" + CONCELIER__AUTHORITY__CLIENTSCOPES__0: "${CONCELIER_AUTHORITY_SCOPE:-concelier.mirror.read}" + CONCELIER__AUTHORITY__AUDIENCES__0: "${CONCELIER_AUTHORITY_AUDIENCE:-api://concelier.mirror}" + CONCELIER__AUTHORITY__BYPASSNETWORKS__0: "10.0.0.0/8" + CONCELIER__AUTHORITY__BYPASSNETWORKS__1: "127.0.0.1/32" + CONCELIER__AUTHORITY__BYPASSNETWORKS__2: "::1/128" + CONCELIER__AUTHORITY__RESILIENCE__ENABLERETRIES: "true" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__0: "00:00:01" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__1: "00:00:02" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__2: "00:00:05" + CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" + CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "00:10:00" + volumes: + - concelier-jobs:/var/lib/concelier/jobs + - concelier-exports:/exports/json + - ./mirror-secrets:/run/secrets:ro + networks: + - mirror + labels: *release-labels + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + restart: unless-stopped + depends_on: + - mongo + environment: + ASPNETCORE_URLS: "http://+:8448" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME:-stellaops_mirror}:${MONGO_INITDB_ROOT_PASSWORD:-mirror-password}@mongo:27017/excititor?authSource=admin" + EXCITITOR__STORAGE__MONGO__DATABASENAME: "${EXCITITOR_MONGO_DATABASE:-excititor}" + EXCITITOR__ARTIFACTS__FILESYSTEM__ROOT: "/exports" + EXCITITOR__ARTIFACTS__FILESYSTEM__OVERWRITEEXISTING: "${EXCITITOR_FILESYSTEM_OVERWRITE:-false}" + EXCITITOR__MIRROR__DOMAINS__0__ID: "${EXCITITOR_MIRROR_DOMAIN_PRIMARY_ID:-primary}" + EXCITITOR__MIRROR__DOMAINS__0__DISPLAYNAME: "${EXCITITOR_MIRROR_DOMAIN_PRIMARY_NAME:-Primary Mirror}" + EXCITITOR__MIRROR__DOMAINS__0__REQUIREAUTHENTICATION: "${EXCITITOR_MIRROR_DOMAIN_PRIMARY_AUTH:-true}" + EXCITITOR__MIRROR__DOMAINS__0__MAXINDEXREQUESTSPERHOUR: "${EXCITITOR_MIRROR_DOMAIN_PRIMARY_INDEX_BUDGET:-300}" + EXCITITOR__MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR: "${EXCITITOR_MIRROR_DOMAIN_PRIMARY_DOWNLOAD_BUDGET:-2400}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__KEY: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_KEY:-consensus-json}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__FORMAT: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_FORMAT:-json}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__VIEW: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_VIEW:-consensus}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__KEY: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_KEY:-consensus-openvex}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__FORMAT: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_FORMAT:-openvex}" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__VIEW: "${EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_VIEW:-consensus}" + EXCITITOR__MIRROR__DOMAINS__1__ID: "${EXCITITOR_MIRROR_DOMAIN_SECONDARY_ID:-community}" + EXCITITOR__MIRROR__DOMAINS__1__DISPLAYNAME: "${EXCITITOR_MIRROR_DOMAIN_SECONDARY_NAME:-Community Mirror}" + EXCITITOR__MIRROR__DOMAINS__1__REQUIREAUTHENTICATION: "${EXCITITOR_MIRROR_DOMAIN_SECONDARY_AUTH:-false}" + EXCITITOR__MIRROR__DOMAINS__1__MAXINDEXREQUESTSPERHOUR: "${EXCITITOR_MIRROR_DOMAIN_SECONDARY_INDEX_BUDGET:-120}" + EXCITITOR__MIRROR__DOMAINS__1__MAXDOWNLOADREQUESTSPERHOUR: "${EXCITITOR_MIRROR_DOMAIN_SECONDARY_DOWNLOAD_BUDGET:-600}" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__KEY: "${EXCITITOR_MIRROR_SECONDARY_EXPORT_KEY:-community-consensus}" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__FORMAT: "${EXCITITOR_MIRROR_SECONDARY_EXPORT_FORMAT:-json}" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__VIEW: "${EXCITITOR_MIRROR_SECONDARY_EXPORT_VIEW:-consensus}" + volumes: + - excititor-exports:/exports + - ./mirror-secrets:/run/secrets:ro + expose: + - "8448" + networks: + - mirror + labels: *release-labels + + mirror-gateway: + image: docker.io/library/nginx@sha256:208b70eefac13ee9be00e486f79c695b15cef861c680527171a27d253d834be9 + restart: unless-stopped + depends_on: + - concelier + - excititor + ports: + - "${MIRROR_GATEWAY_HTTP_PORT:-8080}:80" + - "${MIRROR_GATEWAY_HTTPS_PORT:-9443}:443" + volumes: + - nginx-cache:/var/cache/nginx + - ./mirror-gateway/conf.d:/etc/nginx/conf.d:ro + - ./mirror-gateway/tls:/etc/nginx/tls:ro + - ./mirror-gateway/secrets:/etc/nginx/secrets:ro + networks: + - mirror + labels: *release-labels diff --git a/deploy/compose/docker-compose.stage.yaml b/deploy/compose/docker-compose.stage.yaml new file mode 100644 index 00000000..064d7c8c --- /dev/null +++ b/deploy/compose/docker-compose.stage.yaml @@ -0,0 +1,202 @@ +x-release-labels: &release-labels + com.stellaops.release.version: "2025.09.2" + com.stellaops.release.channel: "stable" + com.stellaops.profile: "stage" + +networks: + stellaops: + driver: bridge + +volumes: + mongo-data: + minio-data: + concelier-jobs: + nats-data: + +services: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + command: ["mongod", "--bind_ip_all"] + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "${MONGO_INITDB_ROOT_USERNAME}" + MONGO_INITDB_ROOT_PASSWORD: "${MONGO_INITDB_ROOT_PASSWORD}" + volumes: + - mongo-data:/data/db + networks: + - stellaops + labels: *release-labels + + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data", "--console-address", ":9001"] + restart: unless-stopped + environment: + MINIO_ROOT_USER: "${MINIO_ROOT_USER}" + MINIO_ROOT_PASSWORD: "${MINIO_ROOT_PASSWORD}" + volumes: + - minio-data:/data + ports: + - "${MINIO_CONSOLE_PORT:-9001}:9001" + networks: + - stellaops + labels: *release-labels + + nats: + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + command: + - "-js" + - "-sd" + - /data + restart: unless-stopped + ports: + - "${NATS_CLIENT_PORT:-4222}:4222" + volumes: + - nats-data:/data + networks: + - stellaops + labels: *release-labels + + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + restart: unless-stopped + depends_on: + - mongo + environment: + STELLAOPS_AUTHORITY__ISSUER: "${AUTHORITY_ISSUER}" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + volumes: + - ../../etc/authority.yaml:/etc/authority.yaml:ro + - ../../etc/authority.plugins:/app/etc/authority.plugins:ro + ports: + - "${AUTHORITY_PORT:-8440}:8440" + networks: + - stellaops + labels: *release-labels + + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + restart: unless-stopped + depends_on: + - authority + environment: + SIGNER__AUTHORITY__BASEURL: "https://authority:8440" + SIGNER__POE__INTROSPECTURL: "${SIGNER_POE_INTROSPECT_URL}" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${SIGNER_PORT:-8441}:8441" + networks: + - stellaops + labels: *release-labels + + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + restart: unless-stopped + depends_on: + - signer + environment: + ATTESTOR__SIGNER__BASEURL: "https://signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + ports: + - "${ATTESTOR_PORT:-8442}:8442" + networks: + - stellaops + labels: *release-labels + + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + restart: unless-stopped + depends_on: + - mongo + - minio + environment: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + CONCELIER__AUTHORITY__BASEURL: "https://authority:8440" + volumes: + - concelier-jobs:/var/lib/concelier/jobs + ports: + - "${CONCELIER_PORT:-8445}:8445" + networks: + - stellaops + labels: *release-labels + + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + restart: unless-stopped + depends_on: + - concelier + - minio + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + ports: + - "${SCANNER_WEB_PORT:-8444}:8444" + networks: + - stellaops + labels: *release-labels + + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + restart: unless-stopped + depends_on: + - scanner-web + - nats + environment: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}" + SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}" + networks: + - stellaops + labels: *release-labels + + notify-web: + image: ${NOTIFY_WEB_IMAGE:-registry.stella-ops.org/stellaops/notify-web:2025.09.2} + restart: unless-stopped + depends_on: + - mongo + - authority + environment: + DOTNET_ENVIRONMENT: Production + volumes: + - ../../etc/notify.stage.yaml:/app/etc/notify.yaml:ro + ports: + - "${NOTIFY_WEB_PORT:-8446}:8446" + networks: + - stellaops + labels: *release-labels + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + restart: unless-stopped + depends_on: + - concelier + environment: + EXCITITOR__CONCELIER__BASEURL: "https://concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017" + networks: + - stellaops + labels: *release-labels + + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + restart: unless-stopped + depends_on: + - scanner-web + environment: + STELLAOPS_UI__BACKEND__BASEURL: "https://scanner-web:8444" + ports: + - "${UI_PORT:-8443}:8443" + networks: + - stellaops + labels: *release-labels diff --git a/deploy/compose/env/airgap.env.example b/deploy/compose/env/airgap.env.example new file mode 100644 index 00000000..e8518edf --- /dev/null +++ b/deploy/compose/env/airgap.env.example @@ -0,0 +1,17 @@ +# Substitutions for docker-compose.airgap.yaml +MONGO_INITDB_ROOT_USERNAME=stellaops +MONGO_INITDB_ROOT_PASSWORD=airgap-password +MINIO_ROOT_USER=stellaops-offline +MINIO_ROOT_PASSWORD=airgap-minio-secret +MINIO_CONSOLE_PORT=29001 +AUTHORITY_ISSUER=https://authority.airgap.local +AUTHORITY_PORT=8440 +SIGNER_POE_INTROSPECT_URL=file:///offline/poe/introspect.json +SIGNER_PORT=8441 +ATTESTOR_PORT=8442 +CONCELIER_PORT=8445 +SCANNER_WEB_PORT=8444 +UI_PORT=9443 +NATS_CLIENT_PORT=24222 +SCANNER_QUEUE_BROKER=nats://nats:4222 +AUTHORITY_OFFLINE_CACHE_TOLERANCE=00:45:00 diff --git a/deploy/compose/env/dev.env.example b/deploy/compose/env/dev.env.example new file mode 100644 index 00000000..3cc077b1 --- /dev/null +++ b/deploy/compose/env/dev.env.example @@ -0,0 +1,16 @@ +# Substitutions for docker-compose.dev.yaml +MONGO_INITDB_ROOT_USERNAME=stellaops +MONGO_INITDB_ROOT_PASSWORD=dev-password +MINIO_ROOT_USER=stellaops +MINIO_ROOT_PASSWORD=dev-minio-secret +MINIO_CONSOLE_PORT=9001 +AUTHORITY_ISSUER=https://authority.localtest.me +AUTHORITY_PORT=8440 +SIGNER_POE_INTROSPECT_URL=https://licensing.svc.local/introspect +SIGNER_PORT=8441 +ATTESTOR_PORT=8442 +CONCELIER_PORT=8445 +SCANNER_WEB_PORT=8444 +UI_PORT=8443 +NATS_CLIENT_PORT=4222 +SCANNER_QUEUE_BROKER=nats://nats:4222 diff --git a/deploy/compose/env/mirror.env.example b/deploy/compose/env/mirror.env.example new file mode 100644 index 00000000..d848cbd6 --- /dev/null +++ b/deploy/compose/env/mirror.env.example @@ -0,0 +1,57 @@ +# Managed mirror profile substitutions + +# Core infrastructure credentials +MONGO_INITDB_ROOT_USERNAME=stellaops_mirror +MONGO_INITDB_ROOT_PASSWORD=mirror-password +MINIO_ROOT_USER=stellaops-mirror +MINIO_ROOT_PASSWORD=mirror-minio-secret + +# Mirror HTTP listeners +MIRROR_GATEWAY_HTTP_PORT=8080 +MIRROR_GATEWAY_HTTPS_PORT=9443 + +# Concelier mirror configuration +CONCELIER_MIRROR_LATEST_SEGMENT=latest +CONCELIER_MIRROR_DIRECTORY_SEGMENT=mirror +CONCELIER_MIRROR_REQUIRE_AUTH=true +CONCELIER_MIRROR_INDEX_BUDGET=600 +CONCELIER_MIRROR_DOMAIN_PRIMARY_ID=primary +CONCELIER_MIRROR_DOMAIN_PRIMARY_NAME=Primary Mirror +CONCELIER_MIRROR_DOMAIN_PRIMARY_AUTH=true +CONCELIER_MIRROR_DOMAIN_PRIMARY_DOWNLOAD_BUDGET=3600 +CONCELIER_MIRROR_DOMAIN_SECONDARY_ID=community +CONCELIER_MIRROR_DOMAIN_SECONDARY_NAME=Community Mirror +CONCELIER_MIRROR_DOMAIN_SECONDARY_AUTH=false +CONCELIER_MIRROR_DOMAIN_SECONDARY_DOWNLOAD_BUDGET=1800 + +# Authority integration (tokens issued by production Authority) +CONCELIER_AUTHORITY_ENABLED=true +CONCELIER_AUTHORITY_ALLOW_ANON=false +CONCELIER_AUTHORITY_ISSUER=https://authority.stella-ops.org +CONCELIER_AUTHORITY_METADATA= +CONCELIER_AUTHORITY_CLIENT_ID=stellaops-concelier-mirror +CONCELIER_AUTHORITY_SCOPE=concelier.mirror.read +CONCELIER_AUTHORITY_AUDIENCE=api://concelier.mirror + +# Excititor mirror configuration +EXCITITOR_MONGO_DATABASE=excititor +EXCITITOR_FILESYSTEM_OVERWRITE=false +EXCITITOR_MIRROR_DOMAIN_PRIMARY_ID=primary +EXCITITOR_MIRROR_DOMAIN_PRIMARY_NAME=Primary Mirror +EXCITITOR_MIRROR_DOMAIN_PRIMARY_AUTH=true +EXCITITOR_MIRROR_DOMAIN_PRIMARY_INDEX_BUDGET=300 +EXCITITOR_MIRROR_DOMAIN_PRIMARY_DOWNLOAD_BUDGET=2400 +EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_KEY=consensus-json +EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_FORMAT=json +EXCITITOR_MIRROR_PRIMARY_EXPORT_CONSENSUS_VIEW=consensus +EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_KEY=consensus-openvex +EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_FORMAT=openvex +EXCITITOR_MIRROR_PRIMARY_EXPORT_OPENVEX_VIEW=consensus +EXCITITOR_MIRROR_DOMAIN_SECONDARY_ID=community +EXCITITOR_MIRROR_DOMAIN_SECONDARY_NAME=Community Mirror +EXCITITOR_MIRROR_DOMAIN_SECONDARY_AUTH=false +EXCITITOR_MIRROR_DOMAIN_SECONDARY_INDEX_BUDGET=120 +EXCITITOR_MIRROR_DOMAIN_SECONDARY_DOWNLOAD_BUDGET=600 +EXCITITOR_MIRROR_SECONDARY_EXPORT_KEY=community-consensus +EXCITITOR_MIRROR_SECONDARY_EXPORT_FORMAT=json +EXCITITOR_MIRROR_SECONDARY_EXPORT_VIEW=consensus diff --git a/deploy/compose/env/stage.env.example b/deploy/compose/env/stage.env.example new file mode 100644 index 00000000..4ac1cb38 --- /dev/null +++ b/deploy/compose/env/stage.env.example @@ -0,0 +1,16 @@ +# Substitutions for docker-compose.stage.yaml +MONGO_INITDB_ROOT_USERNAME=stellaops +MONGO_INITDB_ROOT_PASSWORD=stage-password +MINIO_ROOT_USER=stellaops-stage +MINIO_ROOT_PASSWORD=stage-minio-secret +MINIO_CONSOLE_PORT=19001 +AUTHORITY_ISSUER=https://authority.stage.stella-ops.internal +AUTHORITY_PORT=8440 +SIGNER_POE_INTROSPECT_URL=https://licensing.stage.stella-ops.internal/introspect +SIGNER_PORT=8441 +ATTESTOR_PORT=8442 +CONCELIER_PORT=8445 +SCANNER_WEB_PORT=8444 +UI_PORT=8443 +NATS_CLIENT_PORT=4222 +SCANNER_QUEUE_BROKER=nats://nats:4222 diff --git a/deploy/compose/mirror-data/concelier/.gitkeep b/deploy/compose/mirror-data/concelier/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose/mirror-data/excititor/.gitkeep b/deploy/compose/mirror-data/excititor/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose/mirror-gateway/README.md b/deploy/compose/mirror-gateway/README.md new file mode 100644 index 00000000..69d89496 --- /dev/null +++ b/deploy/compose/mirror-gateway/README.md @@ -0,0 +1,13 @@ +# Mirror Gateway Assets + +This directory holds the reverse-proxy configuration and TLS material for the managed +mirror profile: + +- `conf.d/*.conf` – nginx configuration shipped with the profile. +- `tls/` – place environment-specific certificates and private keys + (`mirror-primary.{crt,key}`, `mirror-community.{crt,key}`, etc.). +- `secrets/` – populate Basic Auth credential stores (`*.htpasswd`) that gate each + mirror domain. Generate with `htpasswd -B`. + +The Compose bundle mounts these paths read-only. Populate `tls/` with the actual +certificates before invoking `docker compose config` or `docker compose up`. diff --git a/deploy/compose/mirror-gateway/conf.d/mirror-locations.conf b/deploy/compose/mirror-gateway/conf.d/mirror-locations.conf new file mode 100644 index 00000000..9298b3f8 --- /dev/null +++ b/deploy/compose/mirror-gateway/conf.d/mirror-locations.conf @@ -0,0 +1,44 @@ +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_redirect off; + +add_header X-Cache-Status $upstream_cache_status always; + +location = /healthz { + default_type application/json; + return 200 '{"status":"ok"}'; +} + +location /concelier/exports/ { + proxy_pass http://concelier_backend/concelier/exports/; + proxy_cache mirror_cache; + proxy_cache_key $mirror_cache_key; + proxy_cache_valid 200 5m; + proxy_cache_valid 404 1m; + add_header Cache-Control "public, max-age=300, immutable" always; +} + +location /concelier/ { + proxy_pass http://concelier_backend/concelier/; + proxy_cache off; +} + +location /excititor/mirror/ { + proxy_pass http://excititor_backend/excititor/mirror/; + proxy_cache mirror_cache; + proxy_cache_key $mirror_cache_key; + proxy_cache_valid 200 5m; + proxy_cache_valid 404 1m; + add_header Cache-Control "public, max-age=300, immutable" always; +} + +location /excititor/ { + proxy_pass http://excititor_backend/excititor/; + proxy_cache off; +} + +location / { + return 404; +} diff --git a/deploy/compose/mirror-gateway/conf.d/mirror.conf b/deploy/compose/mirror-gateway/conf.d/mirror.conf new file mode 100644 index 00000000..c759ffc7 --- /dev/null +++ b/deploy/compose/mirror-gateway/conf.d/mirror.conf @@ -0,0 +1,51 @@ +proxy_cache_path /var/cache/nginx/mirror levels=1:2 keys_zone=mirror_cache:100m max_size=10g inactive=12h use_temp_path=off; + +map $request_uri $mirror_cache_key { + default $scheme$request_method$host$request_uri; +} + +upstream concelier_backend { + server concelier:8445; + keepalive 32; +} + +upstream excititor_backend { + server excititor:8448; + keepalive 32; +} + +server { + listen 80; + server_name _; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + server_name mirror-primary.stella-ops.org; + + ssl_certificate /etc/nginx/tls/mirror-primary.crt; + ssl_certificate_key /etc/nginx/tls/mirror-primary.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + auth_basic "StellaOps Mirror – primary"; + auth_basic_user_file /etc/nginx/secrets/mirror-primary.htpasswd; + + include /etc/nginx/conf.d/mirror-locations.conf; +} + +server { + listen 443 ssl http2; + server_name mirror-community.stella-ops.org; + + ssl_certificate /etc/nginx/tls/mirror-community.crt; + ssl_certificate_key /etc/nginx/tls/mirror-community.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + auth_basic "StellaOps Mirror – community"; + auth_basic_user_file /etc/nginx/secrets/mirror-community.htpasswd; + + include /etc/nginx/conf.d/mirror-locations.conf; +} diff --git a/deploy/compose/mirror-gateway/secrets/.gitkeep b/deploy/compose/mirror-gateway/secrets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose/mirror-gateway/tls/.gitkeep b/deploy/compose/mirror-gateway/tls/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/compose/mirror-secrets/.gitkeep b/deploy/compose/mirror-secrets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/deploy/helm/stellaops/Chart.yaml b/deploy/helm/stellaops/Chart.yaml new file mode 100644 index 00000000..e3d8a061 --- /dev/null +++ b/deploy/helm/stellaops/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: stellaops +description: Stella Ops core stack (authority, signing, scanner, UI) with infrastructure primitives. +type: application +version: 0.1.0 +appVersion: "2025.10.0" diff --git a/deploy/helm/stellaops/templates/_helpers.tpl b/deploy/helm/stellaops/templates/_helpers.tpl new file mode 100644 index 00000000..aa727d83 --- /dev/null +++ b/deploy/helm/stellaops/templates/_helpers.tpl @@ -0,0 +1,31 @@ +{{- define "stellaops.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "stellaops.fullname" -}} +{{- $name := default .root.Chart.Name .root.Values.fullnameOverride -}} +{{- printf "%s-%s" $name .name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "stellaops.selectorLabels" -}} +app.kubernetes.io/name: {{ include "stellaops.name" .root | quote }} +app.kubernetes.io/instance: {{ .root.Release.Name | quote }} +app.kubernetes.io/component: {{ .name | quote }} +{{- if .svc.class }} +app.kubernetes.io/part-of: {{ printf "stellaops-%s" .svc.class | quote }} +{{- else }} +app.kubernetes.io/part-of: "stellaops-core" +{{- end }} +{{- end -}} + +{{- define "stellaops.labels" -}} +{{ include "stellaops.selectorLabels" . }} +helm.sh/chart: {{ printf "%s-%s" .root.Chart.Name .root.Chart.Version | quote }} +app.kubernetes.io/version: {{ .root.Values.global.release.version | quote }} +app.kubernetes.io/managed-by: {{ .root.Release.Service | quote }} +stellaops.release/channel: {{ .root.Values.global.release.channel | quote }} +stellaops.profile: {{ .root.Values.global.profile | quote }} +{{- range $k, $v := .root.Values.global.labels }} +{{ $k }}: {{ $v | quote }} +{{- end }} +{{- end -}} diff --git a/deploy/helm/stellaops/templates/configmap-release.yaml b/deploy/helm/stellaops/templates/configmap-release.yaml new file mode 100644 index 00000000..920fc251 --- /dev/null +++ b/deploy/helm/stellaops/templates/configmap-release.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "stellaops.fullname" (dict "root" . "name" "release") }} + labels: + {{- include "stellaops.labels" (dict "root" . "name" "release" "svc" (dict "class" "meta")) | nindent 4 }} +data: + version: {{ .Values.global.release.version | quote }} + channel: {{ .Values.global.release.channel | quote }} + manifestSha256: {{ default "" .Values.global.release.manifestSha256 | quote }} diff --git a/deploy/helm/stellaops/templates/configmaps.yaml b/deploy/helm/stellaops/templates/configmaps.yaml new file mode 100644 index 00000000..1ce44b80 --- /dev/null +++ b/deploy/helm/stellaops/templates/configmaps.yaml @@ -0,0 +1,15 @@ +{{- $root := . -}} +{{- range $name, $cfg := .Values.configMaps }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "stellaops.fullname" (dict "root" $root "name" $name) }} + labels: + {{- include "stellaops.labels" (dict "root" $root "name" $name "svc" (dict "class" "config")) | nindent 4 }} +data: +{{- range $fileName, $content := $cfg.data }} + {{ $fileName }}: | +{{ $content | nindent 4 }} +{{- end }} +--- +{{- end }} diff --git a/deploy/helm/stellaops/templates/core.yaml b/deploy/helm/stellaops/templates/core.yaml new file mode 100644 index 00000000..54e96cd6 --- /dev/null +++ b/deploy/helm/stellaops/templates/core.yaml @@ -0,0 +1,154 @@ +{{- $root := . -}} +{{- range $name, $svc := .Values.services }} +{{- $configMounts := (default (list) $svc.configMounts) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "stellaops.fullname" (dict "root" $root "name" $name) }} + labels: + {{- include "stellaops.labels" (dict "root" $root "name" $name "svc" $svc) | nindent 4 }} +spec: + replicas: {{ default 1 $svc.replicas }} + selector: + matchLabels: + {{- include "stellaops.selectorLabels" (dict "root" $root "name" $name "svc" $svc) | nindent 6 }} + template: + metadata: + labels: + {{- include "stellaops.selectorLabels" (dict "root" $root "name" $name "svc" $svc) | nindent 8 }} + annotations: + stellaops.release/version: {{ $root.Values.global.release.version | quote }} + stellaops.release/channel: {{ $root.Values.global.release.channel | quote }} + spec: + containers: + - name: {{ $name }} + image: {{ $svc.image | quote }} + imagePullPolicy: {{ default $root.Values.global.image.pullPolicy $svc.imagePullPolicy }} +{{- if $svc.command }} + command: +{{- range $cmd := $svc.command }} + - {{ $cmd | quote }} +{{- end }} +{{- end }} +{{- if $svc.args }} + args: +{{- range $arg := $svc.args }} + - {{ $arg | quote }} +{{- end }} +{{- end }} +{{- if $svc.env }} + env: +{{- range $envName, $envValue := $svc.env }} + - name: {{ $envName }} + value: {{ $envValue | quote }} +{{- end }} +{{- end }} +{{- if $svc.envFrom }} + envFrom: +{{ toYaml $svc.envFrom | nindent 12 }} +{{- end }} +{{- if $svc.ports }} + ports: +{{- range $port := $svc.ports }} + - name: {{ default (printf "%s-%v" $name $port.containerPort) $port.name | trunc 63 | trimSuffix "-" }} + containerPort: {{ $port.containerPort }} + protocol: {{ default "TCP" $port.protocol }} +{{- end }} +{{- else if and $svc.service (hasKey $svc.service "port") }} + {{- $svcService := $svc.service }} + ports: + - name: {{ printf "%s-http" $name | trunc 63 | trimSuffix "-" }} + containerPort: {{ default (index $svcService "port") (index $svcService "targetPort") }} + protocol: {{ default "TCP" (index $svcService "protocol") }} +{{- end }} +{{- if $svc.resources }} + resources: +{{ toYaml $svc.resources | nindent 12 }} +{{- end }} +{{- if $svc.livenessProbe }} + livenessProbe: +{{ toYaml $svc.livenessProbe | nindent 12 }} +{{- end }} +{{- if $svc.readinessProbe }} + readinessProbe: +{{ toYaml $svc.readinessProbe | nindent 12 }} +{{- end }} +{{- if or $svc.volumeMounts $configMounts }} + volumeMounts: +{{- if $svc.volumeMounts }} +{{ toYaml $svc.volumeMounts | nindent 12 }} +{{- end }} +{{- range $mount := $configMounts }} + - name: {{ $mount.name }} + mountPath: {{ $mount.mountPath }} +{{- if $mount.subPath }} + subPath: {{ $mount.subPath }} +{{- end }} +{{- if hasKey $mount "readOnly" }} + readOnly: {{ $mount.readOnly }} +{{- else }} + readOnly: true +{{- end }} +{{- end }} +{{- end }} + {{- if or $svc.volumes (or $svc.volumeClaims $configMounts) }} + volumes: +{{- if $svc.volumes }} +{{ toYaml $svc.volumes | nindent 8 }} +{{- end }} +{{- if $svc.volumeClaims }} +{{- range $claim := $svc.volumeClaims }} + - name: {{ $claim.name }} + persistentVolumeClaim: + claimName: {{ $claim.claimName }} +{{- end }} +{{- end }} +{{- range $mount := $configMounts }} + - name: {{ $mount.name }} + configMap: + name: {{ include "stellaops.fullname" (dict "root" $root "name" $mount.configMap) }} +{{- if $mount.items }} + items: +{{ toYaml $mount.items | nindent 12 }} +{{- else if $mount.subPath }} + items: + - key: {{ $mount.subPath }} + path: {{ $mount.subPath }} +{{- end }} +{{- end }} + {{- end }} + {{- if $svc.serviceAccount }} + serviceAccountName: {{ $svc.serviceAccount | quote }} + {{- end }} + {{- if $svc.nodeSelector }} + nodeSelector: +{{ toYaml $svc.nodeSelector | nindent 8 }} + {{- end }} + {{- if $svc.affinity }} + affinity: +{{ toYaml $svc.affinity | nindent 8 }} + {{- end }} + {{- if $svc.tolerations }} + tolerations: +{{ toYaml $svc.tolerations | nindent 8 }} + {{- end }} +--- +{{- if $svc.service }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "stellaops.fullname" (dict "root" $root "name" $name) }} + labels: + {{- include "stellaops.labels" (dict "root" $root "name" $name "svc" $svc) | nindent 4 }} +spec: + type: {{ default "ClusterIP" $svc.service.type }} + selector: + {{- include "stellaops.selectorLabels" (dict "root" $root "name" $name "svc" $svc) | nindent 4 }} + ports: + - name: {{ default "http" $svc.service.portName }} + port: {{ $svc.service.port }} + targetPort: {{ $svc.service.targetPort | default $svc.service.port }} + protocol: {{ default "TCP" $svc.service.protocol }} +--- +{{- end }} +{{- end }} diff --git a/deploy/helm/stellaops/values-airgap.yaml b/deploy/helm/stellaops/values-airgap.yaml new file mode 100644 index 00000000..8e9bf1f2 --- /dev/null +++ b/deploy/helm/stellaops/values-airgap.yaml @@ -0,0 +1,187 @@ +global: + profile: airgap + release: + version: "2025.09.2-airgap" + channel: airgap + manifestSha256: "b787b833dddd73960c31338279daa0b0a0dce2ef32bd32ef1aaf953d66135f94" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: airgap + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.prod.svc.cluster.local:27017" + database: "stellaops_notify" + commandTimeoutSeconds: 60 + + authority: + enabled: true + issuer: "https://authority.stella-ops.org" + metadataAddress: "https://authority.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/var/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Warning +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:5551a3269b7008cd5aceecf45df018c67459ed519557ccbe48b093b926a39bcc + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__ALLOWANONYMOUSFALLBACK: "false" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:ddbbd664a42846cea6b40fca6465bc679b30f72851158f300d01a8571c5478fc + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "file:///offline/poe/introspect.json" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:1ff0a3124d66d3a2702d8e421df40fbd98cc75cb605d95510598ebbae1433c50 + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:29e2e1a0972707e092cbd3d370701341f9fec2aa9316fb5d8100480f2a1c76b5 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" + CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "00:45:00" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 + service: + port: 8444 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:65c0ee13f773efe920d7181512349a09d363ab3f3e177d276136bd2742325a68 + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:bee9668011ff414572131dc777faab4da24473fe12c230893f161cabee092a1d + service: + port: 9443 + targetPort: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops-airgap + MONGO_INITDB_ROOT_PASSWORD: stellaops-airgap + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops-airgap + MINIO_ROOT_PASSWORD: airgap-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumeClaims: + - name: nats-data + claimName: stellaops-nats-data diff --git a/deploy/helm/stellaops/values-dev.yaml b/deploy/helm/stellaops/values-dev.yaml new file mode 100644 index 00000000..9cacb1ef --- /dev/null +++ b/deploy/helm/stellaops/values-dev.yaml @@ -0,0 +1,185 @@ +global: + profile: dev + release: + version: "2025.10.0-edge" + channel: edge + manifestSha256: "822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: edge + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.dev.svc.cluster.local:27017" + database: "stellaops_notify_dev" + commandTimeoutSeconds: 30 + + authority: + enabled: true + issuer: "https://authority.dev.stella-ops.local" + metadataAddress: "https://authority.dev.stella-ops.local/.well-known/openid-configuration" + requireHttpsMetadata: false + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify.dev + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "../" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Debug +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8bfef9a75783883d49fc18e3566553934e970b00ee090abee9cb110d2d5c3298 + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.svc.local/introspect" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:5cc417948c029da01dccf36e4645d961a3f6d8de7e62fe98d845f07cd2282114 + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumes: + - name: concelier-jobs + emptyDir: {} + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 + service: + port: 8444 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.10.0-edge + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Development + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops + MONGO_INITDB_ROOT_PASSWORD: stellaops + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumes: + - name: mongo-data + emptyDir: {} + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops + MINIO_ROOT_PASSWORD: dev-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data + volumes: + - name: minio-data + emptyDir: {} + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumes: + - name: nats-data + emptyDir: {} diff --git a/deploy/helm/stellaops/values-mirror.yaml b/deploy/helm/stellaops/values-mirror.yaml new file mode 100644 index 00000000..2edce43f --- /dev/null +++ b/deploy/helm/stellaops/values-mirror.yaml @@ -0,0 +1,282 @@ +global: + profile: mirror-managed + release: + version: "2025.10.0-edge" + channel: edge + manifestSha256: "822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: edge + +configMaps: + mirror-gateway: + data: + mirror.conf: | + proxy_cache_path /var/cache/nginx/mirror levels=1:2 keys_zone=mirror_cache:100m max_size=10g inactive=12h use_temp_path=off; + + map $request_uri $mirror_cache_key { + default $scheme$request_method$host$request_uri; + } + + upstream concelier_backend { + server stellaops-concelier:8445; + keepalive 32; + } + + upstream excititor_backend { + server stellaops-excititor:8448; + keepalive 32; + } + + server { + listen 80; + server_name _; + return 301 https://$host$request_uri; + } + + server { + listen 443 ssl http2; + server_name mirror-primary.stella-ops.org; + + ssl_certificate /etc/nginx/tls/mirror-primary.crt; + ssl_certificate_key /etc/nginx/tls/mirror-primary.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + auth_basic "StellaOps Mirror – primary"; + auth_basic_user_file /etc/nginx/secrets/mirror-primary.htpasswd; + + include /etc/nginx/conf.d/mirror-locations.conf; + } + + server { + listen 443 ssl http2; + server_name mirror-community.stella-ops.org; + + ssl_certificate /etc/nginx/tls/mirror-community.crt; + ssl_certificate_key /etc/nginx/tls/mirror-community.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + + auth_basic "StellaOps Mirror – community"; + auth_basic_user_file /etc/nginx/secrets/mirror-community.htpasswd; + + include /etc/nginx/conf.d/mirror-locations.conf; + } + mirror-locations.conf: | + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + + add_header X-Cache-Status $upstream_cache_status always; + + location = /healthz { + default_type application/json; + return 200 '{"status":"ok"}'; + } + + location /concelier/exports/ { + proxy_pass http://concelier_backend/concelier/exports/; + proxy_cache mirror_cache; + proxy_cache_key $mirror_cache_key; + proxy_cache_valid 200 5m; + proxy_cache_valid 404 1m; + add_header Cache-Control "public, max-age=300, immutable" always; + } + + location /concelier/ { + proxy_pass http://concelier_backend/concelier/; + proxy_cache off; + } + + location /excititor/mirror/ { + proxy_pass http://excititor_backend/excititor/mirror/; + proxy_cache mirror_cache; + proxy_cache_key $mirror_cache_key; + proxy_cache_valid 200 5m; + proxy_cache_valid 404 1m; + add_header Cache-Control "public, max-age=300, immutable" always; + } + + location /excititor/ { + proxy_pass http://excititor_backend/excititor/; + proxy_cache off; + } + + location / { + return 404; + } + +services: + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + service: + port: 8445 + env: + ASPNETCORE_URLS: "http://+:8445" + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops_mirror:mirror-password@stellaops-mongo:27017/concelier?authSource=admin" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-mirror" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "mirror-minio-secret" + CONCELIER__TELEMETRY__SERVICENAME: "stellaops-concelier-mirror" + CONCELIER__MIRROR__ENABLED: "true" + CONCELIER__MIRROR__EXPORTROOT: "/exports/json" + CONCELIER__MIRROR__LATESTDIRECTORYNAME: "latest" + CONCELIER__MIRROR__MIRRORDIRECTORYNAME: "mirror" + CONCELIER__MIRROR__REQUIREAUTHENTICATION: "true" + CONCELIER__MIRROR__MAXINDEXREQUESTSPERHOUR: "600" + CONCELIER__MIRROR__DOMAINS__0__ID: "primary" + CONCELIER__MIRROR__DOMAINS__0__DISPLAYNAME: "Primary Mirror" + CONCELIER__MIRROR__DOMAINS__0__REQUIREAUTHENTICATION: "true" + CONCELIER__MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR: "3600" + CONCELIER__MIRROR__DOMAINS__1__ID: "community" + CONCELIER__MIRROR__DOMAINS__1__DISPLAYNAME: "Community Mirror" + CONCELIER__MIRROR__DOMAINS__1__REQUIREAUTHENTICATION: "false" + CONCELIER__MIRROR__DOMAINS__1__MAXDOWNLOADREQUESTSPERHOUR: "1800" + CONCELIER__AUTHORITY__ENABLED: "true" + CONCELIER__AUTHORITY__ALLOWANONYMOUSFALLBACK: "false" + CONCELIER__AUTHORITY__ISSUER: "https://authority.stella-ops.org" + CONCELIER__AUTHORITY__METADATAADDRESS: "" + CONCELIER__AUTHORITY__CLIENTID: "stellaops-concelier-mirror" + CONCELIER__AUTHORITY__CLIENTSECRETFILE: "/run/secrets/concelier-authority-client" + CONCELIER__AUTHORITY__CLIENTSCOPES__0: "concelier.mirror.read" + CONCELIER__AUTHORITY__AUDIENCES__0: "api://concelier.mirror" + CONCELIER__AUTHORITY__BYPASSNETWORKS__0: "10.0.0.0/8" + CONCELIER__AUTHORITY__BYPASSNETWORKS__1: "127.0.0.1/32" + CONCELIER__AUTHORITY__BYPASSNETWORKS__2: "::1/128" + CONCELIER__AUTHORITY__RESILIENCE__ENABLERETRIES: "true" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__0: "00:00:01" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__1: "00:00:02" + CONCELIER__AUTHORITY__RESILIENCE__RETRYDELAYS__2: "00:00:05" + CONCELIER__AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK: "true" + CONCELIER__AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE: "00:10:00" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + - name: concelier-exports + mountPath: /exports/json + - name: concelier-secrets + mountPath: /run/secrets + readOnly: true + volumes: + - name: concelier-jobs + persistentVolumeClaim: + claimName: concelier-mirror-jobs + - name: concelier-exports + persistentVolumeClaim: + claimName: concelier-mirror-exports + - name: concelier-secrets + secret: + secretName: concelier-mirror-auth + + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + env: + ASPNETCORE_URLS: "http://+:8448" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops_mirror:mirror-password@stellaops-mongo:27017/excititor?authSource=admin" + EXCITITOR__STORAGE__MONGO__DATABASENAME: "excititor" + EXCITITOR__ARTIFACTS__FILESYSTEM__ROOT: "/exports" + EXCITITOR__ARTIFACTS__FILESYSTEM__OVERWRITEEXISTING: "false" + EXCITITOR__MIRROR__DOMAINS__0__ID: "primary" + EXCITITOR__MIRROR__DOMAINS__0__DISPLAYNAME: "Primary Mirror" + EXCITITOR__MIRROR__DOMAINS__0__REQUIREAUTHENTICATION: "true" + EXCITITOR__MIRROR__DOMAINS__0__MAXINDEXREQUESTSPERHOUR: "300" + EXCITITOR__MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR: "2400" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__KEY: "consensus-json" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__FORMAT: "json" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__0__VIEW: "consensus" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__KEY: "consensus-openvex" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__FORMAT: "openvex" + EXCITITOR__MIRROR__DOMAINS__0__EXPORTS__1__VIEW: "consensus" + EXCITITOR__MIRROR__DOMAINS__1__ID: "community" + EXCITITOR__MIRROR__DOMAINS__1__DISPLAYNAME: "Community Mirror" + EXCITITOR__MIRROR__DOMAINS__1__REQUIREAUTHENTICATION: "false" + EXCITITOR__MIRROR__DOMAINS__1__MAXINDEXREQUESTSPERHOUR: "120" + EXCITITOR__MIRROR__DOMAINS__1__MAXDOWNLOADREQUESTSPERHOUR: "600" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__KEY: "community-consensus" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__FORMAT: "json" + EXCITITOR__MIRROR__DOMAINS__1__EXPORTS__0__VIEW: "consensus" + volumeMounts: + - name: excititor-exports + mountPath: /exports + - name: excititor-secrets + mountPath: /run/secrets + readOnly: true + volumes: + - name: excititor-exports + persistentVolumeClaim: + claimName: excititor-mirror-exports + - name: excititor-secrets + secret: + secretName: excititor-mirror-auth + + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: "stellaops_mirror" + MONGO_INITDB_ROOT_PASSWORD: "mirror-password" + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: mirror-mongo-data + + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: "stellaops-mirror" + MINIO_ROOT_PASSWORD: "mirror-minio-secret" + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaims: + - name: minio-data + claimName: mirror-minio-data + + mirror-gateway: + image: docker.io/library/nginx@sha256:208b70eefac13ee9be00e486f79c695b15cef861c680527171a27d253d834be9 + service: + type: LoadBalancer + port: 443 + portName: https + targetPort: 443 + configMounts: + - name: mirror-gateway-conf + mountPath: /etc/nginx/conf.d + configMap: mirror-gateway + volumeMounts: + - name: mirror-gateway-tls + mountPath: /etc/nginx/tls + readOnly: true + - name: mirror-gateway-secrets + mountPath: /etc/nginx/secrets + readOnly: true + - name: mirror-cache + mountPath: /var/cache/nginx + volumes: + - name: mirror-gateway-tls + secret: + secretName: mirror-gateway-tls + - name: mirror-gateway-secrets + secret: + secretName: mirror-gateway-htpasswd + - name: mirror-cache + emptyDir: {} diff --git a/deploy/helm/stellaops/values-stage.yaml b/deploy/helm/stellaops/values-stage.yaml new file mode 100644 index 00000000..29b906fb --- /dev/null +++ b/deploy/helm/stellaops/values-stage.yaml @@ -0,0 +1,186 @@ +global: + profile: stage + release: + version: "2025.09.2" + channel: stable + manifestSha256: "dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7" + image: + pullPolicy: IfNotPresent + labels: + stellaops.io/channel: stable + +configMaps: + notify-config: + data: + notify.yaml: | + storage: + driver: mongo + connectionString: "mongodb://notify-mongo.stage.svc.cluster.local:27017" + database: "stellaops_notify_stage" + commandTimeoutSeconds: 45 + + authority: + enabled: true + issuer: "https://authority.stage.stella-ops.org" + metadataAddress: "https://authority.stage.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + + api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + + plugins: + baseDirectory: "/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + + telemetry: + enableRequestLogging: true + minimumLogLevel: Information +services: + authority: + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + service: + port: 8440 + env: + STELLAOPS_AUTHORITY__ISSUER: "https://stellaops-authority:8440" + STELLAOPS_AUTHORITY__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0: "/app/plugins" + STELLAOPS_AUTHORITY__PLUGINS__CONFIGURATIONDIRECTORY: "/app/etc/authority.plugins" + signer: + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + service: + port: 8441 + env: + SIGNER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + SIGNER__POE__INTROSPECTURL: "https://licensing.stage.stella-ops.internal/introspect" + SIGNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + attestor: + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + service: + port: 8442 + env: + ATTESTOR__SIGNER__BASEURL: "https://stellaops-signer:8441" + ATTESTOR__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + service: + port: 8445 + env: + CONCELIER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + CONCELIER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + CONCELIER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + CONCELIER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + CONCELIER__AUTHORITY__BASEURL: "https://stellaops-authority:8440" + volumeMounts: + - name: concelier-jobs + mountPath: /var/lib/concelier/jobs + volumeClaims: + - name: concelier-jobs + claimName: stellaops-concelier-jobs + scanner-web: + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + service: + port: 8444 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + scanner-worker: + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + replicas: 2 + env: + SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000" + SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage" + SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret" + SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222" + notify-web: + image: registry.stella-ops.org/stellaops/notify-web:2025.09.2 + service: + port: 8446 + env: + DOTNET_ENVIRONMENT: Production + configMounts: + - name: notify-config + mountPath: /app/etc/notify.yaml + subPath: notify.yaml + configMap: notify-config + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + env: + EXCITITOR__CONCELIER__BASEURL: "https://stellaops-concelier:8445" + EXCITITOR__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017" + web-ui: + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + service: + port: 8443 + env: + STELLAOPS_UI__BACKEND__BASEURL: "https://stellaops-scanner-web:8444" + mongo: + class: infrastructure + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + service: + port: 27017 + command: + - mongod + - --bind_ip_all + env: + MONGO_INITDB_ROOT_USERNAME: stellaops-stage + MONGO_INITDB_ROOT_PASSWORD: stellaops-stage + volumeMounts: + - name: mongo-data + mountPath: /data/db + volumeClaims: + - name: mongo-data + claimName: stellaops-mongo-data + minio: + class: infrastructure + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + service: + port: 9000 + command: + - server + - /data + - --console-address + - :9001 + env: + MINIO_ROOT_USER: stellaops-stage + MINIO_ROOT_PASSWORD: stage-minio-secret + volumeMounts: + - name: minio-data + mountPath: /data + volumeClaims: + - name: minio-data + claimName: stellaops-minio-data + nats: + class: infrastructure + image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e + service: + port: 4222 + command: + - -js + - -sd + - /data + volumeMounts: + - name: nats-data + mountPath: /data + volumeClaims: + - name: nats-data + claimName: stellaops-nats-data diff --git a/deploy/helm/stellaops/values.yaml b/deploy/helm/stellaops/values.yaml new file mode 100644 index 00000000..1370a282 --- /dev/null +++ b/deploy/helm/stellaops/values.yaml @@ -0,0 +1,10 @@ +global: + release: + version: "" + channel: "" + manifestSha256: "" + profile: "" + image: + pullPolicy: IfNotPresent + labels: {} +services: {} diff --git a/deploy/releases/2025.09-airgap.yaml b/deploy/releases/2025.09-airgap.yaml new file mode 100644 index 00000000..b4b02c7e --- /dev/null +++ b/deploy/releases/2025.09-airgap.yaml @@ -0,0 +1,29 @@ +release: + version: "2025.09.2-airgap" + channel: "airgap" + date: "2025-09-20T00:00:00Z" + calendar: "2025.09" + components: + - name: authority + image: registry.stella-ops.org/stellaops/authority@sha256:5551a3269b7008cd5aceecf45df018c67459ed519557ccbe48b093b926a39bcc + - name: signer + image: registry.stella-ops.org/stellaops/signer@sha256:ddbbd664a42846cea6b40fca6465bc679b30f72851158f300d01a8571c5478fc + - name: attestor + image: registry.stella-ops.org/stellaops/attestor@sha256:1ff0a3124d66d3a2702d8e421df40fbd98cc75cb605d95510598ebbae1433c50 + - name: scanner-web + image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718 + - name: scanner-worker + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5 + - name: concelier + image: registry.stella-ops.org/stellaops/concelier@sha256:29e2e1a0972707e092cbd3d370701341f9fec2aa9316fb5d8100480f2a1c76b5 + - name: excititor + image: registry.stella-ops.org/stellaops/excititor@sha256:65c0ee13f773efe920d7181512349a09d363ab3f3e177d276136bd2742325a68 + - name: web-ui + image: registry.stella-ops.org/stellaops/web-ui@sha256:bee9668011ff414572131dc777faab4da24473fe12c230893f161cabee092a1d + infrastructure: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + checksums: + releaseManifestSha256: b787b833dddd73960c31338279daa0b0a0dce2ef32bd32ef1aaf953d66135f94 diff --git a/deploy/releases/2025.09-stable.yaml b/deploy/releases/2025.09-stable.yaml new file mode 100644 index 00000000..1ac9b33f --- /dev/null +++ b/deploy/releases/2025.09-stable.yaml @@ -0,0 +1,29 @@ +release: + version: "2025.09.2" + channel: "stable" + date: "2025-09-20T00:00:00Z" + calendar: "2025.09" + components: + - name: authority + image: registry.stella-ops.org/stellaops/authority@sha256:b0348bad1d0b401cc3c71cb40ba034c8043b6c8874546f90d4783c9dbfcc0bf5 + - name: signer + image: registry.stella-ops.org/stellaops/signer@sha256:8ad574e61f3a9e9bda8a58eb2700ae46813284e35a150b1137bc7c2b92ac0f2e + - name: attestor + image: registry.stella-ops.org/stellaops/attestor@sha256:0534985f978b0b5d220d73c96fddd962cd9135f616811cbe3bff4666c5af568f + - name: scanner-web + image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7 + - name: scanner-worker + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab + - name: concelier + image: registry.stella-ops.org/stellaops/concelier@sha256:c58cdcaee1d266d68d498e41110a589dd204b487d37381096bd61ab345a867c5 + - name: excititor + image: registry.stella-ops.org/stellaops/excititor@sha256:59022e2016aebcef5c856d163ae705755d3f81949d41195256e935ef40a627fa + - name: web-ui + image: registry.stella-ops.org/stellaops/web-ui@sha256:10d924808c48e4353e3a241da62eb7aefe727a1d6dc830eb23a8e181013b3a23 + infrastructure: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + checksums: + releaseManifestSha256: dc3c8fe1ab83941c838ccc5a8a5862f7ddfa38c2078e580b5649db26554565b7 diff --git a/deploy/releases/2025.10-edge.yaml b/deploy/releases/2025.10-edge.yaml new file mode 100644 index 00000000..6c7e75df --- /dev/null +++ b/deploy/releases/2025.10-edge.yaml @@ -0,0 +1,29 @@ +release: + version: "2025.10.0-edge" + channel: "edge" + date: "2025-10-01T00:00:00Z" + calendar: "2025.10" + components: + - name: authority + image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd + - name: signer + image: registry.stella-ops.org/stellaops/signer@sha256:8bfef9a75783883d49fc18e3566553934e970b00ee090abee9cb110d2d5c3298 + - name: attestor + image: registry.stella-ops.org/stellaops/attestor@sha256:5cc417948c029da01dccf36e4645d961a3f6d8de7e62fe98d845f07cd2282114 + - name: scanner-web + image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11 + - name: scanner-worker + image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37 + - name: concelier + image: registry.stella-ops.org/stellaops/concelier@sha256:dafef3954eb4b837e2c424dd2d23e1e4d60fa83794840fac9cd3dea1d43bd085 + - name: excititor + image: registry.stella-ops.org/stellaops/excititor@sha256:d9bd5cadf1eab427447ce3df7302c30ded837239771cc6433b9befb895054285 + - name: web-ui + image: registry.stella-ops.org/stellaops/web-ui@sha256:38b225fa7767a5b94ebae4dae8696044126aac429415e93de514d5dd95748dcf + infrastructure: + mongo: + image: docker.io/library/mongo@sha256:c258b26dbb7774f97f52aff52231ca5f228273a84329c5f5e451c3739457db49 + minio: + image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + checksums: + releaseManifestSha256: 822f82987529ea38d2321dbdd2ef6874a4062a117116a20861c26a8df1807beb diff --git a/deploy/tools/validate-profiles.sh b/deploy/tools/validate-profiles.sh new file mode 100644 index 00000000..f8ac4af1 --- /dev/null +++ b/deploy/tools/validate-profiles.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +COMPOSE_DIR="$ROOT_DIR/compose" +HELM_DIR="$ROOT_DIR/helm/stellaops" + +compose_profiles=( + "docker-compose.dev.yaml:env/dev.env.example" + "docker-compose.stage.yaml:env/stage.env.example" + "docker-compose.airgap.yaml:env/airgap.env.example" + "docker-compose.mirror.yaml:env/mirror.env.example" +) + +docker_ready=false +if command -v docker >/dev/null 2>&1; then + if docker compose version >/dev/null 2>&1; then + docker_ready=true + else + echo "⚠️ docker CLI present but Compose plugin unavailable; skipping compose validation" >&2 + fi +else + echo "⚠️ docker CLI not found; skipping compose validation" >&2 +fi + +if [[ "$docker_ready" == "true" ]]; then + for entry in "${compose_profiles[@]}"; do + IFS=":" read -r compose_file env_file <<<"$entry" + printf '→ validating %s with %s\n' "$compose_file" "$env_file" + docker compose \ + --env-file "$COMPOSE_DIR/$env_file" \ + -f "$COMPOSE_DIR/$compose_file" config >/dev/null + done +fi + +helm_values=( + "$HELM_DIR/values-dev.yaml" + "$HELM_DIR/values-stage.yaml" + "$HELM_DIR/values-airgap.yaml" + "$HELM_DIR/values-mirror.yaml" +) + +if command -v helm >/dev/null 2>&1; then + for values in "${helm_values[@]}"; do + printf '→ linting Helm chart with %s\n' "$(basename "$values")" + helm lint "$HELM_DIR" -f "$values" + helm template test-release "$HELM_DIR" -f "$values" >/dev/null + done +else + echo "⚠️ helm CLI not found; skipping Helm lint/template" >&2 +fi + +printf 'Profiles validated (where tooling was available).\n' diff --git a/docs/07_HIGH_LEVEL_ARCHITECTURE.md b/docs/07_HIGH_LEVEL_ARCHITECTURE.md index e27f49ee..3a11036b 100755 --- a/docs/07_HIGH_LEVEL_ARCHITECTURE.md +++ b/docs/07_HIGH_LEVEL_ARCHITECTURE.md @@ -1,8 +1,3 @@ -Below is the **revised, consolidated** `high_level_architecture.md`. -It **absorbs** all content from `components.md` so you have a single, authoritative file. No separate components doc is required. - ---- - # High‑Level Architecture — **Stella Ops** (Consolidated • 2025Q4) > **Purpose.** A complete, implementation‑ready map of Stella Ops: product vision, all runtime components, trust boundaries, tokens/licensing, control/data flows, storage, APIs, security, scale, DevOps, and verification logic. @@ -30,28 +25,32 @@ It **absorbs** all content from `components.md` so you have a single, authoritat ### 1.1 Runtime inventory (first‑party) -| Service / Tool | Container image | Core role | Scale pattern | -| ------------------------------- | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -| **Scanner.WebService** | `stellaops/scanner-web` | Control plane for scans; catalog; SBOM composition (inventory & usage); diff; exports. | Stateless; N replicas behind LB. | -| **Scanner.Worker** | `stellaops/scanner-worker` | Runs analyzers (OS, Lang: Java/Node/Python/Go/.NET/Rust, Native ELF/PE/Mach‑O, EntryTrace); emits per‑layer SBOMs and composes image SBOMs. | Horizontal; queue‑driven; sharded by layer digest. | -| **Scanner.Sbomer.BuildXPlugin** | `stellaops/sbom-indexer` | BuildKit **generator** for build‑time SBOMs as OCI **referrers**. | CI‑side; ephemeral. | -| **Scanner.Sbomer.DockerImage** | `stellaops/scanner-cli` | CLI‑orchestrated scanner container for post‑build scans. | Local/CI; ephemeral. | -| **Feedser.WebService** | `stellaops/feedser-web` | Vulnerability ingest/normalize/merge/export (JSON + Trivy DB). | HA via Mongo locks. | -| **Vexer.WebService** | `stellaops/vexer-web` | VEX ingest/normalize/consensus; conflict retention; exports. | HA via Mongo locks. | -| **Policy Engine** | (in `scanner-web`) | YAML DSL evaluator (waivers, vendor preferences, KEV/EPSS, license, usage‑gating); produces **policy digest**. | In‑process; cache per digest. | -| **Signer** | `stellaops/signer` | **Hard gate:** validates entitlement + release integrity; mints signing cert (Fulcio keyless) or uses KMS; signs DSSE. | Stateless; HPA by QPS. | -| **Attestor** | `stellaops/attestor` | Posts DSSE bundles to **Rekor v2**; verification endpoints. | Stateless; HPA by QPS. | -| **Authority** | `stellaops/authority` | On‑prem OIDC issuing **short‑lived OpToks** with DPoP/mTLS sender constraint. | HA behind LB. | -| **Zastava** (Runtime) | `stellaops/zastava` | Runtime inspector/enforcer (observer + optional Admission Webhook). | DaemonSet + Webhook. | -| **Web UI** | `stellaops/ui` | Angular app for scans, diffs, policy, VEX, runtime, reports. | Stateless. | -| **StellaOps.Cli** | `stellaops/cli` | CLI for init/scan/export/diff/policy/report/verify; Buildx helper. | Local/CI. | +| Service / Tool | Container image | Core role | Scale pattern | +| ------------------------------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| **Scanner.WebService** | `stellaops/scanner-web` | Control plane for scans; catalog; SBOM composition (inventory & usage); diff; exports; **analysis‑only report runs** for Scheduler. | Stateless; N replicas behind LB. | +| **Scanner.Worker** | `stellaops/scanner-worker` | Runs analyzers (OS, Lang: Java/Node/Python/Go/.NET/Rust, Native ELF/PE/Mach‑O, EntryTrace); emits per‑layer SBOMs and composes image SBOMs. | Horizontal; queue‑driven; sharded by layer digest. | +| **Scanner.Sbomer.BuildXPlugin** | `stellaops/sbom-indexer` | BuildKit **generator** for build‑time SBOMs as OCI **referrers**. | CI‑side; ephemeral. | +| **Scanner.Sbomer.DockerImage** | `stellaops/scanner-cli` | CLI‑orchestrated scanner container for post‑build scans. | Local/CI; ephemeral. | +| **Concelier.WebService** | `stellaops/concelier-web` | Vulnerability ingest/normalize/merge/export (JSON + Trivy DB). | HA via Mongo locks. | +| **Excititor.WebService** | `stellaops/excititor-web` | VEX ingest/normalize/consensus; conflict retention; exports. | HA via Mongo locks. | +| **Policy Engine** | (in `scanner-web`) | YAML DSL evaluator (waivers, vendor preferences, KEV/EPSS, license, usage‑gating); produces **policy digest**. | In‑process; cache per digest. | +| **Scheduler.WebService** | `stellaops/scheduler-web` | Schedules **re‑evaluation** runs; consumes Concelier/Excititor deltas; selects **impacted images** via BOM‑Index; orchestrates analysis‑only reports. | Stateless API. | +| **Scheduler.Worker** | `stellaops/scheduler-worker` | Executes selection and enqueues batches toward Scanner; enforces rate/limits and windows; maintains impact cursors. | Horizontal; queue‑driven. | +| **Notify.WebService** | `stellaops/notify-web` | Rules engine for outbound notifications; manages channels, templates, throttle/digest logic. | Stateless API. | +| **Notify.Worker** | `stellaops/notify-worker` | Delivers to Slack/Teams/Email/Webhooks; idempotent retries; digests. | Horizontal; per‑channel rate limits. | +| **Signer** | `stellaops/signer` | **Hard gate:** validates entitlement + release integrity; mints signing cert (Fulcio keyless) or uses KMS; signs DSSE. | Stateless; HPA by QPS. | +| **Attestor** | `stellaops/attestor` | Posts DSSE bundles to **Rekor v2**; verification endpoints. | Stateless; HPA by QPS. | +| **Authority** | `stellaops/authority` | On‑prem OIDC issuing **short‑lived OpToks** with DPoP/mTLS sender constraint. | HA behind LB. | +| **Zastava** (Runtime) | `stellaops/zastava` | Runtime inspector/enforcer (observer + optional Admission Webhook). | DaemonSet + Webhook. | +| **Web UI** | `stellaops/ui` | Angular app for scans, diffs, policy, VEX, **Scheduler**, **Notify**, runtime, reports. | Stateless. | +| **StellaOps.Cli** | `stellaops/cli` | CLI for init/scan/export/diff/policy/report/verify; Buildx helper; **schedule** and **notify** verbs. | Local/CI. | ### 1.2 Third‑party (self‑hosted) * **Fulcio** (Sigstore CA) — issues short‑lived signing certs (keyless). * **Rekor v2** (tile‑backed transparency log). * **MinIO** — S3‑compatible object store with lifecycle & Object Lock. -* **MongoDB** — catalog, advisories, VEX. +* **MongoDB** — catalog, advisories, VEX, scheduler, notify. * **Queue** — Redis Streams / NATS / RabbitMQ (pluggable). * **OCI Registry** — must support **Referrers API** (discover SBOMs/signatures). @@ -71,8 +70,12 @@ flowchart LR Auth[Authority (OIDC)\nOpTok (DPoP/mTLS)] SW[Scanner.WebService] WK[Scanner.Worker xN] - FEED[Feedser] - VEX[Vexer] + CONC[Concelier] + EXC[Excititor] + SCHW[Scheduler.Web] + SCH[Scheduler.Worker xN] + NOTW[Notify.Web] + NOT[Notify.Worker xN] POL[Policy Engine (in Scanner.Web)] SGN[Signer\n(entitlement + signing)] ATT[Attestor\n(Rekor v2 submit/verify)] @@ -93,11 +96,19 @@ flowchart LR QUE --> WK WK --> MIN SW --> MGO - FEED --> MGO - VEX --> MGO + CONC --> MGO + EXC --> MGO UI --> SW Z --> SW + %% New event-driven loop + CONC -- export.delta --> SCHW + EXC -- export.delta --> SCHW + SCHW --> SCH + SCH --> SW + SW -- report.ready --> NOTW + Z -- admission/observe --> NOTW + SGN <--> Auth SGN --> FUL SGN -->|mTLS| ATT @@ -106,7 +117,7 @@ flowchart LR SGN <-->|verify referrers| REG ``` -**Trust boundaries.** Only **Signer** can sign; only **Attestor** can write to **Rekor v2**. Scanner/UI never sign. +**Trust boundaries.** Only **Signer** can sign; only **Attestor** can write to **Rekor v2**. Scanner/UI/Scheduler/Notify never sign. --- @@ -116,7 +127,7 @@ flowchart LR * **License Token (LT)** — long‑lived JWT from **Licensing Service**; used **once** to enroll the installation; never used in hot path. * **Proof‑of‑Entitlement (PoE)** — bound to the installation key (mTLS client cert **or** DPoP‑bound JWT with `cnf`); medium‑lived; renewable; revocable. -* **Operational token (OpTok)** — 2–5 min OIDC token from **Authority**, **sender‑constrained** (DPoP or mTLS). Used to authenticate to **Signer**/**Scanner.WebService**. +* **Operational token (OpTok)** — 2–5 min OIDC token from **Authority**, **sender‑constrained** (DPoP or mTLS). Used to authenticate to **Signer**/**Scanner.WebService**/**Scheduler.Web**/**Notify.Web**. **Signer enforces both:** PoE proves entitlement; OpTok proves “who is calling now”. It also **independently verifies** the **scanner image digest** is **Stella Ops‑signed** via **Referrers + cosign** before signing anything. @@ -173,16 +184,21 @@ LS --> IA: PoE (mTLS client cert or JWT with cnf=K_inst), CRL/OCSP/introspect * Buildx **generator** runs analyzers during `docker buildx build --attest=type=sbom,generator=stellaops/sbom-indexer`, attaches SBOMs as **OCI referrers**. * Scanner.WebService can trust these (policy‑configurable) and **skip** re‑scan; DSSE + Rekor v2 can be done either at build time or post‑push via Signer/Attestor. +### 3.5 Events / integrations + +* **Out:** `report.ready` (summary + verdict + Rekor UUID) → internal bus for **Notify** & UI. +* **Expose:** image‑level **BOM‑Index** metadata for **Scheduler** impact selection. + --- ## 4) Backend evaluation (decider) -### 4.1 Feedser (advisories) +### 4.1 Concelier (advisories) * Ingests vendor, distro, OSS feeds; normalizes & merges; persists canonical advisories in Mongo; exports **deterministic JSON** and **Trivy DB**. * Offline kit bundles for air‑gapped sites. -### 4.2 Vexer (VEX) +### 4.2 Excititor (VEX) * Ingests **OpenVEX / CSAF VEX / CycloneDX VEX**; normalizes claims; retains conflicts; computes **consensus** with provider trust weights and justification gates. @@ -194,8 +210,8 @@ LS --> IA: PoE (mTLS client cert or JWT with cnf=K_inst), CRL/OCSP/introspect ### 4.4 PASS/FAIL flow -1. SBOM (Inventory / Usage) → join with **Feedser** advisories. -2. Apply **Vexer** consensus (statuses & justifications). +1. SBOM (Inventory / Usage) → join with **Concelier** advisories. +2. Apply **Excititor** consensus (statuses & justifications). 3. Apply **Policy**; compute PASS/FAIL with waiver TTLs. 4. Sign the **final report** (DSSE via **Signer**) and log to **Rekor v2** via **Attestor**. @@ -227,6 +243,8 @@ s3://stellaops/ * `artifacts` (type/format/sha/size/rekor/ttl/immutable/refCount/createdAt) * `images`, `layers`, `links`, `lifecycleRules` +* **Scheduler:** `schedules`, `runs`, `locks`, `impact_cursors` +* **Notify:** `rules`, `deliveries`, `channels`, `templates` **Retention** @@ -239,13 +257,13 @@ s3://stellaops/ ### 7.1 Scanner.WebService ``` -POST /api/scans { imageRef|digest, force? } → { scanId } -GET /api/scans/{id} → { status, digests, artifacts[] } -GET /api/sboms/{imageDigest} ?format=cdx-json|cdx-pb|spdx-json&view=inventory|usage +POST /api/scans { imageRef|digest, force? } → { scanId } +GET /api/scans/{id} → { status, digests, artifacts[] } +GET /api/sboms/{imageDigest} ?format=cdx-json|cdx-pb|spdx-json&view=inventory|usage GET /api/diff?old=&new= → { added[], removed[], changed[], byLayer[] } -POST /api/exports { imageDigest, format, view } → { artifactId, rekorUrl } -POST /api/reports { imageDigest, policyRevision? } → { reportId, rekorUrl } -GET /api/catalog/artifacts/{id} → { size, ttl, immutable, rekor, refs } +POST /api/exports { imageDigest, format, view } → { artifactId, rekorUrl } +POST /api/reports { imageDigest, policyRevision?, vexSnapshot? } → { reportId, verdict, rekorUrl } +GET /api/catalog/artifacts/{id} → { size, ttl, immutable, rekor, refs } GET /healthz | /readyz | /metrics ``` @@ -276,6 +294,25 @@ POST /license/introspect { poe } → { active, claims, exp } POST /attest/endorse { bundle } → endorsement bundle (optional) ``` +### 7.6 Scheduler + +``` +POST /api/v1/scheduler/schedules {yaml|json} → { scheduleId } +GET /api/v1/scheduler/schedules → [ { id, nextRun, status, stats } ] +POST /api/v1/scheduler/run { id|selector } → { runId } +GET /api/v1/scheduler/runs/{id} → { status, counts, links } +GET /api/v1/scheduler/cursor → { lastConcelierExportId, lastExcititorExportId } +``` + +### 7.7 Notify + +``` +POST /api/v1/notify/test { channel, target } → { delivered } +POST /api/v1/notify/rules {yaml|json} → { ruleId } +GET /api/v1/notify/rules → [ { id, match, actions, enabled } ] +GET /api/v1/notify/deliveries → [ { id, eventId, channel, status, attempts } ] +``` + --- ## 8) Security & verifiability @@ -283,8 +320,9 @@ POST /attest/endorse { bundle } → endorsement bundle (optio * **Sender‑constrained tokens.** All operational calls use **DPoP** (RFC 9449) or **mTLS‑bound** tokens (RFC 8705). * **Entitlement.** **PoE** is mandatory; revocation honored online. * **Release integrity.** **Signer** independently verifies **scanner image digest** via **Referrers + cosign** before signing. -* **Separation of duties.** Scanner/UI cannot sign; only **Signer** can sign; only **Attestor** can write to **Rekor v2**. +* **Separation of duties.** Scanner/UI/Scheduler/Notify cannot sign; only **Signer** can sign; only **Attestor** can write to **Rekor v2**. * **Verifiers.** Anyone can verify: DSSE signature → certificate chain to **Stella Ops Fulcio/KMS root** → **Rekor v2** inclusion. +* **RBAC.** Roles: `scanner.admin|read`, `scheduler.admin|read`, `notify.admin|read`, `zastava.admin|read`. * **Community vs Authorized.** Free/community runs throttled with no official attestations; authorized runs full speed and produce **Stella Ops‑verified** bundles. **DSSE predicate (SBOM/report)** @@ -321,6 +359,8 @@ Binary header + purl table + roaring bitmaps; optional `usedByEntrypoint` flags * Build‑time path P95 ≤ 3–5 s on warmed bases. * Post‑build delta scan P95 ≤ 10 s for 200 MB images. * Policy + VEX evaluation ≤ 500 ms for 5k components using BOM‑Index. + * **Event → notification** p95 ≤ **30–60 s** under nominal load. + * **Export delta → re‑evaluation verdict** p95 ≤ **5 min** for 10k impacted images. * **Quotas:** license plan enforces QPS/concurrency/size; **Signer** throttles and can deny DSSE. --- @@ -337,32 +377,37 @@ Binary header + purl table + roaring bitmaps; optional `usedByEntrypoint` flags ```yaml services: - authority: { image: stellaops/authority } - fulcio: { image: sigstore/fulcio } - rekor: { image: sigstore/rekor-v2 } - minio: { image: minio/minio, command: server /data --console-address ":9001" } - mongo: { image: mongo:7 } - signer: { image: stellaops/signer, depends_on: [authority, fulcio] } - attestor: { image: stellaops/attestor, depends_on: [rekor, signer] } - scanner-web:{ image: stellaops/scanner-web, depends_on: [mongo, minio, signer, attestor] } - scanner-worker: - image: stellaops/scanner-worker - deploy: { replicas: 4 } - depends_on: [scanner-web] - feedser: { image: stellaops/feedser-web, depends_on: [mongo] } - vexer: { image: stellaops/vexer-web, depends_on: [mongo] } - ui: { image: stellaops/ui, depends_on: [scanner-web, feedser, vexer] } + authority: { image: stellaops/authority } + fulcio: { image: sigstore/fulcio } + rekor: { image: sigstore/rekor-v2 } + minio: { image: minio/minio, command: server /data --console-address ":9001" } + mongo: { image: mongo:7 } + signer: { image: stellaops/signer, depends_on: [authority, fulcio] } + attestor: { image: stellaops/attestor, depends_on: [rekor, signer] } + scanner-web: { image: stellaops/scanner-web, depends_on: [mongo, minio, signer, attestor] } + scanner-worker: { image: stellaops/scanner-worker, deploy: { replicas: 4 }, depends_on: [scanner-web] } + concelier: { image: stellaops/concelier-web, depends_on: [mongo] } + excititor: { image: stellaops/excititor-web, depends_on: [mongo] } + scheduler-web: { image: stellaops/scheduler-web, depends_on: [mongo] } + scheduler-worker:{ image: stellaops/scheduler-worker, deploy: { replicas: 2 }, depends_on: [scheduler-web] } + notify-web: { image: stellaops/notify-web, depends_on: [mongo] } + notify-worker: { image: stellaops/notify-worker, deploy: { replicas: 2 }, depends_on: [notify-web] } + ui: { image: stellaops/ui, depends_on: [scanner-web, concelier, excititor, scheduler-web, notify-web] } ``` * **Backups:** Mongo dumps; MinIO versioned buckets & replication; Rekor v2 DB snapshots; JWKS/Fulcio/KMS key rotation. +* **Ops runbooks:** Scheduler catch‑up after Concelier/Excititor recovery; connector key rotation (Slack/Teams/SMTP). +* **SLOs & alerts:** lag between Concelier/Excititor export and first rescan verdict; delivery failure rates by channel. --- ## 11) Observability & audit * **Metrics:** scan latency, layer cache hit %, artifact bytes, DSSE/Rekor latency, policy evaluation time, queue depth, admission decisions (Zastava). -* **Tracing:** per‑stage spans; correlation IDs across Scanner→Signer→Attestor. -* **Audit logs:** every signing records `license_id`, `image_digest`, `policy_digest`, and Rekor UUID. +* **Scheduler metrics:** `scheduler.impacted_images_total`, `scheduler.jobs_enqueued_total`, `scheduler.selection_ms`, end‑to‑end p95 (event → verdict). +* **Notify metrics:** `notify.sent_total{channel}`, `notify.dropped_total{reason}`, `notify.digest_coalesced_total`, `notify.latency_ms`. +* **Tracing:** per‑stage spans; correlation IDs across Scanner→Signer→Attestor and Concelier/Excititor→Scheduler→Scanner→Notify. +* **Audit logs:** every signing records `license_id`, `image_digest`, `policy_digest`, and Rekor UUID; Scheduler records who scheduled what; Notify records where, when, and why messages were sent or deduped. * **Compliance:** MinIO **Object Lock** for immutable artifacts; reproducible outputs via policy digest + SBOM digest in predicate. --- @@ -373,11 +418,13 @@ services: * M2: Buildx generator certified flows; cross‑registry trust policies. * M3: Patch‑Presence plugin (signature‑based backport detection), opt‑in. * M3: Zastava Admission control GA with policy presets and dry‑run→enforce stages. -* Continuous: Policy UX (waiver TTLs, vendor rules), Vexer connectors expansion. +* M3: **Scheduler GA** with export‑delta impact routing and capacity‑aware pacing. +* M3: **Notify GA** with digests, Slack/Teams/Email/Webhooks; **M4:** PagerDuty/Opsgenie connectors. +* Continuous: Policy UX (waiver TTLs, vendor rules), Excititor connectors expansion. --- -## 13) Canonical sequences (verification & signing) +## 13) Canonical sequences (verification, re‑evaluation & notify) **Sign & log (OpTok + PoE, image verify, DSSE, Rekor).** @@ -409,22 +456,62 @@ sequenceDiagram end ``` -**Verification (third party).** +**Event‑driven re‑evaluation & notify.** -```plantuml -@startuml -actor Verifier -participant "stellaops verify" as Tool -database "Fulcio/KMS root" as Root -participant "Rekor v2" as R2 -Verifier -> Tool: bundle (URL/file) -Tool -> Tool: Verify DSSE signature -Tool -> Root: Verify cert chain to StellaOps root -Tool -> R2: Verify inclusion proof / query by UUID -Tool -> Verifier: OK + claims (license_id, policy_digest, version) -@enduml +```mermaid +sequenceDiagram + participant CONC as Concelier + participant EXC as Excititor + participant SCH as Scheduler + participant SC as Scanner.WebService + participant NO as Notify + + CONC->>SCH: export.delta {changedProductKeys, exportId} + EXC ->>SCH: export.delta {changedProductKeys, exportId} + SCH->>SCH: Impact select via BOM-Index bitmaps + SCH->>SC: Enqueue analysis-only reports (batches) + SC-->>SCH: verdict stream (PASS/FAIL, deltas) + SCH->>NO: rescan.delta {imageDigest, newCriticals, links} + NO-->>Slack/Teams/Email/Webhook: deliver (throttle/digest rules applied) ``` --- -**End of `high_level_architecture.md` (Consolidated).** +## 14) Minimal data shapes (Scheduler & Notify) + +**Scheduler schedule (YAML via UI/CLI)** + +```yaml +name: nightly-eu +when: "0 2 * * * Europe/Sofia" +mode: analysis-only # or content-refresh +selection: + scope: all-images # or tenant/ns/repo label selectors + onlyIf: { lastReportOlderThanDays: 7 } +notify: + onNewFindings: true + minSeverity: high +limits: + maxJobs: 5000 + ratePerSecond: 50 +``` + +**Notify rule (YAML)** + +```yaml +name: high-critical-alerts +match: + eventKinds: ["report.ready","rescan.delta","zastava.admission"] + minSeverity: high + namespaces: ["prod-*"] + vex: { includeAcceptedJustifications: false } +actions: + - channel: slack + target: "#sec-alerts" + template: "concise" + throttle: "5m" + - channel: email + target: "soc@acme.org" + digest: "hourly" +enabled: true +``` diff --git a/docs/09_API_CLI_REFERENCE.md b/docs/09_API_CLI_REFERENCE.md index 564ae818..94daac59 100755 --- a/docs/09_API_CLI_REFERENCE.md +++ b/docs/09_API_CLI_REFERENCE.md @@ -149,14 +149,288 @@ Client then generates SBOM **only** for the `missing` layers and re‑posts `/sc --- -### 2.3 Policy Endpoints +### 2.3 Policy Endpoints *(preview feature flag: `scanner.features.enablePolicyPreview`)* -| Method | Path | Purpose | -| ------ | ------------------ | ------------------------------------ | -| `GET` | `/policy/export` | Download live YAML ruleset | -| `POST` | `/policy/import` | Upload YAML or Rego; replaces active | -| `POST` | `/policy/validate` | Lint only; returns 400 on error | -| `GET` | `/policy/history` | Paginated change log (audit trail) | +All policy APIs require **`scanner.reports`** scope (or anonymous access while auth is disabled). + +**Fetch schema** + +``` +GET /api/v1/policy/schema +Authorization: Bearer +Accept: application/schema+json +``` + +Returns the embedded `policy-schema@1` JSON schema used by the binder. + +**Run diagnostics** + +``` +POST /api/v1/policy/diagnostics +Content-Type: application/json +Authorization: Bearer +``` + +```json +{ + "policy": { + "format": "yaml", + "actor": "cli", + "description": "dev override", + "content": "version: \"1.0\"\nrules:\n - name: Quiet Dev\n environments: [dev]\n action:\n type: ignore\n justification: dev waiver\n" + } +} +``` + +**Response 200**: + +```json +{ + "success": false, + "version": "1.0", + "ruleCount": 1, + "errorCount": 0, + "warningCount": 1, + "generatedAt": "2025-10-19T03:25:14.112Z", + "issues": [ + { "code": "policy.rule.quiet.missing_vex", "message": "Quiet flag ignored: rule must specify requireVex justifications.", "severity": "Warning", "path": "$.rules[0]" } + ], + "recommendations": [ + "Review policy warnings and ensure intentional overrides are documented." + ] +} +``` + +`success` is `false` when blocking issues remain; recommendations aggregate YAML ignore rules, VEX include/exclude hints, and vendor precedence guidance. + +**Preview impact** + +``` +POST /api/v1/policy/preview +Authorization: Bearer +Content-Type: application/json +``` + +```json +{ + "imageDigest": "sha256:abc123", + "findings": [ + { "id": "finding-1", "severity": "Critical", "source": "NVD" } + ], + "policy": { + "format": "yaml", + "content": "version: \"1.0\"\nrules:\n - name: Block Critical\n severity: [Critical]\n action: block\n" + } +} +``` + +**Response 200**: + +```json +{ + "success": true, + "policyDigest": "9c5e...", + "revisionId": "preview", + "changed": 1, + "diffs": [ + { + "findingId": "finding-1", + "baseline": {"findingId": "finding-1", "status": "Pass"}, + "projected": { + "findingId": "finding-1", + "status": "Blocked", + "ruleName": "Block Critical", + "ruleAction": "Block", + "score": 5.0, + "configVersion": "1.0", + "inputs": {"severityWeight": 5.0} + }, + "changed": true + } + ], + "issues": [] +} +``` + +- Provide `policy` to preview staged changes; omit it to compare against the active snapshot. +- Baseline verdicts are optional; when omitted, the API synthesises pass baselines before computing diffs. +- Quieted verdicts include `quietedBy` and `quiet` flags; score inputs now surface reachability/vendor trust weights (`reachability.*`, `trustWeight.*`). + +**OpenAPI**: the full API document (including these endpoints) is exposed at `/openapi/v1.json` and can be fetched for tooling or contract regeneration. + +### 2.4 Scanner – Queue a Scan Job *(SP9 milestone)* + +``` +POST /api/v1/scans +Authorization: Bearer +Content-Type: application/json +``` + +```json +{ + "image": { + "reference": "registry.example.com/acme/app:1.2.3" + }, + "force": false, + "clientRequestId": "ci-build-1845", + "metadata": { + "pipeline": "github", + "trigger": "pull-request" + } +} +``` + +| Field | Required | Notes | +| ------------------- | -------- | ------------------------------------------------------------------------------------------------ | +| `image.reference` | no\* | Full repo/tag (`registry/repo:tag`). Provide **either** `reference` or `digest` (sha256:…). | +| `image.digest` | no\* | OCI digest (e.g. `sha256:…`). | +| `force` | no | `true` forces a re-run even if an identical scan (`scanId`) already exists. Default **false**. | +| `clientRequestId` | no | Free-form string surfaced in audit logs. | +| `metadata` | no | Optional string map stored with the job and surfaced in observability feeds. | + +\* At least one of `image.reference` or `image.digest` must be supplied. + +**Response 202** – job accepted (idempotent): + +```http +HTTP/1.1 202 Accepted +Location: /api/v1/scans/2f6c17f9b3f548e2a28b9c412f4d63f8 +``` + +```json +{ + "scanId": "2f6c17f9b3f548e2a28b9c412f4d63f8", + "status": "Pending", + "location": "/api/v1/scans/2f6c17f9b3f548e2a28b9c412f4d63f8", + "created": true +} +``` + +- `scanId` is deterministic – resubmitting an identical payload returns the same identifier with `"created": false`. +- API is cancellation-aware; aborting the HTTP request cancels the submission attempt. +- Required scope: **`scanner.scans.enqueue`**. + +**Response 400** – validation problem (`Content-Type: application/problem+json`) when both `image.reference` and `image.digest` are blank. + +### 2.5 Scanner – Fetch Scan Status + +``` +GET /api/v1/scans/{scanId} +Authorization: Bearer +Accept: application/json +``` + +**Response 200**: + +```json +{ + "scanId": "2f6c17f9b3f548e2a28b9c412f4d63f8", + "status": "Pending", + "image": { + "reference": "registry.example.com/acme/app:1.2.3", + "digest": null + }, + "createdAt": "2025-10-18T20:15:12.482Z", + "updatedAt": "2025-10-18T20:15:12.482Z", + "failureReason": null +} +``` + +Statuses: `Pending`, `Running`, `Succeeded`, `Failed`, `Cancelled`. + +### 2.6 Scanner – Stream Progress (SSE / JSONL) + +``` +GET /api/v1/scans/{scanId}/events?format=sse|jsonl +Authorization: Bearer +Accept: text/event-stream +``` + +When `format` is omitted the endpoint emits **Server-Sent Events** (SSE). Specify `format=jsonl` to receive newline-delimited JSON (`application/x-ndjson`). Response headers include `Cache-Control: no-store` and `X-Accel-Buffering: no` so intermediaries avoid buffering the stream. + +**SSE frame** (default): + +``` +id: 1 +event: pending +data: {"scanId":"2f6c17f9b3f548e2a28b9c412f4d63f8","sequence":1,"state":"Pending","message":"queued","timestamp":"2025-10-19T03:12:45.118Z","correlationId":"2f6c17f9b3f548e2a28b9c412f4d63f8:0001","data":{"force":false,"meta.pipeline":"github"}} +``` + +**JSONL frame** (`format=jsonl`): + +```json +{"scanId":"2f6c17f9b3f548e2a28b9c412f4d63f8","sequence":1,"state":"Pending","message":"queued","timestamp":"2025-10-19T03:12:45.118Z","correlationId":"2f6c17f9b3f548e2a28b9c412f4d63f8:0001","data":{"force":false,"meta.pipeline":"github"}} +``` + +- `sequence` is monotonic starting at `1`. +- `correlationId` is deterministic (`{scanId}:{sequence:0000}`) unless a custom identifier is supplied by the publisher. +- `timestamp` is ISO‑8601 UTC with millisecond precision, ensuring deterministic ordering for consumers. +- The stream completes when the client disconnects or the coordinator stops publishing events. + +### 2.7 Scanner – Assemble Report (Signed Envelope) + +``` +POST /api/v1/reports +Authorization: Bearer +Content-Type: application/json +``` + +Request body mirrors policy preview inputs (image digest plus findings). The service evaluates the active policy snapshot, assembles a verdict, and signs the canonical report payload. + +**Response 200**: + +```json +{ + "report": { + "reportId": "report-3def5f362aa475ef14b6", + "imageDigest": "sha256:deadbeef", + "verdict": "blocked", + "policy": { "revisionId": "rev-1", "digest": "27d2ec2b34feedc304fc564d252ecee1c8fa14ea581a5ff5c1ea8963313d5c8d" }, + "summary": { "total": 1, "blocked": 1, "warned": 0, "ignored": 0, "quieted": 0 }, + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "ruleName": "Block Critical", + "ruleAction": "Block", + "score": 40.5, + "configVersion": "1.0", + "inputs": { + "reachabilityWeight": 0.45, + "baseScore": 40.5, + "severityWeight": 90, + "trustWeight": 1, + "trustWeight.NVD": 1, + "reachability.runtime": 0.45 + }, + "quiet": false, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ], + "issues": [] + }, + "dsse": { + "payloadType": "application/vnd.stellaops.report+json", + "payload": "", + "signatures": [ + { + "keyId": "scanner-report-signing", + "algorithm": "hs256", + "signature": "" + } + ] + } +} +``` + +- The `report` object omits null fields and is deterministic (ISO timestamps, sorted keys). +- `dsse` follows the DSSE (Dead Simple Signing Envelope) shape; `payload` is the canonical UTF-8 JSON and `signatures[0].signature` is the base64 HMAC/Ed25519 value depending on configuration. +- A runnable sample envelope is available at `samples/api/reports/report-sample.dsse.json` for tooling tests or signature verification. + +**Response 404** – `application/problem+json` payload with type `https://stellaops.org/problems/not-found` when the scan identifier is unknown. + +> **Tip** – poll `Location` from the submission call until `status` transitions away from `Pending`/`Running`. ```yaml # Example import payload (YAML) @@ -181,6 +455,23 @@ Validation errors come back as: } ``` +```json +# Preview response excerpt +{ + "success": true, + "policyDigest": "9c5e...", + "revisionId": "rev-12", + "changed": 1, + "diffs": [ + { + "baseline": {"findingId": "finding-1", "status": "pass"}, + "projected": {"findingId": "finding-1", "status": "blocked", "ruleName": "Block Critical"}, + "changed": true + } + ] +} +``` + --- ### 2.4 Attestation (Planned – Q1‑2026) @@ -200,7 +491,7 @@ Returns `202 Accepted` and `Location: /attest/{id}` for async verify. ## 3 StellaOps CLI (`stellaops-cli`) -The new CLI is built on **System.CommandLine 2.0.0‑beta5** and mirrors the Feedser backend REST API. +The new CLI is built on **System.CommandLine 2.0.0‑beta5** and mirrors the Concelier backend REST API. Configuration follows the same precedence chain everywhere: 1. Environment variables (e.g. `API_KEY`, `STELLAOPS_BACKEND_URL`, `StellaOps:ApiKey`) @@ -231,9 +522,12 @@ See `docs/dev/32_AUTH_CLIENT_GUIDE.md` for recommended profiles (online vs. air- | `stellaops-cli auth revoke export` | Export the Authority revocation bundle | `--output ` (defaults to CWD) | Writes `revocation-bundle.json`, `.json.jws`, and `.json.sha256`; verifies the digest locally and includes key metadata in the log summary. | | `stellaops-cli auth revoke verify` | Validate a revocation bundle offline | `--bundle ` `--signature ` `--key `
`--verbose` | Verifies detached JWS signatures, reports the computed SHA-256, and can fall back to cached JWKS when `--key` is omitted. | | `stellaops-cli config show` | Display resolved configuration | — | Masks secret values; helpful for air‑gapped installs | +| `stellaops-cli runtime policy test` | Ask Scanner.WebService for runtime verdicts (Webhook parity) | `--image/-i ` (repeatable, comma/space lists supported)
`--file/-f `
`--namespace/--ns `
`--label/-l key=value` (repeatable)
`--json` | Posts to `POST /api/v1/scanner/policy/runtime`, deduplicates image digests, and prints TTL/policy revision plus per-image columns for signed state, SBOM referrers, quieted-by metadata, confidence, and Rekor attestation (uuid + verified flag). Accepts newline/whitespace-delimited stdin when piped; `--json` emits the raw response without additional logging. | When running on an interactive terminal without explicit override flags, the CLI uses Spectre.Console prompts to let you choose per-run ORAS/offline bundle behaviour. +Runtime verdict output reflects the SCANNER-RUNTIME-12-302 contract sign-off (quieted provenance, confidence band, attestation verification). CLI-RUNTIME-13-008 now mirrors those fields in both table and `--json` formats. + **Startup diagnostics** - `stellaops-cli` now loads Authority plug-in manifests during startup (respecting `Authority:Plugins:*`) and surfaces analyzer warnings when a plug-in weakens the baseline password policy (minimum length **12** and all character classes required). @@ -250,7 +544,7 @@ When running on an interactive terminal without explicit override flags, the CLI - Downloads are verified against the `X-StellaOps-Digest` header (SHA-256). When `StellaOps:ScannerSignaturePublicKeyPath` points to a PEM-encoded RSA key, the optional `X-StellaOps-Signature` header is validated as well. - Metadata for each bundle is written alongside the artefact (`*.metadata.json`) with digest, signature, source URL, and timestamps. - Retry behaviour is controlled via `StellaOps:ScannerDownloadAttempts` (default **3** with exponential backoff). -- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory` plus a `scan-run-*.json` metadata envelope documenting the runner, arguments, timing, and stdout/stderr. The artefact is posted back to Feedser automatically. +- Successful `scan run` executions create timestamped JSON artefacts inside `ResultsDirectory` plus a `scan-run-*.json` metadata envelope documenting the runner, arguments, timing, and stdout/stderr. The artefact is posted back to Concelier automatically. #### Trivy DB export metadata (`metadata.json`) @@ -265,18 +559,18 @@ When running on an interactive terminal without explicit override flags, the CLI | `treeDigest` | string | Canonical SHA-256 digest of the JSON tree used to build the database. | | `treeBytes` | number | Total bytes across exported JSON files. | | `advisoryCount` | number | Count of advisories included in the export. | -| `exporterVersion` | string | Version stamp of `StellaOps.Feedser.Exporter.TrivyDb`. | +| `exporterVersion` | string | Version stamp of `StellaOps.Concelier.Exporter.TrivyDb`. | | `builder` | object? | Raw metadata emitted by `trivy-db build` (version, update cadence, etc.). | | `delta.changedFiles[]` | array | Present when `mode = delta`. Each entry lists `{ "path": "", "length": , "digest": "sha256:..." }`. | | `delta.removedPaths[]` | array | Paths that existed in the previous manifest but were removed in the new run. | When the planner opts for a delta run, the exporter copies unmodified blobs from the baseline layout identified by `baseManifestDigest`. Consumers that cache OCI blobs only need to fetch the `changedFiles` and the new manifest/metadata unless `resetBaseline` is true. -When pushing to ORAS, set `feedser:exporters:trivyDb:oras:publishFull` / `publishDelta` to control whether full or delta runs are copied to the registry. Offline bundles follow the analogous `includeFull` / `includeDelta` switches under `offlineBundle`. +When pushing to ORAS, set `concelier:exporters:trivyDb:oras:publishFull` / `publishDelta` to control whether full or delta runs are copied to the registry. Offline bundles follow the analogous `includeFull` / `includeDelta` switches under `offlineBundle`. Example configuration (`appsettings.yaml`): ```yaml -feedser: +concelier: exporters: trivyDb: oras: @@ -293,7 +587,7 @@ feedser: **Authentication** - API key is sent as `Authorization: Bearer ` automatically when configured. -- Anonymous operation is permitted only when Feedser runs with +- Anonymous operation is permitted only when Concelier runs with `authority.allowAnonymousFallback: true`. This flag is temporary—plan to disable it before **2025-12-31 UTC** so bearer tokens become mandatory. @@ -303,7 +597,7 @@ Authority-backed auth workflow: 3. Execute CLI commands as normal—the backend client injects the cached bearer token automatically and retries on transient 401/403 responses with operator guidance. 4. Inspect the cache with `stellaops-cli auth status` (shows expiry, scope, mode) or clear it via `stellaops-cli auth logout`. 5. Run `stellaops-cli auth whoami` to dump token subject, audience, issuer, scopes, and remaining lifetime (verbose mode prints additional claims). -6. Expect Feedser to emit audit logs for each `/jobs*` request showing `subject`, +6. Expect Concelier to emit audit logs for each `/jobs*` request showing `subject`, `clientId`, `scopes`, `status`, and whether network bypass rules were applied. Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirectory` overrides it. Cached tokens are reused offline until they expire; the CLI surfaces clear errors if refresh fails. @@ -314,7 +608,7 @@ Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirec { "StellaOps": { "ApiKey": "your-api-token", - "BackendUrl": "https://feedser.example.org", + "BackendUrl": "https://concelier.example.org", "ScannerCacheDirectory": "scanners", "ResultsDirectory": "results", "DefaultRunner": "docker", @@ -322,11 +616,11 @@ Tokens live in `~/.stellaops/tokens` unless `StellaOps:Authority:TokenCacheDirec "ScannerDownloadAttempts": 3, "Authority": { "Url": "https://authority.example.org", - "ClientId": "feedser-cli", + "ClientId": "concelier-cli", "ClientSecret": "REDACTED", "Username": "", "Password": "", - "Scope": "feedser.jobs.trigger", + "Scope": "concelier.jobs.trigger", "TokenCacheDirectory": "" } } diff --git a/docs/10_FEEDSER_CLI_QUICKSTART.md b/docs/10_CONCELIER_CLI_QUICKSTART.md similarity index 80% rename from docs/10_FEEDSER_CLI_QUICKSTART.md rename to docs/10_CONCELIER_CLI_QUICKSTART.md index 7283d6d8..3c02e5fc 100644 --- a/docs/10_FEEDSER_CLI_QUICKSTART.md +++ b/docs/10_CONCELIER_CLI_QUICKSTART.md @@ -1,289 +1,289 @@ -# 10 · Feedser + CLI Quickstart - -This guide walks through configuring the Feedser web service and the `stellaops-cli` -tool so an operator can ingest advisories, merge them, and publish exports from a -single workstation. It focuses on deployment-facing surfaces only (configuration, -runtime wiring, CLI usage) and leaves connector/internal customization for later. - ---- - -## 0 · Prerequisites - -- .NET SDK **10.0.100-preview** (matches `global.json`) -- MongoDB instance reachable from the host (local Docker or managed) -- `trivy-db` binary on `PATH` for Trivy exports (and `oras` if publishing to OCI) -- Plugin assemblies present in `PluginBinaries/` (already included in the repo) -- Optional: Docker/Podman runtime if you plan to run scanners locally - -> **Tip** – air-gapped installs should preload `trivy-db` and `oras` binaries into the -> runner image since Feedser never fetches them dynamically. - ---- - -## 1 · Configure Feedser - -1. Copy the sample config to the expected location (CI/CD pipelines can stamp values - into this file during deployment—see the “Deployment automation” note below): - - ```bash - mkdir -p etc - cp etc/feedser.yaml.sample etc/feedser.yaml - ``` - -2. Edit `etc/feedser.yaml` and update the MongoDB DSN (and optional database name). - The default template configures plug-in discovery to look in `PluginBinaries/` - and disables remote telemetry exporters by default. - -3. (Optional) Override settings via environment variables. All keys are prefixed with - `FEEDSER_`. Example: - - ```bash - export FEEDSER_STORAGE__DSN="mongodb://user:pass@mongo:27017/feedser" - export FEEDSER_TELEMETRY__ENABLETRACING=false - ``` - -4. Start the web service from the repository root: - - ```bash - dotnet run --project src/StellaOps.Feedser.WebService - ``` - - On startup Feedser validates the options, boots MongoDB indexes, loads plug-ins, - and exposes: - - - `GET /health` – returns service status and telemetry settings - - `GET /ready` – performs a MongoDB `ping` - - `GET /jobs` + `POST /jobs/{kind}` – inspect and trigger connector/export jobs - - > **Security note** – authentication now ships via StellaOps Authority. Keep - > `authority.allowAnonymousFallback: true` only during the staged rollout and - > disable it before **2025-12-31 UTC** so tokens become mandatory. - -### Authority companion configuration (preview) - -1. Copy the Authority sample configuration: - - ```bash - cp etc/authority.yaml.sample etc/authority.yaml - ``` - -2. Update the issuer URL, token lifetimes, and plug-in descriptors to match your - environment. Authority expects per-plugin manifests in `etc/authority.plugins/`; - sample `standard.yaml` and `ldap.yaml` files are provided as starting points. - For air-gapped installs keep the default plug-in binary directory - (`../PluginBinaries/Authority`) so packaged plug-ins load without outbound access. - -3. Environment variables prefixed with `STELLAOPS_AUTHORITY_` override individual - fields. Example: - - ```bash - export STELLAOPS_AUTHORITY__ISSUER="https://authority.stella-ops.local" - export STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0="/srv/authority/plugins" - ``` - ---- - -## 2 · Configure the CLI - -The CLI reads configuration from JSON/YAML files *and* environment variables. The -defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at runtime. - -| Setting | Environment variable | Default | Purpose | -| ------- | -------------------- | ------- | ------- | -| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Feedser web service | -| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority | -| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder | -| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written | -| `Authority.Url` | `STELLAOPS_AUTHORITY_URL` | _empty_ | StellaOps Authority issuer/token endpoint | -| `Authority.ClientId` | `STELLAOPS_AUTHORITY_CLIENT_ID` | _empty_ | Client identifier for the CLI | -| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) | -| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows | -| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows | -| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `feedser.jobs.trigger` | OAuth scope requested for backend operations | -| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens | -| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls | -| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) | -| `Authority.Resilience.AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Allow CLI to reuse cached discovery/JWKS metadata when Authority is offline | -| `Authority.Resilience.OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window applied to cached metadata | - -Example bootstrap: - -```bash -export STELLAOPS_BACKEND_URL="http://localhost:5000" -export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results" -export STELLAOPS_AUTHORITY_URL="https://authority.local" -export STELLAOPS_AUTHORITY_CLIENT_ID="feedser-cli" -export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t" -dotnet run --project src/StellaOps.Cli -- db merge - -# Acquire a bearer token and confirm cache state -dotnet run --project src/StellaOps.Cli -- auth login -dotnet run --project src/StellaOps.Cli -- auth status -dotnet run --project src/StellaOps.Cli -- auth whoami -``` - -Refer to `docs/dev/32_AUTH_CLIENT_GUIDE.md` for deeper guidance on tuning retry/offline settings and rollout checklists. - -To persist configuration, you can create `stellaops-cli.yaml` next to the binary or -rely on environment variables for ephemeral runners. - ---- - -## 3 · Operating Workflow - -1. **Trigger connector fetch stages** - - ```bash - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage fetch - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage parse - dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage map - ``` - - Use `--mode resume` when continuing from a previous window: - - ```bash - dotnet run --project src/StellaOps.Cli -- db fetch --source redhat --stage fetch --mode resume - ``` - -2. **Merge canonical advisories** - - ```bash - dotnet run --project src/StellaOps.Cli -- db merge - ``` - -3. **Produce exports** - - ```bash - # JSON tree (vuln-list style) - dotnet run --project src/StellaOps.Cli -- db export --format json - - # Trivy DB (delta example) - dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta - ``` - - Feedser always produces a deterministic OCI layout. The first run after a clean - bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous - baseline’s blobs when only JSON manifests change. If the exporter detects that a - prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically - upgrades the next run to a full export and resets the baseline so operators never - chain deltas indefinitely. The CLI exposes `--publish-full/--publish-delta` (for - ORAS pushes) and `--include-full/--include-delta` (for offline bundles) should you - need to override the defaults interactively. - - **Smoke-check delta reuse:** after the first baseline completes, run the export a - second time with `--delta` and verify that the new directory reports `mode=delta` - while reusing the previous layer blob. - - ```bash - export_root=${FEEDSER_EXPORT_ROOT:-exports/trivy} - base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1) - delta=$(ls -1d "$export_root"/* | sort | tail -n1) - - jq -r '.mode,.baseExportId' "$delta/metadata.json" - - base_manifest=$(jq -r '.manifests[0].digest' "$base/index.json") - delta_manifest=$(jq -r '.manifests[0].digest' "$delta/index.json") - printf 'baseline manifest: %s\ndelta manifest: %s\n' "$base_manifest" "$delta_manifest" - - layer_digest=$(jq -r '.layers[0].digest' "$base/blobs/sha256/${base_manifest#sha256:}") - cmp "$base/blobs/sha256/${layer_digest#sha256:}" \ - "$delta/blobs/sha256/${layer_digest#sha256:}" - ``` - - `cmp` returning exit code `0` confirms the delta export reuses the baseline’s - `db.tar.gz` layer instead of rebuilding it. - -4. **Manage scanners (optional)** - - ```bash - dotnet run --project src/StellaOps.Cli -- scanner download --channel stable - dotnet run --project src/StellaOps.Cli -- scan run --entry scanners/latest/Scanner.dll --target ./sboms - dotnet run --project src/StellaOps.Cli -- scan upload --file results/scan-001.json - ``` - -Add `--verbose` to any command for structured console logs. All commands honour -`Ctrl+C` cancellation and exit with non-zero status codes when the backend returns -a problem document. - ---- - -## 4 · Verification Checklist - -- Feedser `/health` returns `"status":"healthy"` and Storage bootstrap is marked - complete after startup. -- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when - triggering Feedser jobs. -- Export artefacts are materialised under the configured output directories and - their manifests record digests. -- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state` - collections after a run. - ---- - -## 5 · Deployment Automation - -- Treat `etc/feedser.yaml.sample` as the canonical template. CI/CD should copy it to - the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron - overrides) with environment-specific secrets. -- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository; - inject them via secret stores or pipeline variables at stamp time. -- When building container images, include `trivy-db` (and `oras` if used) so air-gapped - clusters do not need outbound downloads at runtime. - ---- - -## 5 · Next Steps - -- Enable authority-backed authentication in non-production first. Set - `authority.enabled: true` while keeping `authority.allowAnonymousFallback: true` - to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens. -- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs). -- Pair with the Feedser connector teams when enabling additional sources so their - module-specific requirements are pulled in safely. - ---- - -## 6 · Authority Integration - -- Feedser now authenticates callers through StellaOps Authority using OAuth 2.0 - resource server flows. Populate the `authority` block in `feedser.yaml`: - - ```yaml - authority: - enabled: true - allowAnonymousFallback: false # keep true only during the staged rollout window - issuer: "https://authority.example.org" - audiences: - - "api://feedser" - requiredScopes: - - "feedser.jobs.trigger" - clientId: "feedser-jobs" - clientSecretFile: "../secrets/feedser-jobs.secret" - clientScopes: - - "feedser.jobs.trigger" - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - ``` - -- Store the client secret outside of source control. Either provide it via - `authority.clientSecret` (environment variable `FEEDSER_AUTHORITY__CLIENTSECRET`) - or point `authority.clientSecretFile` to a file mounted at runtime. -- Cron jobs running on the same host can keep using the API thanks to the loopback - bypass mask. Add additional CIDR ranges as needed; every bypass is logged. -- Export the same configuration to Kubernetes or systemd by setting environment - variables such as: - - ```bash - export FEEDSER_AUTHORITY__ENABLED=true - export FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false - export FEEDSER_AUTHORITY__ISSUER="https://authority.example.org" - export FEEDSER_AUTHORITY__CLIENTID="feedser-jobs" - export FEEDSER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/feedser/authority-client" - ``` - -- CLI commands already pass `Authorization` headers when credentials are supplied. - Configure the CLI with matching Authority settings (`docs/09_API_CLI_REFERENCE.md`) - so that automation can obtain tokens with the same client credentials. Feedser - logs every job request with the client ID, subject (if present), scopes, and - a `bypass` flag so operators can audit cron traffic. +# 10 · Concelier + CLI Quickstart + +This guide walks through configuring the Concelier web service and the `stellaops-cli` +tool so an operator can ingest advisories, merge them, and publish exports from a +single workstation. It focuses on deployment-facing surfaces only (configuration, +runtime wiring, CLI usage) and leaves connector/internal customization for later. + +--- + +## 0 · Prerequisites + +- .NET SDK **10.0.100-preview** (matches `global.json`) +- MongoDB instance reachable from the host (local Docker or managed) +- `trivy-db` binary on `PATH` for Trivy exports (and `oras` if publishing to OCI) +- Plugin assemblies present in `StellaOps.Concelier.PluginBinaries/` (already included in the repo) +- Optional: Docker/Podman runtime if you plan to run scanners locally + +> **Tip** – air-gapped installs should preload `trivy-db` and `oras` binaries into the +> runner image since Concelier never fetches them dynamically. + +--- + +## 1 · Configure Concelier + +1. Copy the sample config to the expected location (CI/CD pipelines can stamp values + into this file during deployment—see the “Deployment automation” note below): + + ```bash + mkdir -p etc + cp etc/concelier.yaml.sample etc/concelier.yaml + ``` + +2. Edit `etc/concelier.yaml` and update the MongoDB DSN (and optional database name). + The default template configures plug-in discovery to look in `StellaOps.Concelier.PluginBinaries/` + and disables remote telemetry exporters by default. + +3. (Optional) Override settings via environment variables. All keys are prefixed with + `CONCELIER_`. Example: + + ```bash + export CONCELIER_STORAGE__DSN="mongodb://user:pass@mongo:27017/concelier" + export CONCELIER_TELEMETRY__ENABLETRACING=false + ``` + +4. Start the web service from the repository root: + + ```bash + dotnet run --project src/StellaOps.Concelier.WebService + ``` + + On startup Concelier validates the options, boots MongoDB indexes, loads plug-ins, + and exposes: + + - `GET /health` – returns service status and telemetry settings + - `GET /ready` – performs a MongoDB `ping` + - `GET /jobs` + `POST /jobs/{kind}` – inspect and trigger connector/export jobs + + > **Security note** – authentication now ships via StellaOps Authority. Keep + > `authority.allowAnonymousFallback: true` only during the staged rollout and + > disable it before **2025-12-31 UTC** so tokens become mandatory. + +### Authority companion configuration (preview) + +1. Copy the Authority sample configuration: + + ```bash + cp etc/authority.yaml.sample etc/authority.yaml + ``` + +2. Update the issuer URL, token lifetimes, and plug-in descriptors to match your + environment. Authority expects per-plugin manifests in `etc/authority.plugins/`; + sample `standard.yaml` and `ldap.yaml` files are provided as starting points. + For air-gapped installs keep the default plug-in binary directory + (`../StellaOps.Authority.PluginBinaries`) so packaged plug-ins load without outbound access. + +3. Environment variables prefixed with `STELLAOPS_AUTHORITY_` override individual + fields. Example: + + ```bash + export STELLAOPS_AUTHORITY__ISSUER="https://authority.stella-ops.local" + export STELLAOPS_AUTHORITY__PLUGINDIRECTORIES__0="/srv/authority/plugins" + ``` + +--- + +## 2 · Configure the CLI + +The CLI reads configuration from JSON/YAML files *and* environment variables. The +defaults live in `src/StellaOps.Cli/appsettings.json` and expect overrides at runtime. + +| Setting | Environment variable | Default | Purpose | +| ------- | -------------------- | ------- | ------- | +| `BackendUrl` | `STELLAOPS_BACKEND_URL` | _empty_ | Base URL of the Concelier web service | +| `ApiKey` | `API_KEY` | _empty_ | Reserved for legacy key auth; leave empty when using Authority | +| `ScannerCacheDirectory` | `STELLAOPS_SCANNER_CACHE_DIRECTORY` | `scanners` | Local cache folder | +| `ResultsDirectory` | `STELLAOPS_RESULTS_DIRECTORY` | `results` | Where scan outputs are written | +| `Authority.Url` | `STELLAOPS_AUTHORITY_URL` | _empty_ | StellaOps Authority issuer/token endpoint | +| `Authority.ClientId` | `STELLAOPS_AUTHORITY_CLIENT_ID` | _empty_ | Client identifier for the CLI | +| `Authority.ClientSecret` | `STELLAOPS_AUTHORITY_CLIENT_SECRET` | _empty_ | Client secret (omit when using username/password grant) | +| `Authority.Username` | `STELLAOPS_AUTHORITY_USERNAME` | _empty_ | Username for password grant flows | +| `Authority.Password` | `STELLAOPS_AUTHORITY_PASSWORD` | _empty_ | Password for password grant flows | +| `Authority.Scope` | `STELLAOPS_AUTHORITY_SCOPE` | `concelier.jobs.trigger` | OAuth scope requested for backend operations | +| `Authority.TokenCacheDirectory` | `STELLAOPS_AUTHORITY_TOKEN_CACHE_DIR` | `~/.stellaops/tokens` | Directory that persists cached tokens | +| `Authority.Resilience.EnableRetries` | `STELLAOPS_AUTHORITY_ENABLE_RETRIES` | `true` | Toggle Polly retry handler for Authority HTTP calls | +| `Authority.Resilience.RetryDelays` | `STELLAOPS_AUTHORITY_RETRY_DELAYS` | `1s,2s,5s` | Comma- or space-separated backoff delays (hh:mm:ss) | +| `Authority.Resilience.AllowOfflineCacheFallback` | `STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK` | `true` | Allow CLI to reuse cached discovery/JWKS metadata when Authority is offline | +| `Authority.Resilience.OfflineCacheTolerance` | `STELLAOPS_AUTHORITY_OFFLINE_CACHE_TOLERANCE` | `00:10:00` | Additional tolerance window applied to cached metadata | + +Example bootstrap: + +```bash +export STELLAOPS_BACKEND_URL="http://localhost:5000" +export STELLAOPS_RESULTS_DIRECTORY="$HOME/.stellaops/results" +export STELLAOPS_AUTHORITY_URL="https://authority.local" +export STELLAOPS_AUTHORITY_CLIENT_ID="concelier-cli" +export STELLAOPS_AUTHORITY_CLIENT_SECRET="s3cr3t" +dotnet run --project src/StellaOps.Cli -- db merge + +# Acquire a bearer token and confirm cache state +dotnet run --project src/StellaOps.Cli -- auth login +dotnet run --project src/StellaOps.Cli -- auth status +dotnet run --project src/StellaOps.Cli -- auth whoami +``` + +Refer to `docs/dev/32_AUTH_CLIENT_GUIDE.md` for deeper guidance on tuning retry/offline settings and rollout checklists. + +To persist configuration, you can create `stellaops-cli.yaml` next to the binary or +rely on environment variables for ephemeral runners. + +--- + +## 3 · Operating Workflow + +1. **Trigger connector fetch stages** + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage fetch + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage parse + dotnet run --project src/StellaOps.Cli -- db fetch --source osv --stage map + ``` + + Use `--mode resume` when continuing from a previous window: + + ```bash + dotnet run --project src/StellaOps.Cli -- db fetch --source redhat --stage fetch --mode resume + ``` + +2. **Merge canonical advisories** + + ```bash + dotnet run --project src/StellaOps.Cli -- db merge + ``` + +3. **Produce exports** + + ```bash + # JSON tree (vuln-list style) + dotnet run --project src/StellaOps.Cli -- db export --format json + + # Trivy DB (delta example) + dotnet run --project src/StellaOps.Cli -- db export --format trivy-db --delta + ``` + + Concelier always produces a deterministic OCI layout. The first run after a clean + bootstrap emits a **full** baseline; subsequent `--delta` runs reuse the previous + baseline’s blobs when only JSON manifests change. If the exporter detects that a + prior delta is still active (i.e., `LastDeltaDigest` is recorded) it automatically + upgrades the next run to a full export and resets the baseline so operators never + chain deltas indefinitely. The CLI exposes `--publish-full/--publish-delta` (for + ORAS pushes) and `--include-full/--include-delta` (for offline bundles) should you + need to override the defaults interactively. + + **Smoke-check delta reuse:** after the first baseline completes, run the export a + second time with `--delta` and verify that the new directory reports `mode=delta` + while reusing the previous layer blob. + + ```bash + export_root=${CONCELIER_EXPORT_ROOT:-exports/trivy} + base=$(ls -1d "$export_root"/* | sort | tail -n2 | head -n1) + delta=$(ls -1d "$export_root"/* | sort | tail -n1) + + jq -r '.mode,.baseExportId' "$delta/metadata.json" + + base_manifest=$(jq -r '.manifests[0].digest' "$base/index.json") + delta_manifest=$(jq -r '.manifests[0].digest' "$delta/index.json") + printf 'baseline manifest: %s\ndelta manifest: %s\n' "$base_manifest" "$delta_manifest" + + layer_digest=$(jq -r '.layers[0].digest' "$base/blobs/sha256/${base_manifest#sha256:}") + cmp "$base/blobs/sha256/${layer_digest#sha256:}" \ + "$delta/blobs/sha256/${layer_digest#sha256:}" + ``` + + `cmp` returning exit code `0` confirms the delta export reuses the baseline’s + `db.tar.gz` layer instead of rebuilding it. + +4. **Manage scanners (optional)** + + ```bash + dotnet run --project src/StellaOps.Cli -- scanner download --channel stable + dotnet run --project src/StellaOps.Cli -- scan run --entry scanners/latest/Scanner.dll --target ./sboms + dotnet run --project src/StellaOps.Cli -- scan upload --file results/scan-001.json + ``` + +Add `--verbose` to any command for structured console logs. All commands honour +`Ctrl+C` cancellation and exit with non-zero status codes when the backend returns +a problem document. + +--- + +## 4 · Verification Checklist + +- Concelier `/health` returns `"status":"healthy"` and Storage bootstrap is marked + complete after startup. +- CLI commands return HTTP 202 with a `Location` header (job tracking URL) when + triggering Concelier jobs. +- Export artefacts are materialised under the configured output directories and + their manifests record digests. +- MongoDB contains the expected `document`, `dto`, `advisory`, and `export_state` + collections after a run. + +--- + +## 5 · Deployment Automation + +- Treat `etc/concelier.yaml.sample` as the canonical template. CI/CD should copy it to + the deployment artifact and replace placeholders (DSN, telemetry endpoints, cron + overrides) with environment-specific secrets. +- Keep secret material (Mongo credentials, OTLP tokens) outside of the repository; + inject them via secret stores or pipeline variables at stamp time. +- When building container images, include `trivy-db` (and `oras` if used) so air-gapped + clusters do not need outbound downloads at runtime. + +--- + +## 5 · Next Steps + +- Enable authority-backed authentication in non-production first. Set + `authority.enabled: true` while keeping `authority.allowAnonymousFallback: true` + to observe logs, then flip it to `false` before 2025-12-31 UTC to enforce tokens. +- Automate the workflow above via CI/CD (compose stack or Kubernetes CronJobs). +- Pair with the Concelier connector teams when enabling additional sources so their + module-specific requirements are pulled in safely. + +--- + +## 6 · Authority Integration + +- Concelier now authenticates callers through StellaOps Authority using OAuth 2.0 + resource server flows. Populate the `authority` block in `concelier.yaml`: + + ```yaml + authority: + enabled: true + allowAnonymousFallback: false # keep true only during the staged rollout window + issuer: "https://authority.example.org" + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + clientId: "concelier-jobs" + clientSecretFile: "../secrets/concelier-jobs.secret" + clientScopes: + - "concelier.jobs.trigger" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + ``` + +- Store the client secret outside of source control. Either provide it via + `authority.clientSecret` (environment variable `CONCELIER_AUTHORITY__CLIENTSECRET`) + or point `authority.clientSecretFile` to a file mounted at runtime. +- Cron jobs running on the same host can keep using the API thanks to the loopback + bypass mask. Add additional CIDR ranges as needed; every bypass is logged. +- Export the same configuration to Kubernetes or systemd by setting environment + variables such as: + + ```bash + export CONCELIER_AUTHORITY__ENABLED=true + export CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=false + export CONCELIER_AUTHORITY__ISSUER="https://authority.example.org" + export CONCELIER_AUTHORITY__CLIENTID="concelier-jobs" + export CONCELIER_AUTHORITY__CLIENTSECRETFILE="/var/run/secrets/concelier/authority-client" + ``` + +- CLI commands already pass `Authorization` headers when credentials are supplied. + Configure the CLI with matching Authority settings (`docs/09_API_CLI_REFERENCE.md`) + so that automation can obtain tokens with the same client credentials. Concelier + logs every job request with the client ID, subject (if present), scopes, and + a `bypass` flag so operators can audit cron traffic. diff --git a/docs/10_PLUGIN_SDK_GUIDE.md b/docs/10_PLUGIN_SDK_GUIDE.md index 0956c046..d5a02314 100755 --- a/docs/10_PLUGIN_SDK_GUIDE.md +++ b/docs/10_PLUGIN_SDK_GUIDE.md @@ -82,28 +82,53 @@ Add this to **`MyPlugin.Schedule.csproj`** so the signed DLL + `.sig` land in th --- -## 5 Dependency‑Injection Entry‑point - -Back‑end auto‑discovers the static method below: - -~~~csharp -namespace StellaOps.DependencyInjection; - -public static class IoCConfigurator -{ - public static IServiceCollection Configure(this IServiceCollection services, - IConfiguration cfg) - { - services.AddSingleton(); // schedule job - services.Configure(cfg.GetSection("Plugins:MyPlugin")); - return services; - } -} -~~~ - ---- - -## 6 Schedule Plug‑ins +## 5 Dependency‑Injection Entry‑point + +Back‑end auto‑discovers restart‑time bindings through two mechanisms: + +1. **Service binding metadata** for simple contracts. +2. **`IDependencyInjectionRoutine`** implementations when you need full control. + +### 5.1 Service binding metadata + +Annotate implementations with `[ServiceBinding]` to declare their lifetime and service contract. +The loader honours scoped lifetimes and will register the service before executing any custom DI routines. + +~~~csharp +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; + +[ServiceBinding(typeof(IJob), ServiceLifetime.Scoped, RegisterAsSelf = true)] +public sealed class MyJob : IJob +{ + // IJob dependencies can now use scoped services (Mongo sessions, etc.) +} +~~~ + +Use `RegisterAsSelf = true` when you also want to resolve the concrete type. +Set `ReplaceExisting = true` to override default descriptors if the host already provides one. + +### 5.2 Dependency injection routines + +For advanced scenarios continue to expose a routine: + +~~~csharp +namespace StellaOps.DependencyInjection; + +public sealed class IoCConfigurator : IDependencyInjectionRoutine +{ + public IServiceCollection Register(IServiceCollection services, IConfiguration cfg) + { + services.AddSingleton(); // schedule job + services.Configure(cfg.GetSection("Plugins:MyPlugin")); + return services; + } +} +~~~ + +--- + +## 6 Schedule Plug‑ins ### 6.1 Minimal Job @@ -191,4 +216,4 @@ On merge, the plug‑in shows up in the UI Marketplace. | NotDetected | .sig missing | cosign sign … | | VersionGateMismatch | Backend 2.1 vs plug‑in 2.0 | Re‑compile / bump attribute | | FileLoadException | Duplicate | StellaOps.Common Ensure PrivateAssets="all" | -| Redis | timeouts Large writes | Batch or use Mongo | \ No newline at end of file +| Redis | timeouts Large writes | Batch or use Mongo | diff --git a/docs/11_AUTHORITY.md b/docs/11_AUTHORITY.md index 3fad4471..58e8ffb4 100644 --- a/docs/11_AUTHORITY.md +++ b/docs/11_AUTHORITY.md @@ -1,162 +1,176 @@ -# StellaOps Authority Service - -> **Status:** Drafted 2025-10-12 (CORE5B.DOC / DOC1.AUTH) – aligns with Authority revocation store, JWKS rotation, and bootstrap endpoints delivered in Sprint 1. - -## 1. Purpose -The **StellaOps Authority** service issues OAuth2/OIDC tokens for every StellaOps module (Feedser, Backend, Agent, Zastava) and exposes the policy controls required in sovereign/offline environments. Authority is built as a minimal ASP.NET host that: - -- brokers password, client-credentials, and device-code flows through pluggable identity providers; -- persists access/refresh/device tokens in MongoDB with deterministic schemas for replay analysis and air-gapped audit copies; -- distributes revocation bundles and JWKS material so downstream services can enforce lockouts without direct database access; -- offers bootstrap APIs for first-run provisioning and key rotation without redeploying binaries. - -Authority is deployed alongside Feedser in air-gapped environments and never requires outbound internet access. All trusted metadata (OpenIddict discovery, JWKS, revocation bundles) is cacheable, signed, and reproducible. - -## 2. Component Architecture -Authority is composed of five cooperating subsystems: - -1. **Minimal API host** – configures OpenIddict endpoints (`/token`, `/authorize`, `/revoke`, `/jwks`) and structured logging/telemetry. Rate limiting hooks (`AuthorityRateLimiter`) wrap every request. -2. **Plugin host** – loads `StellaOps.Authority.Plugin.*.dll` assemblies, applies capability metadata, and exposes password/client provisioning surfaces through dependency injection. -3. **Mongo storage** – persists tokens, revocations, bootstrap invites, and plugin state in deterministic collections indexed for offline sync (`authority_tokens`, `authority_revocations`, etc.). -4. **Cryptography layer** – `StellaOps.Cryptography` abstractions manage password hashing, signing keys, JWKS export, and detached JWS generation. -5. **Offline ops APIs** – internal endpoints under `/internal/*` provide administrative flows (bootstrap users/clients, revocation export) guarded by API keys and deterministic audit events. - -A high-level sequence for password logins: - -``` -Client -> /token (password grant) - -> Rate limiter & audit hooks - -> Plugin credential store (Argon2id verification) - -> Token persistence (Mongo authority_tokens) - -> Response (access/refresh tokens + deterministic claims) -``` - -## 3. Token Lifecycle & Persistence -Authority persists every issued token in MongoDB so operators can audit or revoke without scanning distributed caches. - -- **Collection:** `authority_tokens` -- **Key fields:** - - `tokenId`, `type` (`access_token`, `refresh_token`, `device_code`, `authorization_code`) - - `subjectId`, `clientId`, ordered `scope` array - - `status` (`valid`, `revoked`, `expired`), `createdAt`, optional `expiresAt` - - `revokedAt`, machine-readable `revokedReason`, optional `revokedReasonDescription` - - `revokedMetadata` (string dictionary for plugin-specific context) -- **Persistence flow:** `PersistTokensHandler` stamps missing JWT IDs, normalises scopes, and stores every principal emitted by OpenIddict. -- **Revocation flow:** `AuthorityTokenStore.UpdateStatusAsync` flips status, records the reason metadata, and is invoked by token revocation handlers and plugin provisioning events (e.g., disabling a user). -- **Expiry maintenance:** `AuthorityTokenStore.DeleteExpiredAsync` prunes non-revoked tokens past their `expiresAt` timestamp. Operators should schedule this in maintenance windows if large volumes of tokens are issued. - -### Expectations for resource servers -Resource servers (Feedser WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should: - -- cache `/jwks` and `/revocations/export` responses within configured lifetimes; -- honour `revokedReason` metadata when shaping audit trails; -- treat `status != "valid"` or missing tokens as immediate denial conditions. - -## 4. Revocation Pipeline -Authority centralises revocation in `authority_revocations` with deterministic categories: - -| Category | Meaning | Required fields | -| --- | --- | --- | -| `token` | Specific OAuth token revoked early. | `revocationId` (token id), `tokenType`, optional `clientId`, `subjectId` | -| `subject` | All tokens for a subject disabled. | `revocationId` (= subject id) | -| `client` | OAuth client registration revoked. | `revocationId` (= client id) | -| `key` | Signing/JWE key withdrawn. | `revocationId` (= key id) | - -`RevocationBundleBuilder` flattens Mongo documents into canonical JSON, sorts entries by (`category`, `revocationId`, `revokedAt`), and signs exports using detached JWS (RFC 7797) with cosign-compatible headers. - -**Export surfaces** (deterministic output, suitable for Offline Kit): - -- CLI: `stella auth revoke export --output ./out` writes `revocation-bundle.json`, `.jws`, `.sha256`. -- Verification: `stella auth revoke verify --bundle --signature --key ` validates detached JWS signatures before distribution, selecting the crypto provider advertised in the detached header (see `docs/security/revocation-bundle.md`). -- API: `GET /internal/revocations/export` (requires bootstrap API key) returns the same payload. -- Verification: `stella auth revoke verify` validates schema, digest, and detached JWS using cached JWKS or offline keys, automatically preferring the hinted provider (libsodium builds honour `provider=libsodium`; other builds fall back to the managed provider). - -**Consumer guidance:** - -1. Mirror `revocation-bundle.json*` alongside Feedser exports. Offline agents fetch both over the existing update channel. -2. Use bundle `sequence` and `bundleId` to detect replay or monotonicity regressions. Ignore bundles with older sequence numbers unless `bundleId` changes and `issuedAt` advances. -3. Treat `revokedReason` taxonomy as machine-friendly codes (`compromised`, `rotation`, `policy`, `lifecycle`). Translating to human-readable logs is the consumer’s responsibility. - -## 5. Signing Keys & JWKS Rotation -Authority signs revocation bundles and publishes JWKS entries via the new signing manager: - -- **Configuration (`authority.yaml`):** - ```yaml - signing: - enabled: true - algorithm: ES256 # Defaults to ES256 - keySource: file # Loader identifier (file, vault, etc.) - provider: default # Optional preferred crypto provider - activeKeyId: authority-signing-dev - keyPath: "../certificates/authority-signing-dev.pem" - additionalKeys: - - keyId: authority-signing-dev-2024 - path: "../certificates/authority-signing-dev-2024.pem" - source: "file" - ``` -- **Sources:** The default loader supports PEM files relative to the content root; additional loaders can be registered via `IAuthoritySigningKeySource`. -- **Providers:** Keys are registered against the `ICryptoProviderRegistry`, so alternative implementations (HSM, libsodium) can be plugged in without changing host code. -- **JWKS output:** `GET /jwks` lists every signing key with `status` metadata (`active`, `retired`). Old keys remain until operators remove them from configuration, allowing verification of historical bundles/tokens. - -### Rotation SOP (no downtime) -1. Generate a new P-256 private key (PEM) on an offline workstation and place it where the Authority host can read it (e.g., `../certificates/authority-signing-2025.pem`). -2. Call the authenticated admin API: - ```bash - curl -sS -X POST https://authority.example.com/internal/signing/rotate \ - -H "x-stellaops-bootstrap-key: ${BOOTSTRAP_KEY}" \ - -H "Content-Type: application/json" \ - -d '{ - "keyId": "authority-signing-2025", - "location": "../certificates/authority-signing-2025.pem", - "source": "file" - }' - ``` -3. Verify the response reports the previous key as retired and fetch `/jwks` to confirm the new `kid` appears with `status: "active"`. -4. Persist the old key path in `signing.additionalKeys` (the rotation API updates in-memory options; rewrite the YAML to match so restarts remain consistent). -5. If you prefer automation, trigger the `.gitea/workflows/authority-key-rotation.yml` workflow with the new `keyId`/`keyPath`; it wraps `ops/authority/key-rotation.sh` and reads environment-specific secrets. The older key will be marked `retired` and appended to `signing.additionalKeys`. -6. Re-run `stella auth revoke export` so revocation bundles are signed with the new key. Downstream caches should refresh JWKS within their configured lifetime (`StellaOpsAuthorityOptions.Signing` + client cache tolerance). - -The rotation API leverages the same cryptography abstractions as revocation signing; no restart is required and the previous key is marked `retired` but kept available for verification. - -## 6. Bootstrap & Administrative Endpoints -Administrative APIs live under `/internal/*` and require the bootstrap API key plus rate-limiter compliance. - -| Endpoint | Method | Description | -| --- | --- | --- | -| `/internal/users` | `POST` | Provision initial administrative accounts through the registered password-capable plug-in. Emits structured audit events. | -| `/internal/clients` | `POST` | Provision OAuth clients (client credentials / device code). | -| `/internal/revocations/export` | `GET` | Export revocation bundle + detached JWS + digest. | -| `/internal/signing/rotate` | `POST` | Promote a new signing key (see SOP above). Request body accepts `keyId`, `location`, optional `source`, `algorithm`, `provider`, and metadata. | - -All administrative calls emit `AuthEventRecord` entries enriched with correlation IDs, PII tags, and network metadata for offline SOC ingestion. - -## 7. Configuration Reference - -| Section | Key | Description | Notes | -| --- | --- | --- | --- | -| Root | `issuer` | Absolute HTTPS issuer advertised to clients. | Required. Loopback HTTP allowed only for development. | -| Tokens | `accessTokenLifetime`, `refreshTokenLifetime`, etc. | Lifetimes for each grant (access, refresh, device, authorization code, identity). | Enforced during issuance; persisted on each token document. | -| Storage | `storage.connectionString` | MongoDB connection string. | Required even for tests; offline kits ship snapshots for seeding. | -| Signing | `signing.enabled` | Enable JWKS/revocation signing. | Disable only for development. | -| Signing | `signing.algorithm` | Signing algorithm identifier. | Currently ES256; additional curves can be wired through crypto providers. | -| Signing | `signing.keySource` | Loader identifier (`file`, `vault`, custom). | Determines which `IAuthoritySigningKeySource` resolves keys. | -| Signing | `signing.keyPath` | Relative/absolute path understood by the loader. | Stored as-is; rotation request should keep it in sync with filesystem layout. | -| Signing | `signing.activeKeyId` | Active JWKS / revocation signing key id. | Exposed as `kid` in JWKS and bundles. | -| Signing | `signing.additionalKeys[].keyId` | Retired key identifier retained for verification. | Manager updates this automatically after rotation; keep YAML aligned. | -| Signing | `signing.additionalKeys[].source` | Loader identifier per retired key. | Defaults to `signing.keySource` if omitted. | -| Security | `security.rateLimiting` | Fixed-window limits for `/token`, `/authorize`, `/internal/*`. | See `docs/security/rate-limits.md` for tuning. | -| Bootstrap | `bootstrap.apiKey` | Shared secret required for `/internal/*`. | Only required when `bootstrap.enabled` is true. | - -## 8. Offline & Sovereign Operation -- **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. -- **Structured logging:** Every revocation export, signing rotation, bootstrap action, and token issuance emits structured logs with `traceId`, `client_id`, `subjectId`, and `network.remoteIp` where applicable. Mirror logs to your SIEM to retain audit trails without central connectivity. -- **Determinism:** Sorting rules in token and revocation exports guarantee byte-for-byte identical artefacts given the same datastore state. Hashes and signatures remain stable across machines. - -## 9. Operational Checklist -- [ ] Protect the bootstrap API key and disable bootstrap endpoints (`bootstrap.enabled: false`) once initial setup is complete. -- [ ] Schedule `stella auth revoke export` (or `/internal/revocations/export`) at the same cadence as Feedser exports so bundles remain in lockstep. -- [ ] Rotate signing keys before expiration; keep at least one retired key until all cached bundles/tokens signed with it have expired. -- [ ] Monitor `/health` and `/ready` plus rate-limiter metrics to detect plugin outages early. -- [ ] Ensure downstream services cache JWKS and revocation bundles within tolerances; stale caches risk accepting revoked tokens. - -For plug-in specific requirements, refer to **[Authority Plug-in Developer Guide](dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md)**. For revocation bundle validation workflow, see **[Authority Revocation Bundle](security/revocation-bundle.md)**. +# StellaOps Authority Service + +> **Status:** Drafted 2025-10-12 (CORE5B.DOC / DOC1.AUTH) – aligns with Authority revocation store, JWKS rotation, and bootstrap endpoints delivered in Sprint 1. + +## 1. Purpose +The **StellaOps Authority** service issues OAuth2/OIDC tokens for every StellaOps module (Concelier, Backend, Agent, Zastava) and exposes the policy controls required in sovereign/offline environments. Authority is built as a minimal ASP.NET host that: + +- brokers password, client-credentials, and device-code flows through pluggable identity providers; +- persists access/refresh/device tokens in MongoDB with deterministic schemas for replay analysis and air-gapped audit copies; +- distributes revocation bundles and JWKS material so downstream services can enforce lockouts without direct database access; +- offers bootstrap APIs for first-run provisioning and key rotation without redeploying binaries. + +Authority is deployed alongside Concelier in air-gapped environments and never requires outbound internet access. All trusted metadata (OpenIddict discovery, JWKS, revocation bundles) is cacheable, signed, and reproducible. + +## 2. Component Architecture +Authority is composed of five cooperating subsystems: + +1. **Minimal API host** – configures OpenIddict endpoints (`/token`, `/authorize`, `/revoke`, `/jwks`) and structured logging/telemetry. Rate limiting hooks (`AuthorityRateLimiter`) wrap every request. +2. **Plugin host** – loads `StellaOps.Authority.Plugin.*.dll` assemblies, applies capability metadata, and exposes password/client provisioning surfaces through dependency injection. +3. **Mongo storage** – persists tokens, revocations, bootstrap invites, and plugin state in deterministic collections indexed for offline sync (`authority_tokens`, `authority_revocations`, etc.). +4. **Cryptography layer** – `StellaOps.Cryptography` abstractions manage password hashing, signing keys, JWKS export, and detached JWS generation. +5. **Offline ops APIs** – internal endpoints under `/internal/*` provide administrative flows (bootstrap users/clients, revocation export) guarded by API keys and deterministic audit events. + +A high-level sequence for password logins: + +``` +Client -> /token (password grant) + -> Rate limiter & audit hooks + -> Plugin credential store (Argon2id verification) + -> Token persistence (Mongo authority_tokens) + -> Response (access/refresh tokens + deterministic claims) +``` + +## 3. Token Lifecycle & Persistence +Authority persists every issued token in MongoDB so operators can audit or revoke without scanning distributed caches. + +- **Collection:** `authority_tokens` +- **Key fields:** + - `tokenId`, `type` (`access_token`, `refresh_token`, `device_code`, `authorization_code`) + - `subjectId`, `clientId`, ordered `scope` array + - `status` (`valid`, `revoked`, `expired`), `createdAt`, optional `expiresAt` + - `revokedAt`, machine-readable `revokedReason`, optional `revokedReasonDescription` + - `revokedMetadata` (string dictionary for plugin-specific context) +- **Persistence flow:** `PersistTokensHandler` stamps missing JWT IDs, normalises scopes, and stores every principal emitted by OpenIddict. +- **Revocation flow:** `AuthorityTokenStore.UpdateStatusAsync` flips status, records the reason metadata, and is invoked by token revocation handlers and plugin provisioning events (e.g., disabling a user). +- **Expiry maintenance:** `AuthorityTokenStore.DeleteExpiredAsync` prunes non-revoked tokens past their `expiresAt` timestamp. Operators should schedule this in maintenance windows if large volumes of tokens are issued. + +### Expectations for resource servers +Resource servers (Concelier WebService, Backend, Agent) **must not** assume in-memory caches are authoritative. They should: + +- cache `/jwks` and `/revocations/export` responses within configured lifetimes; +- honour `revokedReason` metadata when shaping audit trails; +- treat `status != "valid"` or missing tokens as immediate denial conditions. + +## 4. Revocation Pipeline +Authority centralises revocation in `authority_revocations` with deterministic categories: + +| Category | Meaning | Required fields | +| --- | --- | --- | +| `token` | Specific OAuth token revoked early. | `revocationId` (token id), `tokenType`, optional `clientId`, `subjectId` | +| `subject` | All tokens for a subject disabled. | `revocationId` (= subject id) | +| `client` | OAuth client registration revoked. | `revocationId` (= client id) | +| `key` | Signing/JWE key withdrawn. | `revocationId` (= key id) | + +`RevocationBundleBuilder` flattens Mongo documents into canonical JSON, sorts entries by (`category`, `revocationId`, `revokedAt`), and signs exports using detached JWS (RFC 7797) with cosign-compatible headers. + +**Export surfaces** (deterministic output, suitable for Offline Kit): + +- CLI: `stella auth revoke export --output ./out` writes `revocation-bundle.json`, `.jws`, `.sha256`. +- Verification: `stella auth revoke verify --bundle --signature --key ` validates detached JWS signatures before distribution, selecting the crypto provider advertised in the detached header (see `docs/security/revocation-bundle.md`). +- API: `GET /internal/revocations/export` (requires bootstrap API key) returns the same payload. +- Verification: `stella auth revoke verify` validates schema, digest, and detached JWS using cached JWKS or offline keys, automatically preferring the hinted provider (libsodium builds honour `provider=libsodium`; other builds fall back to the managed provider). + +**Consumer guidance:** + +1. Mirror `revocation-bundle.json*` alongside Concelier exports. Offline agents fetch both over the existing update channel. +2. Use bundle `sequence` and `bundleId` to detect replay or monotonicity regressions. Ignore bundles with older sequence numbers unless `bundleId` changes and `issuedAt` advances. +3. Treat `revokedReason` taxonomy as machine-friendly codes (`compromised`, `rotation`, `policy`, `lifecycle`). Translating to human-readable logs is the consumer’s responsibility. + +## 5. Signing Keys & JWKS Rotation +Authority signs revocation bundles and publishes JWKS entries via the new signing manager: + +- **Configuration (`authority.yaml`):** + ```yaml + signing: + enabled: true + algorithm: ES256 # Defaults to ES256 + keySource: file # Loader identifier (file, vault, etc.) + provider: default # Optional preferred crypto provider + activeKeyId: authority-signing-dev + keyPath: "../certificates/authority-signing-dev.pem" + additionalKeys: + - keyId: authority-signing-dev-2024 + path: "../certificates/authority-signing-dev-2024.pem" + source: "file" + ``` +- **Sources:** The default loader supports PEM files relative to the content root; additional loaders can be registered via `IAuthoritySigningKeySource`. +- **Providers:** Keys are registered against the `ICryptoProviderRegistry`, so alternative implementations (HSM, libsodium) can be plugged in without changing host code. +- **JWKS output:** `GET /jwks` lists every signing key with `status` metadata (`active`, `retired`). Old keys remain until operators remove them from configuration, allowing verification of historical bundles/tokens. + +### Rotation SOP (no downtime) +1. Generate a new P-256 private key (PEM) on an offline workstation and place it where the Authority host can read it (e.g., `../certificates/authority-signing-2025.pem`). +2. Call the authenticated admin API: + ```bash + curl -sS -X POST https://authority.example.com/internal/signing/rotate \ + -H "x-stellaops-bootstrap-key: ${BOOTSTRAP_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "keyId": "authority-signing-2025", + "location": "../certificates/authority-signing-2025.pem", + "source": "file" + }' + ``` +3. Verify the response reports the previous key as retired and fetch `/jwks` to confirm the new `kid` appears with `status: "active"`. +4. Persist the old key path in `signing.additionalKeys` (the rotation API updates in-memory options; rewrite the YAML to match so restarts remain consistent). +5. If you prefer automation, trigger the `.gitea/workflows/authority-key-rotation.yml` workflow with the new `keyId`/`keyPath`; it wraps `ops/authority/key-rotation.sh` and reads environment-specific secrets. The older key will be marked `retired` and appended to `signing.additionalKeys`. +6. Re-run `stella auth revoke export` so revocation bundles are signed with the new key. Downstream caches should refresh JWKS within their configured lifetime (`StellaOpsAuthorityOptions.Signing` + client cache tolerance). + +The rotation API leverages the same cryptography abstractions as revocation signing; no restart is required and the previous key is marked `retired` but kept available for verification. + +## 6. Bootstrap & Administrative Endpoints +Administrative APIs live under `/internal/*` and require the bootstrap API key plus rate-limiter compliance. + +| Endpoint | Method | Description | +| --- | --- | --- | +| `/internal/users` | `POST` | Provision initial administrative accounts through the registered password-capable plug-in. Emits structured audit events. | +| `/internal/clients` | `POST` | Provision OAuth clients (client credentials / device code). | +| `/internal/revocations/export` | `GET` | Export revocation bundle + detached JWS + digest. | +| `/internal/signing/rotate` | `POST` | Promote a new signing key (see SOP above). Request body accepts `keyId`, `location`, optional `source`, `algorithm`, `provider`, and metadata. | + +All administrative calls emit `AuthEventRecord` entries enriched with correlation IDs, PII tags, and network metadata for offline SOC ingestion. + +## 7. Configuration Reference + +| Section | Key | Description | Notes | +| --- | --- | --- | --- | +| Root | `issuer` | Absolute HTTPS issuer advertised to clients. | Required. Loopback HTTP allowed only for development. | +| Tokens | `accessTokenLifetime`, `refreshTokenLifetime`, etc. | Lifetimes for each grant (access, refresh, device, authorization code, identity). | Enforced during issuance; persisted on each token document. | +| Storage | `storage.connectionString` | MongoDB connection string. | Required even for tests; offline kits ship snapshots for seeding. | +| Signing | `signing.enabled` | Enable JWKS/revocation signing. | Disable only for development. | +| Signing | `signing.algorithm` | Signing algorithm identifier. | Currently ES256; additional curves can be wired through crypto providers. | +| Signing | `signing.keySource` | Loader identifier (`file`, `vault`, custom). | Determines which `IAuthoritySigningKeySource` resolves keys. | +| Signing | `signing.keyPath` | Relative/absolute path understood by the loader. | Stored as-is; rotation request should keep it in sync with filesystem layout. | +| Signing | `signing.activeKeyId` | Active JWKS / revocation signing key id. | Exposed as `kid` in JWKS and bundles. | +| Signing | `signing.additionalKeys[].keyId` | Retired key identifier retained for verification. | Manager updates this automatically after rotation; keep YAML aligned. | +| Signing | `signing.additionalKeys[].source` | Loader identifier per retired key. | Defaults to `signing.keySource` if omitted. | +| Security | `security.rateLimiting` | Fixed-window limits for `/token`, `/authorize`, `/internal/*`. | See `docs/security/rate-limits.md` for tuning. | +| Bootstrap | `bootstrap.apiKey` | Shared secret required for `/internal/*`. | Only required when `bootstrap.enabled` is true. | + +### 7.1 Sender-constrained clients (DPoP & mTLS) + +Authority now understands two flavours of sender-constrained OAuth clients: + +- **DPoP proof-of-possession** – clients sign a `DPoP` header for `/token` requests. Authority validates the JWK thumbprint, HTTP method/URI, and replay window, then stamps the resulting access token with `cnf.jkt` so downstream services can verify the same key is reused. + - Configure under `security.senderConstraints.dpop`. `allowedAlgorithms`, `proofLifetime`, and `replayWindow` are enforced at validation time. + - `security.senderConstraints.dpop.nonce.enabled` enables nonce challenges for high-value audiences (`requiredAudiences`, normalised to case-insensitive strings). When a nonce is required but missing or expired, `/token` replies with `WWW-Authenticate: DPoP error="use_dpop_nonce"` (and, when available, a fresh `DPoP-Nonce` header). Clients must retry with the issued nonce embedded in the proof. + - `security.senderConstraints.dpop.nonce.store` selects `memory` (default) or `redis`. When `redis` is configured, set `security.senderConstraints.dpop.nonce.redisConnectionString` so replicas share nonce issuance and high-value clients avoid replay gaps during failover. + - Declare client `audiences` in bootstrap manifests or plug-in provisioning metadata; Authority now defaults the token `aud` claim and `resource` indicator from this list, which is also used to trigger nonce enforcement for audiences such as `signer` and `attestor`. +- **Mutual TLS clients** – client registrations may declare an mTLS binding (`senderConstraint: mtls`). When enabled via `security.senderConstraints.mtls`, Authority validates the presented client certificate against stored bindings (`certificateBindings[]`), optional chain verification, and timing windows. Successful requests embed `cnf.x5t#S256` into the access token so resource servers can enforce the certificate thumbprint. + - Certificate bindings record the certificate thumbprint, optional SANs, subject/issuer metadata, and activation windows. Operators can enforce subject regexes, SAN type allow-lists (`dns`, `uri`, `ip`), trusted certificate authorities, and rotation grace via `security.senderConstraints.mtls.*`. + +Both modes persist additional metadata in `authority_tokens`: `senderConstraint` records the enforced policy, while `senderKeyThumbprint` stores the DPoP JWK thumbprint or mTLS certificate hash captured at issuance. Downstream services can rely on these fields (and the corresponding `cnf` claim) when auditing offline copies of the token store. + +## 8. Offline & Sovereign Operation +- **No outbound dependencies:** Authority only contacts MongoDB and local plugins. Discovery and JWKS are cached by clients with offline tolerances (`AllowOfflineCacheFallback`, `OfflineCacheTolerance`). Operators should mirror these responses for air-gapped use. +- **Structured logging:** Every revocation export, signing rotation, bootstrap action, and token issuance emits structured logs with `traceId`, `client_id`, `subjectId`, and `network.remoteIp` where applicable. Mirror logs to your SIEM to retain audit trails without central connectivity. +- **Determinism:** Sorting rules in token and revocation exports guarantee byte-for-byte identical artefacts given the same datastore state. Hashes and signatures remain stable across machines. + +## 9. Operational Checklist +- [ ] Protect the bootstrap API key and disable bootstrap endpoints (`bootstrap.enabled: false`) once initial setup is complete. +- [ ] Schedule `stella auth revoke export` (or `/internal/revocations/export`) at the same cadence as Concelier exports so bundles remain in lockstep. +- [ ] Rotate signing keys before expiration; keep at least one retired key until all cached bundles/tokens signed with it have expired. +- [ ] Monitor `/health` and `/ready` plus rate-limiter metrics to detect plugin outages early. +- [ ] Ensure downstream services cache JWKS and revocation bundles within tolerances; stale caches risk accepting revoked tokens. + +For plug-in specific requirements, refer to **[Authority Plug-in Developer Guide](dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md)**. For revocation bundle validation workflow, see **[Authority Revocation Bundle](security/revocation-bundle.md)**. diff --git a/docs/11_DATA_SCHEMAS.md b/docs/11_DATA_SCHEMAS.md index b68ac7ec..c110a897 100755 --- a/docs/11_DATA_SCHEMAS.md +++ b/docs/11_DATA_SCHEMAS.md @@ -86,16 +86,152 @@ Only enabled when `MONGO_URI` is supplied (for long‑term audit). Schema detail for **policy_versions**: -```jsonc -{ - "_id": "6619e90b8c5e1f76", - "yaml": "version: 1.0\nrules:\n - …", - "rego": null, // filled when Rego uploaded - "authorId": "u_1021", - "created": "2025-07-14T08:15:04Z", - "comment": "Imported via API" -} -``` +Samples live under `samples/api/scheduler/` (e.g., `schedule.json`, `run.json`, `impact-set.json`, `audit.json`) and mirror the canonical serializer output shown below. + +```jsonc +{ + "_id": "6619e90b8c5e1f76", + "yaml": "version: 1.0\nrules:\n - …", + "rego": null, // filled when Rego uploaded + "authorId": "u_1021", + "created": "2025-07-14T08:15:04Z", + "comment": "Imported via API" +} +``` + +### 3.1 Scheduler Sprints 16 Artifacts + +**Collections.** `schedules`, `runs`, `impact_snapshots`, `audit` (module‑local). All documents reuse the canonical JSON emitted by `StellaOps.Scheduler.Models` so agents and fixtures remain deterministic. + +#### 3.1.1 Schedule (`schedules`) + +```jsonc +{ + "_id": "sch_20251018a", + "tenantId": "tenant-alpha", + "name": "Nightly Prod", + "enabled": true, + "cronExpression": "0 2 * * *", + "timezone": "UTC", + "mode": "analysis-only", + "selection": { + "scope": "by-namespace", + "namespaces": ["team-a", "team-b"], + "repositories": ["app/service-api"], + "includeTags": ["canary", "prod"], + "labels": [{"key": "env", "values": ["prod", "staging"]}], + "resolvesTags": true + }, + "onlyIf": {"lastReportOlderThanDays": 7, "policyRevision": "policy@42"}, + "notify": {"onNewFindings": true, "minSeverity": "high", "includeKev": true}, + "limits": {"maxJobs": 1000, "ratePerSecond": 25, "parallelism": 4}, + "subscribers": ["notify.ops"], + "createdAt": "2025-10-18T22:00:00Z", + "createdBy": "svc_scheduler", + "updatedAt": "2025-10-18T22:00:00Z", + "updatedBy": "svc_scheduler" +} +``` + +*Constraints*: arrays are alphabetically sorted; `selection.tenantId` is optional but when present must match `tenantId`. Cron expressions are validated for newline/length, timezones are validated via `TimeZoneInfo`. + +#### 3.1.2 Run (`runs`) + +```jsonc +{ + "_id": "run_20251018_0001", + "tenantId": "tenant-alpha", + "scheduleId": "sch_20251018a", + "trigger": "feedser", + "state": "running", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 624, + "completed": 310, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "reason": {"feedserExportId": "exp-20251018-03"}, + "createdAt": "2025-10-18T22:03:14Z", + "startedAt": "2025-10-18T22:03:20Z", + "finishedAt": null, + "error": null, + "deltas": [ + { + "imageDigest": "sha256:a1b2c3", + "newFindings": 3, + "newCriticals": 1, + "newHigh": 1, + "newMedium": 1, + "newLow": 0, + "kevHits": ["CVE-2025-0002"], + "topFindings": [ + { + "purl": "pkg:rpm/openssl@3.0.12-5.el9", + "vulnerabilityId": "CVE-2025-0002", + "severity": "critical", + "link": "https://ui.internal/scans/sha256:a1b2c3" + } + ], + "attestation": {"uuid": "rekor-314", "verified": true}, + "detectedAt": "2025-10-18T22:03:21Z" + } + ] +} +``` + +Counters are clamped to ≥0, timestamps are converted to UTC, and delta arrays are sorted (critical → info severity precedence, then vulnerability id). Missing `deltas` implies "no change" snapshots. + +#### 3.1.3 Impact Snapshot (`impact_snapshots`) + +```jsonc +{ + "selector": { + "scope": "all-images", + "tenantId": "tenant-alpha" + }, + "images": [ + { + "imageDigest": "sha256:f1e2d3", + "registry": "registry.internal", + "repository": "app/api", + "namespaces": ["team-a"], + "tags": ["prod"], + "usedByEntrypoint": true, + "labels": {"env": "prod"} + } + ], + "usageOnly": true, + "generatedAt": "2025-10-18T22:02:58Z", + "total": 412, + "snapshotId": "impact-20251018-1" +} +``` + +Images are deduplicated and sorted by digest. Label keys are normalised to lowercase to avoid case‑sensitive duplicates during reconciliation. `snapshotId` enables run planners to compare subsequent snapshots for drift. + +#### 3.1.4 Audit (`audit`) + +```jsonc +{ + "_id": "audit_169754", + "tenantId": "tenant-alpha", + "category": "scheduler", + "action": "pause", + "occurredAt": "2025-10-18T22:10:00Z", + "actor": {"actorId": "user_admin", "displayName": "Cluster Admin", "kind": "user"}, + "scheduleId": "sch_20251018a", + "correlationId": "corr-123", + "metadata": {"details": "schedule paused", "reason": "maintenance"}, + "message": "Paused via API" +} +``` + +Metadata keys are lowercased, first‑writer wins (duplicates with different casing are ignored), and optional IDs (`scheduleId`, `runId`) are trimmed when empty. Use the canonical serializer when emitting events so audit digests remain reproducible. --- @@ -120,20 +256,66 @@ rules: action: escalate ``` -Validation is performed by `policy:mapping.yaml` JSON‑Schema embedded in backend. +Validation is performed by `policy:mapping.yaml` JSON‑Schema embedded in backend. + +Canonical schema source: `src/StellaOps.Policy/Schemas/policy-schema@1.json` (embedded into `StellaOps.Policy`). +`PolicyValidationCli` (see `src/StellaOps.Policy/PolicyValidationCli.cs`) provides the reusable command handler that the main CLI wires up; in the interim it can be invoked from a short host like: + +```csharp +await new PolicyValidationCli().RunAsync(new PolicyValidationCliOptions +{ + Inputs = new[] { "policies/root.yaml" }, + Strict = true, +}); +``` -### 4.1 Rego Variant (Advanced – TODO) - -*Accepted but stored as‑is in `rego` field.* -Evaluated via internal **OPA** side‑car once feature graduates from TODO list. - ---- - -## 5 SLSA Attestation Schema ⭑ - -Planned for Q1‑2026 (kept here for early plug‑in authors). - -```jsonc +### 4.1 Rego Variant (Advanced – TODO) + +*Accepted but stored as‑is in `rego` field.* +Evaluated via internal **OPA** side‑car once feature graduates from TODO list. + +### 4.2 Policy Scoring Config (JSON) + +*Schema id.* `https://schemas.stella-ops.org/policy/policy-scoring-schema@1.json` +*Source.* `src/StellaOps.Policy/Schemas/policy-scoring-schema@1.json` (embedded in `StellaOps.Policy`), default fixture at `src/StellaOps.Policy/Schemas/policy-scoring-default.json`. + +```jsonc +{ + "version": "1.0", + "severityWeights": {"Critical": 90, "High": 75, "Unknown": 60, "...": 0}, + "quietPenalty": 45, + "warnPenalty": 15, + "ignorePenalty": 35, + "trustOverrides": {"vendor": 1.0, "distro": 0.85}, + "reachabilityBuckets": {"entrypoint": 1.0, "direct": 0.85, "runtime": 0.45, "unknown": 0.5}, + "unknownConfidence": { + "initial": 0.8, + "decayPerDay": 0.05, + "floor": 0.2, + "bands": [ + {"name": "high", "min": 0.65}, + {"name": "medium", "min": 0.35}, + {"name": "low", "min": 0.0} + ] + } +} +``` + +Validation occurs alongside policy binding (`PolicyScoringConfigBinder`), producing deterministic digests via `PolicyScoringConfigDigest`. Bands are ordered descending by `min` so consumers can resolve confidence tiers deterministically. Reachability buckets are case-insensitive keys (`entrypoint`, `direct`, `indirect`, `runtime`, `unreachable`, `unknown`) with numeric multipliers (default ≤1.0). + +**Runtime usage** +- `trustOverrides` are matched against `finding.tags` (`trust:`) first, then `finding.source`/`finding.vendor`; missing keys default to `1.0`. +- `reachabilityBuckets` consume `finding.tags` with prefix `reachability:` (fallback `usage:` or `unknown`). Missing buckets fall back to `unknown` weight when present, otherwise `1.0`. +- Policy verdicts expose scoring inputs (`severityWeight`, `trustWeight`, `reachabilityWeight`, `baseScore`, penalties) plus unknown-state metadata (`unknownConfidence`, `unknownAgeDays`, `confidenceBand`) for auditability. See `samples/policy/policy-preview-unknown.json` for an end-to-end preview payload. +- Unknown confidence derives from `unknown-age-days:` (preferred) or `unknown-since:` + `observed-at:` tags; with no hints the engine keeps `initial` confidence. Values decay by `decayPerDay` down to `floor`, then resolve to the first matching `bands[].name`. + +--- + +## 5 SLSA Attestation Schema ⭑ + +Planned for Q1‑2026 (kept here for early plug‑in authors). + +```jsonc { "id": "prov_0291", "imageDigest": "sha256:e2b9…", @@ -153,8 +335,70 @@ Planned for Q1‑2026 (kept here for early plug‑in authors). {"uri": "git+https://git…", "digest": {"sha1": "f6a1…"}} ], "rekorLogIndex": 99817 // entry in local Rekor mirror -} -``` +} +``` + +--- + +## 6 Notify Foundations (Rule · Channel · Event) + +*Sprint 15 target* – canonically describe the Notify data shapes that UI, workers, and storage consume. JSON Schemas live under `docs/notify/schemas/` and deterministic fixtures under `docs/notify/samples/`. + +| Artifact | Schema | Sample | +|----------|--------|--------| +| **Rule** (catalogued routing logic) | `docs/notify/schemas/notify-rule@1.json` | `docs/notify/samples/notify-rule@1.sample.json` | +| **Channel** (delivery endpoint definition) | `docs/notify/schemas/notify-channel@1.json` | `docs/notify/samples/notify-channel@1.sample.json` | +| **Template** (rendering payload) | `docs/notify/schemas/notify-template@1.json` | `docs/notify/samples/notify-template@1.sample.json` | +| **Event envelope** (Notify ingest surface) | `docs/notify/schemas/notify-event@1.json` | `docs/notify/samples/notify-event@1.sample.json` | + +### 6.1 Rule highlights (`notify-rule@1`) + +* Keys are lower‑cased camelCase. `schemaVersion` (`notify.rule@1`), `ruleId`, `tenantId`, `name`, `match`, `actions`, `createdAt`, and `updatedAt` are mandatory. +* `match.eventKinds`, `match.verdicts`, and other array selectors are pre‑sorted and case‑normalized (e.g. `scanner.report.ready`). +* `actions[].throttle` serialises as ISO 8601 duration (`PT5M`), mirroring worker backoff guardrails. +* `vex` gates let operators exclude accepted/not‑affected justifications; omit the block to inherit default behaviour. +* Use `StellaOps.Notify.Models.NotifySchemaMigration.UpgradeRule(JsonNode)` when deserialising legacy payloads that might lack `schemaVersion` or retain older revisions. +* Soft deletions persist `deletedAt` in Mongo (and disable the rule); repository queries automatically filter them. + +### 6.2 Channel highlights (`notify-channel@1`) + +* `schemaVersion` is pinned to `notify.channel@1` and must accompany persisted documents. +* `type` matches plug‑in identifiers (`slack`, `teams`, `email`, `webhook`, `custom`). +* `config.secretRef` stores an external secret handle (Authority, Vault, K8s). Notify never persists raw credentials. +* Optional `config.limits.timeout` uses ISO 8601 durations identical to rule throttles; concurrency/RPM defaults apply when absent. +* `StellaOps.Notify.Models.NotifySchemaMigration.UpgradeChannel(JsonNode)` backfills the schema version when older documents omit it. +* Channels share the same soft-delete marker (`deletedAt`) so operators can restore prior configuration without purging history. + +### 6.3 Event envelope (`notify-event@1`) + +* Aligns with the platform event contract—`eventId` UUID, RFC 3339 `ts`, tenant isolation enforced. +* Enumerated `kind` covers the initial Notify surface (`scanner.report.ready`, `scheduler.rescan.delta`, `zastava.admission`, etc.). +* `scope.labels`/`scope.attributes` and top-level `attributes` mirror the metadata dictionaries workers surface for templating and audits. +* Notify workers use the same migration helper to wrap event payloads before template rendering, so schema additions remain additive. + +### 6.4 Template highlights (`notify-template@1`) + +* Carries the presentation key (`channelType`, `key`, `locale`) and the raw template body; `schemaVersion` is fixed to `notify.template@1`. +* `renderMode` enumerates supported engines (`markdown`, `html`, `adaptiveCard`, `plainText`, `json`) aligning with `NotifyTemplateRenderMode`. +* `format` signals downstream connector expectations (`slack`, `teams`, `email`, `webhook`, `json`). +* Upgrade legacy definitions with `NotifySchemaMigration.UpgradeTemplate(JsonNode)` to auto-apply the new schema version and ordering. +* Templates also record soft deletes via `deletedAt`; UI/API skip them by default while retaining revision history. + +**Validation loop:** + +```bash +# Validate Notify schemas and samples (matches Docs CI) +for schema in docs/notify/schemas/*.json; do + npx ajv compile -c ajv-formats -s "$schema" +done + +for sample in docs/notify/samples/*.sample.json; do + schema="docs/notify/schemas/$(basename "${sample%.sample.json}").json" + npx ajv validate -c ajv-formats -s "$schema" -d "$sample" +done +``` + +Integration tests can embed the sample fixtures to guarantee deterministic serialisation from the `StellaOps.Notify.Models` DTOs introduced in Sprint 15. --- diff --git a/docs/13_SECURITY_POLICY.md b/docs/13_SECURITY_POLICY.md index 1956a68d..f74c380c 100755 --- a/docs/13_SECURITY_POLICY.md +++ b/docs/13_SECURITY_POLICY.md @@ -81,7 +81,7 @@ cosign verify \ ## 5 · Private‑feed mirrors 🌐 -The **Feedser (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge: +The **Concelier (vulnerability ingest/merge/export service)** provides signed JSON and Trivy DB snapshots that merge: * OSV + GHSA * (optional) NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU regionals diff --git a/docs/14_GLOSSARY_OF_TERMS.md b/docs/14_GLOSSARY_OF_TERMS.md index 300c0ade..d16782c2 100755 --- a/docs/14_GLOSSARY_OF_TERMS.md +++ b/docs/14_GLOSSARY_OF_TERMS.md @@ -20,7 +20,7 @@ open a PR and append it alphabetically.* | **ADR** | *Architecture Decision Record* – lightweight Markdown file that captures one irreversible design decision. | ADR template lives at `/docs/adr/` | | **AIRE** | *AI Risk Evaluator* – optional Plus/Pro plug‑in that suggests mute rules using an ONNX model. | Commercial feature | | **Azure‑Pipelines** | CI/CD service in Microsoft Azure DevOps. | Recipe in Pipeline Library | -| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Feedser (vulnerability ingest/merge/export service) | +| **BDU** | Russian (FSTEC) national vulnerability database: *База данных уязвимостей*. | Merged with NVD by Concelier (vulnerability ingest/merge/export service) | | **BuildKit** | Modern Docker build engine with caching and concurrency. | Needed for layer cache patterns | | **CI** | *Continuous Integration* – automated build/test pipeline. | Stella integrates via CLI | | **Cosign** | Open‑source Sigstore tool that signs & verifies container images **and files**. | Images & OUK tarballs | @@ -36,7 +36,7 @@ open a PR and append it alphabetically.* | **Digest (image)** | SHA‑256 hash uniquely identifying a container image or layer. | Pin digests for reproducible builds | | **Docker‑in‑Docker (DinD)** | Running Docker daemon inside a CI container. | Used in GitHub / GitLab recipes | | **DTO** | *Data Transfer Object* – C# record serialised to JSON. | Schemas in doc 11 | -| **Feedser** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `0 1 * * *` | +| **Concelier** | Vulnerability ingest/merge/export service consolidating OVN, GHSA, NVD 2.0, CNNVD, CNVD, ENISA, JVN and BDU feeds into the canonical MongoDB store and export artifacts. | Cron default `0 1 * * *` | | **FSTEC** | Russian regulator issuing SOBIT certificates. | Pro GA target | | **Gitea** | Self‑hosted Git service – mirrors GitHub repo. | OSS hosting | | **GOST TLS** | TLS cipher‑suites defined by Russian GOST R 34.10‑2012 / 34.11‑2012. | Provided by `OpenSslGost` or CryptoPro | diff --git a/docs/17_SECURITY_HARDENING_GUIDE.md b/docs/17_SECURITY_HARDENING_GUIDE.md index 9b95eaa6..355f4c0d 100755 --- a/docs/17_SECURITY_HARDENING_GUIDE.md +++ b/docs/17_SECURITY_HARDENING_GUIDE.md @@ -145,28 +145,28 @@ cosign verify ghcr.io/stellaops/backend@sha256: \ | Audit events | Redis stream audit; export daily to SIEM | | Alert rules | Feed age  ≥ 48 h, P95 wall‑time > 5 s, Redis used memory > 75 % | -###  7.1 Feedser authorization audits +###  7.1 Concelier authorization audits -- Enable the Authority integration for Feedser (`authority.enabled=true`). Keep +- Enable the Authority integration for Concelier (`authority.enabled=true`). Keep `authority.allowAnonymousFallback` set to `true` only during migration and plan to disable it before **2025-12-31 UTC** so the `/jobs*` surface always demands a bearer token. - Store the Authority client secret using Docker/Kubernetes secrets and point `authority.clientSecretFile` at the mounted path; the value is read at startup and never logged. -- Watch the `Feedser.Authorization.Audit` logger. Each entry contains the HTTP +- Watch the `Concelier.Authorization.Audit` logger. Each entry contains the HTTP status, subject, client ID, scopes, remote IP, and a boolean `bypass` flag showing whether a network bypass CIDR allowed the request. Configure your SIEM to alert when unauthenticated requests (`status=401`) appear with `bypass=true`, or when unexpected scopes invoke job triggers. - Detailed monitoring and response guidance lives in `docs/ops/feedser-authority-audit-runbook.md`. + Detailed monitoring and response guidance lives in `docs/ops/concelier-authority-audit-runbook.md`. ##  8 Update & patch strategy | Layer | Cadence | Method | | -------------------- | -------------------------------------------------------- | ------------------------------ | | Backend & CLI images | Monthly or CVE‑driven docker pull + docker compose up -d | -| Trivy DB | 24 h scheduler via Feedser (vulnerability ingest/merge/export service) | configurable via Feedser scheduler options | +| Trivy DB | 24 h scheduler via Concelier (vulnerability ingest/merge/export service) | configurable via Concelier scheduler options | | Docker Engine | vendor LTS | distro package manager | | Host OS | security repos enabled | unattended‑upgrades | diff --git a/docs/19_TEST_SUITE_OVERVIEW.md b/docs/19_TEST_SUITE_OVERVIEW.md index f204be35..991e8d07 100755 --- a/docs/19_TEST_SUITE_OVERVIEW.md +++ b/docs/19_TEST_SUITE_OVERVIEW.md @@ -16,7 +16,7 @@ contributors who need to extend coverage or diagnose failures. | **1. Unit** | `xUnit` (dotnet test) | `*.Tests.csproj` | per PR / push | | **2. Property‑based** | `FsCheck` | `SbomPropertyTests` | per PR | | **3. Integration (API)** | `Testcontainers` suite | `test/Api.Integration` | per PR + nightly | -| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Feedser.Integration` (vulnerability ingest/merge/export service) | per PR | +| **4. Integration (DB-merge)** | in-memory Mongo + Redis | `Concelier.Integration` (vulnerability ingest/merge/export service) | per PR | | **5. Contract (gRPC)** | `Buf breaking` | `buf.yaml` files | per PR | | **6. Front‑end unit** | `Jest` | `ui/src/**/*.spec.ts` | per PR | | **7. Front‑end E2E** | `Playwright` | `ui/e2e/**` | nightly | @@ -59,17 +59,17 @@ The script spins up MongoDB/Redis via Testcontainers and requires: --- -### Feedser OSV↔GHSA parity fixtures +### Concelier OSV↔GHSA parity fixtures -The Feedser connector suite includes a regression test (`OsvGhsaParityRegressionTests`) +The Concelier connector suite includes a regression test (`OsvGhsaParityRegressionTests`) that checks a curated set of GHSA identifiers against OSV responses. The fixture -snapshots live in `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/` and are kept +snapshots live in `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/` and are kept deterministic so the parity report remains reproducible. To refresh the fixtures when GHSA/OSV payloads change: 1. Ensure outbound HTTPS access to `https://api.osv.dev` and `https://api.github.com`. -2. Run `UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`. +2. Run `UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`. 3. Commit the regenerated `osv-ghsa.*.json` files that the test emits (raw snapshots and canonical advisories). The regen flow logs `[Parity]` messages and normalises `recordedAt` timestamps so the @@ -88,7 +88,7 @@ flowchart LR I1 --> FE[Jest] FE --> E2E[Playwright] E2E --> Lighthouse - Lighthouse --> INTEG2[Feedser] + Lighthouse --> INTEG2[Concelier] INTEG2 --> LOAD[k6] LOAD --> CHAOS[pumba] CHAOS --> RELEASE[Attestation diff] diff --git a/docs/21_INSTALL_GUIDE.md b/docs/21_INSTALL_GUIDE.md index 0a363545..5667bb31 100755 --- a/docs/21_INSTALL_GUIDE.md +++ b/docs/21_INSTALL_GUIDE.md @@ -76,51 +76,57 @@ UI: [https://\<host\>:8443](https://<host>:8443) (self‑signed cert > `stella-ops:latest` with the immutable digest printed by > `docker images --digests`. -### 1.1 · Feedser authority configuration +> **Repo bundles** – Development, staging, and air‑gapped Compose profiles live +> under `deploy/compose/`, already tied to the release manifests in +> `deploy/releases/`. Helm users can pull the same channel overlays from +> `deploy/helm/stellaops/values-*.yaml` and validate everything with +> `deploy/tools/validate-profiles.sh`. -The Feedser container reads configuration from `etc/feedser.yaml` plus -`FEEDSER_` environment variables. To enable the new Authority integration: +### 1.1 · Concelier authority configuration + +The Concelier container reads configuration from `etc/concelier.yaml` plus +`CONCELIER_` environment variables. To enable the new Authority integration: 1. Add the following keys to `.env` (replace values for your environment): ```bash - FEEDSER_AUTHORITY__ENABLED=true - FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true # temporary rollout only - FEEDSER_AUTHORITY__ISSUER="https://authority.internal" - FEEDSER_AUTHORITY__AUDIENCES__0="api://feedser" - FEEDSER_AUTHORITY__REQUIREDSCOPES__0="feedser.jobs.trigger" - FEEDSER_AUTHORITY__CLIENTID="feedser-jobs" - FEEDSER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/feedser_authority_client" - FEEDSER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32" - FEEDSER_AUTHORITY__BYPASSNETWORKS__1="::1/128" - FEEDSER_AUTHORITY__RESILIENCE__ENABLERETRIES=true - FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__0="00:00:01" - FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__1="00:00:02" - FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__2="00:00:05" - FEEDSER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK=true - FEEDSER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE="00:10:00" + CONCELIER_AUTHORITY__ENABLED=true + CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true # temporary rollout only + CONCELIER_AUTHORITY__ISSUER="https://authority.internal" + CONCELIER_AUTHORITY__AUDIENCES__0="api://concelier" + CONCELIER_AUTHORITY__REQUIREDSCOPES__0="concelier.jobs.trigger" + CONCELIER_AUTHORITY__CLIENTID="concelier-jobs" + CONCELIER_AUTHORITY__CLIENTSECRETFILE="/run/secrets/concelier_authority_client" + CONCELIER_AUTHORITY__BYPASSNETWORKS__0="127.0.0.1/32" + CONCELIER_AUTHORITY__BYPASSNETWORKS__1="::1/128" + CONCELIER_AUTHORITY__RESILIENCE__ENABLERETRIES=true + CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__0="00:00:01" + CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__1="00:00:02" + CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__2="00:00:05" + CONCELIER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK=true + CONCELIER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE="00:10:00" ``` Store the client secret outside source control (Docker secrets, mounted file, - or Kubernetes Secret). Feedser loads the secret during post-configuration, so + or Kubernetes Secret). Concelier loads the secret during post-configuration, so the value never needs to appear in the YAML template. Connected sites can keep the retry ladder short (1 s, 2 s, 5 s) so job triggers fail fast when Authority is down. For air‑gapped or intermittently connected deployments, extend `RESILIENCE__OFFLINECACHETOLERANCE` (e.g. `00:30:00`) so cached discovery/JWKS data remains valid while the Offline Kit synchronises upstream changes. -2. Redeploy Feedser: +2. Redeploy Concelier: ```bash - docker compose --env-file .env -f docker-compose.stella-ops.yml up -d feedser + docker compose --env-file .env -f docker-compose.stella-ops.yml up -d concelier ``` -3. Tail the logs: `docker compose logs -f feedser`. Successful `/jobs*` calls now - emit `Feedser.Authorization.Audit` entries with `route`, `status`, `subject`, +3. Tail the logs: `docker compose logs -f concelier`. Successful `/jobs*` calls now + emit `Concelier.Authorization.Audit` entries with `route`, `status`, `subject`, `clientId`, `scopes`, `bypass`, and `remote` fields. 401 denials keep the same shape—watch for `bypass=True`, which indicates a bypass CIDR accepted an anonymous - call. See `docs/ops/feedser-authority-audit-runbook.md` for a full audit/alerting checklist. + call. See `docs/ops/concelier-authority-audit-runbook.md` for a full audit/alerting checklist. -> **Enforcement deadline** – keep `FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true` -> only while validating the rollout. Set it to `false` (and restart Feedser) +> **Enforcement deadline** – keep `CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK=true` +> only while validating the rollout. Set it to `false` (and restart Concelier) > before **2025-12-31 UTC** to require tokens in production. --- diff --git a/docs/24_OFFLINE_KIT.md b/docs/24_OFFLINE_KIT.md index df6427da..f4f47355 100755 --- a/docs/24_OFFLINE_KIT.md +++ b/docs/24_OFFLINE_KIT.md @@ -18,7 +18,7 @@ completely isolated network: | **Attested manifest** | `offline-manifest.json` + detached JWS covering bundle metadata, signed during export. | | **Delta patches** | Daily diff bundles keep size \< 350 MB | -**RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `feedser:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. +**RU BDU note:** ship the official Russian Trusted Root/Sub CA bundle (`certificates/russian_trusted_bundle.pem`) inside the kit so `concelier:httpClients:source.bdu:trustedRootPaths` can resolve it when the service runs in an air‑gapped network. Drop the most recent `vulxml.zip` alongside the kit if operators need a cold-start cache. *Scanner core:* C# 12 on **.NET {{ dotnet }}**. *Imports are idempotent and atomic — no service downtime.* @@ -110,4 +110,4 @@ See the detailed rules in * **Install guide:** `/install/#air-gapped` * **Sovereign mode rationale:** `/sovereign/` * **Security policy:** `/security/#reporting-a-vulnerability` -* **CERT-Bund snapshots:** `python tools/certbund_offline_snapshot.py --help` (see `docs/ops/feedser-certbund-operations.md`) +* **CERT-Bund snapshots:** `python tools/certbund_offline_snapshot.py --help` (see `docs/ops/concelier-certbund-operations.md`) diff --git a/docs/40_ARCHITECTURE_OVERVIEW.md b/docs/40_ARCHITECTURE_OVERVIEW.md index 15b2c05c..01f0469c 100755 --- a/docs/40_ARCHITECTURE_OVERVIEW.md +++ b/docs/40_ARCHITECTURE_OVERVIEW.md @@ -32,7 +32,7 @@ why the system leans *monolith‑plus‑plug‑ins*, and where extension points graph TD A(API Gateway) B1(Scanner Core
.NET latest LTS) - B2(Feedser service\n(vuln ingest/merge/export)) + B2(Concelier service\n(vuln ingest/merge/export)) B3(Policy Engine OPA) C1(Redis 7) C2(MongoDB 7) @@ -53,7 +53,7 @@ graph TD | ---------------------------- | --------------------- | ---------------------------------------------------- | | **API Gateway** | ASP.NET Minimal API | Auth (JWT), quotas, request routing | | **Scanner Core** | C# 12, Polly | Layer diffing, SBOM generation, vuln correlation | -| **Feedser (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports | +| **Concelier (vulnerability ingest/merge/export service)** | C# source-gen workers | Consolidate NVD + regional CVE feeds into the canonical MongoDB store and drive JSON / Trivy DB exports | | **Policy Engine** | OPA (Rego) | admission decisions, custom org rules | | **Redis 7** | Key‑DB compatible | LRU cache, quota counters | | **MongoDB 7** | WiredTiger | SBOM & findings storage | @@ -121,7 +121,7 @@ Hot‑plugging is deferred until after v 1.0 for security review. Although the default deployment is a single container, each sub‑service can be extracted: -* Feedser → standalone cron pod. +* Concelier → standalone cron pod. * Policy Engine → side‑car (OPA) with gRPC contract. * ResultSink → queue worker (RabbitMQ or Azure Service Bus). diff --git a/docs/AGENTS.md b/docs/AGENTS.md index a313840b..a501466b 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -1,20 +1,20 @@ -# Docs & Enablement Guild - -## Mission -Produce and maintain offline-friendly documentation for StellaOps modules, covering architecture, configuration, operator workflows, and developer onboarding. - -## Scope Highlights -- Authority docs (`docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, upcoming `docs/11_AUTHORITY.md`). -- Feedser quickstarts, CLI guides, Offline Kit manuals. -- Release notes and migration playbooks. - -## Operating Principles -- Keep guides deterministic and in sync with shipped configuration samples. -- Prefer tables/checklists for operator steps; flag security-sensitive actions. -- When work involves a specific `StellaOps.` project, consult both `docs/07_HIGH_LEVEL_ARCHITECTURE.md` and the matching dossier `docs/ARCHITECTURE_.md` before drafting or editing content. -- Update `docs/TASKS.md` whenever work items change status (TODO/DOING/REVIEW/DONE/BLOCKED). - -## Coordination -- Authority Core & Plugin teams for auth-related changes. -- Security Guild for threat-model outputs and mitigations. -- DevEx for tooling diagrams and documentation pipeline. +# Docs & Enablement Guild + +## Mission +Produce and maintain offline-friendly documentation for StellaOps modules, covering architecture, configuration, operator workflows, and developer onboarding. + +## Scope Highlights +- Authority docs (`docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, upcoming `docs/11_AUTHORITY.md`). +- Concelier quickstarts, CLI guides, Offline Kit manuals. +- Release notes and migration playbooks. + +## Operating Principles +- Keep guides deterministic and in sync with shipped configuration samples. +- Prefer tables/checklists for operator steps; flag security-sensitive actions. +- When work involves a specific `StellaOps.` project, consult both `docs/07_HIGH_LEVEL_ARCHITECTURE.md` and the matching dossier `docs/ARCHITECTURE_.md` before drafting or editing content. +- Update `docs/TASKS.md` whenever work items change status (TODO/DOING/REVIEW/DONE/BLOCKED). + +## Coordination +- Authority Core & Plugin teams for auth-related changes. +- Security Guild for threat-model outputs and mitigations. +- DevEx for tooling diagrams and documentation pipeline. diff --git a/docs/ARCHITECTURE_ATTESTOR.md b/docs/ARCHITECTURE_ATTESTOR.md index f2057ede..9ea10b2c 100644 --- a/docs/ARCHITECTURE_ATTESTOR.md +++ b/docs/ARCHITECTURE_ATTESTOR.md @@ -1,6 +1,6 @@ # component_architecture_attestor.md — **Stella Ops Attestor** (2025Q4) -> **Scope.** Implementation‑ready architecture for the **Attestor**: the service that **submits** DSSE envelopes to **Rekor v2**, retrieves/validates inclusion proofs, caches results, and exposes verification APIs. It accepts DSSE **only** from the **Signer** over mTLS, enforces chain‑of‑trust to Stella Ops roots, and returns `{uuid, index, proof, logURL}` to calling services (Scanner.WebService for SBOMs; backend for final reports; Vexer exports when configured). +> **Scope.** Implementation‑ready architecture for the **Attestor**: the service that **submits** DSSE envelopes to **Rekor v2**, retrieves/validates inclusion proofs, caches results, and exposes verification APIs. It accepts DSSE **only** from the **Signer** over mTLS, enforces chain‑of‑trust to Stella Ops roots, and returns `{uuid, index, proof, logURL}` to calling services (Scanner.WebService for SBOMs; backend for final reports; Excititor exports when configured). --- @@ -200,7 +200,8 @@ Indexes: * Predicate `predicateType` must be on allowlist (sbom/report/vex-export). * `subject.digest.sha256` values must be present and well‑formed (hex). * **No public submission** path. **Never** accept bundles from untrusted clients. -* **Rate limits**: per mTLS thumbprint/license (from Signer‑forwarded claims) to avoid flooding the log. +* **Client certificate allowlists**: optional `security.mtls.allowedSubjects` / `allowedThumbprints` tighten peer identity checks beyond CA pinning. +* **Rate limits**: token-bucket per caller derived from `quotas.perCaller` (QPS/burst) returns `429` + `Retry-After` when exceeded. * **Redaction**: Attestor never logs secret material; DSSE payloads **should** be public by design (SBOMs/reports). If customers require redaction, enforce policy at Signer (predicate minimization) **before** Attestor. --- @@ -233,6 +234,10 @@ Indexes: * `attestor.dedupe_hits_total` * `attestor.errors_total{type}` +**Correlation**: + +* HTTP callers may supply `X-Correlation-Id`; Attestor will echo the header and push `CorrelationId` into the log scope for cross-service tracing. + **Tracing**: * Spans: `validate`, `rekor.submit`, `rekor.poll`, `persist`, `archive`, `verify`. diff --git a/docs/ARCHITECTURE_AUTHORITY.md b/docs/ARCHITECTURE_AUTHORITY.md index 956f52c6..f4f92f4d 100644 --- a/docs/ARCHITECTURE_AUTHORITY.md +++ b/docs/ARCHITECTURE_AUTHORITY.md @@ -6,7 +6,7 @@ ## 0) Mission & boundaries -**Mission.** Provide **fast, local, verifiable** authentication for Stella Ops microservices and tools by minting **very short‑lived** OAuth2/OIDC tokens that are **sender‑constrained** (DPoP or mTLS‑bound). Support RBAC scopes, multi‑tenant claims, and deterministic validation for APIs (Scanner, Signer, Attestor, Vexer, Feedser, UI, CLI, Zastava). +**Mission.** Provide **fast, local, verifiable** authentication for Stella Ops microservices and tools by minting **very short‑lived** OAuth2/OIDC tokens that are **sender‑constrained** (DPoP or mTLS‑bound). Support RBAC scopes, multi‑tenant claims, and deterministic validation for APIs (Scanner, Signer, Attestor, Excititor, Concelier, UI, CLI, Zastava). **Boundaries.** @@ -43,7 +43,7 @@ ``` iss = https://authority. sub = -aud = +aud = exp = (<= 300 s from iat) iat = nbf = iat - 30 @@ -140,7 +140,7 @@ plan? = // optional hint for UIs; not used for e ### 4.1 Audiences * `signer` — only the **Signer** service should accept tokens with `aud=signer`. -* `attestor`, `scanner`, `feedser`, `vexer`, `ui`, `zastava` similarly. +* `attestor`, `scanner`, `concelier`, `excititor`, `ui`, `zastava` similarly. Services **must** verify `aud` and **sender constraint** (DPoP/mTLS) per their policy. @@ -153,8 +153,8 @@ Services **must** verify `aud` and **sender constraint** (DPoP/mTLS) per their p | `scanner.scan` | Scanner.WebService | Submit scan jobs | | `scanner.export` | Scanner.WebService | Export SBOMs | | `scanner.read` | Scanner.WebService | Read catalog/SBOMs | -| `vex.read` / `vex.admin` | Vexer | Query/operate | -| `feedser.read` / `feedser.export` | Feedser | Query/exports | +| `vex.read` / `vex.admin` | Excititor | Query/operate | +| `concelier.read` / `concelier.export` | Concelier | Query/exports | | `ui.read` / `ui.admin` | UI | View/admin | | `zastava.emit` / `zastava.enforce` | Scanner/Zastava | Runtime events / admission | @@ -229,6 +229,8 @@ GET /admin/metrics # Prometheus exposition (token issue rates, GET /admin/healthz|readyz # health/readiness ``` +Declared client `audiences` flow through to the issued JWT `aud` claim and the token request's `resource` indicators. Authority relies on this metadata to enforce DPoP nonce challenges for `signer`, `attestor`, and other high-value services without requiring clients to repeat the audience parameter on every request. + --- ## 11) Integration hard lines (what resource servers must enforce) @@ -286,6 +288,8 @@ authority: nonce: enable: true ttlSeconds: 600 + store: redis + redisConnectionString: "redis://authority-redis:6379?ssl=false" mtls: enable: true caBundleFile: /etc/ssl/mtls/clients-ca.pem @@ -302,6 +306,18 @@ authority: auth: { type: "mtls" } senderConstraint: "mtls" scopes: [ "signer.sign" ] + - clientId: notify-web-dev + grantTypes: [ "client_credentials" ] + audiences: [ "notify.dev" ] + auth: { type: "client_secret", secretFile: "/secrets/notify-web-dev.secret" } + senderConstraint: "dpop" + scopes: [ "notify.read", "notify.admin" ] + - clientId: notify-web + grantTypes: [ "client_credentials" ] + audiences: [ "notify" ] + auth: { type: "client_secret", secretFile: "/secrets/notify-web.secret" } + senderConstraint: "dpop" + scopes: [ "notify.read", "notify.admin" ] ``` --- diff --git a/docs/ARCHITECTURE_CLI.md b/docs/ARCHITECTURE_CLI.md index e0ac962a..f3a0e68c 100644 --- a/docs/ARCHITECTURE_CLI.md +++ b/docs/ARCHITECTURE_CLI.md @@ -1,6 +1,6 @@ # component_architecture_cli.md — **Stella Ops CLI** (2025Q4) -> **Scope.** Implementation‑ready architecture for **Stella Ops CLI**: command surface, process model, auth (Authority/DPoP), integration with Scanner/Vexer/Feedser/Signer/Attestor, Buildx plug‑in management, offline kit behavior, packaging, observability, security posture, and CI ergonomics. +> **Scope.** Implementation‑ready architecture for **Stella Ops CLI**: command surface, process model, auth (Authority/DPoP), integration with Scanner/Excititor/Concelier/Signer/Attestor, Buildx plug‑in management, offline kit behavior, packaging, observability, security posture, and CI ergonomics. --- @@ -18,7 +18,7 @@ * CLI **never** signs; it only calls **Signer**/**Attestor** via backend APIs when needed (e.g., `report --attest`). * CLI **does not** store long‑lived credentials beyond OS keychain; tokens are **short** (Authority OpToks). -* Heavy work (scanning, merging, policy) is executed **server‑side** (Scanner/Vexer/Feedser). +* Heavy work (scanning, merging, policy) is executed **server‑side** (Scanner/Excititor/Concelier). --- @@ -37,6 +37,8 @@ src/ **Language/runtime**: .NET 10 **Native AOT** for speed/startup; Linux builds use **musl** static when possible. +**Plug-in verbs.** Non-core verbs (Excititor, runtime helpers, future integrations) ship as restart-time plug-ins under `plugins/cli/**` with manifest descriptors. The launcher loads plug-ins on startup; hot reloading is intentionally unsupported. + **OS targets**: linux‑x64/arm64, windows‑x64/arm64, macOS‑x64/arm64. --- @@ -76,8 +78,8 @@ src/ ### 2.4 Policy & data * `policy get/set/apply` — fetch active policy, apply staged policy, compute digest. -* `feedser export` — trigger/export canonical JSON or Trivy DB (admin). -* `vexer export` — trigger/export consensus/raw claims (admin). +* `concelier export` — trigger/export canonical JSON or Trivy DB (admin). +* `excititor export` — trigger/export consensus/raw claims (admin). ### 2.5 Verification @@ -87,12 +89,12 @@ src/ ### 2.6 Runtime (Zastava helper) -* `runtime policy test --images [--ns --labels k=v,...]` — ask backend `/policy/runtime` like the webhook would. +* `runtime policy test --image/-i [--file --ns --label key=value --json]` — ask backend `/policy/runtime` like the webhook would (accepts multiple `--image`, comma/space lists, or stdin pipelines). ### 2.7 Offline kit -* `offline kit pull` — fetch latest **Feedser JSON + Trivy DB + Vexer exports** as a tarball from a mirror. -* `offline kit import ` — upload the kit to on‑prem services (Feedser/Vexer). +* `offline kit pull` — fetch latest **Concelier JSON + Trivy DB + Excititor exports** as a tarball from a mirror. +* `offline kit import ` — upload the kit to on‑prem services (Concelier/Excititor). * `offline kit status` — list current seed versions. ### 2.8 Utilities @@ -122,7 +124,7 @@ src/ * `scanner` for scan/export/report/diff * `signer` (indirect; usually backend calls Signer) * `attestor` for verify - * `feedser`/`vexer` for admin verbs + * `concelier`/`excititor` for admin verbs CLI rejects verbs if required scopes are missing. @@ -167,8 +169,8 @@ cli: backend: scanner: "https://scanner-web.internal" attestor: "https://attestor.internal" - feedser: "https://feedser-web.internal" - vexer: "https://vexer-web.internal" + concelier: "https://concelier-web.internal" + excititor: "https://excititor-web.internal" auth: audienceDefault: "scanner" deviceCode: true @@ -263,7 +265,7 @@ Exit code: 2 ## 13) Admin & advanced flags -* `--authority`, `--scanner`, `--attestor`, `--feedser`, `--vexer` override config URLs. +* `--authority`, `--scanner`, `--attestor`, `--concelier`, `--excititor` override config URLs. * `--no-color`, `--quiet`, `--json`. * `--timeout`, `--retries`, `--retry-backoff-ms`. * `--ca-bundle`, `--insecure` (dev only; prints warning). @@ -386,4 +388,3 @@ script: * macOS: 13–15 (x64, arm64). * Windows: 10/11, Server 2019/2022 (x64, arm64). * Docker engines: Docker Desktop, containerd‑based runners. - diff --git a/docs/ARCHITECTURE_FEEDSER.md b/docs/ARCHITECTURE_CONCELIER.md similarity index 74% rename from docs/ARCHITECTURE_FEEDSER.md rename to docs/ARCHITECTURE_CONCELIER.md index d394d635..5bf71631 100644 --- a/docs/ARCHITECTURE_FEEDSER.md +++ b/docs/ARCHITECTURE_CONCELIER.md @@ -1,433 +1,466 @@ -# component_architecture_feedser.md — **Stella Ops Feedser** (2025Q4) - -> **Scope.** Implementation‑ready architecture for **Feedser**: the vulnerability ingest/normalize/merge/export subsystem that produces deterministic advisory data for the Scanner + Policy + Vexer pipeline. Covers domain model, connectors, merge rules, storage schema, exports, APIs, performance, security, and test matrices. - ---- - -## 0) Mission & boundaries - -**Mission.** Acquire authoritative **vulnerability advisories** (vendor PSIRTs, distros, OSS ecosystems, CERTs), normalize them into a **canonical model**, reconcile aliases and version ranges, and export **deterministic artifacts** (JSON, Trivy DB) for fast backend joins. - -**Boundaries.** - -* Feedser **does not** sign with private keys. When attestation is required, the export artifact is handed to the **Signer**/**Attestor** pipeline (out‑of‑process). -* Feedser **does not** decide PASS/FAIL; it provides data to the **Policy** engine. -* Online operation is **allowlist‑only**; air‑gapped deployments use the **Offline Kit**. - ---- - -## 1) Topology & processes - -**Process shape:** single ASP.NET Core service `StellaOps.Feedser.WebService` hosting: - -* **Scheduler** with distributed locks (Mongo backed). -* **Connectors** (fetch/parse/map). -* **Merger** (canonical record assembly + precedence). -* **Exporters** (JSON, Trivy DB). -* **Minimal REST** for health/status/trigger/export. - -**Scale:** HA by running N replicas; **locks** prevent overlapping jobs per source/exporter. - ---- - -## 2) Canonical domain model - -> Stored in MongoDB (database `feedser`), serialized with a **canonical JSON** writer (stable order, camelCase, normalized timestamps). - -### 2.1 Core entities - -**Advisory** - -``` -advisoryId // internal GUID -advisoryKey // stable string key (e.g., CVE-2025-12345 or vendor ID) -title // short title (best-of from sources) -summary // normalized summary (English; i18n optional) -published // earliest source timestamp -modified // latest source timestamp -severity // normalized {none, low, medium, high, critical} -cvss // {v2?, v3?, v4?} objects (vector, baseScore, severity, source) -exploitKnown // bool (e.g., KEV/active exploitation flags) -references[] // typed links (advisory, kb, patch, vendor, exploit, blog) -sources[] // provenance for traceability (doc digests, URIs) -``` - -**Alias** - -``` -advisoryId -scheme // CVE, GHSA, RHSA, DSA, USN, MSRC, etc. -value // e.g., "CVE-2025-12345" -``` - -**Affected** - -``` -advisoryId -productKey // canonical product identity (see 2.2) -rangeKind // semver | evr | nvra | apk | rpm | deb | generic | exact -introduced? // string (format depends on rangeKind) -fixed? // string (format depends on rangeKind) -lastKnownSafe? // optional explicit safe floor -arch? // arch or platform qualifier if source declares (x86_64, aarch64) -distro? // distro qualifier when applicable (rhel:9, debian:12, alpine:3.19) -ecosystem? // npm|pypi|maven|nuget|golang|… -notes? // normalized notes per source -``` - -**Reference** - -``` -advisoryId -url -kind // advisory | patch | kb | exploit | mitigation | blog | cvrf | csaf -sourceTag // e.g., vendor/redhat, distro/debian, oss/ghsa -``` - -**MergeEvent** - -``` -advisoryKey -beforeHash // canonical JSON hash before merge -afterHash // canonical JSON hash after merge -mergedAt -inputs[] // source doc digests that contributed -``` - -**ExportState** - -``` -exportKind // json | trivydb -baseExportId? // last full baseline -baseDigest? // digest of last full baseline -lastFullDigest? // digest of last full export -lastDeltaDigest? // digest of last delta export -cursor // per-kind incremental cursor -files[] // last manifest snapshot (path → sha256) -``` - -### 2.2 Product identity (`productKey`) - -* **Primary:** `purl` (Package URL). -* **OS packages:** RPM (NEVRA→purl:rpm), DEB (dpkg→purl:deb), APK (apk→purl:alpine), with **EVR/NVRA** preserved. -* **Secondary:** `cpe` retained for compatibility; advisory records may carry both. -* **Image/platform:** `oci:/@` for image‑level advisories (rare). -* **Unmappable:** if a source is non‑deterministic, keep native string under `productKey="native::"` and mark **non‑joinable**. - ---- - -## 3) Source families & precedence - -### 3.1 Families - -* **Vendor PSIRTs**: Microsoft, Oracle, Cisco, Adobe, Apple, VMware, Chromium… -* **Linux distros**: Red Hat, SUSE, Ubuntu, Debian, Alpine… -* **OSS ecosystems**: OSV, GHSA (GitHub Security Advisories), PyPI, npm, Maven, NuGet, Go. -* **CERTs / national CSIRTs**: CISA (KEV, ICS), JVN, ACSC, CCCS, KISA, CERT‑FR/BUND, etc. - -### 3.2 Precedence (when claims conflict) - -1. **Vendor PSIRT** (authoritative for their product). -2. **Distro** (authoritative for packages they ship, including backports). -3. **Ecosystem** (OSV/GHSA) for library semantics. -4. **CERTs/aggregators** for enrichment (KEV/known exploited). - -> Precedence affects **Affected** ranges and **fixed** info; **severity** is normalized to the **maximum** credible severity unless policy overrides. Conflicts are retained with **source provenance**. - ---- - -## 4) Connectors & normalization - -### 4.1 Connector contract - -```csharp -public interface IFeedConnector { - string SourceName { get; } - Task FetchAsync(IServiceProvider sp, CancellationToken ct); // -> document collection - Task ParseAsync(IServiceProvider sp, CancellationToken ct); // -> dto collection (validated) - Task MapAsync(IServiceProvider sp, CancellationToken ct); // -> advisory/alias/affected/reference -} -``` - -* **Fetch**: windowed (cursor), conditional GET (ETag/Last‑Modified), retry/backoff, rate limiting. -* **Parse**: schema validation (JSON Schema, XSD/CSAF), content type checks; write **DTO** with normalized casing. -* **Map**: build canonical records; all outputs carry **provenance** (doc digest, URI, anchors). - -### 4.2 Version range normalization - -* **SemVer** ecosystems (npm, pypi, maven, nuget, golang): normalize to `introduced`/`fixed` semver ranges (use `~`, `^`, `<`, `>=` canonicalized to intervals). -* **RPM EVR**: `epoch:version-release` with `rpmvercmp` semantics; store raw EVR strings and also **computed order keys** for query. -* **DEB**: dpkg version comparison semantics mirrored; store computed keys. -* **APK**: Alpine version semantics; compute order keys. -* **Generic**: if provider uses text, retain raw; do **not** invent ranges. - -### 4.3 Severity & CVSS - -* Normalize **CVSS v2/v3/v4** where available (vector, baseScore, severity). -* If multiple CVSS sources exist, track them all; **effective severity** defaults to **max** by policy (configurable). -* **ExploitKnown** toggled by KEV and equivalent sources; store **evidence** (source, date). - ---- - -## 5) Merge engine - -### 5.1 Keying & identity - -* Identity graph: **CVE** is primary node; vendor/distro IDs resolved via **Alias** edges (from connectors and Feedser’s alias tables). -* `advisoryKey` is the canonical primary key (CVE if present, else vendor/distro key). - -### 5.2 Merge algorithm (deterministic) - -1. **Gather** all rows for `advisoryKey` (across sources). -2. **Select title/summary** by precedence source (vendor>distro>ecosystem>cert). -3. **Union aliases** (dedupe by scheme+value). -4. **Merge `Affected`** with rules: - - * Prefer **vendor** ranges for vendor products; prefer **distro** for **distro‑shipped** packages. - * If both exist for same `productKey`, keep **both**; mark `sourceTag` and `precedence` so **Policy** can decide. - * Never collapse range semantics across different families (e.g., rpm EVR vs semver). -5. **CVSS/severity**: record all CVSS sets; compute **effectiveSeverity** = max (unless policy override). -6. **References**: union with type precedence (advisory > patch > kb > exploit > blog); dedupe by URL; preserve `sourceTag`. -7. Produce **canonical JSON**; compute **afterHash**; store **MergeEvent** with inputs and hashes. - -> The merge is **pure** given inputs. Any change in inputs or precedence matrices changes the **hash** predictably. - ---- - -## 6) Storage schema (MongoDB) - -**Collections & indexes** - -* `source` `{_id, type, baseUrl, enabled, notes}` -* `source_state` `{sourceName(unique), enabled, cursor, lastSuccess, backoffUntil, paceOverrides}` -* `document` `{_id, sourceName, uri, fetchedAt, sha256, contentType, status, metadata, gridFsId?, etag?, lastModified?}` - - * Index: `{sourceName:1, uri:1}` unique, `{fetchedAt:-1}` -* `dto` `{_id, sourceName, documentId, schemaVer, payload, validatedAt}` - - * Index: `{sourceName:1, documentId:1}` -* `advisory` `{_id, advisoryKey, title, summary, published, modified, severity, cvss, exploitKnown, sources[]}` - - * Index: `{advisoryKey:1}` unique, `{modified:-1}`, `{severity:1}`, text index (title, summary) -* `alias` `{advisoryId, scheme, value}` - - * Index: `{scheme:1,value:1}`, `{advisoryId:1}` -* `affected` `{advisoryId, productKey, rangeKind, introduced?, fixed?, arch?, distro?, ecosystem?}` - - * Index: `{productKey:1}`, `{advisoryId:1}`, `{productKey:1, rangeKind:1}` -* `reference` `{advisoryId, url, kind, sourceTag}` - - * Index: `{advisoryId:1}`, `{kind:1}` -* `merge_event` `{advisoryKey, beforeHash, afterHash, mergedAt, inputs[]}` - - * Index: `{advisoryKey:1, mergedAt:-1}` -* `export_state` `{_id(exportKind), baseExportId?, baseDigest?, lastFullDigest?, lastDeltaDigest?, cursor, files[]}` -* `locks` `{_id(jobKey), holder, acquiredAt, heartbeatAt, leaseMs, ttlAt}` (TTL cleans dead locks) -* `jobs` `{_id, type, args, state, startedAt, heartbeatAt, endedAt, error}` - -**GridFS buckets**: `fs.documents` for raw payloads. - ---- - -## 7) Exporters - -### 7.1 Deterministic JSON (vuln‑list style) - -* Folder structure mirroring `////…` with one JSON per advisory; deterministic ordering, stable timestamps, normalized whitespace. -* `manifest.json` lists all files with SHA‑256 and a top‑level **export digest**. - -### 7.2 Trivy DB exporter - -* Builds Bolt DB archives compatible with Trivy; supports **full** and **delta** modes. -* In delta, unchanged blobs are reused from the base; metadata captures: - - ``` - { - "mode": "delta|full", - "baseExportId": "...", - "baseManifestDigest": "sha256:...", - "changed": ["path1", "path2"], - "removed": ["path3"] - } - ``` -* Optional ORAS push (OCI layout) for registries. -* Offline kit bundles include Trivy DB + JSON tree + export manifest. - -### 7.3 Hand‑off to Signer/Attestor (optional) - -* On export completion, if `attest: true` is set in job args, Feedser **posts** the artifact metadata to **Signer**/**Attestor**; Feedser itself **does not** hold signing keys. -* Export record stores returned `{ uuid, index, url }` from **Rekor v2**. - ---- - -## 8) REST APIs - -All under `/api/v1/feedser`. - -**Health & status** - -``` -GET /healthz | /readyz -GET /status → sources, last runs, export cursors -``` - -**Sources & jobs** - -``` -GET /sources → list of configured sources -POST /sources/{name}/trigger → { jobId } -POST /sources/{name}/pause | /resume → toggle -GET /jobs/{id} → job status -``` - -**Exports** - -``` -POST /exports/json { full?:bool, force?:bool, attest?:bool } → { exportId, digest, rekor? } -POST /exports/trivy { full?:bool, force?:bool, publish?:bool, attest?:bool } → { exportId, digest, rekor? } -GET /exports/{id} → export metadata (kind, digest, createdAt, rekor?) -``` - -**Search (operator debugging)** - -``` -GET /advisories/{key} -GET /advisories?scheme=CVE&value=CVE-2025-12345 -GET /affected?productKey=pkg:rpm/openssl&limit=100 -``` - -**AuthN/Z:** Authority tokens (OpTok) with roles: `feedser.read`, `feedser.admin`, `feedser.export`. - ---- - -## 9) Configuration (YAML) - -```yaml -feedser: - mongo: { uri: "mongodb://mongo/feedser" } - s3: - endpoint: "http://minio:9000" - bucket: "stellaops-feedser" - scheduler: - windowSeconds: 30 - maxParallelSources: 4 - sources: - - name: redhat - kind: csaf - baseUrl: https://access.redhat.com/security/data/csaf/v2/ - signature: { type: pgp, keys: [ "…redhat PGP…" ] } - enabled: true - windowDays: 7 - - name: suse - kind: csaf - baseUrl: https://ftp.suse.com/pub/projects/security/csaf/ - signature: { type: pgp, keys: [ "…suse PGP…" ] } - - name: ubuntu - kind: usn-json - baseUrl: https://ubuntu.com/security/notices.json - signature: { type: none } - - name: osv - kind: osv - baseUrl: https://api.osv.dev/v1/ - signature: { type: none } - - name: ghsa - kind: ghsa - baseUrl: https://api.github.com/graphql - auth: { tokenRef: "env:GITHUB_TOKEN" } - exporters: - json: - enabled: true - output: s3://stellaops-feedser/json/ - trivy: - enabled: true - mode: full - output: s3://stellaops-feedser/trivy/ - oras: - enabled: false - repo: ghcr.io/org/feedser - precedence: - vendorWinsOverDistro: true - distroWinsOverOsv: true - severity: - policy: max # or 'vendorPreferred' / 'distroPreferred' -``` - ---- - -## 10) Security & compliance - -* **Outbound allowlist** per connector (domains, protocols); proxy support; TLS pinning where possible. -* **Signature verification** for raw docs (PGP/cosign/x509) with results stored in `document.metadata.sig`. Docs failing verification may still be ingested but flagged; **merge** can down‑weight or ignore them by config. -* **No secrets in logs**; auth material via `env:` or mounted files; HTTP redaction of `Authorization` headers. -* **Multi‑tenant**: per‑tenant DBs or prefixes; per‑tenant S3 prefixes; tenant‑scoped API tokens. -* **Determinism**: canonical JSON writer; export digests stable across runs given same inputs. - ---- - -## 11) Performance targets & scale - -* **Ingest**: ≥ 5k documents/min on 4 cores (CSAF/OpenVEX/JSON). -* **Normalize/map**: ≥ 50k `Affected` rows/min on 4 cores. -* **Merge**: ≤ 10 ms P95 per advisory at steady‑state updates. -* **Export**: 1M advisories JSON in ≤ 90 s (streamed, zstd), Trivy DB in ≤ 60 s on 8 cores. -* **Memory**: hard cap per job; chunked streaming writers; backpressure to avoid GC spikes. - -**Scale pattern**: add Feedser replicas; Mongo scaling via indices and read/write concerns; GridFS only for oversized docs. - ---- - -## 12) Observability - -* **Metrics** - - * `feedser.fetch.docs_total{source}` - * `feedser.fetch.bytes_total{source}` - * `feedser.parse.failures_total{source}` - * `feedser.map.affected_total{source}` - * `feedser.merge.changed_total` - * `feedser.export.bytes{kind}` - * `feedser.export.duration_seconds{kind}` -* **Tracing** around fetch/parse/map/merge/export. -* **Logs**: structured with `source`, `uri`, `docDigest`, `advisoryKey`, `exportId`. - ---- - -## 13) Testing matrix - -* **Connectors:** fixture suites for each provider/format (happy path; malformed; signature fail). -* **Version semantics:** EVR vs dpkg vs semver edge cases (epoch bumps, tilde versions, pre‑releases). -* **Merge:** conflicting sources (vendor vs distro vs OSV); verify precedence & dual retention. -* **Export determinism:** byte‑for‑byte stable outputs across runs; digest equality. -* **Performance:** soak tests with 1M advisories; cap memory; verify backpressure. -* **API:** pagination, filters, RBAC, error envelopes (RFC 7807). -* **Offline kit:** bundle build & import correctness. - ---- - -## 14) Failure modes & recovery - -* **Source outages:** scheduler backs off with exponential delay; `source_state.backoffUntil`; alerts on staleness. -* **Schema drifts:** parse stage marks DTO invalid; job fails with clear diagnostics; connector version flags track supported schema ranges. -* **Partial exports:** exporters write to temp prefix; **manifest commit** is atomic; only then move to final prefix and update `export_state`. -* **Resume:** all stages idempotent; `source_state.cursor` supports window resume. - ---- - -## 15) Operator runbook (quick) - -* **Trigger all sources:** `POST /api/v1/feedser/sources/*/trigger` -* **Force full export JSON:** `POST /api/v1/feedser/exports/json { "full": true, "force": true }` -* **Force Trivy DB delta publish:** `POST /api/v1/feedser/exports/trivy { "full": false, "publish": true }` -* **Inspect advisory:** `GET /api/v1/feedser/advisories?scheme=CVE&value=CVE-2025-12345` -* **Pause noisy source:** `POST /api/v1/feedser/sources/osv/pause` - ---- - -## 16) Rollout plan - -1. **MVP**: Red Hat (CSAF), SUSE (CSAF), Ubuntu (USN JSON), OSV; JSON export. -2. **Add**: GHSA GraphQL, Debian (DSA HTML/JSON), Alpine secdb; Trivy DB export. -3. **Attestation hand‑off**: integrate with **Signer/Attestor** (optional). -4. **Scale & diagnostics**: provider dashboards, staleness alerts, export cache reuse. -5. **Offline kit**: end‑to‑end verified bundles for air‑gap. - +# component_architecture_concelier.md — **Stella Ops Concelier** (2025Q4) + +> **Scope.** Implementation‑ready architecture for **Concelier**: the vulnerability ingest/normalize/merge/export subsystem that produces deterministic advisory data for the Scanner + Policy + Excititor pipeline. Covers domain model, connectors, merge rules, storage schema, exports, APIs, performance, security, and test matrices. + +--- + +## 0) Mission & boundaries + +**Mission.** Acquire authoritative **vulnerability advisories** (vendor PSIRTs, distros, OSS ecosystems, CERTs), normalize them into a **canonical model**, reconcile aliases and version ranges, and export **deterministic artifacts** (JSON, Trivy DB) for fast backend joins. + +**Boundaries.** + +* Concelier **does not** sign with private keys. When attestation is required, the export artifact is handed to the **Signer**/**Attestor** pipeline (out‑of‑process). +* Concelier **does not** decide PASS/FAIL; it provides data to the **Policy** engine. +* Online operation is **allowlist‑only**; air‑gapped deployments use the **Offline Kit**. + +--- + +## 1) Topology & processes + +**Process shape:** single ASP.NET Core service `StellaOps.Concelier.WebService` hosting: + +* **Scheduler** with distributed locks (Mongo backed). +* **Connectors** (fetch/parse/map). +* **Merger** (canonical record assembly + precedence). +* **Exporters** (JSON, Trivy DB). +* **Minimal REST** for health/status/trigger/export. + +**Scale:** HA by running N replicas; **locks** prevent overlapping jobs per source/exporter. + +--- + +## 2) Canonical domain model + +> Stored in MongoDB (database `concelier`), serialized with a **canonical JSON** writer (stable order, camelCase, normalized timestamps). + +### 2.1 Core entities + +**Advisory** + +``` +advisoryId // internal GUID +advisoryKey // stable string key (e.g., CVE-2025-12345 or vendor ID) +title // short title (best-of from sources) +summary // normalized summary (English; i18n optional) +published // earliest source timestamp +modified // latest source timestamp +severity // normalized {none, low, medium, high, critical} +cvss // {v2?, v3?, v4?} objects (vector, baseScore, severity, source) +exploitKnown // bool (e.g., KEV/active exploitation flags) +references[] // typed links (advisory, kb, patch, vendor, exploit, blog) +sources[] // provenance for traceability (doc digests, URIs) +``` + +**Alias** + +``` +advisoryId +scheme // CVE, GHSA, RHSA, DSA, USN, MSRC, etc. +value // e.g., "CVE-2025-12345" +``` + +**Affected** + +``` +advisoryId +productKey // canonical product identity (see 2.2) +rangeKind // semver | evr | nvra | apk | rpm | deb | generic | exact +introduced? // string (format depends on rangeKind) +fixed? // string (format depends on rangeKind) +lastKnownSafe? // optional explicit safe floor +arch? // arch or platform qualifier if source declares (x86_64, aarch64) +distro? // distro qualifier when applicable (rhel:9, debian:12, alpine:3.19) +ecosystem? // npm|pypi|maven|nuget|golang|… +notes? // normalized notes per source +``` + +**Reference** + +``` +advisoryId +url +kind // advisory | patch | kb | exploit | mitigation | blog | cvrf | csaf +sourceTag // e.g., vendor/redhat, distro/debian, oss/ghsa +``` + +**MergeEvent** + +``` +advisoryKey +beforeHash // canonical JSON hash before merge +afterHash // canonical JSON hash after merge +mergedAt +inputs[] // source doc digests that contributed +``` + +**AdvisoryStatement (event log)** + +``` +statementId // GUID (immutable) +vulnerabilityKey // canonical advisory key (e.g., CVE-2025-12345) +advisoryKey // merge snapshot advisory key (may reference variant) +statementHash // canonical hash of advisory payload +asOf // timestamp of snapshot (UTC) +recordedAt // persistence timestamp (UTC) +inputDocuments[] // document IDs contributing to the snapshot +payload // canonical advisory document (BSON / canonical JSON) +``` + +**AdvisoryConflict** + +``` +conflictId // GUID +vulnerabilityKey // canonical advisory key +conflictHash // deterministic hash of conflict payload +asOf // timestamp aligned with originating statement set +recordedAt // persistence timestamp +statementIds[] // related advisoryStatement identifiers +details // structured conflict explanation / merge reasoning +``` + +- `AdvisoryEventLog` (Concelier.Core) provides the public API for appending immutable statements/conflicts and querying replay history. Inputs are normalized by trimming and lower-casing `vulnerabilityKey`, serializing advisories with `CanonicalJsonSerializer`, and computing SHA-256 hashes (`statementHash`, `conflictHash`) over the canonical JSON payloads. Consumers can replay by key with an optional `asOf` filter to obtain deterministic snapshots ordered by `asOf` then `recordedAt`. +- Concelier.WebService exposes the immutable log via `GET /concelier/advisories/{vulnerabilityKey}/replay[?asOf=UTC_ISO8601]`, returning the latest statements (with hex-encoded hashes) and any conflict explanations for downstream exporters and APIs. + +**ExportState** + +``` +exportKind // json | trivydb +baseExportId? // last full baseline +baseDigest? // digest of last full baseline +lastFullDigest? // digest of last full export +lastDeltaDigest? // digest of last delta export +cursor // per-kind incremental cursor +files[] // last manifest snapshot (path → sha256) +``` + +### 2.2 Product identity (`productKey`) + +* **Primary:** `purl` (Package URL). +* **OS packages:** RPM (NEVRA→purl:rpm), DEB (dpkg→purl:deb), APK (apk→purl:alpine), with **EVR/NVRA** preserved. +* **Secondary:** `cpe` retained for compatibility; advisory records may carry both. +* **Image/platform:** `oci:/@` for image‑level advisories (rare). +* **Unmappable:** if a source is non‑deterministic, keep native string under `productKey="native::"` and mark **non‑joinable**. + +--- + +## 3) Source families & precedence + +### 3.1 Families + +* **Vendor PSIRTs**: Microsoft, Oracle, Cisco, Adobe, Apple, VMware, Chromium… +* **Linux distros**: Red Hat, SUSE, Ubuntu, Debian, Alpine… +* **OSS ecosystems**: OSV, GHSA (GitHub Security Advisories), PyPI, npm, Maven, NuGet, Go. +* **CERTs / national CSIRTs**: CISA (KEV, ICS), JVN, ACSC, CCCS, KISA, CERT‑FR/BUND, etc. + +### 3.2 Precedence (when claims conflict) + +1. **Vendor PSIRT** (authoritative for their product). +2. **Distro** (authoritative for packages they ship, including backports). +3. **Ecosystem** (OSV/GHSA) for library semantics. +4. **CERTs/aggregators** for enrichment (KEV/known exploited). + +> Precedence affects **Affected** ranges and **fixed** info; **severity** is normalized to the **maximum** credible severity unless policy overrides. Conflicts are retained with **source provenance**. + +--- + +## 4) Connectors & normalization + +### 4.1 Connector contract + +```csharp +public interface IFeedConnector { + string SourceName { get; } + Task FetchAsync(IServiceProvider sp, CancellationToken ct); // -> document collection + Task ParseAsync(IServiceProvider sp, CancellationToken ct); // -> dto collection (validated) + Task MapAsync(IServiceProvider sp, CancellationToken ct); // -> advisory/alias/affected/reference +} +``` + +* **Fetch**: windowed (cursor), conditional GET (ETag/Last‑Modified), retry/backoff, rate limiting. +* **Parse**: schema validation (JSON Schema, XSD/CSAF), content type checks; write **DTO** with normalized casing. +* **Map**: build canonical records; all outputs carry **provenance** (doc digest, URI, anchors). + +### 4.2 Version range normalization + +* **SemVer** ecosystems (npm, pypi, maven, nuget, golang): normalize to `introduced`/`fixed` semver ranges (use `~`, `^`, `<`, `>=` canonicalized to intervals). +* **RPM EVR**: `epoch:version-release` with `rpmvercmp` semantics; store raw EVR strings and also **computed order keys** for query. +* **DEB**: dpkg version comparison semantics mirrored; store computed keys. +* **APK**: Alpine version semantics; compute order keys. +* **Generic**: if provider uses text, retain raw; do **not** invent ranges. + +### 4.3 Severity & CVSS + +* Normalize **CVSS v2/v3/v4** where available (vector, baseScore, severity). +* If multiple CVSS sources exist, track them all; **effective severity** defaults to **max** by policy (configurable). +* **ExploitKnown** toggled by KEV and equivalent sources; store **evidence** (source, date). + +--- + +## 5) Merge engine + +### 5.1 Keying & identity + +* Identity graph: **CVE** is primary node; vendor/distro IDs resolved via **Alias** edges (from connectors and Concelier’s alias tables). +* `advisoryKey` is the canonical primary key (CVE if present, else vendor/distro key). + +### 5.2 Merge algorithm (deterministic) + +1. **Gather** all rows for `advisoryKey` (across sources). +2. **Select title/summary** by precedence source (vendor>distro>ecosystem>cert). +3. **Union aliases** (dedupe by scheme+value). +4. **Merge `Affected`** with rules: + + * Prefer **vendor** ranges for vendor products; prefer **distro** for **distro‑shipped** packages. + * If both exist for same `productKey`, keep **both**; mark `sourceTag` and `precedence` so **Policy** can decide. + * Never collapse range semantics across different families (e.g., rpm EVR vs semver). +5. **CVSS/severity**: record all CVSS sets; compute **effectiveSeverity** = max (unless policy override). +6. **References**: union with type precedence (advisory > patch > kb > exploit > blog); dedupe by URL; preserve `sourceTag`. +7. Produce **canonical JSON**; compute **afterHash**; store **MergeEvent** with inputs and hashes. + +> The merge is **pure** given inputs. Any change in inputs or precedence matrices changes the **hash** predictably. + +--- + +## 6) Storage schema (MongoDB) + +**Collections & indexes** + +* `source` `{_id, type, baseUrl, enabled, notes}` +* `source_state` `{sourceName(unique), enabled, cursor, lastSuccess, backoffUntil, paceOverrides}` +* `document` `{_id, sourceName, uri, fetchedAt, sha256, contentType, status, metadata, gridFsId?, etag?, lastModified?}` + + * Index: `{sourceName:1, uri:1}` unique, `{fetchedAt:-1}` +* `dto` `{_id, sourceName, documentId, schemaVer, payload, validatedAt}` + + * Index: `{sourceName:1, documentId:1}` +* `advisory` `{_id, advisoryKey, title, summary, published, modified, severity, cvss, exploitKnown, sources[]}` + + * Index: `{advisoryKey:1}` unique, `{modified:-1}`, `{severity:1}`, text index (title, summary) +* `alias` `{advisoryId, scheme, value}` + + * Index: `{scheme:1,value:1}`, `{advisoryId:1}` +* `affected` `{advisoryId, productKey, rangeKind, introduced?, fixed?, arch?, distro?, ecosystem?}` + + * Index: `{productKey:1}`, `{advisoryId:1}`, `{productKey:1, rangeKind:1}` +* `reference` `{advisoryId, url, kind, sourceTag}` + + * Index: `{advisoryId:1}`, `{kind:1}` +* `merge_event` `{advisoryKey, beforeHash, afterHash, mergedAt, inputs[]}` + + * Index: `{advisoryKey:1, mergedAt:-1}` +* `export_state` `{_id(exportKind), baseExportId?, baseDigest?, lastFullDigest?, lastDeltaDigest?, cursor, files[]}` +* `locks` `{_id(jobKey), holder, acquiredAt, heartbeatAt, leaseMs, ttlAt}` (TTL cleans dead locks) +* `jobs` `{_id, type, args, state, startedAt, heartbeatAt, endedAt, error}` + +**GridFS buckets**: `fs.documents` for raw payloads. + +--- + +## 7) Exporters + +### 7.1 Deterministic JSON (vuln‑list style) + +* Folder structure mirroring `////…` with one JSON per advisory; deterministic ordering, stable timestamps, normalized whitespace. +* `manifest.json` lists all files with SHA‑256 and a top‑level **export digest**. + +### 7.2 Trivy DB exporter + +* Builds Bolt DB archives compatible with Trivy; supports **full** and **delta** modes. +* In delta, unchanged blobs are reused from the base; metadata captures: + + ``` + { + "mode": "delta|full", + "baseExportId": "...", + "baseManifestDigest": "sha256:...", + "changed": ["path1", "path2"], + "removed": ["path3"] + } + ``` +* Optional ORAS push (OCI layout) for registries. +* Offline kit bundles include Trivy DB + JSON tree + export manifest. +* Mirror-ready bundles: when `concelier.trivy.mirror` defines domains, the exporter emits `mirror/index.json` plus per-domain `manifest.json`, `metadata.json`, and `db.tar.gz` files with SHA-256 digests so Concelier mirrors can expose domain-scoped download endpoints. + +### 7.3 Hand‑off to Signer/Attestor (optional) + +* On export completion, if `attest: true` is set in job args, Concelier **posts** the artifact metadata to **Signer**/**Attestor**; Concelier itself **does not** hold signing keys. +* Export record stores returned `{ uuid, index, url }` from **Rekor v2**. + +--- + +## 8) REST APIs + +All under `/api/v1/concelier`. + +**Health & status** + +``` +GET /healthz | /readyz +GET /status → sources, last runs, export cursors +``` + +**Sources & jobs** + +``` +GET /sources → list of configured sources +POST /sources/{name}/trigger → { jobId } +POST /sources/{name}/pause | /resume → toggle +GET /jobs/{id} → job status +``` + +**Exports** + +``` +POST /exports/json { full?:bool, force?:bool, attest?:bool } → { exportId, digest, rekor? } +POST /exports/trivy { full?:bool, force?:bool, publish?:bool, attest?:bool } → { exportId, digest, rekor? } +GET /exports/{id} → export metadata (kind, digest, createdAt, rekor?) +GET /concelier/exports/index.json → mirror index describing available domains/bundles +GET /concelier/exports/mirror/{domain}/manifest.json +GET /concelier/exports/mirror/{domain}/bundle.json +GET /concelier/exports/mirror/{domain}/bundle.json.jws +``` + +**Search (operator debugging)** + +``` +GET /advisories/{key} +GET /advisories?scheme=CVE&value=CVE-2025-12345 +GET /affected?productKey=pkg:rpm/openssl&limit=100 +``` + +**AuthN/Z:** Authority tokens (OpTok) with roles: `concelier.read`, `concelier.admin`, `concelier.export`. + +--- + +## 9) Configuration (YAML) + +```yaml +concelier: + mongo: { uri: "mongodb://mongo/concelier" } + s3: + endpoint: "http://minio:9000" + bucket: "stellaops-concelier" + scheduler: + windowSeconds: 30 + maxParallelSources: 4 + sources: + - name: redhat + kind: csaf + baseUrl: https://access.redhat.com/security/data/csaf/v2/ + signature: { type: pgp, keys: [ "…redhat PGP…" ] } + enabled: true + windowDays: 7 + - name: suse + kind: csaf + baseUrl: https://ftp.suse.com/pub/projects/security/csaf/ + signature: { type: pgp, keys: [ "…suse PGP…" ] } + - name: ubuntu + kind: usn-json + baseUrl: https://ubuntu.com/security/notices.json + signature: { type: none } + - name: osv + kind: osv + baseUrl: https://api.osv.dev/v1/ + signature: { type: none } + - name: ghsa + kind: ghsa + baseUrl: https://api.github.com/graphql + auth: { tokenRef: "env:GITHUB_TOKEN" } + exporters: + json: + enabled: true + output: s3://stellaops-concelier/json/ + trivy: + enabled: true + mode: full + output: s3://stellaops-concelier/trivy/ + oras: + enabled: false + repo: ghcr.io/org/concelier + precedence: + vendorWinsOverDistro: true + distroWinsOverOsv: true + severity: + policy: max # or 'vendorPreferred' / 'distroPreferred' +``` + +--- + +## 10) Security & compliance + +* **Outbound allowlist** per connector (domains, protocols); proxy support; TLS pinning where possible. +* **Signature verification** for raw docs (PGP/cosign/x509) with results stored in `document.metadata.sig`. Docs failing verification may still be ingested but flagged; **merge** can down‑weight or ignore them by config. +* **No secrets in logs**; auth material via `env:` or mounted files; HTTP redaction of `Authorization` headers. +* **Multi‑tenant**: per‑tenant DBs or prefixes; per‑tenant S3 prefixes; tenant‑scoped API tokens. +* **Determinism**: canonical JSON writer; export digests stable across runs given same inputs. + +--- + +## 11) Performance targets & scale + +* **Ingest**: ≥ 5k documents/min on 4 cores (CSAF/OpenVEX/JSON). +* **Normalize/map**: ≥ 50k `Affected` rows/min on 4 cores. +* **Merge**: ≤ 10 ms P95 per advisory at steady‑state updates. +* **Export**: 1M advisories JSON in ≤ 90 s (streamed, zstd), Trivy DB in ≤ 60 s on 8 cores. +* **Memory**: hard cap per job; chunked streaming writers; backpressure to avoid GC spikes. + +**Scale pattern**: add Concelier replicas; Mongo scaling via indices and read/write concerns; GridFS only for oversized docs. + +--- + +## 12) Observability + +* **Metrics** + + * `concelier.fetch.docs_total{source}` + * `concelier.fetch.bytes_total{source}` + * `concelier.parse.failures_total{source}` + * `concelier.map.affected_total{source}` + * `concelier.merge.changed_total` + * `concelier.export.bytes{kind}` + * `concelier.export.duration_seconds{kind}` +* **Tracing** around fetch/parse/map/merge/export. +* **Logs**: structured with `source`, `uri`, `docDigest`, `advisoryKey`, `exportId`. + +--- + +## 13) Testing matrix + +* **Connectors:** fixture suites for each provider/format (happy path; malformed; signature fail). +* **Version semantics:** EVR vs dpkg vs semver edge cases (epoch bumps, tilde versions, pre‑releases). +* **Merge:** conflicting sources (vendor vs distro vs OSV); verify precedence & dual retention. +* **Export determinism:** byte‑for‑byte stable outputs across runs; digest equality. +* **Performance:** soak tests with 1M advisories; cap memory; verify backpressure. +* **API:** pagination, filters, RBAC, error envelopes (RFC 7807). +* **Offline kit:** bundle build & import correctness. + +--- + +## 14) Failure modes & recovery + +* **Source outages:** scheduler backs off with exponential delay; `source_state.backoffUntil`; alerts on staleness. +* **Schema drifts:** parse stage marks DTO invalid; job fails with clear diagnostics; connector version flags track supported schema ranges. +* **Partial exports:** exporters write to temp prefix; **manifest commit** is atomic; only then move to final prefix and update `export_state`. +* **Resume:** all stages idempotent; `source_state.cursor` supports window resume. + +--- + +## 15) Operator runbook (quick) + +* **Trigger all sources:** `POST /api/v1/concelier/sources/*/trigger` +* **Force full export JSON:** `POST /api/v1/concelier/exports/json { "full": true, "force": true }` +* **Force Trivy DB delta publish:** `POST /api/v1/concelier/exports/trivy { "full": false, "publish": true }` +* **Inspect advisory:** `GET /api/v1/concelier/advisories?scheme=CVE&value=CVE-2025-12345` +* **Pause noisy source:** `POST /api/v1/concelier/sources/osv/pause` + +--- + +## 16) Rollout plan + +1. **MVP**: Red Hat (CSAF), SUSE (CSAF), Ubuntu (USN JSON), OSV; JSON export. +2. **Add**: GHSA GraphQL, Debian (DSA HTML/JSON), Alpine secdb; Trivy DB export. +3. **Attestation hand‑off**: integrate with **Signer/Attestor** (optional). +4. **Scale & diagnostics**: provider dashboards, staleness alerts, export cache reuse. +5. **Offline kit**: end‑to‑end verified bundles for air‑gap. + diff --git a/docs/ARCHITECTURE_DEVOPS.md b/docs/ARCHITECTURE_DEVOPS.md index 901a6f6c..9654b9cf 100644 --- a/docs/ARCHITECTURE_DEVOPS.md +++ b/docs/ARCHITECTURE_DEVOPS.md @@ -42,7 +42,7 @@ Semantic core + calendar tag: A release is a **bundle** of image digests + charts + manifests. All services in a bundle are **wire‑compatible**. Mixed minor versions are allowed within a bounded skew: * **Web UI ↔ backend**: `±1 minor`. -* **Scanner ↔ Policy/Vexer/Feedser**: `±1 minor`. +* **Scanner ↔ Policy/Excititor/Concelier**: `±1 minor`. * **Authority/Signer/Attestor triangle**: **must** be same minor (crypto and DPoP/mTLS binding rules). At startup, services **self‑advertise** their semver & channel; the UI surfaces **mismatch warnings**. @@ -75,7 +75,7 @@ At startup, services **self‑advertise** their semver & channel; the UI surface * **Static**: linters, codegen checks, protobuf API freeze (backward‑compat tests). * **Unit/integration**: per‑component, plus **end‑to‑end** flows (scan→vex→policy→sign→attest). * **Perf SLOs**: hot paths (SBOM compose, diff, export) measured against budgets. -* **Security**: dependency audit vs Feedser export; container hardening tests; minimal caps. +* **Security**: dependency audit vs Concelier export; container hardening tests; minimal caps. * **Canary cohort**: internal staging + selected customers; one week on **edge** before **stable** tag. --- @@ -86,11 +86,12 @@ At startup, services **self‑advertise** their semver & channel; the UI surface * **Primary**: `registry.stella-ops.org` (OCI v2, supports Referrers API). * **Mirrors**: GHCR (read‑only), regional mirrors for latency. + * Operational runbook: see `docs/ops/concelier-mirror-operations.md` for deployment profiles, CDN guidance, and sync automation. * **Pull by digest only** in Kubernetes/Compose manifests. **Gating policy**: -* **Core images** (Authority, Scanner, Feedser, Vexer, Attestor, UI): public **read**. +* **Core images** (Authority, Scanner, Concelier, Excititor, Attestor, UI): public **read**. * **Enterprise add‑ons** (if any) and **pre‑release**: private repos via OAuth2 token service. > Monetization lever is **signing** (PoE gate), not image pulls, so the core remains simple to consume. @@ -115,7 +116,7 @@ At startup, services **self‑advertise** their semver & channel; the UI surface /attest/ DSSE bundles + Rekor proofs /charts/ Helm charts + values templates /compose/ docker-compose.yml + .env template - /plugins/ Feedser/Vexer connectors (restart-time) + /plugins/ Concelier/Excititor connectors (restart-time) /policy/ example policies /manifest/ release.yaml (see §6.1) ``` @@ -169,8 +170,8 @@ helm install stella stellaops/platform \ --set authority.issuer=https://authority.stella.local \ --set scanner.minio.endpoint=http://minio.stella.local:9000 \ --set scanner.mongo.uri=mongodb://mongo/scanner \ - --set feedser.mongo.uri=mongodb://mongo/feedser \ - --set vexer.mongo.uri=mongodb://mongo/vexer + --set concelier.mongo.uri=mongodb://mongo/concelier \ + --set excititor.mongo.uri=mongodb://mongo/excititor ``` * Post‑install job registers **Authority clients** (Scanner, Signer, Attestor, UI) and prints **bootstrap** URLs and client credentials (sealed secrets). @@ -185,7 +186,7 @@ helm install stella stellaops/platform \ 1. Authority (stateless, dual‑key rotation ready) 2. Signer/Attestor (same minor) 3. Scanner WebService & Workers - 4. Feedser, then Vexer (schema migrations are expand/contract) + 4. Concelier, then Excititor (schema migrations are expand/contract) 5. UI last * **DB migrations** are **expand/contract**: @@ -234,6 +235,11 @@ release: The manifest is **cosign‑signed**; UI/CLI can verify a bundle without talking to registries. +> Deployment guardrails – The repository keeps channel-aligned Compose bundles +> in `deploy/compose/` and Helm overlays in `deploy/helm/stellaops/`. Both sets +> pull their digests from `deploy/releases/` and are validated by +> `deploy/tools/validate-profiles.sh` to guarantee lint/dry-run cleanliness. + ### 6.2 Image labels (release metadata) Each image sets OCI labels: @@ -263,10 +269,10 @@ s3://stellaops/ images//usage.cdx.pb diffs/_/diff.json.zst attest/.dsse.json - feedser/ + concelier/ json//... trivy//... - vexer/ + excititor/ exports//... attestor/ dsse/.json @@ -289,14 +295,20 @@ s3://stellaops/ ### 7.4 Mongo retention * **Scanner**: `runtime.events` use TTL (e.g., 30–90 days); **catalog** permanent. -* **Feedser/Vexer**: raw docs keep **last N windows**; canonical stores permanent. +* **Concelier/Excititor**: raw docs keep **last N windows**; canonical stores permanent. * **Attestor**: `entries` permanent; `dedupe` TTL 24–48h. +### 7.5 Mongo server baseline + +* **Minimum supported server:** MongoDB **4.2+**. Driver 3.5.0 removes compatibility shims for 4.0; upstream has already announced 4.0 support will be dropped in upcoming C# driver releases. citeturn1open1 +* **Deploy images:** Compose/Helm defaults stay on `mongo:7.x`. For air-gapped installs, refresh Offline Kit bundles so the packaged `mongod` matches ≥4.2. +* **Upgrade guard:** During rollout, verify replica sets reach FCV `4.2` or above before swapping binaries; automation should hard-stop if FCV is <4.2. + --- ## 8) Observability & SLOs (operations) -* **Uptime SLO**: 99.9% for Signer/Authority/Attestor; 99.5% for Scanner WebService; Vexer/Feedser 99.0%. +* **Uptime SLO**: 99.9% for Signer/Authority/Attestor; 99.5% for Scanner WebService; Excititor/Concelier 99.0%. * **Error budgets**: tracked per month; dashboards show burn rates. * **Golden signals**: @@ -324,7 +336,8 @@ Prometheus + OTLP; Grafana dashboards ship in the charts. * **Vulnerability response**: - * Feedser red‑flag advisories trigger accelerated **stable** patch rollout; UI/CLI “security patch available” notice. + * Concelier red-flag advisories trigger accelerated **stable** patch rollout; UI/CLI “security patch available” notice. + * 2025-10: Pinned `MongoDB.Driver` **3.5.0** and `SharpCompress` **0.41.0** across services (DEVOPS-SEC-10-301) to eliminate NU1902/NU1903 warnings surfaced during scanner cache/worker test runs; future dependency bumps follow the same central override pattern. * **Backups/DR**: @@ -408,10 +421,10 @@ services: scanner-worker: image: registry.stella-ops.org/stellaops/scanner-worker@sha256:... deploy: { replicas: 4 } - feedser: - image: registry.stella-ops.org/stellaops/feedser@sha256:... - vexer: - image: registry.stella-ops.org/stellaops/vexer@sha256:... + concelier: + image: registry.stella-ops.org/stellaops/concelier@sha256:... + excititor: + image: registry.stella-ops.org/stellaops/excititor@sha256:... web-ui: image: registry.stella-ops.org/stellaops/web-ui@sha256:... mongo: @@ -446,7 +459,7 @@ services: * `signer.requests_total{result="success"}/minute` > 0 (when scans occur). * `attestor.submit_latency_seconds{quantile=0.95}` < 0.3. * `scanner.scan_latency_seconds{quantile=0.95}` < target per image size. -* `feedser.export.duration_seconds` stable; `vexer.consensus.conflicts_total` not exploding after policy changes. +* `concelier.export.duration_seconds` stable; `excititor.consensus.conflicts_total` not exploding after policy changes. * MinIO `s3_requests_errors_total` near zero; Mongo `opcounters` hit expected baseline. ### Appendix B — Upgrade safety checklist diff --git a/docs/ARCHITECTURE_EXCITITOR.md b/docs/ARCHITECTURE_EXCITITOR.md new file mode 100644 index 00000000..34dbab49 --- /dev/null +++ b/docs/ARCHITECTURE_EXCITITOR.md @@ -0,0 +1,500 @@ +# component_architecture_excititor.md — **Stella Ops Excititor** (2025Q4) + +> **Scope.** This document specifies the **Excititor** service: its purpose, trust model, data structures, APIs, plug‑in contracts, storage schema, normalization/consensus algorithms, performance budgets, testing matrix, and how it integrates with Scanner, Policy, Concelier, and the attestation chain. It is implementation‑ready. + +--- + +## 0) Mission & role in the platform + +**Mission.** Convert heterogeneous **VEX** statements (OpenVEX, CSAF VEX, CycloneDX VEX; vendor/distro/platform sources) into **canonical, queryable claims**; compute **deterministic consensus** per *(vuln, product)*; preserve **conflicts with provenance**; publish **stable, attestable exports** that the backend uses to suppress non‑exploitable findings, prioritize remaining risk, and explain decisions. + +**Boundaries.** + +* Excititor **does not** decide PASS/FAIL. It supplies **evidence** (statuses + justifications + provenance weights). +* Excititor preserves **conflicting claims** unchanged; consensus encodes how we would pick, but the raw set is always exportable. +* VEX consumption is **backend‑only**: Scanner never applies VEX. The backend’s **Policy Engine** asks Excititor for status evidence and then decides what to show. + +--- + +## 1) Inputs, outputs & canonical domain + +### 1.1 Accepted input formats (ingest) + +* **OpenVEX** JSON documents (attested or raw). +* **CSAF VEX** 2.x (vendor PSIRTs and distros commonly publish CSAF). +* **CycloneDX VEX** 1.4+ (standalone VEX or embedded VEX blocks). +* **OCI‑attached attestations** (VEX statements shipped as OCI referrers) — optional connectors. + +All connectors register **source metadata**: provider identity, trust tier, signature expectations (PGP/cosign/PKI), fetch windows, rate limits, and time anchors. + +### 1.2 Canonical model (normalized) + +Every incoming statement becomes a set of **VexClaim** records: + +``` +VexClaim +- providerId // 'redhat', 'suse', 'ubuntu', 'github', 'vendorX' +- vulnId // 'CVE-2025-12345', 'GHSA-xxxx', canonicalized +- productKey // canonical product identity (see §2.2) +- status // affected | not_affected | fixed | under_investigation +- justification? // for 'not_affected'/'affected' where provided +- introducedVersion? // semantics per provider (range or exact) +- fixedVersion? // where provided (range or exact) +- lastObserved // timestamp from source or fetch time +- provenance // doc digest, signature status, fetch URI, line/offset anchors +- evidence[] // raw source snippets for explainability +- supersedes? // optional cross-doc chain (docDigest → docDigest) +``` + +### 1.3 Exports (consumption) + +* **VexConsensus** per `(vulnId, productKey)` with: + + * `rollupStatus` (after policy weights/justification gates), + * `sources[]` (winning + losing claims with weights & reasons), + * `policyRevisionId` (identifier of the Excititor policy used), + * `consensusDigest` (stable SHA‑256 over canonical JSON). +* **Raw claims** export for auditing (unchanged, with provenance). +* **Provider snapshots** (per source, last N days) for operator debugging. +* **Index** optimized for backend joins: `(productKey, vulnId) → (status, confidence, sourceSet)`. + +All exports are **deterministic**, and (optionally) **attested** via DSSE and logged to Rekor v2. + +--- + +## 2) Identity model — products & joins + +### 2.1 Vuln identity + +* Accepts **CVE**, **GHSA**, vendor IDs (MSRC, RHSA…), distro IDs (DSA/USN/RHSA…) — normalized to `vulnId` with alias sets. +* **Alias graph** maintained (from Concelier) to map vendor/distro IDs → CVE (primary) and to **GHSA** where applicable. + +### 2.2 Product identity (`productKey`) + +* **Primary:** `purl` (Package URL). +* **Secondary links:** `cpe`, **OS package NVRA/EVR**, NuGet/Maven/Golang identity, and **OS package name** when purl unavailable. +* **Fallback:** `oci:/@` for image‑level VEX. +* **Special cases:** kernel modules, firmware, platforms → provider‑specific mapping helpers (connector captures provider’s product taxonomy → canonical `productKey`). + +> Excititor does not invent identities. If a provider cannot be mapped to purl/CPE/NVRA deterministically, we keep the native **product string** and mark the claim as **non‑joinable**; the backend will ignore it unless a policy explicitly whitelists that provider mapping. + +--- + +## 3) Storage schema (MongoDB) + +Database: `excititor` + +### 3.1 Collections + +**`vex.providers`** + +``` +_id: providerId +name, homepage, contact +trustTier: enum {vendor, distro, platform, hub, attestation} +signaturePolicy: { type: pgp|cosign|x509|none, keys[], certs[], cosignKeylessRoots[] } +fetch: { baseUrl, kind: http|oci|file, rateLimit, etagSupport, windowDays } +enabled: bool +createdAt, modifiedAt +``` + +**`vex.raw`** (immutable raw documents) + +``` +_id: sha256(doc bytes) +providerId +uri +ingestedAt +contentType +sig: { verified: bool, method: pgp|cosign|x509|none, keyId|certSubject, bundle? } +payload: GridFS pointer (if large) +disposition: kept|replaced|superseded +correlation: { replaces?: sha256, replacedBy?: sha256 } +``` + +**`vex.statements`** (immutable normalized rows; append-only event log) + +``` +_id: ObjectId +providerId +vulnId +productKey +status +justification? +introducedVersion? +fixedVersion? +lastObserved +docDigest +provenance { uri, line?, pointer?, signatureState } +evidence[] { key, value, locator } +signals? { + severity? { scheme, score?, label?, vector? } + kev?: bool + epss?: double +} +insertedAt +indices: + - {vulnId:1, productKey:1} + - {providerId:1, insertedAt:-1} + - {docDigest:1} + - {status:1} + - text index (optional) on evidence.value for debugging +``` + +**`vex.consensus`** (rollups) + +``` +_id: sha256(canonical(vulnId, productKey, policyRevision)) +vulnId +productKey +rollupStatus +sources[]: [ + { providerId, status, justification?, weight, lastObserved, accepted:bool, reason } +] +policyRevisionId +evaluatedAt +signals? { + severity? { scheme, score?, label?, vector? } + kev?: bool + epss?: double +} +consensusDigest // same as _id +indices: + - {vulnId:1, productKey:1} + - {policyRevisionId:1, evaluatedAt:-1} +``` + +**`vex.exports`** (manifest of emitted artifacts) + +``` +_id +querySignature +format: raw|consensus|index +artifactSha256 +rekor { uuid, index, url }? +createdAt +policyRevisionId +cacheable: bool +``` + +**`vex.cache`** + +``` +querySignature -> exportId (for fast reuse) +ttl, hits +``` + +**`vex.migrations`** + +* ordered migrations applied at bootstrap to ensure indexes. +* `20251019-consensus-signals-statements` introduces the statements log indexes and the `policyRevisionId + evaluatedAt` lookup for consensus — rerun consensus writers once to hydrate newly persisted signals. + +### 3.2 Indexing strategy + +* Hot path queries use exact `(vulnId, productKey)` and time‑bounded windows; compound indexes cover both. +* Providers list view by `lastObserved` for monitoring staleness. +* `vex.consensus` keyed by `(vulnId, productKey, policyRevision)` for deterministic reuse. + +--- + +## 4) Ingestion pipeline + +### 4.1 Connector contract + +```csharp +public interface IVexConnector +{ + string ProviderId { get; } + Task FetchAsync(VexConnectorContext ctx, CancellationToken ct); // raw docs + Task NormalizeAsync(VexConnectorContext ctx, CancellationToken ct); // raw -> VexClaim[] +} +``` + +* **Fetch** must implement: window scheduling, conditional GET (ETag/If‑Modified‑Since), rate limiting, retry/backoff. +* **Normalize** parses the format, validates schema, maps product identities deterministically, emits `VexClaim` records with **provenance**. + +### 4.2 Signature verification (per provider) + +* **cosign (keyless or keyful)** for OCI referrers or HTTP‑served JSON with Sigstore bundles. +* **PGP** (provider keyrings) for distro/vendor feeds that sign docs. +* **x509** (mutual TLS / provider‑pinned certs) where applicable. +* Signature state is stored on **vex.raw.sig** and copied into **provenance.signatureState** on claims. + +> Claims from sources failing signature policy are marked `"signatureState.verified=false"` and **policy** can down‑weight or ignore them. + +### 4.3 Time discipline + +* For each doc, prefer **provider’s document timestamp**; if absent, use fetch time. +* Claims carry `lastObserved` which drives **tie‑breaking** within equal weight tiers. + +--- + +## 5) Normalization: product & status semantics + +### 5.1 Product mapping + +* **purl** first; **cpe** second; OS package NVRA/EVR mapping helpers (distro connectors) produce purls via canonical tables (e.g., rpm→purl:rpm, deb→purl:deb). +* Where a provider publishes **platform‑level** VEX (e.g., “RHEL 9 not affected”), connectors expand to known product inventory rules (e.g., map to sets of packages/components shipped in the platform). Expansion tables are versioned and kept per provider; every expansion emits **evidence** indicating the rule applied. +* If expansion would be speculative, the claim remains **platform‑scoped** with `productKey="platform:redhat:rhel:9"` and is flagged **non‑joinable**; backend can decide to use platform VEX only when Scanner proves the platform runtime. + +### 5.2 Status + justification mapping + +* Canonical **status**: `affected | not_affected | fixed | under_investigation`. +* **Justifications** normalized to a controlled vocabulary (CISA‑aligned), e.g.: + + * `component_not_present` + * `vulnerable_code_not_in_execute_path` + * `vulnerable_configuration_unused` + * `inline_mitigation_applied` + * `fix_available` (with `fixedVersion`) + * `under_investigation` +* Providers with free‑text justifications are mapped by deterministic tables; raw text preserved as `evidence`. + +--- + +## 6) Consensus algorithm + +**Goal:** produce a **stable**, explainable `rollupStatus` per `(vulnId, productKey)` given possibly conflicting claims. + +### 6.1 Inputs + +* Set **S** of `VexClaim` for the key. +* **Excititor policy snapshot**: + + * **weights** per provider tier and per provider overrides. + * **justification gates** (e.g., require justification for `not_affected` to be acceptable). + * **minEvidence** rules (e.g., `not_affected` must come from ≥1 vendor or 2 distros). + * **signature requirements** (e.g., require verified signature for ‘fixed’ to be considered). + +### 6.2 Steps + +1. **Filter invalid** claims by signature policy & justification gates → set `S'`. +2. **Score** each claim: + `score = weight(provider) * freshnessFactor(lastObserved)` where freshnessFactor ∈ [0.8, 1.0] for staleness decay (configurable; small effect). +3. **Aggregate** scores per status: `W(status) = Σ score(claims with that status)`. +4. **Pick** `rollupStatus = argmax_status W(status)`. +5. **Tie‑breakers** (in order): + + * Higher **max single** provider score wins (vendor > distro > platform > hub). + * More **recent** lastObserved wins. + * Deterministic lexicographic order of status (`fixed` > `not_affected` > `under_investigation` > `affected`) as final tiebreaker. +6. **Explain**: mark accepted sources (`accepted=true; reason="weight"`/`"freshness"`), mark rejected sources with explicit `reason` (`"insufficient_justification"`, `"signature_unverified"`, `"lower_weight"`). + +> The algorithm is **pure** given S and policy snapshot; result is reproducible and hashed into `consensusDigest`. + +--- + +## 7) Query & export APIs + +All endpoints are versioned under `/api/v1/vex`. + +### 7.1 Query (online) + +``` +POST /claims/search + body: { vulnIds?: string[], productKeys?: string[], providers?: string[], since?: timestamp, limit?: int, pageToken?: string } + → { claims[], nextPageToken? } + +POST /consensus/search + body: { vulnIds?: string[], productKeys?: string[], policyRevisionId?: string, since?: timestamp, limit?: int, pageToken?: string } + → { entries[], nextPageToken? } + +POST /resolve + body: { purls: string[], vulnIds: string[], policyRevisionId?: string } + → { results: [ { vulnId, productKey, rollupStatus, sources[] } ] } +``` + +### 7.2 Exports (cacheable snapshots) + +``` +POST /exports + body: { signature: { vulnFilter?, productFilter?, providers?, since? }, format: raw|consensus|index, policyRevisionId?: string, force?: bool } + → { exportId, artifactSha256, rekor? } + +GET /exports/{exportId} → bytes (application/json or binary index) +GET /exports/{exportId}/meta → { signature, policyRevisionId, createdAt, artifactSha256, rekor? } +``` + +### 7.3 Provider operations + +``` +GET /providers → provider list & signature policy +POST /providers/{id}/refresh → trigger fetch/normalize window +GET /providers/{id}/status → last fetch, doc counts, signature stats +``` + +**Auth:** service‑to‑service via Authority tokens; operator operations via UI/CLI with RBAC. + +--- + +## 8) Attestation integration + +* Exports can be **DSSE‑signed** via **Signer** and logged to **Rekor v2** via **Attestor** (optional but recommended for regulated pipelines). +* `vex.exports.rekor` stores `{uuid, index, url}` when present. +* **Predicate type**: `https://stella-ops.org/attestations/vex-export/1` with fields: + + * `querySignature`, `policyRevisionId`, `artifactSha256`, `createdAt`. + +--- + +## 9) Configuration (YAML) + +```yaml +excititor: + mongo: { uri: "mongodb://mongo/excititor" } + s3: + endpoint: http://minio:9000 + bucket: stellaops + policy: + weights: + vendor: 1.0 + distro: 0.9 + platform: 0.7 + hub: 0.5 + attestation: 0.6 + ceiling: 1.25 + scoring: + alpha: 0.25 + beta: 0.5 + providerOverrides: + redhat: 1.0 + suse: 0.95 + requireJustificationForNotAffected: true + signatureRequiredForFixed: true + minEvidence: + not_affected: + vendorOrTwoDistros: true + connectors: + - providerId: redhat + kind: csaf + baseUrl: https://access.redhat.com/security/data/csaf/v2/ + signaturePolicy: { type: pgp, keys: [ "…redhat-pgp-key…" ] } + windowDays: 7 + - providerId: suse + kind: csaf + baseUrl: https://ftp.suse.com/pub/projects/security/csaf/ + signaturePolicy: { type: pgp, keys: [ "…suse-pgp-key…" ] } + - providerId: ubuntu + kind: openvex + baseUrl: https://…/vex/ + signaturePolicy: { type: none } + - providerId: vendorX + kind: cyclonedx-vex + ociRef: ghcr.io/vendorx/vex@sha256:… + signaturePolicy: { type: cosign, cosignKeylessRoots: [ "sigstore-root" ] } +``` + +### 9.1 WebService endpoints + +With storage configured, the WebService exposes the following ingress and diagnostic APIs: + +* `GET /excititor/status` – returns the active storage configuration and registered artifact stores. +* `GET /excititor/health` – simple liveness probe. +* `POST /excititor/statements` – accepts normalized VEX statements and persists them via `IVexClaimStore`; use this for migrations/backfills. +* `GET /excititor/statements/{vulnId}/{productKey}?since=` – returns the immutable statement log for a vulnerability/product pair. + +Run the ingestion endpoint once after applying migration `20251019-consensus-signals-statements` to repopulate historical statements with the new severity/KEV/EPSS signal fields. + +* `weights.ceiling` raises the deterministic clamp applied to provider tiers/overrides (range 1.0‒5.0). Values outside the range are clamped with warnings so operators can spot typos. +* `scoring.alpha` / `scoring.beta` configure KEV/EPSS boosts for the Phase 1 → Phase 2 scoring pipeline. Defaults (0.25, 0.5) preserve prior behaviour; negative or excessively large values fall back with diagnostics. + +--- + +## 10) Security model + +* **Input signature verification** enforced per provider policy (PGP, cosign, x509). +* **Connector allowlists**: outbound fetch constrained to configured domains. +* **Tenant isolation**: per‑tenant DB prefixes or separate DBs; per‑tenant S3 prefixes; per‑tenant policies. +* **AuthN/Z**: Authority‑issued OpToks; RBAC roles (`vex.read`, `vex.admin`, `vex.export`). +* **No secrets in logs**; deterministic logging contexts include providerId, docDigest, claim keys. + +--- + +## 11) Performance & scale + +* **Targets:** + + * Normalize 10k VEX claims/minute/core. + * Consensus compute ≤ 50 ms for 1k unique `(vuln, product)` pairs in hot cache. + * Export (consensus) 1M rows in ≤ 60 s on 8 cores with streaming writer. + +* **Scaling:** + + * WebService handles control APIs; **Worker** background services (same image) execute fetch/normalize in parallel with rate‑limits; Mongo writes batched; upserts by natural keys. + * Exports stream straight to S3 (MinIO) with rolling buffers. + +* **Caching:** + + * `vex.cache` maps query signatures → export; TTL to avoid stampedes; optimistic reuse unless `force`. + +--- + +## 12) Observability + +* **Metrics:** + + * `vex.ingest.docs_total{provider}` + * `vex.normalize.claims_total{provider}` + * `vex.signature.failures_total{provider,method}` + * `vex.consensus.conflicts_total{vulnId}` + * `vex.exports.bytes{format}` / `vex.exports.latency_seconds` +* **Tracing:** spans for fetch, verify, parse, map, consensus, export. +* **Dashboards:** provider staleness, top conflicting vulns/components, signature posture, export cache hit‑rate. + +--- + +## 13) Testing matrix + +* **Connectors:** golden raw docs → deterministic claims (fixtures per provider/format). +* **Signature policies:** valid/invalid PGP/cosign/x509 samples; ensure rejects are recorded but not accepted. +* **Normalization edge cases:** platform‑only claims, free‑text justifications, non‑purl products. +* **Consensus:** conflict scenarios across tiers; check tie‑breakers; justification gates. +* **Performance:** 1M‑row export timing; memory ceilings; stream correctness. +* **Determinism:** same inputs + policy → identical `consensusDigest` and export bytes. +* **API contract tests:** pagination, filters, RBAC, rate limits. + +--- + +## 14) Integration points + +* **Backend Policy Engine** (in Scanner.WebService): calls `POST /resolve` with batched `(purl, vulnId)` pairs to fetch `rollupStatus + sources`. +* **Concelier**: provides alias graph (CVE↔vendor IDs) and may supply VEX‑adjacent metadata (e.g., KEV flag) for policy escalation. +* **UI**: VEX explorer screens use `/claims/search` and `/consensus/search`; show conflicts & provenance. +* **CLI**: `stellaops vex export --consensus --since 7d --out vex.json` for audits. + +--- + +## 15) Failure modes & fallback + +* **Provider unreachable:** stale thresholds trigger warnings; policy can down‑weight stale providers automatically (freshness factor). +* **Signature outage:** continue to ingest but mark `signatureState.verified=false`; consensus will likely exclude or down‑weight per policy. +* **Schema drift:** unknown fields are preserved as `evidence`; normalization rejects only on **invalid identity** or **status**. + +--- + +## 16) Rollout plan (incremental) + +1. **MVP**: OpenVEX + CSAF connectors for 3 major providers (e.g., Red Hat/SUSE/Ubuntu), normalization + consensus + `/resolve`. +2. **Signature policies**: PGP for distros; cosign for OCI. +3. **Exports + optional attestation**. +4. **CycloneDX VEX** connectors; platform claim expansion tables; UI explorer. +5. **Scale hardening**: export indexes; conflict analytics. + +--- + +## 17) Operational runbooks + +* **Statement backfill** — see `docs/dev/EXCITITOR_STATEMENT_BACKFILL.md` for the CLI workflow, required permissions, observability guidance, and rollback steps. + +--- + +## 18) Appendix — canonical JSON (stable ordering) + +All exports and consensus entries are serialized via `VexCanonicalJsonSerializer`: + +* UTF‑8 without BOM; +* keys sorted (ASCII); +* arrays sorted by `(providerId, vulnId, productKey, lastObserved)` unless semantic order mandated; +* timestamps in `YYYY‑MM‑DDThh:mm:ssZ`; +* no insignificant whitespace. + diff --git a/docs/ARCHITECTURE_EXCITITOR_MIRRORS.md b/docs/ARCHITECTURE_EXCITITOR_MIRRORS.md new file mode 100644 index 00000000..37fb948f --- /dev/null +++ b/docs/ARCHITECTURE_EXCITITOR_MIRRORS.md @@ -0,0 +1,138 @@ +# architecture_excititor_mirrors.md — Excititor Mirror Distribution + +> **Status:** Draft (Sprint 7). Complements `docs/ARCHITECTURE_EXCITITOR.md` by describing the mirror export surface exposed by `Excititor.WebService` and the configuration hooks used by operators and downstream mirrors. + +--- + +## 0) Purpose + +Excititor publishes canonical VEX consensus data. Operators (or StellaOps-managed mirrors) need a deterministic way to sync those exports into downstream environments. Mirror distribution provides: + +* A declarative map of export bundles (`json`, `jsonl`, `openvex`, `csaf`) reachable via signed HTTP endpoints under `/excititor/mirror`. +* Thin quota/authentication controls on top of the existing export cache so mirrors cannot starve the web service. +* Stable payload shapes that downstream automation can monitor (index → fetch updates → download artifact → verify signature). + +Mirror endpoints are intentionally **read-only**. Write paths (export generation, attestation, cache) remain the responsibility of the export pipeline. + +--- + +## 1) Configuration model + +The web service reads mirror configuration from `Excititor:Mirror` (YAML/JSON/appsettings). Each domain groups a set of exports that share rate limits and authentication rules. + +```yaml +Excititor: + Mirror: + Domains: + - id: primary + displayName: Primary Mirror + requireAuthentication: false + maxIndexRequestsPerHour: 600 + maxDownloadRequestsPerHour: 1200 + exports: + - key: consensus + format: json + filters: + vulnId: CVE-2025-0001 + productKey: pkg:test/demo + sort: + createdAt: false # descending + limit: 1000 + - key: consensus-openvex + format: openvex + filters: + vulnId: CVE-2025-0001 +``` + +### Field reference + +| Field | Required | Description | +| --- | --- | --- | +| `id` | ✅ | Stable identifier. Appears in URLs (`/excititor/mirror/domains/{id}`) and download filenames. | +| `displayName` | – | Human-friendly label surfaced in the `/domains` listing. Falls back to `id`. | +| `requireAuthentication` | – | When `true` the service enforces that the caller is authenticated (Authority token). | +| `maxIndexRequestsPerHour` | – | Per-domain quota for index endpoints. `0`/negative disables the guard. | +| `maxDownloadRequestsPerHour` | – | Per-domain quota for artifact downloads. | +| `exports` | ✅ | Collection of export projections. | + +Export-level fields: + +| Field | Required | Description | +| --- | --- | --- | +| `key` | ✅ | Unique key within the domain. Used in URLs (`/exports/{key}`) and filenames. | +| `format` | ✅ | One of `json`, `jsonl`, `openvex`, `csaf`. Maps to `VexExportFormat`. | +| `filters` | – | Key/value pairs executed via `VexQueryFilter`. Keys must match export data source columns (e.g., `vulnId`, `productKey`). | +| `sort` | – | Key/boolean map (false = descending). | +| `limit`, `offset`, `view` | – | Optional query bounds passed through to the export query. | + +⚠️ **Misconfiguration:** invalid formats or missing keys cause exports to be flagged with `status` in the index response; they are not exposed downstream. + +--- + +## 2) HTTP surface + +Routes are grouped under `/excititor/mirror`. + +| Method | Path | Description | +| --- | --- | --- | +| `GET` | `/domains` | Returns configured domains with quota metadata. | +| `GET` | `/domains/{domainId}` | Domain detail (auth/quota + export keys). `404` for unknown domains. | +| `GET` | `/domains/{domainId}/index` | Lists exports with exportId, query signature, format, artifact digest, attestation metadata, and size. Applies index quota. | +| `GET` | `/domains/{domainId}/exports/{exportKey}` | Returns manifest metadata (single export). `404` if unknown/missing. | +| `GET` | `/domains/{domainId}/exports/{exportKey}/download` | Streams export content from the artifact store. Applies download quota. | + +Responses are serialized via `VexCanonicalJsonSerializer` ensuring stable ordering. Download responses include a content-disposition header naming the file `-.`. + +### Error handling + +* `401` – authentication required (`requireAuthentication=true`). +* `404` – domain/export not found or manifest not persisted. +* `429` – per-domain quota exceeded (`Retry-After` header set in seconds). +* `503` – export misconfiguration (invalid format/query). + +--- + +## 3) Rate limiting + +`MirrorRateLimiter` implements a simple rolling 1-hour window using `IMemoryCache`. Each domain has two quotas: + +* `index` scope → `maxIndexRequestsPerHour` +* `download` scope → `maxDownloadRequestsPerHour` + +`0` or negative limits disable enforcement. Quotas are best-effort (per-instance). For HA deployments, configure sticky routing at the ingress or replace the limiter with a distributed implementation. + +--- + +## 4) Interaction with export pipeline + +Mirror endpoints consume manifests produced by the export engine (`MongoVexExportStore`). They do **not** trigger new exports. Operators must configure connectors/exporters to keep targeted exports fresh (see `EXCITITOR-EXPORT-01-005/006/007`). + +Recommended workflow: + +1. Define export plans at the export layer (JSON/OpenVEX/CSAF). +2. Configure mirror domains mapping to those plans. +3. Downstream mirror automation: + * `GET /domains/{id}/index` + * Compare `exportId` / `consensusRevision` + * `GET /download` when new + * Verify digest + attestation + +When the export team lands deterministic mirror bundles (Sprint 7 tasks 01-005/006/007), these configurations can be generated automatically. + +--- + +## 5) Operational guidance + +* Track quota utilisation via HTTP 429 metrics (configure structured logging or OTEL counters when rate limiting triggers). +* Mirror domains can be deployed per tenant (e.g., `tenant-a`, `tenant-b`) with different auth requirements. +* Ensure the underlying artifact stores (`FileSystem`, `S3`, offline bundle) retain artefacts long enough for mirrors to sync. +* For air-gapped mirrors, combine mirror endpoints with the Offline Kit (see `docs/24_OFFLINE_KIT.md`). + +--- + +## 6) Future alignment + +* Replace manual export definitions with generated mirror bundle manifests once `EXCITITOR-EXPORT-01-007` ships. +* Extend `/index` payload with quiet-provenance when `EXCITITOR-EXPORT-01-006` adds that metadata. +* Integrate domain manifests with DevOps mirror profiles (`DEVOPS-MIRROR-08-001`) so helm/compose overlays can enable or disable domains declaratively. + diff --git a/docs/ARCHITECTURE_NOTIFY.md b/docs/ARCHITECTURE_NOTIFY.md new file mode 100644 index 00000000..f9006782 --- /dev/null +++ b/docs/ARCHITECTURE_NOTIFY.md @@ -0,0 +1,515 @@ +> **Scope.** Implementation‑ready architecture for **Notify**: a rules‑driven, tenant‑aware notification service that consumes platform events (scan completed, report ready, rescan deltas, attestation logged, admission decisions, etc.), evaluates operator‑defined routing rules, renders **channel‑specific messages** (Slack/Teams/Email/Webhook), and delivers them **reliably** with idempotency, throttling, and digests. It is UI‑managed, auditable, and safe by default (no secrets leakage, no spam storms). + +--- + +## 0) Mission & boundaries + +**Mission.** Convert **facts** from Stella Ops into **actionable, noise‑controlled** signals where teams already live (chat/email/webhooks), with **explainable** reasons and deep links to the UI. + +**Boundaries.** + +* Notify **does not make policy decisions** and **does not rescan**; it **consumes** events from Scanner/Scheduler/Vexer/Feedser/Attestor/Zastava and routes them. +* Attachments are **links** (UI/attestation pages); Notify **does not** attach SBOMs or large blobs to messages. +* Secrets for channels (Slack tokens, SMTP creds) are **referenced**, not stored raw in Mongo. + +--- + +## 1) Runtime shape & projects + +``` +src/ + ├─ StellaOps.Notify.WebService/ # REST: rules/channels CRUD, test send, deliveries browse + ├─ StellaOps.Notify.Worker/ # consumers + evaluators + renderers + delivery workers + ├─ StellaOps.Notify.Connectors.* / # channel plug-ins: Slack, Teams, Email, Webhook (v1) + │ └─ *.Tests/ + ├─ StellaOps.Notify.Engine/ # rules engine, templates, idempotency, digests, throttles + ├─ StellaOps.Notify.Models/ # DTOs (Rule, Channel, Event, Delivery, Template) + ├─ StellaOps.Notify.Storage.Mongo/ # rules, channels, deliveries, digests, locks + ├─ StellaOps.Notify.Queue/ # bus client (Redis Streams/NATS JetStream) + └─ StellaOps.Notify.Tests.* # unit/integration/e2e +``` + +**Deployables**: + +* **Notify.WebService** (stateless API) +* **Notify.Worker** (horizontal scale) + +**Dependencies**: Authority (OpToks; DPoP/mTLS), MongoDB, Redis/NATS (bus), HTTP egress to Slack/Teams/Webhooks, SMTP relay for Email. + +> **Configuration.** Notify.WebService bootstraps from `notify.yaml` (see `etc/notify.yaml.sample`). Use `storage.driver: mongo` with a production connection string; the optional `memory` driver exists only for tests. Authority settings follow the platform defaults—when running locally without Authority, set `authority.enabled: false` and supply `developmentSigningKey` so JWTs can be validated offline. +> +> `api.rateLimits` exposes token-bucket controls for delivery history queries and test-send previews (`deliveryHistory`, `testSend`). Default values allow generous browsing while preventing accidental bursts; operators can relax/tighten the buckets per deployment. + +> **Plug-ins.** All channel connectors are packaged under `/plugins/notify`. The ordered load list must start with Slack/Teams before Email/Webhook so chat-first actions are registered deterministically for Offline Kit bundles: +> +> ```yaml +> plugins: +> baseDirectory: "/var/opt/stellaops" +> directory: "plugins/notify" +> orderedPlugins: +> - StellaOps.Notify.Connectors.Slack +> - StellaOps.Notify.Connectors.Teams +> - StellaOps.Notify.Connectors.Email +> - StellaOps.Notify.Connectors.Webhook +> ``` +> +> The Offline Kit job simply copies the `plugins/notify` tree into the air-gapped bundle; the ordered list keeps connector manifests stable across environments. + +> **Authority clients.** Register two OAuth clients in StellaOps Authority: `notify-web-dev` (audience `notify.dev`) for development and `notify-web` (audience `notify`) for staging/production. Both require `notify.read` and `notify.admin` scopes and use DPoP-bound client credentials (`client_secret` in the samples). Reference entries live in `etc/authority.yaml.sample`, with placeholder secrets under `etc/secrets/notify-web*.secret.example`. + +--- + +## 2) Responsibilities + +1. **Ingest** platform events from internal bus with strong ordering per key (e.g., image digest). +2. **Evaluate rules** (tenant‑scoped) with matchers: severity changes, namespaces, repos, labels, KEV flags, provider provenance (VEX), component keys, admission decisions, etc. +3. **Control noise**: **throttle**, **coalesce** (digest windows), and **dedupe** via idempotency keys. +4. **Render** channel‑specific messages using safe templates; include **evidence** and **links**. +5. **Deliver** with retries/backoff; record outcome; expose delivery history to UI. +6. **Test** paths (send test to channel targets) without touching live rules. +7. **Audit**: log who configured what, when, and why a message was sent. + +--- + +## 3) Event model (inputs) + +Notify subscribes to the **internal event bus** (produced by services, escaped JSON; gzip allowed with caps): + +* `scanner.scan.completed` — new SBOM(s) composed; artifacts ready +* `scanner.report.ready` — analysis verdict (policy+vex) available; carries deltas summary +* `scheduler.rescan.delta` — new findings after Feedser/Vexer deltas (already summarized) +* `attestor.logged` — Rekor UUID returned (sbom/report/vex export) +* `zastava.admission` — admit/deny with reasons, namespace, image digests +* `feedser.export.completed` — new export ready (rarely notified directly; usually drives Scheduler) +* `vexer.export.completed` — new consensus snapshot (ditto) + +**Canonical envelope (bus → Notify.Engine):** + +```json +{ + "eventId": "uuid", + "kind": "scanner.report.ready", + "tenant": "tenant-01", + "ts": "2025-10-18T05:41:22Z", + "actor": "scanner-webservice", + "scope": { "namespace":"payments", "repo":"ghcr.io/acme/api", "digest":"sha256:..." }, + "payload": { /* kind-specific fields, see below */ } +} +``` + +**Examples (payload cores):** + +* `scanner.report.ready`: + + ```json + { + "reportId": "report-3def...", + "verdict": "fail", + "summary": {"total": 12, "blocked": 2, "warned": 3, "ignored": 5, "quieted": 2}, + "delta": {"newCritical": 1, "kev": ["CVE-2025-..."]}, + "links": {"ui": "https://ui/.../reports/report-3def...", "rekor": "https://rekor/..."}, + "dsse": { "...": "..." }, + "report": { "...": "..." } + } + ``` + + Payload embeds both the canonical report document and the DSSE envelope so connectors, Notify, and UI tooling can reuse the signed bytes without re-serialising. + +* `scanner.scan.completed`: + + ```json + { + "reportId": "report-3def...", + "digest": "sha256:...", + "verdict": "fail", + "summary": {"total": 12, "blocked": 2, "warned": 3, "ignored": 5, "quieted": 2}, + "delta": {"newCritical": 1, "kev": ["CVE-2025-..."]}, + "policy": {"revisionId": "rev-42", "digest": "27d2..."}, + "findings": [{"id": "finding-1", "severity": "Critical", "cve": "CVE-2025-...", "reachability": "runtime"}], + "dsse": { "...": "..." } + } + ``` + +* `zastava.admission`: + + ```json + { "decision":"deny|allow", "reasons":["unsigned image","missing SBOM"], + "images":[{"digest":"sha256:...","signed":false,"hasSbom":false}] } + ``` + +--- + +## 4) Rules engine — semantics + +**Rule shape (simplified):** + +```yaml +name: "high-critical-alerts-prod" +enabled: true +match: + eventKinds: ["scanner.report.ready","scheduler.rescan.delta","zastava.admission"] + namespaces: ["prod-*"] + repos: ["ghcr.io/acme/*"] + minSeverity: "high" # min of new findings (delta context) + kev: true # require KEV-tagged or allow any if false + verdict: ["fail","deny"] # filter for report/admission + vex: + includeRejectedJustifications: false # notify only on accepted 'affected' +actions: + - channel: "slack:sec-alerts" # reference to Channel object + template: "concise" + throttle: "5m" + - channel: "email:soc" + digest: "hourly" + template: "detailed" +``` + +**Evaluation order** + +1. **Tenant check** → discard if rule tenant ≠ event tenant. +2. **Kind filter** → discard early. +3. **Scope match** (namespace/repo/labels). +4. **Delta/severity gates** (if event carries `delta`). +5. **VEX gate** (drop if event’s finding is not affected under policy consensus unless rule says otherwise). +6. **Throttling/dedup** (idempotency key) — skip if suppressed. +7. **Actions** → enqueue per‑channel job(s). + +**Idempotency key**: `hash(ruleId | actionId | event.kind | scope.digest | delta.hash | day-bucket)`; ensures “same alert” doesn’t fire more than once within throttle window. + +**Digest windows**: maintain per action a **coalescer**: + +* Window: `5m|15m|1h|1d` (configurable); coalesces events by tenant + namespace/repo or by digest group. +* Digest messages summarize top N items and counts, with safe truncation. + +--- + +## 5) Channels & connectors (plug‑ins) + +Channel config is **two‑part**: a **Channel** record (name, type, options) and a Secret **reference** (Vault/K8s Secret). Connectors are **restart-time plug-ins** discovered on service start (same manifest convention as Concelier/Excititor) and live under `plugins/notify//`. + +**Built‑in v1:** + +* **Slack**: Bot token (xoxb‑…), `chat.postMessage` + `blocks`; rate limit aware (HTTP 429). +* **Microsoft Teams**: Incoming Webhook (or Graph card later); adaptive card payloads. +* **Email (SMTP)**: TLS (STARTTLS or implicit), From/To/CC/BCC; HTML+text alt; DKIM optional. +* **Generic Webhook**: POST JSON with HMAC signature (Ed25519 or SHA‑256) in headers. + +**Connector contract:** (implemented by plug-in assemblies) + +```csharp +public interface INotifyConnector { + string Type { get; } // "slack" | "teams" | "email" | "webhook" | ... + Task SendAsync(DeliveryContext ctx, CancellationToken ct); + Task HealthAsync(ChannelConfig cfg, CancellationToken ct); +} +``` + +**DeliveryContext** includes **rendered content** and **raw event** for audit. + +**Test-send previews.** Plug-ins can optionally implement `INotifyChannelTestProvider` to shape `/channels/{id}/test` responses. Providers receive a sanitised `ChannelTestPreviewContext` (channel, tenant, target, timestamp, trace) and return a `NotifyDeliveryRendered` preview + metadata. When no provider is present, the host falls back to a generic preview so the endpoint always responds. + +**Secrets**: `ChannelConfig.secretRef` points to Authority‑managed secret handle or K8s Secret path; workers load at send-time; plug-in manifests (`notify-plugin.json`) declare capabilities and version. + +--- + +## 6) Templates & rendering + +**Template engine**: strongly typed, safe Handlebars‑style; no arbitrary code. Partial templates per channel. Deterministic outputs (prop order, no locale drift unless requested). + +**Variables** (examples): + +* `event.kind`, `event.ts`, `scope.namespace`, `scope.repo`, `scope.digest` +* `payload.verdict`, `payload.delta.newCritical`, `payload.links.ui`, `payload.links.rekor` +* `topFindings[]` with `purl`, `vulnId`, `severity` +* `policy.name`, `policy.revision` (if available) + +**Helpers**: + +* `severity_icon(sev)`, `link(text,url)`, `pluralize(n, "finding")`, `truncate(text, n)`, `code(text)`. + +**Channel mapping**: + +* Slack: title + blocks, limited to 50 blocks/3000 chars per section; long lists → link to UI. +* Teams: Adaptive Card schema 1.5; fallback text for older channels. +* Email: HTML + text; inline table of top N findings, rest behind UI link. +* Webhook: JSON with `event`, `ruleId`, `actionId`, `summary`, `links`, and raw `payload` subset. + +**i18n**: template set per locale (English default; Bulgarian built‑in). + +--- + +## 7) Data model (Mongo) + +Canonical JSON Schemas for rules/channels/events live in `docs/notify/schemas/`. Sample payloads intended for tests/UI mock responses are captured in `docs/notify/samples/`. + +**Database**: `notify` + +* `rules` + + ``` + { _id, tenantId, name, enabled, match, actions, createdBy, updatedBy, createdAt, updatedAt } + ``` + +* `channels` + + ``` + { _id, tenantId, name:"slack:sec-alerts", type:"slack", + config:{ webhookUrl?:"", channel:"#sec-alerts", workspace?: "...", secretRef:"ref://..." }, + createdAt, updatedAt } + ``` + +* `deliveries` + + ``` + { _id, tenantId, ruleId, actionId, eventId, kind, scope, status:"sent|failed|throttled|digested|dropped", + attempts:[{ts, status, code, reason}], + rendered:{ title, body, target }, // redacted for PII; body hash stored + sentAt, lastError? } + ``` + +* `digests` + + ``` + { _id, tenantId, actionKey, window:"hourly", openedAt, items:[{eventId, scope, delta}], status:"open|flushed" } + ``` + +* `throttles` + + ``` + { key:"idem:", ttlAt } // short-lived, also cached in Redis + ``` + +**Indexes**: rules by `{tenantId, enabled}`, deliveries by `{tenantId, sentAt desc}`, digests by `{tenantId, actionKey}`. + +--- + +## 8) External APIs (WebService) + +Base path: `/api/v1/notify` (Authority OpToks; scopes: `notify.admin` for write, `notify.read` for view). + +*All* REST calls require the tenant header `X-StellaOps-Tenant` (matches the canonical `tenantId` stored in Mongo). Payloads are normalised via `NotifySchemaMigration` before persistence to guarantee schema version pinning. + +Authentication today is stubbed with Bearer tokens (`Authorization: Bearer `). When Authority wiring lands, this will switch to OpTok validation + scope enforcement, but the header contract will remain the same. + +Service configuration exposes `notify:auth:*` keys (issuer, audience, signing key, scope names) so operators can wire the Authority JWKS or (in dev) a symmetric test key. `notify:storage:*` keys cover Mongo URI/database/collection overrides. Both sets are required for the new API surface. + +Internal tooling can hit `/internal/notify//normalize` to upgrade legacy JSON and return canonical output used in the docs fixtures. + +* **Channels** + + * `POST /channels` | `GET /channels` | `GET /channels/{id}` | `PATCH /channels/{id}` | `DELETE /channels/{id}` + * `POST /channels/{id}/test` → send sample message (no rule evaluation); returns `202 Accepted` with rendered preview + metadata (base keys: `channelType`, `target`, `previewProvider`, `traceId` + connector-specific entries); governed by `api.rateLimits:testSend`. + * `GET /channels/{id}/health` → connector self‑check + +* **Rules** + + * `POST /rules` | `GET /rules` | `GET /rules/{id}` | `PATCH /rules/{id}` | `DELETE /rules/{id}` + * `POST /rules/{id}/test` → dry‑run rule against a **sample event** (no delivery unless `--send`) + +* **Deliveries** + + * `POST /deliveries` → ingest worker delivery state (idempotent via `deliveryId`). + * `GET /deliveries?since=...&status=...&limit=...` → list envelope `{ items, count, continuationToken }` (most recent first); base metadata keys match the test-send response (`channelType`, `target`, `previewProvider`, `traceId`); rate-limited via `api.rateLimits.deliveryHistory`. See `docs/notify/samples/notify-delivery-list-response.sample.json`. + * `GET /deliveries/{id}` → detail (redacted body + metadata) + * `POST /deliveries/{id}/retry` → force retry (admin, future sprint) + +* **Admin** + + * `GET /stats` (per tenant counts, last hour/day) + * `GET /healthz|readyz` (liveness) + * `POST /locks/acquire` | `POST /locks/release` – worker coordination primitives (short TTL). + * `POST /digests` | `GET /digests/{actionKey}` | `DELETE /digests/{actionKey}` – manage open digest windows. + * `POST /audit` | `GET /audit?since=&limit=` – append/query structured audit trail entries. + +**Ingestion**: workers do **not** expose public ingestion; they **subscribe** to the internal bus. (Optional `/events/test` for integration testing, admin‑only.) + +--- + +## 9) Delivery pipeline (worker) + +``` +[Event bus] → [Ingestor] → [RuleMatcher] → [Throttle/Dedupe] → [DigestCoalescer] → [Renderer] → [Connector] → [Result] + └────────→ [DeliveryStore] +``` + +* **Ingestor**: N consumers with per‑key ordering (key = tenant|digest|namespace). +* **RuleMatcher**: loads active rules snapshot for tenant into memory; vectorized predicate check. +* **Throttle/Dedupe**: consult Redis + Mongo `throttles`; if hit → record `status=throttled`. +* **DigestCoalescer**: append to open digest window or flush when timer expires. +* **Renderer**: select template (channel+locale), inject variables, enforce length limits, compute `bodyHash`. +* **Connector**: send; handle provider‑specific rate limits and backoffs; `maxAttempts` with exponential jitter; overflow → DLQ (dead‑letter topic) + UI surfacing. + +**Idempotency**: per action **idempotency key** stored in Redis (TTL = `throttle window` or `digest window`). Connectors also respect **provider** idempotency where available (e.g., Slack `client_msg_id`). + +--- + +## 10) Reliability & rate controls + +* **Per‑tenant** RPM caps (default 600/min) + **per‑channel** concurrency (Slack 1–4, Teams 1–2, Email 8–32 based on relay). +* **Backoff** map: Slack 429 → respect `Retry‑After`; SMTP 4xx → retry; 5xx → retry with jitter; permanent rejects → drop with status recorded. +* **DLQ**: NATS/Redis stream `notify.dlq` with `{event, rule, action, error}` for operator inspection; UI shows DLQ items. + +--- + +## 11) Security & privacy + +* **AuthZ**: all APIs require **Authority** OpToks; actions scoped by tenant. +* **Secrets**: `secretRef` only; Notify fetches just‑in‑time from Authority Secret proxy or K8s Secret (mounted). No plaintext secrets in Mongo. +* **Egress TLS**: validate SSL; pin domains per channel config; optional CA bundle override for on‑prem SMTP. +* **Webhook signing**: HMAC or Ed25519 signatures in `X-StellaOps-Signature` + replay‑window timestamp; include canonical body hash in header. +* **Redaction**: deliveries store **hashes** of bodies, not full payloads for chat/email to minimize PII retention (configurable). +* **Quiet hours**: per tenant (e.g., 22:00–06:00) route high‑sev only; defer others to digests. +* **Loop prevention**: Webhook target allowlist + event origin tags; do not ingest own webhooks. + +--- + +## 12) Observability (Prometheus + OTEL) + +* `notify.events_consumed_total{kind}` +* `notify.rules_matched_total{ruleId}` +* `notify.throttled_total{reason}` +* `notify.digest_coalesced_total{window}` +* `notify.sent_total{channel}` / `notify.failed_total{channel,code}` +* `notify.delivery_latency_seconds{channel}` (end‑to‑end) +* **Tracing**: spans `ingest`, `match`, `render`, `send`; correlation id = `eventId`. + +**SLO targets** + +* Event→delivery p95 **≤ 30–60 s** under nominal load. +* Failure rate p95 **< 0.5%** per hour (excluding provider outages). +* Duplicate rate **≈ 0** (idempotency working). + +--- + +## 13) Configuration (YAML) + +```yaml +notify: + authority: + issuer: "https://authority.internal" + require: "dpop" # or "mtls" + bus: + kind: "redis" # or "nats" + streams: + - "scanner.events" + - "scheduler.events" + - "attestor.events" + - "zastava.events" + mongo: + uri: "mongodb://mongo/notify" + limits: + perTenantRpm: 600 + perChannel: + slack: { concurrency: 2 } + teams: { concurrency: 1 } + email: { concurrency: 8 } + webhook: { concurrency: 8 } + digests: + defaultWindow: "1h" + maxItems: 100 + quietHours: + enabled: true + window: "22:00-06:00" + minSeverity: "critical" + webhooks: + sign: + method: "ed25519" # or "hmac-sha256" + keyRef: "ref://notify/webhook-sign-key" +``` + +--- + +## 14) UI touch‑points + +* **Notifications → Channels**: add Slack/Teams/Email/Webhook; run **health**; rotate secrets. +* **Notifications → Rules**: create/edit YAML rules with linting; test with sample events; see match rate. +* **Notifications → Deliveries**: timeline with filters (status, channel, rule); inspect last error; retry. +* **Digest preview**: shows current window contents and when it will flush. +* **Quiet hours**: configure per tenant; show overrides. +* **DLQ**: browse dead‑letters; requeue after fix. + +--- + +## 15) Failure modes & responses + +| Condition | Behavior | +| ----------------------------------- | ------------------------------------------------------------------------------------- | +| Slack 429 / Teams 429 | Respect `Retry‑After`, backoff with jitter, reduce concurrency | +| SMTP transient 4xx | Retry up to `maxAttempts`; escalate to DLQ on exhaust | +| Invalid channel secret | Mark channel unhealthy; suppress sends; surface in UI | +| Rule explosion (matches everything) | Safety valve: per‑tenant RPM caps; auto‑pause rule after X drops; UI alert | +| Bus outage | Buffer to local queue (bounded); resume consuming when healthy | +| Mongo slowness | Fall back to Redis throttles; batch write deliveries; shed low‑priority notifications | + +--- + +## 16) Testing matrix + +* **Unit**: matchers, throttle math, digest coalescing, idempotency keys, template rendering edge cases. +* **Connectors**: provider‑level rate limits, payload size truncation, error mapping. +* **Integration**: synthetic event storm (10k/min), ensure p95 latency & duplicate rate. +* **Security**: DPoP/mTLS on APIs; secretRef resolution; webhook signing & replay windows. +* **i18n**: localized templates render deterministically. +* **Chaos**: Slack/Teams API flaps; SMTP greylisting; Redis hiccups; ensure graceful degradation. + +--- + +## 17) Sequences (representative) + +**A) New criticals after Feedser delta (Slack immediate + Email hourly digest)** + +```mermaid +sequenceDiagram + autonumber + participant SCH as Scheduler + participant NO as Notify.Worker + participant SL as Slack + participant SMTP as Email + + SCH->>NO: bus event scheduler.rescan.delta { newCritical:1, digest:sha256:... } + NO->>NO: match rules (Slack immediate; Email hourly digest) + NO->>SL: chat.postMessage (concise) + SL-->>NO: 200 OK + NO->>NO: append to digest window (email:soc) + Note over NO: At window close → render digest email + NO->>SMTP: send email (detailed digest) + SMTP-->>NO: 250 OK +``` + +**B) Admission deny (Teams card + Webhook)** + +```mermaid +sequenceDiagram + autonumber + participant ZA as Zastava + participant NO as Notify.Worker + participant TE as Teams + participant WH as Webhook + + ZA->>NO: bus event zastava.admission { decision: "deny", reasons: [...] } + NO->>TE: POST adaptive card + TE-->>NO: 200 OK + NO->>WH: POST JSON (signed) + WH-->>NO: 2xx +``` + +--- + +## 18) Implementation notes + +* **Language**: .NET 10; minimal API; `System.Text.Json` with canonical writer for body hashing; Channels for pipelines. +* **Bus**: Redis Streams (**XGROUP** consumers) or NATS JetStream for at‑least‑once with ack; per‑tenant consumer groups to localize backpressure. +* **Templates**: compile and cache per rule+channel+locale; version with rule `updatedAt` to invalidate. +* **Rules**: store raw YAML + parsed AST; validate with schema + static checks (e.g., nonsensical combos). +* **Secrets**: pluggable secret resolver (Authority Secret proxy, K8s, Vault). +* **Rate limiting**: `System.Threading.RateLimiting` + per‑connector adapters. + +--- + +## 19) Roadmap (post‑v1) + +* **PagerDuty/Opsgenie** connectors; **Jira** ticket creation. +* **User inbox** (in‑app notifications) + mobile push via webhook relay. +* **Anomaly suppression**: auto‑pause noisy rules with hints (learned thresholds). +* **Graph rules**: “only notify if *not_affected → affected* transition at consensus layer”. +* **Label enrichment**: pluggable taggers (business criticality, data classification) to refine matchers. diff --git a/docs/ARCHITECTURE_SCANNER.md b/docs/ARCHITECTURE_SCANNER.md index abee0e0e..fae81829 100644 --- a/docs/ARCHITECTURE_SCANNER.md +++ b/docs/ARCHITECTURE_SCANNER.md @@ -1,6 +1,6 @@ # component_architecture_scanner.md — **Stella Ops Scanner** (2025Q4) -> **Scope.** Implementation‑ready architecture for the **Scanner** subsystem: WebService, Workers, analyzers, SBOM assembly (inventory & usage), per‑layer caching, three‑way diffs, artifact catalog (MinIO+Mongo), attestation hand‑off, and scale/security posture. This document is the contract between the scanning plane and everything else (Policy, Vexer, Feedser, UI, CLI). +> **Scope.** Implementation‑ready architecture for the **Scanner** subsystem: WebService, Workers, analyzers, SBOM assembly (inventory & usage), per‑layer caching, three‑way diffs, artifact catalog (MinIO+Mongo), attestation hand‑off, and scale/security posture. This document is the contract between the scanning plane and everything else (Policy, Excititor, Concelier, UI, CLI). --- @@ -10,7 +10,7 @@ **Boundaries.** -* Scanner **does not** produce PASS/FAIL. The backend (Policy + Vexer + Feedser) decides presentation and verdicts. +* Scanner **does not** produce PASS/FAIL. The backend (Policy + Excititor + Concelier) decides presentation and verdicts. * Scanner **does not** keep third‑party SBOM warehouses. It may **bind** to existing attestations for exact hashes. * Core analyzers are **deterministic** (no fuzzy identity). Optional heuristic plug‑ins (e.g., patch‑presence) run under explicit flags and never contaminate the core SBOM. @@ -40,6 +40,37 @@ src/ └─ StellaOps.Scanner.Sbomer.DockerImage/ # CLI‑driven scanner container ``` +Analyzer assemblies and buildx generators are packaged as **restart-time plug-ins** under `plugins/scanner/**` with manifests; services must restart to activate new plug-ins. + +### 1.1 Queue backbone (Redis / NATS) + +`StellaOps.Scanner.Queue` exposes a transport-agnostic contract (`IScanQueue`/`IScanQueueLease`) used by the WebService producer and Worker consumers. Sprint 9 introduces two first-party transports: + +- **Redis Streams** (default). Uses consumer groups, deterministic idempotency keys (`scanner:jobs:idemp:*`), and supports lease claim (`XCLAIM`), renewal, exponential-backoff retries, and a `scanner:jobs:dead` stream for exhausted attempts. +- **NATS JetStream**. Provisions the `SCANNER_JOBS` work-queue stream + durable consumer `scanner-workers`, publishes with `MsgId` for dedupe, applies backoff via `NAK` delays, and routes dead-lettered jobs to `SCANNER_JOBS_DEAD`. + +Metrics are emitted via `Meter` counters (`scanner_queue_enqueued_total`, `scanner_queue_retry_total`, `scanner_queue_deadletter_total`), and `ScannerQueueHealthCheck` pings the active backend (Redis `PING`, NATS `PING`). Configuration is bound from `scanner.queue`: + +```yaml +scanner: + queue: + kind: redis # or nats + redis: + connectionString: "redis://queue:6379/0" + streamName: "scanner:jobs" + nats: + url: "nats://queue:4222" + stream: "SCANNER_JOBS" + subject: "scanner.jobs" + durableConsumer: "scanner-workers" + deadLetterSubject: "scanner.jobs.dead" + maxDeliveryAttempts: 5 + retryInitialBackoff: 00:00:05 + retryMaxBackoff: 00:02:00 +``` + +The DI extension (`AddScannerQueue`) wires the selected transport, so future additions (e.g., RabbitMQ) only implement the same contract and register. + **Runtime form‑factor:** two deployables * **Scanner.WebService** (stateless REST) @@ -131,6 +162,10 @@ GET /catalog/artifacts/{id} → { meta } GET /healthz | /readyz | /metrics ``` +### Report events + +When `scanner.events.enabled = true`, the WebService serialises the signed report (canonical JSON + DSSE envelope) with `NotifyCanonicalJsonSerializer` and publishes two Redis Stream entries (`scanner.report.ready`, `scanner.scan.completed`) to the configured stream (default `stella.events`). The stream fields carry the whole envelope plus lightweight headers (`kind`, `tenant`, `ts`) so Notify and UI timelines can consume the event bus without recomputing signatures. Publish timeouts and bounded stream length are controlled via `scanner:events:publishTimeoutSeconds` and `scanner:events:maxStreamLength`. If the queue driver is already Redis and no explicit events DSN is provided, the host reuses the queue connection and auto-enables event emission so deployments get live envelopes without extra wiring. + --- ## 5) Execution flow (Worker) @@ -155,6 +190,12 @@ GET /healthz | /readyz | /metrics * **rpm**: `/var/lib/rpm/Packages` (via librpm or parser) * Record `name`, `version` (epoch/revision), `arch`, source package where present, and **declared file lists**. +> **Data flow note:** Each OS analyzer now writes its canonical output into the shared `ScanAnalysisStore` under +> `analysis.os.packages` (raw results), `analysis.os.fragments` (per-analyzer layer fragments), and contributes to +> `analysis.layers.fragments` (the aggregated view consumed by emit/diff pipelines). Helpers in +> `ScanAnalysisCompositionBuilder` convert these fragments into SBOM composition requests and component graphs so the +> diff/emit stages no longer reach back into individual analyzer implementations. + **B) Language ecosystems (installed state only)** * **Java**: `META-INF/maven/*/pom.properties`, MANIFEST → `pkg:maven/...` @@ -171,6 +212,9 @@ GET /healthz | /readyz | /metrics * **ELF**: parse `PT_INTERP`, `DT_NEEDED`, RPATH/RUNPATH, **GNU symbol versions**; map **SONAMEs** to file paths; link executables → libs. * **PE/Mach‑O** (planned M2): import table, delay‑imports; version resources; code signatures. * Map libs back to **OS packages** if possible (via file lists); else emit `bin:{sha256}` components. +* The exported metadata (`stellaops.os.*` properties, license list, source package) feeds policy scoring and export pipelines + directly – Policy evaluates quiet rules against package provenance while Exporters forward the enriched fields into + downstream JSON/Trivy payloads. **D) EntryTrace (ENTRYPOINT/CMD → terminal program)** @@ -402,6 +446,26 @@ ResolveEntrypoint(ImageConfig cfg, RootFs fs): return Unknown(reason) ``` +### Appendix A.1 — EntryTrace Explainability + +EntryTrace emits structured diagnostics and metrics so operators can quickly understand why resolution succeeded or degraded: + +| Reason | Description | Typical Mitigation | +|--------|-------------|--------------------| +| `CommandNotFound` | A command referenced in the script cannot be located in the layered root filesystem or `PATH`. | Ensure binaries exist in the image or extend `PATH` hints. | +| `MissingFile` | `source`/`.`/`run-parts` targets are missing. | Bundle the script or guard the include. | +| `DynamicEnvironmentReference` | Path depends on `$VARS` that are unknown at scan time. | Provide defaults via scan metadata or accept partial usage. | +| `RecursionLimitReached` | Nested includes exceeded the analyzer depth limit (default 64). | Flatten indirection or increase the limit in options. | +| `RunPartsEmpty` | `run-parts` directory contained no executable entries. | Remove empty directories or ignore if intentional. | +| `JarNotFound` / `ModuleNotFound` | Java/Python targets missing, preventing interpreter tracing. | Ship the jar/module with the image or adjust the launcher. | + +Diagnostics drive two metrics published by `EntryTraceMetrics`: + +- `entrytrace_resolutions_total{outcome}` — resolution attempts segmented by outcome (`resolved`, `partiallyresolved`, `unresolved`). +- `entrytrace_unresolved_total{reason}` — diagnostic counts keyed by reason. + +Structured logs include `entrytrace.path`, `entrytrace.command`, `entrytrace.reason`, and `entrytrace.depth`, all correlated with scan/job IDs. Timestamps are normalized to UTC (microsecond precision) to keep DSSE attestations and UI traces explainable. + ### Appendix B — BOM‑Index sidecar ``` @@ -410,4 +474,3 @@ vector purls map components optional map usedByEntrypoint ``` - diff --git a/docs/ARCHITECTURE_SCHEDULER.md b/docs/ARCHITECTURE_SCHEDULER.md new file mode 100644 index 00000000..21e4d60a --- /dev/null +++ b/docs/ARCHITECTURE_SCHEDULER.md @@ -0,0 +1,424 @@ +# component_architecture_scheduler.md — **Stella Ops Scheduler** (2025Q4) + +> **Scope.** Implementation‑ready architecture for **Scheduler**: a service that (1) **re‑evaluates** already‑cataloged images when intel changes (Feedser/Vexer/policy), (2) orchestrates **nightly** and **ad‑hoc** runs, (3) targets only the **impacted** images using the BOM‑Index, and (4) emits **report‑ready** events that downstream **Notify** fans out. Default mode is **analysis‑only** (no image pull); optional **content‑refresh** can be enabled per schedule. + +--- + +## 0) Mission & boundaries + +**Mission.** Keep scan results **current** without rescanning the world. When new advisories or VEX claims land, **pinpoint** affected images and ask the backend to recompute **verdicts** against the **existing SBOMs**. Surface only **meaningful deltas** to humans and ticket queues. + +**Boundaries.** + +* Scheduler **does not** compute SBOMs and **does not** sign. It calls Scanner/WebService’s **/reports (analysis‑only)** endpoint and lets the backend (Policy + Vexer + Feedser) decide PASS/FAIL. +* Scheduler **may** ask Scanner to **content‑refresh** selected targets (e.g., mutable tags) but the default is **no** image pull. +* Notifications are **not** sent directly; Scheduler emits events consumed by **Notify**. + +--- + +## 1) Runtime shape & projects + +``` +src/ + ├─ StellaOps.Scheduler.WebService/ # REST (schedules CRUD, runs, admin) + ├─ StellaOps.Scheduler.Worker/ # planners + runners (N replicas) + ├─ StellaOps.Scheduler.ImpactIndex/ # purl→images inverted index (roaring bitmaps) + ├─ StellaOps.Scheduler.Models/ # DTOs (Schedule, Run, ImpactSet, Deltas) + ├─ StellaOps.Scheduler.Storage.Mongo/ # schedules, runs, cursors, locks + ├─ StellaOps.Scheduler.Queue/ # Redis Streams / NATS abstraction + ├─ StellaOps.Scheduler.Tests.* # unit/integration/e2e +``` + +**Deployables**: + +* **Scheduler.WebService** (stateless) +* **Scheduler.Worker** (scale‑out; planners + executors) + +**Dependencies**: Authority (OpTok + DPoP/mTLS), Scanner.WebService, Feedser, Vexer, MongoDB, Redis/NATS, (optional) Notify. + +--- + +## 2) Core responsibilities + +1. **Time‑based** runs: cron windows per tenant/timezone (e.g., “02:00 Europe/Sofia”). +2. **Event‑driven** runs: react to **Feedser export** and **Vexer export** deltas (changed product keys / advisories / claims). +3. **Impact targeting**: map changes to **image sets** using a **global inverted index** built from Scanner’s per‑image **BOM‑Index** sidecars. +4. **Run planning**: shard, pace, and rate‑limit jobs to avoid thundering herds. +5. **Execution**: call Scanner **/reports (analysis‑only)** or **/scans (content‑refresh)**; aggregate **delta** results. +6. **Events**: publish `rescan.delta` and `report.ready` summaries for **Notify** & **UI**. +7. **Control plane**: CRUD schedules, **pause/resume**, dry‑run previews, audit. + +--- + +## 3) Data model (Mongo) + +**Database**: `scheduler` + +* `schedules` + + ``` + { _id, tenantId, name, enabled, whenCron, timezone, + mode: "analysis-only" | "content-refresh", + selection: { scope: "all-images" | "by-namespace" | "by-repo" | "by-digest" | "by-labels", + includeTags?: ["prod-*"], digests?: [sha256...], resolvesTags?: bool }, + onlyIf: { lastReportOlderThanDays?: int, policyRevision?: string }, + notify: { onNewFindings: bool, minSeverity: "low|medium|high|critical", includeKEV: bool }, + limits: { maxJobs?: int, ratePerSecond?: int, parallelism?: int }, + createdAt, updatedAt, createdBy, updatedBy } + ``` + +* `runs` + + ``` + { _id, scheduleId?, tenantId, trigger: "cron|feedser|vexer|manual", + reason?: { feedserExportId?, vexerExportId?, cursor? }, + state: "planning|queued|running|completed|error|cancelled", + stats: { candidates: int, deduped: int, queued: int, completed: int, deltas: int, newCriticals: int }, + startedAt, finishedAt, error? } + ``` + +* `impact_cursors` + + ``` + { _id: tenantId, feedserLastExportId, vexerLastExportId, updatedAt } + ``` + +* `locks` (singleton schedulers, run leases) + +* `audit` (CRUD actions, run outcomes) + +**Indexes**: + +* `schedules` on `{tenantId, enabled}`, `{whenCron}`. +* `runs` on `{tenantId, startedAt desc}`, `{state}`. +* TTL optional for completed runs (e.g., 180 days). + +--- + +## 4) ImpactIndex (global inverted index) + +Goal: translate **change keys** → **image sets** in **milliseconds**. + +**Source**: Scanner produces per‑image **BOM‑Index** sidecars (purls, and `usedByEntrypoint` bitmaps). Scheduler ingests/refreshes them to build a **global** index. + +**Representation**: + +* Assign **image IDs** (dense ints) to catalog images. +* Keep **Roaring Bitmaps**: + + * `Contains[purl] → bitmap(imageIds)` + * `UsedBy[purl] → bitmap(imageIds)` (subset of Contains) +* Optionally keep **Owner maps**: `{imageId → {tenantId, namespaces[], repos[]}}` for selection filters. +* Persist in RocksDB/LMDB or Redis‑modules; cache hot shards in memory; snapshot to Mongo for cold start. + +**Update paths**: + +* On new/updated image SBOM: **merge** per‑image set into global maps. +* On image remove/expiry: **clear** id from bitmaps. + +**API (internal)**: + +```csharp +IImpactIndex { + ImpactSet ResolveByPurls(IEnumerable purls, bool usageOnly, Selector sel); + ImpactSet ResolveByVulns(IEnumerable vulnIds, bool usageOnly, Selector sel); // optional (vuln->purl precomputed by Feedser) + ImpactSet ResolveAll(Selector sel); // for nightly +} +``` + +**Selector filters**: tenant, namespaces, repos, labels, digest allowlists, `includeTags` patterns. + +--- + +## 5) External interfaces (REST) + +Base path: `/api/v1/scheduler` (Authority OpToks; scopes: `scheduler.read`, `scheduler.admin`). + +### 5.1 Schedules CRUD + +* `POST /schedules` → create +* `GET /schedules` → list (filter by tenant) +* `GET /schedules/{id}` → details + next run +* `PATCH /schedules/{id}` → pause/resume/update +* `DELETE /schedules/{id}` → delete (soft delete, optional) + +### 5.2 Run control & introspection + +* `POST /run` — ad‑hoc run + + ```json + { "mode": "analysis-only|content-refresh", "selection": {...}, "reason": "manual" } + ``` +* `GET /runs` — list with paging +* `GET /runs/{id}` — status, stats, links to deltas +* `POST /runs/{id}/cancel` — best‑effort cancel + +### 5.3 Previews (dry‑run) + +* `POST /preview/impact` — returns **candidate count** and a small sample of impacted digests for given change keys or selection. + +### 5.4 Event webhooks (optional push from Feedser/Vexer) + +* `POST /events/feedser-export` + + ```json + { "exportId":"...", "changedProductKeys":["pkg:rpm/openssl", ...], "kev": ["CVE-..."], "window": { "from":"...","to":"..." } } + ``` +* `POST /events/vexer-export` + + ```json + { "exportId":"...", "changedClaims":[ { "productKey":"pkg:deb/...", "vulnId":"CVE-...", "status":"not_affected→affected"} ], ... } + ``` + +**Security**: webhook requires **mTLS** or an **HMAC** `X-Scheduler-Signature` (Ed25519 / SHA‑256) plus Authority token. + +--- + +## 6) Planner → Runner pipeline + +### 6.1 Planning algorithm (event‑driven) + +``` +On Export Event (Feedser/Vexer): + keys = Normalize(change payload) # productKeys or vulnIds→productKeys + usageOnly = schedule/policy hint? # default true + sel = Selector for tenant/scope from schedules subscribed to events + + impacted = ImpactIndex.ResolveByPurls(keys, usageOnly, sel) + impacted = ApplyOwnerFilters(impacted, sel) # namespaces/repos/labels + impacted = DeduplicateByDigest(impacted) + impacted = EnforceLimits(impacted, limits.maxJobs) + shards = Shard(impacted, byHashPrefix, n=limits.parallelism) + + For each shard: + Enqueue RunSegment (runId, shard, rate=limits.ratePerSecond) +``` + +**Fairness & pacing** + +* Use **leaky bucket** per tenant and per registry host. +* Prioritize **KEV‑tagged** and **critical** first if oversubscribed. + +### 6.2 Nightly planning + +``` +At cron tick: + sel = resolve selection + candidates = ImpactIndex.ResolveAll(sel) + if lastReportOlderThanDays present → filter by report age (via Scanner catalog) + shard & enqueue as above +``` + +### 6.3 Execution (Runner) + +* Pop **RunSegment** job → for each image digest: + + * **analysis‑only**: `POST scanner/reports { imageDigest, policyRevision? }` + * **content‑refresh**: resolve tag→digest if needed; `POST scanner/scans { imageRef, attest? false }` then `POST /reports` +* Collect **delta**: `newFindings`, `newCriticals`/`highs`, `links` (UI deep link, Rekor if present). +* Persist per‑image outcome in `runs.{id}.stats` (incremental counters). +* Emit `scheduler.rescan.delta` events to **Notify** only when **delta > 0** and matches severity rule. + +--- + +## 7) Event model (outbound) + +**Topic**: `rescan.delta` (internal bus → Notify; UI subscribes via backend). + +```json +{ + "tenant": "tenant-01", + "runId": "324af…", + "imageDigest": "sha256:…", + "newCriticals": 1, + "newHigh": 2, + "kevHits": ["CVE-2025-..."], + "topFindings": [ + { "purl":"pkg:rpm/openssl@3.0.12-...","vulnId":"CVE-2025-...","severity":"critical","link":"https://ui/scans/..." } + ], + "reportUrl": "https://ui/.../scans/sha256:.../report", + "attestation": { "uuid":"rekor-uuid", "verified": true }, + "ts": "2025-10-18T03:12:45Z" +} +``` + +**Also**: `report.ready` for “no‑change” summaries (digest + zero delta), which Notify can ignore by rule. + +--- + +## 8) Security posture + +* **AuthN/Z**: Authority OpToks with `aud=scheduler`; DPoP (preferred) or mTLS. +* **Multi‑tenant**: every schedule, run, and event carries `tenantId`; ImpactIndex filters by tenant‑visible images. +* **Webhook** callers (Feedser/Vexer) present **mTLS** or **HMAC** and Authority token. +* **Input hardening**: size caps on changed key lists; reject >100k keys per event; compress (zstd/gzip) allowed with limits. +* **No secrets** in logs; redact tokens and signatures. + +--- + +## 9) Observability & SLOs + +**Metrics (Prometheus)** + +* `scheduler.events_total{source, result}` +* `scheduler.impact_resolve_seconds{quantile}` +* `scheduler.images_selected_total{mode}` +* `scheduler.jobs_enqueued_total{mode}` +* `scheduler.run_latency_seconds{quantile}` // event → first verdict +* `scheduler.delta_images_total{severity}` +* `scheduler.rate_limited_total{reason}` + +**Targets** + +* Resolve 10k changed keys → impacted set in **<300 ms** (hot cache). +* Event → first rescan verdict in **≤60 s** (p95). +* Nightly coverage 50k images in **≤10 min** with 10 workers (analysis‑only). + +**Tracing** (OTEL): spans `plan`, `resolve`, `enqueue`, `report_call`, `persist`, `emit`. + +--- + +## 10) Configuration (YAML) + +```yaml +scheduler: + authority: + issuer: "https://authority.internal" + require: "dpop" # or "mtls" + queue: + kind: "redis" # or "nats" + url: "redis://redis:6379/4" + mongo: + uri: "mongodb://mongo/scheduler" + impactIndex: + storage: "rocksdb" # "rocksdb" | "redis" | "memory" + warmOnStart: true + usageOnlyDefault: true + limits: + defaultRatePerSecond: 50 + defaultParallelism: 8 + maxJobsPerRun: 50000 + integrates: + scannerUrl: "https://scanner-web.internal" + feedserWebhook: true + vexerWebhook: true + notifications: + emitBus: "internal" # deliver to Notify via internal bus +``` + +--- + +## 11) UI touch‑points + +* **Schedules** page: CRUD, enable/pause, next run, last run stats, mode (analysis/content), selector preview. +* **Runs** page: timeline; heat‑map of deltas; drill‑down to affected images. +* **Dry‑run preview** modal: “This Feedser export touches ~3,214 images; projected deltas: ~420 (34 KEV).” + +--- + +## 12) Failure modes & degradations + +| Condition | Behavior | +| ------------------------------------ | ---------------------------------------------------------------------------------------- | +| ImpactIndex cold / incomplete | Fall back to **All** selection for nightly; for events, cap to KEV+critical until warmed | +| Feedser/Vexer webhook storm | Coalesce by exportId; debounce 30–60 s; keep last | +| Scanner under load (429) | Backoff with jitter; respect per‑tenant/leaky bucket | +| Oversubscription (too many impacted) | Prioritize KEV/critical first; spillover to next window; UI banner shows backlog | +| Notify down | Buffer outbound events in queue (TTL 24h) | +| Mongo slow | Cut batch sizes; sample‑log; alert ops; don’t drop runs unless critical | + +--- + +## 13) Testing matrix + +* **ImpactIndex**: correctness (purl→image sets), performance, persistence after restart, memory pressure with 1M purls. +* **Planner**: dedupe, shard, fairness, limit enforcement, KEV prioritization. +* **Runner**: parallel report calls, error backoff, partial failures, idempotency. +* **End‑to‑end**: Feedser export → deltas visible in UI in ≤60 s. +* **Security**: webhook auth (mTLS/HMAC), DPoP nonce dance, tenant isolation. +* **Chaos**: drop scanner availability; simulate registry throttles (content‑refresh mode). +* **Nightly**: cron tick correctness across timezones and DST. + +--- + +## 14) Implementation notes + +* **Language**: .NET 10 minimal API; Channels‑based pipeline; `System.Threading.RateLimiting`. +* **Bitmaps**: Roaring via `RoaringBitmap` bindings; memory‑map large shards if RocksDB used. +* **Cron**: Quartz‑style parser with timezone support; clock skew tolerated ±60 s. +* **Dry‑run**: use ImpactIndex only; never call scanner. +* **Idempotency**: run segments carry deterministic keys; retries safe. +* **Backpressure**: per‑tenant buckets; per‑host registry budgets respected when content‑refresh enabled. + +--- + +## 15) Sequences (representative) + +**A) Event‑driven rescan (Feedser delta)** + +```mermaid +sequenceDiagram + autonumber + participant FE as Feedser + participant SCH as Scheduler.Worker + participant IDX as ImpactIndex + participant SC as Scanner.WebService + participant NO as Notify + + FE->>SCH: POST /events/feedser-export {exportId, changedProductKeys} + SCH->>IDX: ResolveByPurls(keys, usageOnly=true, sel) + IDX-->>SCH: bitmap(imageIds) → digests list + SCH->>SC: POST /reports {imageDigest} (batch/sequenced) + SC-->>SCH: report deltas (new criticals/highs) + alt delta>0 + SCH->>NO: rescan.delta {digest, newCriticals, links} + end +``` + +**B) Nightly rescan** + +```mermaid +sequenceDiagram + autonumber + participant CRON as Cron + participant SCH as Scheduler.Worker + participant IDX as ImpactIndex + participant SC as Scanner.WebService + + CRON->>SCH: tick (02:00 Europe/Sofia) + SCH->>IDX: ResolveAll(selector) + IDX-->>SCH: candidates + SCH->>SC: POST /reports {digest} (paced) + SC-->>SCH: results + SCH-->>SCH: aggregate, store run stats +``` + +**C) Content‑refresh (tag followers)** + +```mermaid +sequenceDiagram + autonumber + participant SCH as Scheduler + participant SC as Scanner + SCH->>SC: resolve tag→digest (if changed) + alt digest changed + SCH->>SC: POST /scans {imageRef} # new SBOM + SC-->>SCH: scan complete (artifacts) + SCH->>SC: POST /reports {imageDigest} + else unchanged + SCH->>SC: POST /reports {imageDigest} # analysis-only + end +``` + +--- + +## 16) Roadmap + +* **Vuln‑centric impact**: pre‑join vuln→purl→images to rank by **KEV** and **exploited‑in‑the‑wild** signals. +* **Policy diff preview**: when a staged policy changes, show projected breakage set before promotion. +* **Cross‑cluster federation**: one Scheduler instance driving many Scanner clusters (tenant isolation). +* **Windows containers**: integrate Zastava runtime hints for Usage view tightening. + +--- + +**End — component_architecture_scheduler.md** diff --git a/docs/ARCHITECTURE_SIGNER.md b/docs/ARCHITECTURE_SIGNER.md index fda371ea..b116f879 100644 --- a/docs/ARCHITECTURE_SIGNER.md +++ b/docs/ARCHITECTURE_SIGNER.md @@ -223,7 +223,7 @@ Supported **predicate types** (extensible): * `https://stella-ops.org/attestations/sbom/1` (SBOM emissions) * `https://stella-ops.org/attestations/report/1` (final PASS/FAIL reports) -* `https://stella-ops.org/attestations/vex-export/1` (Vexer exports; optional) +* `https://stella-ops.org/attestations/vex-export/1` (Excititor exports; optional) **Validation**: diff --git a/docs/ARCHITECTURE_UI.md b/docs/ARCHITECTURE_UI.md index 19d5d156..58c9c373 100644 --- a/docs/ARCHITECTURE_UI.md +++ b/docs/ARCHITECTURE_UI.md @@ -1,6 +1,6 @@ # component_architecture_web_ui.md — **Stella Ops Web UI** (2025Q4) -> **Scope.** Implementation‑ready architecture for the **Angular SPA** that operators and developers use to drive Stella Ops. This document defines UX surfaces, module boundaries, data flows, auth, RBAC, real‑time updates, performance targets, i18n/a11y, security headers, testing and deployment. The UI is a *consumer* of backend APIs (Scanner, Policy, Vexer, Feedser, Attestor, Authority) and never performs scanning, merging, or signing on its own. +> **Scope.** Implementation‑ready architecture for the **Angular SPA** that operators and developers use to drive Stella Ops. This document defines UX surfaces, module boundaries, data flows, auth, RBAC, real‑time updates, performance targets, i18n/a11y, security headers, testing and deployment. The UI is a *consumer* of backend APIs (Scanner, Policy, Excititor, Concelier, Attestor, Authority) and never performs scanning, merging, or signing on its own. --- @@ -10,7 +10,7 @@ * Scans (status, SBOMs, diffs, EntryTrace, attestation). * Policy management (rules, exemptions, VEX consumption view). -* Vulnerability intel (Feedser status), VEX consensus exploration (Vexer). +* Vulnerability intel (Concelier status), VEX consensus exploration (Excititor). * Runtime posture (Zastava observer + admission). * Admin operations (tenants, tokens, quotas, licensing posture). @@ -42,7 +42,7 @@ ├─ runtime/ # Zastava posture, drift events, admission decisions ├─ policy/ # rules editor (YAML/Rego), exemptions, previews ├─ vex/ # VEX explorer (claims, consensus, conflicts) - ├─ feedser/ # source health, export cursors, rebuild/export triggers + ├─ concelier/ # source health, export cursors, rebuild/export triggers ├─ attest/ # attestation proofs, verification bundles, Rekor links ├─ admin/ # tenants, roles, clients, quotas, licensing posture └─ plugins/ # route plug-ins (lazy remote modules, governed) @@ -86,13 +86,13 @@ Each feature folder builds as a **standalone route** (lazy loaded). All HTTP sha * **VEX inclusion controls**: weight sliders (visualization only), provider allow/deny toggles. * **Preview**: select SBOM (or image digest) → show verdict under staged policy. -### 3.5 Vexer +### 3.5 Excititor * **Claims explorer**: search by vulnId/productKey/provider; show raw claim (status, justification, evidence). * **Consensus view**: rollup per (vuln, product) with accepted/rejected sources, weights, timestamps. * **Conflicts**: grid of top conflicts; filters for justification gates failed. -### 3.6 Feedser +### 3.6 Concelier * **Sources** table: staleness, last run, errors. * **Advisory search**: by CVE/alias; show normalized affected ranges. @@ -136,7 +136,7 @@ Each feature folder builds as a **standalone route** (lazy loaded). All HTTP sha * **`core/http/api-client.ts`** centralizes: - * Base URLs (Scanner, Vexer, Feedser, Attestor). + * Base URLs (Scanner, Excititor, Concelier, Attestor). * **Retry** policies on idempotent GETs (backoff + jitter). * **Problem+JSON** parser → uniform error toasts with correlation ID. * **SSE** helper (EventSource) with auto‑reconnect & backpressure. @@ -144,7 +144,7 @@ Each feature folder builds as a **standalone route** (lazy loaded). All HTTP sha * Typed API clients (DTOs in `core/api/models.ts`): - * `ScannerApi`, `PolicyApi`, `VexerApi`, `FeedserApi`, `AttestorApi`, `AuthorityApi`. + * `ScannerApi`, `PolicyApi`, `ExcititorApi`, `ConcelierApi`, `AttestorApi`, `AuthorityApi`. **DTO examples (abbrev):** @@ -166,6 +166,28 @@ export interface VexConsensus { } ``` +*Upcoming:* `NotifyApi` consumes delivery history using the new paginated envelope returned by `/api/v1/notify/deliveries`. + +```ts +export interface NotifyDeliveryListResponse { + items: NotifyDelivery[]; + count: number; + continuationToken?: string; +} + +export interface NotifyDelivery { + deliveryId: string; + ruleId: string; + actionId: string; + status: 'pending'|'sent'|'failed'|'throttled'|'digested'|'dropped'; + rendered: NotifyDeliveryRendered; + metadata: Record; // includes channelType, target, previewProvider, traceId, and provider-specific entries + createdAt: string; + sentAt?: string; + completedAt?: string; +} +``` + --- ## 6) State, caching & real‑time @@ -184,7 +206,7 @@ export interface VexConsensus { * **Huge tables** rendered with **virtual scrolling** (CDK Virtual Scroll); sort/filter performed client‑side for ≤ 20k rows; beyond that, server‑side queries via BOM‑Index endpoints. * **Component row** shows purl, version, origin (OS pkg / metadata / linker / attested), licenses, and **used** badge (Usage view). -* **Diff**: compact heatmap per layer; clicking opens a right‑pane with evidence: introducing paths, file hashes, VEX notes (from Vexer consensus) and links to advisories (Feedser). +* **Diff**: compact heatmap per layer; clicking opens a right‑pane with evidence: introducing paths, file hashes, VEX notes (from Excititor consensus) and links to advisories (Concelier). --- diff --git a/docs/ARCHITECTURE_ZASTAVA.md b/docs/ARCHITECTURE_ZASTAVA.md index 60f017ae..419a008b 100644 --- a/docs/ARCHITECTURE_ZASTAVA.md +++ b/docs/ARCHITECTURE_ZASTAVA.md @@ -1,451 +1,451 @@ -# component_architecture_zastava.md — **Stella Ops Zastava** (2025Q4) - -> **Scope.** Implementation‑ready architecture for **Zastava**: the **runtime inspector/enforcer** that watches real workloads, detects drift from the scanned baseline, verifies image/SBOM/attestation posture, and (optionally) **admits/blocks** deployments. Includes Kubernetes & plain‑Docker topologies, data contracts, APIs, security posture, performance targets, test matrices, and failure modes. - ---- - -## 0) Mission & boundaries - -**Mission.** Give operators **ground‑truth** from running environments and a **fast guardrail** before workloads land: - -* **Observer:** inventory containers, entrypoints actually executed, and DSOs actually loaded; verify **image signature**, **SBOM referrers**, and **attestation** presence; detect **drift** (unexpected processes/paths) and **policy violations**; publish **runtime events** to Scanner.WebService. -* **Admission (optional):** Kubernetes ValidatingAdmissionWebhook that enforces minimal posture (signed images, SBOM availability, known base images, policy PASS) **pre‑flight**. - -**Boundaries.** - -* Zastava **does not** compute SBOMs and does not sign; it **consumes** Scanner/WebService outputs and **enforces** backend policy verdicts. -* Zastava can **request** a delta scan when the baseline is missing/stale, but scanning is done by **Scanner.Worker**. -* On non‑K8s Docker hosts, Zastava runs as a host service with **observer‑only** features. - ---- - -## 1) Topology & processes - -### 1.1 Components (Kubernetes) - -``` -stellaops/zastava-observer # DaemonSet on every node (read-only host mounts) -stellaops/zastava-webhook # ValidatingAdmissionWebhook (Deployment, 2+ replicas) -``` - -### 1.2 Components (Docker/VM) - -``` -stellaops/zastava-agent # System service; watch Docker events; observer only -``` - -### 1.3 Dependencies - -* **Authority** (OIDC): short OpToks (DPoP/mTLS) for API calls to Scanner.WebService. -* **Scanner.WebService**: `/runtime/events` ingestion; `/policy/runtime` fetch. -* **OCI Registry** (optional): for direct referrers/sig checks if not delegated to backend. -* **Container runtime**: containerd/CRI‑O/Docker (read interfaces only). -* **Kubernetes API** (watch Pods in cluster; validating webhook). -* **Host mounts** (K8s DaemonSet): `/proc`, `/var/lib/containerd` (or CRI‑O), `/run/containerd/containerd.sock` (optional, read‑only). - ---- - -## 2) Data contracts - -### 2.1 Runtime event (observer → Scanner.WebService) - -```json -{ - "eventId": "9f6a…", - "when": "2025-10-17T12:34:56Z", - "kind": "CONTAINER_START|CONTAINER_STOP|DRIFT|POLICY_VIOLATION|ATTESTATION_STATUS", - "tenant": "tenant-01", - "node": "ip-10-0-1-23", - "runtime": { "engine": "containerd", "version": "1.7.19" }, - "workload": { - "platform": "kubernetes", - "namespace": "payments", - "pod": "api-7c9fbbd8b7-ktd84", - "container": "api", - "containerId": "containerd://...", - "imageRef": "ghcr.io/acme/api@sha256:abcd…", - "owner": { "kind": "Deployment", "name": "api" } - }, - "process": { - "pid": 12345, - "entrypoint": ["/entrypoint.sh", "--serve"], - "entryTrace": [ - {"file":"/entrypoint.sh","line":3,"op":"exec","target":"/usr/bin/python3"}, - {"file":"","op":"python","target":"/opt/app/server.py"} - ] - }, - "loadedLibs": [ - { "path": "/lib/x86_64-linux-gnu/libssl.so.3", "inode": 123456, "sha256": "…"}, - { "path": "/usr/lib/x86_64-linux-gnu/libcrypto.so.3", "inode": 123457, "sha256": "…"} - ], - "posture": { - "imageSigned": true, - "sbomReferrer": "present|missing", - "attestation": { "uuid": "rekor-uuid", "verified": true } - }, - "delta": { - "baselineImageDigest": "sha256:abcd…", - "changedFiles": ["/opt/app/server.py"], // optional quick signal - "newBinaries": [{ "path":"/usr/local/bin/helper","sha256":"…" }] - }, - "evidence": [ - {"signal":"procfs.maps","value":"/lib/.../libssl.so.3@0x7f..."}, - {"signal":"cri.task.inspect","value":"pid=12345"}, - {"signal":"registry.referrers","value":"sbom: application/vnd.cyclonedx+json"} - ] -} -``` - -### 2.2 Admission decision (webhook → API server) - -```json -{ - "admissionId": "…", - "namespace": "payments", - "podSpecDigest": "sha256:…", - "images": [ - { - "name": "ghcr.io/acme/api:1.2.3", - "resolved": "ghcr.io/acme/api@sha256:abcd…", - "signed": true, - "hasSbomReferrers": true, - "policyVerdict": "pass|warn|fail", - "reasons": ["unsigned base image", "missing SBOM"] - } - ], - "decision": "Allow|Deny", - "ttlSeconds": 300 -} -``` - ---- - -## 3) Observer — node agent (DaemonSet) - -### 3.1 Responsibilities - -* **Watch** container lifecycle (start/stop) via CRI (`/run/containerd/containerd.sock` gRPC read‑only) or `/var/log/containers/*.log` tail fallback. -* **Resolve** container → image digest, mount point rootfs. -* **Trace entrypoint**: attach **short‑lived** nsenter/exec to PID 1 in container, parse shell for `exec` chain (bounded depth), record **terminal program**. -* **Sample loaded libs**: read `/proc//maps` and `exe` symlink to collect **actually loaded** DSOs; compute **sha256** for each mapped file (bounded count/size). -* **Posture check** (cheap): - - * Image signature presence (if cosign policies are local; else ask backend). - * SBOM **referrers** presence (HEAD to registry, optional). - * Rekor UUID known (query Scanner.WebService by image digest). -* **Publish runtime events** to Scanner.WebService `/runtime/events` (batch & compress). -* **Request delta scan** if: no SBOM in catalog OR base differs from known baseline. - -### 3.2 Privileges & mounts (K8s) - -* **SecurityContext:** `runAsUser: 0`, `readOnlyRootFilesystem: true`, `allowPrivilegeEscalation: false`. -* **Capabilities:** `CAP_SYS_PTRACE` (optional if using nsenter trace), `CAP_DAC_READ_SEARCH`. -* **Host mounts (read‑only):** - - * `/proc` (host) → `/host/proc` - * `/run/containerd/containerd.sock` (or CRI‑O socket) - * `/var/lib/containerd/io.containerd.runtime.v2.task` (rootfs paths & pids) -* **Networking:** cluster‑internal egress to Scanner.WebService only. -* **Rate limits:** hard caps for bytes hashed and file count per container to avoid noisy tenants. - -### 3.3 Event batching - -* Buffer ND‑JSON; flush by **N events** or **2 s**. -* Backpressure: local disk ring buffer (50 MB default) if Scanner is temporarily unavailable; drop oldest after cap with **metrics** and **warning** event. - ---- - -## 4) Admission Webhook (Kubernetes) - -### 4.1 Gate criteria - -Configurable policy (fetched from backend and cached): - -* **Image signature**: must be cosign‑verifiable to configured key(s) or keyless identities. -* **SBOM availability**: at least one **CycloneDX** referrer or **Scanner.WebService** catalog entry. -* **Scanner policy verdict**: backend `PASS` required for namespaces/labels matching rules; allow `WARN` if configured. -* **Registry allowlists/denylists**. -* **Tag bans** (e.g., `:latest`). -* **Base image allowlists** (by digest). - -### 4.2 Flow - -```mermaid -sequenceDiagram - autonumber - participant K8s as API Server - participant WH as Zastava Webhook - participant SW as Scanner.WebService - - K8s->>WH: AdmissionReview(Pod) - WH->>WH: Resolve images to digests (remote HEAD/pull if needed) - WH->>SW: POST /policy/runtime { digests, namespace, labels } - SW-->>WH: { per-image: {signed, hasSbom, verdict, reasons}, ttl } - alt All pass - WH-->>K8s: AdmissionResponse(Allow, ttl) - else Any fail (enforce=true) - WH-->>K8s: AdmissionResponse(Deny, message) - end -``` - -**Caching:** Per‑digest result cached `ttlSeconds` (default 300 s). **Fail‑open** or **fail‑closed** is configurable per namespace. - -### 4.3 TLS & HA - -* Webhook has its own **serving cert** signed by cluster CA (or custom cert + CA bundle on configuration). -* Deployment ≥ 2 replicas; **leaderless**; stateless. - ---- - -## 5) Backend integration (Scanner.WebService) - -### 5.1 Ingestion endpoint - -`POST /api/v1/scanner/runtime/events` *(OpTok + DPoP/mTLS)* - -* Validates event schema; enforces rate caps by tenant/node; persists to **Mongo** (`runtime.events` capped collection or regular with TTL). -* Performs **correlation**: - - * Attach nearest **image SBOM** (inventory/usage) and **BOM‑Index** if known. - * If unknown/missing, schedule **delta scan** and return `202 Accepted`. -* Emits **derived signals** (usedByEntrypoint per component based on `/proc//maps`). - -### 5.2 Policy decision API (for webhook) - -`POST /api/v1/scanner/policy/runtime` - -Request: - -```json -{ - "namespace": "payments", - "labels": { "app": "api", "env": "prod" }, - "images": ["ghcr.io/acme/api@sha256:...", "ghcr.io/acme/nginx@sha256:..."] -} -``` - -Response: - -```json -{ - "ttlSeconds": 300, - "results": { - "ghcr.io/acme/api@sha256:...": { - "signed": true, - "hasSbom": true, - "policyVerdict": "pass", - "reasons": [], - "rekor": { "uuid": "..." } - }, - "ghcr.io/acme/nginx@sha256:...": { - "signed": false, - "hasSbom": false, - "policyVerdict": "fail", - "reasons": ["unsigned", "missing SBOM"] - } - } -} -``` - ---- - -## 6) Configuration (YAML) - -```yaml -zastava: - mode: - observer: true - webhook: true - authority: - issuer: "https://authority.internal" - aud: ["scanner","zastava"] # tokens for backend and self-id - backend: - url: "https://scanner-web.internal" - connectTimeoutMs: 500 - requestTimeoutMs: 1500 - retry: { attempts: 3, backoffMs: 200 } - runtime: - engine: "auto" # containerd|cri-o|docker|auto - procfs: "/host/proc" - collect: - entryTrace: true - loadedLibs: true - maxLibs: 256 - maxHashBytesPerContainer: 64_000_000 - maxDepth: 48 - admission: - enforce: true - failOpenNamespaces: ["dev", "test"] - verify: - imageSignature: true - sbomReferrer: true - scannerPolicyPass: true - cacheTtlSeconds: 300 - resolveTags: true # do remote digest resolution for tag-only images - limits: - eventsPerSecond: 50 - burst: 200 - perNodeQueue: 10_000 - security: - mounts: - containerdSock: "/run/containerd/containerd.sock:ro" - proc: "/proc:/host/proc:ro" - runtimeState: "/var/lib/containerd:ro" -``` - ---- - -## 7) Security posture - -* **AuthN/Z**: Authority OpToks (DPoP preferred) to backend; webhook does **not** require client auth from API server (K8s handles). -* **Least privileges**: read‑only host mounts; optional `CAP_SYS_PTRACE`; **no** host networking; **no** write mounts. -* **Isolation**: never exec untrusted code; nsenter only to **read** `/proc/`. -* **Data minimization**: do not exfiltrate env vars or command arguments unless policy explicitly enables diagnostic mode. -* **Rate limiting**: per‑node caps; per‑tenant caps at backend. -* **Hard caps**: bytes hashed, files inspected, depth of shell parsing. - ---- - -## 8) Metrics, logs, tracing - -**Observer** - -* `zastava.events_emitted_total{kind}` -* `zastava.proc_maps_samples_total{result}` -* `zastava.entrytrace_depth{p99}` -* `zastava.hash_bytes_total` -* `zastava.buffer_drops_total` - -**Webhook** - -* `zastava.admission_requests_total{decision}` -* `zastava.admission_latency_seconds` -* `zastava.cache_hits_total` -* `zastava.backend_failures_total` - -**Logs** (structured): node, pod, image digest, decision, reasons. -**Tracing**: spans for observe→batch→post; webhook request→resolve→respond. - ---- - -## 9) Performance & scale targets - -* **Observer**: ≤ **30 ms** to sample `/proc//maps` and compute quick hashes for ≤ 64 files; ≤ **200 ms** for full library set (256 libs). -* **Webhook**: P95 ≤ **8 ms** with warm cache; ≤ **50 ms** with one backend round‑trip. -* **Throughput**: 1k admission requests/min/replica; 5k runtime events/min/node with batching. - ---- - -## 10) Drift detection model - -**Signals** - -* **Process drift**: terminal program differs from **EntryTrace** baseline. -* **Library drift**: loaded DSOs not present in **Usage** SBOM view. -* **Filesystem drift**: new executable files under `/usr/local/bin`, `/opt`, `/app` with **mtime** after image creation. -* **Network drift** (optional): listening sockets on unexpected ports (from policy). - -**Action** - -* Emit `DRIFT` event with evidence; backend can **auto‑queue** a delta scan; policy may **escalate** to alert/block (Admission cannot block already‑running pods; rely on K8s policies/PodSecurity or operator action). - ---- - -## 11) Test matrix - -* **Engines**: containerd, CRI‑O, Docker; ensure PID resolution and rootfs mapping. -* **EntryTrace**: bash features (case, if, run‑parts, `.`/`source`), language launchers (python/node/java). -* **Procfs**: multiple arches, musl/glibc images; static binaries (maps minimal). -* **Admission**: unsigned images, missing SBOM referrers, tag‑only images, digest resolution, backend latency, cache TTL. -* **Perf/soak**: 500 Pods/node churn; webhook under HPA growth. -* **Security**: attempt privilege escalation disabled, read‑only mounts enforced, rate‑limit abuse. -* **Failure injection**: backend down (observer buffers, webhook fail‑open/closed), registry throttling, containerd socket unavailable. - ---- - -## 12) Failure modes & responses - -| Condition | Observer behavior | Webhook behavior | -| ------------------------------- | ---------------------------------------------- | ------------------------------------------------------ | -| Backend unreachable | Buffer to disk; drop after cap; emit metric | **Fail‑open/closed** per namespace config | -| PID vanished mid‑sample | Retry once; emit partial evidence | N/A | -| CRI socket missing | Fallback to K8s events only (reduced fidelity) | N/A | -| Registry digest resolve blocked | Defer to backend; mark `resolve=unknown` | Deny or allow per `resolveTags` & `failOpenNamespaces` | -| Excessive events | Apply local rate limit, coalesce | N/A | - ---- - -## 13) Deployment notes (K8s) - -**DaemonSet (snippet):** - -```yaml -apiVersion: apps/v1 -kind: DaemonSet -metadata: { name: zastava-observer, namespace: stellaops } -spec: - template: - spec: - serviceAccountName: zastava - hostPID: true - containers: - - name: observer - image: stellaops/zastava-observer:2.3 - securityContext: - runAsUser: 0 - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - capabilities: { add: ["SYS_PTRACE","DAC_READ_SEARCH"] } - volumeMounts: - - { name: proc, mountPath: /host/proc, readOnly: true } - - { name: containerd-sock, mountPath: /run/containerd/containerd.sock, readOnly: true } - - { name: containerd-state, mountPath: /var/lib/containerd, readOnly: true } - volumes: - - { name: proc, hostPath: { path: /proc } } - - { name: containerd-sock, hostPath: { path: /run/containerd/containerd.sock } } - - { name: containerd-state, hostPath: { path: /var/lib/containerd } } -``` - -**Webhook (snippet):** - -```yaml -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -webhooks: -- name: gate.zastava.stella-ops.org - admissionReviewVersions: ["v1"] - sideEffects: None - failurePolicy: Ignore # or Fail - rules: - - operations: ["CREATE","UPDATE"] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] - clientConfig: - service: - namespace: stellaops - name: zastava-webhook - path: /admit - caBundle: -``` - ---- - -## 14) Implementation notes - -* **Language**: Rust (observer) for low‑latency `/proc` parsing; Go/.NET viable too. Webhook can be .NET 10 for parity with backend. -* **CRI drivers**: pluggable (`containerd`, `cri-o`, `docker`). Prefer CRI over parsing logs. -* **Shell parser**: re‑use Scanner.EntryTrace grammar for consistent results (compile to WASM if observer is Rust/Go). -* **Hashing**: `BLAKE3` for speed locally, then convert to `sha256` (or compute `sha256` directly when budget allows). -* **Resilience**: never block container start; observer is **passive**; only webhook decides allow/deny. - ---- - -## 15) Roadmap - -* **eBPF** option for syscall/library load tracing (kernel‑level, opt‑in). -* **Windows containers** support (ETW providers, loaded modules). -* **Network posture** checks: listening ports vs policy. -* **Live **used‑by‑entrypoint** synthesis**: send compact bitset diff to backend to tighten Usage view. -* **Admission dry‑run** dashboards (simulate block lists before enforcing). - +# component_architecture_zastava.md — **Stella Ops Zastava** (2025Q4) + +> **Scope.** Implementation‑ready architecture for **Zastava**: the **runtime inspector/enforcer** that watches real workloads, detects drift from the scanned baseline, verifies image/SBOM/attestation posture, and (optionally) **admits/blocks** deployments. Includes Kubernetes & plain‑Docker topologies, data contracts, APIs, security posture, performance targets, test matrices, and failure modes. + +--- + +## 0) Mission & boundaries + +**Mission.** Give operators **ground‑truth** from running environments and a **fast guardrail** before workloads land: + +* **Observer:** inventory containers, entrypoints actually executed, and DSOs actually loaded; verify **image signature**, **SBOM referrers**, and **attestation** presence; detect **drift** (unexpected processes/paths) and **policy violations**; publish **runtime events** to Scanner.WebService. +* **Admission (optional):** Kubernetes ValidatingAdmissionWebhook that enforces minimal posture (signed images, SBOM availability, known base images, policy PASS) **pre‑flight**. + +**Boundaries.** + +* Zastava **does not** compute SBOMs and does not sign; it **consumes** Scanner/WebService outputs and **enforces** backend policy verdicts. +* Zastava can **request** a delta scan when the baseline is missing/stale, but scanning is done by **Scanner.Worker**. +* On non‑K8s Docker hosts, Zastava runs as a host service with **observer‑only** features. + +--- + +## 1) Topology & processes + +### 1.1 Components (Kubernetes) + +``` +stellaops/zastava-observer # DaemonSet on every node (read-only host mounts) +stellaops/zastava-webhook # ValidatingAdmissionWebhook (Deployment, 2+ replicas) +``` + +### 1.2 Components (Docker/VM) + +``` +stellaops/zastava-agent # System service; watch Docker events; observer only +``` + +### 1.3 Dependencies + +* **Authority** (OIDC): short OpToks (DPoP/mTLS) for API calls to Scanner.WebService. +* **Scanner.WebService**: `/runtime/events` ingestion; `/policy/runtime` fetch. +* **OCI Registry** (optional): for direct referrers/sig checks if not delegated to backend. +* **Container runtime**: containerd/CRI‑O/Docker (read interfaces only). +* **Kubernetes API** (watch Pods in cluster; validating webhook). +* **Host mounts** (K8s DaemonSet): `/proc`, `/var/lib/containerd` (or CRI‑O), `/run/containerd/containerd.sock` (optional, read‑only). + +--- + +## 2) Data contracts + +### 2.1 Runtime event (observer → Scanner.WebService) + +```json +{ + "eventId": "9f6a…", + "when": "2025-10-17T12:34:56Z", + "kind": "CONTAINER_START|CONTAINER_STOP|DRIFT|POLICY_VIOLATION|ATTESTATION_STATUS", + "tenant": "tenant-01", + "node": "ip-10-0-1-23", + "runtime": { "engine": "containerd", "version": "1.7.19" }, + "workload": { + "platform": "kubernetes", + "namespace": "payments", + "pod": "api-7c9fbbd8b7-ktd84", + "container": "api", + "containerId": "containerd://...", + "imageRef": "ghcr.io/acme/api@sha256:abcd…", + "owner": { "kind": "Deployment", "name": "api" } + }, + "process": { + "pid": 12345, + "entrypoint": ["/entrypoint.sh", "--serve"], + "entryTrace": [ + {"file":"/entrypoint.sh","line":3,"op":"exec","target":"/usr/bin/python3"}, + {"file":"","op":"python","target":"/opt/app/server.py"} + ] + }, + "loadedLibs": [ + { "path": "/lib/x86_64-linux-gnu/libssl.so.3", "inode": 123456, "sha256": "…"}, + { "path": "/usr/lib/x86_64-linux-gnu/libcrypto.so.3", "inode": 123457, "sha256": "…"} + ], + "posture": { + "imageSigned": true, + "sbomReferrer": "present|missing", + "attestation": { "uuid": "rekor-uuid", "verified": true } + }, + "delta": { + "baselineImageDigest": "sha256:abcd…", + "changedFiles": ["/opt/app/server.py"], // optional quick signal + "newBinaries": [{ "path":"/usr/local/bin/helper","sha256":"…" }] + }, + "evidence": [ + {"signal":"procfs.maps","value":"/lib/.../libssl.so.3@0x7f..."}, + {"signal":"cri.task.inspect","value":"pid=12345"}, + {"signal":"registry.referrers","value":"sbom: application/vnd.cyclonedx+json"} + ] +} +``` + +### 2.2 Admission decision (webhook → API server) + +```json +{ + "admissionId": "…", + "namespace": "payments", + "podSpecDigest": "sha256:…", + "images": [ + { + "name": "ghcr.io/acme/api:1.2.3", + "resolved": "ghcr.io/acme/api@sha256:abcd…", + "signed": true, + "hasSbomReferrers": true, + "policyVerdict": "pass|warn|fail", + "reasons": ["unsigned base image", "missing SBOM"] + } + ], + "decision": "Allow|Deny", + "ttlSeconds": 300 +} +``` + +--- + +## 3) Observer — node agent (DaemonSet) + +### 3.1 Responsibilities + +* **Watch** container lifecycle (start/stop) via CRI (`/run/containerd/containerd.sock` gRPC read‑only) or `/var/log/containers/*.log` tail fallback. +* **Resolve** container → image digest, mount point rootfs. +* **Trace entrypoint**: attach **short‑lived** nsenter/exec to PID 1 in container, parse shell for `exec` chain (bounded depth), record **terminal program**. +* **Sample loaded libs**: read `/proc//maps` and `exe` symlink to collect **actually loaded** DSOs; compute **sha256** for each mapped file (bounded count/size). +* **Posture check** (cheap): + + * Image signature presence (if cosign policies are local; else ask backend). + * SBOM **referrers** presence (HEAD to registry, optional). + * Rekor UUID known (query Scanner.WebService by image digest). +* **Publish runtime events** to Scanner.WebService `/runtime/events` (batch & compress). +* **Request delta scan** if: no SBOM in catalog OR base differs from known baseline. + +### 3.2 Privileges & mounts (K8s) + +* **SecurityContext:** `runAsUser: 0`, `readOnlyRootFilesystem: true`, `allowPrivilegeEscalation: false`. +* **Capabilities:** `CAP_SYS_PTRACE` (optional if using nsenter trace), `CAP_DAC_READ_SEARCH`. +* **Host mounts (read‑only):** + + * `/proc` (host) → `/host/proc` + * `/run/containerd/containerd.sock` (or CRI‑O socket) + * `/var/lib/containerd/io.containerd.runtime.v2.task` (rootfs paths & pids) +* **Networking:** cluster‑internal egress to Scanner.WebService only. +* **Rate limits:** hard caps for bytes hashed and file count per container to avoid noisy tenants. + +### 3.3 Event batching + +* Buffer ND‑JSON; flush by **N events** or **2 s**. +* Backpressure: local disk ring buffer (50 MB default) if Scanner is temporarily unavailable; drop oldest after cap with **metrics** and **warning** event. + +--- + +## 4) Admission Webhook (Kubernetes) + +### 4.1 Gate criteria + +Configurable policy (fetched from backend and cached): + +* **Image signature**: must be cosign‑verifiable to configured key(s) or keyless identities. +* **SBOM availability**: at least one **CycloneDX** referrer or **Scanner.WebService** catalog entry. +* **Scanner policy verdict**: backend `PASS` required for namespaces/labels matching rules; allow `WARN` if configured. +* **Registry allowlists/denylists**. +* **Tag bans** (e.g., `:latest`). +* **Base image allowlists** (by digest). + +### 4.2 Flow + +```mermaid +sequenceDiagram + autonumber + participant K8s as API Server + participant WH as Zastava Webhook + participant SW as Scanner.WebService + + K8s->>WH: AdmissionReview(Pod) + WH->>WH: Resolve images to digests (remote HEAD/pull if needed) + WH->>SW: POST /policy/runtime { digests, namespace, labels } + SW-->>WH: { per-image: {signed, hasSbom, verdict, reasons}, ttl } + alt All pass + WH-->>K8s: AdmissionResponse(Allow, ttl) + else Any fail (enforce=true) + WH-->>K8s: AdmissionResponse(Deny, message) + end +``` + +**Caching:** Per‑digest result cached `ttlSeconds` (default 300 s). **Fail‑open** or **fail‑closed** is configurable per namespace. + +### 4.3 TLS & HA + +* Webhook has its own **serving cert** signed by cluster CA (or custom cert + CA bundle on configuration). +* Deployment ≥ 2 replicas; **leaderless**; stateless. + +--- + +## 5) Backend integration (Scanner.WebService) + +### 5.1 Ingestion endpoint + +`POST /api/v1/scanner/runtime/events` *(OpTok + DPoP/mTLS)* + +* Validates event schema; enforces rate caps by tenant/node; persists to **Mongo** (`runtime.events` capped collection or regular with TTL). +* Performs **correlation**: + + * Attach nearest **image SBOM** (inventory/usage) and **BOM‑Index** if known. + * If unknown/missing, schedule **delta scan** and return `202 Accepted`. +* Emits **derived signals** (usedByEntrypoint per component based on `/proc//maps`). + +### 5.2 Policy decision API (for webhook) + +`POST /api/v1/scanner/policy/runtime` + +Request: + +```json +{ + "namespace": "payments", + "labels": { "app": "api", "env": "prod" }, + "images": ["ghcr.io/acme/api@sha256:...", "ghcr.io/acme/nginx@sha256:..."] +} +``` + +Response: + +```json +{ + "ttlSeconds": 300, + "results": { + "ghcr.io/acme/api@sha256:...": { + "signed": true, + "hasSbom": true, + "policyVerdict": "pass", + "reasons": [], + "rekor": { "uuid": "..." } + }, + "ghcr.io/acme/nginx@sha256:...": { + "signed": false, + "hasSbom": false, + "policyVerdict": "fail", + "reasons": ["unsigned", "missing SBOM"] + } + } +} +``` + +--- + +## 6) Configuration (YAML) + +```yaml +zastava: + mode: + observer: true + webhook: true + authority: + issuer: "https://authority.internal" + aud: ["scanner","zastava"] # tokens for backend and self-id + backend: + url: "https://scanner-web.internal" + connectTimeoutMs: 500 + requestTimeoutMs: 1500 + retry: { attempts: 3, backoffMs: 200 } + runtime: + engine: "auto" # containerd|cri-o|docker|auto + procfs: "/host/proc" + collect: + entryTrace: true + loadedLibs: true + maxLibs: 256 + maxHashBytesPerContainer: 64_000_000 + maxDepth: 48 + admission: + enforce: true + failOpenNamespaces: ["dev", "test"] + verify: + imageSignature: true + sbomReferrer: true + scannerPolicyPass: true + cacheTtlSeconds: 300 + resolveTags: true # do remote digest resolution for tag-only images + limits: + eventsPerSecond: 50 + burst: 200 + perNodeQueue: 10_000 + security: + mounts: + containerdSock: "/run/containerd/containerd.sock:ro" + proc: "/proc:/host/proc:ro" + runtimeState: "/var/lib/containerd:ro" +``` + +--- + +## 7) Security posture + +* **AuthN/Z**: Authority OpToks (DPoP preferred) to backend; webhook does **not** require client auth from API server (K8s handles). +* **Least privileges**: read‑only host mounts; optional `CAP_SYS_PTRACE`; **no** host networking; **no** write mounts. +* **Isolation**: never exec untrusted code; nsenter only to **read** `/proc/`. +* **Data minimization**: do not exfiltrate env vars or command arguments unless policy explicitly enables diagnostic mode. +* **Rate limiting**: per‑node caps; per‑tenant caps at backend. +* **Hard caps**: bytes hashed, files inspected, depth of shell parsing. + +--- + +## 8) Metrics, logs, tracing + +**Observer** + +* `zastava.events_emitted_total{kind}` +* `zastava.proc_maps_samples_total{result}` +* `zastava.entrytrace_depth{p99}` +* `zastava.hash_bytes_total` +* `zastava.buffer_drops_total` + +**Webhook** + +* `zastava.admission_requests_total{decision}` +* `zastava.admission_latency_seconds` +* `zastava.cache_hits_total` +* `zastava.backend_failures_total` + +**Logs** (structured): node, pod, image digest, decision, reasons. +**Tracing**: spans for observe→batch→post; webhook request→resolve→respond. + +--- + +## 9) Performance & scale targets + +* **Observer**: ≤ **30 ms** to sample `/proc//maps` and compute quick hashes for ≤ 64 files; ≤ **200 ms** for full library set (256 libs). +* **Webhook**: P95 ≤ **8 ms** with warm cache; ≤ **50 ms** with one backend round‑trip. +* **Throughput**: 1k admission requests/min/replica; 5k runtime events/min/node with batching. + +--- + +## 10) Drift detection model + +**Signals** + +* **Process drift**: terminal program differs from **EntryTrace** baseline. +* **Library drift**: loaded DSOs not present in **Usage** SBOM view. +* **Filesystem drift**: new executable files under `/usr/local/bin`, `/opt`, `/app` with **mtime** after image creation. +* **Network drift** (optional): listening sockets on unexpected ports (from policy). + +**Action** + +* Emit `DRIFT` event with evidence; backend can **auto‑queue** a delta scan; policy may **escalate** to alert/block (Admission cannot block already‑running pods; rely on K8s policies/PodSecurity or operator action). + +--- + +## 11) Test matrix + +* **Engines**: containerd, CRI‑O, Docker; ensure PID resolution and rootfs mapping. +* **EntryTrace**: bash features (case, if, run‑parts, `.`/`source`), language launchers (python/node/java). +* **Procfs**: multiple arches, musl/glibc images; static binaries (maps minimal). +* **Admission**: unsigned images, missing SBOM referrers, tag‑only images, digest resolution, backend latency, cache TTL. +* **Perf/soak**: 500 Pods/node churn; webhook under HPA growth. +* **Security**: attempt privilege escalation disabled, read‑only mounts enforced, rate‑limit abuse. +* **Failure injection**: backend down (observer buffers, webhook fail‑open/closed), registry throttling, containerd socket unavailable. + +--- + +## 12) Failure modes & responses + +| Condition | Observer behavior | Webhook behavior | +| ------------------------------- | ---------------------------------------------- | ------------------------------------------------------ | +| Backend unreachable | Buffer to disk; drop after cap; emit metric | **Fail‑open/closed** per namespace config | +| PID vanished mid‑sample | Retry once; emit partial evidence | N/A | +| CRI socket missing | Fallback to K8s events only (reduced fidelity) | N/A | +| Registry digest resolve blocked | Defer to backend; mark `resolve=unknown` | Deny or allow per `resolveTags` & `failOpenNamespaces` | +| Excessive events | Apply local rate limit, coalesce | N/A | + +--- + +## 13) Deployment notes (K8s) + +**DaemonSet (snippet):** + +```yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: { name: zastava-observer, namespace: stellaops } +spec: + template: + spec: + serviceAccountName: zastava + hostPID: true + containers: + - name: observer + image: stellaops/zastava-observer:2.3 + securityContext: + runAsUser: 0 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: { add: ["SYS_PTRACE","DAC_READ_SEARCH"] } + volumeMounts: + - { name: proc, mountPath: /host/proc, readOnly: true } + - { name: containerd-sock, mountPath: /run/containerd/containerd.sock, readOnly: true } + - { name: containerd-state, mountPath: /var/lib/containerd, readOnly: true } + volumes: + - { name: proc, hostPath: { path: /proc } } + - { name: containerd-sock, hostPath: { path: /run/containerd/containerd.sock } } + - { name: containerd-state, hostPath: { path: /var/lib/containerd } } +``` + +**Webhook (snippet):** + +```yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +webhooks: +- name: gate.zastava.stella-ops.org + admissionReviewVersions: ["v1"] + sideEffects: None + failurePolicy: Ignore # or Fail + rules: + - operations: ["CREATE","UPDATE"] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + clientConfig: + service: + namespace: stellaops + name: zastava-webhook + path: /admit + caBundle: +``` + +--- + +## 14) Implementation notes + +* **Language**: Rust (observer) for low‑latency `/proc` parsing; Go/.NET viable too. Webhook can be .NET 10 for parity with backend. +* **CRI drivers**: pluggable (`containerd`, `cri-o`, `docker`). Prefer CRI over parsing logs. +* **Shell parser**: re‑use Scanner.EntryTrace grammar for consistent results (compile to WASM if observer is Rust/Go). +* **Hashing**: `BLAKE3` for speed locally, then convert to `sha256` (or compute `sha256` directly when budget allows). +* **Resilience**: never block container start; observer is **passive**; only webhook decides allow/deny. + +--- + +## 15) Roadmap + +* **eBPF** option for syscall/library load tracing (kernel‑level, opt‑in). +* **Windows containers** support (ETW providers, loaded modules). +* **Network posture** checks: listening ports vs policy. +* **Live **used‑by‑entrypoint** synthesis**: send compact bitset diff to backend to tighten Usage view. +* **Admission dry‑run** dashboards (simulate block lists before enforcing). + diff --git a/docs/EXCITITOR_SCORRING.md b/docs/EXCITITOR_SCORRING.md new file mode 100644 index 00000000..a11ae41f --- /dev/null +++ b/docs/EXCITITOR_SCORRING.md @@ -0,0 +1,104 @@ +## Status + +This document tracks the future-looking risk scoring model for Excititor. The calculation below is not active yet; Sprint 7 work will add the required schema fields, policy controls, and services. Until that ships, Excititor emits consensus statuses without numeric scores. + +## Scoring model (target state) + +**S = Gate(VEX_status) × W_trust(source) × [Severity_base × (1 + α·KEV + β·EPSS)]** + +* **Gate(VEX_status)**: `affected`/`under_investigation` → 1, `not_affected`/`fixed` → 0. A trusted “not affected” or “fixed” still zeroes the score. +* **W_trust(source)**: normalized policy weight (baseline 0‒1). Policies may opt into >1 boosts for signed vendor feeds once Phase 1 closes. +* **Severity_base**: canonical numeric severity from Concelier (CVSS or org-defined scale). +* **KEV flag**: 0/1 boost when CISA Known Exploited Vulnerabilities applies. +* **EPSS**: probability [0,1]; bounded multiplier. +* **α, β**: configurable coefficients (default α=0.25, β=0.5) stored in policy. + +Safeguards: freeze boosts when product identity is unknown, clamp outputs ≥0, and log every factor in the audit trail. + +## Implementation roadmap + +| Phase | Scope | Artifacts | +| --- | --- | --- | +| **Phase 1 – Schema foundations** | Extend Excititor consensus/claims and Concelier canonical advisories with severity, KEV, EPSS, and expose α/β + weight ceilings in policy. | Sprint 7 tasks `EXCITITOR-CORE-02-001`, `EXCITITOR-POLICY-02-001`, `EXCITITOR-STORAGE-02-001`, `FEEDCORE-ENGINE-07-001`. | +| **Phase 2 – Deterministic score engine** | Implement a scoring component that executes alongside consensus and persists score envelopes with hashes. | Planned task `EXCITITOR-CORE-02-002` (backlog). | +| **Phase 3 – Surfacing & enforcement** | Expose scores via WebService/CLI, integrate with Concelier noise priors, and enforce policy-based suppressions. | To be scheduled after Phase 2. | + +## Policy controls (Phase 1) + +Operators tune scoring inputs through the Excititor policy document: + +```yaml +excititor: + policy: + weights: + vendor: 1.10 # per-tier weight + ceiling: 1.40 # max clamp applied to tiers and overrides (1.0‒5.0) + providerOverrides: + trusted.vendor: 1.35 + scoring: + alpha: 0.30 # KEV boost coefficient (defaults to 0.25) + beta: 0.60 # EPSS boost coefficient (defaults to 0.50) +``` + +* All weights (tiers + overrides) are clamped to `[0, weights.ceiling]` with structured warnings when a value is out of range or not a finite number. +* `weights.ceiling` itself is constrained to `[1.0, 5.0]`, preserving prior behaviour when omitted. +* `scoring.alpha` / `scoring.beta` accept non-negative values up to 5.0; values outside the range fall back to defaults and surface diagnostics to operators. + +## Data model (after Phase 1) + +```json +{ + "vulnerabilityId": "CVE-2025-12345", + "product": "pkg:name@version", + "consensus": { + "status": "affected", + "policyRevisionId": "rev-12", + "policyDigest": "0D9AEC…" + }, + "signals": { + "severity": {"scheme": "CVSS:3.1", "score": 7.5}, + "kev": true, + "epss": 0.40 + }, + "policy": { + "weight": 1.15, + "alpha": 0.25, + "beta": 0.5 + }, + "score": { + "value": 10.8, + "generatedAt": "2025-11-05T14:12:30Z", + "audit": [ + "gate:affected", + "weight:1.15", + "severity:7.5", + "kev:1", + "epss:0.40" + ] + } +} +``` + +## Operational guidance + +* **Inputs**: Concelier delivers severity/KEV/EPSS via the advisory event log; Excititor connectors load VEX statements. Policy owns trust tiers and coefficients. +* **Processing**: the scoring engine (Phase 2) runs next to consensus, storing results with deterministic hashes so exports and attestations can reference them. +* **Consumption**: WebService/CLI will return consensus plus score; scanners may suppress findings only when policy-authorized VEX gating and signed score envelopes agree. + +## Pseudocode (Phase 2 preview) + +```python +def risk_score(gate, weight, severity, kev, epss, alpha, beta, freeze_boosts=False): + if gate == 0: + return 0 + if freeze_boosts: + kev, epss = 0, 0 + boost = 1 + alpha * kev + beta * epss + return max(0, weight * severity * boost) +``` + +## FAQ + +* **Can operators opt out?** Set α=β=0 or keep weights ≤1.0 via policy. +* **What about missing signals?** Treat them as zero and log the omission. +* **When will this ship?** Phase 1 is planned for Sprint 7; later phases depend on connector coverage and attestation delivery. diff --git a/docs/README.md b/docs/README.md index 884b0798..43c91e5d 100755 --- a/docs/README.md +++ b/docs/README.md @@ -34,28 +34,34 @@ Everything here is open‑source and versioned — when you check out a git ta ### Reference & concepts - **05 – [System Requirements Specification](05_SYSTEM_REQUIREMENTS_SPEC.md)** - **07 – [High‑Level Architecture](07_HIGH_LEVEL_ARCHITECTURE.md)** +- **08 – [Architecture Decision Records](adr/index.md)** - **08 – Module Architecture Dossiers** - [Scanner](ARCHITECTURE_SCANNER.md) - - [Feedser](ARCHITECTURE_FEEDSER.md) - - [Vexer](ARCHITECTURE_VEXER.md) - - [Signer](ARCHITECTURE_SIGNER.md) - - [Attestor](ARCHITECTURE_ATTESTOR.md) - - [Authority](ARCHITECTURE_AUTHORITY.md) - - [CLI](ARCHITECTURE_CLI.md) - - [Web UI](ARCHITECTURE_UI.md) - - [Zastava Runtime](ARCHITECTURE_ZASTAVA.md) - - [Release & Operations](ARCHITECTURE_DEVOPS.md) -- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** + - [Concelier](ARCHITECTURE_CONCELIER.md) + - [Excititor](ARCHITECTURE_EXCITITOR.md) + - [Excititor Mirrors](ARCHITECTURE_EXCITITOR_MIRRORS.md) + - [Signer](ARCHITECTURE_SIGNER.md) + - [Attestor](ARCHITECTURE_ATTESTOR.md) + - [Authority](ARCHITECTURE_AUTHORITY.md) + - [Notify](ARCHITECTURE_NOTIFY.md) + - [Scheduler](ARCHITECTURE_SCHEDULER.md) + - [CLI](ARCHITECTURE_CLI.md) + - [Web UI](ARCHITECTURE_UI.md) + - [Zastava Runtime](ARCHITECTURE_ZASTAVA.md) + - [Release & Operations](ARCHITECTURE_DEVOPS.md) +- **09 – [API & CLI Reference](09_API_CLI_REFERENCE.md)** - **10 – [Plug‑in SDK Guide](10_PLUGIN_SDK_GUIDE.md)** -- **10 – [Feedser CLI Quickstart](10_FEEDSER_CLI_QUICKSTART.md)** -- **30 – [Vexer Connector Packaging Guide](dev/30_VEXER_CONNECTOR_GUIDE.md)** -- **30 – Developer Templates** - - [Vexer Connector Skeleton](dev/templates/vexer-connector/) -- **11 – [Authority Service](11_AUTHORITY.md)** -- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** -- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** -- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** -- **30 – [Fixture Maintenance](dev/fixtures.md)** +- **10 – [Concelier CLI Quickstart](10_CONCELIER_CLI_QUICKSTART.md)** +- **10 – [BuildX Generator Quickstart](dev/BUILDX_PLUGIN_QUICKSTART.md)** +- **10 – [Scanner Cache Configuration](dev/SCANNER_CACHE_CONFIGURATION.md)** +- **30 – [Excititor Connector Packaging Guide](dev/30_EXCITITOR_CONNECTOR_GUIDE.md)** +- **30 – Developer Templates** + - [Excititor Connector Skeleton](dev/templates/excititor-connector/) +- **11 – [Authority Service](11_AUTHORITY.md)** +- **11 – [Data Schemas](11_DATA_SCHEMAS.md)** +- **12 – [Performance Workbook](12_PERFORMANCE_WORKBOOK.md)** +- **13 – [Release‑Engineering Playbook](13_RELEASE_ENGINEERING_PLAYBOOK.md)** +- **30 – [Fixture Maintenance](dev/fixtures.md)** ### User & operator guides - **14 – [Glossary](14_GLOSSARY_OF_TERMS.md)** @@ -64,18 +70,19 @@ Everything here is open‑source and versioned — when you check out a git ta - **18 – [Coding Standards](18_CODING_STANDARDS.md)** - **19 – [Test‑Suite Overview](19_TEST_SUITE_OVERVIEW.md)** - **21 – [Install Guide](21_INSTALL_GUIDE.md)** -- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** -- **23 – [FAQ](23_FAQ_MATRIX.md)** +- **22 – [CI/CD Recipes Library](ci/20_CI_RECIPES.md)** +- **23 – [FAQ](23_FAQ_MATRIX.md)** - **24 – [Offline Update Kit Admin Guide](24_OFFLINE_KIT.md)** -- **25 – [Feedser Apple Connector Operations](ops/feedser-apple-operations.md)** -- **26 – [Authority Key Rotation Playbook](ops/authority-key-rotation.md)** -- **27 – [Feedser CCCS Connector Operations](ops/feedser-cccs-operations.md)** -- **28 – [Feedser CISA ICS Connector Operations](ops/feedser-icscisa-operations.md)** -- **29 – [Feedser CERT-Bund Connector Operations](ops/feedser-certbund-operations.md)** -- **30 – [Feedser MSRC Connector – AAD Onboarding](ops/feedser-msrc-operations.md)** +- **25 – [Mirror Operations Runbook](ops/concelier-mirror-operations.md)** +- **26 – [Concelier Apple Connector Operations](ops/concelier-apple-operations.md)** +- **27 – [Authority Key Rotation Playbook](ops/authority-key-rotation.md)** +- **28 – [Concelier CCCS Connector Operations](ops/concelier-cccs-operations.md)** +- **29 – [Concelier CISA ICS Connector Operations](ops/concelier-icscisa-operations.md)** +- **30 – [Concelier CERT-Bund Connector Operations](ops/concelier-certbund-operations.md)** +- **31 – [Concelier MSRC Connector – AAD Onboarding](ops/concelier-msrc-operations.md)** ### Legal & licence -- **31 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** +- **32 – [Legal & Quota FAQ](29_LEGAL_FAQ_QUOTA.md)** diff --git a/docs/TASKS.md b/docs/TASKS.md index 2b14c092..ad2480e8 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3,12 +3,19 @@ | ID | Status | Owner(s) | Depends on | Description | Exit Criteria | |----|--------|----------|------------|-------------|---------------| | DOC7.README-INDEX | DONE (2025-10-17) | Docs Guild | — | Refresh index docs (docs/README.md + root README) after architecture dossier split and Offline Kit overhaul. | ✅ ToC reflects new component architecture docs; ✅ root README highlights updated doc set; ✅ Offline Kit guide linked correctly. | -| DOC4.AUTH-PDG | REVIEW | Docs Guild, Plugin Team | PLG6.DOC | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | ✅ PR merged with polish; ✅ Diagram committed; ✅ Slack handoff posted. | +| DOC4.AUTH-PDG | DONE (2025-10-19) | Docs Guild, Plugin Team | PLG6.DOC | Copy-edit `docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md`, export lifecycle diagram, add LDAP RFC cross-link. | ✅ PR merged with polish; ✅ Diagram committed; ✅ Slack handoff posted. | | DOC1.AUTH | DONE (2025-10-12) | Docs Guild, Authority Core | CORE5B.DOC | Draft `docs/11_AUTHORITY.md` covering architecture, configuration, bootstrap flows. | ✅ Architecture + config sections approved by Core; ✅ Samples reference latest options; ✅ Offline note added. | -| DOC3.Feedser-Authority | DONE (2025-10-12) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Feedser authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated with audit runbook references; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. | -| DOC5.Feedser-Runbook | DONE (2025-10-12) | Docs Guild | DOC3.Feedser-Authority | Produce dedicated Feedser authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. | -| FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Feedser conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/feedser-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Feedser team. | -| FEEDDOCS-DOCS-05-002 | DONE (2025-10-16) | Docs Guild, Feedser Ops | FEEDDOCS-DOCS-05-001 | Ops sign-off captured: conflict runbook circulated, alert thresholds tuned, and rollout decisions documented in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised using `docs/ops/feedser-authority-audit-runbook.md`; ✅ change-log entry linked from runbook once GHSA/NVD/OSV regression fixtures land. | +| DOC3.Concelier-Authority | DONE (2025-10-12) | Docs Guild, DevEx | FSR4 | Polish operator/runbook sections (DOC3/DOC5) to document Concelier authority rollout, bypass logging, and enforcement checklist. | ✅ DOC3/DOC5 updated with audit runbook references; ✅ enforcement deadline highlighted; ✅ Docs guild sign-off. | +| DOC5.Concelier-Runbook | DONE (2025-10-12) | Docs Guild | DOC3.Concelier-Authority | Produce dedicated Concelier authority audit runbook covering log fields, monitoring recommendations, and troubleshooting steps. | ✅ Runbook published; ✅ linked from DOC3/DOC5; ✅ alerting guidance included. | +| FEEDDOCS-DOCS-05-001 | DONE (2025-10-11) | Docs Guild | FEEDMERGE-ENGINE-04-001, FEEDMERGE-ENGINE-04-002 | Publish Concelier conflict resolution runbook covering precedence workflow, merge-event auditing, and Sprint 3 metrics. | ✅ `docs/ops/concelier-conflict-resolution.md` committed; ✅ metrics/log tables align with latest merge code; ✅ Ops alert guidance handed to Concelier team. | +| FEEDDOCS-DOCS-05-002 | DONE (2025-10-16) | Docs Guild, Concelier Ops | FEEDDOCS-DOCS-05-001 | Ops sign-off captured: conflict runbook circulated, alert thresholds tuned, and rollout decisions documented in change log. | ✅ Ops review recorded; ✅ alert thresholds finalised using `docs/ops/concelier-authority-audit-runbook.md`; ✅ change-log entry linked from runbook once GHSA/NVD/OSV regression fixtures land. | +| DOCS-ADR-09-001 | DONE (2025-10-19) | Docs Guild, DevEx | — | Establish ADR process (`docs/adr/0000-template.md`) and document usage guidelines. | Template published; README snippet linking ADR process; announcement posted (`docs/updates/2025-10-18-docs-guild.md`). | +| DOCS-EVENTS-09-002 | DONE (2025-10-19) | Docs Guild, Platform Events | SCANNER-EVENTS-15-201 | Publish event schema catalog (`docs/events/`) for `scanner.report.ready@1`, `scheduler.rescan.delta@1`, `attestor.logged@1`. | Schemas validated (Ajv CI hooked); docs/events/README summarises usage; Platform Events notified via `docs/updates/2025-10-18-docs-guild.md`. | +| DOCS-EVENTS-09-003 | DONE (2025-10-19) | Docs Guild | DOCS-EVENTS-09-002 | Add human-readable envelope field references and canonical payload samples for published events, including offline validation workflow. | Tables explain common headers/payload segments; versioned sample payloads committed; README links to validation instructions and samples. | +| DOCS-EVENTS-09-004 | DONE (2025-10-19) | Docs Guild, Scanner WebService | SCANNER-EVENTS-15-201 | Refresh scanner event docs to mirror DSSE-backed report fields, document `scanner.scan.completed`, and capture canonical sample validation. | Schemas updated for new payload shape; README references DSSE reuse and validation test; samples align with emitted events. | +| PLATFORM-EVENTS-09-401 | DONE (2025-10-19) | Platform Events Guild | DOCS-EVENTS-09-003 | Embed canonical event samples into contract/integration tests and ensure CI validates payloads against published schemas. | Notify/Scheduler contract suites exercise samples; CI job validates samples with `ajv-cli`; Platform Events changelog notes coverage. | +| RUNTIME-GUILD-09-402 | DONE (2025-10-19) | Runtime Guild | SCANNER-POLICY-09-107 | Confirm Scanner WebService surfaces `quietedFindingCount` and progress hints to runtime consumers; document readiness checklist. | Runtime verification run captures enriched payload; checklist/doc updates merged; stakeholders acknowledge availability. | +| DOCS-RUNTIME-17-004 | TODO | Docs Guild, Runtime Guild | SCANNER-EMIT-17-701, ZASTAVA-OBS-17-005, DEVOPS-REL-17-002 | Document build-id workflows: SBOM exposure, runtime event payloads, debug-store layout, and operator guidance for symbol retrieval. | Architecture + operator docs updated with build-id sections, examples show `readelf` output + debuginfod usage, references linked from Offline Kit/Release guides. | > Update statuses (TODO/DOING/REVIEW/DONE/BLOCKED) as progress changes. Keep guides in sync with configuration samples under `etc/`. diff --git a/docs/adr/0000-template.md b/docs/adr/0000-template.md new file mode 100644 index 00000000..803dc08a --- /dev/null +++ b/docs/adr/0000-template.md @@ -0,0 +1,34 @@ +# ADR-0000: Title + +## Status +Proposed + +## Date +YYYY-MM-DD + +## Authors +- Name (team) + +## Deciders +- Names of approvers / reviewers + +## Context +- What decision needs to be made? +- What are the forces (requirements, constraints, stakeholders)? +- Why now? What triggers the ADR? + +## Decision +- Summary of the chosen option. +- Key rationale points. + +## Consequences +- Positive/negative consequences. +- Follow-up actions or tasks. +- Rollback plan or re-evaluation criteria. + +## Alternatives Considered +- Option A — pros/cons. +- Option B — pros/cons. + +## References +- Links to related ADRs, issues, documents. diff --git a/docs/adr/index.md b/docs/adr/index.md new file mode 100644 index 00000000..bbe4c9b6 --- /dev/null +++ b/docs/adr/index.md @@ -0,0 +1,41 @@ +# Architecture Decision Records (ADRs) + +Architecture Decision Records document long-lived choices that shape StellaOps architecture, security posture, and operator experience. They complement RFCs by capturing the final call and the context that led to it. + +## When to file an ADR +- Decisions that affect cross-module contracts, persistence models, or external interfaces. +- Security or compliance controls with on-going operational ownership. +- Rollout strategies that require coordination across guilds or sprints. +- Reversals or deprecations of previously accepted ADRs. + +Small, module-local refactors that do not modify public behaviour can live in commit messages instead. + +## Workflow at a glance +1. Copy `docs/adr/0000-template.md` to `docs/adr/NNNN-short-slug.md` with a zero-padded sequence (see **Numbering**). +2. Fill in context, decision, consequences, and alternatives. Include links to RFCs, issues, benchmarks, or experiments. +3. Request async review from the impacted guilds. Capture sign-offs in the **Deciders** field. +4. Merge the ADR with the code/config changes (or in a preparatory PR). +5. Announce the accepted ADR in the Docs Guild channel or sprint notes so downstream teams can consume it. + +## Numbering and status +- Use zero-padded integers (e.g., `0001`, `0002`) in file names and the document header. Increment from the highest existing number. +- Valid statuses: `Proposed`, `Accepted`, `Rejected`, `Deprecated`, `Superseded`. Update the status when follow-up work lands. +- When an ADR supersedes another, link them in both documents’ **References** sections. + +## Review expectations +- Highlight edge-case handling, trade-offs, and determinism requirements. +- Include operational checklists for any new runtime path (quota updates, schema migrations, credential rotation, etc.). +- Attach diagrams under `docs/adr/assets/` when visuals improve comprehension. +- Add TODO tasks for follow-up work in the relevant module’s `TASKS.md` and link them from the ADR. + +## Verification checklist +- [ ] `Status`, `Date`, `Authors`, and `Deciders` populated. +- [ ] Links to code/config PRs or experiments recorded under **References**. +- [ ] Consequences call out migration or rollback steps. +- [ ] Announcement posted to Docs Guild updates (or sprint log). + +## Related resources +- [Docs Guild Task Board](../TASKS.md) +- [High-Level Architecture Overview](../07_HIGH_LEVEL_ARCHITECTURE.md) +- [Coding Standards](../18_CODING_STANDARDS.md) +- [Release Engineering Playbook](../13_RELEASE_ENGINEERING_PLAYBOOK.md) diff --git a/docs/artifacts/bom-index/README.md b/docs/artifacts/bom-index/README.md new file mode 100644 index 00000000..b08ce47f --- /dev/null +++ b/docs/artifacts/bom-index/README.md @@ -0,0 +1,50 @@ +# StellaOps BOM Index (`bom-index@1`) + +The BOM index is a deterministic, offline-friendly sidecar that accelerates queries for +layer-to-component membership and entrypoint usage. It is emitted alongside CycloneDX +SBOMs and consumed by Scheduler/Notify services. + +## File Layout + +Binary little-endian encoding, organised as the following sections: + +1. **Header** + - `magic` (`byte[7]`): ASCII `"BOMIDX1"` identifier. + - `version` (`uint16`): current value `1`. + - `flags` (`uint16`): bit `0` set when entrypoint usage bitmaps are present. + - `imageDigestLength` (`uint16`) + UTF-8 digest string (e.g. `sha256:...`). + - `generatedAt` (`int64`): microseconds since Unix epoch. + - `layerCount` (`uint32`), `componentCount` (`uint32`), `entrypointCount` (`uint32`). + +2. **Layer Table** + - For each layer: `length` (`uint16`) + UTF-8 layer digest (canonical order, base image → top layer). + +3. **Component Table** + - For each component: `length` (`uint16`) + UTF-8 identity (CycloneDX purl when available, otherwise canonical key). + +4. **Component ↦ Layer Bitmaps** + - For each component (matching table order): + - `bitmapLength` (`uint32`). + - Roaring bitmap payload (`Collections.Special.RoaringBitmap.Serialize`) encoding layer indexes that introduce or retain the component. + +5. **Entrypoint Table** *(optional; present when `flags & 0x1 == 1`)* + - For each unique entrypoint/launcher string: `length` (`uint16`) + UTF-8 value (sorted ordinally). + +6. **Component ↦ Entrypoint Bitmaps** *(optional)* + - For each component: roaring bitmap whose set bits reference entrypoint indexes used by EntryTrace. Empty bitmap (`length == 0`) indicates the component is not part of any resolved entrypoint closure. + +## Determinism Guarantees + +* Layer, component, and entrypoint tables are strictly ordered (base → top layer, lexicographically for components and entrypoints). +* Roaring bitmaps are optimised prior to serialisation and always produced from sorted indexes. +* Header timestamp is normalised to microsecond precision using UTC. + +## Sample + +`sample-index.bin` is generated from the integration fixture used in unit tests. It contains: + +* 2 layers: `sha256:layer1`, `sha256:layer2`. +* 3 components: `pkg:npm/a`, `pkg:npm/b`, `pkg:npm/c`. +* Entrypoint bitmaps for `/app/start.sh` and `/app/init.sh`. + +The sample can be decoded with the `BomIndexBuilder` unit tests or any RoaringBitmap implementation compatible with `Collections.Special.RoaringBitmap`. diff --git a/docs/artifacts/bom-index/sample-index.bin b/docs/artifacts/bom-index/sample-index.bin new file mode 100644 index 00000000..95af48b5 Binary files /dev/null and b/docs/artifacts/bom-index/sample-index.bin differ diff --git a/docs/assets/authority/authority-rate-limit-flow.svg b/docs/assets/authority/authority-rate-limit-flow.svg index f2db1be5..af02dde1 100644 --- a/docs/assets/authority/authority-rate-limit-flow.svg +++ b/docs/assets/authority/authority-rate-limit-flow.svg @@ -1,105 +1,105 @@ - - Authority rate limit and lockout flow - - - - - - - - - - - - - - - Client / App - - Authority Host - - Rate Limiter - - Standard Plug-in - - Credential Store - - - - - POST /token request - client_id + subject creds - - - - - Authority middleware - enriches context tags - - - - - Rate limiter window - client_id + IP keyed - - - - - Quota exceeded? - emit Retry-After & tags - - - - - Quota OK - pass remaining tokens - - - - - Verify credentials - hash compare + audit tags - - - - - Load lockout state - deterministic counters - - - - - Lockout threshold hit? - follow dedup precedence - - - - - Issue tokens or errors - include limiter metadata - - - - - - - - - - - - - - - - 429 path → add `authority.client_id`, `authority.remote_ip` tags for dashboards. - Lockout path → reuse precedence strategy from Feedser dedup (see DEDUP_CONFLICTS_RESOLUTION_ALGO.md). - + + Authority rate limit and lockout flow + + + + + + + + + + + + + + + Client / App + + Authority Host + + Rate Limiter + + Standard Plug-in + + Credential Store + + + + + POST /token request + client_id + subject creds + + + + + Authority middleware + enriches context tags + + + + + Rate limiter window + client_id + IP keyed + + + + + Quota exceeded? + emit Retry-After & tags + + + + + Quota OK + pass remaining tokens + + + + + Verify credentials + hash compare + audit tags + + + + + Load lockout state + deterministic counters + + + + + Lockout threshold hit? + follow dedup precedence + + + + + Issue tokens or errors + include limiter metadata + + + + + + + + + + + + + + + + 429 path → add `authority.client_id`, `authority.remote_ip` tags for dashboards. + Lockout path → reuse precedence strategy from Concelier dedup (see DEDUP_CONFLICTS_RESOLUTION_ALGO.md). + diff --git a/docs/ci/20_CI_RECIPES.md b/docs/ci/20_CI_RECIPES.md index 4ad86464..dd17e6dc 100755 --- a/docs/ci/20_CI_RECIPES.md +++ b/docs/ci/20_CI_RECIPES.md @@ -241,7 +241,59 @@ jobs: --- -## 4 · Troubleshooting cheat‑sheet +## 4 · Docs CI (Gitea Actions & Offline Mirror) + +StellaOps ships a dedicated Docs workflow at `.gitea/workflows/docs.yml`. When mirroring the pipeline offline or running it locally, install the same toolchain so markdown linting, schema validation, and HTML preview stay deterministic. + +### 4.1 Toolchain bootstrap + +```bash +# Node.js 20.x is required; install once per runner +npm install --no-save \ + markdown-link-check \ + remark-cli \ + remark-preset-lint-recommended \ + ajv \ + ajv-cli \ + ajv-formats + +# Python 3.11+ powers the preview renderer +python -m pip install --upgrade pip +python -m pip install markdown pygments +``` + +**Offline tip.** Add the packages above to your artifact mirror (for example `ops/devops/offline-kit.json`) so runners can install them via `npm --offline` / `pip --no-index`. + +### 4.2 Schema validation step + +Ajv compiles every event schema to guard against syntax or format regressions. The workflow uses `ajv-formats` for UUID/date-time support. + +```bash +for schema in docs/events/*.json; do + npx ajv compile -c ajv-formats -s "$schema" +done +``` + +Run this loop before committing schema changes. For new references, append `-r additional-file.json` so CI and local runs stay aligned. + +### 4.3 Preview build + +```bash +python scripts/render_docs.py --source docs --output artifacts/docs-preview --clean +``` + +Host the resulting bundle via any static file server for review (for example `python -m http.server`). + +### 4.4 Publishing checklist + +- [ ] Toolchain installs succeed without hitting the public internet (mirror or cached tarballs). +- [ ] Ajv validation passes for `scanner.report.ready@1`, `scheduler.rescan.delta@1`, `attestor.logged@1`. +- [ ] Markdown link check (`npx markdown-link-check`) reports no broken references. +- [ ] Preview bundle archived (or attached) for stakeholders. + +--- + +## 5 · Troubleshooting cheat‑sheet | Symptom | Root cause | First things to try | | ------------------------------------- | --------------------------- | --------------------------------------------------------------- | @@ -253,6 +305,7 @@ jobs: --- -### Change log - -* **2025‑08‑04** – Variable clean‑up, removed Docker‑socket & cache mounts, added Jenkins / CircleCI / Gitea examples, clarified Option B comment. +### Change log + +* **2025‑10‑18** – Documented Docs CI toolchain (Ajv validation, static preview) and offline checklist. +* **2025‑08‑04** – Variable clean‑up, removed Docker‑socket & cache mounts, added Jenkins / CircleCI / Gitea examples, clarified Option B comment. diff --git a/docs/feedser-connector-research-20251011.md b/docs/concelier-connector-research-20251011.md similarity index 80% rename from docs/feedser-connector-research-20251011.md rename to docs/concelier-connector-research-20251011.md index 97936ede..84de135e 100644 --- a/docs/feedser-connector-research-20251011.md +++ b/docs/concelier-connector-research-20251011.md @@ -1,4 +1,4 @@ -# Feedser Connector Research – 2025-10-11 +# Concelier Connector Research – 2025-10-11 Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national/vendor connectors in scope. Use alongside each module’s `TASKS.md` notes. @@ -8,7 +8,7 @@ Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national ## CCCS (Canada) - JSON endpoint (`https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=&content_type=cccs_threat`) returns ~5 100 records per language; `page=` still works for segmented pulls and the earliest `date_created` seen is 2018‑06‑08 (EN) / 2018‑06‑08 (FR). Use an explicit `User-Agent` to avoid 403 responses. -- Follow-up: telemetry, sanitiser coverage, and backfill procedures are documented in `docs/ops/feedser-cccs-operations.md` (2025‑10‑15). Adjust `maxEntriesPerFetch` when performing historical sweeps so cursor state remains responsive. +- Follow-up: telemetry, sanitiser coverage, and backfill procedures are documented in `docs/ops/concelier-cccs-operations.md` (2025‑10‑15). Adjust `maxEntriesPerFetch` when performing historical sweeps so cursor state remains responsive. ## CERT-Bund (Germany) - `https://wid.cert-bund.de/content/public/securityAdvisory/rss` responds 200 without cookies (≈250-item window, German taxonomy). Detail links load an Angular SPA that fetches JSON behind the bootstrap session. @@ -16,7 +16,7 @@ Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national - Historical advisories accessible through the SPA search/export endpoints once the `XSRF-TOKEN` cookie (exposed via `GET /portal/api/security/csrf`) is supplied with the `X-XSRF-TOKEN` header: - `POST /portal/api/securityadvisory/search` (`{"page":N,"size":100,"sort":["published,desc"]}`) pages data back to 2014. - `GET /portal/api/securityadvisory/export?format=json&from=YYYY-MM-DD` emits JSON bundles suitable for Offline Kit mirrors. -- Locale note: content is German-only; Feedser preserves `language=de` and Docs will publish a CERT-Bund glossary so operators can bridge terminology without machine translation. +- Locale note: content is German-only; Concelier preserves `language=de` and Docs will publish a CERT-Bund glossary so operators can bridge terminology without machine translation. ## KISA / KNVD (Korea) - `https://knvd.krcert.or.kr/rss/securityInfo.do` and `/rss/securityNotice.do` return UTF-8 RSS (10-item window) with `detailDos.do?IDX=` links. No cookies required for feed fetch. @@ -24,11 +24,11 @@ Snapshot of direct network checks performed on 2025-10-11 (UTC) for the national ## BDU (Russia / FSTEC) - Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml/json`) return 403/404; TLS chain requires Russian Trusted Sub CA and WAF expects additional headers. -- Next actions: acquire official PEM chain, point `feedser:httpClients:source.bdu:trustedRootPaths` (or `feedser:sources:bdu:http:trustedRootPaths`) at the Offline Kit PEM, keep `allowInvalidCertificates=false`, script session bootstrap, then capture RSS/HTML schema for parser work. +- Next actions: acquire official PEM chain, point `concelier:httpClients:source.bdu:trustedRootPaths` (or `concelier:sources:bdu:http:trustedRootPaths`) at the Offline Kit PEM, keep `allowInvalidCertificates=false`, script session bootstrap, then capture RSS/HTML schema for parser work. ## NKTsKI / cert.gov.ru (Russia) - `https://cert.gov.ru/rss/advisories.xml` served via Bitrix returns 403/404 even with `Accept-Language: ru-RU`; TLS chain also requires Russian trust anchors. -- Next actions: source trust store, configure `feedser:httpClients:source.nkcki:trustedRootPaths` (Offline Kit root via `feedser:offline:root`), prepare proxy fallback, and once accessible document taxonomy/retention plus attachment handling. +- Next actions: source trust store, configure `concelier:httpClients:source.nkcki:trustedRootPaths` (Offline Kit root via `concelier:offline:root`), prepare proxy fallback, and once accessible document taxonomy/retention plus attachment handling. ## CISA ICS (United States) - `curl -I https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml` returns HTTP 403 + `x-reference-error` (Akamai). Same for legacy feed paths. diff --git a/docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md b/docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md new file mode 100644 index 00000000..bd2362bb --- /dev/null +++ b/docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md @@ -0,0 +1,220 @@ +# Excititor Connector Packaging Guide + +> **Audience:** teams implementing new Excititor provider plug‑ins (CSAF feeds, +> OpenVEX attestations, etc.) +> **Prerequisites:** read `docs/ARCHITECTURE_EXCITITOR.md` and the module +> `AGENTS.md` in `src/StellaOps.Excititor.Connectors.Abstractions/`. + +The Excititor connector SDK gives you: + +- `VexConnectorBase` – deterministic logging, SHA‑256 helpers, time provider. +- `VexConnectorOptionsBinder` – strongly typed YAML/JSON configuration binding. +- `IVexConnectorOptionsValidator` – custom validation hooks (offline defaults, auth invariants). +- `VexConnectorDescriptor` & metadata helpers for consistent telemetry. + +This guide explains how to package a connector so the Excititor Worker/WebService +can load it via the plugin host. + +--- + +## 1. Project layout + +Start from the template under +`docs/dev/templates/excititor-connector/`. It contains: + +``` +Excititor.MyConnector/ +├── src/ +│ ├── Excititor.MyConnector.csproj +│ ├── MyConnectorOptions.cs +│ ├── MyConnector.cs +│ └── MyConnectorPlugin.cs +└── manifest/ + └── connector.manifest.yaml +``` + +Key points: + +- Target `net10.0`, enable `TreatWarningsAsErrors`, reference the + `StellaOps.Excititor.Connectors.Abstractions` project (or NuGet once published). +- Keep project ID prefix `StellaOps.Excititor.Connectors.` so the + plugin loader can discover it with the default search pattern. + +### 1.1 csproj snippet + +```xml + + + net10.0 + enable + enable + true + + + + + +``` + +Adjust the `ProjectReference` for your checkout (or switch to a NuGet package +once published). + +--- + +## 2. Implement the connector + +1. **Options model** – create an options POCO with data-annotation attributes. + Bind it via `VexConnectorOptionsBinder.Bind` in your connector + constructor or `ValidateAsync`. +2. **Validator** – implement `IVexConnectorOptionsValidator` to add + complex checks (e.g., ensure both `clientId` and `clientSecret` are present). +3. **Connector** – inherit from `VexConnectorBase`. Implement: + - `ValidateAsync` – run binder/validators, log configuration summary. + - `FetchAsync` – stream raw documents to `context.RawSink`. + - `NormalizeAsync` – convert raw documents into `VexClaimBatch` via + format-specific normalizers (`context.Normalizers`). +4. **Plugin adapter** – expose the connector via a plugin entry point so the + host can instantiate it. + +### 2.1 Options binding example + +```csharp +public sealed class MyConnectorOptions +{ + [Required] + [Url] + public string CatalogUri { get; set; } = default!; + + [Required] + public string ApiKey { get; set; } = default!; + + [Range(1, 64)] + public int MaxParallelRequests { get; set; } = 4; +} + +public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList errors) + { + if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + errors.Add("CatalogUri must use HTTPS."); + } + } +} +``` + +Bind inside the connector: + +```csharp +private readonly MyConnectorOptions _options; + +public MyConnector(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider timeProvider) + : base(descriptor, logger, timeProvider) +{ + // `settings` comes from the orchestrator; validators registered via DI. + _options = VexConnectorOptionsBinder.Bind( + descriptor, + VexConnectorSettings.Empty, + validators: new[] { new MyConnectorOptionsValidator() }); +} +``` + +Replace `VexConnectorSettings.Empty` with the actual settings from context +inside `ValidateAsync`. + +--- + +## 3. Plugin adapter & manifest + +Create a simple plugin class that implements +`StellaOps.Plugin.IConnectorPlugin`. The Worker/WebService plugin host uses +this contract today. + +```csharp +public sealed class MyConnectorPlugin : IConnectorPlugin +{ + private static readonly VexConnectorDescriptor Descriptor = + new("excititor:my-provider", VexProviderKind.Vendor, "My Provider VEX"); + + public string Name => Descriptor.DisplayName; + + public bool IsAvailable(IServiceProvider services) => true; // inject feature flags if needed + + public IFeedConnector Create(IServiceProvider services) + { + var logger = services.GetRequiredService>(); + var timeProvider = services.GetRequiredService(); + return new MyConnector(Descriptor, logger, timeProvider); + } +} +``` + +> **Note:** the Excititor Worker currently instantiates connectors through the +> shared `IConnectorPlugin` contract. Once a dedicated Excititor plugin interface +> lands you simply swap the base interface; the descriptor/connector code +> remains unchanged. + +Provide a manifest describing the assembly for operational tooling: + +```yaml +# manifest/connector.manifest.yaml +id: excititor-my-provider +assembly: StellaOps.Excititor.Connectors.MyProvider.dll +entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin +description: > + Official VEX feed for ExampleCorp products (CSAF JSON, daily updates). +tags: + - excititor + - csaf + - vendor +``` + +Store manifests under `/opt/stella/excititor/plugins//manifest/` in +production so the deployment tooling can inventory and verify plug‑ins. + +--- + +## 4. Packaging workflow + +1. `dotnet publish -c Release` → copy the published DLLs to + `/opt/stella/excititor/plugins//`. +2. Place `connector.manifest.yaml` next to the binaries. +3. Restart the Excititor Worker or WebService (hot reload not supported yet). +4. Verify logs: `VEX-ConnectorLoader` should list the connector descriptor. + +### 4.1 Offline kits + +- Add the connector folder (binaries + manifest) to the Offline Kit bundle. +- Include a `settings.sample.yaml` demonstrating offline-friendly defaults. +- Document any external dependencies (e.g., SHA mirrors) in the manifest `notes` + field. + +--- + +## 5. Testing checklist + +- Unit tests around options binding & validators. +- Integration tests (future `StellaOps.Excititor.Connectors.Abstractions.Tests`) + verifying deterministic logging scopes: + `logger.BeginScope` should produce `vex.connector.id`, `vex.connector.kind`, + and `vex.connector.operation`. +- Deterministic SHA tests: repeated `CreateRawDocument` calls with identical + content must return the same digest. + +--- + +## 6. Reference template + +See `docs/dev/templates/excititor-connector/` for the full quick‑start including: + +- Sample options class + validator. +- Connector implementation inheriting from `VexConnectorBase`. +- Plugin adapter + manifest. + +Copy the directory, rename namespaces/IDs, then iterate on provider-specific +logic. + +--- + +*Last updated: 2025-10-17* diff --git a/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md index 788f6044..6949965b 100644 --- a/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md +++ b/docs/dev/31_AUTHORITY_PLUGIN_DEVELOPER_GUIDE.md @@ -1,210 +1,212 @@ -# Authority Plug-in Developer Guide - -> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria. - -## 1. Overview -Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Feedser plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows: - -- **Use cases:** integrate corporate directories (LDAP/AD), delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation. -- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration and input data. -- **Ship targets:** target the same .NET 10 preview as the host, honour offline-first requirements, and provide clear diagnostics so operators can triage issues from `/ready`. - -## 2. Architecture Snapshot -Authority hosts follow a deterministic plug-in lifecycle. The flow below can be rendered as a sequence diagram in the final authored documentation, but all touchpoints are described here for offline viewers: - -1. **Configuration load** – `AuthorityPluginConfigurationLoader` resolves YAML manifests under `etc/authority.plugins/`. -2. **Assembly discovery** – the shared `PluginHost` scans `PluginBinaries/Authority` for `StellaOps.Authority.Plugin.*.dll` assemblies. -3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks. -4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints. - -![Authority plug-in lifecycle diagram](../assets/authority/authority-plugin-lifecycle.svg) - -_Source:_ `docs/assets/authority/authority-plugin-lifecycle.mmd` - -**Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process. - -## 3. Capability Metadata -Capability flags let the host reason about what your plug-in supports: - -- Declare capabilities in your descriptor using the string constants from `AuthorityPluginCapabilities` (`password`, `mfa`, `clientProvisioning`, `bootstrap`). The configuration loader now validates these tokens and rejects unknown values at startup. -- `AuthorityIdentityProviderCapabilities.FromCapabilities` projects those strings into strongly typed booleans (`SupportsPassword`, etc.). Authority Core will use these flags when wiring flows such as the password grant. Built-in plugins (e.g., Standard) will fail fast or force-enable required capabilities if the descriptor is misconfigured, so keep manifests accurate. -- Typical configuration (`etc/authority.plugins/standard.yaml`): - ```yaml - plugins: - descriptors: - standard: - assemblyName: "StellaOps.Authority.Plugin.Standard" - capabilities: - - password - - bootstrap - ``` -- Only declare a capability if the plug-in genuinely implements it. For example, if `SupportsClientProvisioning` is `true`, the plug-in must supply a working `IClientProvisioningStore`. - -**Operational reminder:** the Authority host surfaces capability summaries during startup (see `AuthorityIdentityProviderRegistry` log lines). Use those logs during smoke tests to ensure manifests align with expectations. - -**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected. - -**Password policy guardrails:** The Standard registrar logs a warning when a plug-in weakens the default password policy (minimum length or required character classes). Keep overrides at least as strong as the compiled defaults—operators treat the warning as an actionable security deviation. - -## 4. Project Scaffold -- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `true`. -- Minimum references: - - `StellaOps.Authority.Plugins.Abstractions` (contracts & capability helpers) - - `StellaOps.Plugin` (hosting/DI helpers) - - `StellaOps.Auth.*` libraries as needed for shared token utilities (optional today). -- Example `.csproj` (trimmed from `StellaOps.Authority.Plugin.Standard`): - ```xml - - - net10.0 - enable - true - true - - - - - - - ``` - (Add other references—e.g., MongoDB driver, shared auth libraries—according to your implementation.) - -## 5. Implementing `IAuthorityPluginRegistrar` -- Create a parameterless registrar class that returns your plug-in type name via `PluginType`. -- Use `AuthorityPluginRegistrationContext` to: - - Bind options (`AddOptions(pluginName).Bind(...)`). - - Register singletons for stores/enrichers using manifest metadata. - - Register any hosted bootstrap tasks (e.g., seed admin users). -- Always validate configuration inside `PostConfigure` and throw meaningful `InvalidOperationException` to fail fast during startup. -- Use the provided `ILoggerFactory` from DI; avoid static loggers or console writes. -- Example skeleton: - ```csharp - internal sealed class MyPluginRegistrar : IAuthorityPluginRegistrar - { - public string PluginType => "my-custom"; - - public void Register(AuthorityPluginRegistrationContext context) - { - var name = context.Plugin.Manifest.Name; - - context.Services.AddOptions(name) - .Bind(context.Plugin.Configuration) - .PostConfigure(opts => opts.Validate(name)); - - context.Services.AddSingleton(sp => - new MyIdentityProvider(context.Plugin, sp.GetRequiredService(), - sp.GetRequiredService(), - sp.GetRequiredService>())); - } - } - ``` - -## 6. Identity Provider Surface -- Implement `IIdentityProviderPlugin` to expose: - - `IUserCredentialStore` for password validation and user CRUD. - - `IClaimsEnricher` to append roles/attributes onto issued principals. - - Optional `IClientProvisioningStore` for machine-to-machine clients. - - `AuthorityIdentityProviderCapabilities` to advertise supported flows. -- Password guidance: - - Standard plug-in hashes via `ICryptoProvider` using Argon2id by default and emits PHC-compliant strings. Successful PBKDF2 logins trigger automatic rehashes so migrations complete gradually. See `docs/security/password-hashing.md` for tuning advice. - - Enforce password policies before hashing to avoid storing weak credentials. -- Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues. -- When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers. - -## 7. Configuration & Secrets -- Authority looks for manifests under `etc/authority.plugins/`. Each YAML file maps directly to a plug-in name. -- Support environment overrides using `STELLAOPS_AUTHORITY_PLUGINS__DESCRIPTORS____...`. -- Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory. -- Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`. -- Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning. -- `passwordHashing` inherits defaults from `authority.security.passwordHashing`. Override only when hardware constraints differ per plug-in: - ```yaml - passwordHashing: - algorithm: Argon2id - memorySizeInKib: 19456 - iterations: 2 - parallelism: 1 - ``` - Invalid values (≤0) fail fast during startup, and legacy PBKDF2 hashes rehash automatically once the new algorithm succeeds. - -### 7.1 Token Persistence Contract -- The host automatically persists every issued principal (access, refresh, device, authorization code) in `authority_tokens`. Plug-in code **must not** bypass this store; use the provided `IAuthorityTokenStore` helpers when implementing custom flows. -- When a plug-in disables a subject or client outside the standard handlers, call `IAuthorityTokenStore.UpdateStatusAsync(...)` for each affected token so revocation bundles stay consistent. -- Supply machine-friendly `revokedReason` codes (`compromised`, `rotation`, `policy`, `lifecycle`, etc.) and optional `revokedMetadata` entries when invalidating credentials. These flow straight into `revocation-bundle.json` and should remain deterministic. -- Token scopes should be normalised (trimmed, unique, ordinal sort) before returning from plug-in verification paths. `TokenPersistenceHandlers` will keep that ordering for downstream consumers. - -### 7.2 Claims & Enrichment Checklist -- Authority always sets the OpenID Connect basics: `sub`, `client_id`, `preferred_username`, optional `name`, and `role` (for password flows). Plug-ins must use `IClaimsEnricher` to append additional claims in a **deterministic** order (sort arrays, normalise casing) so resource servers can rely on stable shapes. -- Recommended enrichment keys: - - `stellaops.realm` – plug-in/tenant identifier so services can scope policies. - - `stellaops.subject.type` – values such as `human`, `service`, `bootstrap`. - - `groups` / `projects` – sorted arrays describing operator entitlements. -- Claims visible in tokens should mirror what `/token` and `/userinfo` emit. Avoid injecting sensitive PII directly; mark values with `ClassifiedString.Personal` inside the plug-in so audit sinks can tag them appropriately. -- For client-credential flows, remember to enrich both the client principal and the validation path (`TokenValidationHandlers`) so refresh flows keep the same metadata. - -### 7.3 Revocation Bundles & Reasons -- Use `IAuthorityRevocationStore` to record subject/client/token revocations when credentials are deleted or rotated. Stick to the standard categories (`token`, `subject`, `client`, `key`). -- Include a deterministic `reason` string and optional `reasonDescription` so operators understand *why* a subject was revoked when inspecting bundles offline. -- Plug-ins should populate `metadata` with stable keys (e.g., `revokedBy`, `sourcePlugin`, `ticketId`) to simplify SOC correlation. The keys must be lowercase, ASCII, and free of secrets—bundles are mirrored to air-gapped agents. - -## 8. Rate Limiting & Lockout Interplay -Rate limiting and account lockouts are complementary controls. Plug-ins must surface both deterministically so operators can correlate limiter hits with credential rejections. - -**Baseline quotas** (from `docs/dev/authority-rate-limit-tuning-outline.md`): - -| Endpoint | Default policy | Notes | -|----------|----------------|-------| -| `/token` | 30 requests / 60s, queue 0 | Drop to 10/60s for untrusted ranges; raise only with WAF + monitoring. | -| `/authorize` | 60 requests / 60s, queue 10 | Reduce carefully; interactive UX depends on headroom. | -| `/internal/*` | Disabled by default; recommended 5/60s when enabled | Keep queue 0 for bootstrap APIs. | - -**Retry metadata:** The middleware stamps `Retry-After` plus tags `authority.client_id`, `authority.remote_ip`, and `authority.endpoint`. Plug-ins should keep these tags intact when crafting responses or telemetry so dashboards remain consistent. - -**Lockout counters:** Treat lockouts as **subject-scoped** decisions. When multiple instances update counters, reuse the deterministic tie-breakers documented in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` (freshness overrides, precedence, and stable hashes) to avoid divergent lockout states across replicas. - -**Alerting hooks:** Emit structured logs/metrics when either the limiter or credential store rejects access. Suggested gauges include `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and any custom `auth.plugins..lockouts_total` counter. - -![Authority rate limit and lockout flow](../assets/authority/authority-rate-limit-flow.svg) - -_Source:_ `docs/assets/authority/authority-rate-limit-flow.mmd` - -## 9. Logging, Metrics, and Diagnostics -- Always log via the injected `ILogger`; include `pluginName` and correlation IDs where available. -- Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`). -- Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`. -- Emit metrics with stable names (`auth.plugins..*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes. - -## 10. Testing & Tooling -- Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template). -- Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable. -- Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `PluginBinaries/Authority` and verify `/ready`. -- Example verification snippet: - ```csharp - [Fact] - public async Task VerifyPasswordAsync_ReturnsSuccess() - { - var store = CreateCredentialStore(); - await store.UpsertUserAsync(new AuthorityUserRegistration("alice", "Pa55!", null, null, false, - Array.Empty(), new Dictionary()), CancellationToken.None); - - var result = await store.VerifyPasswordAsync("alice", "Pa55!", CancellationToken.None); - Assert.True(result.Succeeded); - Assert.True(result.User?.Roles.Count == 0); - } - ``` - -## 11. Packaging & Delivery -- Output assembly should follow `StellaOps.Authority.Plugin..dll` so the host’s search pattern picks it up. -- Place the compiled DLL plus dependencies under `PluginBinaries/Authority` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming). -- Document any external prerequisites (e.g., CA cert bundle) in your plug-in README. -- Update `etc/authority.plugins/.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts. - -## 12. Checklist & Handoff -- ✅ Capabilities declared and validated in automated tests. -- ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable. -- ✅ Local smoke test + unit/integration suites green (`dotnet test`). -- ✅ Operational docs updated: configuration keys, secrets guidance, troubleshooting. -- Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off. - ---- -Mermaid sources for the embedded diagrams live under `docs/assets/authority/`. Regenerate the SVG assets with your preferred renderer before committing future updates so the visuals stay in sync with the `.mmd` definitions. +# Authority Plug-in Developer Guide + +> **Status:** Updated 2025-10-11 (AUTHPLUG-DOCS-01-001) with lifecycle + limiter diagrams and refreshed rate-limit guidance aligned to PLG6 acceptance criteria. + +## 1. Overview +Authority plug-ins extend the **StellaOps Authority** service with custom identity providers, credential stores, and client-management logic. Unlike Concelier plug-ins (which ingest or export advisories), Authority plug-ins participate directly in authentication flows: + +- **Use cases:** integrate corporate directories (LDAP/AD)[^ldap-rfc], delegate to external IDPs, enforce bespoke password/lockout policies, or add client provisioning automation. +- **Constraints:** plug-ins load only during service start (no hot-reload), must function without outbound internet access, and must emit deterministic results for identical configuration input. +- **Ship targets:** build against the host’s .NET 10 preview SDK, honour offline-first requirements, and surface actionable diagnostics so operators can triage issues from `/ready`. + +## 2. Architecture Snapshot +Authority hosts follow a deterministic plug-in lifecycle. The exported diagram (`docs/assets/authority/authority-plugin-lifecycle.svg`) mirrors the steps below; regenerate it from the Mermaid source if you update the flow. + +1. **Configuration load** – `AuthorityPluginConfigurationLoader` resolves YAML manifests under `etc/authority.plugins/`. +2. **Assembly discovery** – the shared `PluginHost` scans `StellaOps.Authority.PluginBinaries` for `StellaOps.Authority.Plugin.*.dll` assemblies. +3. **Registrar execution** – each assembly is searched for `IAuthorityPluginRegistrar` implementations. Registrars bind options, register services, and optionally queue bootstrap tasks. +4. **Runtime** – the host resolves `IIdentityProviderPlugin` instances, uses capability metadata to decide which OAuth grants to expose, and invokes health checks for readiness endpoints. + +![Authority plug-in lifecycle diagram](../assets/authority/authority-plugin-lifecycle.svg) + +_Source:_ `docs/assets/authority/authority-plugin-lifecycle.mmd` + +**Data persistence primer:** the standard Mongo-backed plugin stores users in collections named `authority_users_` and lockout metadata in embedded documents. Additional plugins must document their storage layout and provide deterministic collection naming to honour the Offline Kit replication process. + +## 3. Capability Metadata +Capability flags let the host reason about what your plug-in supports: + +- Declare capabilities in your descriptor using the string constants from `AuthorityPluginCapabilities` (`password`, `mfa`, `clientProvisioning`, `bootstrap`). The configuration loader now validates these tokens and rejects unknown values at startup. +- `AuthorityIdentityProviderCapabilities.FromCapabilities` projects those strings into strongly typed booleans (`SupportsPassword`, etc.). Authority Core will use these flags when wiring flows such as the password grant. Built-in plugins (e.g., Standard) will fail fast or force-enable required capabilities if the descriptor is misconfigured, so keep manifests accurate. +- Typical configuration (`etc/authority.plugins/standard.yaml`): + ```yaml + plugins: + descriptors: + standard: + assemblyName: "StellaOps.Authority.Plugin.Standard" + capabilities: + - password + - bootstrap + ``` +- Only declare a capability if the plug-in genuinely implements it. For example, if `SupportsClientProvisioning` is `true`, the plug-in must supply a working `IClientProvisioningStore`. + +**Operational reminder:** the Authority host surfaces capability summaries during startup (see `AuthorityIdentityProviderRegistry` log lines). Use those logs during smoke tests to ensure manifests align with expectations. + +**Configuration path normalisation:** Manifest-relative paths (e.g., `tokenSigning.keyDirectory: "../keys"`) are resolved against the YAML file location and environment variables are expanded before validation. Plug-ins should expect to receive an absolute, canonical path when options are injected. + +**Password policy guardrails:** The Standard registrar logs a warning when a plug-in weakens the default password policy (minimum length or required character classes). Keep overrides at least as strong as the compiled defaults—operators treat the warning as an actionable security deviation. + +## 4. Project Scaffold +- Target **.NET 10 preview**, enable nullable, treat warnings as errors, and mark Authority plug-ins with `true`. +- Minimum references: + - `StellaOps.Authority.Plugins.Abstractions` (contracts & capability helpers) + - `StellaOps.Plugin` (hosting/DI helpers) + - `StellaOps.Auth.*` libraries as needed for shared token utilities (optional today). +- Example `.csproj` (trimmed from `StellaOps.Authority.Plugin.Standard`): + ```xml + + + net10.0 + enable + true + true + + + + + + + ``` + (Add other references—e.g., MongoDB driver, shared auth libraries—according to your implementation.) + +## 5. Implementing `IAuthorityPluginRegistrar` +- Create a parameterless registrar class that returns your plug-in type name via `PluginType`. +- Use `AuthorityPluginRegistrationContext` to: + - Bind options (`AddOptions(pluginName).Bind(...)`). + - Register singletons for stores/enrichers using manifest metadata. + - Register any hosted bootstrap tasks (e.g., seed admin users). +- Always validate configuration inside `PostConfigure` and throw meaningful `InvalidOperationException` to fail fast during startup. +- Use the provided `ILoggerFactory` from DI; avoid static loggers or console writes. +- Example skeleton: + ```csharp + internal sealed class MyPluginRegistrar : IAuthorityPluginRegistrar + { + public string PluginType => "my-custom"; + + public void Register(AuthorityPluginRegistrationContext context) + { + var name = context.Plugin.Manifest.Name; + + context.Services.AddOptions(name) + .Bind(context.Plugin.Configuration) + .PostConfigure(opts => opts.Validate(name)); + + context.Services.AddSingleton(sp => + new MyIdentityProvider(context.Plugin, sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>())); + } + } + ``` + +## 6. Identity Provider Surface +- Implement `IIdentityProviderPlugin` to expose: + - `IUserCredentialStore` for password validation and user CRUD. + - `IClaimsEnricher` to append roles/attributes onto issued principals. + - Optional `IClientProvisioningStore` for machine-to-machine clients. + - `AuthorityIdentityProviderCapabilities` to advertise supported flows. +- Password guidance: + - Standard plug-in hashes via `ICryptoProvider` using Argon2id by default and emits PHC-compliant strings. Successful PBKDF2 logins trigger automatic rehashes so migrations complete gradually. See `docs/security/password-hashing.md` for tuning advice. + - Enforce password policies before hashing to avoid storing weak credentials. +- Health checks should probe backing stores (e.g., Mongo `ping`) and return `AuthorityPluginHealthResult` so `/ready` can surface issues. +- When supporting additional factors (e.g., TOTP), implement `SupportsMfa` and document the enrolment flow for resource servers. + +## 7. Configuration & Secrets +- Authority looks for manifests under `etc/authority.plugins/`. Each YAML file maps directly to a plug-in name. +- Support environment overrides using `STELLAOPS_AUTHORITY_PLUGINS__DESCRIPTORS____...`. +- Never store raw secrets in git: allow operators to supply them via `.local.yaml`, environment variables, or injected secret files. Document which keys are mandatory. +- Validate configuration as soon as the registrar runs; use explicit error messages to guide operators. The Standard plug-in now enforces complete bootstrap credentials (username + password) and positive lockout windows via `StandardPluginOptions.Validate`. +- Cross-reference bootstrap workflows with `docs/ops/authority_bootstrap.md` (to be published alongside CORE6) so operators can reuse the same payload formats for manual provisioning. +- `passwordHashing` inherits defaults from `authority.security.passwordHashing`. Override only when hardware constraints differ per plug-in: + ```yaml + passwordHashing: + algorithm: Argon2id + memorySizeInKib: 19456 + iterations: 2 + parallelism: 1 + ``` + Invalid values (≤0) fail fast during startup, and legacy PBKDF2 hashes rehash automatically once the new algorithm succeeds. + +### 7.1 Token Persistence Contract +- The host automatically persists every issued principal (access, refresh, device, authorization code) in `authority_tokens`. Plug-in code **must not** bypass this store; use the provided `IAuthorityTokenStore` helpers when implementing custom flows. +- When a plug-in disables a subject or client outside the standard handlers, call `IAuthorityTokenStore.UpdateStatusAsync(...)` for each affected token so revocation bundles stay consistent. +- Supply machine-friendly `revokedReason` codes (`compromised`, `rotation`, `policy`, `lifecycle`, etc.) and optional `revokedMetadata` entries when invalidating credentials. These flow straight into `revocation-bundle.json` and should remain deterministic. +- Token scopes should be normalised (trimmed, unique, ordinal sort) before returning from plug-in verification paths. `TokenPersistenceHandlers` will keep that ordering for downstream consumers. + +### 7.2 Claims & Enrichment Checklist +- Authority always sets the OpenID Connect basics: `sub`, `client_id`, `preferred_username`, optional `name`, and `role` (for password flows). Plug-ins must use `IClaimsEnricher` to append additional claims in a **deterministic** order (sort arrays, normalise casing) so resource servers can rely on stable shapes. +- Recommended enrichment keys: + - `stellaops.realm` – plug-in/tenant identifier so services can scope policies. + - `stellaops.subject.type` – values such as `human`, `service`, `bootstrap`. + - `groups` / `projects` – sorted arrays describing operator entitlements. +- Claims visible in tokens should mirror what `/token` and `/userinfo` emit. Avoid injecting sensitive PII directly; mark values with `ClassifiedString.Personal` inside the plug-in so audit sinks can tag them appropriately. +- For client-credential flows, remember to enrich both the client principal and the validation path (`TokenValidationHandlers`) so refresh flows keep the same metadata. + +### 7.3 Revocation Bundles & Reasons +- Use `IAuthorityRevocationStore` to record subject/client/token revocations when credentials are deleted or rotated. Stick to the standard categories (`token`, `subject`, `client`, `key`). +- Include a deterministic `reason` string and optional `reasonDescription` so operators understand *why* a subject was revoked when inspecting bundles offline. +- Plug-ins should populate `metadata` with stable keys (e.g., `revokedBy`, `sourcePlugin`, `ticketId`) to simplify SOC correlation. The keys must be lowercase, ASCII, and free of secrets—bundles are mirrored to air-gapped agents. + +## 8. Rate Limiting & Lockout Interplay +Rate limiting and account lockouts are complementary controls. Plug-ins must surface both deterministically so operators can correlate limiter hits with credential rejections. + +**Baseline quotas** (from `docs/dev/authority-rate-limit-tuning-outline.md`): + +| Endpoint | Default policy | Notes | +|----------|----------------|-------| +| `/token` | 30 requests / 60s, queue 0 | Drop to 10/60s for untrusted ranges; raise only with WAF + monitoring. | +| `/authorize` | 60 requests / 60s, queue 10 | Reduce carefully; interactive UX depends on headroom. | +| `/internal/*` | Disabled by default; recommended 5/60s when enabled | Keep queue 0 for bootstrap APIs. | + +**Retry metadata:** The middleware stamps `Retry-After` plus tags `authority.client_id`, `authority.remote_ip`, and `authority.endpoint`. Plug-ins should keep these tags intact when crafting responses or telemetry so dashboards remain consistent. + +**Lockout counters:** Treat lockouts as **subject-scoped** decisions. When multiple instances update counters, reuse the deterministic tie-breakers documented in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` (freshness overrides, precedence, and stable hashes) to avoid divergent lockout states across replicas. + +**Alerting hooks:** Emit structured logs/metrics when either the limiter or credential store rejects access. Suggested gauges include `aspnetcore_rate_limiting_rejections_total{limiter="authority-token"}` and any custom `auth.plugins..lockouts_total` counter. + +![Authority rate limit and lockout flow](../assets/authority/authority-rate-limit-flow.svg) + +_Source:_ `docs/assets/authority/authority-rate-limit-flow.mmd` + +## 9. Logging, Metrics, and Diagnostics +- Always log via the injected `ILogger`; include `pluginName` and correlation IDs where available. +- Activity/metric names should align with `AuthorityTelemetry` constants (`service.name=stellaops-authority`). +- Expose additional diagnostics via structured logging rather than writing custom HTTP endpoints; the host will integrate these into `/health` and `/ready`. +- Emit metrics with stable names (`auth.plugins..*`) when introducing custom instrumentation; coordinate with the Observability guild to reserve prefixes. + +## 10. Testing & Tooling +- Unit tests: use Mongo2Go (or similar) to exercise credential stores without hitting production infrastructure (`StandardUserCredentialStoreTests` is a template). +- Determinism: fix timestamps to UTC and sort outputs consistently; avoid random GUIDs unless stable. +- Smoke tests: launch `dotnet run --project src/StellaOps.Authority/StellaOps.Authority` with your plug-in under `StellaOps.Authority.PluginBinaries` and verify `/ready`. +- Example verification snippet: + ```csharp + [Fact] + public async Task VerifyPasswordAsync_ReturnsSuccess() + { + var store = CreateCredentialStore(); + await store.UpsertUserAsync(new AuthorityUserRegistration("alice", "Pa55!", null, null, false, + Array.Empty(), new Dictionary()), CancellationToken.None); + + var result = await store.VerifyPasswordAsync("alice", "Pa55!", CancellationToken.None); + Assert.True(result.Succeeded); + Assert.True(result.User?.Roles.Count == 0); + } + ``` + +## 11. Packaging & Delivery +- Output assembly should follow `StellaOps.Authority.Plugin..dll` so the host’s search pattern picks it up. +- Place the compiled DLL plus dependencies under `StellaOps.Authority.PluginBinaries` for offline deployments; include hashes/signatures in release notes (Security Guild guidance forthcoming). +- Document any external prerequisites (e.g., CA cert bundle) in your plug-in README. +- Update `etc/authority.plugins/.yaml` samples and include deterministic SHA256 hashes for optional bootstrap payloads when distributing Offline Kit artefacts. + +[^ldap-rfc]: Lightweight Directory Access Protocol (LDAPv3) specification — [RFC 4511](https://datatracker.ietf.org/doc/html/rfc4511). + +## 12. Checklist & Handoff +- ✅ Capabilities declared and validated in automated tests. +- ✅ Bootstrap workflows documented (if `bootstrap` capability used) and repeatable. +- ✅ Local smoke test + unit/integration suites green (`dotnet test`). +- ✅ Operational docs updated: configuration keys, secrets guidance, troubleshooting. +- Submit the developer guide update referencing PLG6/DOC4 and tag DevEx + Docs reviewers for sign-off. + +--- +Mermaid sources for the embedded diagrams live under `docs/assets/authority/`. Regenerate the SVG assets with your preferred renderer before committing future updates so the visuals stay in sync with the `.mmd` definitions. diff --git a/docs/dev/32_AUTH_CLIENT_GUIDE.md b/docs/dev/32_AUTH_CLIENT_GUIDE.md index 48587a41..96dde2bd 100644 --- a/docs/dev/32_AUTH_CLIENT_GUIDE.md +++ b/docs/dev/32_AUTH_CLIENT_GUIDE.md @@ -1,91 +1,91 @@ -# StellaOps Auth Client — Integration Guide - -> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Feedser, CLI, Agent) should review before wiring the new options into their configuration surfaces. - -The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable. - -This guide explains how to consume the new settings, when to toggle them, and how to test your integration. - -## 1. Registering the client - -```csharp -services.AddStellaOpsAuthClient(options => -{ - options.Authority = configuration["StellaOps:Authority:Url"]!; - options.ClientId = configuration["StellaOps:Authority:ClientId"]!; - options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"]; - options.DefaultScopes.Add("feedser.jobs.trigger"); - - options.EnableRetries = true; - options.RetryDelays.Clear(); - options.RetryDelays.Add(TimeSpan.FromMilliseconds(500)); - options.RetryDelays.Add(TimeSpan.FromSeconds(2)); - - options.AllowOfflineCacheFallback = true; - options.OfflineCacheTolerance = TimeSpan.FromMinutes(5); -}); -``` - -> **Reminder:** `AddStellaOpsAuthClient` binds the options via `IOptionsMonitor` so changes picked up from configuration reloads will be applied to future HTTP calls without restarting the host. - -## 2. Resilience options - -| Option | Default | Notes | -|--------|---------|-------| -| `EnableRetries` | `true` | When disabled, the shared Polly policy is a no-op and HTTP calls will fail fast. | -| `RetryDelays` | `1s, 2s, 5s` | Edit in ascending order; zero/negative entries are ignored. Clearing the list and leaving it empty keeps the defaults. | -| `AllowOfflineCacheFallback` | `true` | When `true`, stale discovery/JWKS responses are reused within the tolerance window if Authority is unreachable. | -| `OfflineCacheTolerance` | `00:10:00` | Added to the normal cache lifetime. E.g. a 10 minute JWKS cache plus 5 minute tolerance keeps keys for 15 minutes if Authority is offline. | - -The HTTP retry policy handles: - -- 5xx responses -- 429 responses -- Transient transport failures (`HttpRequestException`, timeouts, aborted sockets) - -Retries emit warnings via the `StellaOps.Auth.Client.HttpRetry` logger. Tune the delay values to honour your deployment’s SLOs. - -## 3. Configuration mapping - -Suggested configuration keys (coordinate with consuming teams before finalising): - -```yaml -StellaOps: - Authority: - Url: "https://authority.stella-ops.local" - ClientId: "feedser" - ClientSecret: "change-me" - AuthClient: - EnableRetries: true - RetryDelays: - - "00:00:01" - - "00:00:02" - - "00:00:05" - AllowOfflineCacheFallback: true - OfflineCacheTolerance: "00:10:00" -``` - -Environment variable binding follows the usual double-underscore rules, e.g. - -``` -STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02 -STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00 -``` - -CLI and Feedser teams should expose these knobs once they adopt the auth client. - -## 4. Testing recommendations - -1. **Unit tests:** assert option binding by configuring `StellaOpsAuthClientOptions` via a `ConfigurationBuilder` and ensuring `Validate()` normalises the retry delays and scope list. -2. **Offline fallback:** simulate an unreachable Authority by swapping `HttpMessageHandler` to throw `HttpRequestException` after priming the discovery/JWKS caches. Verify that tokens are still issued until the tolerance expires. -3. **Observability:** watch for `StellaOps.Auth.Client.HttpRetry` warnings in your logs. Excessive retries mean the upstream Authority cluster needs attention. -4. **Determinism:** keep retry delays deterministic. Avoid random jitter—operators can introduce jitter at the infrastructure layer if desired. - -## 5. Rollout checklist - -- [ ] Update consuming service/CLI configuration schema to include the new settings. -- [ ] Document recommended defaults for offline (air-gapped) versus connected deployments. -- [ ] Extend smoke tests to cover Authority outage scenarios. -- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs. - -Once Feedser and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it. +# StellaOps Auth Client — Integration Guide + +> **Status:** Drafted 2025-10-10 as part of LIB5. Consumer teams (Concelier, CLI, Agent) should review before wiring the new options into their configuration surfaces. + +The `StellaOps.Auth.Client` library provides a resilient OpenID Connect client for services and tools that talk to **StellaOps Authority**. LIB5 introduced configurable HTTP retry/backoff policies and an offline-fallback window so downstream components stay deterministic even when Authority is briefly unavailable. + +This guide explains how to consume the new settings, when to toggle them, and how to test your integration. + +## 1. Registering the client + +```csharp +services.AddStellaOpsAuthClient(options => +{ + options.Authority = configuration["StellaOps:Authority:Url"]!; + options.ClientId = configuration["StellaOps:Authority:ClientId"]!; + options.ClientSecret = configuration["StellaOps:Authority:ClientSecret"]; + options.DefaultScopes.Add("concelier.jobs.trigger"); + + options.EnableRetries = true; + options.RetryDelays.Clear(); + options.RetryDelays.Add(TimeSpan.FromMilliseconds(500)); + options.RetryDelays.Add(TimeSpan.FromSeconds(2)); + + options.AllowOfflineCacheFallback = true; + options.OfflineCacheTolerance = TimeSpan.FromMinutes(5); +}); +``` + +> **Reminder:** `AddStellaOpsAuthClient` binds the options via `IOptionsMonitor` so changes picked up from configuration reloads will be applied to future HTTP calls without restarting the host. + +## 2. Resilience options + +| Option | Default | Notes | +|--------|---------|-------| +| `EnableRetries` | `true` | When disabled, the shared Polly policy is a no-op and HTTP calls will fail fast. | +| `RetryDelays` | `1s, 2s, 5s` | Edit in ascending order; zero/negative entries are ignored. Clearing the list and leaving it empty keeps the defaults. | +| `AllowOfflineCacheFallback` | `true` | When `true`, stale discovery/JWKS responses are reused within the tolerance window if Authority is unreachable. | +| `OfflineCacheTolerance` | `00:10:00` | Added to the normal cache lifetime. E.g. a 10 minute JWKS cache plus 5 minute tolerance keeps keys for 15 minutes if Authority is offline. | + +The HTTP retry policy handles: + +- 5xx responses +- 429 responses +- Transient transport failures (`HttpRequestException`, timeouts, aborted sockets) + +Retries emit warnings via the `StellaOps.Auth.Client.HttpRetry` logger. Tune the delay values to honour your deployment’s SLOs. + +## 3. Configuration mapping + +Suggested configuration keys (coordinate with consuming teams before finalising): + +```yaml +StellaOps: + Authority: + Url: "https://authority.stella-ops.local" + ClientId: "concelier" + ClientSecret: "change-me" + AuthClient: + EnableRetries: true + RetryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + AllowOfflineCacheFallback: true + OfflineCacheTolerance: "00:10:00" +``` + +Environment variable binding follows the usual double-underscore rules, e.g. + +``` +STELLAOPS__AUTHORITY__AUTHCLIENT__RETRYDELAYS__0=00:00:02 +STELLAOPS__AUTHORITY__AUTHCLIENT__OFFLINECACHETOLERANCE=00:05:00 +``` + +CLI and Concelier teams should expose these knobs once they adopt the auth client. + +## 4. Testing recommendations + +1. **Unit tests:** assert option binding by configuring `StellaOpsAuthClientOptions` via a `ConfigurationBuilder` and ensuring `Validate()` normalises the retry delays and scope list. +2. **Offline fallback:** simulate an unreachable Authority by swapping `HttpMessageHandler` to throw `HttpRequestException` after priming the discovery/JWKS caches. Verify that tokens are still issued until the tolerance expires. +3. **Observability:** watch for `StellaOps.Auth.Client.HttpRetry` warnings in your logs. Excessive retries mean the upstream Authority cluster needs attention. +4. **Determinism:** keep retry delays deterministic. Avoid random jitter—operators can introduce jitter at the infrastructure layer if desired. + +## 5. Rollout checklist + +- [ ] Update consuming service/CLI configuration schema to include the new settings. +- [ ] Document recommended defaults for offline (air-gapped) versus connected deployments. +- [ ] Extend smoke tests to cover Authority outage scenarios. +- [ ] Coordinate with Docs Guild so user-facing quickstarts reference the new knobs. + +Once Concelier and CLI integrate these changes, we can mark LIB5 **DONE**; further packaging work is deferred until the backlog reintroduces it. diff --git a/docs/dev/BUILDX_PLUGIN_QUICKSTART.md b/docs/dev/BUILDX_PLUGIN_QUICKSTART.md new file mode 100644 index 00000000..dadcb2dd --- /dev/null +++ b/docs/dev/BUILDX_PLUGIN_QUICKSTART.md @@ -0,0 +1,119 @@ +# BuildX Generator Quickstart + +This quickstart explains how to run the StellaOps **BuildX SBOM generator** offline, verify the CAS handshake, and emit OCI descriptors that downstream services can attest. + +## 1. Prerequisites + +- Docker 25+ with BuildKit enabled (`docker buildx` available). +- .NET 10 (preview) SDK matching the repository `global.json`. +- Optional: network access to a StellaOps Attestor endpoint (the quickstart uses a mock service). + +## 2. Publish the plug-in binaries + +The BuildX generator publishes as a .NET self-contained executable with its manifest under `plugins/scanner/buildx/`. + +```bash +# From the repository root +DOTNET_CLI_HOME="${PWD}/.dotnet" \ +dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj \ + -c Release \ + -o out/buildx +``` + +- `out/buildx/` now contains `StellaOps.Scanner.Sbomer.BuildXPlugin.dll` and the manifest `stellaops.sbom-indexer.manifest.json`. +- `plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/` receives the same artefacts for release packaging. +- The CI pipeline also tars and signs (SHA-256 manifest) the OS analyzer plug-ins located under + `plugins/scanner/analyzers/os/` so they ship alongside the BuildX generator artefacts. + +## 3. Verify the CAS handshake + +```bash +dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll handshake \ + --manifest out/buildx \ + --cas out/cas +``` + +The command performs a deterministic probe write (`sha256`) into the provided CAS directory and prints the resolved path. + +## 4. Emit a descriptor + provenance placeholder + +1. Build or identify the image you want to describe and capture its digest: + + ```bash + docker buildx build --load -t stellaops/buildx-demo:ci samples/ci/buildx-demo + DIGEST=$(docker image inspect stellaops/buildx-demo:ci --format '{{index .RepoDigests 0}}') + ``` + +2. Generate a CycloneDX SBOM for the built image (any tool works; here we use `docker sbom`): + + ```bash + docker sbom stellaops/buildx-demo:ci --format cyclonedx-json > out/buildx-sbom.cdx.json + ``` + +3. Invoke the `descriptor` command, pointing at the SBOM file and optional metadata: + + ```bash + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image "$DIGEST" \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + --artifact-type application/vnd.stellaops.sbom.layer+json \ + --sbom-format cyclonedx-json \ + --sbom-kind inventory \ + --repository git.stella-ops.org/stellaops/buildx-demo \ + --build-ref $(git rev-parse HEAD) \ + > out/buildx-descriptor.json + ``` + +The output JSON captures: + +- OCI artifact descriptor including size, digest, and annotations (`org.stellaops.*`). +- Provenance placeholder (`expectedDsseSha256`, `nonce`, `attestorUri` when provided). `nonce` is derived deterministically from the image + SBOM metadata so repeated runs produce identical placeholders for identical inputs. +- Generator metadata and deterministic timestamps. + +## 5. (Optional) Send the placeholder to an Attestor + +The plug-in can POST the descriptor metadata to an Attestor endpoint, returning once it receives an HTTP 202. + +```bash +python3 - <<'PY' & +from http.server import BaseHTTPRequestHandler, HTTPServer +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + _ = self.rfile.read(int(self.headers.get('Content-Length', 0))) + self.send_response(202); self.end_headers(); self.wfile.write(b'accepted') + def log_message(self, fmt, *args): + return +server = HTTPServer(('127.0.0.1', 8085), Handler) +try: + server.serve_forever() +except KeyboardInterrupt: + pass +finally: + server.server_close() +PY +MOCK_PID=$! + +dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image "$DIGEST" \ + --sbom out/buildx-sbom.cdx.json \ + --attestor http://127.0.0.1:8085/provenance \ + --attestor-token "$STELLAOPS_ATTESTOR_TOKEN" \ + > out/buildx-descriptor.json + +kill $MOCK_PID +``` + +Set `STELLAOPS_ATTESTOR_TOKEN` (or pass `--attestor-token`) when the Attestor requires bearer authentication. Use `--attestor-insecure` for lab environments with self-signed certificates. + +## 6. CI workflow example + +A reusable GitHub Actions workflow is provided under `samples/ci/buildx-demo/github-actions-buildx-demo.yml`. It publishes the plug-in, runs the handshake, builds the demo image, emits a descriptor, and uploads both the descriptor and the mock-Attestor request as artefacts. + +Add the workflow to your repository (or call it via `workflow_call`) and adjust the SBOM path + Attestor URL as needed. The workflow also re-runs the `descriptor` command and diffs the results (ignoring the `generatedAt` timestamp) so you catch regressions that would break deterministic CI use. + +--- + +For deeper integration guidance (custom SBOM builders, exporting DSSE bundles), track ADRs in `docs/ARCHITECTURE_SCANNER.md` §7 and follow upcoming Attestor API releases. diff --git a/docs/dev/EXCITITOR_STATEMENT_BACKFILL.md b/docs/dev/EXCITITOR_STATEMENT_BACKFILL.md new file mode 100644 index 00000000..11af06ef --- /dev/null +++ b/docs/dev/EXCITITOR_STATEMENT_BACKFILL.md @@ -0,0 +1,86 @@ +# Excititor Statement Backfill Runbook + +Last updated: 2025-10-19 + +## Overview + +Use this runbook when you need to rebuild the `vex.statements` collection from historical raw documents. Typical scenarios: + +- Upgrading the statement schema (e.g., adding severity/KEV/EPSS signals). +- Recovering from a partial ingest outage where statements were never persisted. +- Seeding a freshly provisioned Excititor deployment from an existing raw archive. + +Backfill operates server-side via the Excititor WebService and reuses the same pipeline that powers the `/excititor/statements` ingestion endpoint. Each raw document is normalized, signed metadata is preserved, and duplicate statements are skipped unless the run is forced. + +## Prerequisites + +1. **Connectivity to Excititor WebService** – the CLI uses the backend URL configured in `stellaops.yml` or the `--backend-url` argument. +2. **Authority credentials** – the CLI honours the existing Authority client configuration; ensure the caller has permission to invoke admin endpoints. +3. **Mongo replica set** (recommended) – causal consistency guarantees rely on majority read/write concerns. Standalone deployment works but skips cross-document transactions. + +## CLI command + +``` +stellaops excititor backfill-statements \ + [--retrieved-since ] \ + [--force] \ + [--batch-size ] \ + [--max-documents ] +``` + +| Option | Description | +| ------ | ----------- | +| `--retrieved-since` | Only process raw documents fetched on or after the specified timestamp (UTC by default). | +| `--force` | Reprocess documents even if matching statements already exist (useful after schema upgrades). | +| `--batch-size` | Number of raw documents pulled per batch (default `100`). | +| `--max-documents` | Optional hard limit on the number of raw documents to evaluate. | + +Example – replay the last 48 hours of Red Hat ingest while keeping existing statements: + +``` +stellaops excititor backfill-statements \ + --retrieved-since "$(date -u -d '48 hours ago' +%Y-%m-%dT%H:%M:%SZ)" +``` + +Example – full replay with forced overwrites, capped at 2,000 documents: + +``` +stellaops excititor backfill-statements --force --max-documents 2000 +``` + +The command returns a summary similar to: + +``` +Backfill completed: evaluated 450, backfilled 180, claims written 320, skipped 270, failures 0. +``` + +## Behaviour + +- Raw documents are streamed in ascending `retrievedAt` order. +- Each document is normalized using the registered VEX normalizers (CSAF, CycloneDX, OpenVEX). +- Statements are appended through the same `IVexClaimStore.AppendAsync` path that powers `/excititor/statements`. +- Duplicate detection compares `Document.Digest`; duplicates are skipped unless `--force` is specified. +- Failures are logged with the offending digest and continue with the next document. + +## Observability + +- CLI logs aggregate counts and the backend logs per-digest warnings or errors. +- Mongo writes carry majority write concern; expect backfill throughput to match ingest baselines (≈5 seconds warm, 30 seconds cold). +- Monitor the `excititor.storage.backfill` log scope for detailed telemetry. + +## Post-run verification + +1. Inspect the `vex.statements` collection for the targeted window (check `InsertedAt`). +2. Re-run the Excititor storage test suite if possible: + ``` + dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj + ``` +3. Optionally, call `/excititor/statements/{vulnerabilityId}/{productKey}` to confirm the expected statements exist. + +## Rollback + +If a forced run produced incorrect statements, use the standard Mongo rollback procedure: + +1. Identify the `InsertedAt` window for the backfill run. +2. Delete affected records from `vex.statements` (and any downstream exports if applicable). +3. Rerun the backfill command with corrected parameters. diff --git a/docs/dev/SCANNER_CACHE_CONFIGURATION.md b/docs/dev/SCANNER_CACHE_CONFIGURATION.md new file mode 100644 index 00000000..ffd41f1a --- /dev/null +++ b/docs/dev/SCANNER_CACHE_CONFIGURATION.md @@ -0,0 +1,108 @@ +# Scanner Cache Configuration Guide + +The scanner cache stores layer-level SBOM fragments and file content that can be reused across scans. This document explains how to configure and operate the cache subsystem introduced in Sprint 10 (Group SP10-G5). + +## 1. Overview + +- **Layer cache** persists SBOM fragments per layer digest under `/layers//` with deterministic metadata (`meta.json`). +- **File CAS** (content-addressable store) keeps deduplicated blobs (e.g., analyzer fixtures, imported SBOM layers) under `/cas///`. +- **Maintenance** runs via `ScannerCacheMaintenanceService`, evicting expired entries and compacting the cache to stay within size limits. +- **Metrics** emit on the `StellaOps.Scanner.Cache` meter with counters for hits, misses, evictions, and byte histograms. +- **Offline workflows** use the CAS import/export helpers to package cache warmups inside the Offline Kit. + +## 2. Configuration keys (`scanner:cache`) + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `true` | Globally disable cache if `false`. | +| `rootPath` | `cache/scanner` | Base directory for cache data. Use an SSD-backed path for best warm-scan latency. | +| `layersDirectoryName` | `layers` | Subdirectory for layer cache entries. | +| `fileCasDirectoryName` | `cas` | Subdirectory for file CAS entries. | +| `layerTtl` | `45.00:00:00` | Time-to-live for layer cache entries (`TimeSpan`). `0` disables TTL eviction. | +| `fileTtl` | `30.00:00:00` | Time-to-live for CAS entries. `0` disables TTL eviction. | +| `maxBytes` | `5368709120` (5 GiB) | Hard cap for combined cache footprint. Compaction trims data back to `warmBytesThreshold`. | +| `warmBytesThreshold` | `maxBytes / 5` | Target size after compaction. | +| `coldBytesThreshold` | `maxBytes * 0.8` | Upper bound that triggers compaction. | +| `enableAutoEviction` | `true` | If `false`, callers must invoke `ILayerCacheStore.CompactAsync` / `IFileContentAddressableStore.CompactAsync` manually. | +| `maintenanceInterval` | `00:15:00` | Interval for the maintenance hosted service. | +| `enableFileCas` | `true` | Disable to prevent CAS usage (APIs throw on `PutAsync`). | +| `importDirectory` / `exportDirectory` | `null` | Optional defaults for offline import/export tooling. | + +> **Tip:** configure `scanner:cache:rootPath` to a dedicated volume and mount it into worker containers when running in Kubernetes or Nomad. + +## 3. Metrics + +Instrumentation lives in `ScannerCacheMetrics` on meter `StellaOps.Scanner.Cache`. + +| Instrument | Unit | Description | +| --- | --- | --- | +| `scanner.layer_cache_hits_total` | count | Layer cache hit counter. Tag: `layer`. | +| `scanner.layer_cache_misses_total` | count | Layer cache miss counter. Tag: `layer`. | +| `scanner.layer_cache_evictions_total` | count | Layer entries evicted due to TTL or compaction. Tag: `layer`. | +| `scanner.layer_cache_bytes` | bytes | Histogram of per-entry payload size when stored. | +| `scanner.file_cas_hits_total` | count | File CAS hit counter. Tag: `sha256`. | +| `scanner.file_cas_misses_total` | count | File CAS miss counter. Tag: `sha256`. | +| `scanner.file_cas_evictions_total` | count | CAS eviction counter. Tag: `sha256`. | +| `scanner.file_cas_bytes` | bytes | Histogram of CAS payload sizes on insert. | + +## 4. Import / Export workflow + +1. **Export warm cache** + ```bash + dotnet tool run stellaops-cache export --destination ./offline-kit/cache + ``` + Internally this calls `IFileContentAddressableStore.ExportAsync` which copies each CAS entry (metadata + `content.bin`). + +2. **Import on air-gapped hosts** + ```bash + dotnet tool run stellaops-cache import --source ./offline-kit/cache + ``` + The import API merges newer metadata and skips older snapshots automatically. + +3. **Layer cache seeding** + Layer cache entries are deterministic and can be packaged the same way (copy `/layers`). For now we keep seeding optional because layers are larger; follow-up tooling can compress directories as needed. + +## 5. Hosted maintenance loop + +`ScannerCacheMaintenanceService` runs as a background service within Scanner Worker or WebService hosts when `AddScannerCache` is registered. Behaviour: + +- At startup it performs an immediate eviction/compaction run. +- Every `maintenanceInterval` it triggers: + - `ILayerCacheStore.EvictExpiredAsync` + - `ILayerCacheStore.CompactAsync` + - `IFileContentAddressableStore.EvictExpiredAsync` + - `IFileContentAddressableStore.CompactAsync` +- Failures are logged at `Error` with preserved stack traces; the next tick continues normally. + +Set `enableAutoEviction=false` when hosting the cache inside ephemeral build pipelines that want to drive eviction explicitly. + +## 6. API surface summary + +```csharp +public interface ILayerCacheStore +{ + ValueTask TryGetAsync(string layerDigest, CancellationToken ct = default); + Task PutAsync(LayerCachePutRequest request, CancellationToken ct = default); + Task RemoveAsync(string layerDigest, CancellationToken ct = default); + Task EvictExpiredAsync(CancellationToken ct = default); + Task CompactAsync(CancellationToken ct = default); + Task OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken ct = default); +} + +public interface IFileContentAddressableStore +{ + ValueTask TryGetAsync(string sha256, CancellationToken ct = default); + Task PutAsync(FileCasPutRequest request, CancellationToken ct = default); + Task RemoveAsync(string sha256, CancellationToken ct = default); + Task EvictExpiredAsync(CancellationToken ct = default); + Task CompactAsync(CancellationToken ct = default); + Task ExportAsync(string destinationDirectory, CancellationToken ct = default); + Task ImportAsync(string sourceDirectory, CancellationToken ct = default); +} +``` + +Register both stores via `services.AddScannerCache(configuration);` in WebService or Worker hosts. + +--- + +_Last updated: 2025-10-19_ diff --git a/docs/dev/authority-dpop-mtls-plan.md b/docs/dev/authority-dpop-mtls-plan.md new file mode 100644 index 00000000..f89c7386 --- /dev/null +++ b/docs/dev/authority-dpop-mtls-plan.md @@ -0,0 +1,142 @@ +# Authority DPoP & mTLS Implementation Plan (2025-10-19) + +## Purpose +- Provide the implementation blueprint for AUTH-DPOP-11-001 and AUTH-MTLS-11-002. +- Unify sender-constraint validation across Authority, downstream services, and clients. +- Capture deterministic, testable steps that unblock UI/Signer guilds depending on DPoP/mTLS hardening. + +## Scope +- Token endpoint validation, issuance, and storage changes inside `StellaOps.Authority`. +- Shared security primitives consumed by Authority, Scanner, Signer, CLI, and UI. +- Operator-facing configuration, auditing, and observability. +- Out of scope: PoE enforcement (Signer) and CLI/UI client UX; those teams consume the new capabilities. + +> **Status update (2025-10-19):** `ValidateDpopProofHandler`, `AuthorityClientCertificateValidator`, and the supporting storage/audit plumbing now live in `src/StellaOps.Authority`. DPoP proofs populate `cnf.jkt`, mTLS bindings enforce certificate thumbprints via `cnf.x5t#S256`, and token documents persist the sender constraint metadata. In-memory nonce issuance is wired (Redis implementation to follow). Documentation and configuration references were updated (`docs/11_AUTHORITY.md`). Targeted unit/integration tests were added; running the broader test suite is currently blocked by pre-existing `StellaOps.Concelier.Storage.Mongo` build errors. + +## Design Summary +- Extract the existing Scanner `DpopProofValidator` stack into a shared `StellaOps.Auth.Security` library used by Authority and resource servers. +- Extend Authority configuration (`authority.yaml`) with strongly-typed `senderConstraints.dpop` and `senderConstraints.mtls` sections (map to sample already shown in architecture doc). +- Require DPoP proofs on `/token` when the registered client policy is `senderConstraint=dpop`; bind issued access tokens via `cnf.jkt`. +- Introduce Authority-managed nonce issuance for “high value” audiences (default: `signer`, `attestor`) with Redis-backed persistence and deterministic auditing. +- Enable OAuth 2.0 mTLS (RFC 8705) by storing certificate bindings per client, requesting client certificates at TLS termination, and stamping `cnf.x5t#S256` into issued tokens plus introspection output. +- Surface structured logs and counters for both DPoP and mTLS flows; provide integration tests that cover success, replay, invalid proof, and certificate mismatch cases. + +## AUTH-DPOP-11-001 — Proof Validation & Nonce Handling + +**Shared validator** +- Move `DpopProofValidator`, option types, and replay cache interfaces from `StellaOps.Scanner.Core` into a new assembly `StellaOps.Auth.Security`. +- Provide pluggable caches: `InMemoryDpopReplayCache` (existing) and new `RedisDpopReplayCache` (leveraging the Authority Redis connection). +- Ensure the validator exposes the validated `SecurityKey`, `jti`, and `iat` so Authority can construct the `cnf` claim and compute nonce expiry. + +**Configuration model** +- Extend `StellaOpsAuthorityOptions.Security` with a `SenderConstraints` property containing: + - `Dpop` (`enabled`, `allowedAlgorithms`, `maxAgeSeconds`, `clockSkewSeconds`, `replayWindowSeconds`, `nonce` settings with `enabled`, `ttlSeconds`, `requiredAudiences`, `maxIssuancePerMinute`). + - `Mtls` (`enabled`, `requireChainValidation`, `clientCaBundle`, `allowedSubjectPatterns`, `allowedSanTypes`). +- Bind from YAML (`authority.security.senderConstraints.*`) while preserving backwards compatibility (defaults keep both disabled). + +**Token endpoint pipeline** +- Introduce a scoped OpenIddict handler `ValidateDpopProofHandler` inserted before `ValidateClientCredentialsHandler`. +- Determine the required sender constraint from client metadata: + - Add `AuthorityClientMetadataKeys.SenderConstraint` storing `dpop` or `mtls`. + - Optionally allow per-client overrides for nonce requirement. +- When `dpop` is required: + - Read the `DPoP` header from the ASP.NET request, reject with `invalid_token` + `WWW-Authenticate: DPoP error="invalid_dpop_proof"` if absent. + - Call the shared validator with method/URI. Enforce algorithm allowlist and `iat` window from options. + - Persist the `jkt` thumbprint plus replay cache state in the OpenIddict transaction (`AuthorityOpenIddictConstants.DpopKeyThumbprintProperty`, `DpopIssuedAtProperty`). + - When the requested audience intersects `SenderConstraints.Dpop.Nonce.RequiredAudiences`, require `nonce` in the proof; on first failure respond with HTTP 401, `error="use_dpop_nonce"`, and include `DPoP-Nonce` header (see nonce note below). Cache the rejection reason for audit logging. + +**Nonce service** +- Add `IDpopNonceStore` with methods `IssueAsync(audience, clientId, jkt)` and `TryConsumeAsync(nonce, audience, clientId, jkt)`. +- Default implementation `RedisDpopNonceStore` storing SHA-256 hashes of nonces keyed by `audience:clientId:jkt`. TTL comes from `SenderConstraints.Dpop.Nonce.Ttl`. +- Create helper `DpopNonceIssuer` used by `ValidateDpopProofHandler` to issue nonces when missing/expired, enforcing issuance rate limits (per options) and tagging audit/log records. +- On successful validation (nonce supplied and consumed) stamp metadata into the transaction for auditing. +- Update `ClientCredentialsHandlers` to observe nonce enforcement: when a nonce challenge was sent, emit structured audit with `nonce_issued`, `audiences`, and `retry`. + +**Token issuance** +- In `HandleClientCredentialsHandler`, if the transaction contains a validated DPoP key: + - Build `cnf.jkt` using thumbprint from validator. + - Include `auth_time`/`dpop_jti` as needed for diagnostics. + - Persist the thumbprint alongside token metadata in Mongo (extend `AuthorityTokenDocument` with `SenderConstraint`, `KeyThumbprint`, `Nonce` fields). + +**Auditing & observability** +- Emit new audit events: + - `authority.dpop.proof.validated` (success/failure, clientId, audience, thumbprint, nonce status, jti). + - `authority.dpop.nonce.issued` and `authority.dpop.nonce.consumed`. +- Metrics (Prometheus style): + - `authority_dpop_validations_total{result,reason}`. + - `authority_dpop_nonce_issued_total{audience}` and `authority_dpop_nonce_fails_total{reason}`. +- Structured logs include `authority.sender_constraint=dpop`, `authority.dpop_thumbprint`, `authority.dpop_nonce`. + +**Testing** +- Unit tests for the handler pipeline using fake OpenIddict transactions. +- Replay/nonce tests with in-memory and Redis stores. +- Integration tests in `StellaOps.Authority.Tests` covering: + - Valid DPoP proof issuing `cnf.jkt`. + - Missing header → challenge with nonce. + - Replayed `jti` rejected. + - Invalid nonce rejected even after issuance. +- Contract tests to ensure `/.well-known/openid-configuration` advertises `dpop_signing_alg_values_supported` and `dpop_nonce_supported` when enabled. + +## AUTH-MTLS-11-002 — Certificate-Bound Tokens + +**Configuration model** +- Reuse `SenderConstraints.Mtls` described above; include: + - `enforceForAudiences` list (defaults `signer`, `attestor`, `scheduler`). + - `certificateRotationGraceSeconds` for overlap. + - `allowedClientCertificateAuthorities` absolute paths. + +**Kestrel/TLS pipeline** +- Configure Kestrel with `ClientCertificateMode.AllowCertificate` globally and implement middleware that enforces certificate presence only when the resolved client requires mTLS. +- Add `IAuthorityClientCertificateValidator` that validates presented certificate chain, SANs (`dns`, `uri`, optional SPIFFE), and thumbprint matches one of the stored bindings. +- Cache validation results per connection id to avoid rehashing on every request. + +**Client registration & storage** +- Extend `AuthorityClientDocument` with `List` containing: + - `Thumbprint`, `SerialNumber`, `Subject`, `NotBefore`, `NotAfter`, `Sans`, `CreatedAt`, `UpdatedAt`, `Label`. +- Provide admin API mutations (`/admin/clients/{id}/certificates`) for ops tooling (deferred implementation but schema ready). +- Update plugin provisioning store (`StandardClientProvisioningStore`) to map descriptors with certificate bindings and `senderConstraint`. +- Persist binding state in Mongo migrations (index on `{clientId, thumbprint}`). + +**Token issuance & introspection** +- Add a transaction property capturing the validated client certificate thumbprint. +- `HandleClientCredentialsHandler`: + - When mTLS required, ensure certificate info present; reject otherwise. + - Stamp `cnf` claim: `principal.SetClaim("cnf", JsonSerializer.Serialize(new { x5t#S256 = thumbprint }))`. + - Store binding metadata in issued token document for audit. +- Update `ValidateAccessTokenHandler` and introspection responses to surface `cnf.x5t#S256`. +- Ensure refresh tokens (if ever enabled) copy the binding data. + +**Auditing & observability** +- Audit events: + - `authority.mtls.handshake` (success/failure, clientId, thumbprint, issuer, subject). + - `authority.mtls.binding.missing` when a required client posts without a cert. +- Metrics: + - `authority_mtls_handshakes_total{result}`. + - `authority_mtls_certificate_rotations_total`. +- Logs include `authority.sender_constraint=mtls`, `authority.mtls_thumbprint`, `authority.mtls_subject`. + +**Testing** +- Unit tests for certificate validation rules (SAN mismatches, expiry, CA trust). +- Integration tests running Kestrel with test certificates: + - Successful token issuance with bound certificate. + - Request without certificate → `invalid_client`. + - Token introspection reveals `cnf.x5t#S256`. + - Rotation scenario (old + new cert allowed during grace window). + +## Implementation Checklist + +**DPoP work-stream** +1. Extract shared validator into `StellaOps.Auth.Security`; update Scanner references. +2. Introduce configuration classes and bind from YAML/environment. +3. Implement nonce store (Redis + in-memory), handler integration, and OpenIddict transaction plumbing. +4. Stamp `cnf.jkt`, audit events, and metrics; update Mongo documents and migrations. +5. Extend docs: `docs/ARCHITECTURE_AUTHORITY.md`, `docs/security/audit-events.md`, `docs/security/rate-limits.md`, CLI/UI references. + +**mTLS work-stream** +1. Extend client document/schema and provisioning stores with certificate bindings + sender constraint flag. +2. Configure Kestrel/middleware for optional client certificates and validation service. +3. Update token issuance/introspection to honour certificate bindings and emit `cnf.x5t#S256`. +4. Add auditing/metrics and integration tests (happy path + failure). +5. Refresh operator documentation (`docs/ops/authority-backup-restore.md`, `docs/ops/authority-monitoring.md`, sample `authority.yaml`) to cover certificate lifecycle. + +Both streams should conclude with `dotnet test src/StellaOps.Authority.sln` and documentation cross-links so dependent guilds can unblock UI/Signer work. diff --git a/docs/dev/authority-plugin-di-coordination.md b/docs/dev/authority-plugin-di-coordination.md new file mode 100644 index 00000000..c79470ba --- /dev/null +++ b/docs/dev/authority-plugin-di-coordination.md @@ -0,0 +1,52 @@ +# Authority Plug-in Scoped Service Coordination + +> Created: 2025-10-19 — Plugin Platform Guild & Authority Core +> Status: Scheduled (session confirmed for 2025-10-20 15:00–16:00 UTC) + +This document tracks preparation, agenda, and outcomes for the scoped-service workshop required before implementing PLUGIN-DI-08-002. + +## Objectives + +- Inventory Authority plug-in surfaces that need scoped service lifetimes. +- Confirm session/scope handling for identity-provider registrars and background jobs. +- Assign follow-up tasks/actions with owners and due dates. + +## Scheduling Snapshot + +- **Meeting time:** 2025-10-20 15:00–16:00 UTC (10:00–11:00 CDT / 08:00–09:00 PDT). +- **Facilitator:** Plugin Platform Guild — Alicia Rivera. +- **Attendees (confirmed):** Authority Core — Jasmin Patel; Authority Security Guild — Mohan Singh; Plugin Platform — Alicia Rivera, Leah Chen. +- **Optional invitees:** DevOps liaison — Sofia Ortega (accepted). +- **Logistics:** Invites sent via shared calendar on 2025-10-19 15:30 UTC with Teams bridge + offline dial-in. Meeting notes will be captured here. +- **Preparation deadline:** 2025-10-20 12:00 UTC — complete checklist below. + +## Pre-work Checklist + +- Review `ServiceBindingAttribute` contract introduced by PLUGIN-DI-08-001. +- Collect existing Authority plug-in registration code paths to evaluate. +- Audit background jobs that assume singleton lifetimes. +- Identify plug-in health checks/telemetry surfaces impacted by scoped lifetimes. + +### Pre-work References + +- _Add links, file paths, or notes here prior to the session._ + +## Draft Agenda + +1. Context recap (5 min) — why scoped DI is needed; summary of PLUGIN-DI-08-001 changes. +2. Authority plug-in surfaces (15 min) — registrars, background services, telemetry. +3. Session handling strategy (10 min) — scope creation semantics, cancellation propagation. +4. Action items & owners (10 min) — capture code/docs/test tasks with due dates. +5. Risks & follow-ups (5 min) — dependencies, rollout sequencing. + +## Notes + +- _Pending coordination session; populate afterwards._ + +## Action Item Log + +| Item | Owner | Due | Status | Notes | +|------|-------|-----|--------|-------| +| Confirm meeting time | Alicia Rivera | 2025-10-19 15:30 UTC | DONE | Calendar invite sent; all required attendees accepted | +| Compile Authority plug-in DI entry points | Jasmin Patel | 2025-10-20 | IN PROGRESS | Gather current Authority plug-in registrars, background jobs, and helper factories that assume singleton lifetimes; add the list with file paths to **Pre-work References** in this document before 2025-10-20 12:00 UTC. | +| Outline scoped-session pattern for background jobs | Leah Chen | Post-session | BLOCKED | Requires meeting outcomes | diff --git a/docs/dev/fixtures.md b/docs/dev/fixtures.md index 15f759be..876905b5 100644 --- a/docs/dev/fixtures.md +++ b/docs/dev/fixtures.md @@ -1,45 +1,45 @@ -# Feedser Fixture Maintenance +# Concelier Fixture Maintenance -Feedser uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the +Concelier uses a handful of deterministic fixtures to keep connector regressions in check. This guide lists the fixture sets, where they live, and how to regenerate them safely. --- ## GHSA ↔ OSV parity fixtures -- **Location:** `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.*.json` +- **Location:** `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.*.json` - **Purpose:** Exercised by `OsvGhsaParityRegressionTests` to ensure OSV + GHSA outputs stay aligned on aliases, ranges, references, and credits. -- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`) +- **Regeneration:** Either run the test harness with online regeneration (`UPDATE_PARITY_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`) or execute the fixture updater (`dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj`). Both paths normalise timestamps and canonical ordering. - **SemVer provenance:** The regenerated fixtures should show `normalizedVersions[].notes` in the `osv:{ecosystem}:{advisoryId}:{identifier}` shape emitted by `SemVerRangeRuleBuilder`. Confirm the constraints and notes line up with GHSA/NVD composites before committing. -- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj` to confirm parity. +- **Verification:** Inspect the diff, then re-run `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj` to confirm parity. ## GHSA credit parity fixtures -- **Location:** `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json` +- **Location:** `src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.{ghsa,osv,nvd}.json` - **Purpose:** Exercised by `GhsaCreditParityRegressionTests` to guarantee GHSA/NVD/OSV acknowledgements remain in lockstep. - **Regeneration:** `dotnet run --project tools/FixtureUpdater/FixtureUpdater.csproj` rewrites all three canonical snapshots. -- **Verification:** `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj`. +- **Verification:** `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj`. > Always commit fixture changes together with the code that motivated them and reference the regression test that guards the behaviour. ## Apple security update fixtures -- **Location:** `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`. +- **Location:** `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/*.html` and `.expected.json`. - **Purpose:** Exercised by `AppleLiveRegressionTests` to guarantee the Apple HTML parser and mapper stay deterministic while covering Rapid Security Responses and multi-device advisories. - **Regeneration:** Use the helper scripts (`scripts/update-apple-fixtures.sh` or `scripts/update-apple-fixtures.ps1`). They export `UPDATE_APPLE_FIXTURES=1`, propagate the flag through `WSLENV`, touch `.update-apple-fixtures`, and then run the Apple test project. This keeps WSL/VSCode test invocations in sync while the refresh workflow fetches live Apple support pages, sanitises them, and rewrites both the HTML and expected DTO snapshots with normalised ordering. -- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj` without the env var to confirm determinism. +- **Verification:** Inspect the generated diffs and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the env var to confirm determinism. > **Tip for other connector owners:** mirror the sentinel + `WSLENV` pattern (`touch .update--fixtures`, append the env var via `WSLENV`) when you add fixture refresh scripts so contributors running under WSL inherit the regeneration flag automatically. ## KISA advisory fixtures -- **Location:** `src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-{feed,detail}.(xml|json)` +- **Location:** `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-{feed,detail}.(xml|json)` - **Purpose:** Used by `KisaConnectorTests` to verify Hangul-aware fetch → parse → map flows and to assert telemetry counters stay wired. -- **Regeneration:** `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj` +- **Regeneration:** `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj` - **Verification:** Re-run the same test suite without the env var; confirm advisory content remains NFC-normalised and HTML is sanitised. Metrics assertions will fail if counters drift. - **Localisation note:** RSS `category` values (e.g. `취약점정보`) remain in Hangul—do not translate them in fixtures; they feed directly into metrics/log tags. diff --git a/docs/dev/kisa_connector_notes.md b/docs/dev/kisa_connector_notes.md index ae8dba52..5ebd37b4 100644 --- a/docs/dev/kisa_connector_notes.md +++ b/docs/dev/kisa_connector_notes.md @@ -4,7 +4,7 @@ The KISA/KNVD connector now ships with structured telemetry, richer logging, and ## Telemetry counters -All metrics are emitted from `KisaDiagnostics` (`Meter` name `StellaOps.Feedser.Source.Kisa`). +All metrics are emitted from `KisaDiagnostics` (`Meter` name `StellaOps.Concelier.Connector.Kisa`). | Metric | Description | Tags | | --- | --- | --- | @@ -39,7 +39,7 @@ The messages use structured properties (`Idx`, `Category`, `DocumentId`, `Severi - Hangul fields (`title`, `summary`, `category`, `reference.label`, product vendor/name) are normalised to NFC before storage. Sample category `취약점정보` roughly translates to “vulnerability information”. - Advisory HTML is sanitised via `HtmlContentSanitizer`, stripping script/style while preserving inline anchors for translation pipelines. - Metrics carry Hangul `category` tags and logging keeps Hangul strings intact; this ensures air-gapped operators can validate native-language content without relying on MT. -- Fixtures live under `src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/`. Regenerate with `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj`. +- Fixtures live under `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/`. Regenerate with `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`. - The regression suite asserts canonical mapping, state cleanup, and telemetry counters (`KisaConnectorTests.Telemetry_RecordsMetrics`) so QA can track instrumentation drift. For operator docs, link to this brief when documenting Hangul handling or counter dashboards so localisation reviewers have a single reference point. diff --git a/docs/dev/merge_semver_playbook.md b/docs/dev/merge_semver_playbook.md index 7ae3044a..c45777cf 100644 --- a/docs/dev/merge_semver_playbook.md +++ b/docs/dev/merge_semver_playbook.md @@ -1,154 +1,154 @@ -# Feedser SemVer Merge Playbook (Sprint 1–2) - -This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint 1–2, how those primitives become normalized version rules, and how downstream jobs query them deterministically. - -## 1. What landed in Sprint 1–2 - -- `RangePrimitives.SemVer` now infers a canonical `style` (`range`, `exact`, `lt`, `lte`, `gt`, `gte`) and captures `exactValue` when the constraint is a single version. -- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`. -- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison. - -See `src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions. - -## 2. Mapper pattern - -Connectors should emit SemVer primitives as soon as they can normalize a vendor constraint. The helper `SemVerPrimitiveExtensions.ToNormalizedVersionRule` turns those primitives into the persisted rules: - -```csharp -var primitive = new SemVerPrimitive( - introduced: "1.2.3", - introducedInclusive: true, - fixed: "2.0.0", - fixedInclusive: false, - lastAffected: null, - lastAffectedInclusive: false, - constraintExpression: ">=1.2.3 <2.0.0", - exactValue: null); - -var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234"); -// rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false -``` - -If you omit the optional `notes` argument, `ToNormalizedVersionRule` now falls back to the primitive’s `ConstraintExpression`, ensuring the original comparator expression is preserved for provenance/audit queries. - -Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility. - -## 3. Merge dedupe flow - -During merge, feed all package candidates through `NormalizedVersionRuleComparer.Instance` prior to persistence. The comparer orders by scheme → type → min → minInclusive → max → maxInclusive → value → notes, guaranteeing consistent document layout and making `$unwind` pipelines deterministic. - -If multiple connectors emit identical constraints, the merge layer should: - -1. Combine provenance entries (preserving one per source). -2. Preserve a single normalized rule instance (thanks to `NormalizedVersionRuleEqualityComparer.Instance`). -3. Attach `decisionReason="precedence"` if one source overrides another. - -## 4. Example Mongo pipeline - -Use the following aggregation to locate advisories that affect a specific SemVer: - -```javascript -db.advisories.aggregate([ - { $match: { "affectedPackages.type": "semver", "affectedPackages.identifier": "pkg:npm/lodash" } }, - { $unwind: "$affectedPackages" }, - { $unwind: "$affectedPackages.normalizedVersions" }, - { $match: { - $or: [ - { "affectedPackages.normalizedVersions.type": "exact", - "affectedPackages.normalizedVersions.value": "4.17.21" }, - { "affectedPackages.normalizedVersions.type": "range", - "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" }, - "affectedPackages.normalizedVersions.max": { $gt: "4.17.21" } }, - { "affectedPackages.normalizedVersions.type": "gte", - "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" } }, - { "affectedPackages.normalizedVersions.type": "lte", - "affectedPackages.normalizedVersions.max": { $gte: "4.17.21" } } - ] - }}, - { $project: { advisoryKey: 1, title: 1, "affectedPackages.identifier": 1 } } -]); -``` - -Pair this query with the indexes listed in [Normalized Versions Query Guide](mongo_indices.md). - -## 5. Recommended indexes - -| Collection | Index | Purpose | -|------------|-------|---------| -| `advisory` | `{ "affectedPackages.identifier": 1, "affectedPackages.normalizedVersions.scheme": 1, "affectedPackages.normalizedVersions.type": 1 }` (compound, multikey) | Speeds up `$match` on identifier + rule style. | -| `advisory` | `{ "affectedPackages.normalizedVersions.value": 1 }` (sparse) | Optimizes lookups for exact version hits. | - -Coordinate with the Storage team when enabling these indexes so deployment windows account for collection size. - -## 6. Dual-write rollout - -Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The summary: - -1. **Dual write (now)** – emit both legacy `versionRanges` and the new `normalizedVersions`. -2. **Backfill** – follow the storage migration in `docs/ops/migrations/SEMVER_STYLE.md` to rewrite historical advisories before switching consumers. -3. **Verify** – run the aggregation above (with `explain("executionStats")`) to ensure the new indexes are used. -4. **Cutover** – after consumers switch to normalized rules, mark the old `rangeExpression` as deprecated. - -## 7. Checklist for connectors & merge - -- [ ] Populate `SemVerPrimitive` for every SemVer-friendly constraint. -- [ ] Call `ToNormalizedVersionRule` and store the result. -- [ ] Emit provenance masks covering both `versionRanges[].primitives.semver` and `normalizedVersions[]`. -- [ ] Ensure merge deduping relies on the canonical comparer. -- [ ] Capture merge decisions via `decisionReason`. -- [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles. - -For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md). - -## 8. Storage projection reference - -`NormalizedVersionDocumentFactory` copies each normalized rule into MongoDB using the shape below. Use this as a contract when reviewing connector fixtures or diagnosing merge/storage diffs: - -```json -{ - "packageId": "pkg:npm/example", - "packageType": "npm", - "scheme": "semver", - "type": "range", - "style": "range", - "min": "1.2.3", - "minInclusive": true, - "max": "2.0.0", - "maxInclusive": false, - "value": null, - "notes": "ghsa:GHSA-xxxx-yyyy", - "decisionReason": "ghsa-precedence-over-nvd", - "constraint": ">= 1.2.3 < 2.0.0", - "source": "ghsa", - "recordedAt": "2025-10-11T00:00:00Z" -} -``` - -For distro-specific ranges (`nevra`, `evr`) the same envelope applies with `scheme` switched accordingly. Example: - -```json -{ - "packageId": "bash", - "packageType": "rpm", - "scheme": "nevra", - "type": "range", - "style": "range", - "min": "0:4.4.18-2.el7", - "minInclusive": true, - "max": "0:4.4.20-1.el7", - "maxInclusive": false, - "value": null, - "notes": "redhat:RHSA-2025:1234", - "decisionReason": "rhel-priority-over-nvd", - "constraint": "<= 0:4.4.20-1.el7", - "source": "redhat", - "recordedAt": "2025-10-11T00:00:00Z" -} -``` - -If a new scheme is required (for example, `apple.build` or `ios.semver`), raise it with the Models team before emitting documents so merge comparers and hashing logic can incorporate the change deterministically. - -## 9. Observability signals - -- `feedser.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) – increments once per normalized rule retained after precedence merge. -- `feedser.merge.normalized_rules_missing` (counter, tags: `package_type`) – increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet. +# Concelier SemVer Merge Playbook (Sprint 1–2) + +This playbook describes how the merge layer and connector teams should emit the new SemVer primitives introduced in Sprint 1–2, how those primitives become normalized version rules, and how downstream jobs query them deterministically. + +## 1. What landed in Sprint 1–2 + +- `RangePrimitives.SemVer` now infers a canonical `style` (`range`, `exact`, `lt`, `lte`, `gt`, `gte`) and captures `exactValue` when the constraint is a single version. +- `NormalizedVersionRule` documents the analytics-friendly projection of each `AffectedPackage` coverage entry and is persisted alongside legacy `versionRanges`. +- `AdvisoryProvenance.decisionReason` records whether merge resolution favored precedence, freshness, or a tie-breaker comparison. + +See `src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md` for the full schema and field descriptions. + +## 2. Mapper pattern + +Connectors should emit SemVer primitives as soon as they can normalize a vendor constraint. The helper `SemVerPrimitiveExtensions.ToNormalizedVersionRule` turns those primitives into the persisted rules: + +```csharp +var primitive = new SemVerPrimitive( + introduced: "1.2.3", + introducedInclusive: true, + fixed: "2.0.0", + fixedInclusive: false, + lastAffected: null, + lastAffectedInclusive: false, + constraintExpression: ">=1.2.3 <2.0.0", + exactValue: null); + +var rule = primitive.ToNormalizedVersionRule(notes: "nvd:CVE-2025-1234"); +// rule => scheme=semver, type=range, min=1.2.3, minInclusive=true, max=2.0.0, maxInclusive=false +``` + +If you omit the optional `notes` argument, `ToNormalizedVersionRule` now falls back to the primitive’s `ConstraintExpression`, ensuring the original comparator expression is preserved for provenance/audit queries. + +Emit the resulting rule inside `AffectedPackage.NormalizedVersions` while continuing to populate `AffectedVersionRange.RangeExpression` for backward compatibility. + +## 3. Merge dedupe flow + +During merge, feed all package candidates through `NormalizedVersionRuleComparer.Instance` prior to persistence. The comparer orders by scheme → type → min → minInclusive → max → maxInclusive → value → notes, guaranteeing consistent document layout and making `$unwind` pipelines deterministic. + +If multiple connectors emit identical constraints, the merge layer should: + +1. Combine provenance entries (preserving one per source). +2. Preserve a single normalized rule instance (thanks to `NormalizedVersionRuleEqualityComparer.Instance`). +3. Attach `decisionReason="precedence"` if one source overrides another. + +## 4. Example Mongo pipeline + +Use the following aggregation to locate advisories that affect a specific SemVer: + +```javascript +db.advisories.aggregate([ + { $match: { "affectedPackages.type": "semver", "affectedPackages.identifier": "pkg:npm/lodash" } }, + { $unwind: "$affectedPackages" }, + { $unwind: "$affectedPackages.normalizedVersions" }, + { $match: { + $or: [ + { "affectedPackages.normalizedVersions.type": "exact", + "affectedPackages.normalizedVersions.value": "4.17.21" }, + { "affectedPackages.normalizedVersions.type": "range", + "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" }, + "affectedPackages.normalizedVersions.max": { $gt: "4.17.21" } }, + { "affectedPackages.normalizedVersions.type": "gte", + "affectedPackages.normalizedVersions.min": { $lte: "4.17.21" } }, + { "affectedPackages.normalizedVersions.type": "lte", + "affectedPackages.normalizedVersions.max": { $gte: "4.17.21" } } + ] + }}, + { $project: { advisoryKey: 1, title: 1, "affectedPackages.identifier": 1 } } +]); +``` + +Pair this query with the indexes listed in [Normalized Versions Query Guide](mongo_indices.md). + +## 5. Recommended indexes + +| Collection | Index | Purpose | +|------------|-------|---------| +| `advisory` | `{ "affectedPackages.identifier": 1, "affectedPackages.normalizedVersions.scheme": 1, "affectedPackages.normalizedVersions.type": 1 }` (compound, multikey) | Speeds up `$match` on identifier + rule style. | +| `advisory` | `{ "affectedPackages.normalizedVersions.value": 1 }` (sparse) | Optimizes lookups for exact version hits. | + +Coordinate with the Storage team when enabling these indexes so deployment windows account for collection size. + +## 6. Dual-write rollout + +Follow the operational checklist in `docs/ops/migrations/SEMVER_STYLE.md`. The summary: + +1. **Dual write (now)** – emit both legacy `versionRanges` and the new `normalizedVersions`. +2. **Backfill** – follow the storage migration in `docs/ops/migrations/SEMVER_STYLE.md` to rewrite historical advisories before switching consumers. +3. **Verify** – run the aggregation above (with `explain("executionStats")`) to ensure the new indexes are used. +4. **Cutover** – after consumers switch to normalized rules, mark the old `rangeExpression` as deprecated. + +## 7. Checklist for connectors & merge + +- [ ] Populate `SemVerPrimitive` for every SemVer-friendly constraint. +- [ ] Call `ToNormalizedVersionRule` and store the result. +- [ ] Emit provenance masks covering both `versionRanges[].primitives.semver` and `normalizedVersions[]`. +- [ ] Ensure merge deduping relies on the canonical comparer. +- [ ] Capture merge decisions via `decisionReason`. +- [ ] Confirm integration tests include fixtures with normalized rules and SemVer styles. + +For deeper query examples and maintenance tasks, continue with [Normalized Versions Query Guide](mongo_indices.md). + +## 8. Storage projection reference + +`NormalizedVersionDocumentFactory` copies each normalized rule into MongoDB using the shape below. Use this as a contract when reviewing connector fixtures or diagnosing merge/storage diffs: + +```json +{ + "packageId": "pkg:npm/example", + "packageType": "npm", + "scheme": "semver", + "type": "range", + "style": "range", + "min": "1.2.3", + "minInclusive": true, + "max": "2.0.0", + "maxInclusive": false, + "value": null, + "notes": "ghsa:GHSA-xxxx-yyyy", + "decisionReason": "ghsa-precedence-over-nvd", + "constraint": ">= 1.2.3 < 2.0.0", + "source": "ghsa", + "recordedAt": "2025-10-11T00:00:00Z" +} +``` + +For distro-specific ranges (`nevra`, `evr`) the same envelope applies with `scheme` switched accordingly. Example: + +```json +{ + "packageId": "bash", + "packageType": "rpm", + "scheme": "nevra", + "type": "range", + "style": "range", + "min": "0:4.4.18-2.el7", + "minInclusive": true, + "max": "0:4.4.20-1.el7", + "maxInclusive": false, + "value": null, + "notes": "redhat:RHSA-2025:1234", + "decisionReason": "rhel-priority-over-nvd", + "constraint": "<= 0:4.4.20-1.el7", + "source": "redhat", + "recordedAt": "2025-10-11T00:00:00Z" +} +``` + +If a new scheme is required (for example, `apple.build` or `ios.semver`), raise it with the Models team before emitting documents so merge comparers and hashing logic can incorporate the change deterministically. + +## 9. Observability signals + +- `concelier.merge.normalized_rules` (counter, tags: `package_type`, `scheme`) – increments once per normalized rule retained after precedence merge. +- `concelier.merge.normalized_rules_missing` (counter, tags: `package_type`) – increments when a merged package still carries version ranges but no normalized rules; watch for spikes to catch connectors that have not emitted normalized arrays yet. diff --git a/docs/dev/mongo_indices.md b/docs/dev/mongo_indices.md index 53992628..4df1a0d0 100644 --- a/docs/dev/mongo_indices.md +++ b/docs/dev/mongo_indices.md @@ -1,108 +1,108 @@ -# Normalized Versions Query Guide - -This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`. - -For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section 8 of the [Feedser SemVer Merge Playbook](merge_semver_playbook.md). - -## 1. Recommended indexes - -When `feedser.storage.enableSemVerStyle` is enabled, advisories expose a flattened -`normalizedVersions` array at the document root. Create these indexes in `mongosh` -after the migration completes (adjust collection name if you use a prefix): - -```javascript -db.advisories.createIndex( - { - "normalizedVersions.packageId": 1, - "normalizedVersions.scheme": 1, - "normalizedVersions.type": 1 - }, - { name: "advisory_normalizedVersions_pkg_scheme_type" } -); - -db.advisories.createIndex( - { "normalizedVersions.value": 1 }, - { name: "advisory_normalizedVersions_value", sparse: true } -); -``` - -- The compound index accelerates `$match` stages that filter by package identifier and rule style without unwinding `affectedPackages`. -- The sparse index keeps storage costs low while supporting pure exact-version lookups (type `exact`). - -The storage bootstrapper creates the same indexes automatically when the feature flag is enabled. - -## 2. Query patterns - -### 2.1 Determine if a specific version is affected - -```javascript -db.advisories.aggregate([ - { $match: { "normalizedVersions.packageId": "pkg:npm/lodash" } }, - { $unwind: "$normalizedVersions" }, - { $match: { - $or: [ - { "normalizedVersions.type": "exact", - "normalizedVersions.value": "4.17.21" }, - { "normalizedVersions.type": "range", - "normalizedVersions.min": { $lte: "4.17.21" }, - "normalizedVersions.max": { $gt: "4.17.21" } }, - { "normalizedVersions.type": "gte", - "normalizedVersions.min": { $lte: "4.17.21" } }, - { "normalizedVersions.type": "lte", - "normalizedVersions.max": { $gte: "4.17.21" } } - ] - }}, - { $project: { advisoryKey: 1, title: 1, "normalizedVersions.packageId": 1 } } -]); -``` - -Use this pipeline during Sprint 2 staging validation runs. Invoke `explain("executionStats")` to confirm the compound index is selected. - -### 2.2 Locate advisories missing normalized rules - -```javascript -db.advisories.aggregate([ - { $match: { $or: [ - { "normalizedVersions": { $exists: false } }, - { "normalizedVersions": { $size: 0 } } - ] } }, - { $project: { advisoryKey: 1, affectedPackages: 1 } } -]); -``` - -Run this query after backfill jobs to identify gaps that still rely solely on `rangeExpression`. - -### 2.3 Deduplicate overlapping rules - -```javascript -db.advisories.aggregate([ - { $unwind: "$normalizedVersions" }, - { $group: { - _id: { - identifier: "$normalizedVersions.packageId", - scheme: "$normalizedVersions.scheme", - type: "$normalizedVersions.type", - min: "$normalizedVersions.min", - minInclusive: "$normalizedVersions.minInclusive", - max: "$normalizedVersions.max", - maxInclusive: "$normalizedVersions.maxInclusive", - value: "$normalizedVersions.value" - }, - advisories: { $addToSet: "$advisoryKey" }, - notes: { $addToSet: "$normalizedVersions.notes" } - }}, - { $match: { "advisories.1": { $exists: true } } }, - { $sort: { "_id.identifier": 1, "_id.type": 1 } } -]); -``` - -Use this to confirm the merge dedupe logic keeps only one normalized rule per unique constraint. - -## 3. Operational checklist - -- [ ] Create the indexes in staging before toggling dual-write in production. -- [ ] Capture explain plans and attach them to the release notes. -- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array. -- [ ] Update export fixtures once dedupe verification passes. - -Additional background and mapper examples live in [Feedser SemVer Merge Playbook](merge_semver_playbook.md). +# Normalized Versions Query Guide + +This guide complements the Sprint 1–2 normalized versions rollout. It documents recommended indexes and aggregation patterns for querying `AffectedPackage.normalizedVersions`. + +For a field-by-field look at how normalized rules persist in MongoDB (including provenance metadata), see Section 8 of the [Concelier SemVer Merge Playbook](merge_semver_playbook.md). + +## 1. Recommended indexes + +When `concelier.storage.enableSemVerStyle` is enabled, advisories expose a flattened +`normalizedVersions` array at the document root. Create these indexes in `mongosh` +after the migration completes (adjust collection name if you use a prefix): + +```javascript +db.advisories.createIndex( + { + "normalizedVersions.packageId": 1, + "normalizedVersions.scheme": 1, + "normalizedVersions.type": 1 + }, + { name: "advisory_normalizedVersions_pkg_scheme_type" } +); + +db.advisories.createIndex( + { "normalizedVersions.value": 1 }, + { name: "advisory_normalizedVersions_value", sparse: true } +); +``` + +- The compound index accelerates `$match` stages that filter by package identifier and rule style without unwinding `affectedPackages`. +- The sparse index keeps storage costs low while supporting pure exact-version lookups (type `exact`). + +The storage bootstrapper creates the same indexes automatically when the feature flag is enabled. + +## 2. Query patterns + +### 2.1 Determine if a specific version is affected + +```javascript +db.advisories.aggregate([ + { $match: { "normalizedVersions.packageId": "pkg:npm/lodash" } }, + { $unwind: "$normalizedVersions" }, + { $match: { + $or: [ + { "normalizedVersions.type": "exact", + "normalizedVersions.value": "4.17.21" }, + { "normalizedVersions.type": "range", + "normalizedVersions.min": { $lte: "4.17.21" }, + "normalizedVersions.max": { $gt: "4.17.21" } }, + { "normalizedVersions.type": "gte", + "normalizedVersions.min": { $lte: "4.17.21" } }, + { "normalizedVersions.type": "lte", + "normalizedVersions.max": { $gte: "4.17.21" } } + ] + }}, + { $project: { advisoryKey: 1, title: 1, "normalizedVersions.packageId": 1 } } +]); +``` + +Use this pipeline during Sprint 2 staging validation runs. Invoke `explain("executionStats")` to confirm the compound index is selected. + +### 2.2 Locate advisories missing normalized rules + +```javascript +db.advisories.aggregate([ + { $match: { $or: [ + { "normalizedVersions": { $exists: false } }, + { "normalizedVersions": { $size: 0 } } + ] } }, + { $project: { advisoryKey: 1, affectedPackages: 1 } } +]); +``` + +Run this query after backfill jobs to identify gaps that still rely solely on `rangeExpression`. + +### 2.3 Deduplicate overlapping rules + +```javascript +db.advisories.aggregate([ + { $unwind: "$normalizedVersions" }, + { $group: { + _id: { + identifier: "$normalizedVersions.packageId", + scheme: "$normalizedVersions.scheme", + type: "$normalizedVersions.type", + min: "$normalizedVersions.min", + minInclusive: "$normalizedVersions.minInclusive", + max: "$normalizedVersions.max", + maxInclusive: "$normalizedVersions.maxInclusive", + value: "$normalizedVersions.value" + }, + advisories: { $addToSet: "$advisoryKey" }, + notes: { $addToSet: "$normalizedVersions.notes" } + }}, + { $match: { "advisories.1": { $exists: true } } }, + { $sort: { "_id.identifier": 1, "_id.type": 1 } } +]); +``` + +Use this to confirm the merge dedupe logic keeps only one normalized rule per unique constraint. + +## 3. Operational checklist + +- [ ] Create the indexes in staging before toggling dual-write in production. +- [ ] Capture explain plans and attach them to the release notes. +- [ ] Notify downstream services that consume advisory snapshots about the new `normalizedVersions` array. +- [ ] Update export fixtures once dedupe verification passes. + +Additional background and mapper examples live in [Concelier SemVer Merge Playbook](merge_semver_playbook.md). diff --git a/docs/dev/normalized_versions_rollout.md b/docs/dev/normalized_versions_rollout.md index 94a91193..2a5b6091 100644 --- a/docs/dev/normalized_versions_rollout.md +++ b/docs/dev/normalized_versions_rollout.md @@ -1,11 +1,11 @@ -# Normalized Versions Rollout Dashboard (Sprint 2 – Feedser) +# Normalized Versions Rollout Dashboard (Sprint 2 – Concelier) _Status date: 2025-10-12 17:05 UTC_ This dashboard tracks connector readiness for emitting `AffectedPackage.NormalizedVersions` arrays and highlights upcoming coordination checkpoints. Use it alongside: -- [`src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines. -- [Feedser SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes. +- [`src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md`](../../src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md) for detailed guidance and timelines. +- [Concelier SemVer Merge Playbook](merge_semver_playbook.md) §8 for persisted Mongo document shapes. - [Normalized Versions Query Guide](mongo_indices.md) for index/query validation steps. ## Key milestones @@ -18,28 +18,28 @@ This dashboard tracks connector readiness for emitting `AffectedPackage.Normaliz | Connector | Owner team | Normalized versions status | Last update | Next action / link | |-----------|------------|---------------------------|-------------|--------------------| -| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Feedser.Source.Acsc/TASKS.md`. | -| Cccs | BE-Conn-CCCS | ❌ Not started – mapper pending | 2025-10-11 | Add normalized SemVer array in canonical mapper; coordinate fixtures per `TASKS.md`. | -| CertBund | BE-Conn-CERTBUND | ✅ Canonical mapper emitting vendor ranges | 2025-10-14 | Normalized vendor range payloads landed alongside telemetry/docs updates; see `src/StellaOps.Feedser.Source.CertBund/TASKS.md`. | -| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Feedser.Source.CertCc/TASKS.md`. | -| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Feedser.Source.Kev.Tests`; `src/StellaOps.Feedser.Source.Kev/TASKS.md`. | -| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Feedser.Source.Cve.Tests`); `src/StellaOps.Feedser.Source.Cve/TASKS.md`. | -| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Feedser.Source.Ghsa/TASKS.md`. | -| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Feedser.Source.Osv.Tests`); `src/StellaOps.Feedser.Source.Osv/TASKS.md`. | -| Ics.Cisa | BE-Conn-ICS-CISA | ❌ Not started – mapper TODO | 2025-10-11 | Plan SemVer/firmware scheme selection; `src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md`. | -| Kisa | BE-Conn-KISA | ✅ Landed 2025-10-14 (mapper + telemetry) | 2025-10-11 | Hangul-aware mapper emits normalized rules; see `docs/dev/kisa_connector_notes.md` for localisation/metric details. | -| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md`. | -| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md`. | -| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`. | -| Vndr.Cisco | BE-Conn-Cisco | ✅ SemVer + vendor extensions emitted | 2025-10-14 | Connector outputs SemVer primitives with `cisco.productId` notes; see `CiscoMapper` and fixtures for coverage. | -| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md`. | -| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Feedser.Source.Nvd/TASKS.md`. | +| Acsc | BE-Conn-ACSC | ❌ Not started – mapper pending | 2025-10-11 | Design DTOs + mapper with normalized rule array; see `src/StellaOps.Concelier.Connector.Acsc/TASKS.md`. | +| Cccs | BE-Conn-CCCS | ⚠️ Scheduled – helper ready, implementation due 2025-10-21 | 2025-10-19 | Apply Merge-provided trailing-version helper to emit `NormalizedVersions`; update mapper/tests per `src/StellaOps.Concelier.Connector.Cccs/TASKS.md`. | +| CertBund | BE-Conn-CERTBUND | ⚠️ Follow-up – translate `versions` strings to normalized rules | 2025-10-19 | Build `bis`/`alle` translator + fixtures before 2025-10-22 per `src/StellaOps.Concelier.Connector.CertBund/TASKS.md`. | +| CertCc | BE-Conn-CERTCC | ⚠️ In progress – fetch pipeline DOING | 2025-10-11 | Implement VINCE mapper with SemVer/NEVRA rules; unblock snapshot regeneration; `src/StellaOps.Concelier.Connector.CertCc/TASKS.md`. | +| Kev | BE-Conn-KEV | ✅ Normalized catalog/due-date rules verified | 2025-10-12 | Fixtures reconfirmed via `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests`; `src/StellaOps.Concelier.Connector.Kev/TASKS.md`. | +| Cve | BE-Conn-CVE | ✅ Normalized SemVer rules verified | 2025-10-12 | Snapshot parity green (`dotnet test src/StellaOps.Concelier.Connector.Cve.Tests`); `src/StellaOps.Concelier.Connector.Cve/TASKS.md`. | +| Ghsa | BE-Conn-GHSA | ⚠️ DOING – normalized rollout task active | 2025-10-11 18:45 UTC | Wire `SemVerRangeRuleBuilder` + refresh fixtures; `src/StellaOps.Concelier.Connector.Ghsa/TASKS.md`. | +| Osv | BE-Conn-OSV | ✅ SemVer mapper & parity fixtures verified | 2025-10-12 | GHSA parity regression passing (`dotnet test src/StellaOps.Concelier.Connector.Osv.Tests`); `src/StellaOps.Concelier.Connector.Osv/TASKS.md`. | +| Ics.Cisa | BE-Conn-ICS-CISA | ⚠️ Decision pending – normalize SemVer exacts or escalate scheme | 2025-10-19 | Promote `SemVerPrimitive` outputs into `NormalizedVersions` or file Models ticket by 2025-10-23 (`src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md`). | +| Kisa | BE-Conn-KISA | ⚠️ Proposal required – firmware scheme due 2025-10-24 | 2025-10-19 | Draft `kisa.build` (or equivalent) scheme with Models, then emit normalized rules; track in `src/StellaOps.Concelier.Connector.Kisa/TASKS.md`. | +| Ru.Bdu | BE-Conn-BDU | ✅ Raw scheme emitted | 2025-10-14 | Mapper now writes `ru-bdu.raw` normalized rules with provenance + telemetry; `src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md`. | +| Ru.Nkcki | BE-Conn-Nkcki | ❌ Not started – mapper TODO | 2025-10-11 | Similar to BDU; ensure Cyrillic provenance preserved; `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md`. | +| Vndr.Apple | BE-Conn-Apple | ✅ Shipped – emitting normalized arrays | 2025-10-11 | Continue fixture/tooling work; `src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`. | +| Vndr.Cisco | BE-Conn-Cisco | ⚠️ Scheduled – normalized rule emission due 2025-10-21 | 2025-10-19 | Use Merge helper to persist `NormalizedVersions` alongside SemVer primitives; see `src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md`. | +| Vndr.Msrc | BE-Conn-MSRC | ✅ Map + normalized build rules landed | 2025-10-15 | `MsrcMapper` emits `msrc.build` normalized rules with CVRF references; see `src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md`. | +| Nvd | BE-Conn-NVD | ⚠️ Needs follow-up – mapper complete but normalized array MR pending | 2025-10-11 | Align CVE notes + normalized payload flag; `src/StellaOps.Concelier.Connector.Nvd/TASKS.md`. | Legend: ✅ complete, ⚠️ in progress/partial, ❌ not started. ## Monitoring -- Merge now emits `feedser.merge.normalized_rules` (tags: `package_type`, `scheme`) and `feedser.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out. +- Merge now emits `concelier.merge.normalized_rules` (tags: `package_type`, `scheme`) and `concelier.merge.normalized_rules_missing` (tags: `package_type`). Track these counters to confirm normalized arrays land as connectors roll out. - Expect `normalized_rules_missing` to trend toward zero as each connector flips on normalized output. Investigate any sustained counts by checking the corresponding module `TASKS.md`. ## Implementation tips diff --git a/docs/dev/templates/excititor-connector/manifest/connector.manifest.yaml b/docs/dev/templates/excititor-connector/manifest/connector.manifest.yaml new file mode 100644 index 00000000..089316d2 --- /dev/null +++ b/docs/dev/templates/excititor-connector/manifest/connector.manifest.yaml @@ -0,0 +1,8 @@ +id: excititor-my-provider +assembly: StellaOps.Excititor.Connectors.MyProvider.dll +entryPoint: StellaOps.Excititor.Connectors.MyProvider.MyConnectorPlugin +description: | + Example connector template. Replace metadata before shipping. +tags: + - excititor + - template diff --git a/docs/dev/templates/excititor-connector/src/Excititor.MyConnector.csproj b/docs/dev/templates/excititor-connector/src/Excititor.MyConnector.csproj new file mode 100644 index 00000000..1e2e996c --- /dev/null +++ b/docs/dev/templates/excititor-connector/src/Excititor.MyConnector.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + true + + + + + + diff --git a/docs/dev/templates/excititor-connector/src/MyConnector.cs b/docs/dev/templates/excititor-connector/src/MyConnector.cs new file mode 100644 index 00000000..209ca2bb --- /dev/null +++ b/docs/dev/templates/excititor-connector/src/MyConnector.cs @@ -0,0 +1,72 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.MyProvider; + +public sealed class MyConnector : VexConnectorBase +{ + private readonly IEnumerable> _validators; + private MyConnectorOptions? _options; + + public MyConnector(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider timeProvider, IEnumerable> validators) + : base(descriptor, logger, timeProvider) + { + _validators = validators; + } + + public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + LogConnectorEvent(LogLevel.Information, "validate", "MyConnector configuration loaded.", + new Dictionary + { + ["catalogUri"] = _options.CatalogUri, + ["maxParallelRequests"] = _options.MaxParallelRequests, + }); + + return ValueTask.CompletedTask; + } + + public override IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken) + { + if (_options is null) + { + throw new InvalidOperationException("Connector not validated."); + } + + return FetchInternalAsync(context, cancellationToken); + } + + private async IAsyncEnumerable FetchInternalAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + LogConnectorEvent(LogLevel.Information, "fetch", "Fetching catalog window..."); + + // Replace with real HTTP logic. + await Task.Delay(10, cancellationToken); + + var metadata = BuildMetadata(builder => builder + .Add("sourceUri", _options!.CatalogUri) + .Add("window", context.Since?.ToString("O") ?? "full")); + + yield return CreateRawDocument( + VexDocumentFormat.CsafJson, + new Uri($"{_options.CatalogUri.TrimEnd('/')}/sample.json"), + new byte[] { 0x7B, 0x7D }, + metadata); + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + var claims = ImmutableArray.Empty; + var diagnostics = ImmutableDictionary.Empty; + return ValueTask.FromResult(new VexClaimBatch(document, claims, diagnostics)); + } +} diff --git a/docs/dev/templates/excititor-connector/src/MyConnectorOptions.cs b/docs/dev/templates/excititor-connector/src/MyConnectorOptions.cs new file mode 100644 index 00000000..585288a2 --- /dev/null +++ b/docs/dev/templates/excititor-connector/src/MyConnectorOptions.cs @@ -0,0 +1,16 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Excititor.Connectors.MyProvider; + +public sealed class MyConnectorOptions +{ + [Required] + [Url] + public string CatalogUri { get; set; } = default!; + + [Required] + public string ApiKey { get; set; } = default!; + + [Range(1, 32)] + public int MaxParallelRequests { get; set; } = 4; +} diff --git a/docs/dev/templates/excititor-connector/src/MyConnectorOptionsValidator.cs b/docs/dev/templates/excititor-connector/src/MyConnectorOptionsValidator.cs new file mode 100644 index 00000000..addb1c23 --- /dev/null +++ b/docs/dev/templates/excititor-connector/src/MyConnectorOptionsValidator.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.MyProvider; + +public sealed class MyConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + public void Validate(VexConnectorDescriptor descriptor, MyConnectorOptions options, IList errors) + { + if (!options.CatalogUri.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + errors.Add("CatalogUri must use HTTPS."); + } + } +} diff --git a/docs/dev/templates/excititor-connector/src/MyConnectorPlugin.cs b/docs/dev/templates/excititor-connector/src/MyConnectorPlugin.cs new file mode 100644 index 00000000..8257857f --- /dev/null +++ b/docs/dev/templates/excititor-connector/src/MyConnectorPlugin.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.MyProvider; + +public sealed class MyConnectorPlugin : IConnectorPlugin +{ + private static readonly VexConnectorDescriptor Descriptor = new( + id: "excititor:my-provider", + kind: VexProviderKind.Vendor, + displayName: "My Provider VEX"); + + public string Name => Descriptor.DisplayName; + + public bool IsAvailable(IServiceProvider services) => true; + + public IFeedConnector Create(IServiceProvider services) + { + var logger = services.GetRequiredService>(); + var timeProvider = services.GetRequiredService(); + var validators = services.GetServices>(); + return new MyConnector(Descriptor, logger, timeProvider, validators); + } +} diff --git a/docs/events/README.md b/docs/events/README.md new file mode 100644 index 00000000..7e97d049 --- /dev/null +++ b/docs/events/README.md @@ -0,0 +1,65 @@ +# Event Envelope Schemas + +Platform services publish strongly typed events; the JSON Schemas in this directory define those envelopes. File names follow `@.json` so producers and consumers can negotiate contracts explicitly. + +## Catalog +- `scanner.report.ready@1.json` — emitted by Scanner.WebService once a signed report is persisted (payload embeds the canonical report plus DSSE envelope). Consumers: Notify, UI timeline. +- `scanner.scan.completed@1.json` — emitted alongside the signed report to capture scan outcomes/summary data for downstream automation. Consumers: Notify, Scheduler backfills, UI timelines. +- `scheduler.rescan.delta@1.json` — emitted by Scheduler when BOM-Index diffs require fresh scans. Consumers: Notify, Policy Engine. +- `attestor.logged@1.json` — emitted by Attestor after storing the Rekor inclusion proof. Consumers: UI attestation panel, Governance exports. + +Additive payload changes (new optional fields) can stay within the same version. Any breaking change (removing a field, tightening validation, altering semantics) must increment the `@` suffix and update downstream consumers. + +## Envelope structure +All event envelopes share the same deterministic header. Use the following table as the quick reference when emitting or parsing events: + +| Field | Type | Notes | +|-------|------|-------| +| `eventId` | `uuid` | Must be globally unique per occurrence; producers log duplicates as fatal. | +| `kind` | `string` | Fixed per schema (e.g., `scanner.report.ready`). Downstream services reject unknown kinds or versions. | +| `tenant` | `string` | Multi‑tenant isolation key; mirror the value recorded in queue/Mongo metadata. | +| `ts` | `date-time` | RFC 3339 UTC timestamp. Use monotonic clocks or atomic offsets so ordering survives retries. | +| `scope` | `object` | Optional block used when the event concerns a specific image or repository. See schema for required fields (e.g., `repo`, `digest`). | +| `payload` | `object` | Event-specific body. Schemas allow additional properties so producers can add optional hints (e.g., `reportId`, `quietedFindingCount`) without breaking consumers. For scanner events, payloads embed both the canonical report document and the DSSE envelope so consumers can reuse signatures without recomputing them. See `docs/runtime/SCANNER_RUNTIME_READINESS.md` for the runtime consumer checklist covering these hints. | + +When adding new optional fields, document the behaviour in the schema’s `description` block and update the consumer checklist in the next sprint sync. + +## Canonical samples & validation +Reference payloads live under `docs/events/samples/`, mirroring the schema version (`@.sample.json`). They illustrate common field combinations, including the optional attributes that downstream teams rely on for UI affordances and audit trails. Scanner samples reuse the exact DSSE envelope checked into `samples/api/reports/report-sample.dsse.json`, and a unit test (`ReportSamplesTests`) guards that the payload/base64 remain canonical. + +Run the following loop offline to validate both schemas and samples: + +```bash +# Validate schemas (same check as CI) +for schema in docs/events/*.json; do + npx ajv compile -c ajv-formats -s "$schema" +done + +# Validate canonical samples against their schemas +for sample in docs/events/samples/*.sample.json; do + schema="docs/events/$(basename "${sample%.sample.json}").json" + npx ajv validate -c ajv-formats -s "$schema" -d "$sample" +done +``` + +Consumers can copy the samples into integration tests to guarantee backwards compatibility. When emitting new event versions, include a matching sample and update this README so air-gapped operators stay in sync. + +## CI validation +The Docs CI workflow (`.gitea/workflows/docs.yml`) installs `ajv-cli` and compiles every schema on pull requests. Run the same check locally before opening a PR: + +```bash +for schema in docs/events/*.json; do + npx ajv compile -c ajv-formats -s "$schema" +done +``` + +Tip: run `npm install --no-save ajv ajv-cli ajv-formats` once per clone so `npx` can resolve the tooling offline. + +If a schema references additional files, include `-r` flags so CI and local runs stay consistent. + +## Working with schemas +- Producers should validate outbound payloads using the matching schema during unit tests. +- Consumers should pin to a specific version and log when encountering unknown versions to catch missing migrations early. +- Store real payload samples under `docs/events/samples/` (mirrors the schema version) and mirror them into `samples/events/` when you need fixtures in integration repositories. + +Contact the Platform Events group in Docs Guild if you need help shaping a new event or version strategy. diff --git a/docs/events/attestor.logged@1.json b/docs/events/attestor.logged@1.json new file mode 100644 index 00000000..54631e86 --- /dev/null +++ b/docs/events/attestor.logged@1.json @@ -0,0 +1,38 @@ +{ + "$id": "https://stella-ops.org/schemas/events/attestor.logged@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["eventId", "kind", "tenant", "ts", "payload"], + "properties": { + "eventId": {"type": "string", "format": "uuid"}, + "kind": {"const": "attestor.logged"}, + "tenant": {"type": "string"}, + "ts": {"type": "string", "format": "date-time"}, + "payload": { + "type": "object", + "required": ["artifactSha256", "rekor", "subject"], + "properties": { + "artifactSha256": {"type": "string"}, + "rekor": { + "type": "object", + "required": ["uuid", "url"], + "properties": { + "uuid": {"type": "string"}, + "url": {"type": "string", "format": "uri"}, + "index": {"type": "integer", "minimum": 0} + } + }, + "subject": { + "type": "object", + "required": ["type", "name"], + "properties": { + "type": {"enum": ["sbom", "report", "vex-export"]}, + "name": {"type": "string"} + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/docs/events/samples/attestor.logged@1.sample.json b/docs/events/samples/attestor.logged@1.sample.json new file mode 100644 index 00000000..120140a1 --- /dev/null +++ b/docs/events/samples/attestor.logged@1.sample.json @@ -0,0 +1,21 @@ +{ + "eventId": "1fdcaa1a-7a27-4154-8bac-cf813d8f4f6f", + "kind": "attestor.logged", + "tenant": "tenant-acme-solar", + "ts": "2025-10-18T15:45:27+00:00", + "payload": { + "artifactSha256": "sha256:8927d9151ad3f44e61a9c647511f9a31af2b4d245e7e031fe5cb4a0e8211c5d9", + "dsseEnvelopeDigest": "sha256:51c1dd189d5f16cfe87e82841d67b4fbc27d6fa9f5a09af0cd7e18945fb4c2a9", + "rekor": { + "index": 563421, + "url": "https://rekor.example/api/v1/log/entries/d6d0f897e7244edc9cb0bb2c68b05c96", + "uuid": "d6d0f897e7244edc9cb0bb2c68b05c96" + }, + "signer": "cosign-stellaops", + "subject": { + "name": "scanner/report/sha256-0f0a8de5c1f93d6716b7249f6f4ea3a8", + "type": "report" + } + }, + "attributes": {} +} diff --git a/docs/events/samples/scanner.report.ready@1.sample.json b/docs/events/samples/scanner.report.ready@1.sample.json new file mode 100644 index 00000000..d76a96de --- /dev/null +++ b/docs/events/samples/scanner.report.ready@1.sample.json @@ -0,0 +1,70 @@ +{ + "eventId": "6d2d1b77-f3c3-4f70-8a9d-6f2d0c8801ab", + "kind": "scanner.report.ready", + "tenant": "tenant-alpha", + "ts": "2025-10-19T12:34:56+00:00", + "scope": { + "namespace": "acme/edge", + "repo": "api", + "digest": "sha256:feedface", + "labels": {}, + "attributes": {} + }, + "payload": { + "delta": { + "kev": ["CVE-2024-9999"], + "newCritical": 1 + }, + "dsse": { + "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", + "payloadType": "application/vnd.stellaops.report\u002Bjson", + "signatures": [{ + "algorithm": "hs256", + "keyId": "test-key", + "signature": "signature-value" + }] + }, + "generatedAt": "2025-10-19T12:34:56+00:00", + "links": { + "ui": "https://scanner.example/ui/reports/report-abc" + }, + "quietedFindingCount": 0, + "report": { + "generatedAt": "2025-10-19T12:34:56+00:00", + "imageDigest": "sha256:feedface", + "issues": [], + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "reportId": "report-abc", + "summary": { + "blocked": 1, + "ignored": 0, + "quieted": 0, + "total": 1, + "warned": 0 + }, + "verdict": "blocked", + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "score": 47.5, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ] + }, + "reportId": "report-abc", + "summary": { + "blocked": 1, + "ignored": 0, + "quieted": 0, + "total": 1, + "warned": 0 + }, + "verdict": "fail" + }, + "attributes": {} +} diff --git a/docs/events/samples/scanner.scan.completed@1.sample.json b/docs/events/samples/scanner.scan.completed@1.sample.json new file mode 100644 index 00000000..0d9c5fff --- /dev/null +++ b/docs/events/samples/scanner.scan.completed@1.sample.json @@ -0,0 +1,78 @@ +{ + "eventId": "08a6de24-4a94-4d14-8432-9d14f36f6da3", + "kind": "scanner.scan.completed", + "tenant": "tenant-alpha", + "ts": "2025-10-19T12:34:56+00:00", + "scope": { + "namespace": "acme/edge", + "repo": "api", + "digest": "sha256:feedface", + "labels": {}, + "attributes": {} + }, + "payload": { + "delta": { + "kev": ["CVE-2024-9999"], + "newCritical": 1 + }, + "digest": "sha256:feedface", + "dsse": { + "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC1hYmMiLCJpbWFnZURpZ2VzdCI6InNoYTI1NjpmZWVkZmFjZSIsImdlbmVyYXRlZEF0IjoiMjAyNS0xMC0xOVQxMjozNDo1NiswMDowMCIsInZlcmRpY3QiOiJibG9ja2VkIiwicG9saWN5Ijp7InJldmlzaW9uSWQiOiJyZXYtNDIiLCJkaWdlc3QiOiJkaWdlc3QtMTIzIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwic2NvcmUiOjQ3LjUsInNvdXJjZVRydXN0IjoiTlZEIiwicmVhY2hhYmlsaXR5IjoicnVudGltZSJ9XSwiaXNzdWVzIjpbXX0=", + "payloadType": "application/vnd.stellaops.report\u002Bjson", + "signatures": [{ + "algorithm": "hs256", + "keyId": "test-key", + "signature": "signature-value" + }] + }, + "findings": [ + { + "cve": "CVE-2024-9999", + "id": "finding-1", + "reachability": "runtime", + "severity": "Critical" + } + ], + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "report": { + "generatedAt": "2025-10-19T12:34:56+00:00", + "imageDigest": "sha256:feedface", + "issues": [], + "policy": { + "digest": "digest-123", + "revisionId": "rev-42" + }, + "reportId": "report-abc", + "summary": { + "blocked": 1, + "ignored": 0, + "quieted": 0, + "total": 1, + "warned": 0 + }, + "verdict": "blocked", + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "score": 47.5, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ] + }, + "reportId": "report-abc", + "summary": { + "blocked": 1, + "ignored": 0, + "quieted": 0, + "total": 1, + "warned": 0 + }, + "verdict": "fail" + }, + "attributes": {} +} diff --git a/docs/events/samples/scheduler.rescan.delta@1.sample.json b/docs/events/samples/scheduler.rescan.delta@1.sample.json new file mode 100644 index 00000000..7cbedbdb --- /dev/null +++ b/docs/events/samples/scheduler.rescan.delta@1.sample.json @@ -0,0 +1,20 @@ +{ + "eventId": "51d0ef8d-3a17-4af3-b2d7-4ad3db3d9d2c", + "kind": "scheduler.rescan.delta", + "tenant": "tenant-acme-solar", + "ts": "2025-10-18T15:40:11+00:00", + "payload": { + "impactedDigests": [ + "sha256:0f0a8de5c1f93d6716b7249f6f4ea3a8db451dc3f3c3ff823f53c9cbde5d5e8a", + "sha256:ab921f9679dd8d0832f3710a4df75dbadbd58c2d95f26a4d4efb2fa8c3d9b4ce" + ], + "reason": "policy-change:scoring/v2", + "scheduleId": "rescan-weekly-critical", + "summary": { + "newCritical": 0, + "newHigh": 1, + "total": 4 + } + }, + "attributes": {} +} diff --git a/docs/events/scanner.report.ready@1.json b/docs/events/scanner.report.ready@1.json new file mode 100644 index 00000000..31523de6 --- /dev/null +++ b/docs/events/scanner.report.ready@1.json @@ -0,0 +1,84 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scanner.report.ready@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["eventId", "kind", "tenant", "ts", "scope", "payload"], + "properties": { + "eventId": {"type": "string", "format": "uuid"}, + "kind": {"const": "scanner.report.ready"}, + "tenant": {"type": "string"}, + "ts": {"type": "string", "format": "date-time"}, + "scope": { + "type": "object", + "required": ["repo", "digest"], + "properties": { + "namespace": {"type": "string"}, + "repo": {"type": "string"}, + "digest": {"type": "string"} + } + }, + "payload": { + "type": "object", + "required": ["verdict", "delta", "links"], + "properties": { + "reportId": {"type": "string"}, + "generatedAt": {"type": "string", "format": "date-time"}, + "verdict": {"enum": ["pass", "warn", "fail"]}, + "summary": { + "type": "object", + "properties": { + "total": {"type": "integer", "minimum": 0}, + "blocked": {"type": "integer", "minimum": 0}, + "warned": {"type": "integer", "minimum": 0}, + "ignored": {"type": "integer", "minimum": 0}, + "quieted": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "delta": { + "type": "object", + "properties": { + "newCritical": {"type": "integer", "minimum": 0}, + "newHigh": {"type": "integer", "minimum": 0}, + "kev": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + }, + "links": { + "type": "object", + "properties": { + "ui": {"type": "string", "format": "uri"}, + "rekor": {"type": "string", "format": "uri"} + }, + "additionalProperties": false + }, + "quietedFindingCount": {"type": "integer", "minimum": 0}, + "report": {"type": "object"}, + "dsse": { + "type": "object", + "required": ["payloadType", "payload", "signatures"], + "properties": { + "payloadType": {"type": "string"}, + "payload": {"type": "string"}, + "signatures": { + "type": "array", + "items": { + "type": "object", + "required": ["keyId", "algorithm", "signature"], + "properties": { + "keyId": {"type": "string"}, + "algorithm": {"type": "string"}, + "signature": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/docs/events/scanner.scan.completed@1.json b/docs/events/scanner.scan.completed@1.json new file mode 100644 index 00000000..7e6e51a8 --- /dev/null +++ b/docs/events/scanner.scan.completed@1.json @@ -0,0 +1,97 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scanner.scan.completed@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["eventId", "kind", "tenant", "ts", "scope", "payload"], + "properties": { + "eventId": {"type": "string", "format": "uuid"}, + "kind": {"const": "scanner.scan.completed"}, + "tenant": {"type": "string"}, + "ts": {"type": "string", "format": "date-time"}, + "scope": { + "type": "object", + "required": ["repo", "digest"], + "properties": { + "namespace": {"type": "string"}, + "repo": {"type": "string"}, + "digest": {"type": "string"} + } + }, + "payload": { + "type": "object", + "required": ["reportId", "digest", "verdict", "summary"], + "properties": { + "reportId": {"type": "string"}, + "digest": {"type": "string"}, + "verdict": {"enum": ["pass", "warn", "fail"]}, + "summary": { + "type": "object", + "properties": { + "total": {"type": "integer", "minimum": 0}, + "blocked": {"type": "integer", "minimum": 0}, + "warned": {"type": "integer", "minimum": 0}, + "ignored": {"type": "integer", "minimum": 0}, + "quieted": {"type": "integer", "minimum": 0} + }, + "additionalProperties": false + }, + "delta": { + "type": "object", + "properties": { + "newCritical": {"type": "integer", "minimum": 0}, + "newHigh": {"type": "integer", "minimum": 0}, + "kev": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + }, + "policy": { + "type": "object", + "properties": { + "revisionId": {"type": "string"}, + "digest": {"type": "string"} + }, + "additionalProperties": false + }, + "findings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "severity": {"type": "string"}, + "cve": {"type": "string"}, + "purl": {"type": "string"}, + "reachability": {"type": "string"} + }, + "additionalProperties": true + } + }, + "report": {"type": "object"}, + "dsse": { + "type": "object", + "required": ["payloadType", "payload", "signatures"], + "properties": { + "payloadType": {"type": "string"}, + "payload": {"type": "string"}, + "signatures": { + "type": "array", + "items": { + "type": "object", + "required": ["keyId", "algorithm", "signature"], + "properties": { + "keyId": {"type": "string"}, + "algorithm": {"type": "string"}, + "signature": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/docs/events/scheduler.rescan.delta@1.json b/docs/events/scheduler.rescan.delta@1.json new file mode 100644 index 00000000..f95db12b --- /dev/null +++ b/docs/events/scheduler.rescan.delta@1.json @@ -0,0 +1,33 @@ +{ + "$id": "https://stella-ops.org/schemas/events/scheduler.rescan.delta@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["eventId", "kind", "tenant", "ts", "payload"], + "properties": { + "eventId": {"type": "string", "format": "uuid"}, + "kind": {"const": "scheduler.rescan.delta"}, + "tenant": {"type": "string"}, + "ts": {"type": "string", "format": "date-time"}, + "payload": { + "type": "object", + "required": ["scheduleId", "impactedDigests", "summary"], + "properties": { + "scheduleId": {"type": "string"}, + "impactedDigests": { + "type": "array", + "items": {"type": "string"} + }, + "summary": { + "type": "object", + "properties": { + "newCritical": {"type": "integer", "minimum": 0}, + "newHigh": {"type": "integer", "minimum": 0}, + "total": {"type": "integer", "minimum": 0} + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/docs/notify/samples/notify-channel@1.sample.json b/docs/notify/samples/notify-channel@1.sample.json new file mode 100644 index 00000000..f9865169 --- /dev/null +++ b/docs/notify/samples/notify-channel@1.sample.json @@ -0,0 +1,32 @@ +{ + "schemaVersion": "notify.channel@1", + "channelId": "channel-slack-sec-ops", + "tenantId": "tenant-01", + "name": "slack:sec-ops", + "type": "slack", + "displayName": "SecOps Slack", + "description": "Primary incident response channel.", + "config": { + "secretRef": "ref://notify/channels/slack/sec-ops", + "target": "#sec-ops", + "properties": { + "workspace": "stellaops-sec" + }, + "limits": { + "concurrency": 2, + "requestsPerMinute": 60, + "timeout": "PT10S" + } + }, + "enabled": true, + "labels": { + "team": "secops" + }, + "metadata": { + "createdByTask": "NOTIFY-MODELS-15-102" + }, + "createdBy": "ops:amir", + "createdAt": "2025-10-18T17:02:11+00:00", + "updatedBy": "ops:amir", + "updatedAt": "2025-10-18T17:45:00+00:00" +} diff --git a/docs/notify/samples/notify-delivery-list-response.sample.json b/docs/notify/samples/notify-delivery-list-response.sample.json new file mode 100644 index 00000000..9fa108a1 --- /dev/null +++ b/docs/notify/samples/notify-delivery-list-response.sample.json @@ -0,0 +1,46 @@ +{ + "items": [ + { + "deliveryId": "delivery-7f3b6c51", + "tenantId": "tenant-acme", + "ruleId": "rule-critical-slack", + "actionId": "slack-secops", + "eventId": "4f6e9c09-01b4-4c2a-8a57-3d06de182d74", + "kind": "scanner.report.ready", + "status": "Sent", + "statusReason": null, + "rendered": { + "channelType": "Slack", + "format": "Slack", + "target": "#sec-alerts", + "title": "Critical findings detected", + "body": "{\"text\":\"Critical findings detected\",\"blocks\":[{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"*Critical findings detected*\\n1 new critical finding across 2 images.\"}},{\"type\":\"context\",\"elements\":[{\"type\":\"mrkdwn\",\"text\":\"Preview generated 2025-10-19T16:23:41.889Z · Trace `trace-58c212`\"}]}]}", + "summary": "1 new critical finding across 2 images.", + "textBody": "1 new critical finding across 2 images.\nTrace: trace-58c212", + "locale": "en-us", + "bodyHash": "febf9b2a630d862b07f4390edfbf31f5e8b836529f5232c491f4b3f6dba4a4b2", + "attachments": [] + }, + "attempts": [ + { + "timestamp": "2025-10-19T16:23:42.112Z", + "status": "Succeeded", + "statusCode": 200, + "reason": null + } + ], + "metadata": { + "channelType": "slack", + "target": "#sec-alerts", + "previewProvider": "fallback", + "traceId": "trace-58c212", + "slack.channel": "#sec-alerts" + }, + "createdAt": "2025-10-19T16:23:41.889Z", + "sentAt": "2025-10-19T16:23:42.101Z", + "completedAt": "2025-10-19T16:23:42.112Z" + } + ], + "count": 1, + "continuationToken": "2025-10-19T16:23:41.889Z|tenant-acme:delivery-7f3b6c51" +} diff --git a/docs/notify/samples/notify-event@1.sample.json b/docs/notify/samples/notify-event@1.sample.json new file mode 100644 index 00000000..33742cdd --- /dev/null +++ b/docs/notify/samples/notify-event@1.sample.json @@ -0,0 +1,34 @@ +{ + "eventId": "8a8d6a2f-9315-49fe-9d52-8fec79ec7aeb", + "kind": "scanner.report.ready", + "version": "1", + "tenant": "tenant-01", + "ts": "2025-10-19T03:58:42+00:00", + "actor": "scanner-webservice", + "scope": { + "namespace": "prod-payment", + "repo": "ghcr.io/acme/api", + "digest": "sha256:79c1f9e5...", + "labels": { + "environment": "production" + }, + "attributes": {} + }, + "payload": { + "delta": { + "kev": [ + "CVE-2025-40123" + ], + "newCritical": 1, + "newHigh": 2 + }, + "links": { + "rekor": "https://rekor.stella.local/api/v1/log/entries/1", + "ui": "https://ui.stella.local/reports/sha256-79c1f9e5" + }, + "verdict": "fail" + }, + "attributes": { + "correlationId": "scan-23a6" + } +} diff --git a/docs/notify/samples/notify-rule@1.sample.json b/docs/notify/samples/notify-rule@1.sample.json new file mode 100644 index 00000000..5cfabe6d --- /dev/null +++ b/docs/notify/samples/notify-rule@1.sample.json @@ -0,0 +1,63 @@ +{ + "schemaVersion": "notify.rule@1", + "ruleId": "rule-secops-critical", + "tenantId": "tenant-01", + "name": "Critical digests to SecOps", + "description": "Escalate KEV-tagged findings to on-call feeds.", + "enabled": true, + "match": { + "eventKinds": [ + "scanner.report.ready", + "scheduler.rescan.delta" + ], + "namespaces": [ + "prod-*" + ], + "repositories": [], + "digests": [], + "labels": [], + "componentPurls": [], + "minSeverity": "high", + "verdicts": [], + "kevOnly": true, + "vex": { + "includeAcceptedJustifications": false, + "includeRejectedJustifications": false, + "includeUnknownJustifications": false, + "justificationKinds": [ + "component-remediated", + "not-affected" + ] + } + }, + "actions": [ + { + "actionId": "email-digest", + "channel": "email:soc", + "digest": "hourly", + "template": "digest", + "enabled": true, + "metadata": { + "locale": "en-us" + } + }, + { + "actionId": "slack-oncall", + "channel": "slack:sec-ops", + "template": "concise", + "throttle": "PT5M", + "metadata": {}, + "enabled": true + } + ], + "labels": { + "team": "secops" + }, + "metadata": { + "source": "sprint-15" + }, + "createdBy": "ops:zoya", + "createdAt": "2025-10-19T04:12:27+00:00", + "updatedBy": "ops:zoya", + "updatedAt": "2025-10-19T04:45:03+00:00" +} diff --git a/docs/notify/samples/notify-template@1.sample.json b/docs/notify/samples/notify-template@1.sample.json new file mode 100644 index 00000000..de28ed5f --- /dev/null +++ b/docs/notify/samples/notify-template@1.sample.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "notify.template@1", + "templateId": "tmpl-slack-concise", + "tenantId": "tenant-01", + "channelType": "slack", + "key": "concise", + "locale": "en-us", + "body": "{{severity_icon payload.delta.newCritical}} {{summary}}", + "description": "Slack concise message for high severity findings.", + "renderMode": "markdown", + "format": "slack", + "metadata": { + "version": "2025-10-19" + }, + "createdBy": "ops:zoya", + "createdAt": "2025-10-19T05:00:00+00:00", + "updatedBy": "ops:zoya", + "updatedAt": "2025-10-19T05:45:00+00:00" +} diff --git a/docs/notify/schemas/notify-channel@1.json b/docs/notify/schemas/notify-channel@1.json new file mode 100644 index 00000000..4cda3816 --- /dev/null +++ b/docs/notify/schemas/notify-channel@1.json @@ -0,0 +1,73 @@ +{ + "$id": "https://stella-ops.org/schemas/notify/notify-channel@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notify Channel", + "type": "object", + "required": [ + "schemaVersion", + "channelId", + "tenantId", + "name", + "type", + "config", + "enabled", + "createdAt", + "updatedAt" + ], + "properties": { + "schemaVersion": {"type": "string", "const": "notify.channel@1"}, + "channelId": {"type": "string"}, + "tenantId": {"type": "string"}, + "name": {"type": "string"}, + "type": { + "type": "string", + "enum": ["slack", "teams", "email", "webhook", "custom"] + }, + "displayName": {"type": "string"}, + "description": {"type": "string"}, + "config": {"$ref": "#/$defs/channelConfig"}, + "enabled": {"type": "boolean"}, + "labels": {"$ref": "#/$defs/stringMap"}, + "metadata": {"$ref": "#/$defs/stringMap"}, + "createdBy": {"type": "string"}, + "createdAt": {"type": "string", "format": "date-time"}, + "updatedBy": {"type": "string"}, + "updatedAt": {"type": "string", "format": "date-time"} + }, + "additionalProperties": false, + "$defs": { + "channelConfig": { + "type": "object", + "required": ["secretRef"], + "properties": { + "secretRef": {"type": "string"}, + "target": {"type": "string"}, + "endpoint": {"type": "string", "format": "uri"}, + "properties": {"$ref": "#/$defs/stringMap"}, + "limits": {"$ref": "#/$defs/channelLimits"} + }, + "additionalProperties": false + }, + "channelLimits": { + "type": "object", + "properties": { + "concurrency": {"type": "integer", "minimum": 1}, + "requestsPerMinute": {"type": "integer", "minimum": 1}, + "timeout": { + "type": "string", + "pattern": "^P(T.*)?$", + "description": "ISO 8601 duration" + }, + "maxBatchSize": {"type": "integer", "minimum": 1} + }, + "additionalProperties": false + }, + "stringMap": { + "type": "object", + "patternProperties": { + ".*": {"type": "string"} + }, + "additionalProperties": false + } + } +} diff --git a/docs/notify/schemas/notify-event@1.json b/docs/notify/schemas/notify-event@1.json new file mode 100644 index 00000000..63c95f4e --- /dev/null +++ b/docs/notify/schemas/notify-event@1.json @@ -0,0 +1,56 @@ +{ + "$id": "https://stella-ops.org/schemas/notify/notify-event@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notify Event Envelope", + "type": "object", + "required": ["eventId", "kind", "tenant", "ts", "payload"], + "properties": { + "eventId": {"type": "string", "format": "uuid"}, + "kind": { + "type": "string", + "description": "Event kind identifier (e.g. scanner.report.ready).", + "enum": [ + "scanner.report.ready", + "scanner.scan.completed", + "scheduler.rescan.delta", + "attestor.logged", + "zastava.admission", + "feedser.export.completed", + "vexer.export.completed" + ] + }, + "version": {"type": "string"}, + "tenant": {"type": "string"}, + "ts": {"type": "string", "format": "date-time"}, + "actor": {"type": "string"}, + "scope": { + "type": "object", + "properties": { + "namespace": {"type": "string"}, + "repo": {"type": "string"}, + "digest": {"type": "string"}, + "component": {"type": "string"}, + "image": {"type": "string"}, + "labels": {"$ref": "#/$defs/stringMap"}, + "attributes": {"$ref": "#/$defs/stringMap"} + }, + "additionalProperties": false + }, + "payload": { + "type": "object", + "description": "Event specific body; see individual schemas for shapes.", + "additionalProperties": true + }, + "attributes": {"$ref": "#/$defs/stringMap"} + }, + "additionalProperties": false, + "$defs": { + "stringMap": { + "type": "object", + "patternProperties": { + ".*": {"type": "string"} + }, + "additionalProperties": false + } + } +} diff --git a/docs/notify/schemas/notify-rule@1.json b/docs/notify/schemas/notify-rule@1.json new file mode 100644 index 00000000..80627f00 --- /dev/null +++ b/docs/notify/schemas/notify-rule@1.json @@ -0,0 +1,96 @@ +{ + "$id": "https://stella-ops.org/schemas/notify/notify-rule@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notify Rule", + "type": "object", + "required": [ + "schemaVersion", + "ruleId", + "tenantId", + "name", + "enabled", + "match", + "actions", + "createdAt", + "updatedAt" + ], + "properties": { + "schemaVersion": {"type": "string", "const": "notify.rule@1"}, + "ruleId": {"type": "string"}, + "tenantId": {"type": "string"}, + "name": {"type": "string"}, + "description": {"type": "string"}, + "enabled": {"type": "boolean"}, + "match": {"$ref": "#/$defs/ruleMatch"}, + "actions": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/ruleAction"} + }, + "labels": {"$ref": "#/$defs/stringMap"}, + "metadata": {"$ref": "#/$defs/stringMap"}, + "createdBy": {"type": "string"}, + "createdAt": {"type": "string", "format": "date-time"}, + "updatedBy": {"type": "string"}, + "updatedAt": {"type": "string", "format": "date-time"} + }, + "additionalProperties": false, + "$defs": { + "ruleMatch": { + "type": "object", + "properties": { + "eventKinds": {"$ref": "#/$defs/stringArray"}, + "namespaces": {"$ref": "#/$defs/stringArray"}, + "repositories": {"$ref": "#/$defs/stringArray"}, + "digests": {"$ref": "#/$defs/stringArray"}, + "labels": {"$ref": "#/$defs/stringArray"}, + "componentPurls": {"$ref": "#/$defs/stringArray"}, + "minSeverity": {"type": "string"}, + "verdicts": {"$ref": "#/$defs/stringArray"}, + "kevOnly": {"type": "boolean"}, + "vex": {"$ref": "#/$defs/ruleMatchVex"} + }, + "additionalProperties": false + }, + "ruleMatchVex": { + "type": "object", + "properties": { + "includeAcceptedJustifications": {"type": "boolean"}, + "includeRejectedJustifications": {"type": "boolean"}, + "includeUnknownJustifications": {"type": "boolean"}, + "justificationKinds": {"$ref": "#/$defs/stringArray"} + }, + "additionalProperties": false + }, + "ruleAction": { + "type": "object", + "required": ["actionId", "channel", "enabled"], + "properties": { + "actionId": {"type": "string"}, + "channel": {"type": "string"}, + "template": {"type": "string"}, + "digest": {"type": "string"}, + "throttle": { + "type": "string", + "pattern": "^P(T.*)?$", + "description": "ISO 8601 duration" + }, + "locale": {"type": "string"}, + "enabled": {"type": "boolean"}, + "metadata": {"$ref": "#/$defs/stringMap"} + }, + "additionalProperties": false + }, + "stringArray": { + "type": "array", + "items": {"type": "string"} + }, + "stringMap": { + "type": "object", + "patternProperties": { + ".*": {"type": "string"} + }, + "additionalProperties": false + } + } +} diff --git a/docs/notify/schemas/notify-template@1.json b/docs/notify/schemas/notify-template@1.json new file mode 100644 index 00000000..343e370e --- /dev/null +++ b/docs/notify/schemas/notify-template@1.json @@ -0,0 +1,55 @@ +{ + "$id": "https://stella-ops.org/schemas/notify/notify-template@1.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notify Template", + "type": "object", + "required": [ + "schemaVersion", + "templateId", + "tenantId", + "channelType", + "key", + "locale", + "body", + "renderMode", + "format", + "createdAt", + "updatedAt" + ], + "properties": { + "schemaVersion": {"type": "string", "const": "notify.template@1"}, + "templateId": {"type": "string"}, + "tenantId": {"type": "string"}, + "channelType": { + "type": "string", + "enum": ["slack", "teams", "email", "webhook", "custom"] + }, + "key": {"type": "string"}, + "locale": {"type": "string"}, + "body": {"type": "string"}, + "description": {"type": "string"}, + "renderMode": { + "type": "string", + "enum": ["markdown", "html", "adaptiveCard", "plainText", "json"] + }, + "format": { + "type": "string", + "enum": ["slack", "teams", "email", "webhook", "json"] + }, + "metadata": {"$ref": "#/$defs/stringMap"}, + "createdBy": {"type": "string"}, + "createdAt": {"type": "string", "format": "date-time"}, + "updatedBy": {"type": "string"}, + "updatedAt": {"type": "string", "format": "date-time"} + }, + "additionalProperties": false, + "$defs": { + "stringMap": { + "type": "object", + "patternProperties": { + ".*": {"type": "string"} + }, + "additionalProperties": false + } + } +} diff --git a/docs/ops/authority-backup-restore.md b/docs/ops/authority-backup-restore.md index bef10c8c..7c8f9d72 100644 --- a/docs/ops/authority-backup-restore.md +++ b/docs/ops/authority-backup-restore.md @@ -1,97 +1,97 @@ -# Authority Backup & Restore Runbook - -## Scope -- **Applies to:** StellaOps Authority deployments running the official `ops/authority/docker-compose.authority.yaml` stack or equivalent Kubernetes packaging. -- **Artifacts covered:** MongoDB (`stellaops-authority` database), Authority configuration (`etc/authority.yaml`), plugin manifests under `etc/authority.plugins/`, and signing key material stored in the `authority-keys` volume (defaults to `/app/keys` inside the container). -- **Frequency:** Run the full procedure prior to upgrades, before rotating keys, and at least once per 24 h in production. Store snapshots in an encrypted, access-controlled vault. - -## Inventory Checklist -| Component | Location (compose default) | Notes | -| --- | --- | --- | -| Mongo data | `mongo-data` volume (`/var/lib/docker/volumes/.../mongo-data`) | Contains all Authority collections (`AuthorityUser`, `AuthorityClient`, `AuthorityToken`, etc.). | -| Configuration | `etc/authority.yaml` | Mounted read-only into the container at `/etc/authority.yaml`. | -| Plugin manifests | `etc/authority.plugins/*.yaml` | Includes `standard.yaml` with `tokenSigning.keyDirectory`. | -| Signing keys | `authority-keys` volume -> `/app/keys` | Path is derived from `tokenSigning.keyDirectory` (defaults to `../keys` relative to the manifest). | - -> **TIP:** Confirm the deployed key directory via `tokenSigning.keyDirectory` in `etc/authority.plugins/standard.yaml`; some installations relocate keys to `/var/lib/stellaops/authority/keys`. - -## Hot Backup (no downtime) -1. **Create output directory:** `mkdir -p backup/$(date +%Y-%m-%d)` on the host. -2. **Dump Mongo:** - ```bash - docker compose -f ops/authority/docker-compose.authority.yaml exec mongo \ - mongodump --archive=/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz \ - --gzip --db stellaops-authority - docker compose -f ops/authority/docker-compose.authority.yaml cp \ - mongo:/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz backup/ - ``` - The `mongodump` archive preserves indexes and can be restored with `mongorestore --archive --gzip`. -3. **Capture configuration + manifests:** - ```bash - cp etc/authority.yaml backup/ - rsync -a etc/authority.plugins/ backup/authority.plugins/ - ``` -4. **Export signing keys:** the compose file maps `authority-keys` to a local Docker volume. Snapshot it without stopping the service: - ```bash - docker run --rm \ - -v authority-keys:/keys \ - -v "$(pwd)/backup:/backup" \ - busybox tar czf /backup/authority-keys-$(date +%Y%m%dT%H%M%SZ).tar.gz -C /keys . - ``` -5. **Checksum:** generate SHA-256 digests for every file and store them alongside the artefacts. -6. **Encrypt & upload:** wrap the backup folder using your secrets management standard (e.g., age, GPG) and upload to the designated offline vault. - -## Cold Backup (planned downtime) -1. Notify stakeholders and drain traffic (CLI clients should refresh tokens afterwards). -2. Stop services: - ```bash - docker compose -f ops/authority/docker-compose.authority.yaml down - ``` -3. Back up volumes directly using `tar`: - ```bash - docker run --rm -v mongo-data:/data -v "$(pwd)/backup:/backup" \ - busybox tar czf /backup/mongo-data-$(date +%Y%m%d).tar.gz -C /data . - docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ - busybox tar czf /backup/authority-keys-$(date +%Y%m%d).tar.gz -C /keys . - ``` -4. Copy configuration + manifests as in the hot backup (steps 3–6). -5. Restart services and verify health: - ```bash - docker compose -f ops/authority/docker-compose.authority.yaml up -d - curl -fsS http://localhost:8080/ready - ``` - -## Restore Procedure -1. **Provision clean volumes:** remove existing volumes if you’re rebuilding a node (`docker volume rm mongo-data authority-keys`), then recreate the compose stack so empty volumes exist. -2. **Restore Mongo:** - ```bash - docker compose exec -T mongo mongorestore --archive --gzip --drop < backup/authority-YYYYMMDDTHHMMSSZ.gz - ``` - Use `--drop` to replace collections; omit if doing a partial restore. -3. **Restore configuration/manifests:** copy `authority.yaml` and `authority.plugins/*` into place before starting the Authority container. -4. **Restore signing keys:** untar into the mounted volume: - ```bash - docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ - busybox tar xzf /backup/authority-keys-YYYYMMDD.tar.gz -C /keys - ``` - Ensure file permissions remain `600` for private keys (`chmod -R 600`). -5. **Start services & validate:** - ```bash - docker compose up -d - curl -fsS http://localhost:8080/health - ``` -6. **Validate JWKS and tokens:** call `/jwks` and issue a short-lived token via the CLI to confirm key material matches expectations. If the restored environment requires a fresh signing key, follow the rotation SOP in [`docs/11_AUTHORITY.md`](../11_AUTHORITY.md) using `ops/authority/key-rotation.sh` to invoke `/internal/signing/rotate`. - -## Disaster Recovery Notes -- **Air-gapped replication:** replicate archives via the Offline Update Kit transport channels; never attach USB devices without scanning. -- **Retention:** maintain 30 daily snapshots + 12 monthly archival copies. Rotate encryption keys annually. -- **Key compromise:** if signing keys are suspected compromised, restore from the latest clean backup, rotate via OPS3 (see `ops/authority/key-rotation.sh` and `docs/11_AUTHORITY.md`), and publish a revocation notice. -- **Mongo version:** keep dump/restore images pinned to the deployment version (compose uses `mongo:7`). Restoring across major versions requires a compatibility review. - -## Verification Checklist -- [ ] `/ready` reports all identity providers ready. -- [ ] OAuth flows issue tokens signed by the restored keys. -- [ ] `PluginRegistrationSummary` logs expected providers on startup. -- [ ] Revocation manifest export (`dotnet run --project src/StellaOps.Authority`) succeeds. -- [ ] Monitoring dashboards show metrics resuming (see OPS5 deliverables). - +# Authority Backup & Restore Runbook + +## Scope +- **Applies to:** StellaOps Authority deployments running the official `ops/authority/docker-compose.authority.yaml` stack or equivalent Kubernetes packaging. +- **Artifacts covered:** MongoDB (`stellaops-authority` database), Authority configuration (`etc/authority.yaml`), plugin manifests under `etc/authority.plugins/`, and signing key material stored in the `authority-keys` volume (defaults to `/app/keys` inside the container). +- **Frequency:** Run the full procedure prior to upgrades, before rotating keys, and at least once per 24 h in production. Store snapshots in an encrypted, access-controlled vault. + +## Inventory Checklist +| Component | Location (compose default) | Notes | +| --- | --- | --- | +| Mongo data | `mongo-data` volume (`/var/lib/docker/volumes/.../mongo-data`) | Contains all Authority collections (`AuthorityUser`, `AuthorityClient`, `AuthorityToken`, etc.). | +| Configuration | `etc/authority.yaml` | Mounted read-only into the container at `/etc/authority.yaml`. | +| Plugin manifests | `etc/authority.plugins/*.yaml` | Includes `standard.yaml` with `tokenSigning.keyDirectory`. | +| Signing keys | `authority-keys` volume -> `/app/keys` | Path is derived from `tokenSigning.keyDirectory` (defaults to `../keys` relative to the manifest). | + +> **TIP:** Confirm the deployed key directory via `tokenSigning.keyDirectory` in `etc/authority.plugins/standard.yaml`; some installations relocate keys to `/var/lib/stellaops/authority/keys`. + +## Hot Backup (no downtime) +1. **Create output directory:** `mkdir -p backup/$(date +%Y-%m-%d)` on the host. +2. **Dump Mongo:** + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml exec mongo \ + mongodump --archive=/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz \ + --gzip --db stellaops-authority + docker compose -f ops/authority/docker-compose.authority.yaml cp \ + mongo:/dump/authority-$(date +%Y%m%dT%H%M%SZ).gz backup/ + ``` + The `mongodump` archive preserves indexes and can be restored with `mongorestore --archive --gzip`. +3. **Capture configuration + manifests:** + ```bash + cp etc/authority.yaml backup/ + rsync -a etc/authority.plugins/ backup/authority.plugins/ + ``` +4. **Export signing keys:** the compose file maps `authority-keys` to a local Docker volume. Snapshot it without stopping the service: + ```bash + docker run --rm \ + -v authority-keys:/keys \ + -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/authority-keys-$(date +%Y%m%dT%H%M%SZ).tar.gz -C /keys . + ``` +5. **Checksum:** generate SHA-256 digests for every file and store them alongside the artefacts. +6. **Encrypt & upload:** wrap the backup folder using your secrets management standard (e.g., age, GPG) and upload to the designated offline vault. + +## Cold Backup (planned downtime) +1. Notify stakeholders and drain traffic (CLI clients should refresh tokens afterwards). +2. Stop services: + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml down + ``` +3. Back up volumes directly using `tar`: + ```bash + docker run --rm -v mongo-data:/data -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/mongo-data-$(date +%Y%m%d).tar.gz -C /data . + docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ + busybox tar czf /backup/authority-keys-$(date +%Y%m%d).tar.gz -C /keys . + ``` +4. Copy configuration + manifests as in the hot backup (steps 3–6). +5. Restart services and verify health: + ```bash + docker compose -f ops/authority/docker-compose.authority.yaml up -d + curl -fsS http://localhost:8080/ready + ``` + +## Restore Procedure +1. **Provision clean volumes:** remove existing volumes if you’re rebuilding a node (`docker volume rm mongo-data authority-keys`), then recreate the compose stack so empty volumes exist. +2. **Restore Mongo:** + ```bash + docker compose exec -T mongo mongorestore --archive --gzip --drop < backup/authority-YYYYMMDDTHHMMSSZ.gz + ``` + Use `--drop` to replace collections; omit if doing a partial restore. +3. **Restore configuration/manifests:** copy `authority.yaml` and `authority.plugins/*` into place before starting the Authority container. +4. **Restore signing keys:** untar into the mounted volume: + ```bash + docker run --rm -v authority-keys:/keys -v "$(pwd)/backup:/backup" \ + busybox tar xzf /backup/authority-keys-YYYYMMDD.tar.gz -C /keys + ``` + Ensure file permissions remain `600` for private keys (`chmod -R 600`). +5. **Start services & validate:** + ```bash + docker compose up -d + curl -fsS http://localhost:8080/health + ``` +6. **Validate JWKS and tokens:** call `/jwks` and issue a short-lived token via the CLI to confirm key material matches expectations. If the restored environment requires a fresh signing key, follow the rotation SOP in [`docs/11_AUTHORITY.md`](../11_AUTHORITY.md) using `ops/authority/key-rotation.sh` to invoke `/internal/signing/rotate`. + +## Disaster Recovery Notes +- **Air-gapped replication:** replicate archives via the Offline Update Kit transport channels; never attach USB devices without scanning. +- **Retention:** maintain 30 daily snapshots + 12 monthly archival copies. Rotate encryption keys annually. +- **Key compromise:** if signing keys are suspected compromised, restore from the latest clean backup, rotate via OPS3 (see `ops/authority/key-rotation.sh` and `docs/11_AUTHORITY.md`), and publish a revocation notice. +- **Mongo version:** keep dump/restore images pinned to the deployment version (compose uses `mongo:7`). Driver 3.5.0 requires MongoDB **4.2+**—clusters still on 4.0 must be upgraded before restore, and future driver releases will drop 4.0 entirely. citeturn1open1 + +## Verification Checklist +- [ ] `/ready` reports all identity providers ready. +- [ ] OAuth flows issue tokens signed by the restored keys. +- [ ] `PluginRegistrationSummary` logs expected providers on startup. +- [ ] Revocation manifest export (`dotnet run --project src/StellaOps.Authority`) succeeds. +- [ ] Monitoring dashboards show metrics resuming (see OPS5 deliverables). + diff --git a/docs/ops/feedser-apple-operations.md b/docs/ops/concelier-apple-operations.md similarity index 69% rename from docs/ops/feedser-apple-operations.md rename to docs/ops/concelier-apple-operations.md index d87fa39b..ecf39fd2 100644 --- a/docs/ops/feedser-apple-operations.md +++ b/docs/ops/concelier-apple-operations.md @@ -1,77 +1,77 @@ -# Feedser Apple Security Update Connector Operations - -This runbook covers staging and production rollout for the Apple security updates connector (`source:vndr-apple:*`), including observability checks and fixture maintenance. - -## 1. Prerequisites - -- Network egress (or mirrored cache) for `https://gdmf.apple.com/v2/pmv` and the Apple Support domain (`https://support.apple.com/`). -- Optional: corporate proxy exclusions for the Apple hosts if outbound traffic is normally filtered. -- Updated configuration (environment variables or `feedser.yaml`) with an `apple` section. Example baseline: - -```yaml -feedser: - sources: - apple: - softwareLookupUri: "https://gdmf.apple.com/v2/pmv" - advisoryBaseUri: "https://support.apple.com/" - localeSegment: "en-us" - maxAdvisoriesPerFetch: 25 - initialBackfill: "120.00:00:00" - modifiedTolerance: "02:00:00" - failureBackoff: "00:05:00" -``` - -> ℹ️ `softwareLookupUri` and `advisoryBaseUri` must stay absolute and aligned with the HTTP allow-list; Feedser automatically adds both hosts to the connector HttpClient. - -## 2. Staging Smoke Test - -1. Deploy the configuration and restart the Feedser workers to ensure the Apple connector options are bound. -2. Trigger a full connector cycle: - - CLI: `stella db jobs run source:vndr-apple:fetch --and-then source:vndr-apple:parse --and-then source:vndr-apple:map` - - REST: `POST /jobs/run { "kind": "source:vndr-apple:fetch", "chain": ["source:vndr-apple:parse", "source:vndr-apple:map"] }` -3. Validate metrics exported under meter `StellaOps.Feedser.Source.Vndr.Apple`: - - `apple.fetch.items` (documents fetched) - - `apple.fetch.failures` - - `apple.fetch.unchanged` - - `apple.parse.failures` - - `apple.map.affected.count` (histogram of affected package counts) -4. Cross-check the shared HTTP counters: - - `feedser.source.http.requests_total{feedser_source="vndr-apple"}` should increase for both index and detail phases. - - `feedser.source.http.failures_total{feedser_source="vndr-apple"}` should remain flat (0) during a healthy run. -5. Inspect the info logs: - - `Apple software index fetch … processed=X newDocuments=Y` - - `Apple advisory parse complete … aliases=… affected=…` - - `Mapped Apple advisory … pendingMappings=0` -6. Confirm MongoDB state: - - `raw_documents` store contains the HT article HTML with metadata (`apple.articleId`, `apple.postingDate`). - - `dtos` store has `schemaVersion="apple.security.update.v1"`. - - `advisories` collection includes keys `HTxxxxxx` with normalized SemVer rules. - - `source_states` entry for `apple` shows a recent `cursor.lastPosted`. - -## 3. Production Monitoring - -- **Dashboards** – Add the following expressions to your Feedser Grafana board (OTLP/Prometheus naming assumed): - - `rate(apple_fetch_items_total[15m])` vs `rate(feedser_source_http_requests_total{feedser_source="vndr-apple"}[15m])` - - `rate(apple_fetch_failures_total[5m])` for error spikes (`severity=warning` at `>0`) - - `histogram_quantile(0.95, rate(apple_map_affected_count_bucket[1h]))` to watch affected-package fan-out - - `increase(apple_parse_failures_total[6h])` to catch parser drift (alerts at `>0`) -- **Alerts** – Page if `rate(apple_fetch_items_total[2h]) == 0` during business hours while other connectors are active. This often indicates lookup feed failures or misconfigured allow-lists. -- **Logs** – Surface warnings `Apple document {DocumentId} missing GridFS payload` or `Apple parse failed`—repeated hits imply storage issues or HTML regressions. -- **Telemetry pipeline** – `StellaOps.Feedser.WebService` now exports `StellaOps.Feedser.Source.Vndr.Apple` alongside existing Feedser meters; ensure your OTEL collector or Prometheus scraper includes it. - -## 4. Fixture Maintenance - -Regression fixtures live under `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures`. Refresh them whenever Apple reshapes the HT layout or when new platforms appear. - -1. Run the helper script matching your platform: - - Bash: `./scripts/update-apple-fixtures.sh` - - PowerShell: `./scripts/update-apple-fixtures.ps1` -2. Each script exports `UPDATE_APPLE_FIXTURES=1`, updates the `WSLENV` passthrough, and touches `.update-apple-fixtures` so WSL+VS Code test runs observe the flag. The subsequent test execution fetches the live HT articles listed in `AppleFixtureManager`, sanitises the HTML, and rewrites the `.expected.json` DTO snapshots. -3. Review the diff for localisation or nav noise. Once satisfied, re-run the tests without the env var (`dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj`) to verify determinism. -4. Commit fixture updates together with any parser/mapping changes that motivated them. - -## 5. Known Issues & Follow-up Tasks - -- Apple occasionally throttles anonymous requests after bursts. The connector backs off automatically, but persistent `apple.fetch.failures` spikes might require mirroring the HT content or scheduling wider fetch windows. -- Rapid Security Responses may appear before the general patch notes surface in the lookup JSON. When that happens, the fetch run will log `detailFailures>0`. Collect sample HTML and refresh fixtures to confirm parser coverage. -- Multi-locale content is still under regression sweep (`src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md`). Capture non-`en-us` snapshots once the fixture tooling stabilises. +# Concelier Apple Security Update Connector Operations + +This runbook covers staging and production rollout for the Apple security updates connector (`source:vndr-apple:*`), including observability checks and fixture maintenance. + +## 1. Prerequisites + +- Network egress (or mirrored cache) for `https://gdmf.apple.com/v2/pmv` and the Apple Support domain (`https://support.apple.com/`). +- Optional: corporate proxy exclusions for the Apple hosts if outbound traffic is normally filtered. +- Updated configuration (environment variables or `concelier.yaml`) with an `apple` section. Example baseline: + +```yaml +concelier: + sources: + apple: + softwareLookupUri: "https://gdmf.apple.com/v2/pmv" + advisoryBaseUri: "https://support.apple.com/" + localeSegment: "en-us" + maxAdvisoriesPerFetch: 25 + initialBackfill: "120.00:00:00" + modifiedTolerance: "02:00:00" + failureBackoff: "00:05:00" +``` + +> ℹ️ `softwareLookupUri` and `advisoryBaseUri` must stay absolute and aligned with the HTTP allow-list; Concelier automatically adds both hosts to the connector HttpClient. + +## 2. Staging Smoke Test + +1. Deploy the configuration and restart the Concelier workers to ensure the Apple connector options are bound. +2. Trigger a full connector cycle: + - CLI: `stella db jobs run source:vndr-apple:fetch --and-then source:vndr-apple:parse --and-then source:vndr-apple:map` + - REST: `POST /jobs/run { "kind": "source:vndr-apple:fetch", "chain": ["source:vndr-apple:parse", "source:vndr-apple:map"] }` +3. Validate metrics exported under meter `StellaOps.Concelier.Connector.Vndr.Apple`: + - `apple.fetch.items` (documents fetched) + - `apple.fetch.failures` + - `apple.fetch.unchanged` + - `apple.parse.failures` + - `apple.map.affected.count` (histogram of affected package counts) +4. Cross-check the shared HTTP counters: + - `concelier.source.http.requests_total{concelier_source="vndr-apple"}` should increase for both index and detail phases. + - `concelier.source.http.failures_total{concelier_source="vndr-apple"}` should remain flat (0) during a healthy run. +5. Inspect the info logs: + - `Apple software index fetch … processed=X newDocuments=Y` + - `Apple advisory parse complete … aliases=… affected=…` + - `Mapped Apple advisory … pendingMappings=0` +6. Confirm MongoDB state: + - `raw_documents` store contains the HT article HTML with metadata (`apple.articleId`, `apple.postingDate`). + - `dtos` store has `schemaVersion="apple.security.update.v1"`. + - `advisories` collection includes keys `HTxxxxxx` with normalized SemVer rules. + - `source_states` entry for `apple` shows a recent `cursor.lastPosted`. + +## 3. Production Monitoring + +- **Dashboards** – Add the following expressions to your Concelier Grafana board (OTLP/Prometheus naming assumed): + - `rate(apple_fetch_items_total[15m])` vs `rate(concelier_source_http_requests_total{concelier_source="vndr-apple"}[15m])` + - `rate(apple_fetch_failures_total[5m])` for error spikes (`severity=warning` at `>0`) + - `histogram_quantile(0.95, rate(apple_map_affected_count_bucket[1h]))` to watch affected-package fan-out + - `increase(apple_parse_failures_total[6h])` to catch parser drift (alerts at `>0`) +- **Alerts** – Page if `rate(apple_fetch_items_total[2h]) == 0` during business hours while other connectors are active. This often indicates lookup feed failures or misconfigured allow-lists. +- **Logs** – Surface warnings `Apple document {DocumentId} missing GridFS payload` or `Apple parse failed`—repeated hits imply storage issues or HTML regressions. +- **Telemetry pipeline** – `StellaOps.Concelier.WebService` now exports `StellaOps.Concelier.Connector.Vndr.Apple` alongside existing Concelier meters; ensure your OTEL collector or Prometheus scraper includes it. + +## 4. Fixture Maintenance + +Regression fixtures live under `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. Refresh them whenever Apple reshapes the HT layout or when new platforms appear. + +1. Run the helper script matching your platform: + - Bash: `./scripts/update-apple-fixtures.sh` + - PowerShell: `./scripts/update-apple-fixtures.ps1` +2. Each script exports `UPDATE_APPLE_FIXTURES=1`, updates the `WSLENV` passthrough, and touches `.update-apple-fixtures` so WSL+VS Code test runs observe the flag. The subsequent test execution fetches the live HT articles listed in `AppleFixtureManager`, sanitises the HTML, and rewrites the `.expected.json` DTO snapshots. +3. Review the diff for localisation or nav noise. Once satisfied, re-run the tests without the env var (`dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj`) to verify determinism. +4. Commit fixture updates together with any parser/mapping changes that motivated them. + +## 5. Known Issues & Follow-up Tasks + +- Apple occasionally throttles anonymous requests after bursts. The connector backs off automatically, but persistent `apple.fetch.failures` spikes might require mirroring the HT content or scheduling wider fetch windows. +- Rapid Security Responses may appear before the general patch notes surface in the lookup JSON. When that happens, the fetch run will log `detailFailures>0`. Collect sample HTML and refresh fixtures to confirm parser coverage. +- Multi-locale content is still under regression sweep (`src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md`). Capture non-`en-us` snapshots once the fixture tooling stabilises. diff --git a/docs/ops/feedser-authority-audit-runbook.md b/docs/ops/concelier-authority-audit-runbook.md similarity index 57% rename from docs/ops/feedser-authority-audit-runbook.md rename to docs/ops/concelier-authority-audit-runbook.md index 3262037a..9e96535d 100644 --- a/docs/ops/feedser-authority-audit-runbook.md +++ b/docs/ops/concelier-authority-audit-runbook.md @@ -1,150 +1,150 @@ -# Feedser Authority Audit Runbook - -_Last updated: 2025-10-12_ - -This runbook helps operators verify and monitor the StellaOps Feedser ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity. - -## 1. Prerequisites - -- Authority integration is enabled in `feedser.yaml` (or via `FEEDSER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes. -- OTLP metrics/log exporters are configured (`feedser.telemetry.*`) or container stdout is shipped to your SIEM. -- Operators have access to the Feedser job trigger endpoints via CLI or REST for smoke tests. - -### Configuration snippet - -```yaml -feedser: - authority: - enabled: true - allowAnonymousFallback: false # keep true only during initial rollout - issuer: "https://authority.internal" - audiences: - - "api://feedser" - requiredScopes: - - "feedser.jobs.trigger" - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - clientId: "feedser-jobs" - clientSecretFile: "/run/secrets/feedser_authority_client" - tokenClockSkewSeconds: 60 - resilience: - enableRetries: true - retryDelays: - - "00:00:01" - - "00:00:02" - - "00:00:05" - allowOfflineCacheFallback: true - offlineCacheTolerance: "00:10:00" -``` - -> Store secrets outside source control. Feedser reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service. - -### Resilience tuning - -- **Connected sites:** keep the default 1 s / 2 s / 5 s retry ladder so Feedser retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts. -- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (15–30 minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Feedser will fail fast but keep deterministic logs. -- Feedser resolves these knobs through `IOptionsMonitor`. Edits to `feedser.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled. - -## 2. Key Signals - -### 2.1 Audit log channel - -Feedser emits structured audit entries via the `Feedser.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active. - -``` -Feedser authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=feedser-cli scopes=feedser.jobs.trigger bypass=False remote=10.1.4.7 -``` - -| Field | Sample value | Meaning | -|--------------|-------------------------|------------------------------------------------------------------------------------------| -| `route` | `/jobs/definitions` | Endpoint that processed the request. | -| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. | -| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). | -| `clientId` | `feedser-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). | -| `scopes` | `feedser.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. | -| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. | -| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. | - -Use your logging backend (e.g., Loki) to index the logger name and filter for suspicious combinations: - -- `status=401 AND bypass=True` – bypass network accepted an unauthenticated call (should be temporary during rollout). -- `status=202 AND scopes="(none)"` – a token without scopes triggered a job; tighten client configuration. -- Spike in `clientId="(none)"` – indicates upstream Authority is not issuing `client_id` claims or the CLI is outdated. - -### 2.2 Metrics - -Feedser publishes counters under the OTEL meter `StellaOps.Feedser.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`. - -| Metric name | Description | PromQL example | -|-------------------------------|----------------------------------------------------|----------------| -| `web.jobs.triggered` | Accepted job trigger requests. | `sum by (job_kind) (rate(web_jobs_triggered_total[5m]))` | -| `web.jobs.trigger.conflict` | Rejected triggers (already running, disabled…). | `sum(rate(web_jobs_trigger_conflict_total[5m]))` | -| `web.jobs.trigger.failed` | Server-side job failures. | `sum(rate(web_jobs_trigger_failed_total[5m]))` | - -> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipeline’s generated metric names. - -Correlate audit logs with the following global meter exported via `Feedser.SourceDiagnostics`: - -- `feedser.source.http.requests_total{feedser_source="jobs-run"}` – ensures REST/manual triggers route through Authority. -- If Grafana dashboards are deployed, extend the “Feedser Jobs” board with the above counters plus a table of recent audit log entries. - -## 3. Alerting Guidance - -1. **Unauthorized bypass attempt** - - Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", status="401", bypass="True"}[5m])) > 0` - - Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious. - -2. **Missing scopes** - - Query: `sum(rate(log_messages_total{logger="Feedser.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0` - - Action: audit Authority client registration; ensure `requiredScopes` includes `feedser.jobs.trigger`. - -3. **Trigger failure surge** - - Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes. - - Action: inspect correlated audit entries and `Feedser.Telemetry` traces for job execution errors. - -4. **Conflict spike** - - Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold). - - Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly. - -5. **Authority offline** - - Watch `Feedser.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback. - -## 4. Rollout & Verification Procedure - -1. **Pre-checks** - - Confirm `allowAnonymousFallback` is `false` in production; keep `true` only during staged validation. - - Validate Authority issuer metadata is reachable from Feedser (`curl https://authority.internal/.well-known/openid-configuration` from the host). - -2. **Smoke test with valid token** - - Obtain a token via CLI: `stella auth login --scope feedser.jobs.trigger`. - - Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://feedser.internal/jobs/definitions`. - - Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=feedser.jobs.trigger`. - -3. **Negative test without token** - - Call the same endpoint without a token. Expect HTTP 401, `bypass=False`. - - If the request succeeds, double-check `bypassNetworks` and ensure fallback is disabled. - -4. **Bypass check (if applicable)** - - From an allowed maintenance IP, call `/jobs/definitions` without a token. Confirm the audit log shows `bypass=True`. Review business justification and expiry date for such entries. - -5. **Metrics validation** - - Ensure `web.jobs.triggered` counter increments during accepted runs. - - Exporters should show corresponding spans (`feedser.job.trigger`) if tracing is enabled. - -## 5. Troubleshooting - -| Symptom | Probable cause | Remediation | -|---------|----------------|-------------| -| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. | -| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Feedser. | -| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`feedser.jobs.trigger`) and ensure the token audience matches `audiences` config. | -| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `feedser.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Feedser.WebService.Jobs` meter. | -| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Feedser job logs, re-run with tracing enabled, validate Authority latency. | - -## 6. References - -- `docs/21_INSTALL_GUIDE.md` – Authority configuration quick start. -- `docs/17_SECURITY_HARDENING_GUIDE.md` – Security guardrails and enforcement deadlines. -- `docs/ops/authority-monitoring.md` – Authority-side monitoring and alerting playbook. -- `StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs` – source of audit log fields. +# Concelier Authority Audit Runbook + +_Last updated: 2025-10-12_ + +This runbook helps operators verify and monitor the StellaOps Concelier ⇆ Authority integration. It focuses on the `/jobs*` surface, which now requires StellaOps Authority tokens, and the corresponding audit/metric signals that expose authentication and bypass activity. + +## 1. Prerequisites + +- Authority integration is enabled in `concelier.yaml` (or via `CONCELIER_AUTHORITY__*` environment variables) with a valid `clientId`, secret, audience, and required scopes. +- OTLP metrics/log exporters are configured (`concelier.telemetry.*`) or container stdout is shipped to your SIEM. +- Operators have access to the Concelier job trigger endpoints via CLI or REST for smoke tests. + +### Configuration snippet + +```yaml +concelier: + authority: + enabled: true + allowAnonymousFallback: false # keep true only during initial rollout + issuer: "https://authority.internal" + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + clientId: "concelier-jobs" + clientSecretFile: "/run/secrets/concelier_authority_client" + tokenClockSkewSeconds: 60 + resilience: + enableRetries: true + retryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + allowOfflineCacheFallback: true + offlineCacheTolerance: "00:10:00" +``` + +> Store secrets outside source control. Concelier reads `clientSecretFile` on startup; rotate by updating the mounted file and restarting the service. + +### Resilience tuning + +- **Connected sites:** keep the default 1 s / 2 s / 5 s retry ladder so Concelier retries transient Authority hiccups but still surfaces outages quickly. Leave `allowOfflineCacheFallback=true` so cached discovery/JWKS data can bridge short Pathfinder restarts. +- **Air-gapped/Offline Kit installs:** extend `offlineCacheTolerance` (15–30 minutes) to keep the cached metadata valid between manual synchronisations. You can also disable retries (`enableRetries=false`) if infrastructure teams prefer to handle exponential backoff at the network layer; Concelier will fail fast but keep deterministic logs. +- Concelier resolves these knobs through `IOptionsMonitor`. Edits to `concelier.yaml` are applied on configuration reload; restart the container if you change environment variables or do not have file-watch reloads enabled. + +## 2. Key Signals + +### 2.1 Audit log channel + +Concelier emits structured audit entries via the `Concelier.Authorization.Audit` logger for every `/jobs*` request once Authority enforcement is active. + +``` +Concelier authorization audit route=/jobs/definitions status=200 subject=ops@example.com clientId=concelier-cli scopes=concelier.jobs.trigger bypass=False remote=10.1.4.7 +``` + +| Field | Sample value | Meaning | +|--------------|-------------------------|------------------------------------------------------------------------------------------| +| `route` | `/jobs/definitions` | Endpoint that processed the request. | +| `status` | `200` / `401` / `409` | Final HTTP status code returned to the caller. | +| `subject` | `ops@example.com` | User or service principal subject (falls back to `(anonymous)` when unauthenticated). | +| `clientId` | `concelier-cli` | OAuth client ID provided by Authority ( `(none)` if the token lacked the claim). | +| `scopes` | `concelier.jobs.trigger` | Normalised scope list extracted from token claims; `(none)` if the token carried none. | +| `bypass` | `True` / `False` | Indicates whether the request succeeded because its source IP matched a bypass CIDR. | +| `remote` | `10.1.4.7` | Remote IP recorded from the connection / forwarded header test hooks. | + +Use your logging backend (e.g., Loki) to index the logger name and filter for suspicious combinations: + +- `status=401 AND bypass=True` – bypass network accepted an unauthenticated call (should be temporary during rollout). +- `status=202 AND scopes="(none)"` – a token without scopes triggered a job; tighten client configuration. +- Spike in `clientId="(none)"` – indicates upstream Authority is not issuing `client_id` claims or the CLI is outdated. + +### 2.2 Metrics + +Concelier publishes counters under the OTEL meter `StellaOps.Concelier.WebService.Jobs`. Tags: `job.kind`, `job.trigger`, `job.outcome`. + +| Metric name | Description | PromQL example | +|-------------------------------|----------------------------------------------------|----------------| +| `web.jobs.triggered` | Accepted job trigger requests. | `sum by (job_kind) (rate(web_jobs_triggered_total[5m]))` | +| `web.jobs.trigger.conflict` | Rejected triggers (already running, disabled…). | `sum(rate(web_jobs_trigger_conflict_total[5m]))` | +| `web.jobs.trigger.failed` | Server-side job failures. | `sum(rate(web_jobs_trigger_failed_total[5m]))` | + +> Prometheus/OTEL collectors typically surface counters with `_total` suffix. Adjust queries to match your pipeline’s generated metric names. + +Correlate audit logs with the following global meter exported via `Concelier.SourceDiagnostics`: + +- `concelier.source.http.requests_total{concelier_source="jobs-run"}` – ensures REST/manual triggers route through Authority. +- If Grafana dashboards are deployed, extend the “Concelier Jobs” board with the above counters plus a table of recent audit log entries. + +## 3. Alerting Guidance + +1. **Unauthorized bypass attempt** + - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", status="401", bypass="True"}[5m])) > 0` + - Action: verify `bypassNetworks` list; confirm expected maintenance windows; rotate credentials if suspicious. + +2. **Missing scopes** + - Query: `sum(rate(log_messages_total{logger="Concelier.Authorization.Audit", scopes="(none)", status="200"}[5m])) > 0` + - Action: audit Authority client registration; ensure `requiredScopes` includes `concelier.jobs.trigger`. + +3. **Trigger failure surge** + - Query: `sum(rate(web_jobs_trigger_failed_total[10m])) > 0` with severity `warning` if sustained for 10 minutes. + - Action: inspect correlated audit entries and `Concelier.Telemetry` traces for job execution errors. + +4. **Conflict spike** + - Query: `sum(rate(web_jobs_trigger_conflict_total[10m])) > 5` (tune threshold). + - Action: downstream scheduling may be firing repetitive triggers; ensure precedence is configured properly. + +5. **Authority offline** + - Watch `Concelier.Authorization.Audit` logs for `status=503` or `status=500` along with `clientId="(none)"`. Investigate Authority availability before re-enabling anonymous fallback. + +## 4. Rollout & Verification Procedure + +1. **Pre-checks** + - Confirm `allowAnonymousFallback` is `false` in production; keep `true` only during staged validation. + - Validate Authority issuer metadata is reachable from Concelier (`curl https://authority.internal/.well-known/openid-configuration` from the host). + +2. **Smoke test with valid token** + - Obtain a token via CLI: `stella auth login --scope concelier.jobs.trigger`. + - Trigger a read-only endpoint: `curl -H "Authorization: Bearer $TOKEN" https://concelier.internal/jobs/definitions`. + - Expect HTTP 200/202 and an audit log with `bypass=False`, `scopes=concelier.jobs.trigger`. + +3. **Negative test without token** + - Call the same endpoint without a token. Expect HTTP 401, `bypass=False`. + - If the request succeeds, double-check `bypassNetworks` and ensure fallback is disabled. + +4. **Bypass check (if applicable)** + - From an allowed maintenance IP, call `/jobs/definitions` without a token. Confirm the audit log shows `bypass=True`. Review business justification and expiry date for such entries. + +5. **Metrics validation** + - Ensure `web.jobs.triggered` counter increments during accepted runs. + - Exporters should show corresponding spans (`concelier.job.trigger`) if tracing is enabled. + +## 5. Troubleshooting + +| Symptom | Probable cause | Remediation | +|---------|----------------|-------------| +| Audit log shows `clientId=(none)` for all requests | Authority not issuing `client_id` claim or CLI outdated | Update StellaOps Authority configuration (`StellaOpsAuthorityOptions.Token.Claims.ClientId`), or upgrade the CLI token acquisition flow. | +| Requests succeed with `bypass=True` unexpectedly | Local network added to `bypassNetworks` or fallback still enabled | Remove/adjust the CIDR list, disable anonymous fallback, restart Concelier. | +| HTTP 401 with valid token | `requiredScopes` missing from client registration or token audience mismatch | Verify Authority client scopes (`concelier.jobs.trigger`) and ensure the token audience matches `audiences` config. | +| Metrics missing from Prometheus | Telemetry exporters disabled or filter missing OTEL meter | Set `concelier.telemetry.enableMetrics=true`, ensure collector includes `StellaOps.Concelier.WebService.Jobs` meter. | +| Sudden spike in `web.jobs.trigger.failed` | Downstream job failure or Authority timeout mid-request | Inspect Concelier job logs, re-run with tracing enabled, validate Authority latency. | + +## 6. References + +- `docs/21_INSTALL_GUIDE.md` – Authority configuration quick start. +- `docs/17_SECURITY_HARDENING_GUIDE.md` – Security guardrails and enforcement deadlines. +- `docs/ops/authority-monitoring.md` – Authority-side monitoring and alerting playbook. +- `StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs` – source of audit log fields. diff --git a/docs/ops/feedser-cccs-operations.md b/docs/ops/concelier-cccs-operations.md similarity index 83% rename from docs/ops/feedser-cccs-operations.md rename to docs/ops/concelier-cccs-operations.md index 9d423944..12917a2f 100644 --- a/docs/ops/feedser-cccs-operations.md +++ b/docs/ops/concelier-cccs-operations.md @@ -1,72 +1,72 @@ -# Feedser CCCS Connector Operations - -This runbook covers day‑to‑day operation of the Canadian Centre for Cyber Security (`source:cccs:*`) connector, including configuration, telemetry, and historical backfill guidance for English/French advisories. - -## 1. Configuration Checklist - -- Network egress (or mirrored cache) for `https://www.cyber.gc.ca/` and the JSON API endpoints under `/api/cccs/`. -- Set the Feedser options before restarting workers. Example `feedser.yaml` snippet: - -```yaml -feedser: - sources: - cccs: - feeds: - - language: "en" - uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat" - - language: "fr" - uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat" - maxEntriesPerFetch: 80 # increase temporarily for backfill runs - maxKnownEntries: 512 - requestTimeout: "00:00:30" - requestDelay: "00:00:00.250" - failureBackoff: "00:05:00" -``` - -> ℹ️ The `/api/cccs/threats/v1/get` endpoint returns thousands of records per language (≈5 100 rows each as of 2025‑10‑14). The connector honours `maxEntriesPerFetch`, so leave it low for steady‑state and raise it for planned backfills. - -## 2. Telemetry & Logging - -- **Metrics (Meter `StellaOps.Feedser.Source.Cccs`):** - - `cccs.fetch.attempts`, `cccs.fetch.success`, `cccs.fetch.failures` - - `cccs.fetch.documents`, `cccs.fetch.unchanged` - - `cccs.parse.success`, `cccs.parse.failures`, `cccs.parse.quarantine` - - `cccs.map.success`, `cccs.map.failures` -- **Shared HTTP metrics** via `SourceDiagnostics`: - - `feedser.source.http.requests{feedser.source="cccs"}` - - `feedser.source.http.failures{feedser.source="cccs"}` - - `feedser.source.http.duration{feedser.source="cccs"}` -- **Structured logs** - - `CCCS fetch completed feeds=… items=… newDocuments=… pendingDocuments=…` - - `CCCS parse completed parsed=… failures=…` - - `CCCS map completed mapped=… failures=…` - - Warnings fire when GridFS payloads/DTOs go missing or parser sanitisation fails. - -Suggested Grafana alerts: -- `increase(cccs.fetch.failures_total[15m]) > 0` -- `rate(cccs.map.success_total[1h]) == 0` while other connectors are active -- `histogram_quantile(0.95, rate(feedser_source_http_duration_bucket{feedser_source="cccs"}[1h])) > 5s` - -## 3. Historical Backfill Plan - -1. **Snapshot the source** – the API accepts `page=` and `lang=` query parameters. `page=0` returns the full dataset (observed earliest `date_created`: 2018‑06‑08 for EN, 2018‑06‑08 for FR). Mirror those responses into Offline Kit storage when operating air‑gapped. -2. **Stage ingestion**: - - Temporarily raise `maxEntriesPerFetch` (e.g. 500) and restart Feedser workers. - - Run chained jobs until `pendingDocuments` drains: - `stella db jobs run source:cccs:fetch --and-then source:cccs:parse --and-then source:cccs:map` - - Monitor `cccs.fetch.unchanged` growth; once it approaches dataset size the backfill is complete. -3. **Optional pagination sweep** – for incremental mirrors, iterate `page=` (0…N) while `response.Count == 50`, persisting JSON to disk. Store alongside metadata (`language`, `page`, SHA256) so repeated runs detect drift. -4. **Language split** – keep EN/FR payloads separate to preserve canonical language fields. The connector emits `Language` directly from the feed entry, so mixed ingestion simply produces parallel advisories keyed by the same serial number. -5. **Throttle planning** – schedule backfills during maintenance windows; the API tolerates burst downloads but respect the 250 ms request delay or raise it if mirrored traffic is not available. - -## 4. Selector & Sanitiser Notes - -- `CccsHtmlParser` now parses the **unsanitised DOM** (via AngleSharp) and only sanitises when persisting `ContentHtml`. -- Product extraction walks headings (`Affected Products`, `Produits touchés`, `Mesures recommandées`) and consumes nested lists within `div/section/article` containers. -- `HtmlContentSanitizer` allows `

…

` and `
` so stored HTML keeps headings for UI rendering and downstream summarisation. - -## 5. Fixture Maintenance - -- Regression fixtures live in `src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures`. -- Refresh via `UPDATE_CCCS_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Cccs.Tests/StellaOps.Feedser.Source.Cccs.Tests.csproj`. -- Fixtures capture both EN/FR advisories with nested lists to guard against sanitiser regressions; review diffs for heading/list changes before committing. +# Concelier CCCS Connector Operations + +This runbook covers day‑to‑day operation of the Canadian Centre for Cyber Security (`source:cccs:*`) connector, including configuration, telemetry, and historical backfill guidance for English/French advisories. + +## 1. Configuration Checklist + +- Network egress (or mirrored cache) for `https://www.cyber.gc.ca/` and the JSON API endpoints under `/api/cccs/`. +- Set the Concelier options before restarting workers. Example `concelier.yaml` snippet: + +```yaml +concelier: + sources: + cccs: + feeds: + - language: "en" + uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat" + - language: "fr" + uri: "https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat" + maxEntriesPerFetch: 80 # increase temporarily for backfill runs + maxKnownEntries: 512 + requestTimeout: "00:00:30" + requestDelay: "00:00:00.250" + failureBackoff: "00:05:00" +``` + +> ℹ️ The `/api/cccs/threats/v1/get` endpoint returns thousands of records per language (≈5 100 rows each as of 2025‑10‑14). The connector honours `maxEntriesPerFetch`, so leave it low for steady‑state and raise it for planned backfills. + +## 2. Telemetry & Logging + +- **Metrics (Meter `StellaOps.Concelier.Connector.Cccs`):** + - `cccs.fetch.attempts`, `cccs.fetch.success`, `cccs.fetch.failures` + - `cccs.fetch.documents`, `cccs.fetch.unchanged` + - `cccs.parse.success`, `cccs.parse.failures`, `cccs.parse.quarantine` + - `cccs.map.success`, `cccs.map.failures` +- **Shared HTTP metrics** via `SourceDiagnostics`: + - `concelier.source.http.requests{concelier.source="cccs"}` + - `concelier.source.http.failures{concelier.source="cccs"}` + - `concelier.source.http.duration{concelier.source="cccs"}` +- **Structured logs** + - `CCCS fetch completed feeds=… items=… newDocuments=… pendingDocuments=…` + - `CCCS parse completed parsed=… failures=…` + - `CCCS map completed mapped=… failures=…` + - Warnings fire when GridFS payloads/DTOs go missing or parser sanitisation fails. + +Suggested Grafana alerts: +- `increase(cccs.fetch.failures_total[15m]) > 0` +- `rate(cccs.map.success_total[1h]) == 0` while other connectors are active +- `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cccs"}[1h])) > 5s` + +## 3. Historical Backfill Plan + +1. **Snapshot the source** – the API accepts `page=` and `lang=` query parameters. `page=0` returns the full dataset (observed earliest `date_created`: 2018‑06‑08 for EN, 2018‑06‑08 for FR). Mirror those responses into Offline Kit storage when operating air‑gapped. +2. **Stage ingestion**: + - Temporarily raise `maxEntriesPerFetch` (e.g. 500) and restart Concelier workers. + - Run chained jobs until `pendingDocuments` drains: + `stella db jobs run source:cccs:fetch --and-then source:cccs:parse --and-then source:cccs:map` + - Monitor `cccs.fetch.unchanged` growth; once it approaches dataset size the backfill is complete. +3. **Optional pagination sweep** – for incremental mirrors, iterate `page=` (0…N) while `response.Count == 50`, persisting JSON to disk. Store alongside metadata (`language`, `page`, SHA256) so repeated runs detect drift. +4. **Language split** – keep EN/FR payloads separate to preserve canonical language fields. The connector emits `Language` directly from the feed entry, so mixed ingestion simply produces parallel advisories keyed by the same serial number. +5. **Throttle planning** – schedule backfills during maintenance windows; the API tolerates burst downloads but respect the 250 ms request delay or raise it if mirrored traffic is not available. + +## 4. Selector & Sanitiser Notes + +- `CccsHtmlParser` now parses the **unsanitised DOM** (via AngleSharp) and only sanitises when persisting `ContentHtml`. +- Product extraction walks headings (`Affected Products`, `Produits touchés`, `Mesures recommandées`) and consumes nested lists within `div/section/article` containers. +- `HtmlContentSanitizer` allows `

…

` and `
` so stored HTML keeps headings for UI rendering and downstream summarisation. + +## 5. Fixture Maintenance + +- Regression fixtures live in `src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures`. +- Refresh via `UPDATE_CCCS_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Cccs.Tests/StellaOps.Concelier.Connector.Cccs.Tests.csproj`. +- Fixtures capture both EN/FR advisories with nested lists to guard against sanitiser regressions; review diffs for heading/list changes before committing. diff --git a/docs/ops/feedser-certbund-operations.md b/docs/ops/concelier-certbund-operations.md similarity index 87% rename from docs/ops/feedser-certbund-operations.md rename to docs/ops/concelier-certbund-operations.md index 0b09a906..ab16503e 100644 --- a/docs/ops/feedser-certbund-operations.md +++ b/docs/ops/concelier-certbund-operations.md @@ -1,146 +1,146 @@ -# Feedser CERT-Bund Connector Operations - -_Last updated: 2025-10-17_ - -Germany’s Federal Office for Information Security (BSI) operates the Warn- und Informationsdienst (WID) portal. The Feedser CERT-Bund connector (`source:cert-bund:*`) ingests the public RSS feed, hydrates the portal’s JSON detail endpoint, and maps the result into canonical advisories while preserving the original German content. - ---- - -## 1. Configuration Checklist - -- Allow outbound access (or stage mirrors) for: - - `https://wid.cert-bund.de/content/public/securityAdvisory/rss` - - `https://wid.cert-bund.de/portal/` (session/bootstrap) - - `https://wid.cert-bund.de/portal/api/securityadvisory` (detail/search/export JSON) -- Ensure the HTTP client reuses a cookie container (the connector’s dependency injection wiring already sets this up). - -Example `feedser.yaml` fragment: - -```yaml -feedser: - sources: - cert-bund: - feedUri: "https://wid.cert-bund.de/content/public/securityAdvisory/rss" - portalBootstrapUri: "https://wid.cert-bund.de/portal/" - detailApiUri: "https://wid.cert-bund.de/portal/api/securityadvisory" - maxAdvisoriesPerFetch: 50 - maxKnownAdvisories: 512 - requestTimeout: "00:00:30" - requestDelay: "00:00:00.250" - failureBackoff: "00:05:00" -``` - -> Leave `maxAdvisoriesPerFetch` at 50 during normal operation. Raise it only for controlled backfills, then restore the default to avoid overwhelming the portal. - ---- - -## 2. Telemetry & Logging - -- **Meter**: `StellaOps.Feedser.Source.CertBund` -- **Counters / histograms**: - - `certbund.feed.fetch.attempts|success|failures` - - `certbund.feed.items.count` - - `certbund.feed.enqueued.count` - - `certbund.feed.coverage.days` - - `certbund.detail.fetch.attempts|success|not_modified|failures{reason}` - - `certbund.parse.success|failures{reason}` - - `certbund.parse.products.count`, `certbund.parse.cve.count` - - `certbund.map.success|failures{reason}` - - `certbund.map.affected.count`, `certbund.map.aliases.count` -- Shared HTTP metrics remain available through `feedser.source.http.*`. - -**Structured logs** (all emitted at information level when work occurs): - -- `CERT-Bund fetch cycle: … truncated {Truncated}, coverageDays={CoverageDays}` -- `CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, …` -- `CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, …` - -Alerting ideas: - -1. `increase(certbund.detail.fetch.failures_total[10m]) > 0` -2. `rate(certbund.map.success_total[30m]) == 0` -3. `histogram_quantile(0.95, rate(feedser_source_http_duration_bucket{feedser_source="cert-bund"}[15m])) > 5s` - -The WebService now registers the meter so metrics surface automatically once OpenTelemetry metrics are enabled. - ---- - -## 3. Historical Backfill & Export Strategy - -### 3.1 Retention snapshot - -- RSS window: ~250 advisories (≈90 days at current cadence). -- Older advisories are accessible through the JSON search/export APIs once the anti-CSRF token is supplied. - -### 3.2 JSON search pagination - -```bash -# 1. Bootstrap cookies (client_config + XSRF-TOKEN) -curl -s -c cookies.txt "https://wid.cert-bund.de/portal/" > /dev/null -curl -s -b cookies.txt -c cookies.txt \ - -H "X-Requested-With: XMLHttpRequest" \ - "https://wid.cert-bund.de/portal/api/security/csrf" > /dev/null - -XSRF=$(awk '/XSRF-TOKEN/ {print $7}' cookies.txt) - -# 2. Page search results -curl -s -b cookies.txt \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - -H "X-XSRF-TOKEN: ${XSRF}" \ - -X POST \ - --data '{"page":4,"size":100,"sort":["published,desc"]}' \ - "https://wid.cert-bund.de/portal/api/securityadvisory/search" \ - > certbund-page4.json -``` - -Iterate `page` until the response `content` array is empty. Pages 0–9 currently cover 2014→present. Persist JSON responses (plus SHA256) for Offline Kit parity. - -> **Shortcut** – run `python tools/certbund_offline_snapshot.py --output seed-data/cert-bund` -> to bootstrap the session, capture the paginated search responses, and regenerate -> the manifest/checksum files automatically. Supply `--cookie-file` and `--xsrf-token` -> if the portal requires a browser-derived session (see options via `--help`). - -### 3.3 Export bundles - -```bash -python tools/certbund_offline_snapshot.py \ - --output seed-data/cert-bund \ - --start-year 2014 \ - --end-year "$(date -u +%Y)" -``` - -The helper stores yearly exports under `seed-data/cert-bund/export/`, -captures paginated search snapshots in `seed-data/cert-bund/search/`, -and generates the manifest + SHA files in `seed-data/cert-bund/manifest/`. -Split ranges according to your compliance window (default: one file per -calendar year). Feedser can ingest these JSON payloads directly when -operating offline. - -> When automatic bootstrap fails (e.g. portal introduces CAPTCHA), run the -> manual `curl` flow above, then rerun the helper with `--skip-fetch` to -> rebuild the manifest from the existing files. - -### 3.4 Connector-driven catch-up - -1. Temporarily raise `maxAdvisoriesPerFetch` (e.g. 150) and reduce `requestDelay`. -2. Run `stella db jobs run source:cert-bund:fetch --and-then source:cert-bund:parse --and-then source:cert-bund:map` until the fetch log reports `enqueued=0`. -3. Restore defaults and capture the cursor snapshot for audit. - ---- - -## 4. Locale & Translation Guidance - -- Advisories remain in German (`language: "de"`). Preserve wording for provenance and legal accuracy. -- UI localisation: enable the translation bundles documented in `docs/15_UI_GUIDE.md` if English UI copy is required. Operators can overlay machine or human translations, but the canonical database stores the source text. -- Docs guild is compiling a CERT-Bund terminology glossary under `docs/locale/certbund-glossary.md` so downstream teams can reference consistent English equivalents without altering the stored advisories. - ---- - -## 5. Verification Checklist - -1. Observe `certbund.feed.fetch.success` and `certbund.detail.fetch.success` increments after runs; `certbund.feed.coverage.days` should hover near the observed RSS window. -2. Ensure summary logs report `truncated=false` in steady state—`true` indicates the fetch cap was hit. -3. During backfills, watch `certbund.feed.enqueued.count` trend to zero. -4. Spot-check stored advisories in Mongo to confirm `language="de"` and reference URLs match the portal detail endpoint. -5. For Offline Kit exports, validate SHA256 hashes before distribution. +# Concelier CERT-Bund Connector Operations + +_Last updated: 2025-10-17_ + +Germany’s Federal Office for Information Security (BSI) operates the Warn- und Informationsdienst (WID) portal. The Concelier CERT-Bund connector (`source:cert-bund:*`) ingests the public RSS feed, hydrates the portal’s JSON detail endpoint, and maps the result into canonical advisories while preserving the original German content. + +--- + +## 1. Configuration Checklist + +- Allow outbound access (or stage mirrors) for: + - `https://wid.cert-bund.de/content/public/securityAdvisory/rss` + - `https://wid.cert-bund.de/portal/` (session/bootstrap) + - `https://wid.cert-bund.de/portal/api/securityadvisory` (detail/search/export JSON) +- Ensure the HTTP client reuses a cookie container (the connector’s dependency injection wiring already sets this up). + +Example `concelier.yaml` fragment: + +```yaml +concelier: + sources: + cert-bund: + feedUri: "https://wid.cert-bund.de/content/public/securityAdvisory/rss" + portalBootstrapUri: "https://wid.cert-bund.de/portal/" + detailApiUri: "https://wid.cert-bund.de/portal/api/securityadvisory" + maxAdvisoriesPerFetch: 50 + maxKnownAdvisories: 512 + requestTimeout: "00:00:30" + requestDelay: "00:00:00.250" + failureBackoff: "00:05:00" +``` + +> Leave `maxAdvisoriesPerFetch` at 50 during normal operation. Raise it only for controlled backfills, then restore the default to avoid overwhelming the portal. + +--- + +## 2. Telemetry & Logging + +- **Meter**: `StellaOps.Concelier.Connector.CertBund` +- **Counters / histograms**: + - `certbund.feed.fetch.attempts|success|failures` + - `certbund.feed.items.count` + - `certbund.feed.enqueued.count` + - `certbund.feed.coverage.days` + - `certbund.detail.fetch.attempts|success|not_modified|failures{reason}` + - `certbund.parse.success|failures{reason}` + - `certbund.parse.products.count`, `certbund.parse.cve.count` + - `certbund.map.success|failures{reason}` + - `certbund.map.affected.count`, `certbund.map.aliases.count` +- Shared HTTP metrics remain available through `concelier.source.http.*`. + +**Structured logs** (all emitted at information level when work occurs): + +- `CERT-Bund fetch cycle: … truncated {Truncated}, coverageDays={CoverageDays}` +- `CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, …` +- `CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, …` + +Alerting ideas: + +1. `increase(certbund.detail.fetch.failures_total[10m]) > 0` +2. `rate(certbund.map.success_total[30m]) == 0` +3. `histogram_quantile(0.95, rate(concelier_source_http_duration_bucket{concelier_source="cert-bund"}[15m])) > 5s` + +The WebService now registers the meter so metrics surface automatically once OpenTelemetry metrics are enabled. + +--- + +## 3. Historical Backfill & Export Strategy + +### 3.1 Retention snapshot + +- RSS window: ~250 advisories (≈90 days at current cadence). +- Older advisories are accessible through the JSON search/export APIs once the anti-CSRF token is supplied. + +### 3.2 JSON search pagination + +```bash +# 1. Bootstrap cookies (client_config + XSRF-TOKEN) +curl -s -c cookies.txt "https://wid.cert-bund.de/portal/" > /dev/null +curl -s -b cookies.txt -c cookies.txt \ + -H "X-Requested-With: XMLHttpRequest" \ + "https://wid.cert-bund.de/portal/api/security/csrf" > /dev/null + +XSRF=$(awk '/XSRF-TOKEN/ {print $7}' cookies.txt) + +# 2. Page search results +curl -s -b cookies.txt \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -H "X-XSRF-TOKEN: ${XSRF}" \ + -X POST \ + --data '{"page":4,"size":100,"sort":["published,desc"]}' \ + "https://wid.cert-bund.de/portal/api/securityadvisory/search" \ + > certbund-page4.json +``` + +Iterate `page` until the response `content` array is empty. Pages 0–9 currently cover 2014→present. Persist JSON responses (plus SHA256) for Offline Kit parity. + +> **Shortcut** – run `python tools/certbund_offline_snapshot.py --output seed-data/cert-bund` +> to bootstrap the session, capture the paginated search responses, and regenerate +> the manifest/checksum files automatically. Supply `--cookie-file` and `--xsrf-token` +> if the portal requires a browser-derived session (see options via `--help`). + +### 3.3 Export bundles + +```bash +python tools/certbund_offline_snapshot.py \ + --output seed-data/cert-bund \ + --start-year 2014 \ + --end-year "$(date -u +%Y)" +``` + +The helper stores yearly exports under `seed-data/cert-bund/export/`, +captures paginated search snapshots in `seed-data/cert-bund/search/`, +and generates the manifest + SHA files in `seed-data/cert-bund/manifest/`. +Split ranges according to your compliance window (default: one file per +calendar year). Concelier can ingest these JSON payloads directly when +operating offline. + +> When automatic bootstrap fails (e.g. portal introduces CAPTCHA), run the +> manual `curl` flow above, then rerun the helper with `--skip-fetch` to +> rebuild the manifest from the existing files. + +### 3.4 Connector-driven catch-up + +1. Temporarily raise `maxAdvisoriesPerFetch` (e.g. 150) and reduce `requestDelay`. +2. Run `stella db jobs run source:cert-bund:fetch --and-then source:cert-bund:parse --and-then source:cert-bund:map` until the fetch log reports `enqueued=0`. +3. Restore defaults and capture the cursor snapshot for audit. + +--- + +## 4. Locale & Translation Guidance + +- Advisories remain in German (`language: "de"`). Preserve wording for provenance and legal accuracy. +- UI localisation: enable the translation bundles documented in `docs/15_UI_GUIDE.md` if English UI copy is required. Operators can overlay machine or human translations, but the canonical database stores the source text. +- Docs guild is compiling a CERT-Bund terminology glossary under `docs/locale/certbund-glossary.md` so downstream teams can reference consistent English equivalents without altering the stored advisories. + +--- + +## 5. Verification Checklist + +1. Observe `certbund.feed.fetch.success` and `certbund.detail.fetch.success` increments after runs; `certbund.feed.coverage.days` should hover near the observed RSS window. +2. Ensure summary logs report `truncated=false` in steady state—`true` indicates the fetch cap was hit. +3. During backfills, watch `certbund.feed.enqueued.count` trend to zero. +4. Spot-check stored advisories in Mongo to confirm `language="de"` and reference URLs match the portal detail endpoint. +5. For Offline Kit exports, validate SHA256 hashes before distribution. diff --git a/docs/ops/feedser-cisco-operations.md b/docs/ops/concelier-cisco-operations.md similarity index 71% rename from docs/ops/feedser-cisco-operations.md rename to docs/ops/concelier-cisco-operations.md index 3d0ae8e8..8cbda088 100644 --- a/docs/ops/feedser-cisco-operations.md +++ b/docs/ops/concelier-cisco-operations.md @@ -1,94 +1,94 @@ -# Feedser Cisco PSIRT Connector – OAuth Provisioning SOP - -_Last updated: 2025-10-14_ - -## 1. Scope - -This runbook describes how Ops provisions, rotates, and distributes Cisco PSIRT openVuln OAuth client credentials for the Feedser Cisco connector. It covers online and air-gapped (Offline Kit) environments, quota-aware execution, and escalation paths. - -## 2. Prerequisites - -- Active Cisco.com (CCO) account with access to the Cisco API Console. -- Cisco PSIRT openVuln API entitlement (visible under “My Apps & Keys” once granted).citeturn3search0 -- Feedser configuration location (typically `/etc/stella/feedser.yaml` in production) or Offline Kit secret bundle staging directory. - -## 3. Provisioning workflow - -1. **Register the application** - - Sign in at . - - Select **Register a New App** → Application Type: `Service`, Grant Type: `Client Credentials`, API: `Cisco PSIRT openVuln API`.citeturn3search0 - - Record the generated `clientId` and `clientSecret` in the Ops vault. -2. **Verify token issuance** - - Request an access token with: - ```bash - curl -s https://id.cisco.com/oauth2/default/v1/token \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "grant_type=client_credentials" \ - -d "client_id=${CLIENT_ID}" \ - -d "client_secret=${CLIENT_SECRET}" - ``` - - Confirm HTTP 200 and an `expires_in` value of 3600 seconds (tokens live for one hour).citeturn3search0turn3search7 - - Preserve the response only long enough to validate syntax; do **not** persist tokens. -3. **Authorize Feedser runtime** - - Update `feedser:sources:cisco:auth` (or the module-specific secret template) with the stored credentials. - - For Offline Kit delivery, export encrypted secrets into `offline-kit/secrets/cisco-openvuln.json` using the platform’s sealed secret format. -4. **Connectivity validation** - - From the Feedser control plane, run `stella db jobs run source:vndr-cisco:fetch --dry-run`. - - Ensure the Source HTTP diagnostics record `Bearer` authorization headers and no 401/403 responses. - -## 4. Rotation SOP - -| Step | Owner | Notes | -| --- | --- | --- | -| 1. Schedule rotation | Ops (monthly board) | Rotate every 90 days or immediately after suspected credential exposure. | -| 2. Create replacement app | Ops | Repeat §3.1 with “-next” suffix; verify token issuance. | -| 3. Stage dual credentials | Ops + Feedser On-Call | Publish new credentials to secret store alongside current pair. | -| 4. Cut over | Feedser On-Call | Restart connector workers during a low-traffic window (<10 min) to pick up the new secret. | -| 5. Deactivate legacy app | Ops | Delete prior app in Cisco API Console once telemetry confirms successful fetch/parse cycles for 2 consecutive hours. | - -**Automation hooks** -- Rotation reminders are tracked in OpsRunbookOps board (`OPS-RUN-KEYS` swim lane); add checklist items for Feedser Cisco when opening a rotation task. -- Use the secret management pipeline (`ops/secrets/rotate.sh --connector cisco`) to template vault updates; the script renders a redacted diff for audit. - -## 5. Offline Kit packaging - -1. Generate the credential bundle using the Offline Kit CLI: - `offline-kit secrets add cisco-openvuln --client-id … --client-secret …` -2. Store the encrypted payload under `offline-kit/secrets/cisco-openvuln.enc`. -3. Distribute via the Offline Kit channel; update `offline-kit/MANIFEST.md` with the credential fingerprint (SHA256 of plaintext concatenated with metadata). -4. Document validation steps for the receiving site (token request from an air-gapped relay or cached token mirror). - -## 6. Quota and throttling guidance - -- Cisco enforces combined limits of 5 requests/second, 30 requests/minute, and 5 000 requests/day per application.citeturn0search0turn3search6 -- Feedser fetch jobs must respect `Retry-After` headers on HTTP 429 responses; Ops should monitor for sustained quota saturation and consider paging window adjustments. -- Telemetry to watch: `feedser.source.http.requests{feedser.source="vndr-cisco"}`, `feedser.source.http.failures{...}`, and connector-specific metrics once implemented. - -## 7. Telemetry & Monitoring - -- **Metrics (Meter `StellaOps.Feedser.Source.Vndr.Cisco`)** - - `cisco.fetch.documents`, `cisco.fetch.failures`, `cisco.fetch.unchanged` - - `cisco.parse.success`, `cisco.parse.failures` - - `cisco.map.success`, `cisco.map.failures`, `cisco.map.affected.packages` -- **Shared HTTP metrics** via `SourceDiagnostics`: - - `feedser.source.http.requests{feedser.source="vndr-cisco"}` - - `feedser.source.http.failures{feedser.source="vndr-cisco"}` - - `feedser.source.http.duration{feedser.source="vndr-cisco"}` -- **Structured logs** - - `Cisco fetch completed date=… pages=… added=…` (info) - - `Cisco parse completed parsed=… failures=…` (info) - - `Cisco map completed mapped=… failures=…` (info) - - Warnings surface when DTO serialization fails or GridFS payload is missing. -- Suggested alerts: non-zero `cisco.fetch.failures` in 15m, or `cisco.map.success` flatlines while fetch continues. - -## 8. Incident response - -- **Token compromise** – revoke the application in the Cisco API Console, purge cached secrets, rotate immediately per §4. -- **Persistent 401/403** – confirm credentials in vault, then validate token issuance; if unresolved, open a Cisco DevNet support ticket referencing the application ID. -- **429 spikes** – inspect job scheduler cadence and adjust connector options (`maxRequestsPerWindow`) before requesting higher quotas from Cisco. - -## 9. References - -- Cisco PSIRT openVuln API Authentication Guide.citeturn3search0 -- Accessing the openVuln API using curl (token lifetime).citeturn3search7 -- openVuln API rate limit documentation.citeturn0search0turn3search6 +# Concelier Cisco PSIRT Connector – OAuth Provisioning SOP + +_Last updated: 2025-10-14_ + +## 1. Scope + +This runbook describes how Ops provisions, rotates, and distributes Cisco PSIRT openVuln OAuth client credentials for the Concelier Cisco connector. It covers online and air-gapped (Offline Kit) environments, quota-aware execution, and escalation paths. + +## 2. Prerequisites + +- Active Cisco.com (CCO) account with access to the Cisco API Console. +- Cisco PSIRT openVuln API entitlement (visible under “My Apps & Keys” once granted).citeturn3search0 +- Concelier configuration location (typically `/etc/stella/concelier.yaml` in production) or Offline Kit secret bundle staging directory. + +## 3. Provisioning workflow + +1. **Register the application** + - Sign in at . + - Select **Register a New App** → Application Type: `Service`, Grant Type: `Client Credentials`, API: `Cisco PSIRT openVuln API`.citeturn3search0 + - Record the generated `clientId` and `clientSecret` in the Ops vault. +2. **Verify token issuance** + - Request an access token with: + ```bash + curl -s https://id.cisco.com/oauth2/default/v1/token \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=${CLIENT_ID}" \ + -d "client_secret=${CLIENT_SECRET}" + ``` + - Confirm HTTP 200 and an `expires_in` value of 3600 seconds (tokens live for one hour).citeturn3search0turn3search7 + - Preserve the response only long enough to validate syntax; do **not** persist tokens. +3. **Authorize Concelier runtime** + - Update `concelier:sources:cisco:auth` (or the module-specific secret template) with the stored credentials. + - For Offline Kit delivery, export encrypted secrets into `offline-kit/secrets/cisco-openvuln.json` using the platform’s sealed secret format. +4. **Connectivity validation** + - From the Concelier control plane, run `stella db jobs run source:vndr-cisco:fetch --dry-run`. + - Ensure the Source HTTP diagnostics record `Bearer` authorization headers and no 401/403 responses. + +## 4. Rotation SOP + +| Step | Owner | Notes | +| --- | --- | --- | +| 1. Schedule rotation | Ops (monthly board) | Rotate every 90 days or immediately after suspected credential exposure. | +| 2. Create replacement app | Ops | Repeat §3.1 with “-next” suffix; verify token issuance. | +| 3. Stage dual credentials | Ops + Concelier On-Call | Publish new credentials to secret store alongside current pair. | +| 4. Cut over | Concelier On-Call | Restart connector workers during a low-traffic window (<10 min) to pick up the new secret. | +| 5. Deactivate legacy app | Ops | Delete prior app in Cisco API Console once telemetry confirms successful fetch/parse cycles for 2 consecutive hours. | + +**Automation hooks** +- Rotation reminders are tracked in OpsRunbookOps board (`OPS-RUN-KEYS` swim lane); add checklist items for Concelier Cisco when opening a rotation task. +- Use the secret management pipeline (`ops/secrets/rotate.sh --connector cisco`) to template vault updates; the script renders a redacted diff for audit. + +## 5. Offline Kit packaging + +1. Generate the credential bundle using the Offline Kit CLI: + `offline-kit secrets add cisco-openvuln --client-id … --client-secret …` +2. Store the encrypted payload under `offline-kit/secrets/cisco-openvuln.enc`. +3. Distribute via the Offline Kit channel; update `offline-kit/MANIFEST.md` with the credential fingerprint (SHA256 of plaintext concatenated with metadata). +4. Document validation steps for the receiving site (token request from an air-gapped relay or cached token mirror). + +## 6. Quota and throttling guidance + +- Cisco enforces combined limits of 5 requests/second, 30 requests/minute, and 5 000 requests/day per application.citeturn0search0turn3search6 +- Concelier fetch jobs must respect `Retry-After` headers on HTTP 429 responses; Ops should monitor for sustained quota saturation and consider paging window adjustments. +- Telemetry to watch: `concelier.source.http.requests{concelier.source="vndr-cisco"}`, `concelier.source.http.failures{...}`, and connector-specific metrics once implemented. + +## 7. Telemetry & Monitoring + +- **Metrics (Meter `StellaOps.Concelier.Connector.Vndr.Cisco`)** + - `cisco.fetch.documents`, `cisco.fetch.failures`, `cisco.fetch.unchanged` + - `cisco.parse.success`, `cisco.parse.failures` + - `cisco.map.success`, `cisco.map.failures`, `cisco.map.affected.packages` +- **Shared HTTP metrics** via `SourceDiagnostics`: + - `concelier.source.http.requests{concelier.source="vndr-cisco"}` + - `concelier.source.http.failures{concelier.source="vndr-cisco"}` + - `concelier.source.http.duration{concelier.source="vndr-cisco"}` +- **Structured logs** + - `Cisco fetch completed date=… pages=… added=…` (info) + - `Cisco parse completed parsed=… failures=…` (info) + - `Cisco map completed mapped=… failures=…` (info) + - Warnings surface when DTO serialization fails or GridFS payload is missing. +- Suggested alerts: non-zero `cisco.fetch.failures` in 15m, or `cisco.map.success` flatlines while fetch continues. + +## 8. Incident response + +- **Token compromise** – revoke the application in the Cisco API Console, purge cached secrets, rotate immediately per §4. +- **Persistent 401/403** – confirm credentials in vault, then validate token issuance; if unresolved, open a Cisco DevNet support ticket referencing the application ID. +- **429 spikes** – inspect job scheduler cadence and adjust connector options (`maxRequestsPerWindow`) before requesting higher quotas from Cisco. + +## 9. References + +- Cisco PSIRT openVuln API Authentication Guide.citeturn3search0 +- Accessing the openVuln API using curl (token lifetime).citeturn3search7 +- openVuln API rate limit documentation.citeturn0search0turn3search6 diff --git a/docs/ops/feedser-conflict-resolution.md b/docs/ops/concelier-conflict-resolution.md similarity index 51% rename from docs/ops/feedser-conflict-resolution.md rename to docs/ops/concelier-conflict-resolution.md index 804e87dd..493c304c 100644 --- a/docs/ops/feedser-conflict-resolution.md +++ b/docs/ops/concelier-conflict-resolution.md @@ -1,160 +1,160 @@ -# Feedser Conflict Resolution Runbook (Sprint 3) - -This runbook equips Feedser operators to detect, triage, and resolve advisory conflicts now that the Sprint 3 merge engine landed (`AdvisoryPrecedenceMerger`, merge-event hashing, and telemetry counters). It builds on the canonical rules defined in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` and the metrics/logging instrumentation delivered this sprint. - ---- - -## 1. Precedence Model (recap) - -- **Default ranking:** `GHSA -> NVD -> OSV`, with distro/vendor PSIRTs outranking ecosystem feeds (`AdvisoryPrecedenceDefaults`). Use `feedser:merge:precedence:ranks` to override per source when incident response requires it. -- **Freshness override:** if a lower-ranked source is >= 48 hours newer for a freshness-sensitive field (title, summary, affected ranges, references, credits), it wins. Every override stamps `provenance[].decisionReason = freshness`. -- **Tie-breakers:** when precedence and freshness tie, the engine falls back to (1) primary source order, (2) shortest normalized text, (3) lowest stable hash. Merge-generated provenance records set `decisionReason = tie-breaker`. -- **Audit trail:** each merged advisory receives a `merge` provenance entry listing the participating sources plus a `merge_event` record with canonical before/after SHA-256 hashes. - ---- - -## 2. Telemetry Shipped This Sprint - -| Instrument | Type | Key Tags | Purpose | -|------------|------|----------|---------| -| `feedser.merge.operations` | Counter | `inputs` | Total precedence merges executed. | -| `feedser.merge.overrides` | Counter | `primary_source`, `suppressed_source`, `primary_rank`, `suppressed_rank` | Field-level overrides chosen by precedence. | -| `feedser.merge.range_overrides` | Counter | `advisory_key`, `package_type`, `primary_source`, `suppressed_source`, `primary_range_count`, `suppressed_range_count` | Package range overrides emitted by `AffectedPackagePrecedenceResolver`. | -| `feedser.merge.conflicts` | Counter | `type` (`severity`, `precedence_tie`), `reason` (`mismatch`, `primary_missing`, `equal_rank`) | Conflicts requiring operator review. | -| `feedser.merge.identity_conflicts` | Counter | `scheme`, `alias_value`, `advisory_count` | Alias collisions surfaced by the identity graph. | - -### Structured logs - -- `AdvisoryOverride` (EventId 1000) - logs merge suppressions with alias/provenance counts. -- `PackageRangeOverride` (EventId 1001) - logs package-level precedence decisions. -- `PrecedenceConflict` (EventId 1002) - logs mismatched severity or equal-rank scenarios. -- `Alias collision ...` (no EventId) - emitted when `feedser.merge.identity_conflicts` increments. - -Expect all logs at `Information`. Ensure OTEL exporters include the scope `StellaOps.Feedser.Merge`. - ---- - -## 3. Detection & Alerting - -1. **Dashboard panels** - - `feedser.merge.conflicts` - table grouped by `type/reason`. Alert when > 0 in a 15 minute window. - - `feedser.merge.range_overrides` - stacked bar by `package_type`. Spikes highlight vendor PSIRT overrides over registry data. - - `feedser.merge.overrides` with `primary_source|suppressed_source` - catches unexpected precedence flips (e.g., OSV overtaking GHSA). - - `feedser.merge.identity_conflicts` - single-stat; alert when alias collisions occur more than once per day. -2. **Log based alerts** - - `eventId=1002` with `reason="equal_rank"` - indicates precedence table gaps; page merge owners. - - `eventId=1002` with `reason="mismatch"` - severity disagreement; open connector bug if sustained. -3. **Job health** - - `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #feedser-ops. - -### Threshold updates (2025-10-12) - -- `feedser.merge.conflicts` – Page only when ≥ 2 events fire within 30 minutes; the synthetic conflict fixture run produces 0 conflicts, so the first event now routes to Slack for manual review instead of paging. -- `feedser.merge.overrides` – Raise a warning when the 30-minute sum exceeds 10 (canonical triple yields exactly 1 summary override with `primary_source=osv`, `suppressed_source=ghsa`). -- `feedser.merge.range_overrides` – Maintain the 15-minute alert at ≥ 3 but annotate dashboards that the regression triple emits a single `package_type=semver` override so ops can spot unexpected spikes. - ---- - -## 4. Triage Workflow - -1. **Confirm job context** - - `stellaops-cli db merge` (CLI) or `POST /jobs/merge:reconcile` (API) to rehydrate the merge job. Use `--verbose` to stream structured logs during triage. -2. **Inspect metrics** - - Correlate spikes in `feedser.merge.conflicts` with `primary_source`/`suppressed_source` tags from `feedser.merge.overrides`. -3. **Pull structured logs** - - Example (vector output): - ``` - jq 'select(.EventId.Name=="PrecedenceConflict") | {advisory: .State[0].Value, type: .ConflictType, reason: .Reason, primary: .PrimarySources, suppressed: .SuppressedSources}' stellaops-feedser.log - ``` -4. **Review merge events** - - `mongosh`: - ```javascript - use feedser; - db.merge_event.find({ advisoryKey: "CVE-2025-1234" }).sort({ mergedAt: -1 }).limit(5); - ``` - - Compare `beforeHash` vs `afterHash` to confirm the merge actually changed canonical output. -5. **Interrogate provenance** - - `db.advisories.findOne({ advisoryKey: "CVE-2025-1234" }, { title: 1, severity: 1, provenance: 1, "affectedPackages.provenance": 1 })` - - Check `provenance[].decisionReason` values (`precedence`, `freshness`, `tie-breaker`) to understand why the winning field was chosen. - ---- - -## 5. Conflict Classification Matrix - -| Signal | Likely Cause | Immediate Action | -|--------|--------------|------------------| -| `reason="mismatch"` with `type="severity"` | Upstream feeds disagree on CVSS vector/severity. | Verify which feed is freshest; if correctness is known, adjust connector mapping or precedence override. | -| `reason="primary_missing"` | Higher-ranked source lacks the field entirely. | Backfill connector data or temporarily allow lower-ranked source via precedence override. | -| `reason="equal_rank"` | Two feeds share the same precedence rank (custom config or missing entry). | Update `feedser:merge:precedence:ranks` to break the tie; restart merge job. | -| Rising `feedser.merge.range_overrides` for a package type | Vendor PSIRT now supplies richer ranges. | Validate connectors emit `decisionReason="precedence"` and update dashboards to treat registry ranges as fallback. | -| `feedser.merge.identity_conflicts` > 0 | Alias scheme mapping produced collisions (duplicate CVE <-> advisory pairs). | Inspect `Alias collision` log payload; reconcile the alias graph by adjusting connector alias output. | - ---- - -## 6. Resolution Playbook - -1. **Connector data fix** - - Re-run the offending connector stages (`stellaops-cli db fetch --source ghsa --stage map` etc.). - - Once fixed, rerun merge and verify `decisionReason` reflects `freshness` or `precedence` as expected. -2. **Temporary precedence override** - - Edit `etc/feedser.yaml`: - ```yaml - feedser: - merge: - precedence: - ranks: - osv: 1 - ghsa: 0 - ``` - - Restart Feedser workers; confirm tags in `feedser.merge.overrides` show the new ranks. - - Document the override with expiry in the change log. -3. **Alias remediation** - - Update connector mapping rules to weed out duplicate aliases (e.g., skip GHSA aliases that mirror CVE IDs). - - Flush cached alias graphs if necessary (`db.alias_graph.drop()` is destructive-coordinate with Storage before issuing). -4. **Escalation** - - If override metrics spike due to upstream regression, open an incident with Security Guild, referencing merge logs and `merge_event` IDs. - ---- - -## 7. Validation Checklist - -- [ ] Merge job rerun returns exit code `0`. -- [ ] `feedser.merge.conflicts` baseline returns to zero after corrective action. -- [ ] Latest `merge_event` entry shows expected hash delta. -- [ ] Affected advisory document shows updated `provenance[].decisionReason`. -- [ ] Ops change log updated with incident summary, config overrides, and rollback plan. - ---- - -## 8. Reference Material - -- Canonical conflict rules: `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md`. -- Merge engine internals: `src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs`. -- Metrics definitions: `src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs` (identity conflicts) and `AdvisoryPrecedenceMerger`. -- Storage audit trail: `src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Feedser.Storage.Mongo/MergeEvents`. - -Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change. - ---- - -## 9. Synthetic Regression Fixtures - -- **Locations** – Canonical conflict snapshots now live at `src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json`, `src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json`, and `src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json`. -- **Validation commands** – To regenerate and verify the fixtures offline, run: - -```bash -dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests -dotnet test src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj --filter NvdConflictFixtureTests -dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj --filter OsvConflictFixtureTests -dotnet test src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj --filter MergeAsync_AppliesCanonicalRulesAndPersistsDecisions -``` - -- **Expected signals** – The triple produces one freshness-driven summary override (`primary_source=osv`, `suppressed_source=ghsa`) and one range override for the npm SemVer package while leaving `feedser.merge.conflicts` at zero. Use these values as the baseline when tuning dashboards or load-testing alert pipelines. - ---- - -## 10. Change Log - -| Date (UTC) | Change | Notes | -|------------|--------|-------| -| 2025-10-16 | Ops review signed off after connector expansion (CCCS, CERT-Bund, KISA, ICS CISA, MSRC) landed. Alert thresholds from §3 reaffirmed; dashboards updated to watch attachment signals emitted by ICS CISA connector. | Ops sign-off recorded by Feedser Ops Guild; no additional overrides required. | +# Concelier Conflict Resolution Runbook (Sprint 3) + +This runbook equips Concelier operators to detect, triage, and resolve advisory conflicts now that the Sprint 3 merge engine landed (`AdvisoryPrecedenceMerger`, merge-event hashing, and telemetry counters). It builds on the canonical rules defined in `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md` and the metrics/logging instrumentation delivered this sprint. + +--- + +## 1. Precedence Model (recap) + +- **Default ranking:** `GHSA -> NVD -> OSV`, with distro/vendor PSIRTs outranking ecosystem feeds (`AdvisoryPrecedenceDefaults`). Use `concelier:merge:precedence:ranks` to override per source when incident response requires it. +- **Freshness override:** if a lower-ranked source is >= 48 hours newer for a freshness-sensitive field (title, summary, affected ranges, references, credits), it wins. Every override stamps `provenance[].decisionReason = freshness`. +- **Tie-breakers:** when precedence and freshness tie, the engine falls back to (1) primary source order, (2) shortest normalized text, (3) lowest stable hash. Merge-generated provenance records set `decisionReason = tie-breaker`. +- **Audit trail:** each merged advisory receives a `merge` provenance entry listing the participating sources plus a `merge_event` record with canonical before/after SHA-256 hashes. + +--- + +## 2. Telemetry Shipped This Sprint + +| Instrument | Type | Key Tags | Purpose | +|------------|------|----------|---------| +| `concelier.merge.operations` | Counter | `inputs` | Total precedence merges executed. | +| `concelier.merge.overrides` | Counter | `primary_source`, `suppressed_source`, `primary_rank`, `suppressed_rank` | Field-level overrides chosen by precedence. | +| `concelier.merge.range_overrides` | Counter | `advisory_key`, `package_type`, `primary_source`, `suppressed_source`, `primary_range_count`, `suppressed_range_count` | Package range overrides emitted by `AffectedPackagePrecedenceResolver`. | +| `concelier.merge.conflicts` | Counter | `type` (`severity`, `precedence_tie`), `reason` (`mismatch`, `primary_missing`, `equal_rank`) | Conflicts requiring operator review. | +| `concelier.merge.identity_conflicts` | Counter | `scheme`, `alias_value`, `advisory_count` | Alias collisions surfaced by the identity graph. | + +### Structured logs + +- `AdvisoryOverride` (EventId 1000) - logs merge suppressions with alias/provenance counts. +- `PackageRangeOverride` (EventId 1001) - logs package-level precedence decisions. +- `PrecedenceConflict` (EventId 1002) - logs mismatched severity or equal-rank scenarios. +- `Alias collision ...` (no EventId) - emitted when `concelier.merge.identity_conflicts` increments. + +Expect all logs at `Information`. Ensure OTEL exporters include the scope `StellaOps.Concelier.Merge`. + +--- + +## 3. Detection & Alerting + +1. **Dashboard panels** + - `concelier.merge.conflicts` - table grouped by `type/reason`. Alert when > 0 in a 15 minute window. + - `concelier.merge.range_overrides` - stacked bar by `package_type`. Spikes highlight vendor PSIRT overrides over registry data. + - `concelier.merge.overrides` with `primary_source|suppressed_source` - catches unexpected precedence flips (e.g., OSV overtaking GHSA). + - `concelier.merge.identity_conflicts` - single-stat; alert when alias collisions occur more than once per day. +2. **Log based alerts** + - `eventId=1002` with `reason="equal_rank"` - indicates precedence table gaps; page merge owners. + - `eventId=1002` with `reason="mismatch"` - severity disagreement; open connector bug if sustained. +3. **Job health** + - `stellaops-cli db merge` exit code `1` signifies unresolved conflicts. Pipe to automation that captures logs and notifies #concelier-ops. + +### Threshold updates (2025-10-12) + +- `concelier.merge.conflicts` – Page only when ≥ 2 events fire within 30 minutes; the synthetic conflict fixture run produces 0 conflicts, so the first event now routes to Slack for manual review instead of paging. +- `concelier.merge.overrides` – Raise a warning when the 30-minute sum exceeds 10 (canonical triple yields exactly 1 summary override with `primary_source=osv`, `suppressed_source=ghsa`). +- `concelier.merge.range_overrides` – Maintain the 15-minute alert at ≥ 3 but annotate dashboards that the regression triple emits a single `package_type=semver` override so ops can spot unexpected spikes. + +--- + +## 4. Triage Workflow + +1. **Confirm job context** + - `stellaops-cli db merge` (CLI) or `POST /jobs/merge:reconcile` (API) to rehydrate the merge job. Use `--verbose` to stream structured logs during triage. +2. **Inspect metrics** + - Correlate spikes in `concelier.merge.conflicts` with `primary_source`/`suppressed_source` tags from `concelier.merge.overrides`. +3. **Pull structured logs** + - Example (vector output): + ``` + jq 'select(.EventId.Name=="PrecedenceConflict") | {advisory: .State[0].Value, type: .ConflictType, reason: .Reason, primary: .PrimarySources, suppressed: .SuppressedSources}' stellaops-concelier.log + ``` +4. **Review merge events** + - `mongosh`: + ```javascript + use concelier; + db.merge_event.find({ advisoryKey: "CVE-2025-1234" }).sort({ mergedAt: -1 }).limit(5); + ``` + - Compare `beforeHash` vs `afterHash` to confirm the merge actually changed canonical output. +5. **Interrogate provenance** + - `db.advisories.findOne({ advisoryKey: "CVE-2025-1234" }, { title: 1, severity: 1, provenance: 1, "affectedPackages.provenance": 1 })` + - Check `provenance[].decisionReason` values (`precedence`, `freshness`, `tie-breaker`) to understand why the winning field was chosen. + +--- + +## 5. Conflict Classification Matrix + +| Signal | Likely Cause | Immediate Action | +|--------|--------------|------------------| +| `reason="mismatch"` with `type="severity"` | Upstream feeds disagree on CVSS vector/severity. | Verify which feed is freshest; if correctness is known, adjust connector mapping or precedence override. | +| `reason="primary_missing"` | Higher-ranked source lacks the field entirely. | Backfill connector data or temporarily allow lower-ranked source via precedence override. | +| `reason="equal_rank"` | Two feeds share the same precedence rank (custom config or missing entry). | Update `concelier:merge:precedence:ranks` to break the tie; restart merge job. | +| Rising `concelier.merge.range_overrides` for a package type | Vendor PSIRT now supplies richer ranges. | Validate connectors emit `decisionReason="precedence"` and update dashboards to treat registry ranges as fallback. | +| `concelier.merge.identity_conflicts` > 0 | Alias scheme mapping produced collisions (duplicate CVE <-> advisory pairs). | Inspect `Alias collision` log payload; reconcile the alias graph by adjusting connector alias output. | + +--- + +## 6. Resolution Playbook + +1. **Connector data fix** + - Re-run the offending connector stages (`stellaops-cli db fetch --source ghsa --stage map` etc.). + - Once fixed, rerun merge and verify `decisionReason` reflects `freshness` or `precedence` as expected. +2. **Temporary precedence override** + - Edit `etc/concelier.yaml`: + ```yaml + concelier: + merge: + precedence: + ranks: + osv: 1 + ghsa: 0 + ``` + - Restart Concelier workers; confirm tags in `concelier.merge.overrides` show the new ranks. + - Document the override with expiry in the change log. +3. **Alias remediation** + - Update connector mapping rules to weed out duplicate aliases (e.g., skip GHSA aliases that mirror CVE IDs). + - Flush cached alias graphs if necessary (`db.alias_graph.drop()` is destructive-coordinate with Storage before issuing). +4. **Escalation** + - If override metrics spike due to upstream regression, open an incident with Security Guild, referencing merge logs and `merge_event` IDs. + +--- + +## 7. Validation Checklist + +- [ ] Merge job rerun returns exit code `0`. +- [ ] `concelier.merge.conflicts` baseline returns to zero after corrective action. +- [ ] Latest `merge_event` entry shows expected hash delta. +- [ ] Affected advisory document shows updated `provenance[].decisionReason`. +- [ ] Ops change log updated with incident summary, config overrides, and rollback plan. + +--- + +## 8. Reference Material + +- Canonical conflict rules: `src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md`. +- Merge engine internals: `src/StellaOps.Concelier.Merge/Services/AdvisoryPrecedenceMerger.cs`. +- Metrics definitions: `src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs` (identity conflicts) and `AdvisoryPrecedenceMerger`. +- Storage audit trail: `src/StellaOps.Concelier.Merge/Services/MergeEventWriter.cs`, `src/StellaOps.Concelier.Storage.Mongo/MergeEvents`. + +Keep this runbook synchronized with future sprint notes and update alert thresholds as baseline volumes change. + +--- + +## 9. Synthetic Regression Fixtures + +- **Locations** – Canonical conflict snapshots now live at `src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json`, `src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json`, and `src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/conflict-osv.canonical.json`. +- **Validation commands** – To regenerate and verify the fixtures offline, run: + +```bash +dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests +dotnet test src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj --filter NvdConflictFixtureTests +dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj --filter OsvConflictFixtureTests +dotnet test src/StellaOps.Concelier.Merge.Tests/StellaOps.Concelier.Merge.Tests.csproj --filter MergeAsync_AppliesCanonicalRulesAndPersistsDecisions +``` + +- **Expected signals** – The triple produces one freshness-driven summary override (`primary_source=osv`, `suppressed_source=ghsa`) and one range override for the npm SemVer package while leaving `concelier.merge.conflicts` at zero. Use these values as the baseline when tuning dashboards or load-testing alert pipelines. + +--- + +## 10. Change Log + +| Date (UTC) | Change | Notes | +|------------|--------|-------| +| 2025-10-16 | Ops review signed off after connector expansion (CCCS, CERT-Bund, KISA, ICS CISA, MSRC) landed. Alert thresholds from §3 reaffirmed; dashboards updated to watch attachment signals emitted by ICS CISA connector. | Ops sign-off recorded by Concelier Ops Guild; no additional overrides required. | diff --git a/docs/ops/feedser-cve-kev-grafana-dashboard.json b/docs/ops/concelier-cve-kev-grafana-dashboard.json similarity index 94% rename from docs/ops/feedser-cve-kev-grafana-dashboard.json rename to docs/ops/concelier-cve-kev-grafana-dashboard.json index 51df8cc4..97ee78b5 100644 --- a/docs/ops/feedser-cve-kev-grafana-dashboard.json +++ b/docs/ops/concelier-cve-kev-grafana-dashboard.json @@ -1,151 +1,151 @@ -{ - "title": "Feedser CVE & KEV Observability", - "uid": "feedser-cve-kev", - "schemaVersion": 38, - "version": 1, - "editable": true, - "timezone": "", - "time": { - "from": "now-24h", - "to": "now" - }, - "refresh": "5m", - "templating": { - "list": [ - { - "name": "datasource", - "type": "datasource", - "query": "prometheus", - "refresh": 1, - "hide": 0 - } - ] - }, - "panels": [ - { - "type": "timeseries", - "title": "CVE fetch success vs failure", - "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10 - } - }, - "overrides": [] - }, - "targets": [ - { - "refId": "A", - "expr": "rate(cve_fetch_success_total[5m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "success" - }, - { - "refId": "B", - "expr": "rate(cve_fetch_failures_total[5m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "failure" - } - ] - }, - { - "type": "timeseries", - "title": "KEV fetch cadence", - "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10 - } - }, - "overrides": [] - }, - "targets": [ - { - "refId": "A", - "expr": "rate(kev_fetch_success_total[30m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "success" - }, - { - "refId": "B", - "expr": "rate(kev_fetch_failures_total[30m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "failure" - }, - { - "refId": "C", - "expr": "rate(kev_fetch_unchanged_total[30m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "unchanged" - } - ] - }, - { - "type": "table", - "title": "KEV parse anomalies (24h)", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "targets": [ - { - "refId": "A", - "expr": "sum by (reason) (increase(kev_parse_anomalies_total[24h]))", - "format": "table", - "datasource": { "type": "prometheus", "uid": "${datasource}" } - } - ], - "transformations": [ - { - "id": "organize", - "options": { - "renameByName": { - "Value": "count" - } - } - } - ] - }, - { - "type": "timeseries", - "title": "Advisories emitted", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "custom": { - "drawStyle": "line", - "lineWidth": 2, - "fillOpacity": 10 - } - }, - "overrides": [] - }, - "targets": [ - { - "refId": "A", - "expr": "rate(cve_map_success_total[15m])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "CVE" - }, - { - "refId": "B", - "expr": "rate(kev_map_advisories_total[24h])", - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "legendFormat": "KEV" - } - ] - } - ] -} +{ + "title": "Concelier CVE & KEV Observability", + "uid": "concelier-cve-kev", + "schemaVersion": 38, + "version": 1, + "editable": true, + "timezone": "", + "time": { + "from": "now-24h", + "to": "now" + }, + "refresh": "5m", + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "hide": 0 + } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "CVE fetch success vs failure", + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(cve_fetch_success_total[5m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "success" + }, + { + "refId": "B", + "expr": "rate(cve_fetch_failures_total[5m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "failure" + } + ] + }, + { + "type": "timeseries", + "title": "KEV fetch cadence", + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 0 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(kev_fetch_success_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "success" + }, + { + "refId": "B", + "expr": "rate(kev_fetch_failures_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "failure" + }, + { + "refId": "C", + "expr": "rate(kev_fetch_unchanged_total[30m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "unchanged" + } + ] + }, + { + "type": "table", + "title": "KEV parse anomalies (24h)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (reason) (increase(kev_parse_anomalies_total[24h]))", + "format": "table", + "datasource": { "type": "prometheus", "uid": "${datasource}" } + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "renameByName": { + "Value": "count" + } + } + } + ] + }, + { + "type": "timeseries", + "title": "Advisories emitted", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineWidth": 2, + "fillOpacity": 10 + } + }, + "overrides": [] + }, + "targets": [ + { + "refId": "A", + "expr": "rate(cve_map_success_total[15m])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "CVE" + }, + { + "refId": "B", + "expr": "rate(kev_map_advisories_total[24h])", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "legendFormat": "KEV" + } + ] + } + ] +} diff --git a/docs/ops/feedser-cve-kev-operations.md b/docs/ops/concelier-cve-kev-operations.md similarity index 76% rename from docs/ops/feedser-cve-kev-operations.md rename to docs/ops/concelier-cve-kev-operations.md index c3e84bd1..66f38377 100644 --- a/docs/ops/feedser-cve-kev-operations.md +++ b/docs/ops/concelier-cve-kev-operations.md @@ -1,4 +1,4 @@ -# Feedser CVE & KEV Connector Operations +# Concelier CVE & KEV Connector Operations This playbook equips operators with the steps required to roll out and monitor the CVE Services and CISA KEV connectors across environments. @@ -7,17 +7,17 @@ This playbook equips operators with the steps required to roll out and monitor t ### 1.1 Prerequisites - CVE Services API credentials (organisation ID, user ID, API key) with access to the JSON 5 API. -- Network egress to `https://cveawg.mitre.org` (or a mirrored endpoint) from the Feedser workers. -- Updated `feedser.yaml` (or the matching environment variables) with the following section: +- Network egress to `https://cveawg.mitre.org` (or a mirrored endpoint) from the Concelier workers. +- Updated `concelier.yaml` (or the matching environment variables) with the following section: ```yaml -feedser: +concelier: sources: cve: baseEndpoint: "https://cveawg.mitre.org/api/" apiOrg: "ORG123" apiUser: "user@example.org" - apiKeyFile: "/var/run/secrets/feedser/cve-api-key" + apiKeyFile: "/var/run/secrets/concelier/cve-api-key" seedDirectory: "./seed-data/cve" pageSize: 200 maxPagesPerFetch: 5 @@ -26,44 +26,44 @@ feedser: failureBackoff: "00:10:00" ``` -> ℹ️ Store the API key outside source control. When using `apiKeyFile`, mount the secret file into the container/host; alternatively supply `apiKey` via `FEEDSER_SOURCES__CVE__APIKEY`. +> ℹ️ Store the API key outside source control. When using `apiKeyFile`, mount the secret file into the container/host; alternatively supply `apiKey` via `CONCELIER_SOURCES__CVE__APIKEY`. > 🪙 When credentials are not yet available, configure `seedDirectory` to point at mirrored CVE JSON (for example, the repo’s `seed-data/cve/` bundle). The connector will ingest those records and log a warning instead of failing the job; live fetching resumes automatically once `apiOrg` / `apiUser` / `apiKey` are supplied. ### 1.2 Smoke Test (staging) -1. Deploy the updated configuration and restart the Feedser service so the connector picks up the credentials. +1. Deploy the updated configuration and restart the Concelier service so the connector picks up the credentials. 2. Trigger one end-to-end cycle: - - Feedser CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map` + - Concelier CLI: `stella db jobs run source:cve:fetch --and-then source:cve:parse --and-then source:cve:map` - REST fallback: `POST /jobs/run { "kind": "source:cve:fetch", "chain": ["source:cve:parse", "source:cve:map"] }` -3. Observe the following metrics (exported via OTEL meter `StellaOps.Feedser.Source.Cve`): +3. Observe the following metrics (exported via OTEL meter `StellaOps.Concelier.Connector.Cve`): - `cve.fetch.attempts`, `cve.fetch.success`, `cve.fetch.documents`, `cve.fetch.failures`, `cve.fetch.unchanged` - `cve.parse.success`, `cve.parse.failures`, `cve.parse.quarantine` - `cve.map.success` -4. Verify Prometheus shows matching `feedser.source.http.requests_total{feedser_source="cve"}` deltas (list vs detail phases) while `feedser.source.http.failures_total{feedser_source="cve"}` stays flat. +4. Verify Prometheus shows matching `concelier.source.http.requests_total{concelier_source="cve"}` deltas (list vs detail phases) while `concelier.source.http.failures_total{concelier_source="cve"}` stays flat. 5. Confirm the info-level summary log `CVEs fetch window … pages=X detailDocuments=Y detailFailures=Z` appears once per fetch run and shows `detailFailures=0`. 6. Verify the MongoDB advisory store contains fresh CVE advisories (`advisoryKey` prefix `cve/`) and that the source cursor (`source_states` collection) advanced. ### 1.3 Production Monitoring -- **Dashboards** – Plot `rate(cve_fetch_success_total[5m])`, `rate(cve_fetch_failures_total[5m])`, and `rate(cve_fetch_documents_total[5m])` alongside `feedser_source_http_requests_total{feedser_source="cve"}` to confirm HTTP and connector counters stay aligned. Keep `feedser.range.primitives{scheme=~"semver|vendor"}` on the same board for range coverage. Example alerts: +- **Dashboards** – Plot `rate(cve_fetch_success_total[5m])`, `rate(cve_fetch_failures_total[5m])`, and `rate(cve_fetch_documents_total[5m])` alongside `concelier_source_http_requests_total{concelier_source="cve"}` to confirm HTTP and connector counters stay aligned. Keep `concelier.range.primitives{scheme=~"semver|vendor"}` on the same board for range coverage. Example alerts: - `rate(cve_fetch_failures_total[5m]) > 0` for 10 minutes (`severity=warning`) - `rate(cve_map_success_total[15m]) == 0` while `rate(cve_fetch_success_total[15m]) > 0` (`severity=critical`) - `sum_over_time(cve_parse_quarantine_total[1h]) > 0` to catch schema anomalies - **Logs** – Monitor warnings such as `Failed fetching CVE record {CveId}` and `Malformed CVE JSON`, and surface the summary info log `CVEs fetch window … detailFailures=0 detailUnchanged=0` on dashboards. A non-zero `detailFailures` usually indicates rate-limit or auth issues on detail requests. -- **Grafana pack** – Import `docs/ops/feedser-cve-kev-grafana-dashboard.json` and filter by panel legend (`CVE`, `KEV`) to reuse the canned layout. -- **Backfill window** – Operators can tighten or widen `initialBackfill` / `maxPagesPerFetch` after validating throughput. Update config and restart Feedser to apply changes. +- **Grafana pack** – Import `docs/ops/concelier-cve-kev-grafana-dashboard.json` and filter by panel legend (`CVE`, `KEV`) to reuse the canned layout. +- **Backfill window** – Operators can tighten or widen `initialBackfill` / `maxPagesPerFetch` after validating throughput. Update config and restart Concelier to apply changes. ### 1.4 Staging smoke log (2025-10-15) While Ops finalises long-lived CVE Services credentials, we validated the connector end-to-end against the recorded CVE-2024-0001 payloads used in regression tests: -- Command: `dotnet test src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj -l "console;verbosity=detailed"` +- Command: `dotnet test src/StellaOps.Concelier.Connector.Cve.Tests/StellaOps.Concelier.Connector.Cve.Tests.csproj -l "console;verbosity=detailed"` - Summary log emitted by the connector: ``` CVEs fetch window 2024-09-01T00:00:00Z->2024-10-01T00:00:00Z pages=1 listSuccess=1 detailDocuments=1 detailFailures=0 detailUnchanged=0 pendingDocuments=0->1 pendingMappings=0->1 hasMorePages=False nextWindowStart=2024-09-15T12:00:00Z nextWindowEnd=(none) nextPage=1 ``` -- Telemetry captured by `Meter` `StellaOps.Feedser.Source.Cve`: +- Telemetry captured by `Meter` `StellaOps.Concelier.Connector.Cve`: | Metric | Value | |--------|-------| | `cve.fetch.attempts` | 1 | @@ -72,7 +72,7 @@ While Ops finalises long-lived CVE Services credentials, we validated the connec | `cve.parse.success` | 1 | | `cve.map.success` | 1 | -The Grafana pack `docs/ops/feedser-cve-kev-grafana-dashboard.json` has been imported into staging so the panels referenced above render against these counters once the live API keys are in place. +The Grafana pack `docs/ops/concelier-cve-kev-grafana-dashboard.json` has been imported into staging so the panels referenced above render against these counters once the live API keys are in place. ## 2. CISA KEV Connector (`source:kev:*`) @@ -80,10 +80,10 @@ The Grafana pack `docs/ops/feedser-cve-kev-grafana-dashboard.json` has been impo - Network egress (or mirrored content) for `https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json`. - No credentials are required, but the HTTP allow-list must include `www.cisa.gov`. -- Confirm the following snippet in `feedser.yaml` (defaults shown; tune as needed): +- Confirm the following snippet in `concelier.yaml` (defaults shown; tune as needed): ```yaml -feedser: +concelier: sources: kev: feedUri: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" @@ -105,15 +105,15 @@ Treat repeated schema failures or growing anomaly counts as an upstream regressi ### 2.3 Smoke Test (staging) -1. Deploy the configuration and restart Feedser. +1. Deploy the configuration and restart Concelier. 2. Trigger a pipeline run: - CLI: `stella db jobs run source:kev:fetch --and-then source:kev:parse --and-then source:kev:map` - REST: `POST /jobs/run { "kind": "source:kev:fetch", "chain": ["source:kev:parse", "source:kev:map"] }` -3. Verify the metrics exposed by meter `StellaOps.Feedser.Source.Kev`: +3. Verify the metrics exposed by meter `StellaOps.Concelier.Connector.Kev`: - `kev.fetch.attempts`, `kev.fetch.success`, `kev.fetch.unchanged`, `kev.fetch.failures` - `kev.parse.entries` (tag `catalogVersion`), `kev.parse.failures`, `kev.parse.anomalies` (tag `reason`) - `kev.map.advisories` (tag `catalogVersion`) -4. Confirm `feedser.source.http.requests_total{feedser_source="kev"}` increments once per fetch and that the paired `feedser.source.http.failures_total` stays flat (zero increase). +4. Confirm `concelier.source.http.requests_total{concelier_source="kev"}` increments once per fetch and that the paired `concelier.source.http.failures_total` stays flat (zero increase). 5. Inspect the info logs `Fetched KEV catalog document … pendingDocuments=…` and `Parsed KEV catalog document … entries=…`—they should appear exactly once per run and `Mapped X/Y… skipped=0` should match the `kev.map.advisories` delta. 6. Confirm MongoDB documents exist for the catalog JSON (`raw_documents` & `dtos`) and that advisories with prefix `kev/` are written. @@ -126,7 +126,7 @@ Treat repeated schema failures or growing anomaly counts as an upstream regressi ### 2.5 Known good dashboard tiles -Add the following panels to the Feedser observability board: +Add the following panels to the Concelier observability board: | Metric | Recommended visualisation | |--------|---------------------------| @@ -140,4 +140,4 @@ Add the following panels to the Feedser observability board: - Record staging/production smoke test results (date, catalog version, advisory counts) in your team’s change log. - Add the CVE/KEV job kinds to the standard maintenance checklist so operators can manually trigger them after planned downtime. - Keep this document in sync with future connector changes (for example, new anomaly reasons or additional metrics). -- Version-control dashboard tweaks alongside `docs/ops/feedser-cve-kev-grafana-dashboard.json` so operations can re-import the observability pack during restores. +- Version-control dashboard tweaks alongside `docs/ops/concelier-cve-kev-grafana-dashboard.json` so operations can re-import the observability pack during restores. diff --git a/docs/ops/feedser-ghsa-operations.md b/docs/ops/concelier-ghsa-operations.md similarity index 85% rename from docs/ops/feedser-ghsa-operations.md rename to docs/ops/concelier-ghsa-operations.md index fb29fe6c..bc5c8177 100644 --- a/docs/ops/feedser-ghsa-operations.md +++ b/docs/ops/concelier-ghsa-operations.md @@ -1,123 +1,123 @@ -# Feedser GHSA Connector – Operations Runbook - -_Last updated: 2025-10-16_ - -## 1. Overview -The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents. - -## 2. Rate-limit telemetry -The connector now surfaces rate-limit headers on every fetch and exposes the following metrics via OpenTelemetry: - -| Metric | Description | Tags | -|--------|-------------|------| -| `ghsa.ratelimit.limit` (histogram) | Samples the reported request quota at fetch time. | `phase` = `list` or `detail`, `resource` (e.g., `core`). | -| `ghsa.ratelimit.remaining` (histogram) | Remaining requests returned by `X-RateLimit-Remaining`. | `phase`, `resource`. | -| `ghsa.ratelimit.reset_seconds` (histogram) | Seconds until `X-RateLimit-Reset`. | `phase`, `resource`. | -| `ghsa.ratelimit.headroom_pct` (histogram) | Percentage of the quota still available (`remaining / limit * 100`). | `phase`, `resource`. | -| `ghsa.ratelimit.headroom_pct_current` (observable gauge) | Latest headroom percentage reported per resource. | `phase`, `resource`. | -| `ghsa.ratelimit.exhausted` (counter) | Incremented whenever GitHub returns a zero remaining quota and the connector delays before retrying. | `phase`. | - -### Dashboards & alerts -- Plot `ghsa.ratelimit.remaining` as the latest value to watch the runway. Alert when the value stays below **`RateLimitWarningThreshold`** (default `500`) for more than 5 minutes. -- Use `ghsa.ratelimit.headroom_pct_current` to visualise remaining quota % — paging once it sits below **10 %** for longer than a single reset window helps avoid secondary limits. -- Raise a separate alert on `increase(ghsa.ratelimit.exhausted[15m]) > 0` to catch hard throttles. -- Overlay `ghsa.fetch.attempts` vs `ghsa.fetch.failures` to confirm retries are effective. - -## 3. Logging signals -When `X-RateLimit-Remaining` falls below `RateLimitWarningThreshold`, the connector emits: -``` -GHSA rate limit warning: remaining {Remaining}/{Limit} for {Phase} {Resource} (headroom {Headroom}%) -``` -When GitHub reports zero remaining calls, the connector logs and sleeps for the reported `Retry-After`/`X-RateLimit-Reset` interval (falling back to `SecondaryRateLimitBackoff`). - -After the quota recovers above the warning threshold the connector writes an informational log with the refreshed remaining/headroom, letting operators clear alerts quickly. - -## 4. Configuration knobs (`feedser.yaml`) -```yaml -feedser: - sources: - ghsa: - apiToken: "${GITHUB_PAT}" - pageSize: 50 - requestDelay: "00:00:00.200" - failureBackoff: "00:05:00" - rateLimitWarningThreshold: 500 # warn below this many remaining calls - secondaryRateLimitBackoff: "00:02:00" # fallback delay when GitHub omits Retry-After -``` - -### Recommendations -- Increase `requestDelay` in air-gapped or burst-heavy deployments to smooth token consumption. -- Lower `rateLimitWarningThreshold` only if your dashboards already page on the new histogram; never set it negative. -- For bots using a low-privilege PAT, keep `secondaryRateLimitBackoff` at ≥60 seconds to respect GitHub’s secondary-limit guidance. - -#### Default job schedule - -| Job kind | Cron | Timeout | Lease | -|----------|------|---------|-------| -| `source:ghsa:fetch` | `1,11,21,31,41,51 * * * *` | 6 minutes | 4 minutes | -| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes | -| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes | - -These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `feedser.jobs.definitions[...]` when coordinating multiple connectors on the same runner. - -## 5. Provisioning credentials - -Feedser requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `feedser.sources.ghsa.apiToken`. - -### Docker Compose (stack operators) -```yaml -services: - feedser: - environment: - FEEDSER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat - secrets: - - ghsa_pat - -secrets: - ghsa_pat: - file: ./secrets/ghsa_pat.txt # contains only the PAT value -``` - -### Helm values (cluster operators) -```yaml -feedser: - extraEnv: - - name: FEEDSER__SOURCES__GHSA__APITOKEN - valueFrom: - secretKeyRef: - name: feedser-ghsa - key: apiToken - -extraSecrets: - feedser-ghsa: - apiToken: "" -``` - -After rotating the PAT, restart the Feedser workers (or run `kubectl rollout restart deployment/feedser`) to ensure the configuration reloads. - -When enabling GHSA the first time, run a staged backfill: - -1. Trigger `source:ghsa:fetch` manually (CLI or API) outside of peak hours. -2. Watch `feedser.jobs.health` for the GHSA jobs until they report `healthy`. -3. Allow the scheduled cron cadence to resume once the initial backlog drains (typically < 30 minutes). - -## 6. Runbook steps when throttled -1. Check `ghsa.ratelimit.exhausted` for the affected phase (`list` vs `detail`). -2. Confirm the connector is delaying—logs will show `GHSA rate limit exhausted...` with the chosen backoff. -3. If rate limits stay exhausted: - - Verify no other jobs are sharing the PAT. - - Temporarily reduce `MaxPagesPerFetch` or `PageSize` to shrink burst size. - - Consider provisioning a dedicated PAT (GHSA permissions only) for Feedser. -4. After the quota resets, reset `rateLimitWarningThreshold`/`requestDelay` to their normal values and monitor the histograms for at least one hour. - -## 7. Alert integration quick reference -- Prometheus: `ghsa_ratelimit_remaining_bucket` (from histogram) – use `histogram_quantile(0.99, ...)` to trend capacity. -- VictoriaMetrics: `LAST_over_time(ghsa_ratelimit_remaining_sum[5m])` for simple last-value graphs. -- Grafana: stack remaining + used to visualise total limit per resource. - -## 8. Canonical metric fallback analytics -When GitHub omits CVSS vectors/scores, the connector now assigns a deterministic canonical metric id in the form `ghsa:severity/` and publishes it to Merge so severity precedence still resolves against GHSA even without CVSS data. - -- Metric: `ghsa.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `reason=no_cvss`. -- Monitor the counter alongside Merge parity checks; a sudden spike suggests GitHub is shipping advisories without vectors and warrants cross-checking downstream exporters. -- Because the canonical id feeds Merge, parity dashboards should overlay this metric to confirm fallback advisories continue to merge ahead of downstream sources when GHSA supplies more recent data. +# Concelier GHSA Connector – Operations Runbook + +_Last updated: 2025-10-16_ + +## 1. Overview +The GitHub Security Advisories (GHSA) connector pulls advisory metadata from the GitHub REST API `/security/advisories` endpoint. GitHub enforces both primary and secondary rate limits, so operators must monitor usage and configure retries to avoid throttling incidents. + +## 2. Rate-limit telemetry +The connector now surfaces rate-limit headers on every fetch and exposes the following metrics via OpenTelemetry: + +| Metric | Description | Tags | +|--------|-------------|------| +| `ghsa.ratelimit.limit` (histogram) | Samples the reported request quota at fetch time. | `phase` = `list` or `detail`, `resource` (e.g., `core`). | +| `ghsa.ratelimit.remaining` (histogram) | Remaining requests returned by `X-RateLimit-Remaining`. | `phase`, `resource`. | +| `ghsa.ratelimit.reset_seconds` (histogram) | Seconds until `X-RateLimit-Reset`. | `phase`, `resource`. | +| `ghsa.ratelimit.headroom_pct` (histogram) | Percentage of the quota still available (`remaining / limit * 100`). | `phase`, `resource`. | +| `ghsa.ratelimit.headroom_pct_current` (observable gauge) | Latest headroom percentage reported per resource. | `phase`, `resource`. | +| `ghsa.ratelimit.exhausted` (counter) | Incremented whenever GitHub returns a zero remaining quota and the connector delays before retrying. | `phase`. | + +### Dashboards & alerts +- Plot `ghsa.ratelimit.remaining` as the latest value to watch the runway. Alert when the value stays below **`RateLimitWarningThreshold`** (default `500`) for more than 5 minutes. +- Use `ghsa.ratelimit.headroom_pct_current` to visualise remaining quota % — paging once it sits below **10 %** for longer than a single reset window helps avoid secondary limits. +- Raise a separate alert on `increase(ghsa.ratelimit.exhausted[15m]) > 0` to catch hard throttles. +- Overlay `ghsa.fetch.attempts` vs `ghsa.fetch.failures` to confirm retries are effective. + +## 3. Logging signals +When `X-RateLimit-Remaining` falls below `RateLimitWarningThreshold`, the connector emits: +``` +GHSA rate limit warning: remaining {Remaining}/{Limit} for {Phase} {Resource} (headroom {Headroom}%) +``` +When GitHub reports zero remaining calls, the connector logs and sleeps for the reported `Retry-After`/`X-RateLimit-Reset` interval (falling back to `SecondaryRateLimitBackoff`). + +After the quota recovers above the warning threshold the connector writes an informational log with the refreshed remaining/headroom, letting operators clear alerts quickly. + +## 4. Configuration knobs (`concelier.yaml`) +```yaml +concelier: + sources: + ghsa: + apiToken: "${GITHUB_PAT}" + pageSize: 50 + requestDelay: "00:00:00.200" + failureBackoff: "00:05:00" + rateLimitWarningThreshold: 500 # warn below this many remaining calls + secondaryRateLimitBackoff: "00:02:00" # fallback delay when GitHub omits Retry-After +``` + +### Recommendations +- Increase `requestDelay` in air-gapped or burst-heavy deployments to smooth token consumption. +- Lower `rateLimitWarningThreshold` only if your dashboards already page on the new histogram; never set it negative. +- For bots using a low-privilege PAT, keep `secondaryRateLimitBackoff` at ≥60 seconds to respect GitHub’s secondary-limit guidance. + +#### Default job schedule + +| Job kind | Cron | Timeout | Lease | +|----------|------|---------|-------| +| `source:ghsa:fetch` | `1,11,21,31,41,51 * * * *` | 6 minutes | 4 minutes | +| `source:ghsa:parse` | `3,13,23,33,43,53 * * * *` | 5 minutes | 4 minutes | +| `source:ghsa:map` | `5,15,25,35,45,55 * * * *` | 5 minutes | 4 minutes | + +These defaults spread GHSA stages across the hour so fetch completes before parse/map fire. Override them via `concelier.jobs.definitions[...]` when coordinating multiple connectors on the same runner. + +## 5. Provisioning credentials + +Concelier requires a GitHub personal access token (classic) with the **`read:org`** and **`security_events`** scopes to pull GHSA data. Store it as a secret and reference it via `concelier.sources.ghsa.apiToken`. + +### Docker Compose (stack operators) +```yaml +services: + concelier: + environment: + CONCELIER__SOURCES__GHSA__APITOKEN: /run/secrets/ghsa_pat + secrets: + - ghsa_pat + +secrets: + ghsa_pat: + file: ./secrets/ghsa_pat.txt # contains only the PAT value +``` + +### Helm values (cluster operators) +```yaml +concelier: + extraEnv: + - name: CONCELIER__SOURCES__GHSA__APITOKEN + valueFrom: + secretKeyRef: + name: concelier-ghsa + key: apiToken + +extraSecrets: + concelier-ghsa: + apiToken: "" +``` + +After rotating the PAT, restart the Concelier workers (or run `kubectl rollout restart deployment/concelier`) to ensure the configuration reloads. + +When enabling GHSA the first time, run a staged backfill: + +1. Trigger `source:ghsa:fetch` manually (CLI or API) outside of peak hours. +2. Watch `concelier.jobs.health` for the GHSA jobs until they report `healthy`. +3. Allow the scheduled cron cadence to resume once the initial backlog drains (typically < 30 minutes). + +## 6. Runbook steps when throttled +1. Check `ghsa.ratelimit.exhausted` for the affected phase (`list` vs `detail`). +2. Confirm the connector is delaying—logs will show `GHSA rate limit exhausted...` with the chosen backoff. +3. If rate limits stay exhausted: + - Verify no other jobs are sharing the PAT. + - Temporarily reduce `MaxPagesPerFetch` or `PageSize` to shrink burst size. + - Consider provisioning a dedicated PAT (GHSA permissions only) for Concelier. +4. After the quota resets, reset `rateLimitWarningThreshold`/`requestDelay` to their normal values and monitor the histograms for at least one hour. + +## 7. Alert integration quick reference +- Prometheus: `ghsa_ratelimit_remaining_bucket` (from histogram) – use `histogram_quantile(0.99, ...)` to trend capacity. +- VictoriaMetrics: `LAST_over_time(ghsa_ratelimit_remaining_sum[5m])` for simple last-value graphs. +- Grafana: stack remaining + used to visualise total limit per resource. + +## 8. Canonical metric fallback analytics +When GitHub omits CVSS vectors/scores, the connector now assigns a deterministic canonical metric id in the form `ghsa:severity/` and publishes it to Merge so severity precedence still resolves against GHSA even without CVSS data. + +- Metric: `ghsa.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `reason=no_cvss`. +- Monitor the counter alongside Merge parity checks; a sudden spike suggests GitHub is shipping advisories without vectors and warrants cross-checking downstream exporters. +- Because the canonical id feeds Merge, parity dashboards should overlay this metric to confirm fallback advisories continue to merge ahead of downstream sources when GHSA supplies more recent data. diff --git a/docs/ops/feedser-icscisa-operations.md b/docs/ops/concelier-icscisa-operations.md similarity index 76% rename from docs/ops/feedser-icscisa-operations.md rename to docs/ops/concelier-icscisa-operations.md index 06b2db0d..914c7267 100644 --- a/docs/ops/feedser-icscisa-operations.md +++ b/docs/ops/concelier-icscisa-operations.md @@ -1,122 +1,122 @@ -# Feedser CISA ICS Connector Operations - -This runbook documents how to provision, rotate, and validate credentials for the CISA Industrial Control Systems (ICS) connector (`source:ics-cisa:*`). Follow it before enabling the connector in staging or offline installations. - -## 1. Credential Provisioning - -1. **Create a service mailbox** reachable by the Ops crew (shared mailbox recommended). -2. Browse to `https://public.govdelivery.com/accounts/USDHSCISA/subscriber/new` and subscribe the mailbox to the following GovDelivery topics: - - `USDHSCISA_16` — ICS-CERT advisories (legacy numbering: `ICSA-YY-###`). - - `USDHSCISA_19` — ICS medical advisories (`ICSMA-YY-###`). - - `USDHSCISA_17` — ICS alerts (`IR-ALERT-YY-###`) for completeness. -3. Complete the verification email. After confirmation, note the **personalised subscription code** included in the “Manage Preferences” link. It has the shape `code=AB12CD34EF`. -4. Store the code in the shared secret vault (or Offline Kit secrets bundle) as `feedser/sources/icscisa/govdelivery/code`. - -> ℹ️ GovDelivery does not expose a one-time API key; the personalised code is what authenticates the RSS pull. Never commit it to git. - -## 2. Feed Validation - -Use the following command to confirm the feed is reachable before wiring it into Feedser (substitute `` with the personalised value): - -```bash -curl -H "User-Agent: StellaOpsFeedser/ics-cisa" \ - "https://content.govdelivery.com/accounts/USDHSCISA/topics/ICS-CERT/feed.rss?format=xml&code=" -``` - -If the endpoint returns HTTP 200 and an RSS payload, record the sample response under `docs/artifacts/icscisa/` (see Task `FEEDCONN-ICSCISA-02-007`). HTTP 403 or 406 usually means the subscription was not confirmed or the code was mistyped. - -## 3. Configuration Snippet - -Add the connector configuration to `feedser.yaml` (or equivalent environment variables): - -```yaml -feedser: - sources: - icscisa: - govDelivery: - code: "${FEEDSER_ICS_CISA_GOVDELIVERY_CODE}" - topics: - - "USDHSCISA_16" - - "USDHSCISA_19" - - "USDHSCISA_17" - rssBaseUri: "https://content.govdelivery.com/accounts/USDHSCISA" - requestDelay: "00:00:01" - failureBackoff: "00:05:00" -``` - -Environment variable example: - -```bash -export FEEDSER_SOURCES_ICSCISA_GOVDELIVERY_CODE="AB12CD34EF" -``` - -Feedser automatically register the host with the Source.Common HTTP allow-list when the connector assembly is loaded. - - -Optional tuning keys (set only when needed): - -- `proxyUri` — HTTP/HTTPS proxy URL used when Akamai blocks direct pulls. -- `requestVersion` / `requestVersionPolicy` — override HTTP negotiation when the proxy requires HTTP/1.1. -- `enableDetailScrape` — toggle HTML detail fallback (defaults to true). -- `captureAttachments` — collect PDF attachments from detail pages (defaults to true). -- `detailBaseUri` — alternate host for detail enrichment if CISA changes their layout. - -## 4. Seeding Without GovDelivery - -If credentials are still pending, populate the connector with the community CSV dataset before enabling the live fetch: - -1. Run `./scripts/fetch-ics-cisa-seed.sh` (or `.ps1`) to download the latest `CISA_ICS_ADV_*.csv` files into `seed-data/ics-cisa/`. -2. Copy the CSVs (and the generated `.sha256` files) into your Offline Kit staging area so they ship alongside the other feeds. -3. Import the kit as usual. The connector can parse the seed data for historical context, but **live GovDelivery credentials are still required** for fresh advisories. -4. Once credentials arrive, update `feedser:sources:icscisa:govDelivery:code` and re-trigger `source:ics-cisa:fetch` so the connector switches to the authorised feed. - -> The CSVs are licensed under ODbL 1.0 by the ICS Advisory Project. Preserve the attribution when redistributing them. - -## 4. Integration Validation - -1. Ensure secrets are in place and restart the Feedser workers. -2. Run a dry-run fetch/parse/map chain against an Akamai-protected topic: - ```bash - FEEDSER_SOURCES_ICSCISA_GOVDELIVERY_CODE=... \ - FEEDSER_SOURCES_ICSCISA_ENABLEDETAILSCRAPE=1 \ - stella db jobs run source:ics-cisa:fetch --and-then source:ics-cisa:parse --and-then source:ics-cisa:map - ``` -3. Confirm logs contain `ics-cisa detail fetch` entries and that new documents/DTOs include attachments (see `docs/artifacts/icscisa`). Canonical advisories should expose PDF links as `references.kind == "attachment"` and affected packages should surface `primitives.semVer.exactValue` for single-version hits. -4. If Akamai blocks direct fetches, set `feedser:sources:icscisa:proxyUri` to your allow-listed egress proxy and rerun the dry-run. - -## 4. Rotation & Incident Response - -- Review GovDelivery access quarterly. Rotate the personalised code whenever Ops changes the service mailbox password or membership. -- Revoking the subscription in GovDelivery invalidates the code immediately; update the vault and configuration in the same change. -- If the code leaks, remove the subscription (`https://public.govdelivery.com/accounts/USDHSCISA/subscriber/manage_preferences?code=`), resubscribe, and distribute the new value via the vault. - -## 5. Offline Kit Handling - -Include the personalised code in `offline-kit/secrets/feedser/icscisa.env`: - -``` -FEEDSER_SOURCES_ICSCISA_GOVDELIVERY_CODE=AB12CD34EF -``` - -The Offline Kit deployment script copies this file into the container secret directory mounted at `/run/secrets/feedser`. Ensure permissions are `600` and ownership matches the Feedser runtime user. - -## 6. Telemetry & Monitoring - -The connector emits metrics under the meter `StellaOps.Feedser.Source.Ics.Cisa`. They allow operators to track Akamai fallbacks, detail enrichment health, and advisory fan-out. - -- `icscisa.fetch.*` – counters for `attempts`, `success`, `failures`, `not_modified`, and `fallbacks`, plus histogram `icscisa.fetch.documents` showing documents added per topic pull (tags: `feedser.source`, `icscisa.topic`). -- `icscisa.parse.*` – counters for `success`/`failures` and histograms `icscisa.parse.advisories`, `icscisa.parse.attachments`, `icscisa.parse.detail_fetches` to monitor enrichment workload per feed document. -- `icscisa.detail.*` – counters `success` / `failures` per advisory (tagged with `icscisa.advisory`) to alert when Akamai blocks detail pages. -- `icscisa.map.*` – counters for `success`/`failures` and histograms `icscisa.map.references`, `icscisa.map.packages`, `icscisa.map.aliases` capturing canonical fan-out. - -Suggested alerts: - -- `increase(icscisa.fetch.failures_total[15m]) > 0` or `increase(icscisa.fetch.fallbacks_total[15m]) > 5` — sustained Akamai or proxy issues. -- `increase(icscisa.detail.failures_total[30m]) > 0` — detail enrichment breaking (potential HTML layout change). -- `histogram_quantile(0.95, rate(icscisa.map.references_bucket[1h]))` trending sharply higher — sudden advisory reference explosion worth investigating. -- Keep an eye on shared HTTP metrics (`feedser.source.http.*{feedser.source="ics-cisa"}`) for request latency and retry patterns. - -## 6. Related Tasks - -- `FEEDCONN-ICSCISA-02-009` (GovDelivery credential onboarding) — completed once this runbook is followed and secrets are placed in the vault. -- `FEEDCONN-ICSCISA-02-007` (document inventory) — archive the first successful RSS response and any attachment URL schema under `docs/artifacts/icscisa/`. +# Concelier CISA ICS Connector Operations + +This runbook documents how to provision, rotate, and validate credentials for the CISA Industrial Control Systems (ICS) connector (`source:ics-cisa:*`). Follow it before enabling the connector in staging or offline installations. + +## 1. Credential Provisioning + +1. **Create a service mailbox** reachable by the Ops crew (shared mailbox recommended). +2. Browse to `https://public.govdelivery.com/accounts/USDHSCISA/subscriber/new` and subscribe the mailbox to the following GovDelivery topics: + - `USDHSCISA_16` — ICS-CERT advisories (legacy numbering: `ICSA-YY-###`). + - `USDHSCISA_19` — ICS medical advisories (`ICSMA-YY-###`). + - `USDHSCISA_17` — ICS alerts (`IR-ALERT-YY-###`) for completeness. +3. Complete the verification email. After confirmation, note the **personalised subscription code** included in the “Manage Preferences” link. It has the shape `code=AB12CD34EF`. +4. Store the code in the shared secret vault (or Offline Kit secrets bundle) as `concelier/sources/icscisa/govdelivery/code`. + +> ℹ️ GovDelivery does not expose a one-time API key; the personalised code is what authenticates the RSS pull. Never commit it to git. + +## 2. Feed Validation + +Use the following command to confirm the feed is reachable before wiring it into Concelier (substitute `` with the personalised value): + +```bash +curl -H "User-Agent: StellaOpsConcelier/ics-cisa" \ + "https://content.govdelivery.com/accounts/USDHSCISA/topics/ICS-CERT/feed.rss?format=xml&code=" +``` + +If the endpoint returns HTTP 200 and an RSS payload, record the sample response under `docs/artifacts/icscisa/` (see Task `FEEDCONN-ICSCISA-02-007`). HTTP 403 or 406 usually means the subscription was not confirmed or the code was mistyped. + +## 3. Configuration Snippet + +Add the connector configuration to `concelier.yaml` (or equivalent environment variables): + +```yaml +concelier: + sources: + icscisa: + govDelivery: + code: "${CONCELIER_ICS_CISA_GOVDELIVERY_CODE}" + topics: + - "USDHSCISA_16" + - "USDHSCISA_19" + - "USDHSCISA_17" + rssBaseUri: "https://content.govdelivery.com/accounts/USDHSCISA" + requestDelay: "00:00:01" + failureBackoff: "00:05:00" +``` + +Environment variable example: + +```bash +export CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE="AB12CD34EF" +``` + +Concelier automatically register the host with the Source.Common HTTP allow-list when the connector assembly is loaded. + + +Optional tuning keys (set only when needed): + +- `proxyUri` — HTTP/HTTPS proxy URL used when Akamai blocks direct pulls. +- `requestVersion` / `requestVersionPolicy` — override HTTP negotiation when the proxy requires HTTP/1.1. +- `enableDetailScrape` — toggle HTML detail fallback (defaults to true). +- `captureAttachments` — collect PDF attachments from detail pages (defaults to true). +- `detailBaseUri` — alternate host for detail enrichment if CISA changes their layout. + +## 4. Seeding Without GovDelivery + +If credentials are still pending, populate the connector with the community CSV dataset before enabling the live fetch: + +1. Run `./scripts/fetch-ics-cisa-seed.sh` (or `.ps1`) to download the latest `CISA_ICS_ADV_*.csv` files into `seed-data/ics-cisa/`. +2. Copy the CSVs (and the generated `.sha256` files) into your Offline Kit staging area so they ship alongside the other feeds. +3. Import the kit as usual. The connector can parse the seed data for historical context, but **live GovDelivery credentials are still required** for fresh advisories. +4. Once credentials arrive, update `concelier:sources:icscisa:govDelivery:code` and re-trigger `source:ics-cisa:fetch` so the connector switches to the authorised feed. + +> The CSVs are licensed under ODbL 1.0 by the ICS Advisory Project. Preserve the attribution when redistributing them. + +## 4. Integration Validation + +1. Ensure secrets are in place and restart the Concelier workers. +2. Run a dry-run fetch/parse/map chain against an Akamai-protected topic: + ```bash + CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE=... \ + CONCELIER_SOURCES_ICSCISA_ENABLEDETAILSCRAPE=1 \ + stella db jobs run source:ics-cisa:fetch --and-then source:ics-cisa:parse --and-then source:ics-cisa:map + ``` +3. Confirm logs contain `ics-cisa detail fetch` entries and that new documents/DTOs include attachments (see `docs/artifacts/icscisa`). Canonical advisories should expose PDF links as `references.kind == "attachment"` and affected packages should surface `primitives.semVer.exactValue` for single-version hits. +4. If Akamai blocks direct fetches, set `concelier:sources:icscisa:proxyUri` to your allow-listed egress proxy and rerun the dry-run. + +## 4. Rotation & Incident Response + +- Review GovDelivery access quarterly. Rotate the personalised code whenever Ops changes the service mailbox password or membership. +- Revoking the subscription in GovDelivery invalidates the code immediately; update the vault and configuration in the same change. +- If the code leaks, remove the subscription (`https://public.govdelivery.com/accounts/USDHSCISA/subscriber/manage_preferences?code=`), resubscribe, and distribute the new value via the vault. + +## 5. Offline Kit Handling + +Include the personalised code in `offline-kit/secrets/concelier/icscisa.env`: + +``` +CONCELIER_SOURCES_ICSCISA_GOVDELIVERY_CODE=AB12CD34EF +``` + +The Offline Kit deployment script copies this file into the container secret directory mounted at `/run/secrets/concelier`. Ensure permissions are `600` and ownership matches the Concelier runtime user. + +## 6. Telemetry & Monitoring + +The connector emits metrics under the meter `StellaOps.Concelier.Connector.Ics.Cisa`. They allow operators to track Akamai fallbacks, detail enrichment health, and advisory fan-out. + +- `icscisa.fetch.*` – counters for `attempts`, `success`, `failures`, `not_modified`, and `fallbacks`, plus histogram `icscisa.fetch.documents` showing documents added per topic pull (tags: `concelier.source`, `icscisa.topic`). +- `icscisa.parse.*` – counters for `success`/`failures` and histograms `icscisa.parse.advisories`, `icscisa.parse.attachments`, `icscisa.parse.detail_fetches` to monitor enrichment workload per feed document. +- `icscisa.detail.*` – counters `success` / `failures` per advisory (tagged with `icscisa.advisory`) to alert when Akamai blocks detail pages. +- `icscisa.map.*` – counters for `success`/`failures` and histograms `icscisa.map.references`, `icscisa.map.packages`, `icscisa.map.aliases` capturing canonical fan-out. + +Suggested alerts: + +- `increase(icscisa.fetch.failures_total[15m]) > 0` or `increase(icscisa.fetch.fallbacks_total[15m]) > 5` — sustained Akamai or proxy issues. +- `increase(icscisa.detail.failures_total[30m]) > 0` — detail enrichment breaking (potential HTML layout change). +- `histogram_quantile(0.95, rate(icscisa.map.references_bucket[1h]))` trending sharply higher — sudden advisory reference explosion worth investigating. +- Keep an eye on shared HTTP metrics (`concelier.source.http.*{concelier.source="ics-cisa"}`) for request latency and retry patterns. + +## 6. Related Tasks + +- `FEEDCONN-ICSCISA-02-009` (GovDelivery credential onboarding) — completed once this runbook is followed and secrets are placed in the vault. +- `FEEDCONN-ICSCISA-02-007` (document inventory) — archive the first successful RSS response and any attachment URL schema under `docs/artifacts/icscisa/`. diff --git a/docs/ops/feedser-kisa-operations.md b/docs/ops/concelier-kisa-operations.md similarity index 82% rename from docs/ops/feedser-kisa-operations.md rename to docs/ops/concelier-kisa-operations.md index d2d25caf..85d060bb 100644 --- a/docs/ops/feedser-kisa-operations.md +++ b/docs/ops/concelier-kisa-operations.md @@ -1,74 +1,74 @@ -# Feedser KISA Connector Operations - -Operational guidance for the Korea Internet & Security Agency (KISA / KNVD) connector (`source:kisa:*`). Pair this with the engineering brief in `docs/dev/kisa_connector_notes.md`. - -## 1. Prerequisites - -- Outbound HTTPS (or mirrored cache) for `https://knvd.krcert.or.kr/`. -- Connector options defined under `feedser:sources:kisa`: - -```yaml -feedser: - sources: - kisa: - feedUri: "https://knvd.krcert.or.kr/rss/securityInfo.do" - detailApiUri: "https://knvd.krcert.or.kr/rssDetailData.do" - detailPageUri: "https://knvd.krcert.or.kr/detailDos.do" - maxAdvisoriesPerFetch: 10 - requestDelay: "00:00:01" - failureBackoff: "00:05:00" -``` - -> Ensure the URIs stay absolute—Feedser adds the `feedUri`/`detailApiUri` hosts to the HttpClient allow-list automatically. - -## 2. Staging Smoke Test - -1. Restart the Feedser workers so the KISA options bind. -2. Run a full connector cycle: - - CLI: `stella db jobs run source:kisa:fetch --and-then source:kisa:parse --and-then source:kisa:map` - - REST: `POST /jobs/run { "kind": "source:kisa:fetch", "chain": ["source:kisa:parse", "source:kisa:map"] }` -3. Confirm telemetry (Meter `StellaOps.Feedser.Source.Kisa`): - - `kisa.feed.success`, `kisa.feed.items` - - `kisa.detail.success` / `.failures` - - `kisa.parse.success` / `.failures` - - `kisa.map.success` / `.failures` - - `kisa.cursor.updates` -4. Inspect logs for structured entries: - - `KISA feed returned {ItemCount}` - - `KISA fetched detail for {Idx} … category={Category}` - - `KISA mapped advisory {AdvisoryId} (severity={Severity})` - - Absence of warnings such as `document missing GridFS payload`. -5. Validate MongoDB state: - - `raw_documents.metadata` has `kisa.idx`, `kisa.category`, `kisa.title`. - - DTO store contains `schemaVersion="kisa.detail.v1"`. - - Advisories include aliases (`IDX`, CVE) and `language="ko"`. - - `source_states` entry for `kisa` shows recent `cursor.lastFetchAt`. - -## 3. Production Monitoring - -- **Dashboards** – Add the following Prometheus/OTEL expressions: - - `rate(kisa_feed_items_total[15m])` versus `rate(feedser_source_http_requests_total{feedser_source="kisa"}[15m])` - - `increase(kisa_detail_failures_total{reason!="empty-document"}[1h])` alert at `>0` - - `increase(kisa_parse_failures_total[1h])` for storage/JSON issues - - `increase(kisa_map_failures_total[1h])` to flag schema drift - - `increase(kisa_cursor_updates_total[6h]) == 0` during active windows → warn -- **Alerts** – Page when `rate(kisa_feed_success_total[2h]) == 0` while other connectors are active; back off for maintenance windows announced on `https://knvd.krcert.or.kr/`. -- **Logs** – Watch for repeated warnings (`document missing`, `DTO missing`) or errors with reason tags `HttpRequestException`, `download`, `parse`, `map`. - -## 4. Localisation Handling - -- Hangul categories (for example `취약점정보`) flow into telemetry tags (`category=…`) and logs. Dashboards must render UTF‑8 and avoid transliteration. -- HTML content is sanitised before storage; translation teams can consume the `ContentHtml` field safely. -- Advisory severity remains as provided by KISA (`High`, `Medium`, etc.). Map-level failures include the severity tag for filtering. - -## 5. Fixture & Regression Maintenance - -- Regression fixtures: `src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-feed.xml` and `kisa-detail.json`. -- Refresh via `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj`. -- The telemetry regression (`KisaConnectorTests.Telemetry_RecordsMetrics`) will fail if counters/log wiring drifts—treat failures as gating. - -## 6. Known Issues - -- RSS feeds only expose the latest 10 advisories; long outages require replay via archived feeds or manual IDX seeds. -- Detail endpoint occasionally throttles; the connector honours `requestDelay` and reports failures with reason `HttpRequestException`. Consider increasing delay for weekend backfills. -- If `kisa.category` tags suddenly appear as `unknown`, verify KISA has not renamed RSS elements; update the parser fixtures before production rollout. +# Concelier KISA Connector Operations + +Operational guidance for the Korea Internet & Security Agency (KISA / KNVD) connector (`source:kisa:*`). Pair this with the engineering brief in `docs/dev/kisa_connector_notes.md`. + +## 1. Prerequisites + +- Outbound HTTPS (or mirrored cache) for `https://knvd.krcert.or.kr/`. +- Connector options defined under `concelier:sources:kisa`: + +```yaml +concelier: + sources: + kisa: + feedUri: "https://knvd.krcert.or.kr/rss/securityInfo.do" + detailApiUri: "https://knvd.krcert.or.kr/rssDetailData.do" + detailPageUri: "https://knvd.krcert.or.kr/detailDos.do" + maxAdvisoriesPerFetch: 10 + requestDelay: "00:00:01" + failureBackoff: "00:05:00" +``` + +> Ensure the URIs stay absolute—Concelier adds the `feedUri`/`detailApiUri` hosts to the HttpClient allow-list automatically. + +## 2. Staging Smoke Test + +1. Restart the Concelier workers so the KISA options bind. +2. Run a full connector cycle: + - CLI: `stella db jobs run source:kisa:fetch --and-then source:kisa:parse --and-then source:kisa:map` + - REST: `POST /jobs/run { "kind": "source:kisa:fetch", "chain": ["source:kisa:parse", "source:kisa:map"] }` +3. Confirm telemetry (Meter `StellaOps.Concelier.Connector.Kisa`): + - `kisa.feed.success`, `kisa.feed.items` + - `kisa.detail.success` / `.failures` + - `kisa.parse.success` / `.failures` + - `kisa.map.success` / `.failures` + - `kisa.cursor.updates` +4. Inspect logs for structured entries: + - `KISA feed returned {ItemCount}` + - `KISA fetched detail for {Idx} … category={Category}` + - `KISA mapped advisory {AdvisoryId} (severity={Severity})` + - Absence of warnings such as `document missing GridFS payload`. +5. Validate MongoDB state: + - `raw_documents.metadata` has `kisa.idx`, `kisa.category`, `kisa.title`. + - DTO store contains `schemaVersion="kisa.detail.v1"`. + - Advisories include aliases (`IDX`, CVE) and `language="ko"`. + - `source_states` entry for `kisa` shows recent `cursor.lastFetchAt`. + +## 3. Production Monitoring + +- **Dashboards** – Add the following Prometheus/OTEL expressions: + - `rate(kisa_feed_items_total[15m])` versus `rate(concelier_source_http_requests_total{concelier_source="kisa"}[15m])` + - `increase(kisa_detail_failures_total{reason!="empty-document"}[1h])` alert at `>0` + - `increase(kisa_parse_failures_total[1h])` for storage/JSON issues + - `increase(kisa_map_failures_total[1h])` to flag schema drift + - `increase(kisa_cursor_updates_total[6h]) == 0` during active windows → warn +- **Alerts** – Page when `rate(kisa_feed_success_total[2h]) == 0` while other connectors are active; back off for maintenance windows announced on `https://knvd.krcert.or.kr/`. +- **Logs** – Watch for repeated warnings (`document missing`, `DTO missing`) or errors with reason tags `HttpRequestException`, `download`, `parse`, `map`. + +## 4. Localisation Handling + +- Hangul categories (for example `취약점정보`) flow into telemetry tags (`category=…`) and logs. Dashboards must render UTF‑8 and avoid transliteration. +- HTML content is sanitised before storage; translation teams can consume the `ContentHtml` field safely. +- Advisory severity remains as provided by KISA (`High`, `Medium`, etc.). Map-level failures include the severity tag for filtering. + +## 5. Fixture & Regression Maintenance + +- Regression fixtures: `src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-feed.xml` and `kisa-detail.json`. +- Refresh via `UPDATE_KISA_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj`. +- The telemetry regression (`KisaConnectorTests.Telemetry_RecordsMetrics`) will fail if counters/log wiring drifts—treat failures as gating. + +## 6. Known Issues + +- RSS feeds only expose the latest 10 advisories; long outages require replay via archived feeds or manual IDX seeds. +- Detail endpoint occasionally throttles; the connector honours `requestDelay` and reports failures with reason `HttpRequestException`. Consider increasing delay for weekend backfills. +- If `kisa.category` tags suddenly appear as `unknown`, verify KISA has not renamed RSS elements; update the parser fixtures before production rollout. diff --git a/docs/ops/concelier-mirror-operations.md b/docs/ops/concelier-mirror-operations.md new file mode 100644 index 00000000..5d3e15eb --- /dev/null +++ b/docs/ops/concelier-mirror-operations.md @@ -0,0 +1,196 @@ +# Concelier & Excititor Mirror Operations + +This runbook describes how Stella Ops operates the managed mirrors under `*.stella-ops.org`. +It covers Docker Compose and Helm deployment overlays, secret handling for multi-tenant +authn, CDN fronting, and the recurring sync pipeline that keeps mirror bundles current. + +## 1. Prerequisites + +- **Authority access** – client credentials (`client_id` + secret) authorised for + `concelier.mirror.read` and `excititor.mirror.read` scopes. Secrets live outside git. +- **Signed TLS certificates** – wildcard or per-domain (`mirror-primary`, `mirror-community`). + Store them under `deploy/compose/mirror-gateway/tls/` or in Kubernetes secrets. +- **Mirror gateway credentials** – Basic Auth htpasswd files per domain. Generate with + `htpasswd -B`. Operators distribute credentials to downstream consumers. +- **Export artifact source** – read access to the canonical S3 buckets (or rsync share) + that hold `concelier` JSON bundles and `excititor` VEX exports. +- **Persistent volumes** – storage for Concelier job metadata and mirror export trees. + For Helm, provision PVCs (`concelier-mirror-jobs`, `concelier-mirror-exports`, + `excititor-mirror-exports`, `mirror-mongo-data`, `mirror-minio-data`) before rollout. + +## 2. Secret & certificate layout + +### Docker Compose (`deploy/compose/docker-compose.mirror.yaml`) + +- `deploy/compose/env/mirror.env.example` – copy to `.env` and adjust quotas or domain IDs. +- `deploy/compose/mirror-secrets/` – mount read-only into `/run/secrets`. Place: + - `concelier-authority-client` – Authority client secret. + - `excititor-authority-client` (optional) – reserve for future authn. +- `deploy/compose/mirror-gateway/tls/` – PEM-encoded cert/key pairs: + - `mirror-primary.crt`, `mirror-primary.key` + - `mirror-community.crt`, `mirror-community.key` +- `deploy/compose/mirror-gateway/secrets/` – htpasswd files: + - `mirror-primary.htpasswd` + - `mirror-community.htpasswd` + +### Helm (`deploy/helm/stellaops/values-mirror.yaml`) + +Create secrets in the target namespace: + +```bash +kubectl create secret generic concelier-mirror-auth \ + --from-file=concelier-authority-client=concelier-authority-client + +kubectl create secret generic excititor-mirror-auth \ + --from-file=excititor-authority-client=excititor-authority-client + +kubectl create secret tls mirror-gateway-tls \ + --cert=mirror-primary.crt --key=mirror-primary.key + +kubectl create secret generic mirror-gateway-htpasswd \ + --from-file=mirror-primary.htpasswd --from-file=mirror-community.htpasswd +``` + +> Keep Basic Auth lists short-lived (rotate quarterly) and document credential recipients. + +## 3. Deployment + +### 3.1 Docker Compose (edge mirrors, lab validation) + +1. `cp deploy/compose/env/mirror.env.example deploy/compose/env/mirror.env` +2. Populate secrets/tls directories as described above. +3. Sync mirror bundles (see §4) into `deploy/compose/mirror-data/…` and ensure they are mounted + on the host path backing the `concelier-exports` and `excititor-exports` volumes. +4. Run the profile validator: `deploy/tools/validate-profiles.sh`. +5. Launch: `docker compose --env-file env/mirror.env -f docker-compose.mirror.yaml up -d`. + +### 3.2 Helm (production mirrors) + +1. Provision PVCs sized for mirror bundles (baseline: 20 GiB per domain). +2. Create secrets/tls config maps (§2). +3. `helm upgrade --install mirror deploy/helm/stellaops -f deploy/helm/stellaops/values-mirror.yaml`. +4. Annotate the `stellaops-mirror-gateway` service with ingress/LoadBalancer metadata required by + your CDN (e.g., AWS load balancer scheme internal + NLB idle timeout). + +## 4. Artifact sync workflow + +Mirrors never generate exports—they ingest signed bundles produced by the Concelier and Excititor +export jobs. Recommended sync pattern: + +### 4.1 Compose host (systemd timer) + +`/usr/local/bin/mirror-sync.sh`: + +```bash +#!/usr/bin/env bash +set -euo pipefail +export AWS_ACCESS_KEY_ID=… +export AWS_SECRET_ACCESS_KEY=… + +aws s3 sync s3://mirror-stellaops/concelier/latest \ + /opt/stellaops/mirror-data/concelier --delete --size-only + +aws s3 sync s3://mirror-stellaops/excititor/latest \ + /opt/stellaops/mirror-data/excititor --delete --size-only +``` + +Schedule with a systemd timer every 5 minutes. The Compose volumes mount `/opt/stellaops/mirror-data/*` +into the containers read-only, matching `CONCELIER__MIRROR__EXPORTROOT=/exports/json` and +`EXCITITOR__ARTIFACTS__FILESYSTEM__ROOT=/exports`. + +### 4.2 Kubernetes (CronJob) + +Create a CronJob running the AWS CLI (or rclone) in the same namespace, writing into the PVCs: + +```yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mirror-sync +spec: + schedule: "*/5 * * * *" + jobTemplate: + spec: + template: + spec: + containers: + - name: sync + image: public.ecr.aws/aws-cli/aws-cli@sha256:5df5f52c29f5e3ba46d0ad9e0e3afc98701c4a0f879400b4c5f80d943b5fadea + command: + - /bin/sh + - -c + - > + aws s3 sync s3://mirror-stellaops/concelier/latest /exports/concelier --delete --size-only && + aws s3 sync s3://mirror-stellaops/excititor/latest /exports/excititor --delete --size-only + volumeMounts: + - name: concelier-exports + mountPath: /exports/concelier + - name: excititor-exports + mountPath: /exports/excititor + envFrom: + - secretRef: + name: mirror-sync-aws + restartPolicy: OnFailure + volumes: + - name: concelier-exports + persistentVolumeClaim: + claimName: concelier-mirror-exports + - name: excititor-exports + persistentVolumeClaim: + claimName: excititor-mirror-exports +``` + +## 5. CDN integration + +1. Point the CDN origin at the mirror gateway (Compose host or Kubernetes LoadBalancer). +2. Honour the response headers emitted by the gateway and Concelier/Excititor: + `Cache-Control: public, max-age=300, immutable` for mirror payloads. +3. Configure origin shields in the CDN to prevent cache stampedes. Recommended TTLs: + - Index (`/concelier/exports/index.json`, `/excititor/mirror/*/index`) → 60 s. + - Bundle/manifest payloads → 300 s. +4. Forward the `Authorization` header—Basic Auth terminates at the gateway. +5. Enforce per-domain rate limits at the CDN (matching gateway budgets) and enable logging + to SIEM for anomaly detection. + +## 6. Smoke tests + +After each deployment or sync cycle: + +```bash +# Index with Basic Auth +curl -u $PRIMARY_CREDS https://mirror-primary.stella-ops.org/concelier/exports/index.json | jq 'keys' + +# Mirror manifest signature +curl -u $PRIMARY_CREDS -I https://mirror-primary.stella-ops.org/concelier/exports/mirror/primary/manifest.json + +# Excititor consensus bundle metadata +curl -u $COMMUNITY_CREDS https://mirror-community.stella-ops.org/excititor/mirror/community/index \ + | jq '.exports[].exportKey' + +# Signed bundle + detached JWS (spot check digests) +curl -u $PRIMARY_CREDS https://mirror-primary.stella-ops.org/concelier/exports/mirror/primary/bundle.json.jws \ + -o bundle.json.jws +cosign verify-blob --signature bundle.json.jws --key mirror-key.pub bundle.json +``` + +Watch the gateway metrics (`nginx_vts` or access logs) for cache hits. In Kubernetes, `kubectl logs deploy/stellaops-mirror-gateway` +should show `X-Cache-Status: HIT/MISS`. + +## 7. Maintenance & rotation + +- **Bundle freshness** – alert if sync job lag exceeds 15 minutes or if `concelier` logs + `Mirror export root is not configured`. +- **Secret rotation** – change Authority client secrets and Basic Auth credentials quarterly. + Update the mounted secrets and restart deployments (`docker compose restart concelier` or + `kubectl rollout restart deploy/stellaops-concelier`). +- **TLS renewal** – reissue certificates, place new files, and reload gateway (`docker compose exec mirror-gateway nginx -s reload`). +- **Quota tuning** – adjust per-domain `MAXDOWNLOADREQUESTSPERHOUR` in `.env` or values file. + Align CDN rate limits and inform downstreams. + +## 8. References + +- Deployment profiles: `deploy/compose/docker-compose.mirror.yaml`, + `deploy/helm/stellaops/values-mirror.yaml` +- Mirror architecture dossiers: `docs/ARCHITECTURE_CONCELIER.md`, + `docs/ARCHITECTURE_EXCITITOR_MIRRORS.md` +- Export bundling: `docs/ARCHITECTURE_DEVOPS.md` §3, `docs/ARCHITECTURE_EXCITITOR.md` §7 diff --git a/docs/ops/feedser-msrc-operations.md b/docs/ops/concelier-msrc-operations.md similarity index 82% rename from docs/ops/feedser-msrc-operations.md rename to docs/ops/concelier-msrc-operations.md index 828b5a9c..e91aa25c 100644 --- a/docs/ops/feedser-msrc-operations.md +++ b/docs/ops/concelier-msrc-operations.md @@ -1,86 +1,86 @@ -# Feedser MSRC Connector – Azure AD Onboarding Brief - -_Drafted: 2025-10-15_ - -## 1. App registration requirements - -- **Tenant**: shared StellaOps production Azure AD. -- **Application type**: confidential client (web/API) issuing client credentials. -- **API permissions**: `api://api.msrc.microsoft.com/.default` (Application). Admin consent required once. -- **Token audience**: `https://api.msrc.microsoft.com/`. -- **Grant type**: client credentials. Feedser will request tokens via `POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token`. - -## 2. Secret/credential policy - -- Maintain two client secrets (primary + standby) rotating every 90 days. -- Store secrets in the Feedser secrets vault; Offline Kit deployments must mirror the secret payloads in their encrypted store. -- Record rotation cadence in Ops runbook and update Feedser configuration (`FEEDSER__SOURCES__VNDR__MSRC__CLIENTSECRET`) ahead of expiry. - -## 3. Feedser configuration sample - -```yaml -feedser: - sources: - vndr.msrc: - tenantId: "" - clientId: "" - clientSecret: "" - apiVersion: "2024-08-01" - locale: "en-US" - requestDelay: "00:00:00.250" - failureBackoff: "00:05:00" - cursorOverlapMinutes: 10 - downloadCvrf: false # set true to persist CVRF ZIP alongside JSON detail -``` - -## 4. CVRF artefacts - -- The MSRC REST payload exposes `cvrfUrl` per advisory. Current connector persists the link as advisory metadata and reference; it does **not** download the ZIP by default. -- Ops should mirror CVRF ZIPs when preparing Offline Kits so air-gapped deployments can reconcile advisories without direct internet access. -- Once Offline Kit storage guidelines are finalised, extend the connector configuration with `downloadCvrf: true` to enable automatic attachment retrieval. - -### 4.1 State seeding helper - -Use `tools/SourceStateSeeder` to queue historical advisories (detail JSON + optional CVRF artefacts) for replay without manual Mongo edits. Example seed file: - -```json -{ - "source": "vndr.msrc", - "cursor": { - "lastModifiedCursor": "2024-01-01T00:00:00Z" - }, - "documents": [ - { - "uri": "https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV2024-0001", - "contentFile": "./seeds/adv2024-0001.json", - "contentType": "application/json", - "metadata": { "msrc.vulnerabilityId": "ADV2024-0001" }, - "addToPendingDocuments": true - }, - { - "uri": "https://download.microsoft.com/msrc/2024/ADV2024-0001.cvrf.zip", - "contentFile": "./seeds/adv2024-0001.cvrf.zip", - "contentType": "application/zip", - "status": "mapped", - "addToPendingDocuments": false - } - ] -} -``` - -Run the helper: - -```bash -dotnet run --project tools/SourceStateSeeder -- \ - --connection-string "mongodb://localhost:27017" \ - --database feedser \ - --input seeds/msrc-backfill.json -``` - -Any documents marked `addToPendingDocuments` will appear in the connector cursor; `DownloadCvrf` can remain disabled if the ZIP artefact is pre-seeded. - -## 5. Outstanding items - -- Ops to confirm tenant/app names and provide client credentials through the secure channel. -- Connector team monitors token cache health (already implemented); validate instrumentation once Ops supplies credentials. -- Offline Kit packaging: add encrypted blob containing client credentials with rotation instructions. +# Concelier MSRC Connector – Azure AD Onboarding Brief + +_Drafted: 2025-10-15_ + +## 1. App registration requirements + +- **Tenant**: shared StellaOps production Azure AD. +- **Application type**: confidential client (web/API) issuing client credentials. +- **API permissions**: `api://api.msrc.microsoft.com/.default` (Application). Admin consent required once. +- **Token audience**: `https://api.msrc.microsoft.com/`. +- **Grant type**: client credentials. Concelier will request tokens via `POST https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token`. + +## 2. Secret/credential policy + +- Maintain two client secrets (primary + standby) rotating every 90 days. +- Store secrets in the Concelier secrets vault; Offline Kit deployments must mirror the secret payloads in their encrypted store. +- Record rotation cadence in Ops runbook and update Concelier configuration (`CONCELIER__SOURCES__VNDR__MSRC__CLIENTSECRET`) ahead of expiry. + +## 3. Concelier configuration sample + +```yaml +concelier: + sources: + vndr.msrc: + tenantId: "" + clientId: "" + clientSecret: "" + apiVersion: "2024-08-01" + locale: "en-US" + requestDelay: "00:00:00.250" + failureBackoff: "00:05:00" + cursorOverlapMinutes: 10 + downloadCvrf: false # set true to persist CVRF ZIP alongside JSON detail +``` + +## 4. CVRF artefacts + +- The MSRC REST payload exposes `cvrfUrl` per advisory. Current connector persists the link as advisory metadata and reference; it does **not** download the ZIP by default. +- Ops should mirror CVRF ZIPs when preparing Offline Kits so air-gapped deployments can reconcile advisories without direct internet access. +- Once Offline Kit storage guidelines are finalised, extend the connector configuration with `downloadCvrf: true` to enable automatic attachment retrieval. + +### 4.1 State seeding helper + +Use `tools/SourceStateSeeder` to queue historical advisories (detail JSON + optional CVRF artefacts) for replay without manual Mongo edits. Example seed file: + +```json +{ + "source": "vndr.msrc", + "cursor": { + "lastModifiedCursor": "2024-01-01T00:00:00Z" + }, + "documents": [ + { + "uri": "https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV2024-0001", + "contentFile": "./seeds/adv2024-0001.json", + "contentType": "application/json", + "metadata": { "msrc.vulnerabilityId": "ADV2024-0001" }, + "addToPendingDocuments": true + }, + { + "uri": "https://download.microsoft.com/msrc/2024/ADV2024-0001.cvrf.zip", + "contentFile": "./seeds/adv2024-0001.cvrf.zip", + "contentType": "application/zip", + "status": "mapped", + "addToPendingDocuments": false + } + ] +} +``` + +Run the helper: + +```bash +dotnet run --project tools/SourceStateSeeder -- \ + --connection-string "mongodb://localhost:27017" \ + --database concelier \ + --input seeds/msrc-backfill.json +``` + +Any documents marked `addToPendingDocuments` will appear in the connector cursor; `DownloadCvrf` can remain disabled if the ZIP artefact is pre-seeded. + +## 5. Outstanding items + +- Ops to confirm tenant/app names and provide client credentials through the secure channel. +- Connector team monitors token cache health (already implemented); validate instrumentation once Ops supplies credentials. +- Offline Kit packaging: add encrypted blob containing client credentials with rotation instructions. diff --git a/docs/ops/feedser-nkcki-operations.md b/docs/ops/concelier-nkcki-operations.md similarity index 82% rename from docs/ops/feedser-nkcki-operations.md rename to docs/ops/concelier-nkcki-operations.md index 4424c9ee..4be0ed72 100644 --- a/docs/ops/feedser-nkcki-operations.md +++ b/docs/ops/concelier-nkcki-operations.md @@ -1,48 +1,48 @@ -# NKCKI Connector Operations Guide - -## Overview - -The NKCKI connector ingests JSON bulletin archives from cert.gov.ru, expanding each `*.json.zip` attachment into per-vulnerability DTOs before canonical mapping. The fetch pipeline now supports cache-backed recovery, deterministic pagination, and telemetry suitable for production monitoring. - -## Configuration - -Key options exposed through `feedser:sources:ru-nkcki:http`: - -- `maxBulletinsPerFetch` – limits new bulletin downloads in a single run (default `5`). -- `maxListingPagesPerFetch` – maximum listing pages visited during pagination (default `3`). -- `listingCacheDuration` – minimum interval between listing fetches before falling back to cached artefacts (default `00:10:00`). -- `cacheDirectory` – optional path for persisted bulletin archives used during offline or failure scenarios. -- `requestDelay` – delay inserted between bulletin downloads to respect upstream politeness. - -When operating in offline-first mode, set `cacheDirectory` to a writable path (e.g. `/var/lib/feedser/cache/ru-nkcki`) and pre-populate bulletin archives via the offline kit. - -## Telemetry - -`RuNkckiDiagnostics` emits the following metrics under meter `StellaOps.Feedser.Source.Ru.Nkcki`: - -- `nkcki.listing.fetch.attempts` / `nkcki.listing.fetch.success` / `nkcki.listing.fetch.failures` -- `nkcki.listing.pages.visited` (histogram, `pages`) -- `nkcki.listing.attachments.discovered` / `nkcki.listing.attachments.new` -- `nkcki.bulletin.fetch.success` / `nkcki.bulletin.fetch.cached` / `nkcki.bulletin.fetch.failures` -- `nkcki.entries.processed` (histogram, `entries`) - -Integrate these counters into standard Feedser observability dashboards to track crawl coverage and cache hit rates. - -## Archive Backfill Strategy - -Bitrix pagination surfaces archives via `?PAGEN_1=n`. The connector now walks up to `maxListingPagesPerFetch` pages, deduplicating bulletin IDs and maintaining a rolling `knownBulletins` window. Backfill strategy: - -1. Enumerate pages from newest to oldest, respecting `maxListingPagesPerFetch` and `listingCacheDuration` to avoid refetch storms. -2. Persist every `*.json.zip` attachment to the configured cache directory. This enables replay when listing access is temporarily blocked. -3. During archive replay, `ProcessCachedBulletinsAsync` enqueues missing documents while respecting `maxVulnerabilitiesPerFetch`. -4. For historical HTML-only advisories, collect page URLs and metadata while offline (future work: HTML and PDF extraction pipeline documented in `docs/feedser-connector-research-20251011.md`). - -For large migrations, seed caches with archived zip bundles, then run fetch/parse/map cycles in chronological order to maintain deterministic outputs. - -## Failure Handling - -- Listing failures mark the source state with exponential backoff while attempting cache replay. -- Bulletin fetches fall back to cached copies before surfacing an error. -- Mongo integration tests rely on bundled OpenSSL 1.1 libraries (`tools/openssl/linux-x64`) to keep `Mongo2Go` operational on modern distros. - -Refer to `ru-nkcki` entries in `src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md` for outstanding items. +# NKCKI Connector Operations Guide + +## Overview + +The NKCKI connector ingests JSON bulletin archives from cert.gov.ru, expanding each `*.json.zip` attachment into per-vulnerability DTOs before canonical mapping. The fetch pipeline now supports cache-backed recovery, deterministic pagination, and telemetry suitable for production monitoring. + +## Configuration + +Key options exposed through `concelier:sources:ru-nkcki:http`: + +- `maxBulletinsPerFetch` – limits new bulletin downloads in a single run (default `5`). +- `maxListingPagesPerFetch` – maximum listing pages visited during pagination (default `3`). +- `listingCacheDuration` – minimum interval between listing fetches before falling back to cached artefacts (default `00:10:00`). +- `cacheDirectory` – optional path for persisted bulletin archives used during offline or failure scenarios. +- `requestDelay` – delay inserted between bulletin downloads to respect upstream politeness. + +When operating in offline-first mode, set `cacheDirectory` to a writable path (e.g. `/var/lib/concelier/cache/ru-nkcki`) and pre-populate bulletin archives via the offline kit. + +## Telemetry + +`RuNkckiDiagnostics` emits the following metrics under meter `StellaOps.Concelier.Connector.Ru.Nkcki`: + +- `nkcki.listing.fetch.attempts` / `nkcki.listing.fetch.success` / `nkcki.listing.fetch.failures` +- `nkcki.listing.pages.visited` (histogram, `pages`) +- `nkcki.listing.attachments.discovered` / `nkcki.listing.attachments.new` +- `nkcki.bulletin.fetch.success` / `nkcki.bulletin.fetch.cached` / `nkcki.bulletin.fetch.failures` +- `nkcki.entries.processed` (histogram, `entries`) + +Integrate these counters into standard Concelier observability dashboards to track crawl coverage and cache hit rates. + +## Archive Backfill Strategy + +Bitrix pagination surfaces archives via `?PAGEN_1=n`. The connector now walks up to `maxListingPagesPerFetch` pages, deduplicating bulletin IDs and maintaining a rolling `knownBulletins` window. Backfill strategy: + +1. Enumerate pages from newest to oldest, respecting `maxListingPagesPerFetch` and `listingCacheDuration` to avoid refetch storms. +2. Persist every `*.json.zip` attachment to the configured cache directory. This enables replay when listing access is temporarily blocked. +3. During archive replay, `ProcessCachedBulletinsAsync` enqueues missing documents while respecting `maxVulnerabilitiesPerFetch`. +4. For historical HTML-only advisories, collect page URLs and metadata while offline (future work: HTML and PDF extraction pipeline documented in `docs/concelier-connector-research-20251011.md`). + +For large migrations, seed caches with archived zip bundles, then run fetch/parse/map cycles in chronological order to maintain deterministic outputs. + +## Failure Handling + +- Listing failures mark the source state with exponential backoff while attempting cache replay. +- Bulletin fetches fall back to cached copies before surfacing an error. +- Mongo integration tests rely on bundled OpenSSL 1.1 libraries (`tools/openssl/linux-x64`) to keep `Mongo2Go` operational on modern distros. + +Refer to `ru-nkcki` entries in `src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md` for outstanding items. diff --git a/docs/ops/feedser-osv-operations.md b/docs/ops/concelier-osv-operations.md similarity index 93% rename from docs/ops/feedser-osv-operations.md rename to docs/ops/concelier-osv-operations.md index a00dcac5..bb035e0c 100644 --- a/docs/ops/feedser-osv-operations.md +++ b/docs/ops/concelier-osv-operations.md @@ -1,24 +1,24 @@ -# Feedser OSV Connector – Operations Notes - -_Last updated: 2025-10-16_ - -The OSV connector ingests advisories from OSV.dev across OSS ecosystems. This note highlights the additional merge/export expectations introduced with the canonical metric fallback work in Sprint 4. - -## 1. Canonical metric fallbacks -- When OSV omits CVSS vectors (common for CVSS v4-only payloads) the mapper now emits a deterministic canonical metric id in the form `osv:severity/` and normalises the advisory severity to the same ``. -- Metric: `osv.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `ecosystem`, `reason=no_cvss`. Watch this alongside merge parity dashboards to catch spikes where OSV publishes severity-only advisories. -- Merge precedence still prefers GHSA over OSV; the shared severity-based canonical id keeps Merge/export parity deterministic even when only OSV supplies severity data. - -## 2. CWE provenance -- `database_specific.cwe_ids` now populates provenance decision reasons for every mapped weakness. Expect `decisionReason="database_specific.cwe_ids"` on OSV weakness provenance and confirm exporters preserve the value. -- If OSV ever attaches `database_specific.cwe_notes`, the connector will surface the joined note string in `decisionReason` instead of the default marker. - -## 3. Dashboards & alerts -- Extend existing merge dashboards with the new counter: - - Overlay `sum(osv.map.canonical_metric_fallbacks{ecosystem=~".+"})` with Merge severity overrides to confirm fallback advisories are reconciling cleanly. - - Alert when the 1-hour sum exceeds 50 for any ecosystem; baseline volume is currently <5 per day (mostly GHSA mirrors emitting CVSS v4 only). -- Exporters already surface `canonicalMetricId`; no schema change is required, but ORAS/Trivy bundles should be spot-checked after deploying the connector update. - -## 4. Runbook updates -- Fixture parity suites (`osv-ghsa.*`) now assert the fallback id and provenance notes. Regenerate via `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`. -- When investigating merge severity conflicts, include the fallback counter and confirm OSV advisories carry the expected `osv:severity/` id before raising connector bugs. +# Concelier OSV Connector – Operations Notes + +_Last updated: 2025-10-16_ + +The OSV connector ingests advisories from OSV.dev across OSS ecosystems. This note highlights the additional merge/export expectations introduced with the canonical metric fallback work in Sprint 4. + +## 1. Canonical metric fallbacks +- When OSV omits CVSS vectors (common for CVSS v4-only payloads) the mapper now emits a deterministic canonical metric id in the form `osv:severity/` and normalises the advisory severity to the same ``. +- Metric: `osv.map.canonical_metric_fallbacks` (counter) with tags `severity`, `canonical_metric_id`, `ecosystem`, `reason=no_cvss`. Watch this alongside merge parity dashboards to catch spikes where OSV publishes severity-only advisories. +- Merge precedence still prefers GHSA over OSV; the shared severity-based canonical id keeps Merge/export parity deterministic even when only OSV supplies severity data. + +## 2. CWE provenance +- `database_specific.cwe_ids` now populates provenance decision reasons for every mapped weakness. Expect `decisionReason="database_specific.cwe_ids"` on OSV weakness provenance and confirm exporters preserve the value. +- If OSV ever attaches `database_specific.cwe_notes`, the connector will surface the joined note string in `decisionReason` instead of the default marker. + +## 3. Dashboards & alerts +- Extend existing merge dashboards with the new counter: + - Overlay `sum(osv.map.canonical_metric_fallbacks{ecosystem=~".+"})` with Merge severity overrides to confirm fallback advisories are reconciling cleanly. + - Alert when the 1-hour sum exceeds 50 for any ecosystem; baseline volume is currently <5 per day (mostly GHSA mirrors emitting CVSS v4 only). +- Exporters already surface `canonicalMetricId`; no schema change is required, but ORAS/Trivy bundles should be spot-checked after deploying the connector update. + +## 4. Runbook updates +- Fixture parity suites (`osv-ghsa.*`) now assert the fallback id and provenance notes. Regenerate via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`. +- When investigating merge severity conflicts, include the fallback counter and confirm OSV advisories carry the expected `osv:severity/` id before raising connector bugs. diff --git a/docs/ops/migrations/SEMVER_STYLE.md b/docs/ops/migrations/SEMVER_STYLE.md index 151f078b..d79b2aec 100644 --- a/docs/ops/migrations/SEMVER_STYLE.md +++ b/docs/ops/migrations/SEMVER_STYLE.md @@ -1,50 +1,50 @@ -# SemVer Style Backfill Runbook - -_Last updated: 2025-10-11_ - -## Overview - -The SemVer style migration populates the new `normalizedVersions` field on advisory documents and ensures -provenance `decisionReason` values are preserved during future reads. The migration is idempotent and only -runs when the feature flag `feedser:storage:enableSemVerStyle` is enabled. - -## Preconditions - -1. **Review configuration** – set `feedser.storage.enableSemVerStyle` to `true` on all Feedser services. -2. **Confirm batch size** – adjust `feedser.storage.backfillBatchSize` if you need smaller batches for older - deployments (default: `250`). -3. **Back up** – capture a fresh snapshot of the `advisory` collection or a full MongoDB backup. -4. **Staging dry-run** – enable the flag in a staging environment and observe the migration output before - rolling to production. - -## Execution - -No manual command is required. After deploying the configuration change, restart the Feedser WebService or -any component that hosts the Mongo migration runner. During startup you will see log entries similar to: - -``` -Applying Mongo migration 20251011-semver-style-backfill: Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled. -Mongo migration 20251011-semver-style-backfill applied -``` - -The migration reads advisories in batches (`feedser.storage.backfillBatchSize`) and writes flattened -`normalizedVersions` arrays. Existing documents without SemVer ranges remain untouched. - -## Post-checks - -1. Verify the new indexes exist: - ``` - db.advisory.getIndexes() - ``` - You should see `advisory_normalizedVersions_pkg_scheme_type` and `advisory_normalizedVersions_value`. -2. Spot check a few advisories to confirm the top-level `normalizedVersions` array exists and matches - the embedded package data. -3. Run `dotnet test` for `StellaOps.Feedser.Storage.Mongo.Tests` (optional but recommended) in CI to confirm - the storage suite passes with the feature flag enabled. - -## Rollback - -Set `feedser.storage.enableSemVerStyle` back to `false` and redeploy. The migration will be skipped on -subsequent startups. You can leave the populated `normalizedVersions` arrays in place; they are ignored when -the feature flag is off. If you must remove them entirely, restore from the backup captured during -preparation. +# SemVer Style Backfill Runbook + +_Last updated: 2025-10-11_ + +## Overview + +The SemVer style migration populates the new `normalizedVersions` field on advisory documents and ensures +provenance `decisionReason` values are preserved during future reads. The migration is idempotent and only +runs when the feature flag `concelier:storage:enableSemVerStyle` is enabled. + +## Preconditions + +1. **Review configuration** – set `concelier.storage.enableSemVerStyle` to `true` on all Concelier services. +2. **Confirm batch size** – adjust `concelier.storage.backfillBatchSize` if you need smaller batches for older + deployments (default: `250`). +3. **Back up** – capture a fresh snapshot of the `advisory` collection or a full MongoDB backup. +4. **Staging dry-run** – enable the flag in a staging environment and observe the migration output before + rolling to production. + +## Execution + +No manual command is required. After deploying the configuration change, restart the Concelier WebService or +any component that hosts the Mongo migration runner. During startup you will see log entries similar to: + +``` +Applying Mongo migration 20251011-semver-style-backfill: Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled. +Mongo migration 20251011-semver-style-backfill applied +``` + +The migration reads advisories in batches (`concelier.storage.backfillBatchSize`) and writes flattened +`normalizedVersions` arrays. Existing documents without SemVer ranges remain untouched. + +## Post-checks + +1. Verify the new indexes exist: + ``` + db.advisory.getIndexes() + ``` + You should see `advisory_normalizedVersions_pkg_scheme_type` and `advisory_normalizedVersions_value`. +2. Spot check a few advisories to confirm the top-level `normalizedVersions` array exists and matches + the embedded package data. +3. Run `dotnet test` for `StellaOps.Concelier.Storage.Mongo.Tests` (optional but recommended) in CI to confirm + the storage suite passes with the feature flag enabled. + +## Rollback + +Set `concelier.storage.enableSemVerStyle` back to `false` and redeploy. The migration will be skipped on +subsequent startups. You can leave the populated `normalizedVersions` arrays in place; they are ignored when +the feature flag is off. If you must remove them entirely, restore from the backup captured during +preparation. diff --git a/docs/runtime/SCANNER_RUNTIME_READINESS.md b/docs/runtime/SCANNER_RUNTIME_READINESS.md new file mode 100644 index 00000000..cf8c9c86 --- /dev/null +++ b/docs/runtime/SCANNER_RUNTIME_READINESS.md @@ -0,0 +1,81 @@ +# Scanner Runtime Readiness Checklist + +Last updated: 2025-10-19 + +This runbook confirms that Scanner.WebService now surfaces the metadata Runtime Guild consumers requested: quieted finding counts in the signed report events and progress hints on the scan event stream. Follow the checklist before relying on these fields in production automation. + +--- + +## 1. Prerequisites + +- Scanner.WebService release includes **SCANNER-POLICY-09-107** (adds quieted provenance and score inputs to `/reports`). +- Docs repository at commit containing `docs/events/scanner.report.ready@1.json` with `quietedFindingCount`. +- Access to a Scanner environment (staging or sandbox) with an image capable of producing policy verdicts. + +--- + +## 2. Verify quieted finding hints + +1. **Trigger a report** – run a scan that produces at least one quieted finding (policy with `quiet: true`). After the scan completes, call: + ```http + POST /api/v1/reports + Authorization: Bearer + Content-Type: application/json + ``` + Ensure the JSON response contains `report.summary.quieted` and that the DSSE payload mirrors the same count. +2. **Check emitted event** – pull the latest `scanner.report.ready` event (from the queue or sample capture). Confirm the payload includes: + - `quietedFindingCount` equal to the `summary.quieted` value. + - Updated `summary` block with the quieted counter. +3. **Schema validation** – optionally validate the payload against `docs/events/scanner.report.ready@1.json` to guarantee downstream compatibility: + ```bash + npx ajv validate -c ajv-formats \ + -s docs/events/scanner.report.ready@1.json \ + -d + ``` + (Use `npm install --no-save ajv ajv-cli ajv-formats` once per clone.) + +> Snapshot fixtures: see `docs/events/samples/scanner.report.ready@1.sample.json` for a canonical event that already carries `quietedFindingCount`. + +--- + +## 3. Verify progress hints (SSE / JSONL) + +Scanner streams structured progress messages for each scan. The `data` map inside every frame carries the hints Runtime systems consume (force flag, client metadata, additional stage-specific attributes). + +1. **Submit a scan** with custom metadata (for example `pipeline=github`, `build=1234`). +2. **Stream events**: + ```http + GET /api/v1/scans/{scanId}/events?format=jsonl + Authorization: Bearer + Accept: application/x-ndjson + ``` +3. **Confirm payload** – each frame should resemble: + ```json + { + "scanId": "2f6c17f9b3f548e2a28b9c412f4d63f8", + "sequence": 1, + "state": "Pending", + "message": "queued", + "timestamp": "2025-10-19T03:12:45.118Z", + "correlationId": "2f6c17f9b3f548e2a28b9c412f4d63f8:0001", + "data": { + "force": false, + "meta.pipeline": "github" + } + } + ``` + Subsequent frames include additional hints as analyzers progress (e.g., `stage`, `meta.*`, or analyzer-provided keys). Ensure newline-delimited JSON consumers preserve the `data` dictionary when forwarding to runtime dashboards. + +> The same frame structure is documented in `docs/09_API_CLI_REFERENCE.md` §2.6. Copy that snippet into integration tests to keep compatibility. + +--- + +## 4. Sign-off matrix + +| Stakeholder | Checklist | Status | Notes | +|-------------|-----------|--------|-------| +| Runtime Guild | Sections 2 & 3 completed | ☐ | Capture sample payloads for webhook regression tests. | +| Notify Guild | `quietedFindingCount` consumed in notifications | ☐ | Update templates after Runtime sign-off. | +| Docs Guild | Checklist published & linked from updates | ☑ | 2025-10-19 | + +Mark the stakeholder boxes as each team completes its validation. Once all checks are green, update `docs/TASKS.md` to reflect task completion. diff --git a/docs/scanner-core-contracts.md b/docs/scanner-core-contracts.md new file mode 100644 index 00000000..5745b306 --- /dev/null +++ b/docs/scanner-core-contracts.md @@ -0,0 +1,147 @@ +# Scanner Core Contracts + +The **Scanner Core** library provides shared contracts, observability helpers, and security utilities consumed by `Scanner.WebService`, `Scanner.Worker`, analyzers, and tooling. These primitives guarantee deterministic identifiers, timestamps, and log context for all scanning flows. + +## Canonical DTOs + +- `ScanJob` & `ScanJobStatus` – canonical job metadata (image reference/digest, tenant, correlation ID, timestamps, failure details). Constructors normalise timestamps to UTC microsecond precision and canonicalise image digests. Round-trips with `JsonSerializerDefaults.Web` using `ScannerJsonOptions`. +- `ScanProgressEvent` & `ScanStage`/`ScanProgressEventKind` – stage-level progress surface for queue/stream consumers. Includes deterministic sequence numbers, optional progress percentage, attributes, and attached `ScannerError`. +- `ScannerError` & `ScannerErrorCode` – shared error taxonomy spanning queue, analyzers, storage, exporters, and signing. Carries severity, retryability, structured details, and microsecond-precision timestamps. +- `ScanJobId` – strongly-typed identifier rendered as `Guid` (lowercase `N` format) with deterministic parsing. + +### Canonical JSON samples + +The golden fixtures consumed by `ScannerCoreContractsTests` document the wire shape shared with downstream services. They live under `src/StellaOps.Scanner.Core.Tests/Fixtures/` and a representative extract is shown below. + +```json +{ + "id": "8f4cc9c582454b9d9b4f5ae049631b7d", + "status": "running", + "imageReference": "registry.example.com/stellaops/scanner:1.2.3", + "imageDigest": "sha256:abcdef", + "createdAt": "2025-10-18T14:30:15.123456+00:00", + "updatedAt": "2025-10-18T14:30:20.123456+00:00", + "correlationId": "scan-analyzeoperatingsystem-8f4cc9c582454b9d9b4f5ae049631b7d", + "tenantId": "tenant-a", + "metadata": { + "requestId": "req-1234", + "source": "ci" + }, + "failure": { + "code": "analyzerFailure", + "severity": "error", + "message": "Analyzer failed to parse layer", + "timestamp": "2025-10-18T14:30:15.123456+00:00", + "retryable": false, + "stage": "AnalyzeOperatingSystem", + "component": "os-analyzer", + "details": { + "layerDigest": "sha256:deadbeef", + "attempt": "1" + } + } +} +``` + +Progress events follow the same conventions (`jobId`, `stage`, `kind`, `timestamp`, `attributes`, optional embedded `ScannerError`). The fixtures are verified via deterministic JSON comparison in every CI run. + +## Deterministic helpers + +- `ScannerIdentifiers` – derives `ScanJobId`, correlation IDs, and SHA-256 hashes from normalised inputs (image reference/digest, tenant, salt). Ensures case-insensitive stability and reproducible metric keys. +- `ScannerTimestamps` – trims to microsecond precision, provides ISO-8601 (`yyyy-MM-ddTHH:mm:ss.ffffffZ`) rendering, and parsing helpers. +- `ScannerJsonOptions` – standard JSON options (web defaults, camel-case enums) shared by services/tests. +- `ScanAnalysisStore` & `ScanAnalysisKeys` – shared in-memory analysis cache flowing through Worker stages. OS analyzers populate + `analysis.os.packages` (raw output), `analysis.os.fragments` (per-analyzer component fragments), and merge into + `analysis.layers.fragments` so emit/diff stages can compose SBOMs and diffs without knowledge of individual analyzer + implementations. + +## Observability primitives + +- `ScannerDiagnostics` – global `ActivitySource`/`Meter` for scanner components. `StartActivity` seeds deterministic tags (`job_id`, `stage`, `component`, `correlation_id`). +- `ScannerMetricNames` – centralises metric prefixes (`stellaops.scanner.*`) and deterministic job/event tag builders. +- `ScannerCorrelationContext` & `ScannerCorrelationContextAccessor` – ambient correlation propagation via `AsyncLocal` for log scopes, metrics, and diagnostics. +- `ScannerLogExtensions` – `ILogger` scopes for jobs/progress events with automatic correlation context push, minimal allocations, and consistent structured fields. + +### Observability overhead validation + +A micro-benchmark executed on 2025-10-19 (4 vCPU runner, .NET 10.0.100-rc.1) measured the average scope cost across 1 000 000 iterations: + +| Scope | Mean (µs/call) | +|-------|----------------| +| `BeginScanScope` (logger attached) | 0.80 | +| `BeginScanScope` (noop logger) | 0.31 | +| `BeginProgressScope` | 0.57 | + +To reproduce, run `dotnet test src/StellaOps.Scanner.Core.Tests -c Release` (see `ScannerLogExtensionsPerformanceTests`) or copy the snippet below into a throwaway `dotnet run` console project and execute it with `dotnet run -c Release`: + +```csharp +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Observability; +using StellaOps.Scanner.Core.Utility; + +var factory = LoggerFactory.Create(builder => builder.AddFilter(static _ => true)); +var logger = factory.CreateLogger("bench"); + +var jobId = ScannerIdentifiers.CreateJobId("registry.example.com/stellaops/scanner:1.2.3", "sha256:abcdef", "tenant-a", "benchmark"); +var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, nameof(ScanStage.AnalyzeOperatingSystem)); +var now = ScannerTimestamps.Normalize(new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero)); + +var job = new ScanJob(jobId, ScanJobStatus.Running, "registry.example.com/stellaops/scanner:1.2.3", "sha256:abcdef", now, now, correlationId, "tenant-a", new Dictionary(StringComparer.Ordinal) { ["requestId"] = "req-bench" }); +var progress = new ScanProgressEvent(jobId, ScanStage.AnalyzeOperatingSystem, ScanProgressEventKind.Progress, 42, now, 10.5, "benchmark", new Dictionary(StringComparer.Ordinal) { ["sample"] = "true" }); + +Console.WriteLine("Scanner Core Observability micro-bench (1,000,000 iterations)"); +Report("BeginScanScope (logger)", Measure(static ctx => ctx.Logger.BeginScanScope(ctx.Job, ctx.Stage, ctx.Component), new ScopeContext(logger, job, nameof(ScanStage.AnalyzeOperatingSystem), "os-analyzer"))); +Report("BeginScanScope (no logger)", Measure(static ctx => ScannerLogExtensions.BeginScanScope(null, ctx.Job, ctx.Stage, ctx.Component), new ScopeContext(logger, job, nameof(ScanStage.AnalyzeOperatingSystem), "os-analyzer"))); +Report("BeginProgressScope", Measure(static ctx => ctx.Logger.BeginProgressScope(ctx.Progress!, ctx.Component), new ScopeContext(logger, job, nameof(ScanStage.AnalyzeOperatingSystem), "os-analyzer", progress))); + +static double Measure(Func factory, ScopeContext context) +{ + const int iterations = 1_000_000; + for (var i = 0; i < 10_000; i++) + { + using var scope = factory(context); + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var sw = Stopwatch.StartNew(); + for (var i = 0; i < iterations; i++) + { + using var scope = factory(context); + } + + sw.Stop(); + return sw.Elapsed.TotalSeconds * 1_000_000 / iterations; +} + +static void Report(string label, double microseconds) + => Console.WriteLine($"{label,-28}: {microseconds:F3} µs"); + +readonly record struct ScopeContext(ILogger Logger, ScanJob Job, string? Stage, string? Component, ScanProgressEvent? Progress = null); +``` + +Both guardrails enforce the ≤ 5 µs acceptance target for SP9-G1. + +## Security utilities + +- `AuthorityTokenSource` – caches short-lived OpToks per audience+scope using deterministic keys and refresh skew (default 30 s). Integrates with `StellaOps.Auth.Client`. +- `DpopProofValidator` – validates DPoP proofs (alg allowlist, `htm`/`htu`, nonce, replay window, signature) backed by pluggable `IDpopReplayCache`. Ships with `InMemoryDpopReplayCache` for restart-only deployments. +- `RestartOnlyPluginGuard` – enforces restart-time plug-in registration (deterministic path normalisation; throws if new plug-ins added post-seal). +- `ServiceCollectionExtensions.AddScannerAuthorityCore` – DI helper wiring Authority client, OpTok source, DPoP validation, replay cache, and plug-in guard. + +## Testing guarantees + +Unit tests (`StellaOps.Scanner.Core.Tests`) assert: + +- DTO JSON round-trips are stable and deterministic (`ScannerCoreContractsTests` + golden fixtures). +- Identifier/hash helpers ignore case and emit lowercase hex. +- Timestamp normalisation retains UTC semantics. +- Log scopes push/pop correlation context predictably while staying under the 5 µs envelope. +- Authority token caching honours refresh skew and invalidation. +- DPoP validator accepts valid proofs, rejects nonce mismatch/replay, and enforces signature validation. +- Restart-only plug-in guard blocks runtime additions post-seal. diff --git a/docs/security/authority-threat-model.md b/docs/security/authority-threat-model.md index f9ad5c99..350e0f23 100644 --- a/docs/security/authority-threat-model.md +++ b/docs/security/authority-threat-model.md @@ -1,106 +1,106 @@ -# Authority Threat Model (STRIDE) - -> Prepared by Security Guild — 2025-10-12. Scope covers Authority host, Standard plug-in, CLI, bootstrap workflow, and offline revocation distribution. - -## 1. Scope & Method - -- Methodology: STRIDE applied to primary Authority surfaces (token issuance, bootstrap, revocation, operator tooling, plug-in extensibility). -- Assets in scope: identity credentials, OAuth tokens (access/refresh), bootstrap invites, revocation manifests, signing keys, audit telemetry. -- Out of scope: Third-party IdPs federated via OpenIddict (tracked separately in SEC6 backlog). - -## 2. Assets & Entry Points - -| Asset / Surface | Description | Primary Actors | -|-----------------|-------------|----------------| -| Token issuance APIs (`/token`, `/authorize`) | OAuth/OIDC endpoints mediated by OpenIddict | CLI, UI, automation agents | -| Bootstrap channel | Initial admin invite + bootstrap CLI workflow | Platform operators | -| Revocation bundle | Offline JSON + detached JWS consumed by agents | Feedser, Agents, Zastava | -| Plug-in manifests | Standard plug-in configuration and password policy overrides | Operators, DevOps | -| Signing keys | ES256 signing keys backing tokens and revocation manifests | Security Guild, HSM/KeyOps | -| Audit telemetry | Structured login/audit stream persisted to Mongo/observability stack | SOC, SecOps | - -## 3. Trust Boundaries - -| Boundary | Rationale | Controls | -|----------|-----------|----------| -| TB1 — Public network ↔️ Authority ingress | Internet/extranet exposure for `/token`, `/authorize`, `/bootstrap` | TLS 1.3, reverse proxy ACLs, rate limiting (SEC3.A / CORE8.RL) | -| TB2 — Authority host ↔️ Mongo storage | Credential store, revocation state, audit log persistence | Authenticated Mongo, network segmentation, deterministic serializers | -| TB3 — Authority host ↔️ Plug-in sandbox | Plug-ins may override password policy and bootstrap flows | Code signing, manifest validation, restart-time loading only | -| TB4 — Operator workstation ↔️ CLI | CLI holds bootstrap secrets and revocation bundles | OS keychain storage, MFA on workstations, offline kit checksum | -| TB5 — Authority ↔️ Downstream agents | Revocation bundle consumption, token validation | Mutual TLS (planned), detached JWS signatures, bundle freshness checks | - -## 4. Data Flow Diagrams - -### 4.1 Runtime token issuance - -```mermaid -flowchart LR - subgraph Client Tier - CLI[StellaOps CLI] - UI[UI / Automation] - end - subgraph Perimeter - RP[Reverse Proxy / WAF] - end - subgraph Authority - AUTH[Authority Host] - PLGIN[Standard Plug-in] - STORE[(Mongo Credential Store)] - end - CLI -->|OAuth password / client creds| RP --> AUTH - UI -->|OAuth flows| RP - AUTH -->|PasswordHashOptions + Secrets| PLGIN - AUTH -->|Verify / Persist hashes| STORE - STORE -->|Rehash needed| AUTH - AUTH -->|Access / refresh token| RP --> Client Tier -``` - -### 4.2 Bootstrap & revocation - -```mermaid -flowchart LR - subgraph Operator - OPS[Operator Workstation] - end - subgraph Authority - AUTH[Authority Host] - STORE[(Mongo)] - end - subgraph Distribution - OFFKIT[Offline Kit Bundle] - AGENT[Authorized Agent / Feedser] - end - OPS -->|Bootstrap CLI (`stellaops auth bootstrap`)| AUTH - AUTH -->|One-time invite + Argon2 hash| STORE - AUTH -->|Revocation export (`stellaops auth revoke export`)| OFFKIT - OFFKIT -->|Signed JSON + .jws| AGENT - AGENT -->|Revocation ACK / telemetry| AUTH -``` - -## 5. STRIDE Analysis - -| Threat | STRIDE Vector | Surface | Risk (L×I) | Existing Controls | Gaps / Actions | Owner | -|--------|---------------|---------|------------|-------------------|----------------|-------| -| Spoofed revocation bundle | Spoofing | TB5 — Authority ↔️ Agents | Med×High | Detached JWS signature (planned), offline kit checksums | Finalise signing key registry & verification script (SEC4.B/SEC4.HOST); add bundle freshness requirement | Security Guild (follow-up: **SEC5.B**) | -| Parameter tampering on `/token` | Tampering | TB1 — Public ingress | Med×High | ASP.NET model validation, OpenIddict, rate limiter (CORE8.RL) | Tampered requests emit `authority.token.tamper` audit events (`request.tampered`, unexpected parameter names) correlating with `/token` outcomes (SEC5.C) | Security Guild + Authority Core (follow-up: **SEC5.C**) | -| Bootstrap invite replay | Repudiation | TB4 — Operator CLI ↔️ Authority | Low×High | One-time bootstrap tokens, Argon2id hashing on creation | Invites expire automatically and emit audit events on consumption/expiration (SEC5.D) | Security Guild | -| Token replay by stolen agent | Information Disclosure | TB5 | Med×High | Signed revocation bundles, device fingerprint heuristics, optional mTLS | Monitor revocation acknowledgement latency via Zastava and tune replay alerting thresholds | Security Guild + Zastava (follow-up: **SEC5.E**) | -| Privilege escalation via plug-in override | Elevation of Privilege | TB3 — Plug-in sandbox | Med×High | Signed plug-ins, restart-only loading, configuration validation | Add static analysis on manifest overrides + runtime warning when policy weaker than host | Security Guild + DevOps (follow-up: **SEC5.F**) | -| Offline bundle tampering | Tampering | Distribution | Low×High | SHA256 manifest, signed bundles (planned) | Add supply-chain attestation for Offline Kit, publish verification CLI in docs | Security Guild + Ops (follow-up: **SEC5.G**) | -| Failure to log denied tokens | Repudiation | TB2 — Authority ↔️ Mongo | Med×Med | Serilog structured events (partial), Mongo persistence path (planned) | Finalise audit schema (SEC2.A) and ensure `/token` denies include subject/client/IP fields | Security Guild + Authority Core (follow-up: **SEC5.H**) | - -Risk scoring uses qualitative scale (Low/Med/High) for likelihood × impact; mitigation priority follows High > Med > Low. - -## 6. Follow-up Backlog Hooks - -| Backlog ID | Linked Threat | Summary | Target Owners | -|------------|---------------|---------|---------------| -| SEC5.B | Spoofed revocation bundle | Complete libsodium/Core signing integration and ship revocation verification script. | Security Guild + Authority Core | -| SEC5.C | Parameter tampering on `/token` | Finalise audit contract (`SEC2.A`) and add request tamper logging. | Security Guild + Authority Core | -| SEC5.D | Bootstrap invite replay | Implement expiry enforcement + audit coverage for unused bootstrap invites. | Security Guild | -| SEC5.E | Token replay by stolen agent | Coordinate Zastava alerting with the new device fingerprint heuristics and surface stale revocation acknowledgements. | Security Guild + Zastava | -| SEC5.F | Plug-in override escalation | Static analysis of plug-in manifests; warn on weaker password policy overrides. | Security Guild + DevOps | -| SEC5.G | Offline bundle tampering | Extend Offline Kit build to include attested manifest + verification CLI sample. | Security Guild + Ops | -| SEC5.H | Failure to log denied tokens | Ensure audit persistence for all `/token` denials with correlation IDs. | Security Guild + Authority Core | - -Update `src/StellaOps.Cryptography/TASKS.md` (Security Guild board) with the above backlog entries to satisfy SEC5.A exit criteria. +# Authority Threat Model (STRIDE) + +> Prepared by Security Guild — 2025-10-12. Scope covers Authority host, Standard plug-in, CLI, bootstrap workflow, and offline revocation distribution. + +## 1. Scope & Method + +- Methodology: STRIDE applied to primary Authority surfaces (token issuance, bootstrap, revocation, operator tooling, plug-in extensibility). +- Assets in scope: identity credentials, OAuth tokens (access/refresh), bootstrap invites, revocation manifests, signing keys, audit telemetry. +- Out of scope: Third-party IdPs federated via OpenIddict (tracked separately in SEC6 backlog). + +## 2. Assets & Entry Points + +| Asset / Surface | Description | Primary Actors | +|-----------------|-------------|----------------| +| Token issuance APIs (`/token`, `/authorize`) | OAuth/OIDC endpoints mediated by OpenIddict | CLI, UI, automation agents | +| Bootstrap channel | Initial admin invite + bootstrap CLI workflow | Platform operators | +| Revocation bundle | Offline JSON + detached JWS consumed by agents | Concelier, Agents, Zastava | +| Plug-in manifests | Standard plug-in configuration and password policy overrides | Operators, DevOps | +| Signing keys | ES256 signing keys backing tokens and revocation manifests | Security Guild, HSM/KeyOps | +| Audit telemetry | Structured login/audit stream persisted to Mongo/observability stack | SOC, SecOps | + +## 3. Trust Boundaries + +| Boundary | Rationale | Controls | +|----------|-----------|----------| +| TB1 — Public network ↔️ Authority ingress | Internet/extranet exposure for `/token`, `/authorize`, `/bootstrap` | TLS 1.3, reverse proxy ACLs, rate limiting (SEC3.A / CORE8.RL) | +| TB2 — Authority host ↔️ Mongo storage | Credential store, revocation state, audit log persistence | Authenticated Mongo, network segmentation, deterministic serializers | +| TB3 — Authority host ↔️ Plug-in sandbox | Plug-ins may override password policy and bootstrap flows | Code signing, manifest validation, restart-time loading only | +| TB4 — Operator workstation ↔️ CLI | CLI holds bootstrap secrets and revocation bundles | OS keychain storage, MFA on workstations, offline kit checksum | +| TB5 — Authority ↔️ Downstream agents | Revocation bundle consumption, token validation | Mutual TLS (planned), detached JWS signatures, bundle freshness checks | + +## 4. Data Flow Diagrams + +### 4.1 Runtime token issuance + +```mermaid +flowchart LR + subgraph Client Tier + CLI[StellaOps CLI] + UI[UI / Automation] + end + subgraph Perimeter + RP[Reverse Proxy / WAF] + end + subgraph Authority + AUTH[Authority Host] + PLGIN[Standard Plug-in] + STORE[(Mongo Credential Store)] + end + CLI -->|OAuth password / client creds| RP --> AUTH + UI -->|OAuth flows| RP + AUTH -->|PasswordHashOptions + Secrets| PLGIN + AUTH -->|Verify / Persist hashes| STORE + STORE -->|Rehash needed| AUTH + AUTH -->|Access / refresh token| RP --> Client Tier +``` + +### 4.2 Bootstrap & revocation + +```mermaid +flowchart LR + subgraph Operator + OPS[Operator Workstation] + end + subgraph Authority + AUTH[Authority Host] + STORE[(Mongo)] + end + subgraph Distribution + OFFKIT[Offline Kit Bundle] + AGENT[Authorized Agent / Concelier] + end + OPS -->|Bootstrap CLI (`stellaops auth bootstrap`)| AUTH + AUTH -->|One-time invite + Argon2 hash| STORE + AUTH -->|Revocation export (`stellaops auth revoke export`)| OFFKIT + OFFKIT -->|Signed JSON + .jws| AGENT + AGENT -->|Revocation ACK / telemetry| AUTH +``` + +## 5. STRIDE Analysis + +| Threat | STRIDE Vector | Surface | Risk (L×I) | Existing Controls | Gaps / Actions | Owner | +|--------|---------------|---------|------------|-------------------|----------------|-------| +| Spoofed revocation bundle | Spoofing | TB5 — Authority ↔️ Agents | Med×High | Detached JWS signature (planned), offline kit checksums | Finalise signing key registry & verification script (SEC4.B/SEC4.HOST); add bundle freshness requirement | Security Guild (follow-up: **SEC5.B**) | +| Parameter tampering on `/token` | Tampering | TB1 — Public ingress | Med×High | ASP.NET model validation, OpenIddict, rate limiter (CORE8.RL) | Tampered requests emit `authority.token.tamper` audit events (`request.tampered`, unexpected parameter names) correlating with `/token` outcomes (SEC5.C) | Security Guild + Authority Core (follow-up: **SEC5.C**) | +| Bootstrap invite replay | Repudiation | TB4 — Operator CLI ↔️ Authority | Low×High | One-time bootstrap tokens, Argon2id hashing on creation | Invites expire automatically and emit audit events on consumption/expiration (SEC5.D) | Security Guild | +| Token replay by stolen agent | Information Disclosure | TB5 | Med×High | Signed revocation bundles, device fingerprint heuristics, optional mTLS | Monitor revocation acknowledgement latency via Zastava and tune replay alerting thresholds | Security Guild + Zastava (follow-up: **SEC5.E**) | +| Privilege escalation via plug-in override | Elevation of Privilege | TB3 — Plug-in sandbox | Med×High | Signed plug-ins, restart-only loading, configuration validation | Add static analysis on manifest overrides + runtime warning when policy weaker than host | Security Guild + DevOps (follow-up: **SEC5.F**) | +| Offline bundle tampering | Tampering | Distribution | Low×High | SHA256 manifest, signed bundles (planned) | Add supply-chain attestation for Offline Kit, publish verification CLI in docs | Security Guild + Ops (follow-up: **SEC5.G**) | +| Failure to log denied tokens | Repudiation | TB2 — Authority ↔️ Mongo | Med×Med | Serilog structured events (partial), Mongo persistence path (planned) | Finalise audit schema (SEC2.A) and ensure `/token` denies include subject/client/IP fields | Security Guild + Authority Core (follow-up: **SEC5.H**) | + +Risk scoring uses qualitative scale (Low/Med/High) for likelihood × impact; mitigation priority follows High > Med > Low. + +## 6. Follow-up Backlog Hooks + +| Backlog ID | Linked Threat | Summary | Target Owners | +|------------|---------------|---------|---------------| +| SEC5.B | Spoofed revocation bundle | Complete libsodium/Core signing integration and ship revocation verification script. | Security Guild + Authority Core | +| SEC5.C | Parameter tampering on `/token` | Finalise audit contract (`SEC2.A`) and add request tamper logging. | Security Guild + Authority Core | +| SEC5.D | Bootstrap invite replay | Implement expiry enforcement + audit coverage for unused bootstrap invites. | Security Guild | +| SEC5.E | Token replay by stolen agent | Coordinate Zastava alerting with the new device fingerprint heuristics and surface stale revocation acknowledgements. | Security Guild + Zastava | +| SEC5.F | Plug-in override escalation | Static analysis of plug-in manifests; warn on weaker password policy overrides. | Security Guild + DevOps | +| SEC5.G | Offline bundle tampering | Extend Offline Kit build to include attested manifest + verification CLI sample. | Security Guild + Ops | +| SEC5.H | Failure to log denied tokens | Ensure audit persistence for all `/token` denials with correlation IDs. | Security Guild + Authority Core | + +Update `src/StellaOps.Cryptography/TASKS.md` (Security Guild board) with the above backlog entries to satisfy SEC5.A exit criteria. diff --git a/docs/security/revocation-bundle-example.json b/docs/security/revocation-bundle-example.json index b82477ad..375f7327 100644 --- a/docs/security/revocation-bundle-example.json +++ b/docs/security/revocation-bundle-example.json @@ -1,56 +1,56 @@ -{ - "$schema": "../../etc/authority/revocation_bundle.schema.json", - "schemaVersion": "1.0.0", - "issuer": "https://auth.stella-ops.example", - "bundleId": "6f9d08bfa0c24a0a9f7f59e6c17d2f8e8bca2ef34215c3d3ba5a9a1f0fbe2d10", - "issuedAt": "2025-10-12T15:00:00Z", - "validFrom": "2025-10-12T15:00:00Z", - "sequence": 42, - "signingKeyId": "authority-signing-20251012", - "revocations": [ - { - "id": "7ad4f3d2c21b461d9b3420e1151be9c4", - "category": "token", - "tokenType": "access_token", - "clientId": "feedser-cli", - "subjectId": "user:ops-admin", - "reason": "compromised", - "reasonDescription": "Access token reported by SOC automation run R-2045.", - "revokedAt": "2025-10-12T14:32:05Z", - "scopes": [ - "feedser:export", - "feedser:jobs" - ], - "fingerprint": "AD35E719C12204D7E7C92ED3F6DEBF0A44642D41AAF94233F9A47E183F4C5F18", - "metadata": { - "reportId": "R-2045", - "source": "soc-automation" - } - }, - { - "id": "user:departed-vendor", - "category": "subject", - "subjectId": "user:departed-vendor", - "reason": "lifecycle", - "revokedAt": "2025-10-10T18:15:00Z", - "metadata": { - "ticket": "HR-8821" - } - }, - { - "id": "ci-runner-legacy", - "category": "client", - "clientId": "ci-runner-legacy", - "reason": "rotation", - "revokedAt": "2025-10-09T11:00:00Z", - "expiresAt": "2025-11-01T00:00:00Z", - "metadata": { - "replacement": "ci-runner-2025" - } - } - ], - "metadata": { - "generator": "stellaops-authority@1.4.0", - "jobId": "revocation-export-20251012T1500Z" - } -} +{ + "$schema": "../../etc/authority/revocation_bundle.schema.json", + "schemaVersion": "1.0.0", + "issuer": "https://auth.stella-ops.example", + "bundleId": "6f9d08bfa0c24a0a9f7f59e6c17d2f8e8bca2ef34215c3d3ba5a9a1f0fbe2d10", + "issuedAt": "2025-10-12T15:00:00Z", + "validFrom": "2025-10-12T15:00:00Z", + "sequence": 42, + "signingKeyId": "authority-signing-20251012", + "revocations": [ + { + "id": "7ad4f3d2c21b461d9b3420e1151be9c4", + "category": "token", + "tokenType": "access_token", + "clientId": "concelier-cli", + "subjectId": "user:ops-admin", + "reason": "compromised", + "reasonDescription": "Access token reported by SOC automation run R-2045.", + "revokedAt": "2025-10-12T14:32:05Z", + "scopes": [ + "concelier:export", + "concelier:jobs" + ], + "fingerprint": "AD35E719C12204D7E7C92ED3F6DEBF0A44642D41AAF94233F9A47E183F4C5F18", + "metadata": { + "reportId": "R-2045", + "source": "soc-automation" + } + }, + { + "id": "user:departed-vendor", + "category": "subject", + "subjectId": "user:departed-vendor", + "reason": "lifecycle", + "revokedAt": "2025-10-10T18:15:00Z", + "metadata": { + "ticket": "HR-8821" + } + }, + { + "id": "ci-runner-legacy", + "category": "client", + "clientId": "ci-runner-legacy", + "reason": "rotation", + "revokedAt": "2025-10-09T11:00:00Z", + "expiresAt": "2025-11-01T00:00:00Z", + "metadata": { + "replacement": "ci-runner-2025" + } + } + ], + "metadata": { + "generator": "stellaops-authority@1.4.0", + "jobId": "revocation-export-20251012T1500Z" + } +} diff --git a/docs/security/revocation-bundle.md b/docs/security/revocation-bundle.md index 657c10e0..b176f0d3 100644 --- a/docs/security/revocation-bundle.md +++ b/docs/security/revocation-bundle.md @@ -1,91 +1,91 @@ -# Authority Revocation Bundle - -The Authority service exports revocation information as an offline-friendly JSON document plus a detached JWS signature. Operators can mirror the bundle alongside Feedser exports to ensure air-gapped scanners receive the latest token, subject, and client revocations. - -## File layout - -| Artefact | Description | -| --- | --- | -| `revocation-bundle.json` | Canonical JSON document describing revoked entities. Validates against [`etc/authority/revocation_bundle.schema.json`](../../etc/authority/revocation_bundle.schema.json). | -| `revocation-bundle.json.jws` | Detached JWS signature covering the exact UTF-8 bytes of `revocation-bundle.json`. | -| `revocation-bundle.json.sha256` | Hex-encoded SHA-256 digest used by mirror automation (optional but recommended). | - -All hashes and signatures are generated after applying the deterministic formatting rules below. - -## Deterministic formatting rules - -- JSON is serialised with UTF-8 encoding, 2-space indentation, and lexicographically sorted object keys. -- Arrays are sorted by deterministic keys: - - Top-level `revocations` sorted by (`category`, `id`, `revokedAt`). - - Nested arrays (`scopes`) sorted ascending, unique enforced. -- Numeric values (`sequence`) are emitted without leading zeros. -- Timestamps use UTC ISO-8601 format with `Z` suffix. - -Consumers MUST treat the combination of `schemaVersion` and `sequence` as a monotonic feed. Bundles with older `sequence` values are ignored unless `bundleId` differs and `issuedAt` is newer (supporting replay detection). - -## Revocation entry categories - -| Category | Description | Required fields | -| --- | --- | --- | -| `token` | A single OAuth token (access, refresh, device, authorization code). | `tokenType`, `clientId`, `revokedAt`, optional `subjectId` | -| `subject` | All credentials issued to a subject (user/service account). | `subjectId`, `revokedAt` | -| `client` | Entire OAuth client registration is revoked. | `clientId`, `revokedAt` | -| `key` | Signing/encryption key material revoked. | `id`, `revokedAt` | - -`reason` is a machine-friendly code (`compromised`, `rotation`, `policy`, `lifecycle`, etc). `reasonDescription` may include a short operator note. - -## Detached JWS workflow - -1. Serialise `revocation-bundle.json` using the deterministic rules. -2. Compute SHA-256 digest; write to `revocation-bundle.json.sha256`. -3. Sign using ES256 (default) with the configured Authority signing key. The JWS header uses: - ```json - { - "alg": "ES256", - "kid": "{signingKeyId}", - "provider": "{providerName}", - "typ": "application/vnd.stellaops.revocation-bundle+jws", - "b64": false, - "crit": ["b64"] - } - ``` -4. Persist the detached signature payload to `revocation-bundle.json.jws` (per RFC 7797). - -Verification steps: - -1. Validate `revocation-bundle.json` against the schema. -2. Re-compute SHA-256 and compare with `.sha256` (if present). -3. Resolve the signing key from JWKS (`/.well-known/jwks.json`) or the offline key bundle, preferring the provider declared in the JWS header (`provider` falls back to `default`). -4. Verify the detached JWS using the resolved provider. The CLI mirrors Authority resolution, so builds compiled with `StellaOpsCryptoSodium=true` automatically use the libsodium provider when advertised; otherwise verification downgrades to the managed fallback. - -### CLI verification workflow - -Use the bundled CLI command before distributing a bundle: - -```bash -stellaops auth revoke verify \ - --bundle artifacts/revocation-bundle.json \ - --signature artifacts/revocation-bundle.json.jws \ - --key etc/authority/signing/authority-public.pem \ - --verbose -``` - -The verifier performs three checks: - -1. Prints the computed digest in `sha256:` format. Compare it with the exported `.sha256` artefact. -2. Confirms the detached JWS header advertises `b64: false`, captures the provider hint, and that the algorithm matches the Authority configuration (ES256 unless overridden). -3. Registers the supplied PEM key with the crypto provider registry and validates the signature (falling back to the managed provider when the hinted provider is unavailable). - -A zero exit code means the bundle is ready for mirroring/import. Non-zero codes signal missing arguments, malformed JWS payloads, or signature mismatches; regenerate or re-sign the bundle before distribution. - -## Example - -The repository contains an [example bundle](revocation-bundle-example.json) demonstrating a mixed export of token, subject, and client revocations. Use it as a reference for integration tests and tooling. - -## Operations Quick Reference - -- `stella auth revoke export` emits a canonical JSON bundle, `.sha256` digest, and detached JWS signature in one command. Use `--output` to write into your mirror staging directory. -- `stella auth revoke verify` validates a bundle using cached JWKS or an offline PEM key, honours the `provider` metadata embedded in the signature, and reports digest mismatches before distribution. -- `POST /internal/revocations/export` provides the same payload for orchestrators that already talk to the bootstrap API. -- `POST /internal/signing/rotate` rotates JWKS material without downtime; always export a fresh bundle afterward so downstream mirrors receive signatures from the new `kid`. -- Offline Kit automation should mirror `revocation-bundle.json*` alongside Feedser exports so agents ingest revocations during the same sync pass. +# Authority Revocation Bundle + +The Authority service exports revocation information as an offline-friendly JSON document plus a detached JWS signature. Operators can mirror the bundle alongside Concelier exports to ensure air-gapped scanners receive the latest token, subject, and client revocations. + +## File layout + +| Artefact | Description | +| --- | --- | +| `revocation-bundle.json` | Canonical JSON document describing revoked entities. Validates against [`etc/authority/revocation_bundle.schema.json`](../../etc/authority/revocation_bundle.schema.json). | +| `revocation-bundle.json.jws` | Detached JWS signature covering the exact UTF-8 bytes of `revocation-bundle.json`. | +| `revocation-bundle.json.sha256` | Hex-encoded SHA-256 digest used by mirror automation (optional but recommended). | + +All hashes and signatures are generated after applying the deterministic formatting rules below. + +## Deterministic formatting rules + +- JSON is serialised with UTF-8 encoding, 2-space indentation, and lexicographically sorted object keys. +- Arrays are sorted by deterministic keys: + - Top-level `revocations` sorted by (`category`, `id`, `revokedAt`). + - Nested arrays (`scopes`) sorted ascending, unique enforced. +- Numeric values (`sequence`) are emitted without leading zeros. +- Timestamps use UTC ISO-8601 format with `Z` suffix. + +Consumers MUST treat the combination of `schemaVersion` and `sequence` as a monotonic feed. Bundles with older `sequence` values are ignored unless `bundleId` differs and `issuedAt` is newer (supporting replay detection). + +## Revocation entry categories + +| Category | Description | Required fields | +| --- | --- | --- | +| `token` | A single OAuth token (access, refresh, device, authorization code). | `tokenType`, `clientId`, `revokedAt`, optional `subjectId` | +| `subject` | All credentials issued to a subject (user/service account). | `subjectId`, `revokedAt` | +| `client` | Entire OAuth client registration is revoked. | `clientId`, `revokedAt` | +| `key` | Signing/encryption key material revoked. | `id`, `revokedAt` | + +`reason` is a machine-friendly code (`compromised`, `rotation`, `policy`, `lifecycle`, etc). `reasonDescription` may include a short operator note. + +## Detached JWS workflow + +1. Serialise `revocation-bundle.json` using the deterministic rules. +2. Compute SHA-256 digest; write to `revocation-bundle.json.sha256`. +3. Sign using ES256 (default) with the configured Authority signing key. The JWS header uses: + ```json + { + "alg": "ES256", + "kid": "{signingKeyId}", + "provider": "{providerName}", + "typ": "application/vnd.stellaops.revocation-bundle+jws", + "b64": false, + "crit": ["b64"] + } + ``` +4. Persist the detached signature payload to `revocation-bundle.json.jws` (per RFC 7797). + +Verification steps: + +1. Validate `revocation-bundle.json` against the schema. +2. Re-compute SHA-256 and compare with `.sha256` (if present). +3. Resolve the signing key from JWKS (`/.well-known/jwks.json`) or the offline key bundle, preferring the provider declared in the JWS header (`provider` falls back to `default`). +4. Verify the detached JWS using the resolved provider. The CLI mirrors Authority resolution, so builds compiled with `StellaOpsCryptoSodium=true` automatically use the libsodium provider when advertised; otherwise verification downgrades to the managed fallback. + +### CLI verification workflow + +Use the bundled CLI command before distributing a bundle: + +```bash +stellaops auth revoke verify \ + --bundle artifacts/revocation-bundle.json \ + --signature artifacts/revocation-bundle.json.jws \ + --key etc/authority/signing/authority-public.pem \ + --verbose +``` + +The verifier performs three checks: + +1. Prints the computed digest in `sha256:` format. Compare it with the exported `.sha256` artefact. +2. Confirms the detached JWS header advertises `b64: false`, captures the provider hint, and that the algorithm matches the Authority configuration (ES256 unless overridden). +3. Registers the supplied PEM key with the crypto provider registry and validates the signature (falling back to the managed provider when the hinted provider is unavailable). + +A zero exit code means the bundle is ready for mirroring/import. Non-zero codes signal missing arguments, malformed JWS payloads, or signature mismatches; regenerate or re-sign the bundle before distribution. + +## Example + +The repository contains an [example bundle](revocation-bundle-example.json) demonstrating a mixed export of token, subject, and client revocations. Use it as a reference for integration tests and tooling. + +## Operations Quick Reference + +- `stella auth revoke export` emits a canonical JSON bundle, `.sha256` digest, and detached JWS signature in one command. Use `--output` to write into your mirror staging directory. +- `stella auth revoke verify` validates a bundle using cached JWKS or an offline PEM key, honours the `provider` metadata embedded in the signature, and reports digest mismatches before distribution. +- `POST /internal/revocations/export` provides the same payload for orchestrators that already talk to the bootstrap API. +- `POST /internal/signing/rotate` rotates JWKS material without downtime; always export a fresh bundle afterward so downstream mirrors receive signatures from the new `kid`. +- Offline Kit automation should mirror `revocation-bundle.json*` alongside Concelier exports so agents ingest revocations during the same sync pass. diff --git a/docs/updates/2025-10-18-docs-guild.md b/docs/updates/2025-10-18-docs-guild.md new file mode 100644 index 00000000..caecb2fd --- /dev/null +++ b/docs/updates/2025-10-18-docs-guild.md @@ -0,0 +1,14 @@ +# Docs Guild Update — 2025-10-18 + +**Subject:** ADR process + events schema validation shipped +**Audience:** Docs Guild, DevEx, Platform Events + +- Published the ADR contribution guide at `docs/adr/index.md` and enriched the template to capture authorship, deciders, and alternatives. All new cross-module decisions should follow this workflow. +- Linked the ADR hub from `docs/README.md` so operators and engineers can discover the process without digging through directories. +- Extended Docs CI (`.gitea/workflows/docs.yml`) to compile event schemas with Ajv (including `ajv-formats`) and documented the local loop in `docs/events/README.md`. +- Captured the mirror/offline workflow in `docs/ci/20_CI_RECIPES.md` so runners know how to install the Ajv toolchain and publish previews without internet access. +- Validated `scanner.report.ready@1`, `scheduler.rescan.delta@1`, and `attestor.logged@1` schemas locally to unblock Platform Events acknowledgements. + +Next steps: +- Platform Events to confirm Notify/Scheduler consumers have visibility into the schema docs. +- DevEx to add ADR announcement blurb to the next sprint recap if broader broadcast is needed. diff --git a/docs/updates/2025-10-19-docs-guild.md b/docs/updates/2025-10-19-docs-guild.md new file mode 100644 index 00000000..bd6c5d91 --- /dev/null +++ b/docs/updates/2025-10-19-docs-guild.md @@ -0,0 +1,12 @@ +# Docs Guild Update — 2025-10-19 + +**Subject:** Event envelope reference & canonical samples +**Audience:** Docs Guild, Platform Events, Runtime Guild + +- Extended `docs/events/README.md` with envelope field tables, offline validation commands, and guidance for optional payload fields. +- Added canonical sample payloads under `docs/events/samples/` for `scanner.report.ready@1`, `scheduler.rescan.delta@1`, and `attestor.logged@1`; validated them with `ajv-cli` to match the published schemas. +- Documented the validation loop so air-gapped operators can mirror the CI checks before rolling new event versions. + +Next steps: +- Platform Events to embed the canonical samples into their contract tests. +- Runtime Guild checklist for quieted finding counts & progress hints published in `docs/runtime/SCANNER_RUNTIME_READINESS.md`; gather stakeholder sign-off. diff --git a/docs/updates/2025-10-19-platform-events.md b/docs/updates/2025-10-19-platform-events.md new file mode 100644 index 00000000..840398d0 --- /dev/null +++ b/docs/updates/2025-10-19-platform-events.md @@ -0,0 +1,10 @@ +# Platform Events Update — 2025-10-19 + +**Subject:** Canonical event samples enforced across tests & CI +**Audience:** Platform Events Guild, Notify Guild, Scheduler Guild, Docs Guild + +- Scanner WebService contract tests deserialize `scanner.report.ready@1` and `scanner.scan.completed@1` samples, validating DSSE payloads and canonical ordering via `NotifyCanonicalJsonSerializer`. +- Notify and Scheduler model suites now round-trip the published event samples (including `attestor.logged@1` and `scheduler.rescan.delta@1`) to catch drift in consumer expectations. +- Docs CI (`.gitea/workflows/docs.yml`) validates every sample against its schema with `ajv-cli`, keeping offline bundles and repositories aligned. + +No additional follow-ups — downstream teams can rely on the committed samples for integration coverage. diff --git a/docs/updates/2025-10-19-scanner-policy.md b/docs/updates/2025-10-19-scanner-policy.md new file mode 100644 index 00000000..758a0f16 --- /dev/null +++ b/docs/updates/2025-10-19-scanner-policy.md @@ -0,0 +1,5 @@ +# 2025-10-19 – Scanner ↔ Policy Sync + +- Scanner WebService now emits `scanner.report.ready` and `scanner.scan.completed` via Redis Streams when `scanner.events.enabled=true`; DSSE envelopes are embedded verbatim to keep Notify/UI consumers in sync. +- Config plumbing introduces `scanner:events:*` settings (driver, DSN, stream, publish timeout) with validation and Redis-backed publisher wiring. +- Policy Guild coordination task `POLICY-RUNTIME-17-201` opened to track Zastava runtime feed contract; `SCANNER-RUNTIME-17-401` now depends on it so reachability tags stay aligned once runtime endpoints ship. diff --git a/docs/updates/2025-10-19-scheduler-storage.md b/docs/updates/2025-10-19-scheduler-storage.md new file mode 100644 index 00000000..61541cc3 --- /dev/null +++ b/docs/updates/2025-10-19-scheduler-storage.md @@ -0,0 +1,8 @@ +# Scheduler Storage Update — 2025-10-19 + +**Subject:** Mongo bootstrap + canonical fixtures +**Audience:** Scheduler Storage Guild, Scheduler WebService/Worker teams + +- Added `StellaOps.Scheduler.Storage.Mongo` bootstrap (`AddSchedulerMongoStorage`) with collection/index migrations for schedules, runs (incl. TTL), impact snapshots, audit, and locks. +- Introduced Mongo2Go-backed tests that round-trip the published scheduler samples (`samples/api/scheduler/*.json`) to ensure canonical JSON stays intact. +- `ISchedulerMongoInitializer.EnsureMigrationsAsync` now provides the single entry point for WebService/Worker hosts to apply migrations at startup. diff --git a/etc/authority.yaml.sample b/etc/authority.yaml.sample index d9d887ad..2daf2a95 100644 --- a/etc/authority.yaml.sample +++ b/etc/authority.yaml.sample @@ -1,90 +1,113 @@ -# StellaOps Authority configuration template. -# Copy to ../etc/authority.yaml (relative to the Authority content root) -# and adjust values to fit your environment. Environment variables -# prefixed with STELLAOPS_AUTHORITY_ override these values at runtime. -# Example: STELLAOPS_AUTHORITY__ISSUER=https://authority.example.com - -schemaVersion: 1 - -# Absolute issuer URI advertised to clients. Use HTTPS for anything -# beyond loopback development. -issuer: "https://authority.stella-ops.local" - -# Token lifetimes expressed as HH:MM:SS or DD.HH:MM:SS. -accessTokenLifetime: "00:15:00" -refreshTokenLifetime: "30.00:00:00" -identityTokenLifetime: "00:05:00" -authorizationCodeLifetime: "00:05:00" -deviceCodeLifetime: "00:15:00" - -# MongoDB storage connection details. -storage: - connectionString: "mongodb://localhost:27017/stellaops-authority" - # databaseName: "stellaops_authority" - commandTimeout: "00:00:30" - -# Signing configuration for revocation bundles and JWKS. -signing: - enabled: true - activeKeyId: "authority-signing-2025-dev" - keyPath: "../certificates/authority-signing-2025-dev.pem" - algorithm: "ES256" - keySource: "file" - # provider: "default" - additionalKeys: - - keyId: "authority-signing-dev" - path: "../certificates/authority-signing-dev.pem" - source: "file" - # Rotation flow: - # 1. Generate a new PEM under ./certificates (e.g. authority-signing-2026-dev.pem). - # 2. Trigger the .gitea/workflows/authority-key-rotation.yml workflow (or run - # ops/authority/key-rotation.sh) with the new keyId/keyPath. - # 3. Update activeKeyId/keyPath above and move the previous key into additionalKeys - # so restarts retain retired material for JWKS consumers. - -# Bootstrap administrative endpoints (initial provisioning). -bootstrap: - enabled: false - apiKey: "change-me" - defaultIdentityProvider: "standard" - -# Directories scanned for Authority plug-ins. Relative paths resolve -# against the application content root, enabling air-gapped deployments -# that package plug-ins alongside binaries. -pluginDirectories: - - "../PluginBinaries/Authority" - # "/var/lib/stellaops/authority/plugins" - -# Plug-in manifests live in descriptors below; per-plugin settings are stored -# in the configurationDirectory (YAML files). Authority will load any enabled -# plugins and surface their metadata/capabilities to the host. -plugins: - configurationDirectory: "../etc/authority.plugins" - descriptors: - standard: - type: "standard" - assemblyName: "StellaOps.Authority.Plugin.Standard" - enabled: true - configFile: "standard.yaml" - capabilities: - - password - - bootstrap - - clientProvisioning - metadata: - defaultRole: "operators" - # Example for an external identity provider plugin. Leave disabled unless - # the plug-in package exists under PluginBinaries/Authority. - ldap: - type: "ldap" - assemblyName: "StellaOps.Authority.Plugin.Ldap" - enabled: false - configFile: "ldap.yaml" - capabilities: - - password - - mfa - -# CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). -# Keep the list tight: localhost is sufficient for most air-gapped installs. -bypassNetworks: - - "127.0.0.1/32" - - "::1/128" +# StellaOps Authority configuration template. +# Copy to ../etc/authority.yaml (relative to the Authority content root) +# and adjust values to fit your environment. Environment variables +# prefixed with STELLAOPS_AUTHORITY_ override these values at runtime. +# Example: STELLAOPS_AUTHORITY__ISSUER=https://authority.example.com + +schemaVersion: 1 + +# Absolute issuer URI advertised to clients. Use HTTPS for anything +# beyond loopback development. +issuer: "https://authority.stella-ops.local" + +# Token lifetimes expressed as HH:MM:SS or DD.HH:MM:SS. +accessTokenLifetime: "00:15:00" +refreshTokenLifetime: "30.00:00:00" +identityTokenLifetime: "00:05:00" +authorizationCodeLifetime: "00:05:00" +deviceCodeLifetime: "00:15:00" + +# MongoDB storage connection details. +storage: + connectionString: "mongodb://localhost:27017/stellaops-authority" + # databaseName: "stellaops_authority" + commandTimeout: "00:00:30" + +# Signing configuration for revocation bundles and JWKS. +signing: + enabled: true + activeKeyId: "authority-signing-2025-dev" + keyPath: "../certificates/authority-signing-2025-dev.pem" + algorithm: "ES256" + keySource: "file" + # provider: "default" + additionalKeys: + - keyId: "authority-signing-dev" + path: "../certificates/authority-signing-dev.pem" + source: "file" + # Rotation flow: + # 1. Generate a new PEM under ./certificates (e.g. authority-signing-2026-dev.pem). + # 2. Trigger the .gitea/workflows/authority-key-rotation.yml workflow (or run + # ops/authority/key-rotation.sh) with the new keyId/keyPath. + # 3. Update activeKeyId/keyPath above and move the previous key into additionalKeys + # so restarts retain retired material for JWKS consumers. + +# Bootstrap administrative endpoints (initial provisioning). +bootstrap: + enabled: false + apiKey: "change-me" + defaultIdentityProvider: "standard" + +# Directories scanned for Authority plug-ins. Relative paths resolve +# against the application content root, enabling air-gapped deployments +# that package plug-ins alongside binaries. +pluginDirectories: + - "../StellaOps.Authority.PluginBinaries" + # "/var/lib/stellaops/authority/plugins" + +# Plug-in manifests live in descriptors below; per-plugin settings are stored +# in the configurationDirectory (YAML files). Authority will load any enabled +# plugins and surface their metadata/capabilities to the host. +plugins: + configurationDirectory: "../etc/authority.plugins" + descriptors: + standard: + type: "standard" + assemblyName: "StellaOps.Authority.Plugin.Standard" + enabled: true + configFile: "standard.yaml" + capabilities: + - password + - bootstrap + - clientProvisioning + metadata: + defaultRole: "operators" + # Example for an external identity provider plugin. Leave disabled unless + # the plug-in package exists under StellaOps.Authority.PluginBinaries. + ldap: + type: "ldap" + assemblyName: "StellaOps.Authority.Plugin.Ldap" + enabled: false + configFile: "ldap.yaml" + capabilities: + - password + - mfa + +# OAuth client registrations issued by Authority. These examples cover Notify WebService +# in dev (notify.dev audience) and production (notify audience). Replace the secret files +# with paths to your sealed credentials before enabling bootstrap mode. +clients: + - clientId: "notify-web-dev" + displayName: "Notify WebService (dev)" + grantTypes: [ "client_credentials" ] + audiences: [ "notify.dev" ] + scopes: [ "notify.read", "notify.admin" ] + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/notify-web-dev.secret" + - clientId: "notify-web" + displayName: "Notify WebService" + grantTypes: [ "client_credentials" ] + audiences: [ "notify" ] + scopes: [ "notify.read", "notify.admin" ] + senderConstraint: "dpop" + auth: + type: "client_secret" + secretFile: "../secrets/notify-web.secret" + +# CIDR ranges that bypass network-sensitive policies (e.g. on-host cron jobs). +# Keep the list tight: localhost is sufficient for most air-gapped installs. +bypassNetworks: + - "127.0.0.1/32" + - "::1/128" diff --git a/etc/feedser.yaml.sample b/etc/concelier.yaml.sample similarity index 64% rename from etc/feedser.yaml.sample rename to etc/concelier.yaml.sample index a36cdd1f..77c93fb5 100644 --- a/etc/feedser.yaml.sample +++ b/etc/concelier.yaml.sample @@ -1,97 +1,113 @@ -# Feedser configuration template for StellaOps deployments. -# Copy to ../etc/feedser.yaml (relative to the web service content root) -# and adjust the values to match your environment. Environment variables -# (prefixed with FEEDSER_) override these settings at runtime. - -storage: - driver: mongo - # Mongo connection string. Use SRV URI or standard connection string. - dsn: "mongodb://feedser:feedser@mongo:27017/feedser?authSource=admin" - # Optional database name; defaults to the name embedded in the DSN or 'feedser'. - database: "feedser" - # Mongo command timeout in seconds. - commandTimeoutSeconds: 30 - -plugins: - # Feedser resolves plug-ins relative to the content root; override as needed. - baseDirectory: ".." - directory: "PluginBinaries" - searchPatterns: - - "StellaOps.Feedser.Plugin.*.dll" - -telemetry: - enabled: true - enableTracing: false - enableMetrics: false - enableLogging: true - minimumLogLevel: "Information" - serviceName: "stellaops-feedser" - # Configure OTLP endpoint when shipping traces/metrics/logs out-of-band. - otlpEndpoint: "" - # Optional headers for OTLP exporters, for example authentication tokens. - otlpHeaders: {} - # Attach additional resource attributes to telemetry exports. - resourceAttributes: - deployment.environment: "local" - # Emit console exporters for local debugging. - exportConsole: true - -authority: - enabled: false - # Temporary rollout flag. When true, Feedser logs anonymous access but does not fail requests - # without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully. - allowAnonymousFallback: true - # Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local). - issuer: "https://authority.stella-ops.local" - # Optional explicit metadata address; defaults to {issuer}/.well-known/openid-configuration. - metadataAddress: "" - requireHttpsMetadata: true - backchannelTimeoutSeconds: 30 - tokenClockSkewSeconds: 60 - audiences: - - "api://feedser" - requiredScopes: - - "feedser.jobs.trigger" - # Outbound credentials Feedser can use to call Authority (client credentials flow). - clientId: "feedser-jobs" - # Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile. - clientSecret: "" - clientSecretFile: "" - clientScopes: - - "feedser.jobs.trigger" - resilience: - # Enable deterministic retry/backoff when Authority is briefly unavailable. - enableRetries: true - retryDelays: - - "00:00:01" - - "00:00:02" - - "00:00:05" - # Allow stale discovery/JWKS responses when Authority is offline (extend tolerance as needed for air-gapped mirrors). - allowOfflineCacheFallback: true - offlineCacheTolerance: "00:10:00" - # Networks allowed to bypass authentication (loopback by default for on-host cron jobs). - bypassNetworks: - - "127.0.0.1/32" - - "::1/128" - -sources: - ghsa: - apiToken: "${GITHUB_PAT}" - pageSize: 50 - maxPagesPerFetch: 5 - requestDelay: "00:00:00.200" - failureBackoff: "00:05:00" - rateLimitWarningThreshold: 500 - secondaryRateLimitBackoff: "00:02:00" - cve: - baseEndpoint: "https://cveawg.mitre.org/api/" - apiOrg: "" - apiUser: "" - apiKey: "" - # Optional mirror used when credentials are unavailable. - seedDirectory: "./seed-data/cve" - pageSize: 200 - maxPagesPerFetch: 5 - initialBackfill: "30.00:00:00" - requestDelay: "00:00:00.250" - failureBackoff: "00:10:00" +# Concelier configuration template for StellaOps deployments. +# Copy to ../etc/concelier.yaml (relative to the web service content root) +# and adjust the values to match your environment. Environment variables +# (prefixed with CONCELIER_) override these settings at runtime. + +storage: + driver: mongo + # Mongo connection string. Use SRV URI or standard connection string. + dsn: "mongodb://concelier:concelier@mongo:27017/concelier?authSource=admin" + # Optional database name; defaults to the name embedded in the DSN or 'concelier'. + database: "concelier" + # Mongo command timeout in seconds. + commandTimeoutSeconds: 30 + +plugins: + # Concelier resolves plug-ins relative to the content root; override as needed. + baseDirectory: ".." + directory: "StellaOps.Concelier.PluginBinaries" + searchPatterns: + - "StellaOps.Concelier.Plugin.*.dll" + +telemetry: + enabled: true + enableTracing: false + enableMetrics: false + enableLogging: true + minimumLogLevel: "Information" + serviceName: "stellaops-concelier" + # Configure OTLP endpoint when shipping traces/metrics/logs out-of-band. + otlpEndpoint: "" + # Optional headers for OTLP exporters, for example authentication tokens. + otlpHeaders: {} + # Attach additional resource attributes to telemetry exports. + resourceAttributes: + deployment.environment: "local" + # Emit console exporters for local debugging. + exportConsole: true + +authority: + enabled: false + # Temporary rollout flag. When true, Concelier logs anonymous access but does not fail requests + # without tokens. Set to false before 2025-12-31 UTC to enforce authentication fully. + allowAnonymousFallback: true + # Issuer advertised by StellaOps Authority (e.g. https://authority.stella-ops.local). + issuer: "https://authority.stella-ops.local" + # Optional explicit metadata address; defaults to {issuer}/.well-known/openid-configuration. + metadataAddress: "" + requireHttpsMetadata: true + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - "api://concelier" + requiredScopes: + - "concelier.jobs.trigger" + # Outbound credentials Concelier can use to call Authority (client credentials flow). + clientId: "concelier-jobs" + # Prefer storing the secret outside of the config file. Provide either clientSecret or clientSecretFile. + clientSecret: "" + clientSecretFile: "" + clientScopes: + - "concelier.jobs.trigger" + resilience: + # Enable deterministic retry/backoff when Authority is briefly unavailable. + enableRetries: true + retryDelays: + - "00:00:01" + - "00:00:02" + - "00:00:05" + # Allow stale discovery/JWKS responses when Authority is offline (extend tolerance as needed for air-gapped mirrors). + allowOfflineCacheFallback: true + offlineCacheTolerance: "00:10:00" + # Networks allowed to bypass authentication (loopback by default for on-host cron jobs). + bypassNetworks: + - "127.0.0.1/32" + - "::1/128" + +mirror: + enabled: false + # Directory containing JSON exporter outputs (absolute or relative to content root). + exportRoot: "exports/json" + # Optional explicit export identifier; defaults to `latest` symlink or most recent export. + activeExportId: "" + latestDirectoryName: "latest" + mirrorDirectoryName: "mirror" + requireAuthentication: false + maxIndexRequestsPerHour: 600 + domains: + - id: "primary" + displayName: "Primary Mirror" + requireAuthentication: false + maxDownloadRequestsPerHour: 1200 + +sources: + ghsa: + apiToken: "${GITHUB_PAT}" + pageSize: 50 + maxPagesPerFetch: 5 + requestDelay: "00:00:00.200" + failureBackoff: "00:05:00" + rateLimitWarningThreshold: 500 + secondaryRateLimitBackoff: "00:02:00" + cve: + baseEndpoint: "https://cveawg.mitre.org/api/" + apiOrg: "" + apiUser: "" + apiKey: "" + # Optional mirror used when credentials are unavailable. + seedDirectory: "./seed-data/cve" + pageSize: 200 + maxPagesPerFetch: 5 + initialBackfill: "30.00:00:00" + requestDelay: "00:00:00.250" + failureBackoff: "00:10:00" diff --git a/etc/notify.dev.yaml b/etc/notify.dev.yaml new file mode 100644 index 00000000..5261d81b --- /dev/null +++ b/etc/notify.dev.yaml @@ -0,0 +1,43 @@ +# Notify WebService configuration — development + +storage: + driver: mongo + connectionString: "mongodb://notify-mongo.dev.svc.cluster.local:27017" + database: "stellaops_notify_dev" + commandTimeoutSeconds: 30 + +authority: + enabled: true + issuer: "https://authority.dev.stella-ops.local" + metadataAddress: "https://authority.dev.stella-ops.local/.well-known/openid-configuration" + requireHttpsMetadata: false + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify.dev + readScope: notify.read + adminScope: notify.admin + +api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + +plugins: + baseDirectory: "../" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + +telemetry: + enableRequestLogging: true + minimumLogLevel: Debug + +# Development override: when the Authority service is not available, set +# authority.enabled: false and authority.developmentSigningKey to a 32+ byte secret. diff --git a/etc/notify.prod.yaml b/etc/notify.prod.yaml new file mode 100644 index 00000000..e0c993b7 --- /dev/null +++ b/etc/notify.prod.yaml @@ -0,0 +1,40 @@ +# Notify WebService configuration — production + +storage: + driver: mongo + connectionString: "mongodb://notify-mongo.prod.svc.cluster.local:27017" + database: "stellaops_notify" + commandTimeoutSeconds: 60 + +authority: + enabled: true + issuer: "https://authority.stella-ops.org" + metadataAddress: "https://authority.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + +api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + +plugins: + baseDirectory: "/var/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + +telemetry: + enableRequestLogging: true + minimumLogLevel: Warning diff --git a/etc/notify.stage.yaml b/etc/notify.stage.yaml new file mode 100644 index 00000000..3e336dbd --- /dev/null +++ b/etc/notify.stage.yaml @@ -0,0 +1,40 @@ +# Notify WebService configuration — staging + +storage: + driver: mongo + connectionString: "mongodb://notify-mongo.stage.svc.cluster.local:27017" + database: "stellaops_notify_stage" + commandTimeoutSeconds: 45 + +authority: + enabled: true + issuer: "https://authority.stage.stella-ops.org" + metadataAddress: "https://authority.stage.stella-ops.org/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + +api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + +plugins: + baseDirectory: "/opt/stellaops" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + +telemetry: + enableRequestLogging: true + minimumLogLevel: Information diff --git a/etc/notify.yaml.sample b/etc/notify.yaml.sample new file mode 100644 index 00000000..4015cfac --- /dev/null +++ b/etc/notify.yaml.sample @@ -0,0 +1,59 @@ +# Notify WebService sample configuration + +storage: + # Use "mongo" for production deployments; set to "memory" only for tests/dev harnesses. + driver: mongo + connectionString: "mongodb://localhost:27017" + database: "stellaops_notify" + commandTimeoutSeconds: 30 + +authority: + enabled: true + issuer: "https://authority.stella-ops.local" + metadataAddress: "https://authority.stella-ops.local/.well-known/openid-configuration" + requireHttpsMetadata: true + allowAnonymousFallback: false + backchannelTimeoutSeconds: 30 + tokenClockSkewSeconds: 60 + audiences: + - notify + readScope: notify.read + adminScope: notify.admin + +api: + basePath: "/api/v1/notify" + internalBasePath: "/internal/notify" + tenantHeader: "X-StellaOps-Tenant" + rateLimits: + deliveryHistory: + enabled: true + tokenLimit: 60 + tokensPerPeriod: 30 + replenishmentPeriodSeconds: 60 + queueLimit: 20 + testSend: + enabled: true + tokenLimit: 5 + tokensPerPeriod: 5 + replenishmentPeriodSeconds: 60 + queueLimit: 2 + +plugins: + baseDirectory: "../" + directory: "plugins/notify" + searchPatterns: + - "StellaOps.Notify.Connectors.*.dll" + orderedPlugins: + - StellaOps.Notify.Connectors.Slack + - StellaOps.Notify.Connectors.Teams + - StellaOps.Notify.Connectors.Email + - StellaOps.Notify.Connectors.Webhook + +telemetry: + enableRequestLogging: true + minimumLogLevel: Information + +# When running in development without Authority, set the following instead: +# authority: +# enabled: false +# developmentSigningKey: "change-me-32-bytes-minimum-signing-key" diff --git a/etc/secrets/.gitkeep b/etc/secrets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/etc/secrets/notify-web-dev.secret.example b/etc/secrets/notify-web-dev.secret.example new file mode 100644 index 00000000..ef02eac8 --- /dev/null +++ b/etc/secrets/notify-web-dev.secret.example @@ -0,0 +1,3 @@ +# Replace this file with the actual client secret for the notify-web-dev Authority client. +# Store the secret with restrictive permissions (chmod 600) and mount/read-only in deployments. +NOTIFY_WEB_DEV_CLIENT_SECRET=change-me-dev diff --git a/etc/secrets/notify-web.secret.example b/etc/secrets/notify-web.secret.example new file mode 100644 index 00000000..7b341d49 --- /dev/null +++ b/etc/secrets/notify-web.secret.example @@ -0,0 +1,3 @@ +# Replace this file with the production client secret for the notify-web Authority client. +# Keep outside source control and mount via secrets manager in Kubernetes/offline kit bundles. +NOTIFY_WEB_CLIENT_SECRET=change-me-prod diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json new file mode 100644 index 00000000..85d67b30 --- /dev/null +++ b/node_modules/.package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "git.stella-ops.org", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/ops/deployment/AGENTS.md b/ops/deployment/AGENTS.md new file mode 100644 index 00000000..16730e7d --- /dev/null +++ b/ops/deployment/AGENTS.md @@ -0,0 +1,4 @@ +# Deployment & Operations — Agent Charter + +## Mission +Maintain deployment/upgrade/rollback workflows (Helm/Compose) per `docs/ARCHITECTURE_DEVOPS.md` including environment-specific configs. diff --git a/ops/deployment/TASKS.md b/ops/deployment/TASKS.md new file mode 100644 index 00000000..64eb058a --- /dev/null +++ b/ops/deployment/TASKS.md @@ -0,0 +1,5 @@ +# Deployment Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DEVOPS-OPS-14-003 | TODO | Deployment Guild | DEVOPS-REL-14-001 | Document and script upgrade/rollback flows, channel management, and compatibility matrices per architecture. | Helm/Compose guides updated with digest pinning, automated checks committed, rollback drill recorded. | diff --git a/ops/devops/AGENTS.md b/ops/devops/AGENTS.md new file mode 100644 index 00000000..bc640c47 --- /dev/null +++ b/ops/devops/AGENTS.md @@ -0,0 +1,11 @@ +# DevOps & Release — Agent Charter + +## Mission +Execute deterministic build/release pipeline per `docs/ARCHITECTURE_DEVOPS.md`: +- Reproducible builds with SBOM/provenance, cosign signing, transparency logging. +- Channel manifests (LTS/Stable/Edge) with digests, Helm/Compose profiles. +- Performance guard jobs ensuring budgets. + +## Expectations +- Coordinate with Scanner/Scheduler/Notify teams for artifact availability. +- Maintain CI reliability; update `TASKS.md` as states change. diff --git a/ops/devops/TASKS.md b/ops/devops/TASKS.md new file mode 100644 index 00000000..b3bf217e --- /dev/null +++ b/ops/devops/TASKS.md @@ -0,0 +1,13 @@ +# DevOps Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DEVOPS-HELM-09-001 | DONE | DevOps Guild | SCANNER-WEB-09-101 | Create Helm/Compose environment profiles (dev, staging, airgap) with deterministic digests. | Profiles committed under `deploy/`; docs updated; CI smoke deploy passes. | +| DEVOPS-SCANNER-09-204 | TODO | DevOps Guild, Scanner WebService Guild | SCANNER-EVENTS-15-201 | Surface `SCANNER__EVENTS__*` environment variables across docker-compose (dev/stage/airgap) and Helm values, defaulting to share the Redis queue DSN. | Compose/Helm configs ship enabled Redis event publishing with documented overrides; lint jobs updated; docs cross-link to new knobs. | +| DEVOPS-SCANNER-09-205 | TODO | DevOps Guild, Notify Guild | DEVOPS-SCANNER-09-204 | Add Notify smoke stage that tails the Redis stream and asserts `scanner.report.ready`/`scanner.scan.completed` reach Notify WebService in staging. | CI job reads Redis stream during scanner smoke deploy, confirms Notify ingestion via API, alerts on failure. | +| DEVOPS-PERF-10-001 | DONE | DevOps Guild | BENCH-SCANNER-10-001 | Add perf smoke job (SBOM compose <5 s target) to CI. | CI job runs sample build verifying <5 s; alerts configured. | +| DEVOPS-PERF-10-002 | TODO | DevOps Guild | BENCH-SCANNER-10-002 | Publish analyzer bench metrics to Grafana/perf workbook and alarm on ≥20 % regressions. | CI exports JSON for dashboards; Grafana panel wired; Ops on-call doc updated with alert hook. | +| DEVOPS-REL-14-001 | TODO | DevOps Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Deterministic build/release pipeline with SBOM/provenance, signing, manifest generation. | CI pipeline produces signed images + SBOM/attestations, manifests published with verified hashes, docs updated. | +| DEVOPS-REL-17-002 | TODO | DevOps Guild | DEVOPS-REL-14-001, SCANNER-EMIT-17-701 | Persist stripped-debug artifacts organised by GNU build-id and bundle them into release/offline kits with checksum manifests. | CI job writes `.debug` files under `artifacts/debug/.build-id/`, manifest + checksums published, offline kit includes cache, smoke job proves symbol lookup via build-id. | +| DEVOPS-MIRROR-08-001 | DONE (2025-10-19) | DevOps Guild | DEVOPS-REL-14-001 | Stand up managed mirror profiles for `*.stella-ops.org` (Concelier/Excititor), including Helm/Compose overlays, multi-tenant secrets, CDN caching, and sync documentation. | Infra overlays committed, CI smoke deploy hits mirror endpoints, runbooks published for downstream sync and quota management. | +| DEVOPS-SEC-10-301 | DOING (2025-10-19) | DevOps Guild | Wave 0A complete | Address NU1902/NU1903 advisories for `MongoDB.Driver` 2.12.0 and `SharpCompress` 0.23.0 surfaced during scanner cache and worker test runs. | Dependencies bumped to patched releases, audit logs free of NU1902/NU1903 warnings, regression tests green, change log documents upgrade guidance. | diff --git a/ops/licensing/AGENTS.md b/ops/licensing/AGENTS.md new file mode 100644 index 00000000..d1cce675 --- /dev/null +++ b/ops/licensing/AGENTS.md @@ -0,0 +1,4 @@ +# Licensing & Registry Access — Agent Charter + +## Mission +Implement licensing token service and registry access workflows described in `docs/ARCHITECTURE_DEVOPS.md`. diff --git a/ops/licensing/TASKS.md b/ops/licensing/TASKS.md new file mode 100644 index 00000000..5c9a727c --- /dev/null +++ b/ops/licensing/TASKS.md @@ -0,0 +1,5 @@ +# Licensing Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DEVOPS-LIC-14-004 | TODO | Licensing Guild | AUTH-MTLS-11-002 | Implement registry token service tied to Authority (DPoP/mTLS), plan gating, revocation handling, and monitoring per architecture. | Token service issues scoped tokens, revocation tested, monitoring dashboards in place, docs updated. | diff --git a/ops/offline-kit/AGENTS.md b/ops/offline-kit/AGENTS.md new file mode 100644 index 00000000..e0962c1d --- /dev/null +++ b/ops/offline-kit/AGENTS.md @@ -0,0 +1,4 @@ +# Offline Kit — Agent Charter + +## Mission +Package Offline Update Kit per `docs/ARCHITECTURE_DEVOPS.md` and `docs/24_OFFLINE_KIT.md` with deterministic digests and import tooling. diff --git a/ops/offline-kit/TASKS.md b/ops/offline-kit/TASKS.md new file mode 100644 index 00000000..b14ecc10 --- /dev/null +++ b/ops/offline-kit/TASKS.md @@ -0,0 +1,5 @@ +# Offline Kit Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| DEVOPS-OFFLINE-14-002 | TODO | Offline Kit Guild | DEVOPS-REL-14-001 | Build offline kit packaging workflow (artifact bundling, manifest generation, signature verification). | Offline tarball generated with manifest + checksums + signatures; import script verifies integrity; docs updated. | diff --git a/out/tmp-cdx/Program.cs b/out/tmp-cdx/Program.cs new file mode 100644 index 00000000..479258bd --- /dev/null +++ b/out/tmp-cdx/Program.cs @@ -0,0 +1,6 @@ +using System; +using CycloneDX.Models; + +var dependenciesProperty = typeof(Dependency).GetProperty("Dependencies")!; +Console.WriteLine(dependenciesProperty.PropertyType); +Console.WriteLine(dependenciesProperty.PropertyType.GenericTypeArguments[0]); diff --git a/out/tmp-cdx/tmp-cdx.csproj b/out/tmp-cdx/tmp-cdx.csproj new file mode 100644 index 00000000..1670a49c --- /dev/null +++ b/out/tmp-cdx/tmp-cdx.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + tmp_cdx + enable + enable + + + + + + + diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..85d67b30 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "git.stella-ops.org", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..69a88e3b --- /dev/null +++ b/package.json @@ -0,0 +1 @@ +{} diff --git a/plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json b/plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json new file mode 100644 index 00000000..096ef4d6 --- /dev/null +++ b/plugins/scanner/analyzers/lang/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzer.lang.java", + "displayName": "StellaOps Java / Maven Analyzer", + "version": "0.1.0", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.Lang.Java.dll", + "typeName": "StellaOps.Scanner.Analyzers.Lang.Java.JavaLanguageAnalyzer" + }, + "capabilities": [ + "language-analyzer", + "java", + "maven" + ], + "metadata": { + "org.stellaops.analyzer.language": "java", + "org.stellaops.analyzer.kind": "language", + "org.stellaops.restart.required": "true" + } +} diff --git a/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json new file mode 100644 index 00000000..0aaa333c --- /dev/null +++ b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.apk", + "displayName": "StellaOps Alpine APK Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Apk.dll" + }, + "capabilities": [ + "os-analyzer", + "apk" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "apk" + } +} diff --git a/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json new file mode 100644 index 00000000..8ff2ab98 --- /dev/null +++ b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.dpkg", + "displayName": "StellaOps Debian dpkg Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Dpkg.dll" + }, + "capabilities": [ + "os-analyzer", + "dpkg" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "dpkg" + } +} diff --git a/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json new file mode 100644 index 00000000..ed842a2f --- /dev/null +++ b/plugins/scanner/analyzers/os/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.rpm", + "displayName": "StellaOps RPM Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Rpm.dll" + }, + "capabilities": [ + "os-analyzer", + "rpm" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "rpm" + } +} diff --git a/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/manifest.json b/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/manifest.json new file mode 100644 index 00000000..e3ceac65 --- /dev/null +++ b/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/manifest.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.sbom-indexer", + "displayName": "StellaOps SBOM BuildX Generator", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "executable": "StellaOps.Scanner.Sbomer.BuildXPlugin.dll", + "arguments": [ + "handshake" + ] + }, + "capabilities": [ + "generator", + "sbom" + ], + "cas": { + "protocol": "filesystem", + "defaultRoot": "cas", + "compression": "zstd" + }, + "image": { + "name": "stellaops/sbom-indexer", + "digest": null, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + "metadata": { + "org.stellaops.plugin.kind": "buildx-generator", + "org.stellaops.restart.required": "true" + } +} diff --git a/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json b/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json new file mode 100644 index 00000000..e3ceac65 --- /dev/null +++ b/plugins/scanner/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.sbom-indexer", + "displayName": "StellaOps SBOM BuildX Generator", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "executable": "StellaOps.Scanner.Sbomer.BuildXPlugin.dll", + "arguments": [ + "handshake" + ] + }, + "capabilities": [ + "generator", + "sbom" + ], + "cas": { + "protocol": "filesystem", + "defaultRoot": "cas", + "compression": "zstd" + }, + "image": { + "name": "stellaops/sbom-indexer", + "digest": null, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + "metadata": { + "org.stellaops.plugin.kind": "buildx-generator", + "org.stellaops.restart.required": "true" + } +} diff --git a/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.deps.json b/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.deps.json new file mode 100644 index 00000000..219e38a6 --- /dev/null +++ b/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.deps.json @@ -0,0 +1,191 @@ +{ + "runtimeTarget": { + "name": ".NETCoreApp,Version=v10.0", + "signature": "" + }, + "compilationOptions": {}, + "targets": { + ".NETCoreApp,Version=v10.0": { + "StellaOps.Scanner.EntryTrace/1.0.0": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Options.ConfigurationExtensions": "9.0.0", + "StellaOps.Plugin": "1.0.0" + }, + "runtime": { + "StellaOps.Scanner.EntryTrace.dll": {} + } + }, + "Microsoft.Extensions.Configuration.Abstractions/9.0.0": { + "dependencies": { + "Microsoft.Extensions.Primitives": "9.0.0" + }, + "runtime": { + "lib/net9.0/Microsoft.Extensions.Configuration.Abstractions.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.Configuration.Binder/9.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0" + }, + "runtime": { + "lib/net9.0/Microsoft.Extensions.Configuration.Binder.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions/9.0.0": { + "runtime": { + "lib/net9.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.Logging.Abstractions/9.0.0": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0" + }, + "runtime": { + "lib/net9.0/Microsoft.Extensions.Logging.Abstractions.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.Options/9.0.0": { + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Primitives": "9.0.0" + }, + "runtime": { + "lib/net9.0/Microsoft.Extensions.Options.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions/9.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", + "Microsoft.Extensions.Configuration.Binder": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Primitives": "9.0.0" + }, + "runtime": { + "lib/net9.0/Microsoft.Extensions.Options.ConfigurationExtensions.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "Microsoft.Extensions.Primitives/9.0.0": { + "runtime": { + "lib/net9.0/Microsoft.Extensions.Primitives.dll": { + "assemblyVersion": "9.0.0.0", + "fileVersion": "9.0.24.52809" + } + } + }, + "StellaOps.DependencyInjection/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0" + }, + "runtime": { + "StellaOps.DependencyInjection.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + }, + "StellaOps.Plugin/1.0.0": { + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "9.0.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.0", + "Microsoft.Extensions.Logging.Abstractions": "9.0.0", + "StellaOps.DependencyInjection": "1.0.0" + }, + "runtime": { + "StellaOps.Plugin.dll": { + "assemblyVersion": "1.0.0.0", + "fileVersion": "1.0.0.0" + } + } + } + } + }, + "libraries": { + "StellaOps.Scanner.EntryTrace/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "Microsoft.Extensions.Configuration.Abstractions/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-lqvd7W3FGKUO1+ZoUEMaZ5XDJeWvjpy2/M/ptCGz3tXLD4HWVaSzjufsAsjemasBEg+2SxXVtYVvGt5r2nKDlg==", + "path": "microsoft.extensions.configuration.abstractions/9.0.0", + "hashPath": "microsoft.extensions.configuration.abstractions.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.Configuration.Binder/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-RiScL99DcyngY9zJA2ROrri7Br8tn5N4hP4YNvGdTN/bvg1A3dwvDOxHnNZ3Im7x2SJ5i4LkX1uPiR/MfSFBLQ==", + "path": "microsoft.extensions.configuration.binder/9.0.0", + "hashPath": "microsoft.extensions.configuration.binder.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.DependencyInjection.Abstractions/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-+6f2qv2a3dLwd5w6JanPIPs47CxRbnk+ZocMJUhv9NxP88VlOcJYZs9jY+MYSjxvady08bUZn6qgiNh7DadGgg==", + "path": "microsoft.extensions.dependencyinjection.abstractions/9.0.0", + "hashPath": "microsoft.extensions.dependencyinjection.abstractions.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.Logging.Abstractions/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-g0UfujELzlLbHoVG8kPKVBaW470Ewi+jnptGS9KUi6jcb+k2StujtK3m26DFSGGwQ/+bVgZfsWqNzlP6YOejvw==", + "path": "microsoft.extensions.logging.abstractions/9.0.0", + "hashPath": "microsoft.extensions.logging.abstractions.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.Options/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-y2146b3jrPI3Q0lokKXdKLpmXqakYbDIPDV6r3M8SqvSf45WwOTzkyfDpxnZXJsJQEpAsAqjUq5Pu8RCJMjubg==", + "path": "microsoft.extensions.options/9.0.0", + "hashPath": "microsoft.extensions.options.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.Options.ConfigurationExtensions/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-Ob3FXsXkcSMQmGZi7qP07EQ39kZpSBlTcAZLbJLdI4FIf0Jug8biv2HTavWmnTirchctPlq9bl/26CXtQRguzA==", + "path": "microsoft.extensions.options.configurationextensions/9.0.0", + "hashPath": "microsoft.extensions.options.configurationextensions.9.0.0.nupkg.sha512" + }, + "Microsoft.Extensions.Primitives/9.0.0": { + "type": "package", + "serviceable": true, + "sha512": "sha512-N3qEBzmLMYiASUlKxxFIISP4AiwuPTHF5uCh+2CWSwwzAJiIYx0kBJsS30cp1nvhSySFAVi30jecD307jV+8Kg==", + "path": "microsoft.extensions.primitives/9.0.0", + "hashPath": "microsoft.extensions.primitives.9.0.0.nupkg.sha512" + }, + "StellaOps.DependencyInjection/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + }, + "StellaOps.Plugin/1.0.0": { + "type": "project", + "serviceable": false, + "sha512": "" + } + } +} \ No newline at end of file diff --git a/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/manifest.json b/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/manifest.json new file mode 100644 index 00000000..565793ce --- /dev/null +++ b/plugins/scanner/entrytrace/StellaOps.Scanner.EntryTrace/manifest.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.entrytrace.analyzers", + "displayName": "StellaOps EntryTrace Analyzer Pack", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "executable": "StellaOps.Scanner.EntryTrace.dll", + "arguments": [ + "handshake" + ] + }, + "capabilities": [ + "entrytrace", + "analyzer" + ], + "metadata": { + "org.stellaops.plugin.kind": "entrytrace-analyzer", + "org.stellaops.restart.required": "true" + } +} diff --git a/samples/TASKS.md b/samples/TASKS.md new file mode 100644 index 00000000..d4d98279 --- /dev/null +++ b/samples/TASKS.md @@ -0,0 +1,6 @@ +# Samples Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SAMPLES-10-001 | DONE | Samples Guild, Scanner Team | SCANNER-EMIT-10-605 | Curate sample images (nginx, alpine+busybox, distroless+go, .NET AOT, python venv, npm monorepo) with expected SBOM/BOM-Index sidecars. | Samples committed under `samples/`; golden SBOM/BOM-Index files present; documented usage. | +| SAMPLES-13-004 | TODO | Samples Guild, Policy Guild | POLICY-CORE-09-006, UI-POLICY-13-007 | Add policy preview/report fixtures showing confidence bands and unknown-age tags. | Confidence sample (`samples/policy/policy-preview-unknown.json`) reviewed, documented usage in UI dev guide, ajv validation hook updated. | diff --git a/samples/api/reports/report-sample.dsse.json b/samples/api/reports/report-sample.dsse.json new file mode 100644 index 00000000..9ae89de4 --- /dev/null +++ b/samples/api/reports/report-sample.dsse.json @@ -0,0 +1,52 @@ +{ + "report": { + "reportId": "report-3def5f362aa475ef14b6", + "imageDigest": "sha256:deadbeef", + "generatedAt": "2025-10-19T08:28:09.3699267+00:00", + "verdict": "blocked", + "policy": { + "revisionId": "rev-1", + "digest": "27d2ec2b34feedc304fc564d252ecee1c8fa14ea581a5ff5c1ea8963313d5c8d" + }, + "summary": { + "total": 1, + "blocked": 1, + "warned": 0, + "ignored": 0, + "quieted": 0 + }, + "verdicts": [ + { + "findingId": "finding-1", + "status": "Blocked", + "ruleName": "Block Critical", + "ruleAction": "Block", + "score": 40.5, + "configVersion": "1.0", + "inputs": { + "reachabilityWeight": 0.45, + "baseScore": 40.5, + "severityWeight": 90, + "trustWeight": 1, + "trustWeight.NVD": 1, + "reachability.runtime": 0.45 + }, + "quiet": false, + "sourceTrust": "NVD", + "reachability": "runtime" + } + ], + "issues": [] + }, + "dsse": { + "payloadType": "application/vnd.stellaops.report+json", + "payload": "eyJyZXBvcnRJZCI6InJlcG9ydC0zZGVmNWYzNjJhYTQ3NWVmMTRiNiIsImltYWdlRGlnZXN0Ijoic2hhMjU2OmRlYWRiZWVmIiwiZ2VuZXJhdGVkQXQiOiIyMDI1LTEwLTE5VDA4OjI4OjA5LjM2OTkyNjcrMDA6MDAiLCJ2ZXJkaWN0IjoiYmxvY2tlZCIsInBvbGljeSI6eyJyZXZpc2lvbklkIjoicmV2LTEiLCJkaWdlc3QiOiIyN2QyZWMyYjM0ZmVlZGMzMDRmYzU2NGQyNTJlY2VlMWM4ZmExNGVhNTgxYTVmZjVjMWVhODk2MzMxM2Q1YzhkIn0sInN1bW1hcnkiOnsidG90YWwiOjEsImJsb2NrZWQiOjEsIndhcm5lZCI6MCwiaWdub3JlZCI6MCwicXVpZXRlZCI6MH0sInZlcmRpY3RzIjpbeyJmaW5kaW5nSWQiOiJmaW5kaW5nLTEiLCJzdGF0dXMiOiJCbG9ja2VkIiwicnVsZU5hbWUiOiJCbG9jayBDcml0aWNhbCIsInJ1bGVBY3Rpb24iOiJCbG9jayIsInNjb3JlIjo0MC41LCJjb25maWdWZXJzaW9uIjoiMS4wIiwiaW5wdXRzIjp7InJlYWNoYWJpbGl0eVdlaWdodCI6MC40NSwiYmFzZVNjb3JlIjo0MC41LCJzZXZlcml0eVdlaWdodCI6OTAsInRydXN0V2VpZ2h0IjoxLCJ0cnVzdFdlaWdodC5OVkQiOjEsInJlYWNoYWJpbGl0eS5ydW50aW1lIjowLjQ1fSwicXVpZXQiOmZhbHNlLCJzb3VyY2VUcnVzdCI6Ik5WRCIsInJlYWNoYWJpbGl0eSI6InJ1bnRpbWUifV0sImlzc3VlcyI6W119", + "signatures": [ + { + "keyId": "scanner-report-signing", + "algorithm": "hs256", + "signature": "s3qnWeRsYs+QA/nO84Us8G2xjZcvphc2P7KnOdTVwQs=" + } + ] + } +} diff --git a/samples/api/scheduler/audit.json b/samples/api/scheduler/audit.json new file mode 100644 index 00000000..91913c01 --- /dev/null +++ b/samples/api/scheduler/audit.json @@ -0,0 +1,19 @@ +{ + "id": "audit_169754", + "tenantId": "tenant-alpha", + "category": "scheduler", + "action": "pause", + "occurredAt": "2025-10-18T22:10:00+00:00", + "actor": { + "actorId": "user_admin", + "displayName": "Cluster Admin", + "kind": "user" + }, + "scheduleId": "sch_20251018a", + "correlationId": "corr-123", + "metadata": { + "details": "schedule paused", + "reason": "maintenance" + }, + "message": "Paused via API" +} diff --git a/samples/api/scheduler/impact-set.json b/samples/api/scheduler/impact-set.json new file mode 100644 index 00000000..fb4b5e84 --- /dev/null +++ b/samples/api/scheduler/impact-set.json @@ -0,0 +1,34 @@ +{ + "schemaVersion": "scheduler.impact-set@1", + "selector": { + "scope": "all-images", + "tenantId": "tenant-alpha", + "namespaces": [], + "repositories": [], + "digests": [], + "includeTags": [], + "labels": [], + "resolvesTags": false + }, + "images": [ + { + "imageDigest": "sha256:f1e2d3", + "registry": "registry.internal", + "repository": "app/api", + "namespaces": [ + "team-a" + ], + "tags": [ + "prod" + ], + "usedByEntrypoint": true, + "labels": { + "env": "prod" + } + } + ], + "usageOnly": true, + "generatedAt": "2025-10-18T22:02:58+00:00", + "total": 412, + "snapshotId": "impact-20251018-1" +} diff --git a/samples/api/scheduler/run.json b/samples/api/scheduler/run.json new file mode 100644 index 00000000..9f4a997e --- /dev/null +++ b/samples/api/scheduler/run.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "scheduler.run@1", + "id": "run_20251018_0001", + "tenantId": "tenant-alpha", + "scheduleId": "sch_20251018a", + "trigger": "feedser", + "state": "running", + "stats": { + "candidates": 1280, + "deduped": 910, + "queued": 624, + "completed": 310, + "deltas": 42, + "newCriticals": 7, + "newHigh": 11, + "newMedium": 18, + "newLow": 6 + }, + "reason": { + "feedserExportId": "exp-20251018-03" + }, + "createdAt": "2025-10-18T22:03:14+00:00", + "startedAt": "2025-10-18T22:03:20+00:00", + "deltas": [ + { + "imageDigest": "sha256:a1b2c3", + "newFindings": 3, + "newCriticals": 1, + "newHigh": 1, + "newMedium": 1, + "newLow": 0, + "kevHits": [ + "CVE-2025-0002" + ], + "topFindings": [ + { + "purl": "pkg:rpm/openssl@3.0.12-5.el9", + "vulnerabilityId": "CVE-2025-0002", + "severity": "critical", + "link": "https://ui.internal/scans/sha256:a1b2c3" + } + ], + "attestation": { + "uuid": "rekor-314", + "verified": true + }, + "detectedAt": "2025-10-18T22:03:21+00:00" + } + ] +} diff --git a/samples/api/scheduler/schedule.json b/samples/api/scheduler/schedule.json new file mode 100644 index 00000000..2d94fbb6 --- /dev/null +++ b/samples/api/scheduler/schedule.json @@ -0,0 +1,57 @@ +{ + "schemaVersion": "scheduler.schedule@1", + "id": "sch_20251018a", + "tenantId": "tenant-alpha", + "name": "Nightly Prod", + "enabled": true, + "cronExpression": "0 2 * * *", + "timezone": "UTC", + "mode": "analysis-only", + "selection": { + "scope": "by-namespace", + "tenantId": "tenant-alpha", + "namespaces": [ + "team-a", + "team-b" + ], + "repositories": [ + "app/service-api" + ], + "digests": [], + "includeTags": [ + "canary", + "prod" + ], + "labels": [ + { + "key": "env", + "values": [ + "prod", + "staging" + ] + } + ], + "resolvesTags": true + }, + "onlyIf": { + "lastReportOlderThanDays": 7, + "policyRevision": "policy@42" + }, + "notify": { + "onNewFindings": true, + "minSeverity": "high", + "includeKev": true + }, + "limits": { + "maxJobs": 1000, + "ratePerSecond": 25, + "parallelism": 4 + }, + "subscribers": [ + "notify.ops" + ], + "createdAt": "2025-10-18T22:00:00+00:00", + "createdBy": "svc_scheduler", + "updatedAt": "2025-10-18T22:00:00+00:00", + "updatedBy": "svc_scheduler" +} diff --git a/samples/ci/buildx-demo/Dockerfile b/samples/ci/buildx-demo/Dockerfile new file mode 100644 index 00000000..814082e0 --- /dev/null +++ b/samples/ci/buildx-demo/Dockerfile @@ -0,0 +1,4 @@ +FROM alpine:3.20 +RUN adduser -S stella && echo "hello" > /app.txt +USER stella +CMD ["/bin/sh","-c","cat /app.txt"] diff --git a/samples/ci/buildx-demo/README.md b/samples/ci/buildx-demo/README.md new file mode 100644 index 00000000..adc99eeb --- /dev/null +++ b/samples/ci/buildx-demo/README.md @@ -0,0 +1,42 @@ +# Buildx SBOM Demo Workflow + +This sample GitHub Actions workflow shows how to run the StellaOps BuildX generator alongside a container build. + +## What it does + +1. Publishes the `StellaOps.Scanner.Sbomer.BuildXPlugin` with the manifest copied beside the binaries. +2. Calls the plug-in `handshake` command to verify the local CAS directory. +3. Builds a tiny Alpine-based image via `docker buildx`. +4. Generates a CycloneDX SBOM from the built image with `docker sbom`. +5. Emits a descriptor + provenance placeholder referencing the freshly generated SBOM with the `descriptor` command. +6. Sends the placeholder to a mock Attestor endpoint and uploads the descriptor, SBOM, and captured request as artefacts. (Swap the mock step with your real Attestor URL + `STELLAOPS_ATTESTOR_TOKEN` secret when ready.) + +## Files + +- `github-actions-buildx-demo.yml` – workflow definition (`workflow_dispatch` + `demo/buildx` branch trigger). +- `Dockerfile` – minimal demo image. +- `github-actions-buildx-demo.yml` now captures a real SBOM via `docker sbom`. + +## Running locally + +```bash +dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj -c Release -o out/buildx + +dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll handshake \ + --manifest out/buildx \ + --cas out/cas + +docker buildx build --load -t stellaops/buildx-demo:ci samples/ci/buildx-demo +DIGEST=$(docker image inspect stellaops/buildx-demo:ci --format '{{index .RepoDigests 0}}') + +docker sbom stellaops/buildx-demo:ci --format cyclonedx-json > out/buildx-sbom.cdx.json + +dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image "$DIGEST" \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + > out/buildx-descriptor.json +``` + +The descriptor JSON contains deterministic annotations and provenance placeholders ready for the Attestor. diff --git a/samples/ci/buildx-demo/github-actions-buildx-demo.yml b/samples/ci/buildx-demo/github-actions-buildx-demo.yml new file mode 100644 index 00000000..c79a08ab --- /dev/null +++ b/samples/ci/buildx-demo/github-actions-buildx-demo.yml @@ -0,0 +1,153 @@ +name: Buildx SBOM Demo +on: + workflow_dispatch: + push: + branches: [ demo/buildx ] + +jobs: + buildx-sbom: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up .NET 10 preview + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + - name: Publish StellaOps BuildX generator + run: | + dotnet publish src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj \ + -c Release \ + -o out/buildx + + - name: Handshake CAS + run: | + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll handshake \ + --manifest out/buildx \ + --cas out/cas + + - name: Build demo container image + run: | + docker buildx build --load -t stellaops/buildx-demo:ci samples/ci/buildx-demo + + - name: Capture image digest + id: digest + run: | + DIGEST=$(docker image inspect stellaops/buildx-demo:ci --format '{{index .RepoDigests 0}}') + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + + - name: Generate SBOM from built image + run: | + mkdir -p out + docker sbom stellaops/buildx-demo:ci --format cyclonedx-json > out/buildx-sbom.cdx.json + + - name: Start mock Attestor + id: attestor + run: | + mkdir -p out + cat <<'PY' > out/mock-attestor.py +import json +import os +from http.server import BaseHTTPRequestHandler, HTTPServer + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get('Content-Length') or 0) + body = self.rfile.read(length) + with open(os.path.join('out', 'provenance-request.json'), 'wb') as fp: + fp.write(body) + self.send_response(202) + self.end_headers() + self.wfile.write(b'accepted') + + def log_message(self, format, *args): + return + +if __name__ == '__main__': + server = HTTPServer(('127.0.0.1', 8085), Handler) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() +PY + touch out/provenance-request.json + python3 out/mock-attestor.py & + echo $! > out/mock-attestor.pid + + - name: Emit descriptor with provenance placeholder + env: + IMAGE_DIGEST: ${{ steps.digest.outputs.digest }} + # Uncomment the next line and remove the mock Attestor block to hit a real service. + # STELLAOPS_ATTESTOR_TOKEN: ${{ secrets.STELLAOPS_ATTESTOR_TOKEN }} + run: | + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image "$IMAGE_DIGEST" \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + --artifact-type application/vnd.stellaops.sbom.layer+json \ + --sbom-format cyclonedx-json \ + --sbom-kind inventory \ + --repository ${{ github.repository }} \ + --build-ref ${{ github.sha }} \ + --attestor http://127.0.0.1:8085/provenance \ + > out/buildx-descriptor.json + + - name: Verify descriptor determinism + env: + IMAGE_DIGEST: ${{ steps.digest.outputs.digest }} + run: | + dotnet out/buildx/StellaOps.Scanner.Sbomer.BuildXPlugin.dll descriptor \ + --manifest out/buildx \ + --image "$IMAGE_DIGEST" \ + --sbom out/buildx-sbom.cdx.json \ + --sbom-name buildx-sbom.cdx.json \ + --artifact-type application/vnd.stellaops.sbom.layer+json \ + --sbom-format cyclonedx-json \ + --sbom-kind inventory \ + --repository ${{ github.repository }} \ + --build-ref ${{ github.sha }} \ + > out/buildx-descriptor-repeat.json + + python - <<'PY' +import json + +def normalize(path: str) -> dict: + with open(path, 'r', encoding='utf-8') as handle: + data = json.load(handle) + data.pop('generatedAt', None) + return data + +baseline = normalize('out/buildx-descriptor.json') +repeat = normalize('out/buildx-descriptor-repeat.json') + +if baseline != repeat: + raise SystemExit('Descriptor output changed between runs.') +PY + + - name: Stop mock Attestor + if: always() + run: | + if [ -f out/mock-attestor.pid ]; then + kill $(cat out/mock-attestor.pid) + fi + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: stellaops-buildx-demo + path: | + out/buildx-descriptor.json + out/buildx-sbom.cdx.json + out/provenance-request.json + out/buildx-descriptor-repeat.json + + - name: Show descriptor summary + run: | + cat out/buildx-descriptor.json diff --git a/samples/policy/policy-preview-unknown.json b/samples/policy/policy-preview-unknown.json new file mode 100644 index 00000000..b9e3ae3e --- /dev/null +++ b/samples/policy/policy-preview-unknown.json @@ -0,0 +1,98 @@ +{ + "previewRequest": { + "imageDigest": "sha256:7dbe0c9a5d4f1c8184007e9d94dbe55928f8a2db5ab9c1c2d4a2f7bbcdfe1234", + "findings": [ + { + "id": "library:pkg/openssl@1.1.1w", + "severity": "Unknown", + "source": "NVD", + "tags": [ + "trust:vendor", + "reachability:unknown", + "unknown-age-days:5" + ] + }, + { + "id": "library:pkg/zlib@1.3.1", + "severity": "High", + "source": "NVD", + "tags": [ + "state:unknown", + "reachability:runtime", + "unknown-since:2025-10-10T00:00:00Z", + "observed-at:2025-10-19T12:00:00Z" + ] + } + ] + }, + "previewResponse": { + "success": true, + "policyDigest": "8a0f72f8dc5c51c46991db3bba34e9b3c0c8e944a7a6d0a9c29a9aa6b8439876", + "revisionId": "rev-42", + "changed": 2, + "diffs": [ + { + "findingId": "library:pkg/openssl@1.1.1w", + "baseline": { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Pass", + "score": 0, + "configVersion": "1.0" + }, + "projected": { + "findingId": "library:pkg/openssl@1.1.1w", + "status": "Blocked", + "ruleName": "Block vendor unknowns", + "ruleAction": "block", + "score": 19.5, + "configVersion": "1.0", + "inputs": { + "severityWeight": 50, + "trustWeight": 0.65, + "reachabilityWeight": 0.6, + "baseScore": 19.5, + "trustWeight.vendor": 0.65, + "reachability.unknown": 0.6, + "unknownConfidence": 0.55, + "unknownAgeDays": 5 + }, + "unknownConfidence": 0.55, + "confidenceBand": "medium", + "unknownAgeDays": 5 + }, + "changed": true + }, + { + "findingId": "library:pkg/zlib@1.3.1", + "baseline": { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Pass", + "score": 0, + "configVersion": "1.0" + }, + "projected": { + "findingId": "library:pkg/zlib@1.3.1", + "status": "Warned", + "ruleName": "Runtime mitigation required", + "ruleAction": "warn", + "score": 33.75, + "configVersion": "1.0", + "inputs": { + "severityWeight": 75, + "trustWeight": 1, + "reachabilityWeight": 0.45, + "baseScore": 33.75, + "reachability.runtime": 0.45, + "warnPenalty": 15, + "unknownConfidence": 0.35, + "unknownAgeDays": 9 + }, + "unknownConfidence": 0.35, + "confidenceBand": "medium", + "unknownAgeDays": 9 + }, + "changed": true + } + ] + } +} diff --git a/samples/runtime/README.md b/samples/runtime/README.md new file mode 100644 index 00000000..487205d4 --- /dev/null +++ b/samples/runtime/README.md @@ -0,0 +1,6 @@ +# Runtime Fixtures + +Supporting filesystem snippets consumed by analyzer microbenchmarks and integration tests. They are intentionally lightweight yet deterministic so they can be committed to the repository without bloating history. + +- `npm-monorepo/` – trimmed `node_modules/` tree for workspace-style Node.js projects. +- `python-venv/` – selected `site-packages/` entries highlighting `*.dist-info` metadata. diff --git a/samples/runtime/npm-monorepo/README.md b/samples/runtime/npm-monorepo/README.md new file mode 100644 index 00000000..ad04f617 --- /dev/null +++ b/samples/runtime/npm-monorepo/README.md @@ -0,0 +1,9 @@ +# NPM Monorepo Fixture + +This fixture represents a trimmed monorepo layout used by the analyzer microbench. It contains four packages under `node_modules/` with realistic metadata and dependency edges. + +- `@stella/core` depends on `lodash`. +- `@stella/web` depends on `@stella/core` and `rxjs`. +- Third-party packages expose standard `name`, `version`, and `license` fields. + +The files are intentionally small so that the bench harness focuses on directory traversal and metadata parsing overhead. diff --git a/samples/runtime/npm-monorepo/node_modules/@stella/core/package.json b/samples/runtime/npm-monorepo/node_modules/@stella/core/package.json new file mode 100644 index 00000000..57f99309 --- /dev/null +++ b/samples/runtime/npm-monorepo/node_modules/@stella/core/package.json @@ -0,0 +1,9 @@ +{ + "name": "@stella/core", + "version": "2.0.0", + "description": "Core services shared by the sample monorepo.", + "dependencies": { + "lodash": "4.17.21" + }, + "license": "Apache-2.0" +} diff --git a/samples/runtime/npm-monorepo/node_modules/@stella/web/package.json b/samples/runtime/npm-monorepo/node_modules/@stella/web/package.json new file mode 100644 index 00000000..09c68556 --- /dev/null +++ b/samples/runtime/npm-monorepo/node_modules/@stella/web/package.json @@ -0,0 +1,10 @@ +{ + "name": "@stella/web", + "version": "1.5.3", + "description": "Web layer in the sample monorepo.", + "dependencies": { + "@stella/core": "2.0.0", + "rxjs": "7.8.1" + }, + "license": "MIT" +} diff --git a/samples/runtime/npm-monorepo/node_modules/lodash/package.json b/samples/runtime/npm-monorepo/node_modules/lodash/package.json new file mode 100644 index 00000000..90c5403b --- /dev/null +++ b/samples/runtime/npm-monorepo/node_modules/lodash/package.json @@ -0,0 +1,6 @@ +{ + "name": "lodash", + "version": "4.17.21", + "description": "Lodash modular utilities.", + "license": "MIT" +} diff --git a/samples/runtime/npm-monorepo/node_modules/rxjs/package.json b/samples/runtime/npm-monorepo/node_modules/rxjs/package.json new file mode 100644 index 00000000..34c367e0 --- /dev/null +++ b/samples/runtime/npm-monorepo/node_modules/rxjs/package.json @@ -0,0 +1,6 @@ +{ + "name": "rxjs", + "version": "7.8.1", + "description": "Reactive Extensions for modern JavaScript.", + "license": "Apache-2.0" +} diff --git a/samples/runtime/python-venv/README.md b/samples/runtime/python-venv/README.md new file mode 100644 index 00000000..71bd490e --- /dev/null +++ b/samples/runtime/python-venv/README.md @@ -0,0 +1,5 @@ +# Python Virtual Environment Fixture + +The fixture mimics a trimmed `site-packages/` layout with three common dependencies (`requests`, `urllib3`, `certifi`). Each package exposes a `*.dist-info/METADATA` file so the analyzer bench can validate parsing performance and header extraction. + +Files intentionally omit wheels and bytecode to keep the tree compact while still realistic. diff --git a/samples/runtime/python-venv/lib/python3.11/site-packages/certifi-2024.6.2.dist-info/METADATA b/samples/runtime/python-venv/lib/python3.11/site-packages/certifi-2024.6.2.dist-info/METADATA new file mode 100644 index 00000000..3fb68afd --- /dev/null +++ b/samples/runtime/python-venv/lib/python3.11/site-packages/certifi-2024.6.2.dist-info/METADATA @@ -0,0 +1,5 @@ +Metadata-Version: 2.1 +Name: certifi +Version: 2024.6.2 +Summary: Mozilla SSL Certificates. +License: MPL-2.0 diff --git a/samples/runtime/python-venv/lib/python3.11/site-packages/requests-2.32.0.dist-info/METADATA b/samples/runtime/python-venv/lib/python3.11/site-packages/requests-2.32.0.dist-info/METADATA new file mode 100644 index 00000000..b5003e46 --- /dev/null +++ b/samples/runtime/python-venv/lib/python3.11/site-packages/requests-2.32.0.dist-info/METADATA @@ -0,0 +1,7 @@ +Metadata-Version: 2.1 +Name: requests +Version: 2.32.0 +Summary: Python HTTP for Humans. +License: Apache-2.0 +Requires-Dist: urllib3 (>=1.21.1,<3) +Requires-Dist: certifi (>=2017.4.17) diff --git a/samples/runtime/python-venv/lib/python3.11/site-packages/urllib3-2.2.1.dist-info/METADATA b/samples/runtime/python-venv/lib/python3.11/site-packages/urllib3-2.2.1.dist-info/METADATA new file mode 100644 index 00000000..79dd62bf --- /dev/null +++ b/samples/runtime/python-venv/lib/python3.11/site-packages/urllib3-2.2.1.dist-info/METADATA @@ -0,0 +1,5 @@ +Metadata-Version: 2.1 +Name: urllib3 +Version: 2.2.1 +Summary: HTTP library with thread-safe connection pooling. +License: MIT diff --git a/samples/scanner/README.md b/samples/scanner/README.md new file mode 100644 index 00000000..dea8197c --- /dev/null +++ b/samples/scanner/README.md @@ -0,0 +1,12 @@ +# Scanner Samples + +Curated SBOM and BOM Index fixtures covering representative container types referenced throughout Sprint 10. Each sample folder under `images/` corresponds to a container profile, while `../runtime` holds trimmed filesystem fixtures used by analyzer and perf tests. + +| Sample | Highlights | +| ------ | ---------- | +| `nginx` | Alpine packages with mixed inventory/runtime coverage. | +| `alpine-busybox` | Minimal BusyBox rootfs with musl runtime linkage. | +| `distroless-go` | Go binary with Distroless base and Go build-info evidence. | +| `dotnet-aot` | Ahead-of-time compiled .NET worker exposing NuGet dependencies. | +| `python-venv` | Python virtualenv with `*.dist-info` evidence. | +| `npm-monorepo` | Node workspace packages resolved via `package.json`. | diff --git a/samples/scanner/images/alpine-busybox/README.md b/samples/scanner/images/alpine-busybox/README.md new file mode 100644 index 00000000..7d88123e --- /dev/null +++ b/samples/scanner/images/alpine-busybox/README.md @@ -0,0 +1,3 @@ +# Alpine + BusyBox Sample + +Fixtures showcase the tiny Alpine image that powers many minimal containers. BusyBox and musl appear in usage because they back the entrypoint shell, while alpine-baselayout remains inventory-only. diff --git a/samples/scanner/images/alpine-busybox/bom-index.json b/samples/scanner/images/alpine-busybox/bom-index.json new file mode 100644 index 00000000..5d989caa --- /dev/null +++ b/samples/scanner/images/alpine-busybox/bom-index.json @@ -0,0 +1,42 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "docker.io/library/alpine", + "digest": "sha256:9a214327ec7df5bc8f1d3f12171873be7d778fdbf473d6f9a63d5de6c6bfb2d3", + "tag": "3.20" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:apk/alpine/busybox@1.36.1-r2?arch=x86_64", + "layerDigest": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "usage": ["inventory", "runtime"], + "licenses": ["GPL-2.0-only"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + }, + { + "purl": "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64", + "layerDigest": "sha256:6666666666666666666666666666666666666666666666666666666666666666", + "usage": ["inventory", "runtime"], + "licenses": ["MIT"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + }, + { + "purl": "pkg:apk/alpine/alpine-baselayout@3.4.3-r0?arch=x86_64", + "layerDigest": "sha256:7777777777777777777777777777777777777777777777777777777777777777", + "usage": ["inventory"], + "licenses": ["GPL-2.0-only"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + } + ] +} diff --git a/samples/scanner/images/alpine-busybox/inventory.cdx.json b/samples/scanner/images/alpine-busybox/inventory.cdx.json new file mode 100644 index 00000000..35177625 --- /dev/null +++ b/samples/scanner/images/alpine-busybox/inventory.cdx.json @@ -0,0 +1,34 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "alpine-busybox", + "version": "3.20", + "bomRef": "pkg:docker/library/alpine@sha256:9a214327ec7df5bc8f1d3f12171873be7d778fdbf473d6f9a63d5de6c6bfb2d3" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:apk/alpine/busybox@1.36.1-r2?arch=x86_64", + "name": "busybox", + "version": "1.36.1-r2" + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64", + "name": "musl", + "version": "1.2.5-r0" + }, + { + "type": "application", + "bomRef": "pkg:apk/alpine/alpine-baselayout@3.4.3-r0?arch=x86_64", + "name": "alpine-baselayout", + "version": "3.4.3-r0" + } + ] +} diff --git a/samples/scanner/images/alpine-busybox/usage.cdx.json b/samples/scanner/images/alpine-busybox/usage.cdx.json new file mode 100644 index 00000000..73db0d7d --- /dev/null +++ b/samples/scanner/images/alpine-busybox/usage.cdx.json @@ -0,0 +1,28 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "alpine-busybox", + "version": "3.20", + "bomRef": "pkg:docker/library/alpine@sha256:9a214327ec7df5bc8f1d3f12171873be7d778fdbf473d6f9a63d5de6c6bfb2d3" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:apk/alpine/busybox@1.36.1-r2?arch=x86_64", + "name": "busybox", + "version": "1.36.1-r2" + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/musl@1.2.5-r0?arch=x86_64", + "name": "musl", + "version": "1.2.5-r0" + } + ] +} diff --git a/samples/scanner/images/distroless-go/README.md b/samples/scanner/images/distroless-go/README.md new file mode 100644 index 00000000..db740b3f --- /dev/null +++ b/samples/scanner/images/distroless-go/README.md @@ -0,0 +1,3 @@ +# Distroless + Go Sample + +Demonstrates a Go binary shipped on top of Distroless. Only the compiled service appears in the usage SBOM, while the Go standard library remains inventory-only and still tracked in the BOM Index. diff --git a/samples/scanner/images/distroless-go/bom-index.json b/samples/scanner/images/distroless-go/bom-index.json new file mode 100644 index 00000000..3bd9f1df --- /dev/null +++ b/samples/scanner/images/distroless-go/bom-index.json @@ -0,0 +1,32 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "gcr.io/distroless/base", + "digest": "sha256:0dd2f0f15c9f8abfba6a0ce0d7d6a24e2e1071c977733f6e77cbe51b87f15ad9", + "tag": "nonroot" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:golang/github.com/stellaops/sample-service@v1.4.0", + "layerDigest": "sha256:8888888888888888888888888888888888888888888888888888888888888888", + "usage": ["inventory", "runtime"], + "licenses": ["Apache-2.0"], + "evidence": { + "kind": "go-buildinfo", + "path": "/workspace/service" + } + }, + { + "purl": "pkg:golang/std@go1.22.5", + "layerDigest": "sha256:9999999999999999999999999999999999999999999999999999999999999999", + "usage": ["inventory"], + "licenses": ["BSD-3-Clause"], + "evidence": { + "kind": "go-buildinfo", + "path": "/workspace/service" + } + } + ] +} diff --git a/samples/scanner/images/distroless-go/inventory.cdx.json b/samples/scanner/images/distroless-go/inventory.cdx.json new file mode 100644 index 00000000..fd365323 --- /dev/null +++ b/samples/scanner/images/distroless-go/inventory.cdx.json @@ -0,0 +1,34 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "distroless-go", + "version": "2025.10.0", + "bomRef": "pkg:docker/gcr.io/distroless/base@sha256:0dd2f0f15c9f8abfba6a0ce0d7d6a24e2e1071c977733f6e77cbe51b87f15ad9" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:golang/github.com/stellaops/sample-service@v1.4.0", + "name": "github.com/stellaops/sample-service", + "version": "v1.4.0", + "properties": [ + { + "name": "stellaops.entrypoint", + "value": "/workspace/service" + } + ] + }, + { + "type": "library", + "bomRef": "pkg:golang/std@go1.22.5", + "name": "golang-stdlib", + "version": "go1.22.5" + } + ] +} diff --git a/samples/scanner/images/distroless-go/usage.cdx.json b/samples/scanner/images/distroless-go/usage.cdx.json new file mode 100644 index 00000000..df9faf9c --- /dev/null +++ b/samples/scanner/images/distroless-go/usage.cdx.json @@ -0,0 +1,22 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "distroless-go", + "version": "2025.10.0", + "bomRef": "pkg:docker/gcr.io/distroless/base@sha256:0dd2f0f15c9f8abfba6a0ce0d7d6a24e2e1071c977733f6e77cbe51b87f15ad9" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:golang/github.com/stellaops/sample-service@v1.4.0", + "name": "github.com/stellaops/sample-service", + "version": "v1.4.0" + } + ] +} diff --git a/samples/scanner/images/dotnet-aot/README.md b/samples/scanner/images/dotnet-aot/README.md new file mode 100644 index 00000000..ad6a57ea --- /dev/null +++ b/samples/scanner/images/dotnet-aot/README.md @@ -0,0 +1,3 @@ +# .NET AOT Sample + +An ahead-of-time compiled worker showcasing how native .NET deployments appear in SBOM outputs. The BOM Index ties NuGet packages back to the generated `deps.json` evidence. diff --git a/samples/scanner/images/dotnet-aot/bom-index.json b/samples/scanner/images/dotnet-aot/bom-index.json new file mode 100644 index 00000000..7e72ba00 --- /dev/null +++ b/samples/scanner/images/dotnet-aot/bom-index.json @@ -0,0 +1,52 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "registry.stella-ops.org/sample/dotnet-aot", + "digest": "sha256:5be6f3ad9d2b1e4fcb4c6f40d9c664fca97f5b4d9ccb8e1d8f970e8b2bce1123", + "tag": "1.0.0" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:nuget/Sample.Worker@1.0.0", + "layerDigest": "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "usage": ["inventory", "runtime"], + "licenses": ["MIT"], + "evidence": { + "kind": "deps-json", + "path": "/app/Sample.Worker.deps.json" + } + }, + { + "purl": "pkg:nuget/Microsoft.Extensions.Hosting@8.0.0", + "layerDigest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "usage": ["inventory"], + "licenses": ["MIT"], + "evidence": { + "kind": "deps-json", + "path": "/app/Sample.Worker.deps.json" + } + }, + { + "purl": "pkg:nuget/System.Text.Json@8.0.0", + "layerDigest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "usage": ["inventory", "runtime"], + "licenses": ["MIT"], + "evidence": { + "kind": "deps-json", + "path": "/app/Sample.Worker.deps.json" + } + }, + { + "purl": "pkg:nuget/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross@8.0.0", + "layerDigest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "usage": ["inventory"], + "licenses": ["MIT"], + "evidence": { + "kind": "deps-json", + "path": "/app/Sample.Worker.deps.json" + } + } + ] +} diff --git a/samples/scanner/images/dotnet-aot/inventory.cdx.json b/samples/scanner/images/dotnet-aot/inventory.cdx.json new file mode 100644 index 00000000..e57c3af7 --- /dev/null +++ b/samples/scanner/images/dotnet-aot/inventory.cdx.json @@ -0,0 +1,40 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "dotnet-aot", + "version": "8.0.0", + "bomRef": "pkg:docker/stellaops/sample-dotnet-aot@sha256:5be6f3ad9d2b1e4fcb4c6f40d9c664fca97f5b4d9ccb8e1d8f970e8b2bce1123" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:nuget/Sample.Worker@1.0.0", + "name": "Sample.Worker", + "version": "1.0.0" + }, + { + "type": "library", + "bomRef": "pkg:nuget/Microsoft.Extensions.Hosting@8.0.0", + "name": "Microsoft.Extensions.Hosting", + "version": "8.0.0" + }, + { + "type": "library", + "bomRef": "pkg:nuget/System.Text.Json@8.0.0", + "name": "System.Text.Json", + "version": "8.0.0" + }, + { + "type": "library", + "bomRef": "pkg:nuget/Microsoft.NETCore.App.Runtime.AOT.win-x64.Cross@8.0.0", + "name": "NativeAotRuntime", + "version": "8.0.0" + } + ] +} diff --git a/samples/scanner/images/dotnet-aot/usage.cdx.json b/samples/scanner/images/dotnet-aot/usage.cdx.json new file mode 100644 index 00000000..2007db8f --- /dev/null +++ b/samples/scanner/images/dotnet-aot/usage.cdx.json @@ -0,0 +1,28 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "dotnet-aot", + "version": "8.0.0", + "bomRef": "pkg:docker/stellaops/sample-dotnet-aot@sha256:5be6f3ad9d2b1e4fcb4c6f40d9c664fca97f5b4d9ccb8e1d8f970e8b2bce1123" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:nuget/Sample.Worker@1.0.0", + "name": "Sample.Worker", + "version": "1.0.0" + }, + { + "type": "library", + "bomRef": "pkg:nuget/System.Text.Json@8.0.0", + "name": "System.Text.Json", + "version": "8.0.0" + } + ] +} diff --git a/samples/scanner/images/nginx/README.md b/samples/scanner/images/nginx/README.md new file mode 100644 index 00000000..87a3cee3 --- /dev/null +++ b/samples/scanner/images/nginx/README.md @@ -0,0 +1,3 @@ +# Nginx Inventory Sample + +CycloneDX inventory, usage, and BOM Index fixtures for the `docker.io/library/nginx:1.25.4` image. The SBOMs capture base Alpine packages and the BOM Index links each component to the layer that introduced it. diff --git a/samples/scanner/images/nginx/bom-index.json b/samples/scanner/images/nginx/bom-index.json new file mode 100644 index 00000000..7b81faa9 --- /dev/null +++ b/samples/scanner/images/nginx/bom-index.json @@ -0,0 +1,52 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "docker.io/library/nginx", + "digest": "sha256:8f47d7c6b538c0d9533b78913cba3d5e671e7c4b4e7c6a2bb9a1a1c4d4f8e123", + "tag": "1.25.4" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:apk/alpine/nginx@1.25.4-r1?arch=x86_64", + "layerDigest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "usage": ["inventory", "runtime"], + "licenses": ["BSD-2-Clause"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + }, + { + "purl": "pkg:apk/alpine/openssl@3.2.2-r0?arch=x86_64", + "layerDigest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "usage": ["inventory", "runtime"], + "licenses": ["Apache-2.0"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + }, + { + "purl": "pkg:apk/alpine/pcre2@10.42-r1?arch=x86_64", + "layerDigest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "usage": ["inventory"], + "licenses": ["BSD-3-Clause"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + }, + { + "purl": "pkg:apk/alpine/zlib@1.3-r2?arch=x86_64", + "layerDigest": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "usage": ["inventory"], + "licenses": ["Zlib"], + "evidence": { + "kind": "apk-database", + "path": "/lib/apk/db/installed" + } + } + ] +} diff --git a/samples/scanner/images/nginx/inventory.cdx.json b/samples/scanner/images/nginx/inventory.cdx.json new file mode 100644 index 00000000..ecfbf43a --- /dev/null +++ b/samples/scanner/images/nginx/inventory.cdx.json @@ -0,0 +1,53 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "nginx", + "version": "1.25.4", + "bomRef": "pkg:docker/library/nginx@sha256:8f47d7c6b538c0d9533b78913cba3d5e671e7c4b4e7c6a2bb9a1a1c4d4f8e123" + }, + "tools": [ + { + "name": "StellaOps Scanner", + "version": "10.0.0-preview1" + } + ] + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:apk/alpine/nginx@1.25.4-r1?arch=x86_64", + "name": "nginx", + "version": "1.25.4-r1", + "licenses": [ + { + "license": { + "id": "2BSD" + } + } + ] + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/openssl@3.2.2-r0?arch=x86_64", + "name": "openssl", + "version": "3.2.2-r0" + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/pcre2@10.42-r1?arch=x86_64", + "name": "pcre2", + "version": "10.42-r1" + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/zlib@1.3-r2?arch=x86_64", + "name": "zlib", + "version": "1.3-r2" + } + ] +} diff --git a/samples/scanner/images/nginx/usage.cdx.json b/samples/scanner/images/nginx/usage.cdx.json new file mode 100644 index 00000000..f01468ae --- /dev/null +++ b/samples/scanner/images/nginx/usage.cdx.json @@ -0,0 +1,28 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "nginx", + "version": "1.25.4", + "bomRef": "pkg:docker/library/nginx@sha256:8f47d7c6b538c0d9533b78913cba3d5e671e7c4b4e7c6a2bb9a1a1c4d4f8e123" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:apk/alpine/nginx@1.25.4-r1?arch=x86_64", + "name": "nginx", + "version": "1.25.4-r1" + }, + { + "type": "library", + "bomRef": "pkg:apk/alpine/openssl@3.2.2-r0?arch=x86_64", + "name": "openssl", + "version": "3.2.2-r0" + } + ] +} diff --git a/samples/scanner/images/npm-monorepo/README.md b/samples/scanner/images/npm-monorepo/README.md new file mode 100644 index 00000000..d82ca65f --- /dev/null +++ b/samples/scanner/images/npm-monorepo/README.md @@ -0,0 +1,3 @@ +# NPM Monorepo Sample + +Mirrors the fixture under `samples/runtime/npm-monorepo`. The SBOMs highlight the workspace packages plus transitive dependencies, and the BOM Index pins evidence to individual `package.json` files. diff --git a/samples/scanner/images/npm-monorepo/bom-index.json b/samples/scanner/images/npm-monorepo/bom-index.json new file mode 100644 index 00000000..0c6b066f --- /dev/null +++ b/samples/scanner/images/npm-monorepo/bom-index.json @@ -0,0 +1,52 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "registry.stella-ops.org/samples/npm-monorepo", + "digest": "sha256:1cf2ab9d373086ed5bd1a8f4aa6f491f8844bbb0d6be8df449c16ad6c8fa7c55", + "tag": "2025.10.0" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:npm/%40stella/web@1.5.3", + "layerDigest": "sha256:1212121212121212121212121212121212121212121212121212121212121212", + "usage": ["inventory", "runtime"], + "licenses": ["MIT"], + "evidence": { + "kind": "package-json", + "path": "node_modules/@stella/web/package.json" + } + }, + { + "purl": "pkg:npm/%40stella/core@2.0.0", + "layerDigest": "sha256:1313131313131313131313131313131313131313131313131313131313131313", + "usage": ["inventory", "runtime"], + "licenses": ["Apache-2.0"], + "evidence": { + "kind": "package-json", + "path": "node_modules/@stella/core/package.json" + } + }, + { + "purl": "pkg:npm/lodash@4.17.21", + "layerDigest": "sha256:1414141414141414141414141414141414141414141414141414141414141414", + "usage": ["inventory"], + "licenses": ["MIT"], + "evidence": { + "kind": "package-json", + "path": "node_modules/lodash/package.json" + } + }, + { + "purl": "pkg:npm/rxjs@7.8.1", + "layerDigest": "sha256:1515151515151515151515151515151515151515151515151515151515151515", + "usage": ["inventory", "runtime"], + "licenses": ["Apache-2.0"], + "evidence": { + "kind": "package-json", + "path": "node_modules/rxjs/package.json" + } + } + ] +} diff --git a/samples/scanner/images/npm-monorepo/inventory.cdx.json b/samples/scanner/images/npm-monorepo/inventory.cdx.json new file mode 100644 index 00000000..aeeae21a --- /dev/null +++ b/samples/scanner/images/npm-monorepo/inventory.cdx.json @@ -0,0 +1,40 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "npm-monorepo", + "version": "2025.10.0", + "bomRef": "pkg:docker/registry.stella-ops.org/samples/npm-monorepo@sha256:1cf2ab9d373086ed5bd1a8f4aa6f491f8844bbb0d6be8df449c16ad6c8fa7c55" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:npm/%40stella/core@2.0.0", + "name": "@stella/core", + "version": "2.0.0" + }, + { + "type": "application", + "bomRef": "pkg:npm/%40stella/web@1.5.3", + "name": "@stella/web", + "version": "1.5.3" + }, + { + "type": "library", + "bomRef": "pkg:npm/lodash@4.17.21", + "name": "lodash", + "version": "4.17.21" + }, + { + "type": "library", + "bomRef": "pkg:npm/rxjs@7.8.1", + "name": "rxjs", + "version": "7.8.1" + } + ] +} diff --git a/samples/scanner/images/npm-monorepo/usage.cdx.json b/samples/scanner/images/npm-monorepo/usage.cdx.json new file mode 100644 index 00000000..a2affd11 --- /dev/null +++ b/samples/scanner/images/npm-monorepo/usage.cdx.json @@ -0,0 +1,34 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "npm-monorepo", + "version": "2025.10.0", + "bomRef": "pkg:docker/registry.stella-ops.org/samples/npm-monorepo@sha256:1cf2ab9d373086ed5bd1a8f4aa6f491f8844bbb0d6be8df449c16ad6c8fa7c55" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:npm/%40stella/web@1.5.3", + "name": "@stella/web", + "version": "1.5.3" + }, + { + "type": "application", + "bomRef": "pkg:npm/%40stella/core@2.0.0", + "name": "@stella/core", + "version": "2.0.0" + }, + { + "type": "library", + "bomRef": "pkg:npm/rxjs@7.8.1", + "name": "rxjs", + "version": "7.8.1" + } + ] +} diff --git a/samples/scanner/images/python-venv/README.md b/samples/scanner/images/python-venv/README.md new file mode 100644 index 00000000..df550376 --- /dev/null +++ b/samples/scanner/images/python-venv/README.md @@ -0,0 +1,3 @@ +# Python Virtualenv Sample + +Pairs with the runtime fixture under `samples/runtime/python-venv`. The SBOMs highlight how requests pulls in urllib3 and certifi, and the BOM Index records the `*.dist-info/METADATA` evidence paths used by the Python analyzer. diff --git a/samples/scanner/images/python-venv/bom-index.json b/samples/scanner/images/python-venv/bom-index.json new file mode 100644 index 00000000..755e2d4d --- /dev/null +++ b/samples/scanner/images/python-venv/bom-index.json @@ -0,0 +1,42 @@ +{ + "schema": "stellaops/bom-index@1", + "image": { + "repository": "docker.io/library/python", + "digest": "sha256:dbed08b7d9675c2be627bbecac182a04c36d3f4ffd542c4fba7c7a850a6578dc", + "tag": "3.12-slim" + }, + "generatedAt": "2025-10-19T00:00:00Z", + "generator": "stellaops/scanner@10.0.0-preview1", + "components": [ + { + "purl": "pkg:pypi/requests@2.32.0", + "layerDigest": "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + "usage": ["inventory", "runtime"], + "licenses": ["Apache-2.0"], + "evidence": { + "kind": "dist-info", + "path": "lib/python3.11/site-packages/requests-2.32.0.dist-info/METADATA" + } + }, + { + "purl": "pkg:pypi/urllib3@2.2.1", + "layerDigest": "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "usage": ["inventory", "runtime"], + "licenses": ["MIT"], + "evidence": { + "kind": "dist-info", + "path": "lib/python3.11/site-packages/urllib3-2.2.1.dist-info/METADATA" + } + }, + { + "purl": "pkg:pypi/certifi@2024.6.2", + "layerDigest": "sha256:0000000000000000000000000000000000000000000000000000000000000000", + "usage": ["inventory"], + "licenses": ["MPL-2.0"], + "evidence": { + "kind": "dist-info", + "path": "lib/python3.11/site-packages/certifi-2024.6.2.dist-info/METADATA" + } + } + ] +} diff --git a/samples/scanner/images/python-venv/inventory.cdx.json b/samples/scanner/images/python-venv/inventory.cdx.json new file mode 100644 index 00000000..e00f5a4f --- /dev/null +++ b/samples/scanner/images/python-venv/inventory.cdx.json @@ -0,0 +1,34 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "python-venv", + "version": "3.12-slim", + "bomRef": "pkg:docker/library/python@sha256:dbed08b7d9675c2be627bbecac182a04c36d3f4ffd542c4fba7c7a850a6578dc" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:pypi/requests@2.32.0", + "name": "requests", + "version": "2.32.0" + }, + { + "type": "library", + "bomRef": "pkg:pypi/urllib3@2.2.1", + "name": "urllib3", + "version": "2.2.1" + }, + { + "type": "library", + "bomRef": "pkg:pypi/certifi@2024.6.2", + "name": "certifi", + "version": "2024.6.2" + } + ] +} diff --git a/samples/scanner/images/python-venv/usage.cdx.json b/samples/scanner/images/python-venv/usage.cdx.json new file mode 100644 index 00000000..ed10b73c --- /dev/null +++ b/samples/scanner/images/python-venv/usage.cdx.json @@ -0,0 +1,28 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "timestamp": "2025-10-19T00:00:00Z", + "component": { + "type": "container", + "name": "python-venv", + "version": "3.12-slim", + "bomRef": "pkg:docker/library/python@sha256:dbed08b7d9675c2be627bbecac182a04c36d3f4ffd542c4fba7c7a850a6578dc" + } + }, + "components": [ + { + "type": "application", + "bomRef": "pkg:pypi/requests@2.32.0", + "name": "requests", + "version": "2.32.0" + }, + { + "type": "library", + "bomRef": "pkg:pypi/urllib3@2.2.1", + "name": "urllib3", + "version": "2.2.1" + } + ] +} diff --git a/scripts/update-apple-fixtures.ps1 b/scripts/update-apple-fixtures.ps1 index 34e74881..63cdf4b8 100644 --- a/scripts/update-apple-fixtures.ps1 +++ b/scripts/update-apple-fixtures.ps1 @@ -1,19 +1,19 @@ -#!/usr/bin/env pwsh -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" - -$rootDir = Split-Path -Parent $PSCommandPath -$rootDir = Join-Path $rootDir ".." -$rootDir = Resolve-Path $rootDir - -$env:UPDATE_APPLE_FIXTURES = "1" - -Push-Location $rootDir -try { - $sentinel = Join-Path $rootDir "src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures" - New-Item -ItemType File -Path $sentinel -Force | Out-Null - dotnet test "src\StellaOps.Feedser.Source.Vndr.Apple.Tests\StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj" @Args -} -finally { - Pop-Location -} +#!/usr/bin/env pwsh +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$rootDir = Split-Path -Parent $PSCommandPath +$rootDir = Join-Path $rootDir ".." +$rootDir = Resolve-Path $rootDir + +$env:UPDATE_APPLE_FIXTURES = "1" + +Push-Location $rootDir +try { + $sentinel = Join-Path $rootDir "src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures" + New-Item -ItemType File -Path $sentinel -Force | Out-Null + dotnet test "src\StellaOps.Concelier.Connector.Vndr.Apple.Tests\StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj" @Args +} +finally { + Pop-Location +} diff --git a/scripts/update-apple-fixtures.sh b/scripts/update-apple-fixtures.sh index 60ebc818..96008c98 100644 --- a/scripts/update-apple-fixtures.sh +++ b/scripts/update-apple-fixtures.sh @@ -1,14 +1,14 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -export UPDATE_APPLE_FIXTURES=1 -if [ -n "${WSLENV-}" ]; then - export WSLENV="${WSLENV}:UPDATE_APPLE_FIXTURES/up" -else - export WSLENV="UPDATE_APPLE_FIXTURES/up" -fi - -touch "$ROOT_DIR/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures" -( cd "$ROOT_DIR" && dotnet test "src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj" "$@" ) +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +export UPDATE_APPLE_FIXTURES=1 +if [ -n "${WSLENV-}" ]; then + export WSLENV="${WSLENV}:UPDATE_APPLE_FIXTURES/up" +else + export WSLENV="UPDATE_APPLE_FIXTURES/up" +fi + +touch "$ROOT_DIR/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/.update-apple-fixtures" +( cd "$ROOT_DIR" && dotnet test "src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj" "$@" ) diff --git a/scripts/update-model-goldens.ps1 b/scripts/update-model-goldens.ps1 index e4b5a39a..9bb20554 100644 --- a/scripts/update-model-goldens.ps1 +++ b/scripts/update-model-goldens.ps1 @@ -6,4 +6,4 @@ Param( $Root = Split-Path -Parent $PSScriptRoot $env:UPDATE_GOLDENS = "1" -dotnet test (Join-Path $Root "src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj") @RestArgs +dotnet test (Join-Path $Root "src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj") @RestArgs diff --git a/scripts/update-model-goldens.sh b/scripts/update-model-goldens.sh index e668b616..5e0d0984 100644 --- a/scripts/update-model-goldens.sh +++ b/scripts/update-model-goldens.sh @@ -5,4 +5,4 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" export UPDATE_GOLDENS=1 -dotnet test "$ROOT_DIR/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj" "$@" +dotnet test "$ROOT_DIR/src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj" "$@" diff --git a/seed-data/cert-bund/README.md b/seed-data/cert-bund/README.md index 28c5c642..c31fea90 100644 --- a/seed-data/cert-bund/README.md +++ b/seed-data/cert-bund/README.md @@ -40,7 +40,7 @@ python tools/certbund_offline_snapshot.py --output seed-data/cert-bund ``` See the connector operations guide -(`docs/ops/feedser-certbund-operations.md`) for detailed usage, +(`docs/ops/concelier-certbund-operations.md`) for detailed usage, including how to provide cookies/tokens when the portal requires manual authentication. diff --git a/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md b/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md index 71160900..6c508e5a 100644 --- a/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md +++ b/src/DEDUP_CONFLICTS_RESOLUTION_ALGO.md @@ -1,143 +1,143 @@ -````markdown -# Feedser Vulnerability Conflict Resolution Rules - -This document defines the canonical, deterministic conflict resolution strategy for merging vulnerability data from **NVD**, **GHSA**, and **OSV** in Feedser. - ---- - -## 🧭 Source Precedence - -1. **Primary order:** - `GHSA > NVD > OSV` - - **Rationale:** - GHSA advisories are human-curated and fast to correct; NVD has the broadest CVE coverage; OSV excels in ecosystem-specific precision. - -2. **Freshness override (≥48 h):** - If a **lower-priority** source is **newer by at least 48 hours** for a freshness-sensitive field, its value overrides the higher-priority one. - Always store the decision in a provenance record. - -3. **Merge scope:** - Only merge data referring to the **same CVE ID** or the same GHSA/OSV advisory explicitly mapped to that CVE. - ---- - -## 🧩 Field-Level Precedence - -| Field | Priority | Freshness-Sensitive | Notes | -|-------|-----------|--------------------|-------| -| Title / Summary | GHSA → NVD → OSV | ✅ | Prefer concise structured titles | -| Description | GHSA → NVD → OSV | ✅ | | -| Severity (CVSS) | NVD → GHSA → OSV | ❌ | Keep all under `metrics[]`, mark `canonicalMetric` by order | -| Ecosystem Severity Label | GHSA → OSV | ❌ | Supplemental tag only | -| Affected Packages / Ranges | OSV → GHSA → NVD | ✅ | OSV strongest for SemVer normalization | -| CWE(s) | NVD → GHSA → OSV | ❌ | NVD taxonomy most stable | -| References / Links | Union of all | ✅ | Deduplicate by normalized URL | -| Credits / Acknowledgements | Union of all | ✅ | Sort by role, displayName | -| Published / Modified timestamps | Earliest published / Latest modified | ✅ | | -| EPSS / KEV / Exploit status | Specialized feed only | ❌ | Do not override manually | - ---- - -## ⚖️ Deterministic Tie-Breakers - -If precedence and freshness both tie: - -1. **Source order:** GHSA > NVD > OSV -2. **Lexicographic stability:** Prefer shorter normalized text; if equal, ASCIIbetical -3. **Stable hash of payload:** Lowest hash wins - -Each chosen value must store the merge rationale: - -```json -{ - "provenance": { - "source": "GHSA", - "kind": "merge", - "value": "description", - "decisionReason": "precedence" - } -} -```` - ---- - -## 🧮 Merge Algorithm (Pseudocode) - -```csharp -inputs: records = {ghsa?, nvd?, osv?} -out = new CanonicalVuln(CVE) - -foreach field in CANONICAL_SCHEMA: - candidates = collect(values, source, lastModified) - if freshnessSensitive(field) and newerBy48h(lowerPriority): - pick newest - else: - pick by precedence(field) - if tie: - applyTieBreakers() - out.field = normalize(field, value) - out.provenance[field] = decisionTrail - -out.references = dedupe(union(all.references)) -out.affected = normalizeAndUnion(OSV, GHSA, NVD) -out.metrics = rankAndSetCanonical(NVDv3 → GHSA → OSV → v2) -return out -``` - ---- - -## 🔧 Normalization Rules - -* **SemVer:** - Parse with tolerant builder; normalize `v` prefixes; map comparators (`<=`, `<`, `>=`, `>`); expand OSV events into continuous ranges. - -* **Packages:** - Canonical key = `(ecosystem, packageName, language?)`; maintain aliases (purl, npm, Maven GAV, etc.). - -* **CWE:** - Store both ID and name; validate against current CWE catalog. - -* **CVSS:** - Preserve provided vector and base score; recompute only for validation. - ---- - -## ✅ Output Guarantees - -| Property | Description | -| ---------------- | ------------------------------------------------------------------------------- | -| **Reproducible** | Same input → same canonical output | -| **Auditable** | Provenance stored per field | -| **Complete** | Unions with de-duplication | -| **Composable** | Future layers (KEV, EPSS, vendor advisories) can safely extend precedence rules | - ---- - -## 🧠 Example - -* GHSA summary updated on *2025-10-09* -* NVD last modified *2025-10-05* -* OSV updated *2025-10-10* - -→ **Summary:** OSV wins (freshness override) -→ **CVSS:** NVD v3.1 remains canonical -→ **Affected:** OSV ranges canonical; GHSA aliases merged - ---- - -## 🧰 Optional C# Helper Class - -`StellaOps.Feedser.Core/CanonicalMerger.cs` - -Implements: - -* `FieldPrecedenceMap` -* `FreshnessSensitiveFields` -* `ApplyTieBreakers()` -* `NormalizeAndUnion()` - -Deterministically builds `CanonicalVuln` with full provenance tracking. - -``` -``` +````markdown +# Concelier Vulnerability Conflict Resolution Rules + +This document defines the canonical, deterministic conflict resolution strategy for merging vulnerability data from **NVD**, **GHSA**, and **OSV** in Concelier. + +--- + +## 🧭 Source Precedence + +1. **Primary order:** + `GHSA > NVD > OSV` + + **Rationale:** + GHSA advisories are human-curated and fast to correct; NVD has the broadest CVE coverage; OSV excels in ecosystem-specific precision. + +2. **Freshness override (≥48 h):** + If a **lower-priority** source is **newer by at least 48 hours** for a freshness-sensitive field, its value overrides the higher-priority one. + Always store the decision in a provenance record. + +3. **Merge scope:** + Only merge data referring to the **same CVE ID** or the same GHSA/OSV advisory explicitly mapped to that CVE. + +--- + +## 🧩 Field-Level Precedence + +| Field | Priority | Freshness-Sensitive | Notes | +|-------|-----------|--------------------|-------| +| Title / Summary | GHSA → NVD → OSV | ✅ | Prefer concise structured titles | +| Description | GHSA → NVD → OSV | ✅ | | +| Severity (CVSS) | NVD → GHSA → OSV | ❌ | Keep all under `metrics[]`, mark `canonicalMetric` by order | +| Ecosystem Severity Label | GHSA → OSV | ❌ | Supplemental tag only | +| Affected Packages / Ranges | OSV → GHSA → NVD | ✅ | OSV strongest for SemVer normalization | +| CWE(s) | NVD → GHSA → OSV | ❌ | NVD taxonomy most stable | +| References / Links | Union of all | ✅ | Deduplicate by normalized URL | +| Credits / Acknowledgements | Union of all | ✅ | Sort by role, displayName | +| Published / Modified timestamps | Earliest published / Latest modified | ✅ | | +| EPSS / KEV / Exploit status | Specialized feed only | ❌ | Do not override manually | + +--- + +## ⚖️ Deterministic Tie-Breakers + +If precedence and freshness both tie: + +1. **Source order:** GHSA > NVD > OSV +2. **Lexicographic stability:** Prefer shorter normalized text; if equal, ASCIIbetical +3. **Stable hash of payload:** Lowest hash wins + +Each chosen value must store the merge rationale: + +```json +{ + "provenance": { + "source": "GHSA", + "kind": "merge", + "value": "description", + "decisionReason": "precedence" + } +} +```` + +--- + +## 🧮 Merge Algorithm (Pseudocode) + +```csharp +inputs: records = {ghsa?, nvd?, osv?} +out = new CanonicalVuln(CVE) + +foreach field in CANONICAL_SCHEMA: + candidates = collect(values, source, lastModified) + if freshnessSensitive(field) and newerBy48h(lowerPriority): + pick newest + else: + pick by precedence(field) + if tie: + applyTieBreakers() + out.field = normalize(field, value) + out.provenance[field] = decisionTrail + +out.references = dedupe(union(all.references)) +out.affected = normalizeAndUnion(OSV, GHSA, NVD) +out.metrics = rankAndSetCanonical(NVDv3 → GHSA → OSV → v2) +return out +``` + +--- + +## 🔧 Normalization Rules + +* **SemVer:** + Parse with tolerant builder; normalize `v` prefixes; map comparators (`<=`, `<`, `>=`, `>`); expand OSV events into continuous ranges. + +* **Packages:** + Canonical key = `(ecosystem, packageName, language?)`; maintain aliases (purl, npm, Maven GAV, etc.). + +* **CWE:** + Store both ID and name; validate against current CWE catalog. + +* **CVSS:** + Preserve provided vector and base score; recompute only for validation. + +--- + +## ✅ Output Guarantees + +| Property | Description | +| ---------------- | ------------------------------------------------------------------------------- | +| **Reproducible** | Same input → same canonical output | +| **Auditable** | Provenance stored per field | +| **Complete** | Unions with de-duplication | +| **Composable** | Future layers (KEV, EPSS, vendor advisories) can safely extend precedence rules | + +--- + +## 🧠 Example + +* GHSA summary updated on *2025-10-09* +* NVD last modified *2025-10-05* +* OSV updated *2025-10-10* + +→ **Summary:** OSV wins (freshness override) +→ **CVSS:** NVD v3.1 remains canonical +→ **Affected:** OSV ranges canonical; GHSA aliases merged + +--- + +## 🧰 Optional C# Helper Class + +`StellaOps.Concelier.Core/CanonicalMerger.cs` + +Implements: + +* `FieldPrecedenceMap` +* `FreshnessSensitiveFields` +* `ApplyTieBreakers()` +* `NormalizeAndUnion()` + +Deterministically builds `CanonicalVuln` with full provenance tracking. + +``` +``` diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 1b0a699d..64b50dfa 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -1,33 +1,45 @@ - $(SolutionDir)PluginBinaries - $(MSBuildThisFileDirectory)PluginBinaries - $(SolutionDir)PluginBinaries\Authority - $(MSBuildThisFileDirectory)PluginBinaries\Authority - true - true + $(SolutionDir)StellaOps.Concelier.PluginBinaries + $(MSBuildThisFileDirectory)StellaOps.Concelier.PluginBinaries + $(SolutionDir)StellaOps.Authority.PluginBinaries + $(MSBuildThisFileDirectory)StellaOps.Authority.PluginBinaries + true + true true + $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\buildx\')) + true + $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\analyzers\os\')) + true + $([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)..\plugins\scanner\analyzers\lang\')) + true + true false runtime - - - - - - + + + + + + + + + + + - + - - - - + + + + diff --git a/src/Directory.Build.targets b/src/Directory.Build.targets index a2e810fb..14b78768 100644 --- a/src/Directory.Build.targets +++ b/src/Directory.Build.targets @@ -1,18 +1,18 @@ - + - $(FeedserPluginOutputRoot)\$(MSBuildProjectName) + $(ConcelierPluginOutputRoot)\$(MSBuildProjectName) - + - - - + + + - + @@ -30,4 +30,55 @@ + + + + $(ScannerBuildxPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + + + + + $(ScannerOsAnalyzerPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + + + + + $(ScannerLangAnalyzerPluginOutputRoot)\$(MSBuildProjectName) + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/AGENTS.md b/src/StellaOps.Attestor/AGENTS.md new file mode 100644 index 00000000..70c1828b --- /dev/null +++ b/src/StellaOps.Attestor/AGENTS.md @@ -0,0 +1,21 @@ +# Attestor Guild + +## Mission +Operate the StellaOps Attestor service: accept signed DSSE envelopes from the Signer over mTLS, submit them to Rekor v2, persist inclusion proofs, and expose verification APIs for downstream services and operators. + +## Teams On Call +- Team 11 (Attestor API) +- Team 12 (Attestor Observability) — partners on logging, metrics, and alerting + +## Operating Principles +- Enforce mTLS + Authority tokens for every submission; never accept anonymous callers. +- Deterministic hashing, canonical JSON, and idempotent Rekor interactions (`bundleSha256` is the source of truth). +- Persist everything (entries, dedupe, audit) before acknowledging; background jobs must be resumable. +- Structured logs + metrics for each stage (`validate`, `submit`, `proof`, `persist`, `archive`). +- Update `TASKS.md`, architecture docs, and tests whenever behaviour changes. + +## Key Directories +- `src/StellaOps.Attestor/StellaOps.Attestor.WebService/` — Minimal API host and HTTP surface. +- `src/StellaOps.Attestor/StellaOps.Attestor.Core/` — Domain contracts, submission/verification pipelines. +- `src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/` — Mongo, Redis, Rekor, and archival implementations. +- `src/StellaOps.Attestor/StellaOps.Attestor.Tests/` — Unit and integration tests. diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Audit/AttestorAuditRecord.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Audit/AttestorAuditRecord.cs new file mode 100644 index 00000000..d9712320 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Audit/AttestorAuditRecord.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Attestor.Core.Audit; + +public sealed class AttestorAuditRecord +{ + public string Action { get; init; } = string.Empty; + + public string Result { get; init; } = string.Empty; + + public string? RekorUuid { get; init; } + + public long? Index { get; init; } + + public string ArtifactSha256 { get; init; } = string.Empty; + + public string BundleSha256 { get; init; } = string.Empty; + + public string Backend { get; init; } = string.Empty; + + public long LatencyMs { get; init; } + + public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow; + + public CallerDescriptor Caller { get; init; } = new(); + + public IDictionary Metadata { get; init; } = new Dictionary(); + + public sealed class CallerDescriptor + { + public string? Subject { get; init; } + + public string? Audience { get; init; } + + public string? ClientId { get; init; } + + public string? MtlsThumbprint { get; init; } + + public string? Tenant { get; init; } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Observability/AttestorMetrics.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Observability/AttestorMetrics.cs new file mode 100644 index 00000000..2a605e95 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Observability/AttestorMetrics.cs @@ -0,0 +1,45 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Attestor.Core.Observability; + +public sealed class AttestorMetrics : IDisposable +{ + public const string MeterName = "StellaOps.Attestor"; + + private readonly Meter _meter; + private bool _disposed; + + public AttestorMetrics() + { + _meter = new Meter(MeterName); + SubmitTotal = _meter.CreateCounter("attestor.submit_total", description: "Total submission attempts grouped by result and backend."); + SubmitLatency = _meter.CreateHistogram("attestor.submit_latency_seconds", unit: "s", description: "Submission latency in seconds per backend."); + ProofFetchTotal = _meter.CreateCounter("attestor.proof_fetch_total", description: "Proof fetch attempts grouped by result."); + VerifyTotal = _meter.CreateCounter("attestor.verify_total", description: "Verification attempts grouped by result."); + DedupeHitsTotal = _meter.CreateCounter("attestor.dedupe_hits_total", description: "Number of dedupe hits by outcome."); + ErrorTotal = _meter.CreateCounter("attestor.errors_total", description: "Total errors grouped by type."); + } + + public Counter SubmitTotal { get; } + + public Histogram SubmitLatency { get; } + + public Counter ProofFetchTotal { get; } + + public Counter VerifyTotal { get; } + + public Counter DedupeHitsTotal { get; } + + public Counter ErrorTotal { get; } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _meter.Dispose(); + _disposed = true; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Options/AttestorOptions.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Options/AttestorOptions.cs new file mode 100644 index 00000000..59c0b40f --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Options/AttestorOptions.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; + +namespace StellaOps.Attestor.Core.Options; + +/// +/// Strongly typed configuration for the Attestor service. +/// +public sealed class AttestorOptions +{ + public string Listen { get; set; } = "https://0.0.0.0:8444"; + + public SecurityOptions Security { get; set; } = new(); + + public RekorOptions Rekor { get; set; } = new(); + + public MongoOptions Mongo { get; set; } = new(); + + public RedisOptions Redis { get; set; } = new(); + + public S3Options S3 { get; set; } = new(); + + public QuotaOptions Quotas { get; set; } = new(); + + public TelemetryOptions Telemetry { get; set; } = new(); + + public sealed class SecurityOptions + { + public MtlsOptions Mtls { get; set; } = new(); + + public AuthorityOptions Authority { get; set; } = new(); + + public SignerIdentityOptions SignerIdentity { get; set; } = new(); + } + + public sealed class MtlsOptions + { + public bool RequireClientCertificate { get; set; } = true; + + public string? CaBundle { get; set; } + + public IList AllowedSubjects { get; set; } = new List(); + + public IList AllowedThumbprints { get; set; } = new List(); + } + + public sealed class AuthorityOptions + { + public string? Issuer { get; set; } + + public string? JwksUrl { get; set; } + + public string? RequireSenderConstraint { get; set; } + + public bool RequireHttpsMetadata { get; set; } = true; + + public IList Audiences { get; set; } = new List(); + + public IList RequiredScopes { get; set; } = new List(); + } + + public sealed class SignerIdentityOptions + { + public IList Mode { get; set; } = new List { "keyless", "kms" }; + + public IList FulcioRoots { get; set; } = new List(); + + public IList AllowedSans { get; set; } = new List(); + + public IList KmsKeys { get; set; } = new List(); + } + + public sealed class RekorOptions + { + public RekorBackendOptions Primary { get; set; } = new(); + + public RekorMirrorOptions Mirror { get; set; } = new(); + } + + public class RekorBackendOptions + { + public string? Url { get; set; } + + public int ProofTimeoutMs { get; set; } = 15_000; + + public int PollIntervalMs { get; set; } = 250; + + public int MaxAttempts { get; set; } = 60; + } + + public sealed class RekorMirrorOptions : RekorBackendOptions + { + public bool Enabled { get; set; } + } + + public sealed class MongoOptions + { + public string? Uri { get; set; } + + public string Database { get; set; } = "attestor"; + + public string EntriesCollection { get; set; } = "entries"; + + public string DedupeCollection { get; set; } = "dedupe"; + + public string AuditCollection { get; set; } = "audit"; + } + + public sealed class RedisOptions + { + public string? Url { get; set; } + + public string? DedupePrefix { get; set; } = "attestor:dedupe:"; + } + + public sealed class S3Options + { + public bool Enabled { get; set; } + + public string? Endpoint { get; set; } + + public string? Bucket { get; set; } + + public string? Prefix { get; set; } + + public string? ObjectLockMode { get; set; } + + public bool UseTls { get; set; } = true; + } + + public sealed class QuotaOptions + { + public PerCallerQuotaOptions PerCaller { get; set; } = new(); + } + + public sealed class PerCallerQuotaOptions + { + public int Qps { get; set; } = 50; + + public int Burst { get; set; } = 100; + } + + public sealed class TelemetryOptions + { + public bool EnableLogging { get; set; } = true; + + public bool EnableTracing { get; set; } = false; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/IRekorClient.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/IRekorClient.cs new file mode 100644 index 00000000..dcd3539e --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/IRekorClient.cs @@ -0,0 +1,18 @@ +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Attestor.Core.Submission; + +namespace StellaOps.Attestor.Core.Rekor; + +public interface IRekorClient +{ + Task SubmitAsync( + AttestorSubmissionRequest request, + RekorBackend backend, + CancellationToken cancellationToken = default); + + Task GetProofAsync( + string rekorUuid, + RekorBackend backend, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorBackend.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorBackend.cs new file mode 100644 index 00000000..d89ae6ab --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorBackend.cs @@ -0,0 +1,16 @@ +using System; + +namespace StellaOps.Attestor.Core.Rekor; + +public sealed class RekorBackend +{ + public required string Name { get; init; } + + public required Uri Url { get; init; } + + public TimeSpan ProofTimeout { get; init; } = TimeSpan.FromSeconds(15); + + public TimeSpan PollInterval { get; init; } = TimeSpan.FromMilliseconds(250); + + public int MaxAttempts { get; init; } = 60; +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorProofResponse.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorProofResponse.cs new file mode 100644 index 00000000..1d486ef8 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorProofResponse.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Attestor.Core.Rekor; + +public sealed class RekorProofResponse +{ + [JsonPropertyName("checkpoint")] + public RekorCheckpoint? Checkpoint { get; set; } + + [JsonPropertyName("inclusion")] + public RekorInclusionProof? Inclusion { get; set; } + + public sealed class RekorCheckpoint + { + [JsonPropertyName("origin")] + public string? Origin { get; set; } + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("rootHash")] + public string? RootHash { get; set; } + + [JsonPropertyName("timestamp")] + public DateTimeOffset? Timestamp { get; set; } + } + + public sealed class RekorInclusionProof + { + [JsonPropertyName("leafHash")] + public string? LeafHash { get; set; } + + [JsonPropertyName("path")] + public IReadOnlyList Path { get; set; } = Array.Empty(); + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorSubmissionResponse.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorSubmissionResponse.cs new file mode 100644 index 00000000..a80daf51 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Rekor/RekorSubmissionResponse.cs @@ -0,0 +1,21 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Attestor.Core.Rekor; + +public sealed class RekorSubmissionResponse +{ + [JsonPropertyName("uuid")] + public string Uuid { get; set; } = string.Empty; + + [JsonPropertyName("index")] + public long? Index { get; set; } + + [JsonPropertyName("logURL")] + public string? LogUrl { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } = "included"; + + [JsonPropertyName("proof")] + public RekorProofResponse? Proof { get; set; } +} diff --git a/src/StellaOps.Vexer.Core/StellaOps.Vexer.Core.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.Core/StellaOps.Attestor.Core.csproj similarity index 97% rename from src/StellaOps.Vexer.Core/StellaOps.Vexer.Core.csproj rename to src/StellaOps.Attestor/StellaOps.Attestor.Core/StellaOps.Attestor.Core.csproj index ecc3af66..2b6aadf4 100644 --- a/src/StellaOps.Vexer.Core/StellaOps.Vexer.Core.csproj +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/StellaOps.Attestor.Core.csproj @@ -1,9 +1,9 @@ - - - net10.0 - preview - enable - enable - true - - + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorArchiveBundle.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorArchiveBundle.cs new file mode 100644 index 00000000..cc8afa9b --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorArchiveBundle.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Attestor.Core.Storage; + +public sealed class AttestorArchiveBundle +{ + public string RekorUuid { get; init; } = string.Empty; + + public string ArtifactSha256 { get; init; } = string.Empty; + + public string BundleSha256 { get; init; } = string.Empty; + + public byte[] CanonicalBundleJson { get; init; } = Array.Empty(); + + public byte[] ProofJson { get; init; } = Array.Empty(); + + public IReadOnlyDictionary Metadata { get; init; } = new Dictionary(); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorEntry.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorEntry.cs new file mode 100644 index 00000000..9ec4b0e3 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/AttestorEntry.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Attestor.Core.Storage; + +/// +/// Canonical representation of a Rekor entry persisted in Mongo. +/// +public sealed class AttestorEntry +{ + public string RekorUuid { get; init; } = string.Empty; + + public ArtifactDescriptor Artifact { get; init; } = new(); + + public string BundleSha256 { get; init; } = string.Empty; + + public long? Index { get; init; } + + public ProofDescriptor? Proof { get; init; } + + public LogDescriptor Log { get; init; } = new(); + + public DateTimeOffset CreatedAt { get; init; } + + public string Status { get; init; } = "pending"; + + public SignerIdentityDescriptor SignerIdentity { get; init; } = new(); + + public LogReplicaDescriptor? Mirror { get; init; } + + public sealed class ArtifactDescriptor + { + public string Sha256 { get; init; } = string.Empty; + + public string Kind { get; init; } = string.Empty; + + public string? ImageDigest { get; init; } + + public string? SubjectUri { get; init; } + } + + public sealed class ProofDescriptor + { + public CheckpointDescriptor? Checkpoint { get; init; } + + public InclusionDescriptor? Inclusion { get; init; } + } + + public sealed class CheckpointDescriptor + { + public string? Origin { get; init; } + + public long Size { get; init; } + + public string? RootHash { get; init; } + + public DateTimeOffset? Timestamp { get; init; } + } + + public sealed class InclusionDescriptor + { + public string? LeafHash { get; init; } + + public IReadOnlyList Path { get; init; } = Array.Empty(); + } + + public sealed class LogDescriptor + { + public string Backend { get; init; } = "primary"; + + public string Url { get; init; } = string.Empty; + + public string? LogId { get; init; } + } + + public sealed class SignerIdentityDescriptor + { + public string Mode { get; init; } = string.Empty; + + public string? Issuer { get; init; } + + public string? SubjectAlternativeName { get; init; } + + public string? KeyId { get; init; } + } + + public sealed class LogReplicaDescriptor + { + public string Backend { get; init; } = string.Empty; + + public string Url { get; init; } = string.Empty; + + public string? Uuid { get; init; } + + public long? Index { get; init; } + + public string Status { get; init; } = "pending"; + + public ProofDescriptor? Proof { get; init; } + + public string? LogId { get; init; } + + public string? Error { get; init; } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorArchiveStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorArchiveStore.cs new file mode 100644 index 00000000..d3e670f6 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorArchiveStore.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Storage; + +public interface IAttestorArchiveStore +{ + Task ArchiveBundleAsync(AttestorArchiveBundle bundle, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorAuditSink.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorAuditSink.cs new file mode 100644 index 00000000..8e2ea219 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorAuditSink.cs @@ -0,0 +1,10 @@ +using StellaOps.Attestor.Core.Audit; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Storage; + +public interface IAttestorAuditSink +{ + Task WriteAsync(AttestorAuditRecord record, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorDedupeStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorDedupeStore.cs new file mode 100644 index 00000000..c0563851 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorDedupeStore.cs @@ -0,0 +1,12 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Storage; + +public interface IAttestorDedupeStore +{ + Task TryGetExistingAsync(string bundleSha256, CancellationToken cancellationToken = default); + + Task SetAsync(string bundleSha256, string rekorUuid, TimeSpan ttl, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorEntryRepository.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorEntryRepository.cs new file mode 100644 index 00000000..64e09494 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Storage/IAttestorEntryRepository.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Storage; + +public interface IAttestorEntryRepository +{ + Task GetByBundleShaAsync(string bundleSha256, CancellationToken cancellationToken = default); + + Task GetByUuidAsync(string rekorUuid, CancellationToken cancellationToken = default); + + Task> GetByArtifactShaAsync(string artifactSha256, CancellationToken cancellationToken = default); + + Task SaveAsync(AttestorEntry entry, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionRequest.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionRequest.cs new file mode 100644 index 00000000..31411db7 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionRequest.cs @@ -0,0 +1,79 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Attestor.Core.Submission; + +/// +/// Incoming submission payload for /api/v1/rekor/entries. +/// +public sealed class AttestorSubmissionRequest +{ + [JsonPropertyName("bundle")] + public SubmissionBundle Bundle { get; set; } = new(); + + [JsonPropertyName("meta")] + public SubmissionMeta Meta { get; set; } = new(); + + public sealed class SubmissionBundle + { + [JsonPropertyName("dsse")] + public DsseEnvelope Dsse { get; set; } = new(); + + [JsonPropertyName("certificateChain")] + public IList CertificateChain { get; set; } = new List(); + + [JsonPropertyName("mode")] + public string Mode { get; set; } = "keyless"; + } + + public sealed class DsseEnvelope + { + [JsonPropertyName("payloadType")] + public string PayloadType { get; set; } = string.Empty; + + [JsonPropertyName("payload")] + public string PayloadBase64 { get; set; } = string.Empty; + + [JsonPropertyName("signatures")] + public IList Signatures { get; set; } = new List(); + } + + public sealed class DsseSignature + { + [JsonPropertyName("keyid")] + public string? KeyId { get; set; } + + [JsonPropertyName("sig")] + public string Signature { get; set; } = string.Empty; + } + + public sealed class SubmissionMeta + { + [JsonPropertyName("artifact")] + public ArtifactInfo Artifact { get; set; } = new(); + + [JsonPropertyName("bundleSha256")] + public string BundleSha256 { get; set; } = string.Empty; + + [JsonPropertyName("logPreference")] + public string LogPreference { get; set; } = "primary"; + + [JsonPropertyName("archive")] + public bool Archive { get; set; } = true; + } + + public sealed class ArtifactInfo + { + [JsonPropertyName("sha256")] + public string Sha256 { get; set; } = string.Empty; + + [JsonPropertyName("kind")] + public string Kind { get; set; } = string.Empty; + + [JsonPropertyName("imageDigest")] + public string? ImageDigest { get; set; } + + [JsonPropertyName("subjectUri")] + public string? SubjectUri { get; set; } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionResult.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionResult.cs new file mode 100644 index 00000000..89fba827 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionResult.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Attestor.Core.Submission; + +/// +/// Result returned to callers after processing a submission. +/// +public sealed class AttestorSubmissionResult +{ + [JsonPropertyName("uuid")] + public string? Uuid { get; set; } + + [JsonPropertyName("index")] + public long? Index { get; set; } + + [JsonPropertyName("proof")] + public RekorProof? Proof { get; set; } + + [JsonPropertyName("logURL")] + public string? LogUrl { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } = "pending"; + + [JsonPropertyName("mirror")] + public MirrorLog? Mirror { get; set; } + + public sealed class RekorProof + { + [JsonPropertyName("checkpoint")] + public Checkpoint? Checkpoint { get; set; } + + [JsonPropertyName("inclusion")] + public InclusionProof? Inclusion { get; set; } + } + + public sealed class Checkpoint + { + [JsonPropertyName("origin")] + public string? Origin { get; set; } + + [JsonPropertyName("size")] + public long Size { get; set; } + + [JsonPropertyName("rootHash")] + public string? RootHash { get; set; } + + [JsonPropertyName("timestamp")] + public string? Timestamp { get; set; } + } + + public sealed class InclusionProof + { + [JsonPropertyName("leafHash")] + public string? LeafHash { get; set; } + + [JsonPropertyName("path")] + public IReadOnlyList Path { get; init; } = Array.Empty(); + } + + public sealed class MirrorLog + { + [JsonPropertyName("uuid")] + public string? Uuid { get; set; } + + [JsonPropertyName("index")] + public long? Index { get; set; } + + [JsonPropertyName("logURL")] + public string? LogUrl { get; set; } + + [JsonPropertyName("status")] + public string Status { get; set; } = "pending"; + + [JsonPropertyName("proof")] + public RekorProof? Proof { get; set; } + + [JsonPropertyName("error")] + public string? Error { get; set; } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidationResult.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidationResult.cs new file mode 100644 index 00000000..564bd8f4 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidationResult.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Attestor.Core.Submission; + +public sealed class AttestorSubmissionValidationResult +{ + public AttestorSubmissionValidationResult(byte[] canonicalBundle) + { + CanonicalBundle = canonicalBundle; + } + + public byte[] CanonicalBundle { get; } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidator.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidator.cs new file mode 100644 index 00000000..71ce511f --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorSubmissionValidator.cs @@ -0,0 +1,176 @@ +using System; +using System.Buffers.Text; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Submission; + +public sealed class AttestorSubmissionValidator +{ + private static readonly string[] AllowedKinds = ["sbom", "report", "vex-export"]; + + private readonly IDsseCanonicalizer _canonicalizer; + private readonly HashSet _allowedModes; + + public AttestorSubmissionValidator(IDsseCanonicalizer canonicalizer, IEnumerable? allowedModes = null) + { + _canonicalizer = canonicalizer ?? throw new ArgumentNullException(nameof(canonicalizer)); + _allowedModes = allowedModes is null + ? new HashSet(StringComparer.OrdinalIgnoreCase) + : new HashSet(allowedModes, StringComparer.OrdinalIgnoreCase); + } + + public async Task ValidateAsync(AttestorSubmissionRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + if (request.Bundle is null) + { + throw new AttestorValidationException("bundle_missing", "Submission bundle payload is required."); + } + + if (request.Bundle.Dsse is null) + { + throw new AttestorValidationException("dsse_missing", "DSSE envelope is required."); + } + + if (string.IsNullOrWhiteSpace(request.Bundle.Dsse.PayloadType)) + { + throw new AttestorValidationException("payload_type_missing", "DSSE payloadType is required."); + } + + if (string.IsNullOrWhiteSpace(request.Bundle.Dsse.PayloadBase64)) + { + throw new AttestorValidationException("payload_missing", "DSSE payload must be provided."); + } + + if (request.Bundle.Dsse.Signatures.Count == 0) + { + throw new AttestorValidationException("signature_missing", "At least one DSSE signature is required."); + } + + if (_allowedModes.Count > 0 && !string.IsNullOrWhiteSpace(request.Bundle.Mode) && !_allowedModes.Contains(request.Bundle.Mode)) + { + throw new AttestorValidationException("mode_not_allowed", $"Submission mode '{request.Bundle.Mode}' is not permitted."); + } + + if (request.Meta is null) + { + throw new AttestorValidationException("meta_missing", "Submission metadata is required."); + } + + if (request.Meta.Artifact is null) + { + throw new AttestorValidationException("artifact_missing", "Artifact metadata is required."); + } + + if (string.IsNullOrWhiteSpace(request.Meta.Artifact.Sha256)) + { + throw new AttestorValidationException("artifact_sha_missing", "Artifact sha256 is required."); + } + + if (!IsHex(request.Meta.Artifact.Sha256, expectedLength: 64)) + { + throw new AttestorValidationException("artifact_sha_invalid", "Artifact sha256 must be a 64-character hex string."); + } + + if (string.IsNullOrWhiteSpace(request.Meta.BundleSha256)) + { + throw new AttestorValidationException("bundle_sha_missing", "bundleSha256 is required."); + } + + if (!IsHex(request.Meta.BundleSha256, expectedLength: 64)) + { + throw new AttestorValidationException("bundle_sha_invalid", "bundleSha256 must be a 64-character hex string."); + } + + if (Array.IndexOf(AllowedKinds, request.Meta.Artifact.Kind) < 0) + { + throw new AttestorValidationException("artifact_kind_invalid", $"Artifact kind '{request.Meta.Artifact.Kind}' is not supported."); + } + + if (!Base64UrlDecode(request.Bundle.Dsse.PayloadBase64, out _)) + { + throw new AttestorValidationException("payload_invalid_base64", "DSSE payload must be valid base64."); + } + + var canonical = await _canonicalizer.CanonicalizeAsync(request, cancellationToken).ConfigureAwait(false); + Span hash = stackalloc byte[32]; + if (!SHA256.TryHashData(canonical, hash, out _)) + { + throw new AttestorValidationException("bundle_sha_failure", "Failed to compute canonical bundle hash."); + } + + var hashHex = Convert.ToHexString(hash).ToLowerInvariant(); + if (!string.Equals(hashHex, request.Meta.BundleSha256, StringComparison.OrdinalIgnoreCase)) + { + throw new AttestorValidationException("bundle_sha_mismatch", "bundleSha256 does not match canonical DSSE hash."); + } + + if (!string.Equals(request.Meta.LogPreference, "primary", StringComparison.OrdinalIgnoreCase) + && !string.Equals(request.Meta.LogPreference, "mirror", StringComparison.OrdinalIgnoreCase) + && !string.Equals(request.Meta.LogPreference, "both", StringComparison.OrdinalIgnoreCase)) + { + throw new AttestorValidationException("log_preference_invalid", "logPreference must be 'primary', 'mirror', or 'both'."); + } + + return new AttestorSubmissionValidationResult(canonical); + } + + private static bool IsHex(string value, int expectedLength) + { + if (value.Length != expectedLength) + { + return false; + } + + foreach (var ch in value) + { + var isHex = ch is >= '0' and <= '9' or >= 'a' and <= 'f' or >= 'A' and <= 'F'; + if (!isHex) + { + return false; + } + } + + return true; + } + + private static bool Base64UrlDecode(string value, out byte[] bytes) + { + try + { + bytes = Convert.FromBase64String(Normalise(value)); + return true; + } + catch (FormatException) + { + bytes = Array.Empty(); + return false; + } + } + + private static string Normalise(string value) + { + if (value.Contains('-') || value.Contains('_')) + { + Span buffer = value.ToCharArray(); + for (var i = 0; i < buffer.Length; i++) + { + buffer[i] = buffer[i] switch + { + '-' => '+', + '_' => '/', + _ => buffer[i] + }; + } + + var padding = 4 - (buffer.Length % 4); + return padding == 4 ? new string(buffer) : new string(buffer) + new string('=', padding); + } + + return value; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorValidationException.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorValidationException.cs new file mode 100644 index 00000000..aa28a17d --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/AttestorValidationException.cs @@ -0,0 +1,14 @@ +using System; + +namespace StellaOps.Attestor.Core.Submission; + +public sealed class AttestorValidationException : Exception +{ + public AttestorValidationException(string code, string message) + : base(message) + { + Code = code; + } + + public string Code { get; } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IAttestorSubmissionService.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IAttestorSubmissionService.cs new file mode 100644 index 00000000..64c05c52 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IAttestorSubmissionService.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Submission; + +public interface IAttestorSubmissionService +{ + Task SubmitAsync( + AttestorSubmissionRequest request, + SubmissionContext context, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IDsseCanonicalizer.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IDsseCanonicalizer.cs new file mode 100644 index 00000000..de8a63a6 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/IDsseCanonicalizer.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Attestor.Core.Submission; + +public interface IDsseCanonicalizer +{ + Task CanonicalizeAsync(AttestorSubmissionRequest request, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/SubmissionContext.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/SubmissionContext.cs new file mode 100644 index 00000000..ce15a4af --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Submission/SubmissionContext.cs @@ -0,0 +1,21 @@ +using System.Security.Cryptography.X509Certificates; + +namespace StellaOps.Attestor.Core.Submission; + +/// +/// Ambient information about the caller used for policy and audit decisions. +/// +public sealed class SubmissionContext +{ + public required string CallerSubject { get; init; } + + public required string CallerAudience { get; init; } + + public required string? CallerClientId { get; init; } + + public required string? CallerTenant { get; init; } + + public X509Certificate2? ClientCertificate { get; init; } + + public string? MtlsThumbprint { get; init; } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationException.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationException.cs new file mode 100644 index 00000000..8df8071c --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationException.cs @@ -0,0 +1,14 @@ +using System; + +namespace StellaOps.Attestor.Core.Verification; + +public sealed class AttestorVerificationException : Exception +{ + public AttestorVerificationException(string code, string message) + : base(message) + { + Code = code; + } + + public string Code { get; } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationRequest.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationRequest.cs new file mode 100644 index 00000000..2845196c --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationRequest.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Attestor.Core.Verification; + +/// +/// Payload accepted by the verification service. +/// +public sealed class AttestorVerificationRequest +{ + public string? Uuid { get; set; } + + public Submission.AttestorSubmissionRequest.SubmissionBundle? Bundle { get; set; } + + public string? ArtifactSha256 { get; set; } + + public bool RefreshProof { get; set; } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationResult.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationResult.cs new file mode 100644 index 00000000..b9c95025 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/AttestorVerificationResult.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Attestor.Core.Verification; + +public sealed class AttestorVerificationResult +{ + public bool Ok { get; init; } + + public string? Uuid { get; init; } + + public long? Index { get; init; } + + public string? LogUrl { get; init; } + + public DateTimeOffset CheckedAt { get; init; } = DateTimeOffset.UtcNow; + + public string Status { get; init; } = "unknown"; + + public IReadOnlyList Issues { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/IAttestorVerificationService.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/IAttestorVerificationService.cs new file mode 100644 index 00000000..96304481 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Core/Verification/IAttestorVerificationService.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Core.Verification; + +public interface IAttestorVerificationService +{ + Task VerifyAsync(AttestorVerificationRequest request, CancellationToken cancellationToken = default); + + Task GetEntryAsync(string rekorUuid, bool refreshProof, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Properties/AssemblyInfo.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..9ecf146a --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Attestor.Tests")] diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/HttpRekorClient.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/HttpRekorClient.cs new file mode 100644 index 00000000..63e6696d --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/HttpRekorClient.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Submission; + +namespace StellaOps.Attestor.Infrastructure.Rekor; + +internal sealed class HttpRekorClient : IRekorClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public HttpRekorClient(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient; + _logger = logger; + } + + public async Task SubmitAsync(AttestorSubmissionRequest request, RekorBackend backend, CancellationToken cancellationToken = default) + { + var submissionUri = BuildUri(backend.Url, "api/v2/log/entries"); + + using var httpRequest = new HttpRequestMessage(HttpMethod.Post, submissionUri) + { + Content = JsonContent.Create(BuildSubmissionPayload(request), options: SerializerOptions) + }; + + using var response = await _httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.Conflict) + { + var message = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Rekor reported a conflict: {message}"); + } + + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + var root = document.RootElement; + + long? index = null; + if (root.TryGetProperty("index", out var indexElement) && indexElement.TryGetInt64(out var indexValue)) + { + index = indexValue; + } + + return new RekorSubmissionResponse + { + Uuid = root.TryGetProperty("uuid", out var uuidElement) ? uuidElement.GetString() ?? string.Empty : string.Empty, + Index = index, + LogUrl = root.TryGetProperty("logURL", out var urlElement) ? urlElement.GetString() ?? backend.Url.ToString() : backend.Url.ToString(), + Status = root.TryGetProperty("status", out var statusElement) ? statusElement.GetString() ?? "included" : "included", + Proof = TryParseProof(root.TryGetProperty("proof", out var proofElement) ? proofElement : default) + }; + } + + public async Task GetProofAsync(string rekorUuid, RekorBackend backend, CancellationToken cancellationToken = default) + { + var proofUri = BuildUri(backend.Url, $"api/v2/log/entries/{rekorUuid}/proof"); + + using var request = new HttpRequestMessage(HttpMethod.Get, proofUri); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogDebug("Rekor proof for {Uuid} not found", rekorUuid); + return null; + } + + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + return TryParseProof(document.RootElement); + } + + private static object BuildSubmissionPayload(AttestorSubmissionRequest request) + { + var signatures = new List(); + foreach (var sig in request.Bundle.Dsse.Signatures) + { + signatures.Add(new { keyid = sig.KeyId, sig = sig.Signature }); + } + + return new + { + entries = new[] + { + new + { + dsseEnvelope = new + { + payload = request.Bundle.Dsse.PayloadBase64, + payloadType = request.Bundle.Dsse.PayloadType, + signatures + } + } + } + }; + } + + private static RekorProofResponse? TryParseProof(JsonElement proofElement) + { + if (proofElement.ValueKind == JsonValueKind.Undefined || proofElement.ValueKind == JsonValueKind.Null) + { + return null; + } + + var checkpointElement = proofElement.TryGetProperty("checkpoint", out var cp) ? cp : default; + var inclusionElement = proofElement.TryGetProperty("inclusion", out var inc) ? inc : default; + + return new RekorProofResponse + { + Checkpoint = checkpointElement.ValueKind == JsonValueKind.Object + ? new RekorProofResponse.RekorCheckpoint + { + Origin = checkpointElement.TryGetProperty("origin", out var origin) ? origin.GetString() : null, + Size = checkpointElement.TryGetProperty("size", out var size) && size.TryGetInt64(out var sizeValue) ? sizeValue : 0, + RootHash = checkpointElement.TryGetProperty("rootHash", out var rootHash) ? rootHash.GetString() : null, + Timestamp = checkpointElement.TryGetProperty("timestamp", out var ts) && ts.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(ts.GetString(), out var dto) ? dto : null + } + : null, + Inclusion = inclusionElement.ValueKind == JsonValueKind.Object + ? new RekorProofResponse.RekorInclusionProof + { + LeafHash = inclusionElement.TryGetProperty("leafHash", out var leaf) ? leaf.GetString() : null, + Path = inclusionElement.TryGetProperty("path", out var pathElement) && pathElement.ValueKind == JsonValueKind.Array + ? pathElement.EnumerateArray().Select(p => p.GetString() ?? string.Empty).ToArray() + : Array.Empty() + } + : null + }; + } + + private static Uri BuildUri(Uri baseUri, string relative) + { + if (!relative.StartsWith("/", StringComparison.Ordinal)) + { + relative = "/" + relative; + } + + return new Uri(baseUri, relative); + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/StubRekorClient.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/StubRekorClient.cs new file mode 100644 index 00000000..48e69649 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Rekor/StubRekorClient.cs @@ -0,0 +1,71 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Submission; + +namespace StellaOps.Attestor.Infrastructure.Rekor; + +internal sealed class StubRekorClient : IRekorClient +{ + private readonly ILogger _logger; + + public StubRekorClient(ILogger logger) + { + _logger = logger; + } + + public Task SubmitAsync(AttestorSubmissionRequest request, RekorBackend backend, CancellationToken cancellationToken = default) + { + var uuid = Guid.NewGuid().ToString(); + _logger.LogInformation("Stub Rekor submission for bundle {BundleSha} -> {Uuid}", request.Meta.BundleSha256, uuid); + + var proof = new RekorProofResponse + { + Checkpoint = new RekorProofResponse.RekorCheckpoint + { + Origin = backend.Url.Host, + Size = 1, + RootHash = request.Meta.BundleSha256, + Timestamp = DateTimeOffset.UtcNow + }, + Inclusion = new RekorProofResponse.RekorInclusionProof + { + LeafHash = request.Meta.BundleSha256, + Path = Array.Empty() + } + }; + + var response = new RekorSubmissionResponse + { + Uuid = uuid, + Index = Random.Shared.NextInt64(1, long.MaxValue), + LogUrl = new Uri(backend.Url, $"/api/v2/log/entries/{uuid}").ToString(), + Status = "included", + Proof = proof + }; + + return Task.FromResult(response); + } + + public Task GetProofAsync(string rekorUuid, RekorBackend backend, CancellationToken cancellationToken = default) + { + _logger.LogInformation("Stub Rekor proof fetch for {Uuid}", rekorUuid); + return Task.FromResult(new RekorProofResponse + { + Checkpoint = new RekorProofResponse.RekorCheckpoint + { + Origin = backend.Url.Host, + Size = 1, + RootHash = string.Empty, + Timestamp = DateTimeOffset.UtcNow + }, + Inclusion = new RekorProofResponse.RekorInclusionProof + { + LeafHash = string.Empty, + Path = Array.Empty() + } + }); + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..5b7e4cf1 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/ServiceCollectionExtensions.cs @@ -0,0 +1,123 @@ +using System; +using Amazon.Runtime; +using Amazon.S3; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StackExchange.Redis; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Observability; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Storage; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Infrastructure.Rekor; +using StellaOps.Attestor.Infrastructure.Storage; +using StellaOps.Attestor.Infrastructure.Submission; +using StellaOps.Attestor.Core.Verification; +using StellaOps.Attestor.Infrastructure.Verification; + +namespace StellaOps.Attestor.Infrastructure; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddAttestorInfrastructure(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(sp => + { + var canonicalizer = sp.GetRequiredService(); + var options = sp.GetRequiredService>().Value; + return new AttestorSubmissionValidator(canonicalizer, options.Security.SignerIdentity.Mode); + }); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHttpClient(client => + { + client.Timeout = TimeSpan.FromSeconds(30); + }); + services.AddSingleton(sp => sp.GetRequiredService()); + + services.AddSingleton(sp => + { + var options = sp.GetRequiredService>().Value; + if (string.IsNullOrWhiteSpace(options.Mongo.Uri)) + { + throw new InvalidOperationException("Attestor MongoDB connection string is not configured."); + } + + return new MongoClient(options.Mongo.Uri); + }); + + services.AddSingleton(sp => + { + var opts = sp.GetRequiredService>().Value; + var client = sp.GetRequiredService(); + var databaseName = MongoUrl.Create(opts.Mongo.Uri).DatabaseName ?? opts.Mongo.Database; + return client.GetDatabase(databaseName); + }); + + services.AddSingleton(sp => + { + var opts = sp.GetRequiredService>().Value; + var database = sp.GetRequiredService(); + return database.GetCollection(opts.Mongo.EntriesCollection); + }); + + services.AddSingleton(sp => + { + var opts = sp.GetRequiredService>().Value; + var database = sp.GetRequiredService(); + return database.GetCollection(opts.Mongo.AuditCollection); + }); + + services.AddSingleton(); + services.AddSingleton(); + + + services.AddSingleton(sp => + { + var options = sp.GetRequiredService>().Value; + if (string.IsNullOrWhiteSpace(options.Redis.Url)) + { + return new InMemoryAttestorDedupeStore(); + } + + var multiplexer = sp.GetRequiredService(); + return new RedisAttestorDedupeStore(multiplexer, sp.GetRequiredService>()); + }); + + services.AddSingleton(sp => + { + var options = sp.GetRequiredService>().Value; + if (string.IsNullOrWhiteSpace(options.Redis.Url)) + { + throw new InvalidOperationException("Redis connection string is required when redis dedupe is enabled."); + } + + return ConnectionMultiplexer.Connect(options.Redis.Url); + }); + + services.AddSingleton(sp => + { + var options = sp.GetRequiredService>().Value; + if (options.S3.Enabled && !string.IsNullOrWhiteSpace(options.S3.Endpoint) && !string.IsNullOrWhiteSpace(options.S3.Bucket)) + { + var config = new AmazonS3Config + { + ServiceURL = options.S3.Endpoint, + ForcePathStyle = true, + UseHttp = !options.S3.UseTls + }; + + var client = new AmazonS3Client(FallbackCredentialsFactory.GetCredentials(), config); + return new S3AttestorArchiveStore(client, sp.GetRequiredService>(), sp.GetRequiredService>()); + } + + return new NullAttestorArchiveStore(sp.GetRequiredService>()); + }); + + return services; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj new file mode 100644 index 00000000..054613bd --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/StellaOps.Attestor.Infrastructure.csproj @@ -0,0 +1,21 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/InMemoryAttestorDedupeStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/InMemoryAttestorDedupeStore.cs new file mode 100644 index 00000000..990780aa --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/InMemoryAttestorDedupeStore.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class InMemoryAttestorDedupeStore : IAttestorDedupeStore +{ + private readonly ConcurrentDictionary _store = new(); + + public Task TryGetExistingAsync(string bundleSha256, CancellationToken cancellationToken = default) + { + if (_store.TryGetValue(bundleSha256, out var entry)) + { + if (entry.ExpiresAt > DateTimeOffset.UtcNow) + { + return Task.FromResult(entry.Uuid); + } + + _store.TryRemove(bundleSha256, out _); + } + + return Task.FromResult(null); + } + + public Task SetAsync(string bundleSha256, string rekorUuid, TimeSpan ttl, CancellationToken cancellationToken = default) + { + _store[bundleSha256] = (rekorUuid, DateTimeOffset.UtcNow.Add(ttl)); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorAuditSink.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorAuditSink.cs new file mode 100644 index 00000000..3d1b7763 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorAuditSink.cs @@ -0,0 +1,115 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using StellaOps.Attestor.Core.Audit; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class MongoAttestorAuditSink : IAttestorAuditSink +{ + private readonly IMongoCollection _collection; + + public MongoAttestorAuditSink(IMongoCollection collection) + { + _collection = collection; + } + + public Task WriteAsync(AttestorAuditRecord record, CancellationToken cancellationToken = default) + { + var document = AttestorAuditDocument.FromRecord(record); + return _collection.InsertOneAsync(document, cancellationToken: cancellationToken); + } + + internal sealed class AttestorAuditDocument + { + [BsonId] + public ObjectId Id { get; set; } + + [BsonElement("ts")] + public BsonDateTime Timestamp { get; set; } = BsonDateTime.Create(DateTime.UtcNow); + + [BsonElement("action")] + public string Action { get; set; } = string.Empty; + + [BsonElement("result")] + public string Result { get; set; } = string.Empty; + + [BsonElement("rekorUuid")] + public string? RekorUuid { get; set; } + + [BsonElement("index")] + public long? Index { get; set; } + + [BsonElement("artifactSha256")] + public string ArtifactSha256 { get; set; } = string.Empty; + + [BsonElement("bundleSha256")] + public string BundleSha256 { get; set; } = string.Empty; + + [BsonElement("backend")] + public string Backend { get; set; } = string.Empty; + + [BsonElement("latencyMs")] + public long LatencyMs { get; set; } + + [BsonElement("caller")] + public CallerDocument Caller { get; set; } = new(); + + [BsonElement("metadata")] + public BsonDocument Metadata { get; set; } = new(); + + public static AttestorAuditDocument FromRecord(AttestorAuditRecord record) + { + var metadata = new BsonDocument(); + foreach (var kvp in record.Metadata) + { + metadata[kvp.Key] = kvp.Value; + } + + return new AttestorAuditDocument + { + Id = ObjectId.GenerateNewId(), + Timestamp = BsonDateTime.Create(record.Timestamp.UtcDateTime), + Action = record.Action, + Result = record.Result, + RekorUuid = record.RekorUuid, + Index = record.Index, + ArtifactSha256 = record.ArtifactSha256, + BundleSha256 = record.BundleSha256, + Backend = record.Backend, + LatencyMs = record.LatencyMs, + Caller = new CallerDocument + { + Subject = record.Caller.Subject, + Audience = record.Caller.Audience, + ClientId = record.Caller.ClientId, + MtlsThumbprint = record.Caller.MtlsThumbprint, + Tenant = record.Caller.Tenant + }, + Metadata = metadata + }; + } + + internal sealed class CallerDocument + { + [BsonElement("subject")] + public string? Subject { get; set; } + + [BsonElement("audience")] + public string? Audience { get; set; } + + [BsonElement("clientId")] + public string? ClientId { get; set; } + + [BsonElement("mtlsThumbprint")] + public string? MtlsThumbprint { get; set; } + + [BsonElement("tenant")] + public string? Tenant { get; set; } + } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorEntryRepository.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorEntryRepository.cs new file mode 100644 index 00000000..2673d71e --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/MongoAttestorEntryRepository.cs @@ -0,0 +1,342 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using MongoDB.Driver; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class MongoAttestorEntryRepository : IAttestorEntryRepository +{ + private readonly IMongoCollection _entries; + + public MongoAttestorEntryRepository(IMongoCollection entries) + { + _entries = entries; + } + + public async Task GetByBundleShaAsync(string bundleSha256, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.BundleSha256, bundleSha256); + var document = await _entries.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToDomain(); + } + + public async Task GetByUuidAsync(string rekorUuid, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.Id, rekorUuid); + var document = await _entries.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToDomain(); + } + + public async Task> GetByArtifactShaAsync(string artifactSha256, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.Artifact.Sha256, artifactSha256); + var documents = await _entries.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return documents.ConvertAll(static doc => doc.ToDomain()); + } + + public async Task SaveAsync(AttestorEntry entry, CancellationToken cancellationToken = default) + { + var document = AttestorEntryDocument.FromDomain(entry); + var filter = Builders.Filter.Eq(x => x.Id, document.Id); + await _entries.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + [BsonIgnoreExtraElements] + internal sealed class AttestorEntryDocument + { + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("artifact")] + public ArtifactDocument Artifact { get; set; } = new(); + + [BsonElement("bundleSha256")] + public string BundleSha256 { get; set; } = string.Empty; + + [BsonElement("index")] + public long? Index { get; set; } + + [BsonElement("proof")] + public ProofDocument? Proof { get; set; } + + [BsonElement("log")] + public LogDocument Log { get; set; } = new(); + + [BsonElement("createdAt")] + public BsonDateTime CreatedAt { get; set; } = BsonDateTime.Create(System.DateTimeOffset.UtcNow); + + [BsonElement("status")] + public string Status { get; set; } = "pending"; + + [BsonElement("signerIdentity")] + public SignerIdentityDocument SignerIdentity { get; set; } = new(); + + [BsonElement("mirror")] + public MirrorDocument? Mirror { get; set; } + + public static AttestorEntryDocument FromDomain(AttestorEntry entry) + { + return new AttestorEntryDocument + { + Id = entry.RekorUuid, + Artifact = new ArtifactDocument + { + Sha256 = entry.Artifact.Sha256, + Kind = entry.Artifact.Kind, + ImageDigest = entry.Artifact.ImageDigest, + SubjectUri = entry.Artifact.SubjectUri + }, + BundleSha256 = entry.BundleSha256, + Index = entry.Index, + Proof = entry.Proof is null ? null : new ProofDocument + { + Checkpoint = entry.Proof.Checkpoint is null ? null : new CheckpointDocument + { + Origin = entry.Proof.Checkpoint.Origin, + Size = entry.Proof.Checkpoint.Size, + RootHash = entry.Proof.Checkpoint.RootHash, + Timestamp = entry.Proof.Checkpoint.Timestamp is null + ? null + : BsonDateTime.Create(entry.Proof.Checkpoint.Timestamp.Value) + }, + Inclusion = entry.Proof.Inclusion is null ? null : new InclusionDocument + { + LeafHash = entry.Proof.Inclusion.LeafHash, + Path = entry.Proof.Inclusion.Path + } + }, + Log = new LogDocument + { + Backend = entry.Log.Backend, + Url = entry.Log.Url, + LogId = entry.Log.LogId + }, + CreatedAt = BsonDateTime.Create(entry.CreatedAt.UtcDateTime), + Status = entry.Status, + SignerIdentity = new SignerIdentityDocument + { + Mode = entry.SignerIdentity.Mode, + Issuer = entry.SignerIdentity.Issuer, + SubjectAlternativeName = entry.SignerIdentity.SubjectAlternativeName, + KeyId = entry.SignerIdentity.KeyId + }, + Mirror = entry.Mirror is null ? null : MirrorDocument.FromDomain(entry.Mirror) + }; + } + + public AttestorEntry ToDomain() + { + return new AttestorEntry + { + RekorUuid = Id, + Artifact = new AttestorEntry.ArtifactDescriptor + { + Sha256 = Artifact.Sha256, + Kind = Artifact.Kind, + ImageDigest = Artifact.ImageDigest, + SubjectUri = Artifact.SubjectUri + }, + BundleSha256 = BundleSha256, + Index = Index, + Proof = Proof is null ? null : new AttestorEntry.ProofDescriptor + { + Checkpoint = Proof.Checkpoint is null ? null : new AttestorEntry.CheckpointDescriptor + { + Origin = Proof.Checkpoint.Origin, + Size = Proof.Checkpoint.Size, + RootHash = Proof.Checkpoint.RootHash, + Timestamp = Proof.Checkpoint.Timestamp?.ToUniversalTime() + }, + Inclusion = Proof.Inclusion is null ? null : new AttestorEntry.InclusionDescriptor + { + LeafHash = Proof.Inclusion.LeafHash, + Path = Proof.Inclusion.Path + } + }, + Log = new AttestorEntry.LogDescriptor + { + Backend = Log.Backend, + Url = Log.Url, + LogId = Log.LogId + }, + CreatedAt = CreatedAt.ToUniversalTime(), + Status = Status, + SignerIdentity = new AttestorEntry.SignerIdentityDescriptor + { + Mode = SignerIdentity.Mode, + Issuer = SignerIdentity.Issuer, + SubjectAlternativeName = SignerIdentity.SubjectAlternativeName, + KeyId = SignerIdentity.KeyId + }, + Mirror = Mirror?.ToDomain() + }; + } + + internal sealed class ArtifactDocument + { + [BsonElement("sha256")] + public string Sha256 { get; set; } = string.Empty; + + [BsonElement("kind")] + public string Kind { get; set; } = string.Empty; + + [BsonElement("imageDigest")] + public string? ImageDigest { get; set; } + + [BsonElement("subjectUri")] + public string? SubjectUri { get; set; } + } + + internal sealed class ProofDocument + { + [BsonElement("checkpoint")] + public CheckpointDocument? Checkpoint { get; set; } + + [BsonElement("inclusion")] + public InclusionDocument? Inclusion { get; set; } + } + + internal sealed class CheckpointDocument + { + [BsonElement("origin")] + public string? Origin { get; set; } + + [BsonElement("size")] + public long Size { get; set; } + + [BsonElement("rootHash")] + public string? RootHash { get; set; } + + [BsonElement("timestamp")] + public BsonDateTime? Timestamp { get; set; } + } + + internal sealed class InclusionDocument + { + [BsonElement("leafHash")] + public string? LeafHash { get; set; } + + [BsonElement("path")] + public IReadOnlyList Path { get; set; } = System.Array.Empty(); + } + + internal sealed class LogDocument + { + [BsonElement("backend")] + public string Backend { get; set; } = "primary"; + + [BsonElement("url")] + public string Url { get; set; } = string.Empty; + + [BsonElement("logId")] + public string? LogId { get; set; } + } + + internal sealed class SignerIdentityDocument + { + [BsonElement("mode")] + public string Mode { get; set; } = string.Empty; + + [BsonElement("issuer")] + public string? Issuer { get; set; } + + [BsonElement("san")] + public string? SubjectAlternativeName { get; set; } + + [BsonElement("kid")] + public string? KeyId { get; set; } + } + + internal sealed class MirrorDocument + { + [BsonElement("backend")] + public string Backend { get; set; } = string.Empty; + + [BsonElement("url")] + public string Url { get; set; } = string.Empty; + + [BsonElement("uuid")] + public string? Uuid { get; set; } + + [BsonElement("index")] + public long? Index { get; set; } + + [BsonElement("status")] + public string Status { get; set; } = "pending"; + + [BsonElement("proof")] + public ProofDocument? Proof { get; set; } + + [BsonElement("logId")] + public string? LogId { get; set; } + + [BsonElement("error")] + public string? Error { get; set; } + + public static MirrorDocument FromDomain(AttestorEntry.LogReplicaDescriptor mirror) + { + return new MirrorDocument + { + Backend = mirror.Backend, + Url = mirror.Url, + Uuid = mirror.Uuid, + Index = mirror.Index, + Status = mirror.Status, + Proof = mirror.Proof is null ? null : new ProofDocument + { + Checkpoint = mirror.Proof.Checkpoint is null ? null : new CheckpointDocument + { + Origin = mirror.Proof.Checkpoint.Origin, + Size = mirror.Proof.Checkpoint.Size, + RootHash = mirror.Proof.Checkpoint.RootHash, + Timestamp = mirror.Proof.Checkpoint.Timestamp is null + ? null + : BsonDateTime.Create(mirror.Proof.Checkpoint.Timestamp.Value) + }, + Inclusion = mirror.Proof.Inclusion is null ? null : new InclusionDocument + { + LeafHash = mirror.Proof.Inclusion.LeafHash, + Path = mirror.Proof.Inclusion.Path + } + }, + LogId = mirror.LogId, + Error = mirror.Error + }; + } + + public AttestorEntry.LogReplicaDescriptor ToDomain() + { + return new AttestorEntry.LogReplicaDescriptor + { + Backend = Backend, + Url = Url, + Uuid = Uuid, + Index = Index, + Status = Status, + Proof = Proof is null ? null : new AttestorEntry.ProofDescriptor + { + Checkpoint = Proof.Checkpoint is null ? null : new AttestorEntry.CheckpointDescriptor + { + Origin = Proof.Checkpoint.Origin, + Size = Proof.Checkpoint.Size, + RootHash = Proof.Checkpoint.RootHash, + Timestamp = Proof.Checkpoint.Timestamp?.ToUniversalTime() + }, + Inclusion = Proof.Inclusion is null ? null : new AttestorEntry.InclusionDescriptor + { + LeafHash = Proof.Inclusion.LeafHash, + Path = Proof.Inclusion.Path + } + }, + LogId = LogId, + Error = Error + }; + } + } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/NullAttestorArchiveStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/NullAttestorArchiveStore.cs new file mode 100644 index 00000000..f2643082 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/NullAttestorArchiveStore.cs @@ -0,0 +1,22 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class NullAttestorArchiveStore : IAttestorArchiveStore +{ + private readonly ILogger _logger; + + public NullAttestorArchiveStore(ILogger logger) + { + _logger = logger; + } + + public Task ArchiveBundleAsync(AttestorArchiveBundle bundle, CancellationToken cancellationToken = default) + { + _logger.LogDebug("Archive disabled; skipping bundle {BundleSha}", bundle.BundleSha256); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/RedisAttestorDedupeStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/RedisAttestorDedupeStore.cs new file mode 100644 index 00000000..677da49f --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/RedisAttestorDedupeStore.cs @@ -0,0 +1,34 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using StackExchange.Redis; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class RedisAttestorDedupeStore : IAttestorDedupeStore +{ + private readonly IDatabase _database; + private readonly string _prefix; + + public RedisAttestorDedupeStore(IConnectionMultiplexer multiplexer, IOptions options) + { + _database = multiplexer.GetDatabase(); + _prefix = options.Value.Redis.DedupePrefix ?? "attestor:dedupe:"; + } + + public async Task TryGetExistingAsync(string bundleSha256, CancellationToken cancellationToken = default) + { + var value = await _database.StringGetAsync(BuildKey(bundleSha256)).ConfigureAwait(false); + return value.HasValue ? value.ToString() : null; + } + + public Task SetAsync(string bundleSha256, string rekorUuid, TimeSpan ttl, CancellationToken cancellationToken = default) + { + return _database.StringSetAsync(BuildKey(bundleSha256), rekorUuid, ttl); + } + + private RedisKey BuildKey(string bundleSha256) => new RedisKey(_prefix + bundleSha256); +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/S3AttestorArchiveStore.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/S3AttestorArchiveStore.cs new file mode 100644 index 00000000..c63cba8d --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Storage/S3AttestorArchiveStore.cs @@ -0,0 +1,72 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Amazon.S3; +using Amazon.S3.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Infrastructure.Storage; + +internal sealed class S3AttestorArchiveStore : IAttestorArchiveStore, IDisposable +{ + private readonly IAmazonS3 _s3; + private readonly AttestorOptions.S3Options _options; + private readonly ILogger _logger; + private bool _disposed; + + public S3AttestorArchiveStore(IAmazonS3 s3, IOptions options, ILogger logger) + { + _s3 = s3; + _options = options.Value.S3; + _logger = logger; + } + + public async Task ArchiveBundleAsync(AttestorArchiveBundle bundle, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(_options.Bucket)) + { + _logger.LogWarning("S3 archive bucket is not configured; skipping archive for bundle {Bundle}", bundle.BundleSha256); + return; + } + + var prefix = _options.Prefix ?? "attest/"; + + await PutObjectAsync(prefix + "dsse/" + bundle.BundleSha256 + ".json", bundle.CanonicalBundleJson, cancellationToken).ConfigureAwait(false); + if (bundle.ProofJson.Length > 0) + { + await PutObjectAsync(prefix + "proof/" + bundle.RekorUuid + ".json", bundle.ProofJson, cancellationToken).ConfigureAwait(false); + } + + var metadataObject = JsonSerializer.SerializeToUtf8Bytes(bundle.Metadata); + await PutObjectAsync(prefix + "meta/" + bundle.RekorUuid + ".json", metadataObject, cancellationToken).ConfigureAwait(false); + } + + private Task PutObjectAsync(string key, byte[] content, CancellationToken cancellationToken) + { + using var stream = new MemoryStream(content); + var request = new PutObjectRequest + { + BucketName = _options.Bucket, + Key = key, + InputStream = stream, + AutoCloseStream = false + }; + return _s3.PutObjectAsync(request, cancellationToken); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _s3.Dispose(); + _disposed = true; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/AttestorSubmissionService.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/AttestorSubmissionService.cs new file mode 100644 index 00000000..36c5aa5a --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/AttestorSubmissionService.cs @@ -0,0 +1,624 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Audit; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Observability; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Storage; +using StellaOps.Attestor.Core.Submission; + +namespace StellaOps.Attestor.Infrastructure.Submission; + +internal sealed class AttestorSubmissionService : IAttestorSubmissionService +{ + private static readonly TimeSpan DedupeTtl = TimeSpan.FromHours(48); + + private readonly AttestorSubmissionValidator _validator; + private readonly IAttestorEntryRepository _repository; + private readonly IAttestorDedupeStore _dedupeStore; + private readonly IRekorClient _rekorClient; + private readonly IAttestorArchiveStore _archiveStore; + private readonly IAttestorAuditSink _auditSink; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly AttestorOptions _options; + private readonly AttestorMetrics _metrics; + + public AttestorSubmissionService( + AttestorSubmissionValidator validator, + IAttestorEntryRepository repository, + IAttestorDedupeStore dedupeStore, + IRekorClient rekorClient, + IAttestorArchiveStore archiveStore, + IAttestorAuditSink auditSink, + IOptions options, + ILogger logger, + TimeProvider timeProvider, + AttestorMetrics metrics) + { + _validator = validator; + _repository = repository; + _dedupeStore = dedupeStore; + _rekorClient = rekorClient; + _archiveStore = archiveStore; + _auditSink = auditSink; + _logger = logger; + _timeProvider = timeProvider; + _options = options.Value; + _metrics = metrics; + } + + public async Task SubmitAsync( + AttestorSubmissionRequest request, + SubmissionContext context, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(context); + + var validation = await _validator.ValidateAsync(request, cancellationToken).ConfigureAwait(false); + var canonicalBundle = validation.CanonicalBundle; + + var preference = NormalizeLogPreference(request.Meta.LogPreference); + var requiresPrimary = preference is "primary" or "both"; + var requiresMirror = preference is "mirror" or "both"; + + if (!requiresPrimary && !requiresMirror) + { + requiresPrimary = true; + } + + if (requiresMirror && !_options.Rekor.Mirror.Enabled) + { + throw new AttestorValidationException("mirror_disabled", "Mirror log requested but not configured."); + } + + var existing = await TryGetExistingEntryAsync(request.Meta.BundleSha256, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + _metrics.DedupeHitsTotal.Add(1, new KeyValuePair("result", "hit")); + var updated = await EnsureBackendsAsync(existing, request, context, requiresPrimary, requiresMirror, cancellationToken).ConfigureAwait(false); + return ToResult(updated); + } + + _metrics.DedupeHitsTotal.Add(1, new KeyValuePair("result", "miss")); + + SubmissionOutcome? canonicalOutcome = null; + SubmissionOutcome? mirrorOutcome = null; + + if (requiresPrimary) + { + canonicalOutcome = await SubmitToBackendAsync(request, "primary", _options.Rekor.Primary, cancellationToken).ConfigureAwait(false); + } + + if (requiresMirror) + { + try + { + var mirror = await SubmitToBackendAsync(request, "mirror", _options.Rekor.Mirror, cancellationToken).ConfigureAwait(false); + if (canonicalOutcome is null) + { + canonicalOutcome = mirror; + } + else + { + mirrorOutcome = mirror; + } + } + catch (Exception ex) + { + if (canonicalOutcome is null) + { + throw; + } + + _metrics.ErrorTotal.Add(1, new KeyValuePair("type", "submit_mirror")); + _logger.LogWarning(ex, "Mirror submission failed for bundle {BundleSha}", request.Meta.BundleSha256); + mirrorOutcome = SubmissionOutcome.Failure("mirror", _options.Rekor.Mirror.Url, ex, TimeSpan.Zero); + RecordSubmissionMetrics(mirrorOutcome); + } + } + + if (canonicalOutcome is null) + { + throw new InvalidOperationException("No Rekor submission outcome was produced."); + } + + var entry = CreateEntry(request, context, canonicalOutcome, mirrorOutcome); + await _repository.SaveAsync(entry, cancellationToken).ConfigureAwait(false); + await _dedupeStore.SetAsync(request.Meta.BundleSha256, entry.RekorUuid, DedupeTtl, cancellationToken).ConfigureAwait(false); + + if (request.Meta.Archive) + { + await ArchiveAsync(entry, canonicalBundle, canonicalOutcome.Proof, cancellationToken).ConfigureAwait(false); + } + + await WriteAuditAsync(request, context, entry, canonicalOutcome, cancellationToken).ConfigureAwait(false); + if (mirrorOutcome is not null) + { + await WriteAuditAsync(request, context, entry, mirrorOutcome, cancellationToken).ConfigureAwait(false); + } + + return ToResult(entry); + } + + private static AttestorSubmissionResult ToResult(AttestorEntry entry) + { + var result = new AttestorSubmissionResult + { + Uuid = entry.RekorUuid, + Index = entry.Index, + LogUrl = entry.Log.Url, + Status = entry.Status, + Proof = ToResultProof(entry.Proof) + }; + + if (entry.Mirror is not null) + { + result.Mirror = new AttestorSubmissionResult.MirrorLog + { + Uuid = entry.Mirror.Uuid, + Index = entry.Mirror.Index, + LogUrl = entry.Mirror.Url, + Status = entry.Mirror.Status, + Proof = ToResultProof(entry.Mirror.Proof), + Error = entry.Mirror.Error + }; + } + + return result; + } + + private AttestorEntry CreateEntry( + AttestorSubmissionRequest request, + SubmissionContext context, + SubmissionOutcome canonicalOutcome, + SubmissionOutcome? mirrorOutcome) + { + if (canonicalOutcome.Submission is null) + { + throw new InvalidOperationException("Canonical submission outcome must include a Rekor response."); + } + + var submission = canonicalOutcome.Submission; + var now = _timeProvider.GetUtcNow(); + + return new AttestorEntry + { + RekorUuid = submission.Uuid, + Artifact = new AttestorEntry.ArtifactDescriptor + { + Sha256 = request.Meta.Artifact.Sha256, + Kind = request.Meta.Artifact.Kind, + ImageDigest = request.Meta.Artifact.ImageDigest, + SubjectUri = request.Meta.Artifact.SubjectUri + }, + BundleSha256 = request.Meta.BundleSha256, + Index = submission.Index, + Proof = ConvertProof(canonicalOutcome.Proof), + Log = new AttestorEntry.LogDescriptor + { + Backend = canonicalOutcome.Backend, + Url = submission.LogUrl ?? canonicalOutcome.Url, + LogId = null + }, + CreatedAt = now, + Status = submission.Status ?? "included", + SignerIdentity = new AttestorEntry.SignerIdentityDescriptor + { + Mode = request.Bundle.Mode, + Issuer = context.CallerAudience, + SubjectAlternativeName = context.CallerSubject, + KeyId = context.CallerClientId + }, + Mirror = mirrorOutcome is null ? null : CreateMirrorDescriptor(mirrorOutcome) + }; + } + + private static string NormalizeLogPreference(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "primary"; + } + + var normalized = value.Trim().ToLowerInvariant(); + return normalized switch + { + "primary" => "primary", + "mirror" => "mirror", + "both" => "both", + _ => "primary" + }; + } + + private async Task TryGetExistingEntryAsync(string bundleSha256, CancellationToken cancellationToken) + { + var dedupeUuid = await _dedupeStore.TryGetExistingAsync(bundleSha256, cancellationToken).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(dedupeUuid)) + { + return null; + } + + return await _repository.GetByUuidAsync(dedupeUuid, cancellationToken).ConfigureAwait(false) + ?? await _repository.GetByBundleShaAsync(bundleSha256, cancellationToken).ConfigureAwait(false); + } + + private async Task EnsureBackendsAsync( + AttestorEntry existing, + AttestorSubmissionRequest request, + SubmissionContext context, + bool requiresPrimary, + bool requiresMirror, + CancellationToken cancellationToken) + { + var entry = existing; + var updated = false; + + if (requiresPrimary && !IsPrimary(entry)) + { + var outcome = await SubmitToBackendAsync(request, "primary", _options.Rekor.Primary, cancellationToken).ConfigureAwait(false); + entry = PromoteToPrimary(entry, outcome); + await _repository.SaveAsync(entry, cancellationToken).ConfigureAwait(false); + await _dedupeStore.SetAsync(request.Meta.BundleSha256, entry.RekorUuid, DedupeTtl, cancellationToken).ConfigureAwait(false); + await WriteAuditAsync(request, context, entry, outcome, cancellationToken).ConfigureAwait(false); + updated = true; + } + + if (requiresMirror) + { + var mirrorSatisfied = entry.Mirror is not null + && entry.Mirror.Error is null + && string.Equals(entry.Mirror.Status, "included", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrEmpty(entry.Mirror.Uuid); + + if (!mirrorSatisfied) + { + try + { + var mirrorOutcome = await SubmitToBackendAsync(request, "mirror", _options.Rekor.Mirror, cancellationToken).ConfigureAwait(false); + entry = WithMirror(entry, mirrorOutcome); + await _repository.SaveAsync(entry, cancellationToken).ConfigureAwait(false); + await WriteAuditAsync(request, context, entry, mirrorOutcome, cancellationToken).ConfigureAwait(false); + updated = true; + } + catch (Exception ex) + { + _metrics.ErrorTotal.Add(1, new KeyValuePair("type", "submit_mirror")); + _logger.LogWarning(ex, "Mirror submission failed for deduplicated bundle {BundleSha}", request.Meta.BundleSha256); + var failure = SubmissionOutcome.Failure("mirror", _options.Rekor.Mirror.Url, ex, TimeSpan.Zero); + RecordSubmissionMetrics(failure); + entry = WithMirror(entry, failure); + await _repository.SaveAsync(entry, cancellationToken).ConfigureAwait(false); + await WriteAuditAsync(request, context, entry, failure, cancellationToken).ConfigureAwait(false); + updated = true; + } + } + } + + if (!updated) + { + _metrics.SubmitTotal.Add(1, + new KeyValuePair("result", "dedupe"), + new KeyValuePair("backend", "cache")); + } + + return entry; + } + + private static bool IsPrimary(AttestorEntry entry) => + string.Equals(entry.Log.Backend, "primary", StringComparison.OrdinalIgnoreCase); + + private async Task SubmitToBackendAsync( + AttestorSubmissionRequest request, + string backendName, + AttestorOptions.RekorBackendOptions backendOptions, + CancellationToken cancellationToken) + { + var backend = BuildBackend(backendName, backendOptions); + var stopwatch = Stopwatch.StartNew(); + try + { + var submission = await _rekorClient.SubmitAsync(request, backend, cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + + var proof = submission.Proof; + if (proof is null && string.Equals(submission.Status, "included", StringComparison.OrdinalIgnoreCase)) + { + try + { + proof = await _rekorClient.GetProofAsync(submission.Uuid, backend, cancellationToken).ConfigureAwait(false); + _metrics.ProofFetchTotal.Add(1, + new KeyValuePair("result", proof is null ? "missing" : "ok")); + } + catch (Exception ex) + { + _metrics.ErrorTotal.Add(1, new KeyValuePair("type", "proof_fetch")); + _logger.LogWarning(ex, "Proof fetch failed for {Uuid} on backend {Backend}", submission.Uuid, backendName); + } + } + + var outcome = SubmissionOutcome.Success(backendName, backend.Url, submission, proof, stopwatch.Elapsed); + RecordSubmissionMetrics(outcome); + return outcome; + } + catch (Exception ex) + { + stopwatch.Stop(); + _metrics.ErrorTotal.Add(1, new KeyValuePair("type", $"submit_{backendName}")); + _logger.LogError(ex, "Failed to submit bundle {BundleSha} to Rekor backend {Backend}", request.Meta.BundleSha256, backendName); + throw; + } + } + + private void RecordSubmissionMetrics(SubmissionOutcome outcome) + { + var result = outcome.IsSuccess + ? outcome.Submission!.Status ?? "unknown" + : "failed"; + + _metrics.SubmitTotal.Add(1, + new KeyValuePair("result", result), + new KeyValuePair("backend", outcome.Backend)); + + if (outcome.Latency > TimeSpan.Zero) + { + _metrics.SubmitLatency.Record(outcome.Latency.TotalSeconds, + new KeyValuePair("backend", outcome.Backend)); + } + } + + private async Task ArchiveAsync( + AttestorEntry entry, + byte[] canonicalBundle, + RekorProofResponse? proof, + CancellationToken cancellationToken) + { + var metadata = new Dictionary + { + ["logUrl"] = entry.Log.Url, + ["status"] = entry.Status + }; + + if (entry.Mirror is not null) + { + metadata["mirror.backend"] = entry.Mirror.Backend; + metadata["mirror.uuid"] = entry.Mirror.Uuid ?? string.Empty; + metadata["mirror.status"] = entry.Mirror.Status; + } + + var archiveBundle = new AttestorArchiveBundle + { + RekorUuid = entry.RekorUuid, + ArtifactSha256 = entry.Artifact.Sha256, + BundleSha256 = entry.BundleSha256, + CanonicalBundleJson = canonicalBundle, + ProofJson = proof is null ? Array.Empty() : JsonSerializer.SerializeToUtf8Bytes(proof, JsonSerializerOptions.Default), + Metadata = metadata + }; + + try + { + await _archiveStore.ArchiveBundleAsync(archiveBundle, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to archive bundle {BundleSha}", entry.BundleSha256); + _metrics.ErrorTotal.Add(1, new KeyValuePair("type", "archive")); + } + } + + private Task WriteAuditAsync( + AttestorSubmissionRequest request, + SubmissionContext context, + AttestorEntry entry, + SubmissionOutcome outcome, + CancellationToken cancellationToken) + { + var metadata = new Dictionary(); + if (!outcome.IsSuccess && outcome.Error is not null) + { + metadata["error"] = outcome.Error.Message; + } + + var record = new AttestorAuditRecord + { + Action = "submit", + Result = outcome.IsSuccess + ? outcome.Submission!.Status ?? "included" + : "failed", + RekorUuid = outcome.IsSuccess + ? outcome.Submission!.Uuid + : string.Equals(outcome.Backend, "primary", StringComparison.OrdinalIgnoreCase) + ? entry.RekorUuid + : entry.Mirror?.Uuid, + Index = outcome.Submission?.Index, + ArtifactSha256 = request.Meta.Artifact.Sha256, + BundleSha256 = request.Meta.BundleSha256, + Backend = outcome.Backend, + LatencyMs = (long)outcome.Latency.TotalMilliseconds, + Timestamp = _timeProvider.GetUtcNow(), + Caller = new AttestorAuditRecord.CallerDescriptor + { + Subject = context.CallerSubject, + Audience = context.CallerAudience, + ClientId = context.CallerClientId, + MtlsThumbprint = context.MtlsThumbprint, + Tenant = context.CallerTenant + }, + Metadata = metadata + }; + + return _auditSink.WriteAsync(record, cancellationToken); + } + + private static AttestorEntry.ProofDescriptor? ConvertProof(RekorProofResponse? proof) + { + if (proof is null) + { + return null; + } + + return new AttestorEntry.ProofDescriptor + { + Checkpoint = proof.Checkpoint is null ? null : new AttestorEntry.CheckpointDescriptor + { + Origin = proof.Checkpoint.Origin, + Size = proof.Checkpoint.Size, + RootHash = proof.Checkpoint.RootHash, + Timestamp = proof.Checkpoint.Timestamp + }, + Inclusion = proof.Inclusion is null ? null : new AttestorEntry.InclusionDescriptor + { + LeafHash = proof.Inclusion.LeafHash, + Path = proof.Inclusion.Path + } + }; + } + + private static AttestorSubmissionResult.RekorProof? ToResultProof(AttestorEntry.ProofDescriptor? proof) + { + if (proof is null) + { + return null; + } + + return new AttestorSubmissionResult.RekorProof + { + Checkpoint = proof.Checkpoint is null ? null : new AttestorSubmissionResult.Checkpoint + { + Origin = proof.Checkpoint.Origin, + Size = proof.Checkpoint.Size, + RootHash = proof.Checkpoint.RootHash, + Timestamp = proof.Checkpoint.Timestamp?.ToString("O") + }, + Inclusion = proof.Inclusion is null ? null : new AttestorSubmissionResult.InclusionProof + { + LeafHash = proof.Inclusion.LeafHash, + Path = proof.Inclusion.Path + } + }; + } + + private static AttestorEntry.LogReplicaDescriptor CreateMirrorDescriptor(SubmissionOutcome outcome) + { + return new AttestorEntry.LogReplicaDescriptor + { + Backend = outcome.Backend, + Url = outcome.IsSuccess + ? outcome.Submission!.LogUrl ?? outcome.Url + : outcome.Url, + Uuid = outcome.Submission?.Uuid, + Index = outcome.Submission?.Index, + Status = outcome.IsSuccess + ? outcome.Submission!.Status ?? "included" + : "failed", + Proof = outcome.IsSuccess ? ConvertProof(outcome.Proof) : null, + Error = outcome.Error?.Message + }; + } + + private static AttestorEntry WithMirror(AttestorEntry entry, SubmissionOutcome outcome) + { + return new AttestorEntry + { + RekorUuid = entry.RekorUuid, + Artifact = entry.Artifact, + BundleSha256 = entry.BundleSha256, + Index = entry.Index, + Proof = entry.Proof, + Log = entry.Log, + CreatedAt = entry.CreatedAt, + Status = entry.Status, + SignerIdentity = entry.SignerIdentity, + Mirror = CreateMirrorDescriptor(outcome) + }; + } + + private AttestorEntry PromoteToPrimary(AttestorEntry existing, SubmissionOutcome outcome) + { + if (outcome.Submission is null) + { + throw new InvalidOperationException("Cannot promote to primary without a successful submission."); + } + + var mirrorDescriptor = existing.Mirror; + if (mirrorDescriptor is null && !string.Equals(existing.Log.Backend, outcome.Backend, StringComparison.OrdinalIgnoreCase)) + { + mirrorDescriptor = CreateMirrorDescriptorFromEntry(existing); + } + + return new AttestorEntry + { + RekorUuid = outcome.Submission.Uuid, + Artifact = existing.Artifact, + BundleSha256 = existing.BundleSha256, + Index = outcome.Submission.Index, + Proof = ConvertProof(outcome.Proof), + Log = new AttestorEntry.LogDescriptor + { + Backend = outcome.Backend, + Url = outcome.Submission.LogUrl ?? outcome.Url, + LogId = existing.Log.LogId + }, + CreatedAt = existing.CreatedAt, + Status = outcome.Submission.Status ?? "included", + SignerIdentity = existing.SignerIdentity, + Mirror = mirrorDescriptor + }; + } + + private static AttestorEntry.LogReplicaDescriptor CreateMirrorDescriptorFromEntry(AttestorEntry entry) + { + return new AttestorEntry.LogReplicaDescriptor + { + Backend = entry.Log.Backend, + Url = entry.Log.Url, + Uuid = entry.RekorUuid, + Index = entry.Index, + Status = entry.Status, + Proof = entry.Proof, + LogId = entry.Log.LogId + }; + } + + private sealed record SubmissionOutcome( + string Backend, + string Url, + RekorSubmissionResponse? Submission, + RekorProofResponse? Proof, + TimeSpan Latency, + Exception? Error) + { + public bool IsSuccess => Submission is not null && Error is null; + + public static SubmissionOutcome Success(string backend, Uri backendUrl, RekorSubmissionResponse submission, RekorProofResponse? proof, TimeSpan latency) => + new SubmissionOutcome(backend, backendUrl.ToString(), submission, proof, latency, null); + + public static SubmissionOutcome Failure(string backend, string? url, Exception error, TimeSpan latency) => + new SubmissionOutcome(backend, url ?? string.Empty, null, null, latency, error); + } + + private static RekorBackend BuildBackend(string name, AttestorOptions.RekorBackendOptions options) + { + if (string.IsNullOrWhiteSpace(options.Url)) + { + throw new InvalidOperationException($"Rekor backend '{name}' is not configured."); + } + + return new RekorBackend + { + Name = name, + Url = new Uri(options.Url, UriKind.Absolute), + ProofTimeout = TimeSpan.FromMilliseconds(options.ProofTimeoutMs), + PollInterval = TimeSpan.FromMilliseconds(options.PollIntervalMs), + MaxAttempts = options.MaxAttempts + }; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/DefaultDsseCanonicalizer.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/DefaultDsseCanonicalizer.cs new file mode 100644 index 00000000..cb0abd58 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Submission/DefaultDsseCanonicalizer.cs @@ -0,0 +1,49 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Attestor.Core.Submission; + +namespace StellaOps.Attestor.Infrastructure.Submission; + +public sealed class DefaultDsseCanonicalizer : IDsseCanonicalizer +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }; + + public Task CanonicalizeAsync(AttestorSubmissionRequest request, CancellationToken cancellationToken = default) + { + var node = new JsonObject + { + ["payloadType"] = request.Bundle.Dsse.PayloadType, + ["payload"] = request.Bundle.Dsse.PayloadBase64, + ["signatures"] = CreateSignaturesArray(request) + }; + + var json = node.ToJsonString(SerializerOptions); + return Task.FromResult(JsonSerializer.SerializeToUtf8Bytes(JsonNode.Parse(json)!, SerializerOptions)); + } + + private static JsonArray CreateSignaturesArray(AttestorSubmissionRequest request) + { + var array = new JsonArray(); + foreach (var signature in request.Bundle.Dsse.Signatures) + { + var obj = new JsonObject + { + ["sig"] = signature.Signature + }; + if (!string.IsNullOrWhiteSpace(signature.KeyId)) + { + obj["keyid"] = signature.KeyId; + } + + array.Add(obj); + } + + return array; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/AttestorVerificationService.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/AttestorVerificationService.cs new file mode 100644 index 00000000..ab0ed164 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Infrastructure/Verification/AttestorVerificationService.cs @@ -0,0 +1,754 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Storage; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Core.Verification; +using StellaOps.Attestor.Core.Observability; + +namespace StellaOps.Attestor.Infrastructure.Verification; + +internal sealed class AttestorVerificationService : IAttestorVerificationService +{ + private readonly IAttestorEntryRepository _repository; + private readonly IDsseCanonicalizer _canonicalizer; + private readonly IRekorClient _rekorClient; + private readonly ILogger _logger; + private readonly AttestorOptions _options; + private readonly AttestorMetrics _metrics; + + public AttestorVerificationService( + IAttestorEntryRepository repository, + IDsseCanonicalizer canonicalizer, + IRekorClient rekorClient, + IOptions options, + ILogger logger, + AttestorMetrics metrics) + { + _repository = repository; + _canonicalizer = canonicalizer; + _rekorClient = rekorClient; + _logger = logger; + _options = options.Value; + _metrics = metrics; + } + + public async Task VerifyAsync(AttestorVerificationRequest request, CancellationToken cancellationToken = default) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + var entry = await ResolveEntryAsync(request, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + throw new AttestorVerificationException("not_found", "No attestor entry matched the supplied query."); + } + + var issues = new List(); + + if (request.Bundle is not null) + { + var canonicalBundle = await _canonicalizer.CanonicalizeAsync(new AttestorSubmissionRequest + { + Bundle = request.Bundle, + Meta = new AttestorSubmissionRequest.SubmissionMeta + { + Artifact = new AttestorSubmissionRequest.ArtifactInfo + { + Sha256 = entry.Artifact.Sha256, + Kind = entry.Artifact.Kind + }, + BundleSha256 = entry.BundleSha256 + } + }, cancellationToken).ConfigureAwait(false); + + var computedHash = Convert.ToHexString(SHA256.HashData(canonicalBundle)).ToLowerInvariant(); + if (!string.Equals(computedHash, entry.BundleSha256, StringComparison.OrdinalIgnoreCase)) + { + issues.Add("bundle_hash_mismatch"); + } + + if (!TryDecodeBase64(request.Bundle.Dsse.PayloadBase64, out var payloadBytes)) + { + issues.Add("bundle_payload_invalid_base64"); + } + else + { + var preAuth = ComputePreAuthEncoding(request.Bundle.Dsse.PayloadType, payloadBytes); + VerifySignatures(entry, request.Bundle, preAuth, issues); + } + } + else + { + _logger.LogDebug("No DSSE bundle supplied for verification of {Uuid}; signature checks skipped.", entry.RekorUuid); + } + + if (request.RefreshProof || entry.Proof is null) + { + var backend = BuildBackend("primary", _options.Rekor.Primary); + try + { + var proof = await _rekorClient.GetProofAsync(entry.RekorUuid, backend, cancellationToken).ConfigureAwait(false); + if (proof is not null) + { + var updated = CloneWithProof(entry, proof.ToProofDescriptor()); + await _repository.SaveAsync(updated, cancellationToken).ConfigureAwait(false); + entry = updated; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to refresh proof for entry {Uuid}", entry.RekorUuid); + issues.Add("Proof refresh failed: " + ex.Message); + } + } + + VerifyMerkleProof(entry, issues); + + var ok = issues.Count == 0 && string.Equals(entry.Status, "included", StringComparison.OrdinalIgnoreCase); + + _metrics.VerifyTotal.Add(1, new KeyValuePair("result", ok ? "ok" : "failed")); + + return new AttestorVerificationResult + { + Ok = ok, + Uuid = entry.RekorUuid, + Index = entry.Index, + LogUrl = entry.Log.Url, + Status = entry.Status, + Issues = issues, + CheckedAt = DateTimeOffset.UtcNow + }; + } + + public Task GetEntryAsync(string rekorUuid, bool refreshProof, CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(rekorUuid)) + { + throw new ArgumentException("Value cannot be null or whitespace.", nameof(rekorUuid)); + } + + return ResolveEntryByUuidAsync(rekorUuid, refreshProof, cancellationToken); + } + + private async Task ResolveEntryAsync(AttestorVerificationRequest request, CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(request.Uuid)) + { + return await ResolveEntryByUuidAsync(request.Uuid, request.RefreshProof, cancellationToken).ConfigureAwait(false); + } + + if (request.Bundle is not null) + { + var canonical = await _canonicalizer.CanonicalizeAsync(new AttestorSubmissionRequest + { + Bundle = request.Bundle, + Meta = new AttestorSubmissionRequest.SubmissionMeta + { + Artifact = new AttestorSubmissionRequest.ArtifactInfo + { + Sha256 = string.Empty, + Kind = string.Empty + } + } + }, cancellationToken).ConfigureAwait(false); + + var bundleSha = Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(canonical)).ToLowerInvariant(); + return await ResolveEntryByBundleShaAsync(bundleSha, request.RefreshProof, cancellationToken).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(request.ArtifactSha256)) + { + return await ResolveEntryByArtifactAsync(request.ArtifactSha256, request.RefreshProof, cancellationToken).ConfigureAwait(false); + } + + throw new AttestorVerificationException("invalid_query", "At least one of uuid, bundle, or artifactSha256 must be provided."); + } + + private async Task ResolveEntryByUuidAsync(string uuid, bool refreshProof, CancellationToken cancellationToken) + { + var entry = await _repository.GetByUuidAsync(uuid, cancellationToken).ConfigureAwait(false); + if (entry is null || !refreshProof) + { + return entry; + } + + var backend = BuildBackend("primary", _options.Rekor.Primary); + try + { + var proof = await _rekorClient.GetProofAsync(uuid, backend, cancellationToken).ConfigureAwait(false); + if (proof is not null) + { + var updated = CloneWithProof(entry, proof.ToProofDescriptor()); + await _repository.SaveAsync(updated, cancellationToken).ConfigureAwait(false); + entry = updated; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to refresh proof for entry {Uuid}", uuid); + } + + return entry; + } + + private async Task ResolveEntryByBundleShaAsync(string bundleSha, bool refreshProof, CancellationToken cancellationToken) + { + var entry = await _repository.GetByBundleShaAsync(bundleSha, cancellationToken).ConfigureAwait(false); + if (entry is null || !refreshProof) + { + return entry; + } + + return await ResolveEntryByUuidAsync(entry.RekorUuid, true, cancellationToken).ConfigureAwait(false); + } + + private async Task ResolveEntryByArtifactAsync(string artifactSha256, bool refreshProof, CancellationToken cancellationToken) + { + var entries = await _repository.GetByArtifactShaAsync(artifactSha256, cancellationToken).ConfigureAwait(false); + var entry = entries.OrderByDescending(e => e.CreatedAt).FirstOrDefault(); + if (entry is null) + { + return null; + } + + return refreshProof + ? await ResolveEntryByUuidAsync(entry.RekorUuid, true, cancellationToken).ConfigureAwait(false) + : entry; + } + + private void VerifySignatures(AttestorEntry entry, AttestorSubmissionRequest.SubmissionBundle bundle, byte[] preAuthEncoding, IList issues) + { + var mode = (entry.SignerIdentity.Mode ?? bundle.Mode ?? string.Empty).ToLowerInvariant(); + + if (mode == "kms") + { + if (!VerifyKmsSignature(bundle, preAuthEncoding, issues)) + { + issues.Add("signature_invalid_kms"); + } + + return; + } + + if (mode == "keyless") + { + VerifyKeylessSignature(entry, bundle, preAuthEncoding, issues); + return; + } + + issues.Add(string.IsNullOrEmpty(mode) + ? "signer_mode_unknown" + : $"signer_mode_unsupported:{mode}"); + } + + private bool VerifyKmsSignature(AttestorSubmissionRequest.SubmissionBundle bundle, byte[] preAuthEncoding, IList issues) + { + if (_options.Security.SignerIdentity.KmsKeys.Count == 0) + { + issues.Add("kms_key_missing"); + return false; + } + + var signatures = new List(); + foreach (var signature in bundle.Dsse.Signatures) + { + if (!TryDecodeBase64(signature.Signature, out var signatureBytes)) + { + issues.Add("signature_invalid_base64"); + return false; + } + + signatures.Add(signatureBytes); + } + + foreach (var secret in _options.Security.SignerIdentity.KmsKeys) + { + if (!TryDecodeSecret(secret, out var secretBytes)) + { + continue; + } + + using var hmac = new HMACSHA256(secretBytes); + var computed = hmac.ComputeHash(preAuthEncoding); + + foreach (var signatureBytes in signatures) + { + if (CryptographicOperations.FixedTimeEquals(computed, signatureBytes)) + { + return true; + } + } + } + + return false; + } + + private void VerifyKeylessSignature(AttestorEntry entry, AttestorSubmissionRequest.SubmissionBundle bundle, byte[] preAuthEncoding, IList issues) + { + if (bundle.CertificateChain.Count == 0) + { + issues.Add("certificate_chain_missing"); + return; + } + + var certificates = new List(); + try + { + foreach (var pem in bundle.CertificateChain) + { + certificates.Add(X509Certificate2.CreateFromPem(pem)); + } + } + catch (Exception ex) when (ex is CryptographicException or ArgumentException) + { + issues.Add("certificate_chain_invalid"); + _logger.LogWarning(ex, "Failed to parse certificate chain for {Uuid}", entry.RekorUuid); + return; + } + + var leafCertificate = certificates[0]; + + if (_options.Security.SignerIdentity.FulcioRoots.Count > 0) + { + using var chain = new X509Chain + { + ChainPolicy = + { + RevocationMode = X509RevocationMode.NoCheck, + VerificationFlags = X509VerificationFlags.NoFlag, + TrustMode = X509ChainTrustMode.CustomRootTrust + } + }; + + foreach (var rootPath in _options.Security.SignerIdentity.FulcioRoots) + { + try + { + if (File.Exists(rootPath)) + { + var rootCertificate = X509CertificateLoader.LoadCertificateFromFile(rootPath); + chain.ChainPolicy.CustomTrustStore.Add(rootCertificate); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load Fulcio root {Root}", rootPath); + } + } + + if (!chain.Build(leafCertificate)) + { + var status = string.Join(";", chain.ChainStatus.Select(s => s.StatusInformation.Trim())) + .Trim(';'); + issues.Add(string.IsNullOrEmpty(status) ? "certificate_chain_untrusted" : $"certificate_chain_untrusted:{status}"); + } + } + + if (_options.Security.SignerIdentity.AllowedSans.Count > 0) + { + var sans = GetSubjectAlternativeNames(leafCertificate); + if (!sans.Any(san => _options.Security.SignerIdentity.AllowedSans.Contains(san, StringComparer.OrdinalIgnoreCase))) + { + issues.Add("certificate_san_untrusted"); + } + } + + var signatureVerified = false; + foreach (var signature in bundle.Dsse.Signatures) + { + if (!TryDecodeBase64(signature.Signature, out var signatureBytes)) + { + issues.Add("signature_invalid_base64"); + return; + } + + if (TryVerifyWithCertificate(leafCertificate, preAuthEncoding, signatureBytes)) + { + signatureVerified = true; + break; + } + } + + if (!signatureVerified) + { + issues.Add("signature_invalid"); + } + } + + private static bool TryVerifyWithCertificate(X509Certificate2 certificate, byte[] preAuthEncoding, byte[] signature) + { + try + { + var ecdsa = certificate.GetECDsaPublicKey(); + if (ecdsa is not null) + { + using (ecdsa) + { + return ecdsa.VerifyData(preAuthEncoding, signature, HashAlgorithmName.SHA256); + } + } + + var rsa = certificate.GetRSAPublicKey(); + if (rsa is not null) + { + using (rsa) + { + return rsa.VerifyData(preAuthEncoding, signature, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + } + } + } + catch (CryptographicException) + { + return false; + } + + return false; + } + + private static IEnumerable GetSubjectAlternativeNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (!string.Equals(extension.Oid?.Value, "2.5.29.17", StringComparison.Ordinal)) + { + continue; + } + + var formatted = extension.Format(true); + var lines = formatted.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + foreach (var line in lines) + { + var parts = line.Split('='); + if (parts.Length == 2) + { + yield return parts[1].Trim(); + } + } + } + } + + private static byte[] ComputePreAuthEncoding(string payloadType, byte[] payload) + { + var headerBytes = Encoding.UTF8.GetBytes(payloadType ?? string.Empty); + var buffer = new byte[6 + 8 + headerBytes.Length + 8 + payload.Length]; + var offset = 0; + + Encoding.ASCII.GetBytes("DSSEv1", 0, 6, buffer, offset); + offset += 6; + + BinaryPrimitives.WriteUInt64BigEndian(buffer.AsSpan(offset, 8), (ulong)headerBytes.Length); + offset += 8; + Buffer.BlockCopy(headerBytes, 0, buffer, offset, headerBytes.Length); + offset += headerBytes.Length; + + BinaryPrimitives.WriteUInt64BigEndian(buffer.AsSpan(offset, 8), (ulong)payload.Length); + offset += 8; + Buffer.BlockCopy(payload, 0, buffer, offset, payload.Length); + + return buffer; + } + + private void VerifyMerkleProof(AttestorEntry entry, IList issues) + { + if (entry.Proof is null) + { + issues.Add("proof_missing"); + return; + } + + if (!TryDecodeHash(entry.BundleSha256, out var bundleHash)) + { + issues.Add("bundle_hash_decode_failed"); + return; + } + + if (entry.Proof.Inclusion is null) + { + issues.Add("proof_inclusion_missing"); + return; + } + + if (entry.Proof.Inclusion.LeafHash is not null) + { + if (!TryDecodeHash(entry.Proof.Inclusion.LeafHash, out var proofLeaf)) + { + issues.Add("proof_leafhash_decode_failed"); + return; + } + + if (!CryptographicOperations.FixedTimeEquals(bundleHash, proofLeaf)) + { + issues.Add("proof_leafhash_mismatch"); + } + } + + var current = bundleHash; + + if (entry.Proof.Inclusion.Path.Count > 0) + { + var nodes = new List(); + foreach (var element in entry.Proof.Inclusion.Path) + { + if (!ProofPathNode.TryParse(element, out var node)) + { + issues.Add("proof_path_decode_failed"); + return; + } + + if (!node.HasOrientation) + { + issues.Add("proof_path_orientation_missing"); + return; + } + + nodes.Add(node); + } + + foreach (var node in nodes) + { + current = node.Left + ? HashInternal(node.Hash, current) + : HashInternal(current, node.Hash); + } + } + + if (entry.Proof.Checkpoint is null) + { + issues.Add("checkpoint_missing"); + return; + } + + if (!TryDecodeHash(entry.Proof.Checkpoint.RootHash, out var rootHash)) + { + issues.Add("checkpoint_root_decode_failed"); + return; + } + + if (!CryptographicOperations.FixedTimeEquals(current, rootHash)) + { + issues.Add("proof_root_mismatch"); + } + } + + private static byte[] HashInternal(byte[] left, byte[] right) + { + using var sha = SHA256.Create(); + var buffer = new byte[1 + left.Length + right.Length]; + buffer[0] = 0x01; + Buffer.BlockCopy(left, 0, buffer, 1, left.Length); + Buffer.BlockCopy(right, 0, buffer, 1 + left.Length, right.Length); + return sha.ComputeHash(buffer); + } + + private static bool TryDecodeSecret(string value, out byte[] bytes) + { + if (string.IsNullOrWhiteSpace(value)) + { + bytes = Array.Empty(); + return false; + } + + value = value.Trim(); + + if (value.StartsWith("base64:", StringComparison.OrdinalIgnoreCase)) + { + return TryDecodeBase64(value[7..], out bytes); + } + + if (value.StartsWith("hex:", StringComparison.OrdinalIgnoreCase)) + { + return TryDecodeHex(value[4..], out bytes); + } + + if (TryDecodeBase64(value, out bytes)) + { + return true; + } + + if (TryDecodeHex(value, out bytes)) + { + return true; + } + + bytes = Array.Empty(); + return false; + } + + private static bool TryDecodeBase64(string value, out byte[] bytes) + { + try + { + bytes = Convert.FromBase64String(value); + return true; + } + catch (FormatException) + { + bytes = Array.Empty(); + return false; + } + } + + private static bool TryDecodeHex(string value, out byte[] bytes) + { + try + { + bytes = Convert.FromHexString(value); + return true; + } + catch (FormatException) + { + bytes = Array.Empty(); + return false; + } + } + + private static bool TryDecodeHash(string? value, out byte[] bytes) + { + bytes = Array.Empty(); + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + + if (TryDecodeHex(trimmed, out bytes)) + { + return true; + } + + if (TryDecodeBase64(trimmed, out bytes)) + { + return true; + } + + bytes = Array.Empty(); + return false; + } + + private readonly struct ProofPathNode + { + private ProofPathNode(bool hasOrientation, bool left, byte[] hash) + { + HasOrientation = hasOrientation; + Left = left; + Hash = hash; + } + + public bool HasOrientation { get; } + + public bool Left { get; } + + public byte[] Hash { get; } + + public static bool TryParse(string value, out ProofPathNode node) + { + node = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var parts = trimmed.Split(':', 2); + bool hasOrientation = false; + bool left = false; + string hashPart = trimmed; + + if (parts.Length == 2) + { + var prefix = parts[0].Trim().ToLowerInvariant(); + if (prefix is "l" or "left") + { + hasOrientation = true; + left = true; + } + else if (prefix is "r" or "right") + { + hasOrientation = true; + left = false; + } + + hashPart = parts[1].Trim(); + } + + if (!TryDecodeHash(hashPart, out var hash)) + { + return false; + } + + node = new ProofPathNode(hasOrientation, left, hash); + return true; + } + } + + private static AttestorEntry CloneWithProof(AttestorEntry entry, AttestorEntry.ProofDescriptor? proof) + { + return new AttestorEntry + { + RekorUuid = entry.RekorUuid, + Artifact = entry.Artifact, + BundleSha256 = entry.BundleSha256, + Index = entry.Index, + Proof = proof, + Log = entry.Log, + CreatedAt = entry.CreatedAt, + Status = entry.Status, + SignerIdentity = entry.SignerIdentity + }; + } + + private static RekorBackend BuildBackend(string name, AttestorOptions.RekorBackendOptions options) + { + if (string.IsNullOrWhiteSpace(options.Url)) + { + throw new InvalidOperationException($"Rekor backend '{name}' is not configured."); + } + + return new RekorBackend + { + Name = name, + Url = new Uri(options.Url, UriKind.Absolute), + ProofTimeout = TimeSpan.FromMilliseconds(options.ProofTimeoutMs), + PollInterval = TimeSpan.FromMilliseconds(options.PollIntervalMs), + MaxAttempts = options.MaxAttempts + }; + } +} + +internal static class RekorProofResponseExtensions +{ + public static AttestorEntry.ProofDescriptor ToProofDescriptor(this RekorProofResponse response) + { + return new AttestorEntry.ProofDescriptor + { + Checkpoint = response.Checkpoint is null ? null : new AttestorEntry.CheckpointDescriptor + { + Origin = response.Checkpoint.Origin, + Size = response.Checkpoint.Size, + RootHash = response.Checkpoint.RootHash, + Timestamp = response.Checkpoint.Timestamp + }, + Inclusion = response.Inclusion is null ? null : new AttestorEntry.InclusionDescriptor + { + LeafHash = response.Inclusion.LeafHash, + Path = response.Inclusion.Path + } + }; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorSubmissionServiceTests.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorSubmissionServiceTests.cs new file mode 100644 index 00000000..03c8fb5e --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorSubmissionServiceTests.cs @@ -0,0 +1,321 @@ +using System; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Core.Observability; +using StellaOps.Attestor.Infrastructure.Rekor; +using StellaOps.Attestor.Infrastructure.Storage; +using StellaOps.Attestor.Infrastructure.Submission; +using Xunit; + +namespace StellaOps.Attestor.Tests; + +public sealed class AttestorSubmissionServiceTests +{ + [Fact] + public async Task SubmitAsync_ReturnsDeterministicUuid_OnDuplicateBundle() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions + { + Url = string.Empty + }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.stellaops.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + } + }); + + var canonicalizer = new DefaultDsseCanonicalizer(); + var validator = new AttestorSubmissionValidator(canonicalizer); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var logger = new NullLogger(); + using var metrics = new AttestorMetrics(); + var service = new AttestorSubmissionService( + validator, + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + logger, + TimeProvider.System, + metrics); + + var request = CreateValidRequest(canonicalizer); + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default", + ClientCertificate = null, + MtlsThumbprint = "00" + }; + + var first = await service.SubmitAsync(request, context); + var second = await service.SubmitAsync(request, context); + + Assert.NotNull(first.Uuid); + Assert.Equal(first.Uuid, second.Uuid); + + var stored = await repository.GetByBundleShaAsync(request.Meta.BundleSha256); + Assert.NotNull(stored); + Assert.Equal(first.Uuid, stored!.RekorUuid); + } + + [Fact] + public async Task Validator_ThrowsWhenModeNotAllowed() + { + var canonicalizer = new DefaultDsseCanonicalizer(); + var validator = new AttestorSubmissionValidator(canonicalizer, new[] { "kms" }); + + var request = CreateValidRequest(canonicalizer); + request.Bundle.Mode = "keyless"; + + await Assert.ThrowsAsync(() => validator.ValidateAsync(request)); + } + + [Fact] + public async Task SubmitAsync_Throws_WhenMirrorDisabledButRequested() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions { Url = string.Empty }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.primary.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + } + }); + + var canonicalizer = new DefaultDsseCanonicalizer(); + var validator = new AttestorSubmissionValidator(canonicalizer); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var logger = new NullLogger(); + using var metrics = new AttestorMetrics(); + + var service = new AttestorSubmissionService( + validator, + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + logger, + TimeProvider.System, + metrics); + + var request = CreateValidRequest(canonicalizer); + request.Meta.LogPreference = "mirror"; + + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default" + }; + + var ex = await Assert.ThrowsAsync(() => service.SubmitAsync(request, context)); + Assert.Equal("mirror_disabled", ex.Code); + } + + [Fact] + public async Task SubmitAsync_ReturnsMirrorMetadata_WhenPreferenceBoth() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions { Url = string.Empty }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.primary.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + }, + Mirror = new AttestorOptions.RekorMirrorOptions + { + Enabled = true, + Url = "https://rekor.mirror.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + } + }); + + var canonicalizer = new DefaultDsseCanonicalizer(); + var validator = new AttestorSubmissionValidator(canonicalizer); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var logger = new NullLogger(); + using var metrics = new AttestorMetrics(); + + var service = new AttestorSubmissionService( + validator, + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + logger, + TimeProvider.System, + metrics); + + var request = CreateValidRequest(canonicalizer); + request.Meta.LogPreference = "both"; + + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default" + }; + + var result = await service.SubmitAsync(request, context); + + Assert.NotNull(result.Mirror); + Assert.False(string.IsNullOrEmpty(result.Mirror!.Uuid)); + Assert.Equal("included", result.Mirror.Status); + } + + [Fact] + public async Task SubmitAsync_UsesMirrorAsCanonical_WhenPreferenceMirror() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions { Url = string.Empty }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.primary.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + }, + Mirror = new AttestorOptions.RekorMirrorOptions + { + Enabled = true, + Url = "https://rekor.mirror.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + } + }); + + var canonicalizer = new DefaultDsseCanonicalizer(); + var validator = new AttestorSubmissionValidator(canonicalizer); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var logger = new NullLogger(); + using var metrics = new AttestorMetrics(); + + var service = new AttestorSubmissionService( + validator, + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + logger, + TimeProvider.System, + metrics); + + var request = CreateValidRequest(canonicalizer); + request.Meta.LogPreference = "mirror"; + + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default" + }; + + var result = await service.SubmitAsync(request, context); + + Assert.NotNull(result.Uuid); + var stored = await repository.GetByBundleShaAsync(request.Meta.BundleSha256); + Assert.NotNull(stored); + Assert.Equal("mirror", stored!.Log.Backend); + Assert.Null(result.Mirror); + } + + private static AttestorSubmissionRequest CreateValidRequest(DefaultDsseCanonicalizer canonicalizer) + { + var request = new AttestorSubmissionRequest + { + Bundle = new AttestorSubmissionRequest.SubmissionBundle + { + Mode = "keyless", + Dsse = new AttestorSubmissionRequest.DsseEnvelope + { + PayloadType = "application/vnd.in-toto+json", + PayloadBase64 = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("{}")), + Signatures = + { + new AttestorSubmissionRequest.DsseSignature + { + KeyId = "test", + Signature = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)) + } + } + } + }, + Meta = new AttestorSubmissionRequest.SubmissionMeta + { + Artifact = new AttestorSubmissionRequest.ArtifactInfo + { + Sha256 = new string('a', 64), + Kind = "sbom" + }, + LogPreference = "primary", + Archive = false + } + }; + + var canonical = canonicalizer.CanonicalizeAsync(request).GetAwaiter().GetResult(); + request.Meta.BundleSha256 = Convert.ToHexString(SHA256.HashData(canonical)).ToLowerInvariant(); + return request; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorVerificationServiceTests.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorVerificationServiceTests.cs new file mode 100644 index 00000000..34bcd96a --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/AttestorVerificationServiceTests.cs @@ -0,0 +1,267 @@ +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Core.Verification; +using StellaOps.Attestor.Infrastructure.Storage; +using StellaOps.Attestor.Infrastructure.Submission; +using StellaOps.Attestor.Infrastructure.Verification; +using StellaOps.Attestor.Infrastructure.Rekor; +using StellaOps.Attestor.Core.Observability; +using Xunit; + +namespace StellaOps.Attestor.Tests; + +public sealed class AttestorVerificationServiceTests +{ + private static readonly byte[] HmacSecret = Encoding.UTF8.GetBytes("attestor-hmac-secret"); + private static readonly string HmacSecretBase64 = Convert.ToBase64String(HmacSecret); + + [Fact] + public async Task VerifyAsync_ReturnsOk_ForExistingUuid() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions + { + Url = string.Empty + }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.stellaops.test", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + }, + Security = new AttestorOptions.SecurityOptions + { + SignerIdentity = new AttestorOptions.SignerIdentityOptions + { + Mode = { "kms" }, + KmsKeys = { HmacSecretBase64 } + } + } + }); + + using var metrics = new AttestorMetrics(); + var canonicalizer = new DefaultDsseCanonicalizer(); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var submissionService = new AttestorSubmissionService( + new AttestorSubmissionValidator(canonicalizer), + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + new NullLogger(), + TimeProvider.System, + metrics); + + var submission = CreateSubmissionRequest(canonicalizer, HmacSecret); + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default" + }; + + var response = await submissionService.SubmitAsync(submission, context); + + var verificationService = new AttestorVerificationService( + repository, + canonicalizer, + rekorClient, + options, + new NullLogger(), + metrics); + + var verifyResult = await verificationService.VerifyAsync(new AttestorVerificationRequest + { + Uuid = response.Uuid, + Bundle = submission.Bundle + }); + + Assert.True(verifyResult.Ok); + Assert.Equal(response.Uuid, verifyResult.Uuid); + Assert.Empty(verifyResult.Issues); + } + + [Fact] + public async Task VerifyAsync_FlagsTamperedBundle() + { + var options = Options.Create(new AttestorOptions + { + Redis = new AttestorOptions.RedisOptions { Url = string.Empty }, + Rekor = new AttestorOptions.RekorOptions + { + Primary = new AttestorOptions.RekorBackendOptions + { + Url = "https://rekor.example/", + ProofTimeoutMs = 1000, + PollIntervalMs = 50, + MaxAttempts = 2 + } + }, + Security = new AttestorOptions.SecurityOptions + { + SignerIdentity = new AttestorOptions.SignerIdentityOptions + { + Mode = { "kms" }, + KmsKeys = { HmacSecretBase64 } + } + } + }); + + using var metrics = new AttestorMetrics(); + var canonicalizer = new DefaultDsseCanonicalizer(); + var repository = new InMemoryAttestorEntryRepository(); + var dedupeStore = new InMemoryAttestorDedupeStore(); + var rekorClient = new StubRekorClient(new NullLogger()); + var archiveStore = new NullAttestorArchiveStore(new NullLogger()); + var auditSink = new InMemoryAttestorAuditSink(); + var submissionService = new AttestorSubmissionService( + new AttestorSubmissionValidator(canonicalizer), + repository, + dedupeStore, + rekorClient, + archiveStore, + auditSink, + options, + new NullLogger(), + TimeProvider.System, + metrics); + + var submission = CreateSubmissionRequest(canonicalizer, HmacSecret); + var context = new SubmissionContext + { + CallerSubject = "urn:stellaops:signer", + CallerAudience = "attestor", + CallerClientId = "signer-service", + CallerTenant = "default" + }; + + var response = await submissionService.SubmitAsync(submission, context); + + var verificationService = new AttestorVerificationService( + repository, + canonicalizer, + rekorClient, + options, + new NullLogger(), + metrics); + + var tamperedBundle = CloneBundle(submission.Bundle); + tamperedBundle.Dsse.PayloadBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("{\"tampered\":true}")); + + var result = await verificationService.VerifyAsync(new AttestorVerificationRequest + { + Uuid = response.Uuid, + Bundle = tamperedBundle + }); + + Assert.False(result.Ok); + Assert.Contains(result.Issues, issue => issue.Contains("signature_invalid", StringComparison.OrdinalIgnoreCase)); + } + + private static AttestorSubmissionRequest CreateSubmissionRequest(DefaultDsseCanonicalizer canonicalizer, byte[] hmacSecret) + { + var payload = Encoding.UTF8.GetBytes("{}"); + var request = new AttestorSubmissionRequest + { + Bundle = new AttestorSubmissionRequest.SubmissionBundle + { + Mode = "kms", + Dsse = new AttestorSubmissionRequest.DsseEnvelope + { + PayloadType = "application/vnd.in-toto+json", + PayloadBase64 = Convert.ToBase64String(payload) + } + }, + Meta = new AttestorSubmissionRequest.SubmissionMeta + { + Artifact = new AttestorSubmissionRequest.ArtifactInfo + { + Sha256 = new string('a', 64), + Kind = "sbom" + }, + LogPreference = "primary", + Archive = false + } + }; + + var preAuth = ComputePreAuthEncodingForTests(request.Bundle.Dsse.PayloadType, payload); + using (var hmac = new HMACSHA256(hmacSecret)) + { + var signature = hmac.ComputeHash(preAuth); + request.Bundle.Dsse.Signatures.Add(new AttestorSubmissionRequest.DsseSignature + { + KeyId = "kms-test", + Signature = Convert.ToBase64String(signature) + }); + } + + var canonical = canonicalizer.CanonicalizeAsync(request).GetAwaiter().GetResult(); + request.Meta.BundleSha256 = Convert.ToHexString(SHA256.HashData(canonical)).ToLowerInvariant(); + return request; + } + + private static AttestorSubmissionRequest.SubmissionBundle CloneBundle(AttestorSubmissionRequest.SubmissionBundle source) + { + var clone = new AttestorSubmissionRequest.SubmissionBundle + { + Mode = source.Mode, + Dsse = new AttestorSubmissionRequest.DsseEnvelope + { + PayloadType = source.Dsse.PayloadType, + PayloadBase64 = source.Dsse.PayloadBase64 + } + }; + + foreach (var certificate in source.CertificateChain) + { + clone.CertificateChain.Add(certificate); + } + + foreach (var signature in source.Dsse.Signatures) + { + clone.Dsse.Signatures.Add(new AttestorSubmissionRequest.DsseSignature + { + KeyId = signature.KeyId, + Signature = signature.Signature + }); + } + + return clone; + } + + private static byte[] ComputePreAuthEncodingForTests(string payloadType, byte[] payload) + { + var headerBytes = Encoding.UTF8.GetBytes(payloadType ?? string.Empty); + var buffer = new byte[6 + 8 + headerBytes.Length + 8 + payload.Length]; + var offset = 0; + Encoding.ASCII.GetBytes("DSSEv1", 0, 6, buffer, offset); + offset += 6; + BinaryPrimitives.WriteUInt64BigEndian(buffer.AsSpan(offset, 8), (ulong)headerBytes.Length); + offset += 8; + Buffer.BlockCopy(headerBytes, 0, buffer, offset, headerBytes.Length); + offset += headerBytes.Length; + BinaryPrimitives.WriteUInt64BigEndian(buffer.AsSpan(offset, 8), (ulong)payload.Length); + offset += 8; + Buffer.BlockCopy(payload, 0, buffer, offset, payload.Length); + return buffer; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/HttpRekorClientTests.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/HttpRekorClientTests.cs new file mode 100644 index 00000000..957f66f5 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/HttpRekorClientTests.cs @@ -0,0 +1,149 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Attestor.Core.Rekor; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Infrastructure.Rekor; +using Xunit; + +namespace StellaOps.Attestor.Tests; + +public sealed class HttpRekorClientTests +{ + [Fact] + public async Task SubmitAsync_ParsesResponse() + { + var payload = new + { + uuid = "123", + index = 42, + logURL = "https://rekor.example/api/v2/log/entries/123", + status = "included", + proof = new + { + checkpoint = new { origin = "rekor", size = 10, rootHash = "abc", timestamp = "2025-10-19T00:00:00Z" }, + inclusion = new { leafHash = "leaf", path = new[] { "p1", "p2" } } + } + }; + + var client = CreateClient(HttpStatusCode.Created, payload); + var rekorClient = new HttpRekorClient(client, NullLogger.Instance); + + var request = new AttestorSubmissionRequest + { + Bundle = new AttestorSubmissionRequest.SubmissionBundle + { + Dsse = new AttestorSubmissionRequest.DsseEnvelope + { + PayloadType = "application/json", + PayloadBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("{}")), + Signatures = { new AttestorSubmissionRequest.DsseSignature { Signature = "sig" } } + } + } + }; + + var backend = new RekorBackend + { + Name = "primary", + Url = new Uri("https://rekor.example/"), + ProofTimeout = TimeSpan.FromSeconds(1), + PollInterval = TimeSpan.FromMilliseconds(100), + MaxAttempts = 1 + }; + + var response = await rekorClient.SubmitAsync(request, backend); + + Assert.Equal("123", response.Uuid); + Assert.Equal(42, response.Index); + Assert.Equal("included", response.Status); + Assert.NotNull(response.Proof); + Assert.Equal("leaf", response.Proof!.Inclusion!.LeafHash); + } + + [Fact] + public async Task SubmitAsync_ThrowsOnConflict() + { + var client = CreateClient(HttpStatusCode.Conflict, new { error = "duplicate" }); + var rekorClient = new HttpRekorClient(client, NullLogger.Instance); + + var request = new AttestorSubmissionRequest + { + Bundle = new AttestorSubmissionRequest.SubmissionBundle + { + Dsse = new AttestorSubmissionRequest.DsseEnvelope + { + PayloadType = "application/json", + PayloadBase64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("{}")), + Signatures = { new AttestorSubmissionRequest.DsseSignature { Signature = "sig" } } + } + } + }; + + var backend = new RekorBackend + { + Name = "primary", + Url = new Uri("https://rekor.example/"), + ProofTimeout = TimeSpan.FromSeconds(1), + PollInterval = TimeSpan.FromMilliseconds(100), + MaxAttempts = 1 + }; + + await Assert.ThrowsAsync(() => rekorClient.SubmitAsync(request, backend)); + } + + [Fact] + public async Task GetProofAsync_ReturnsNullOnNotFound() + { + var client = CreateClient(HttpStatusCode.NotFound, new { }); + var rekorClient = new HttpRekorClient(client, NullLogger.Instance); + + var backend = new RekorBackend + { + Name = "primary", + Url = new Uri("https://rekor.example/"), + ProofTimeout = TimeSpan.FromSeconds(1), + PollInterval = TimeSpan.FromMilliseconds(100), + MaxAttempts = 1 + }; + + var proof = await rekorClient.GetProofAsync("abc", backend); + Assert.Null(proof); + } + + private static HttpClient CreateClient(HttpStatusCode statusCode, object payload) + { + var handler = new StubHandler(statusCode, payload); + return new HttpClient(handler) + { + BaseAddress = new Uri("https://rekor.example/") + }; + } + + private sealed class StubHandler : HttpMessageHandler + { + private readonly HttpStatusCode _statusCode; + private readonly object _payload; + + public StubHandler(HttpStatusCode statusCode, object payload) + { + _statusCode = statusCode; + _payload = payload; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(_payload); + var response = new HttpResponseMessage(_statusCode) + { + Content = new StringContent(json, Encoding.UTF8, "application/json") + }; + + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj new file mode 100644 index 00000000..5f7f5ec6 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/StellaOps.Attestor.Tests.csproj @@ -0,0 +1,25 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.Tests/TestDoubles.cs b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/TestDoubles.cs new file mode 100644 index 00000000..af61d6cc --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.Tests/TestDoubles.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Attestor.Core.Audit; +using StellaOps.Attestor.Core.Storage; + +namespace StellaOps.Attestor.Tests; + +internal sealed class InMemoryAttestorEntryRepository : IAttestorEntryRepository +{ + private readonly ConcurrentDictionary _entries = new(); + + public Task GetByBundleShaAsync(string bundleSha256, CancellationToken cancellationToken = default) + { + var entry = _entries.Values.FirstOrDefault(e => string.Equals(e.BundleSha256, bundleSha256, StringComparison.OrdinalIgnoreCase)); + return Task.FromResult(entry); + } + + public Task GetByUuidAsync(string rekorUuid, CancellationToken cancellationToken = default) + { + _entries.TryGetValue(rekorUuid, out var entry); + return Task.FromResult(entry); + } + + public Task> GetByArtifactShaAsync(string artifactSha256, CancellationToken cancellationToken = default) + { + var entries = _entries.Values + .Where(e => string.Equals(e.Artifact.Sha256, artifactSha256, StringComparison.OrdinalIgnoreCase)) + .OrderBy(e => e.CreatedAt) + .ToList(); + + return Task.FromResult>(entries); + } + + public Task SaveAsync(AttestorEntry entry, CancellationToken cancellationToken = default) + { + _entries[entry.RekorUuid] = entry; + return Task.CompletedTask; + } +} + +internal sealed class InMemoryAttestorAuditSink : IAttestorAuditSink +{ + public List Records { get; } = new(); + + public Task WriteAsync(AttestorAuditRecord record, CancellationToken cancellationToken = default) + { + Records.Add(record); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.WebService/Program.cs b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/Program.cs new file mode 100644 index 00000000..339b1d1a --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/Program.cs @@ -0,0 +1,405 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Authentication; +using System.Security.Cryptography; +using System.Security.Claims; +using System.Security.Cryptography.X509Certificates; +using System.Threading.RateLimiting; +using Serilog; +using Serilog.Events; +using StellaOps.Attestor.Core.Options; +using StellaOps.Attestor.Core.Submission; +using StellaOps.Attestor.Infrastructure; +using StellaOps.Configuration; +using StellaOps.Auth.ServerIntegration; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using OpenTelemetry.Metrics; +using StellaOps.Attestor.Core.Observability; +using StellaOps.Attestor.Core.Verification; +using Microsoft.AspNetCore.Server.Kestrel.Https; +using Serilog.Context; + +const string ConfigurationSection = "attestor"; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "ATTESTOR_"; + options.BindingSection = ConfigurationSection; +}); + +builder.Host.UseSerilog((context, services, loggerConfiguration) => +{ + loggerConfiguration + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft", LogEventLevel.Warning) + .Enrich.FromLogContext() + .WriteTo.Console(); +}); + +var attestorOptions = builder.Configuration.BindOptions(ConfigurationSection); + +var clientCertificateAuthorities = LoadClientCertificateAuthorities(attestorOptions.Security.Mtls.CaBundle); + +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddSingleton(attestorOptions); + +builder.Services.AddRateLimiter(options => +{ + options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + options.OnRejected = static (context, _) => + { + context.HttpContext.Response.Headers.TryAdd("Retry-After", "1"); + return ValueTask.CompletedTask; + }; + + options.AddPolicy("attestor-submissions", httpContext => + { + var identity = httpContext.Connection.ClientCertificate?.Thumbprint + ?? httpContext.User.FindFirst("sub")?.Value + ?? httpContext.User.FindFirst("client_id")?.Value + ?? httpContext.Connection.RemoteIpAddress?.ToString() + ?? "anonymous"; + + var quota = attestorOptions.Quotas.PerCaller; + var tokensPerPeriod = Math.Max(1, quota.Qps); + var tokenLimit = Math.Max(tokensPerPeriod, quota.Burst); + var queueLimit = Math.Max(quota.Burst, tokensPerPeriod); + + return RateLimitPartition.GetTokenBucketLimiter(identity, _ => new TokenBucketRateLimiterOptions + { + TokenLimit = tokenLimit, + TokensPerPeriod = tokensPerPeriod, + ReplenishmentPeriod = TimeSpan.FromSeconds(1), + QueueLimit = queueLimit, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + AutoReplenishment = true + }); + }); +}); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(ConfigurationSection)) + .ValidateOnStart(); + +builder.Services.AddProblemDetails(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddAttestorInfrastructure(); +builder.Services.AddHttpContextAccessor(); +builder.Services.AddHealthChecks() + .AddCheck("self", () => HealthCheckResult.Healthy()); + +builder.Services.AddOpenTelemetry() + .WithMetrics(metricsBuilder => + { + metricsBuilder.AddMeter(AttestorMetrics.MeterName); + metricsBuilder.AddAspNetCoreInstrumentation(); + metricsBuilder.AddRuntimeInstrumentation(); + }); + +if (attestorOptions.Security.Authority is { Issuer: not null } authority) +{ + builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = authority.Issuer!; + resourceOptions.RequireHttpsMetadata = authority.RequireHttpsMetadata; + if (!string.IsNullOrWhiteSpace(authority.JwksUrl)) + { + resourceOptions.MetadataAddress = authority.JwksUrl; + } + + foreach (var audience in authority.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + + foreach (var scope in authority.RequiredScopes) + { + resourceOptions.RequiredScopes.Add(scope); + } + }); + + builder.Services.AddAuthorization(options => + { + options.AddPolicy("attestor:write", policy => + { + policy.RequireAuthenticatedUser(); + policy.RequireClaim("scope", authority.RequiredScopes); + }); + }); +} +else +{ + builder.Services.AddAuthorization(); +} + +builder.WebHost.ConfigureKestrel(kestrel => +{ + kestrel.ConfigureHttpsDefaults(https => + { + if (attestorOptions.Security.Mtls.RequireClientCertificate) + { + https.ClientCertificateMode = ClientCertificateMode.RequireCertificate; + } + + https.SslProtocols = SslProtocols.Tls13 | SslProtocols.Tls12; + + https.ClientCertificateValidation = (certificate, _, _) => + { + if (!attestorOptions.Security.Mtls.RequireClientCertificate) + { + return true; + } + + if (certificate is null) + { + Log.Warning("Client certificate missing"); + return false; + } + + if (clientCertificateAuthorities.Count > 0) + { + using var chain = new X509Chain + { + ChainPolicy = + { + RevocationMode = X509RevocationMode.NoCheck, + TrustMode = X509ChainTrustMode.CustomRootTrust + } + }; + + foreach (var authority in clientCertificateAuthorities) + { + chain.ChainPolicy.CustomTrustStore.Add(authority); + } + + if (!chain.Build(certificate)) + { + Log.Warning("Client certificate chain validation failed for {Subject}", certificate.Subject); + return false; + } + } + + if (attestorOptions.Security.Mtls.AllowedThumbprints.Count > 0 && + !attestorOptions.Security.Mtls.AllowedThumbprints.Contains(certificate.Thumbprint ?? string.Empty, StringComparer.OrdinalIgnoreCase)) + { + Log.Warning("Client certificate thumbprint {Thumbprint} rejected", certificate.Thumbprint); + return false; + } + + if (attestorOptions.Security.Mtls.AllowedSubjects.Count > 0 && + !attestorOptions.Security.Mtls.AllowedSubjects.Contains(certificate.Subject, StringComparer.OrdinalIgnoreCase)) + { + Log.Warning("Client certificate subject {Subject} rejected", certificate.Subject); + return false; + } + + return true; + }; + }); +}); + +var app = builder.Build(); + +app.UseSerilogRequestLogging(); + +app.Use(async (context, next) => +{ + var correlationId = context.Request.Headers["X-Correlation-Id"].FirstOrDefault(); + if (string.IsNullOrWhiteSpace(correlationId)) + { + correlationId = Guid.NewGuid().ToString("N"); + } + + context.Response.Headers["X-Correlation-Id"] = correlationId; + + using (LogContext.PushProperty("CorrelationId", correlationId)) + { + await next().ConfigureAwait(false); + } +}); + +app.UseExceptionHandler(static handler => +{ + handler.Run(async context => + { + var result = Results.Problem(statusCode: StatusCodes.Status500InternalServerError); + await result.ExecuteAsync(context); + }); +}); + +app.UseRateLimiter(); + +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapHealthChecks("/health/ready"); +app.MapHealthChecks("/health/live"); + +app.MapPost("/api/v1/rekor/entries", async (AttestorSubmissionRequest request, HttpContext httpContext, IAttestorSubmissionService submissionService, CancellationToken cancellationToken) => +{ + var certificate = httpContext.Connection.ClientCertificate; + if (certificate is null) + { + return Results.Problem(statusCode: StatusCodes.Status403Forbidden, title: "Client certificate required"); + } + + var user = httpContext.User; + if (user?.Identity is not { IsAuthenticated: true }) + { + return Results.Problem(statusCode: StatusCodes.Status401Unauthorized, title: "Authentication required"); + } + + var submissionContext = BuildSubmissionContext(user, certificate); + + try + { + var result = await submissionService.SubmitAsync(request, submissionContext, cancellationToken).ConfigureAwait(false); + return Results.Ok(result); + } + catch (AttestorValidationException validationEx) + { + return Results.Problem(statusCode: StatusCodes.Status400BadRequest, title: validationEx.Message, extensions: new Dictionary + { + ["code"] = validationEx.Code + }); + } +}) +.RequireAuthorization("attestor:write") +.RequireRateLimiting("attestor-submissions"); + +app.MapGet("/api/v1/rekor/entries/{uuid}", async (string uuid, bool? refresh, IAttestorVerificationService verificationService, CancellationToken cancellationToken) => +{ + var entry = await verificationService.GetEntryAsync(uuid, refresh is true, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + return Results.NotFound(); + } + + return Results.Ok(new + { + uuid = entry.RekorUuid, + index = entry.Index, + backend = entry.Log.Backend, + proof = entry.Proof is null ? null : new + { + checkpoint = entry.Proof.Checkpoint is null ? null : new + { + origin = entry.Proof.Checkpoint.Origin, + size = entry.Proof.Checkpoint.Size, + rootHash = entry.Proof.Checkpoint.RootHash, + timestamp = entry.Proof.Checkpoint.Timestamp?.ToString("O") + }, + inclusion = entry.Proof.Inclusion is null ? null : new + { + leafHash = entry.Proof.Inclusion.LeafHash, + path = entry.Proof.Inclusion.Path + } + }, + logURL = entry.Log.Url, + status = entry.Status, + mirror = entry.Mirror is null ? null : new + { + backend = entry.Mirror.Backend, + uuid = entry.Mirror.Uuid, + index = entry.Mirror.Index, + logURL = entry.Mirror.Url, + status = entry.Mirror.Status, + proof = entry.Mirror.Proof is null ? null : new + { + checkpoint = entry.Mirror.Proof.Checkpoint is null ? null : new + { + origin = entry.Mirror.Proof.Checkpoint.Origin, + size = entry.Mirror.Proof.Checkpoint.Size, + rootHash = entry.Mirror.Proof.Checkpoint.RootHash, + timestamp = entry.Mirror.Proof.Checkpoint.Timestamp?.ToString("O") + }, + inclusion = entry.Mirror.Proof.Inclusion is null ? null : new + { + leafHash = entry.Mirror.Proof.Inclusion.LeafHash, + path = entry.Mirror.Proof.Inclusion.Path + } + }, + error = entry.Mirror.Error + }, + artifact = new + { + sha256 = entry.Artifact.Sha256, + kind = entry.Artifact.Kind, + imageDigest = entry.Artifact.ImageDigest, + subjectUri = entry.Artifact.SubjectUri + } + }); +}).RequireAuthorization("attestor:write"); + +app.MapPost("/api/v1/rekor/verify", async (AttestorVerificationRequest request, IAttestorVerificationService verificationService, CancellationToken cancellationToken) => +{ + try + { + var result = await verificationService.VerifyAsync(request, cancellationToken).ConfigureAwait(false); + return Results.Ok(result); + } + catch (AttestorVerificationException ex) + { + return Results.Problem(statusCode: StatusCodes.Status400BadRequest, title: ex.Message, extensions: new Dictionary + { + ["code"] = ex.Code + }); + } +}).RequireAuthorization("attestor:write"); + +app.Run(); + +static SubmissionContext BuildSubmissionContext(ClaimsPrincipal user, X509Certificate2 certificate) +{ + var subject = user.FindFirst("sub")?.Value ?? certificate.Subject; + var audience = user.FindFirst("aud")?.Value ?? string.Empty; + var clientId = user.FindFirst("client_id")?.Value; + var tenant = user.FindFirst("tenant")?.Value; + + return new SubmissionContext + { + CallerSubject = subject, + CallerAudience = audience, + CallerClientId = clientId, + CallerTenant = tenant, + ClientCertificate = certificate, + MtlsThumbprint = certificate.Thumbprint + }; +} + +static List LoadClientCertificateAuthorities(string? path) +{ + var certificates = new List(); + + if (string.IsNullOrWhiteSpace(path)) + { + return certificates; + } + + try + { + if (!File.Exists(path)) + { + Log.Warning("Client CA bundle '{Path}' not found", path); + return certificates; + } + + var collection = new X509Certificate2Collection(); + collection.ImportFromPemFile(path); + + certificates.AddRange(collection.Cast()); + } + catch (Exception ex) when (ex is IOException or CryptographicException) + { + Log.Warning(ex, "Failed to load client CA bundle from {Path}", path); + } + + return certificates; +} diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj new file mode 100644 index 00000000..62c77cc6 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.WebService/StellaOps.Attestor.WebService.csproj @@ -0,0 +1,30 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Attestor/StellaOps.Attestor.sln b/src/StellaOps.Attestor/StellaOps.Attestor.sln new file mode 100644 index 00000000..8abfec51 --- /dev/null +++ b/src/StellaOps.Attestor/StellaOps.Attestor.sln @@ -0,0 +1,118 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Core", "StellaOps.Attestor.Core\StellaOps.Attestor.Core.csproj", "{C0FE77EB-933C-4E47-8195-758AB049157A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Infrastructure", "StellaOps.Attestor.Infrastructure\StellaOps.Attestor.Infrastructure.csproj", "{996D74F8-8683-45FA-90AB-DA7ACE78D4B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.WebService", "StellaOps.Attestor.WebService\StellaOps.Attestor.WebService.csproj", "{B238B098-32B1-4875-99A7-393A63AC3CCF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "..\StellaOps.Configuration\StellaOps.Configuration.csproj", "{988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography", "..\StellaOps.Cryptography\StellaOps.Cryptography.csproj", "{82EFA477-307D-4B47-A4CF-1627F076D60A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "..\StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{21327A4F-2586-49F8-9D4A-3840DE64C48E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Tests", "StellaOps.Attestor.Tests\StellaOps.Attestor.Tests.csproj", "{4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|x64.ActiveCfg = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|x64.Build.0 = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|x86.ActiveCfg = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Debug|x86.Build.0 = Debug|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|Any CPU.Build.0 = Release|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|x64.ActiveCfg = Release|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|x64.Build.0 = Release|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|x86.ActiveCfg = Release|Any CPU + {C0FE77EB-933C-4E47-8195-758AB049157A}.Release|x86.Build.0 = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|x64.Build.0 = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Debug|x86.Build.0 = Debug|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|Any CPU.Build.0 = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|x64.ActiveCfg = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|x64.Build.0 = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|x86.ActiveCfg = Release|Any CPU + {996D74F8-8683-45FA-90AB-DA7ACE78D4B3}.Release|x86.Build.0 = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|x64.ActiveCfg = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|x64.Build.0 = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|x86.ActiveCfg = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Debug|x86.Build.0 = Debug|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|Any CPU.Build.0 = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|x64.ActiveCfg = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|x64.Build.0 = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|x86.ActiveCfg = Release|Any CPU + {B238B098-32B1-4875-99A7-393A63AC3CCF}.Release|x86.Build.0 = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|x64.ActiveCfg = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|x64.Build.0 = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|x86.ActiveCfg = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Debug|x86.Build.0 = Debug|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|Any CPU.Build.0 = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|x64.ActiveCfg = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|x64.Build.0 = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|x86.ActiveCfg = Release|Any CPU + {988E2AC7-50E0-4845-B1C2-BA4931F2FFD7}.Release|x86.Build.0 = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|x64.ActiveCfg = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|x64.Build.0 = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|x86.ActiveCfg = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Debug|x86.Build.0 = Debug|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|Any CPU.Build.0 = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|x64.ActiveCfg = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|x64.Build.0 = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|x86.ActiveCfg = Release|Any CPU + {82EFA477-307D-4B47-A4CF-1627F076D60A}.Release|x86.Build.0 = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|x64.ActiveCfg = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|x64.Build.0 = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|x86.ActiveCfg = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Debug|x86.Build.0 = Debug|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|Any CPU.Build.0 = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|x64.ActiveCfg = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|x64.Build.0 = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|x86.ActiveCfg = Release|Any CPU + {21327A4F-2586-49F8-9D4A-3840DE64C48E}.Release|x86.Build.0 = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|x64.Build.0 = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Debug|x86.Build.0 = Debug|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|Any CPU.Build.0 = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|x64.ActiveCfg = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|x64.Build.0 = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|x86.ActiveCfg = Release|Any CPU + {4B7592CD-D67C-4F4D-82FE-DF99BAAC4275}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Attestor/TASKS.md b/src/StellaOps.Attestor/TASKS.md new file mode 100644 index 00000000..1fe30f81 --- /dev/null +++ b/src/StellaOps.Attestor/TASKS.md @@ -0,0 +1,10 @@ +# Attestor Guild Task Board (UTC 2025-10-19) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| ATTESTOR-API-11-201 | DONE (2025-10-19) | Attestor Guild | — | `/rekor/entries` submission pipeline with dedupe, proof acquisition, and persistence. | ✅ `POST /api/v1/rekor/entries` enforces mTLS + Authority OpTok, validates DSSE bundles, and handles dual-log preferences.
✅ Redis/Mongo idempotency returns existing UUID on duplicate `bundleSha256` without re-submitting to Rekor.
✅ Rekor driver fetches inclusion proofs (or schedules async fetch) and persists canonical entry/proof metadata.
✅ Optional archive path stores DSSE/proof bundles to MinIO/S3; integration tests cover success/pending/error flows. | +| ATTESTOR-VERIFY-11-202 | DONE (2025-10-19) | Attestor Guild | — | `/rekor/verify` + retrieval endpoints validating signatures and Merkle proofs. | ✅ `GET /api/v1/rekor/entries/{uuid}` surfaces cached entries with optional backend refresh and handles not-found/refresh flows.
✅ `POST /api/v1/rekor/verify` accepts UUID, bundle, or artifact hash inputs; verifies DSSE signatures, Merkle proofs, and checkpoint anchors.
✅ Verification output returns `{ok, uuid, index, logURL, checkedAt}` with failure diagnostics for invalid proofs.
✅ Unit/integration tests exercise cache hits, backend refresh, invalid bundle/proof scenarios, and checkpoint trust anchor enforcement. | +| ATTESTOR-OBS-11-203 | DONE (2025-10-19) | Attestor Guild | — | Telemetry, alerting, mTLS hardening, and archive workflow for Attestor. | ✅ Structured logs, metrics, and optional traces record submission latency, proof fetch outcomes, verification results, and Rekor error buckets with correlation IDs.
✅ mTLS enforcement hardened (peer allowlist, SAN checks, rate limiting) and documented; TLS settings audited for modern ciphers only.
✅ Alerting/dashboard pack covers error rates, proof backlog, Redis/Mongo health, and archive job failures; runbook updated.
✅ Archive workflow includes retention policy jobs, failure alerts, and periodic verification of stored bundles and proofs. | + +> Remark (2025-10-19): Wave 0 prerequisites reviewed (none outstanding); ATTESTOR-API-11-201, ATTESTOR-VERIFY-11-202, and ATTESTOR-OBS-11-203 tracked as DOING per Wave 0A kickoff. +> Remark (2025-10-19): Dual-log submissions, signature/proof verification, and observability hardening landed; attestor endpoints now rate-limited per client with correlation-ID logging and updated docs/tests. diff --git a/src/StellaOps.Auth.Security/Dpop/DpopNonceConsumeResult.cs b/src/StellaOps.Auth.Security/Dpop/DpopNonceConsumeResult.cs new file mode 100644 index 00000000..c6f51234 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopNonceConsumeResult.cs @@ -0,0 +1,50 @@ +using System; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Represents the outcome of attempting to consume a DPoP nonce. +/// +public sealed class DpopNonceConsumeResult +{ + private DpopNonceConsumeResult(DpopNonceConsumeStatus status, DateTimeOffset? issuedAt, DateTimeOffset? expiresAt) + { + Status = status; + IssuedAt = issuedAt; + ExpiresAt = expiresAt; + } + + /// + /// Consumption status. + /// + public DpopNonceConsumeStatus Status { get; } + + /// + /// Timestamp the nonce was originally issued (when available). + /// + public DateTimeOffset? IssuedAt { get; } + + /// + /// Expiry timestamp for the nonce (when available). + /// + public DateTimeOffset? ExpiresAt { get; } + + public static DpopNonceConsumeResult Success(DateTimeOffset issuedAt, DateTimeOffset expiresAt) + => new(DpopNonceConsumeStatus.Success, issuedAt, expiresAt); + + public static DpopNonceConsumeResult Expired(DateTimeOffset? issuedAt, DateTimeOffset expiresAt) + => new(DpopNonceConsumeStatus.Expired, issuedAt, expiresAt); + + public static DpopNonceConsumeResult NotFound() + => new(DpopNonceConsumeStatus.NotFound, null, null); +} + +/// +/// Known statuses for nonce consumption attempts. +/// +public enum DpopNonceConsumeStatus +{ + Success, + Expired, + NotFound +} diff --git a/src/StellaOps.Auth.Security/Dpop/DpopNonceIssueResult.cs b/src/StellaOps.Auth.Security/Dpop/DpopNonceIssueResult.cs new file mode 100644 index 00000000..d4bf1259 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopNonceIssueResult.cs @@ -0,0 +1,56 @@ +using System; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Represents the result of issuing a DPoP nonce. +/// +public sealed class DpopNonceIssueResult +{ + private DpopNonceIssueResult(DpopNonceIssueStatus status, string? nonce, DateTimeOffset? expiresAt, string? error) + { + Status = status; + Nonce = nonce; + ExpiresAt = expiresAt; + Error = error; + } + + /// + /// Issue status. + /// + public DpopNonceIssueStatus Status { get; } + + /// + /// Issued nonce when is . + /// + public string? Nonce { get; } + + /// + /// Expiry timestamp for the issued nonce (UTC). + /// + public DateTimeOffset? ExpiresAt { get; } + + /// + /// Additional failure information, where applicable. + /// + public string? Error { get; } + + public static DpopNonceIssueResult Success(string nonce, DateTimeOffset expiresAt) + => new(DpopNonceIssueStatus.Success, nonce, expiresAt, null); + + public static DpopNonceIssueResult RateLimited(string? error = null) + => new(DpopNonceIssueStatus.RateLimited, null, null, error); + + public static DpopNonceIssueResult Failure(string? error = null) + => new(DpopNonceIssueStatus.Failure, null, null, error); +} + +/// +/// Known statuses for nonce issuance. +/// +public enum DpopNonceIssueStatus +{ + Success, + RateLimited, + Failure +} diff --git a/src/StellaOps.Auth.Security/Dpop/DpopNonceUtilities.cs b/src/StellaOps.Auth.Security/Dpop/DpopNonceUtilities.cs new file mode 100644 index 00000000..078118a3 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopNonceUtilities.cs @@ -0,0 +1,66 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Auth.Security.Dpop; + +internal static class DpopNonceUtilities +{ + private static readonly char[] Base64Padding = { '=' }; + + internal static string GenerateNonce() + { + Span buffer = stackalloc byte[32]; + RandomNumberGenerator.Fill(buffer); + + return Convert.ToBase64String(buffer) + .TrimEnd(Base64Padding) + .Replace('+', '-') + .Replace('/', '_'); + } + + internal static byte[] ComputeNonceHash(string nonce) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nonce); + var bytes = Encoding.UTF8.GetBytes(nonce); + return SHA256.HashData(bytes); + } + + internal static string EncodeHash(ReadOnlySpan hash) + => Convert.ToHexString(hash); + + internal static string ComputeStorageKey(string audience, string clientId, string keyThumbprint) + { + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + ArgumentException.ThrowIfNullOrWhiteSpace(clientId); + ArgumentException.ThrowIfNullOrWhiteSpace(keyThumbprint); + + return string.Create( + "dpop-nonce:".Length + audience.Length + clientId.Length + keyThumbprint.Length + 2, + (audience.Trim(), clientId.Trim(), keyThumbprint.Trim()), + static (span, parts) => + { + var index = 0; + const string Prefix = "dpop-nonce:"; + Prefix.CopyTo(span); + index += Prefix.Length; + + index = Append(span, index, parts.Item1); + span[index++] = ':'; + index = Append(span, index, parts.Item2); + span[index++] = ':'; + _ = Append(span, index, parts.Item3); + }); + + static int Append(Span span, int index, string value) + { + if (value.Length == 0) + { + throw new ArgumentException("Value must not be empty after trimming."); + } + + value.AsSpan().CopyTo(span[index..]); + return index + value.Length; + } + } +} diff --git a/src/StellaOps.Auth.Security/Dpop/DpopProofValidator.cs b/src/StellaOps.Auth.Security/Dpop/DpopProofValidator.cs new file mode 100644 index 00000000..07ef05a5 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopProofValidator.cs @@ -0,0 +1,258 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Validates DPoP proofs following RFC 9449. +/// +public sealed class DpopProofValidator : IDpopProofValidator +{ + private static readonly string ProofType = "dpop+jwt"; + private readonly DpopValidationOptions options; + private readonly IDpopReplayCache replayCache; + private readonly TimeProvider timeProvider; + private readonly ILogger? logger; + private readonly JwtSecurityTokenHandler tokenHandler = new(); + + public DpopProofValidator( + IOptions options, + IDpopReplayCache? replayCache = null, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(options); + + var cloned = options.Value ?? throw new InvalidOperationException("DPoP options must be provided."); + cloned.Validate(); + + this.options = cloned; + this.replayCache = replayCache ?? NullReplayCache.Instance; + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + } + + public async ValueTask ValidateAsync(string proof, string httpMethod, Uri httpUri, string? nonce = null, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(proof); + ArgumentException.ThrowIfNullOrWhiteSpace(httpMethod); + ArgumentNullException.ThrowIfNull(httpUri); + + var now = timeProvider.GetUtcNow(); + + if (!TryDecodeSegment(proof, segmentIndex: 0, out var headerElement, out var headerError)) + { + logger?.LogWarning("DPoP header decode failure: {Error}", headerError); + return DpopValidationResult.Failure("invalid_header", headerError ?? "Unable to decode header."); + } + + if (!headerElement.TryGetProperty("typ", out var typElement) || !string.Equals(typElement.GetString(), ProofType, StringComparison.OrdinalIgnoreCase)) + { + return DpopValidationResult.Failure("invalid_header", "DPoP proof missing typ=dpop+jwt header."); + } + + if (!headerElement.TryGetProperty("alg", out var algElement)) + { + return DpopValidationResult.Failure("invalid_header", "DPoP proof missing alg header."); + } + + var algorithm = algElement.GetString()?.Trim().ToUpperInvariant(); + if (string.IsNullOrEmpty(algorithm) || !options.NormalizedAlgorithms.Contains(algorithm)) + { + return DpopValidationResult.Failure("invalid_header", "Unsupported DPoP algorithm."); + } + + if (!headerElement.TryGetProperty("jwk", out var jwkElement)) + { + return DpopValidationResult.Failure("invalid_header", "DPoP proof missing jwk header."); + } + + JsonWebKey jwk; + try + { + jwk = new JsonWebKey(jwkElement.GetRawText()); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to parse DPoP jwk header."); + return DpopValidationResult.Failure("invalid_header", "DPoP proof jwk header is invalid."); + } + + if (!TryDecodeSegment(proof, segmentIndex: 1, out var payloadElement, out var payloadError)) + { + logger?.LogWarning("DPoP payload decode failure: {Error}", payloadError); + return DpopValidationResult.Failure("invalid_payload", payloadError ?? "Unable to decode payload."); + } + + if (!payloadElement.TryGetProperty("htm", out var htmElement)) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing htm claim."); + } + + var method = httpMethod.Trim().ToUpperInvariant(); + if (!string.Equals(htmElement.GetString(), method, StringComparison.Ordinal)) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP htm does not match request method."); + } + + if (!payloadElement.TryGetProperty("htu", out var htuElement)) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing htu claim."); + } + + var normalizedHtu = NormalizeHtu(httpUri); + if (!string.Equals(htuElement.GetString(), normalizedHtu, StringComparison.Ordinal)) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP htu does not match request URI."); + } + + if (!payloadElement.TryGetProperty("iat", out var iatElement) || iatElement.ValueKind is not JsonValueKind.Number) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing iat claim."); + } + + if (!payloadElement.TryGetProperty("jti", out var jtiElement) || jtiElement.ValueKind != JsonValueKind.String) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP proof missing jti claim."); + } + + long iatSeconds; + try + { + iatSeconds = iatElement.GetInt64(); + } + catch (Exception) + { + return DpopValidationResult.Failure("invalid_payload", "DPoP proof iat claim is not a valid number."); + } + + var issuedAt = DateTimeOffset.FromUnixTimeSeconds(iatSeconds).ToUniversalTime(); + if (issuedAt - options.AllowedClockSkew > now) + { + return DpopValidationResult.Failure("invalid_token", "DPoP proof issued in the future."); + } + + if (now - issuedAt > options.GetMaximumAge()) + { + return DpopValidationResult.Failure("invalid_token", "DPoP proof expired."); + } + + string? actualNonce = null; + + if (nonce is not null) + { + if (!payloadElement.TryGetProperty("nonce", out var nonceElement) || nonceElement.ValueKind != JsonValueKind.String) + { + return DpopValidationResult.Failure("invalid_token", "DPoP proof missing nonce claim."); + } + + actualNonce = nonceElement.GetString(); + + if (!string.Equals(actualNonce, nonce, StringComparison.Ordinal)) + { + return DpopValidationResult.Failure("invalid_token", "DPoP nonce mismatch."); + } + } + else if (payloadElement.TryGetProperty("nonce", out var nonceElement) && nonceElement.ValueKind == JsonValueKind.String) + { + actualNonce = nonceElement.GetString(); + } + + var jwtId = jtiElement.GetString()!; + + try + { + var parameters = new TokenValidationParameters + { + ValidateAudience = false, + ValidateIssuer = false, + ValidateLifetime = false, + ValidateTokenReplay = false, + RequireSignedTokens = true, + ValidateIssuerSigningKey = true, + IssuerSigningKey = jwk, + ValidAlgorithms = options.NormalizedAlgorithms.ToArray() + }; + + tokenHandler.ValidateToken(proof, parameters, out _); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "DPoP proof signature validation failed."); + return DpopValidationResult.Failure("invalid_signature", "DPoP proof signature validation failed."); + } + + if (!await replayCache.TryStoreAsync(jwtId, issuedAt + options.ReplayWindow, cancellationToken).ConfigureAwait(false)) + { + return DpopValidationResult.Failure("replay", "DPoP proof already used."); + } + + return DpopValidationResult.Success(jwk, jwtId, issuedAt, actualNonce); + } + + private static string NormalizeHtu(Uri uri) + { + var builder = new UriBuilder(uri) + { + Fragment = null, + Query = null + }; + return builder.Uri.ToString(); + } + + private static bool TryDecodeSegment(string token, int segmentIndex, out JsonElement element, out string? error) + { + element = default; + error = null; + + var segments = token.Split('.'); + if (segments.Length != 3) + { + error = "Token must contain three segments."; + return false; + } + + if (segmentIndex < 0 || segmentIndex > 2) + { + error = "Segment index out of range."; + return false; + } + + try + { + var json = Base64UrlEncoder.Decode(segments[segmentIndex]); + using var document = JsonDocument.Parse(json); + element = document.RootElement.Clone(); + return true; + } + catch (Exception ex) + { + error = ex.Message; + return false; + } + } + + private static class NullReplayCache + { + public static readonly IDpopReplayCache Instance = new Noop(); + + private sealed class Noop : IDpopReplayCache + { + public ValueTask TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jwtId); + return ValueTask.FromResult(true); + } + } + } +} + +file static class DpopValidationOptionsExtensions +{ + public static TimeSpan GetMaximumAge(this DpopValidationOptions options) + => options.ProofLifetime + options.AllowedClockSkew; +} diff --git a/src/StellaOps.Auth.Security/Dpop/DpopValidationOptions.cs b/src/StellaOps.Auth.Security/Dpop/DpopValidationOptions.cs new file mode 100644 index 00000000..600c53b3 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopValidationOptions.cs @@ -0,0 +1,77 @@ +using System.Collections.Immutable; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Configures acceptable algorithms and replay windows for DPoP proof validation. +/// +public sealed class DpopValidationOptions +{ + private readonly HashSet allowedAlgorithms = new(StringComparer.Ordinal); + + public DpopValidationOptions() + { + allowedAlgorithms.Add("ES256"); + allowedAlgorithms.Add("ES384"); + } + + /// + /// Maximum age a proof is considered valid relative to . + /// + public TimeSpan ProofLifetime { get; set; } = TimeSpan.FromMinutes(2); + + /// + /// Allowed clock skew when evaluating iat. + /// + public TimeSpan AllowedClockSkew { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Duration a successfully validated proof is tracked to prevent replay. + /// + public TimeSpan ReplayWindow { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Algorithms (JWA) permitted for DPoP proofs. + /// + public ISet AllowedAlgorithms => allowedAlgorithms; + + /// + /// Normalised, upper-case representation of allowed algorithms. + /// + public IReadOnlySet NormalizedAlgorithms { get; private set; } = ImmutableHashSet.Empty; + + public void Validate() + { + if (ProofLifetime <= TimeSpan.Zero) + { + throw new InvalidOperationException("DPoP proof lifetime must be greater than zero."); + } + + if (AllowedClockSkew < TimeSpan.Zero || AllowedClockSkew > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("DPoP allowed clock skew must be between 0 seconds and 5 minutes."); + } + + if (ReplayWindow < TimeSpan.Zero) + { + throw new InvalidOperationException("DPoP replay window must be greater than or equal to zero."); + } + + if (allowedAlgorithms.Count == 0) + { + throw new InvalidOperationException("At least one allowed DPoP algorithm must be configured."); + } + + NormalizedAlgorithms = allowedAlgorithms + .Select(static algorithm => algorithm.Trim().ToUpperInvariant()) + .Where(static algorithm => algorithm.Length > 0) + .ToImmutableHashSet(StringComparer.Ordinal); + + if (NormalizedAlgorithms.Count == 0) + { + throw new InvalidOperationException("Allowed DPoP algorithms cannot be empty after normalization."); + } + } +} diff --git a/src/StellaOps.Auth.Security/Dpop/DpopValidationResult.cs b/src/StellaOps.Auth.Security/Dpop/DpopValidationResult.cs new file mode 100644 index 00000000..6a36a264 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/DpopValidationResult.cs @@ -0,0 +1,40 @@ +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Represents the outcome of DPoP proof validation. +/// +public sealed class DpopValidationResult +{ + private DpopValidationResult(bool success, string? errorCode, string? errorDescription, SecurityKey? key, string? jwtId, DateTimeOffset? issuedAt, string? nonce) + { + IsValid = success; + ErrorCode = errorCode; + ErrorDescription = errorDescription; + PublicKey = key; + JwtId = jwtId; + IssuedAt = issuedAt; + Nonce = nonce; + } + + public bool IsValid { get; } + + public string? ErrorCode { get; } + + public string? ErrorDescription { get; } + + public SecurityKey? PublicKey { get; } + + public string? JwtId { get; } + + public DateTimeOffset? IssuedAt { get; } + + public string? Nonce { get; } + + public static DpopValidationResult Success(SecurityKey key, string jwtId, DateTimeOffset issuedAt, string? nonce) + => new(true, null, null, key, jwtId, issuedAt, nonce); + + public static DpopValidationResult Failure(string code, string description) + => new(false, code, description, null, null, null, null); +} diff --git a/src/StellaOps.Auth.Security/Dpop/IDpopNonceStore.cs b/src/StellaOps.Auth.Security/Dpop/IDpopNonceStore.cs new file mode 100644 index 00000000..ba89c344 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/IDpopNonceStore.cs @@ -0,0 +1,45 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Provides persistence and validation for DPoP nonces. +/// +public interface IDpopNonceStore +{ + /// + /// Issues a nonce tied to the specified audience, client, and DPoP key thumbprint. + /// + /// Audience the nonce applies to. + /// Client identifier requesting the nonce. + /// Thumbprint of the DPoP public key. + /// Time-to-live for the nonce. + /// Maximum number of nonces that can be issued within a one-minute window for the tuple. + /// Cancellation token. + /// Outcome describing the issued nonce. + ValueTask IssueAsync( + string audience, + string clientId, + string keyThumbprint, + TimeSpan ttl, + int maxIssuancePerMinute, + CancellationToken cancellationToken = default); + + /// + /// Attempts to consume a nonce previously issued for the tuple. + /// + /// Nonce supplied by the client. + /// Audience the nonce should match. + /// Client identifier. + /// Thumbprint of the DPoP public key. + /// Cancellation token. + /// Outcome describing whether the nonce was accepted. + ValueTask TryConsumeAsync( + string nonce, + string audience, + string clientId, + string keyThumbprint, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Auth.Security/Dpop/IDpopProofValidator.cs b/src/StellaOps.Auth.Security/Dpop/IDpopProofValidator.cs new file mode 100644 index 00000000..b2ab6ed4 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/IDpopProofValidator.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Auth.Security.Dpop; + +public interface IDpopProofValidator +{ + ValueTask ValidateAsync(string proof, string httpMethod, Uri httpUri, string? nonce = null, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Auth.Security/Dpop/IDpopReplayCache.cs b/src/StellaOps.Auth.Security/Dpop/IDpopReplayCache.cs new file mode 100644 index 00000000..09d192b1 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/IDpopReplayCache.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Auth.Security.Dpop; + +public interface IDpopReplayCache +{ + ValueTask TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Auth.Security/Dpop/InMemoryDpopNonceStore.cs b/src/StellaOps.Auth.Security/Dpop/InMemoryDpopNonceStore.cs new file mode 100644 index 00000000..97cf4eb9 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/InMemoryDpopNonceStore.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using Microsoft.Extensions.Logging; +using System.Threading.Tasks; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// In-memory implementation of suitable for single-host or test environments. +/// +public sealed class InMemoryDpopNonceStore : IDpopNonceStore +{ + private static readonly TimeSpan IssuanceWindow = TimeSpan.FromMinutes(1); + private readonly ConcurrentDictionary nonces = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary issuanceBuckets = new(StringComparer.Ordinal); + private readonly TimeProvider timeProvider; + private readonly ILogger? logger; + + public InMemoryDpopNonceStore(TimeProvider? timeProvider = null, ILogger? logger = null) + { + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + } + + public ValueTask IssueAsync( + string audience, + string clientId, + string keyThumbprint, + TimeSpan ttl, + int maxIssuancePerMinute, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + ArgumentException.ThrowIfNullOrWhiteSpace(clientId); + ArgumentException.ThrowIfNullOrWhiteSpace(keyThumbprint); + + if (ttl <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(ttl), "Nonce TTL must be greater than zero."); + } + + if (maxIssuancePerMinute < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxIssuancePerMinute), "Max issuance per minute must be at least 1."); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var now = timeProvider.GetUtcNow(); + var bucketKey = BuildBucketKey(audience, clientId, keyThumbprint); + var bucket = issuanceBuckets.GetOrAdd(bucketKey, static _ => new IssuanceBucket()); + + bool allowed; + lock (bucket.SyncRoot) + { + bucket.Prune(now - IssuanceWindow); + + if (bucket.IssuanceTimes.Count >= maxIssuancePerMinute) + { + allowed = false; + } + else + { + bucket.IssuanceTimes.Enqueue(now); + allowed = true; + } + } + + if (!allowed) + { + logger?.LogDebug("DPoP nonce issuance throttled for {BucketKey}.", bucketKey); + return ValueTask.FromResult(DpopNonceIssueResult.RateLimited("rate_limited")); + } + + var nonce = GenerateNonce(); + var nonceKey = BuildNonceKey(audience, clientId, keyThumbprint, nonce); + var expiresAt = now + ttl; + nonces[nonceKey] = new StoredNonce(now, expiresAt); + return ValueTask.FromResult(DpopNonceIssueResult.Success(nonce, expiresAt)); + } + + public ValueTask TryConsumeAsync( + string nonce, + string audience, + string clientId, + string keyThumbprint, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nonce); + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + ArgumentException.ThrowIfNullOrWhiteSpace(clientId); + ArgumentException.ThrowIfNullOrWhiteSpace(keyThumbprint); + + cancellationToken.ThrowIfCancellationRequested(); + + var now = timeProvider.GetUtcNow(); + var nonceKey = BuildNonceKey(audience, clientId, keyThumbprint, nonce); + + if (!nonces.TryRemove(nonceKey, out var stored)) + { + logger?.LogDebug("DPoP nonce {NonceKey} not found during consumption.", nonceKey); + return ValueTask.FromResult(DpopNonceConsumeResult.NotFound()); + } + + if (stored.ExpiresAt <= now) + { + logger?.LogDebug("DPoP nonce {NonceKey} expired at {ExpiresAt:o}.", nonceKey, stored.ExpiresAt); + return ValueTask.FromResult(DpopNonceConsumeResult.Expired(stored.IssuedAt, stored.ExpiresAt)); + } + + return ValueTask.FromResult(DpopNonceConsumeResult.Success(stored.IssuedAt, stored.ExpiresAt)); + } + + private static string BuildBucketKey(string audience, string clientId, string keyThumbprint) + => $"{audience.Trim().ToLowerInvariant()}::{clientId.Trim().ToLowerInvariant()}::{keyThumbprint.Trim().ToLowerInvariant()}"; + + private static string BuildNonceKey(string audience, string clientId, string keyThumbprint, string nonce) + { + var bucketKey = BuildBucketKey(audience, clientId, keyThumbprint); + var digest = ComputeSha256(nonce); + return $"{bucketKey}::{digest}"; + } + + private static string ComputeSha256(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Base64UrlEncode(hash); + } + + private static string Base64UrlEncode(ReadOnlySpan bytes) + { + return Convert.ToBase64String(bytes) + .TrimEnd('=') + .Replace('+', '-') + .Replace('/', '_'); + } + + private static string GenerateNonce() + { + Span buffer = stackalloc byte[32]; + RandomNumberGenerator.Fill(buffer); + return Base64UrlEncode(buffer); + } + + private sealed class StoredNonce + { + internal StoredNonce(DateTimeOffset issuedAt, DateTimeOffset expiresAt) + { + IssuedAt = issuedAt; + ExpiresAt = expiresAt; + } + + internal DateTimeOffset IssuedAt { get; } + + internal DateTimeOffset ExpiresAt { get; } + } + + private sealed class IssuanceBucket + { + internal object SyncRoot { get; } = new(); + internal Queue IssuanceTimes { get; } = new(); + + internal void Prune(DateTimeOffset threshold) + { + while (IssuanceTimes.Count > 0 && IssuanceTimes.Peek() < threshold) + { + IssuanceTimes.Dequeue(); + } + } + } +} diff --git a/src/StellaOps.Auth.Security/Dpop/InMemoryDpopReplayCache.cs b/src/StellaOps.Auth.Security/Dpop/InMemoryDpopReplayCache.cs new file mode 100644 index 00000000..3887123a --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/InMemoryDpopReplayCache.cs @@ -0,0 +1,66 @@ +using System.Collections.Concurrent; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// In-memory replay cache intended for single-process deployments or tests. +/// +public sealed class InMemoryDpopReplayCache : IDpopReplayCache +{ + private readonly ConcurrentDictionary entries = new(StringComparer.Ordinal); + private readonly TimeProvider timeProvider; + + public InMemoryDpopReplayCache(TimeProvider? timeProvider = null) + { + this.timeProvider = timeProvider ?? TimeProvider.System; + } + + public ValueTask TryStoreAsync(string jwtId, DateTimeOffset expiresAt, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jwtId); + + var now = timeProvider.GetUtcNow(); + RemoveExpired(now); + + if (entries.TryAdd(jwtId, expiresAt)) + { + return ValueTask.FromResult(true); + } + + while (!cancellationToken.IsCancellationRequested) + { + if (!entries.TryGetValue(jwtId, out var existing)) + { + if (entries.TryAdd(jwtId, expiresAt)) + { + return ValueTask.FromResult(true); + } + + continue; + } + + if (existing > now) + { + return ValueTask.FromResult(false); + } + + if (entries.TryUpdate(jwtId, expiresAt, existing)) + { + return ValueTask.FromResult(true); + } + } + + return ValueTask.FromResult(false); + } + + private void RemoveExpired(DateTimeOffset now) + { + foreach (var entry in entries) + { + if (entry.Value <= now) + { + entries.TryRemove(entry.Key, out _); + } + } + } +} diff --git a/src/StellaOps.Auth.Security/Dpop/RedisDpopNonceStore.cs b/src/StellaOps.Auth.Security/Dpop/RedisDpopNonceStore.cs new file mode 100644 index 00000000..70bee048 --- /dev/null +++ b/src/StellaOps.Auth.Security/Dpop/RedisDpopNonceStore.cs @@ -0,0 +1,138 @@ +using System; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis; + +namespace StellaOps.Auth.Security.Dpop; + +/// +/// Redis-backed implementation of that supports multi-node deployments. +/// +public sealed class RedisDpopNonceStore : IDpopNonceStore +{ + private const string ConsumeScript = @" +local value = redis.call('GET', KEYS[1]) +if value ~= false and value == ARGV[1] then + redis.call('DEL', KEYS[1]) + return 1 +end +return 0"; + + private readonly IConnectionMultiplexer connection; + private readonly TimeProvider timeProvider; + + public RedisDpopNonceStore(IConnectionMultiplexer connection, TimeProvider? timeProvider = null) + { + this.connection = connection ?? throw new ArgumentNullException(nameof(connection)); + this.timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask IssueAsync( + string audience, + string clientId, + string keyThumbprint, + TimeSpan ttl, + int maxIssuancePerMinute, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + ArgumentException.ThrowIfNullOrWhiteSpace(clientId); + ArgumentException.ThrowIfNullOrWhiteSpace(keyThumbprint); + + if (ttl <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(ttl), "Nonce TTL must be greater than zero."); + } + + if (maxIssuancePerMinute < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxIssuancePerMinute), "Max issuance per minute must be at least 1."); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var database = connection.GetDatabase(); + var issuedAt = timeProvider.GetUtcNow(); + + var baseKey = DpopNonceUtilities.ComputeStorageKey(audience, clientId, keyThumbprint); + var nonceKey = (RedisKey)baseKey; + var metadataKey = (RedisKey)(baseKey + ":meta"); + var rateKey = (RedisKey)(baseKey + ":rate"); + + var rateCount = await database.StringIncrementAsync(rateKey, flags: CommandFlags.DemandMaster).ConfigureAwait(false); + if (rateCount == 1) + { + await database.KeyExpireAsync(rateKey, TimeSpan.FromMinutes(1), CommandFlags.DemandMaster).ConfigureAwait(false); + } + + if (rateCount > maxIssuancePerMinute) + { + return DpopNonceIssueResult.RateLimited("rate_limited"); + } + + var nonce = DpopNonceUtilities.GenerateNonce(); + var hash = (RedisValue)DpopNonceUtilities.EncodeHash(DpopNonceUtilities.ComputeNonceHash(nonce)); + var expiresAt = issuedAt + ttl; + + await database.StringSetAsync(nonceKey, hash, ttl, When.Always, CommandFlags.DemandMaster).ConfigureAwait(false); + var metadataValue = FormattableString.Invariant($"{issuedAt.UtcTicks}|{ttl.Ticks}"); + await database.StringSetAsync(metadataKey, metadataValue, ttl, When.Always, CommandFlags.DemandMaster).ConfigureAwait(false); + + return DpopNonceIssueResult.Success(nonce, expiresAt); + } + + public async ValueTask TryConsumeAsync( + string nonce, + string audience, + string clientId, + string keyThumbprint, + CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(nonce); + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + ArgumentException.ThrowIfNullOrWhiteSpace(clientId); + ArgumentException.ThrowIfNullOrWhiteSpace(keyThumbprint); + + cancellationToken.ThrowIfCancellationRequested(); + + var database = connection.GetDatabase(); + + var baseKey = DpopNonceUtilities.ComputeStorageKey(audience, clientId, keyThumbprint); + var nonceKey = (RedisKey)baseKey; + var metadataKey = (RedisKey)(baseKey + ":meta"); + var hash = (RedisValue)DpopNonceUtilities.EncodeHash(DpopNonceUtilities.ComputeNonceHash(nonce)); + + var rawResult = await database.ScriptEvaluateAsync( + ConsumeScript, + new[] { nonceKey }, + new RedisValue[] { hash }).ConfigureAwait(false); + + if (rawResult.IsNull || (long)rawResult != 1) + { + return DpopNonceConsumeResult.NotFound(); + } + + var metadata = await database.StringGetAsync(metadataKey).ConfigureAwait(false); + await database.KeyDeleteAsync(metadataKey, CommandFlags.DemandMaster).ConfigureAwait(false); + + if (!metadata.IsNull) + { + var parts = metadata.ToString() + .Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + if (parts.Length == 2 && + long.TryParse(parts[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out var issuedTicks) && + long.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var ttlTicks)) + { + var issuedAt = new DateTimeOffset(issuedTicks, TimeSpan.Zero); + var expiresAt = issuedAt + TimeSpan.FromTicks(ttlTicks); + return expiresAt <= timeProvider.GetUtcNow() + ? DpopNonceConsumeResult.Expired(issuedAt, expiresAt) + : DpopNonceConsumeResult.Success(issuedAt, expiresAt); + } + } + + return DpopNonceConsumeResult.Success(timeProvider.GetUtcNow(), timeProvider.GetUtcNow()); + } +} diff --git a/src/StellaOps.Auth.Security/README.md b/src/StellaOps.Auth.Security/README.md new file mode 100644 index 00000000..866d4122 --- /dev/null +++ b/src/StellaOps.Auth.Security/README.md @@ -0,0 +1,3 @@ +# StellaOps.Auth.Security + +Shared sender-constraint helpers (DPoP proof validation, replay caches, future mTLS utilities) used by Authority, Scanner, Signer, and other StellaOps services. This package centralises primitives so services remain deterministic while honouring proof-of-possession guarantees. diff --git a/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj b/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj new file mode 100644 index 00000000..ce4ae69f --- /dev/null +++ b/src/StellaOps.Auth.Security/StellaOps.Auth.Security.csproj @@ -0,0 +1,38 @@ + + + net10.0 + preview + enable + enable + true + + + Sender-constrained authentication primitives (DPoP, mTLS) shared across StellaOps services. + StellaOps.Auth.Security + StellaOps + StellaOps + stellaops;dpop;mtls;oauth2;security + AGPL-3.0-or-later + https://stella-ops.org + https://git.stella-ops.org/stella-ops.org/git.stella-ops.org + git + true + true + true + snupkg + README.md + 1.0.0-preview.1 + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs index 1dd7b2dd..bff2005b 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsPrincipalBuilderTests.cs @@ -1,74 +1,74 @@ -using System; -using System.Linq; -using System.Security.Claims; -using StellaOps.Auth.Abstractions; -using Xunit; - -namespace StellaOps.Auth.Abstractions.Tests; - -public class StellaOpsPrincipalBuilderTests -{ - [Fact] - public void NormalizedScopes_AreSortedDeduplicatedLowerCased() - { - var builder = new StellaOpsPrincipalBuilder() - .WithScopes(new[] { "Feedser.Jobs.Trigger", " feedser.jobs.trigger ", "AUTHORITY.USERS.MANAGE" }) - .WithAudiences(new[] { " api://feedser ", "api://cli", "api://feedser" }); - - Assert.Equal( - new[] { "authority.users.manage", "feedser.jobs.trigger" }, - builder.NormalizedScopes); - - Assert.Equal( - new[] { "api://cli", "api://feedser" }, - builder.Audiences); - } - - [Fact] - public void Build_ConstructsClaimsPrincipalWithNormalisedValues() - { - var now = DateTimeOffset.UtcNow; - var builder = new StellaOpsPrincipalBuilder() - .WithSubject(" user-1 ") - .WithClientId(" cli-01 ") - .WithTenant(" default ") - .WithName(" Jane Doe ") - .WithIdentityProvider(" internal ") - .WithSessionId(" session-123 ") - .WithTokenId(Guid.NewGuid().ToString("N")) - .WithAuthenticationMethod("password") - .WithAuthenticationType(" custom ") - .WithScopes(new[] { "Feedser.Jobs.Trigger", "AUTHORITY.USERS.MANAGE" }) - .WithAudience(" api://feedser ") - .WithIssuedAt(now) - .WithExpires(now.AddMinutes(5)) - .AddClaim(" custom ", " value "); - - var principal = builder.Build(); - var identity = Assert.IsType(principal.Identity); - - Assert.Equal("custom", identity.AuthenticationType); - Assert.Equal("Jane Doe", identity.Name); - Assert.Equal("user-1", principal.FindFirstValue(StellaOpsClaimTypes.Subject)); - Assert.Equal("cli-01", principal.FindFirstValue(StellaOpsClaimTypes.ClientId)); - Assert.Equal("default", principal.FindFirstValue(StellaOpsClaimTypes.Tenant)); - Assert.Equal("internal", principal.FindFirstValue(StellaOpsClaimTypes.IdentityProvider)); - Assert.Equal("session-123", principal.FindFirstValue(StellaOpsClaimTypes.SessionId)); - Assert.Equal("value", principal.FindFirstValue("custom")); - - var scopeClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.ScopeItem).Select(claim => claim.Value).ToArray(); - Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, scopeClaims); - - var scopeList = principal.FindFirstValue(StellaOpsClaimTypes.Scope); - Assert.Equal("authority.users.manage feedser.jobs.trigger", scopeList); - - var audienceClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.Audience).Select(claim => claim.Value).ToArray(); - Assert.Equal(new[] { "api://feedser" }, audienceClaims); - - var issuedAt = principal.FindFirstValue("iat"); - Assert.Equal(now.ToUnixTimeSeconds().ToString(), issuedAt); - - var expires = principal.FindFirstValue("exp"); - Assert.Equal(now.AddMinutes(5).ToUnixTimeSeconds().ToString(), expires); - } -} +using System; +using System.Linq; +using System.Security.Claims; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class StellaOpsPrincipalBuilderTests +{ + [Fact] + public void NormalizedScopes_AreSortedDeduplicatedLowerCased() + { + var builder = new StellaOpsPrincipalBuilder() + .WithScopes(new[] { "Concelier.Jobs.Trigger", " concelier.jobs.trigger ", "AUTHORITY.USERS.MANAGE" }) + .WithAudiences(new[] { " api://concelier ", "api://cli", "api://concelier" }); + + Assert.Equal( + new[] { "authority.users.manage", "concelier.jobs.trigger" }, + builder.NormalizedScopes); + + Assert.Equal( + new[] { "api://cli", "api://concelier" }, + builder.Audiences); + } + + [Fact] + public void Build_ConstructsClaimsPrincipalWithNormalisedValues() + { + var now = DateTimeOffset.UtcNow; + var builder = new StellaOpsPrincipalBuilder() + .WithSubject(" user-1 ") + .WithClientId(" cli-01 ") + .WithTenant(" default ") + .WithName(" Jane Doe ") + .WithIdentityProvider(" internal ") + .WithSessionId(" session-123 ") + .WithTokenId(Guid.NewGuid().ToString("N")) + .WithAuthenticationMethod("password") + .WithAuthenticationType(" custom ") + .WithScopes(new[] { "Concelier.Jobs.Trigger", "AUTHORITY.USERS.MANAGE" }) + .WithAudience(" api://concelier ") + .WithIssuedAt(now) + .WithExpires(now.AddMinutes(5)) + .AddClaim(" custom ", " value "); + + var principal = builder.Build(); + var identity = Assert.IsType(principal.Identity); + + Assert.Equal("custom", identity.AuthenticationType); + Assert.Equal("Jane Doe", identity.Name); + Assert.Equal("user-1", principal.FindFirstValue(StellaOpsClaimTypes.Subject)); + Assert.Equal("cli-01", principal.FindFirstValue(StellaOpsClaimTypes.ClientId)); + Assert.Equal("default", principal.FindFirstValue(StellaOpsClaimTypes.Tenant)); + Assert.Equal("internal", principal.FindFirstValue(StellaOpsClaimTypes.IdentityProvider)); + Assert.Equal("session-123", principal.FindFirstValue(StellaOpsClaimTypes.SessionId)); + Assert.Equal("value", principal.FindFirstValue("custom")); + + var scopeClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.ScopeItem).Select(claim => claim.Value).ToArray(); + Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, scopeClaims); + + var scopeList = principal.FindFirstValue(StellaOpsClaimTypes.Scope); + Assert.Equal("authority.users.manage concelier.jobs.trigger", scopeList); + + var audienceClaims = principal.Claims.Where(claim => claim.Type == StellaOpsClaimTypes.Audience).Select(claim => claim.Value).ToArray(); + Assert.Equal(new[] { "api://concelier" }, audienceClaims); + + var issuedAt = principal.FindFirstValue("iat"); + Assert.Equal(now.ToUnixTimeSeconds().ToString(), issuedAt); + + var expires = principal.FindFirstValue("exp"); + Assert.Equal(now.AddMinutes(5).ToUnixTimeSeconds().ToString(), expires); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs index c9243ccc..fe2e1308 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions.Tests/StellaOpsProblemResultFactoryTests.cs @@ -1,53 +1,53 @@ -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Http.HttpResults; -using Microsoft.AspNetCore.Mvc; -using StellaOps.Auth.Abstractions; -using Xunit; - -namespace StellaOps.Auth.Abstractions.Tests; - -public class StellaOpsProblemResultFactoryTests -{ - [Fact] - public void AuthenticationRequired_ReturnsCanonicalProblem() - { - var result = StellaOpsProblemResultFactory.AuthenticationRequired(instance: "/jobs"); - - Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); - var details = Assert.IsType(result.ProblemDetails); - Assert.Equal("https://docs.stella-ops.org/problems/authentication-required", details.Type); - Assert.Equal("Authentication required", details.Title); - Assert.Equal("/jobs", details.Instance); - Assert.Equal("unauthorized", details.Extensions["error"]); - Assert.Equal(details.Detail, details.Extensions["error_description"]); - } - - [Fact] - public void InvalidToken_UsesProvidedDetail() - { - var result = StellaOpsProblemResultFactory.InvalidToken("expired refresh token"); - - var details = Assert.IsType(result.ProblemDetails); - Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); - Assert.Equal("expired refresh token", details.Detail); - Assert.Equal("invalid_token", details.Extensions["error"]); - } - - [Fact] - public void InsufficientScope_AddsScopeExtensions() - { - var result = StellaOpsProblemResultFactory.InsufficientScope( - new[] { StellaOpsScopes.FeedserJobsTrigger }, - new[] { StellaOpsScopes.AuthorityUsersManage }, - instance: "/jobs/trigger"); - - Assert.Equal(StatusCodes.Status403Forbidden, result.StatusCode); - - var details = Assert.IsType(result.ProblemDetails); - Assert.Equal("https://docs.stella-ops.org/problems/insufficient-scope", details.Type); - Assert.Equal("insufficient_scope", details.Extensions["error"]); - Assert.Equal(new[] { StellaOpsScopes.FeedserJobsTrigger }, Assert.IsType(details.Extensions["required_scopes"])); - Assert.Equal(new[] { StellaOpsScopes.AuthorityUsersManage }, Assert.IsType(details.Extensions["granted_scopes"])); - Assert.Equal("/jobs/trigger", details.Instance); - } -} +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.HttpResults; +using Microsoft.AspNetCore.Mvc; +using StellaOps.Auth.Abstractions; +using Xunit; + +namespace StellaOps.Auth.Abstractions.Tests; + +public class StellaOpsProblemResultFactoryTests +{ + [Fact] + public void AuthenticationRequired_ReturnsCanonicalProblem() + { + var result = StellaOpsProblemResultFactory.AuthenticationRequired(instance: "/jobs"); + + Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal("https://docs.stella-ops.org/problems/authentication-required", details.Type); + Assert.Equal("Authentication required", details.Title); + Assert.Equal("/jobs", details.Instance); + Assert.Equal("unauthorized", details.Extensions["error"]); + Assert.Equal(details.Detail, details.Extensions["error_description"]); + } + + [Fact] + public void InvalidToken_UsesProvidedDetail() + { + var result = StellaOpsProblemResultFactory.InvalidToken("expired refresh token"); + + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal(StatusCodes.Status401Unauthorized, result.StatusCode); + Assert.Equal("expired refresh token", details.Detail); + Assert.Equal("invalid_token", details.Extensions["error"]); + } + + [Fact] + public void InsufficientScope_AddsScopeExtensions() + { + var result = StellaOpsProblemResultFactory.InsufficientScope( + new[] { StellaOpsScopes.ConcelierJobsTrigger }, + new[] { StellaOpsScopes.AuthorityUsersManage }, + instance: "/jobs/trigger"); + + Assert.Equal(StatusCodes.Status403Forbidden, result.StatusCode); + + var details = Assert.IsType(result.ProblemDetails); + Assert.Equal("https://docs.stella-ops.org/problems/insufficient-scope", details.Type); + Assert.Equal("insufficient_scope", details.Extensions["error"]); + Assert.Equal(new[] { StellaOpsScopes.ConcelierJobsTrigger }, Assert.IsType(details.Extensions["required_scopes"])); + Assert.Equal(new[] { StellaOpsScopes.AuthorityUsersManage }, Assert.IsType(details.Extensions["granted_scopes"])); + Assert.Equal("/jobs/trigger", details.Instance); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs index 42587835..7028669b 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Abstractions/StellaOpsScopes.cs @@ -1,79 +1,79 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Auth.Abstractions; - -/// -/// Canonical scope names supported by StellaOps services. -/// -public static class StellaOpsScopes -{ - /// - /// Scope required to trigger Feedser jobs. - /// - public const string FeedserJobsTrigger = "feedser.jobs.trigger"; - - /// - /// Scope required to manage Feedser merge operations. - /// - public const string FeedserMerge = "feedser.merge"; - - /// - /// Scope granting administrative access to Authority user management. - /// - public const string AuthorityUsersManage = "authority.users.manage"; - - /// - /// Scope granting administrative access to Authority client registrations. - /// - public const string AuthorityClientsManage = "authority.clients.manage"; - - /// - /// Scope granting read-only access to Authority audit logs. - /// - public const string AuthorityAuditRead = "authority.audit.read"; - - /// - /// Synthetic scope representing trusted network bypass. - /// - public const string Bypass = "stellaops.bypass"; - - private static readonly HashSet KnownScopes = new(StringComparer.OrdinalIgnoreCase) - { - FeedserJobsTrigger, - FeedserMerge, - AuthorityUsersManage, - AuthorityClientsManage, - AuthorityAuditRead, - Bypass - }; - - /// - /// Normalises a scope string (trim/convert to lower case). - /// - /// Scope raw value. - /// Normalised scope or null when the input is blank. - public static string? Normalize(string? scope) - { - if (string.IsNullOrWhiteSpace(scope)) - { - return null; - } - - return scope.Trim().ToLowerInvariant(); - } - - /// - /// Checks whether the provided scope is registered as a built-in StellaOps scope. - /// - public static bool IsKnown(string scope) - { - ArgumentNullException.ThrowIfNull(scope); - return KnownScopes.Contains(scope); - } - - /// - /// Returns the full set of built-in scopes. - /// - public static IReadOnlyCollection All => KnownScopes; -} +using System; +using System.Collections.Generic; + +namespace StellaOps.Auth.Abstractions; + +/// +/// Canonical scope names supported by StellaOps services. +/// +public static class StellaOpsScopes +{ + /// + /// Scope required to trigger Concelier jobs. + /// + public const string ConcelierJobsTrigger = "concelier.jobs.trigger"; + + /// + /// Scope required to manage Concelier merge operations. + /// + public const string ConcelierMerge = "concelier.merge"; + + /// + /// Scope granting administrative access to Authority user management. + /// + public const string AuthorityUsersManage = "authority.users.manage"; + + /// + /// Scope granting administrative access to Authority client registrations. + /// + public const string AuthorityClientsManage = "authority.clients.manage"; + + /// + /// Scope granting read-only access to Authority audit logs. + /// + public const string AuthorityAuditRead = "authority.audit.read"; + + /// + /// Synthetic scope representing trusted network bypass. + /// + public const string Bypass = "stellaops.bypass"; + + private static readonly HashSet KnownScopes = new(StringComparer.OrdinalIgnoreCase) + { + ConcelierJobsTrigger, + ConcelierMerge, + AuthorityUsersManage, + AuthorityClientsManage, + AuthorityAuditRead, + Bypass + }; + + /// + /// Normalises a scope string (trim/convert to lower case). + /// + /// Scope raw value. + /// Normalised scope or null when the input is blank. + public static string? Normalize(string? scope) + { + if (string.IsNullOrWhiteSpace(scope)) + { + return null; + } + + return scope.Trim().ToLowerInvariant(); + } + + /// + /// Checks whether the provided scope is registered as a built-in StellaOps scope. + /// + public static bool IsKnown(string scope) + { + ArgumentNullException.ThrowIfNull(scope); + return KnownScopes.Contains(scope); + } + + /// + /// Returns the full set of built-in scopes. + /// + public static IReadOnlyCollection All => KnownScopes; +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs index e49deb14..f3042b97 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsAuthClientOptionsTests.cs @@ -1,84 +1,84 @@ -using System; -using StellaOps.Auth.Client; -using Xunit; - -namespace StellaOps.Auth.Client.Tests; - -public class StellaOpsAuthClientOptionsTests -{ - [Fact] - public void Validate_NormalizesScopes() - { - var options = new StellaOpsAuthClientOptions - { - Authority = "https://authority.test", - ClientId = "cli", - HttpTimeout = TimeSpan.FromSeconds(15) - }; - options.DefaultScopes.Add(" Feedser.Jobs.Trigger "); - options.DefaultScopes.Add("feedser.jobs.trigger"); - options.DefaultScopes.Add("AUTHORITY.USERS.MANAGE"); - - options.Validate(); - - Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes); - Assert.Equal(new Uri("https://authority.test"), options.AuthorityUri); - Assert.Equal(options.RetryDelays, options.NormalizedRetryDelays); - } - - [Fact] - public void Validate_Throws_When_AuthorityMissing() - { - var options = new StellaOpsAuthClientOptions(); - - var exception = Assert.Throws(() => options.Validate()); - - Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void Validate_NormalizesRetryDelays() - { - var options = new StellaOpsAuthClientOptions - { - Authority = "https://authority.test" - }; - options.RetryDelays.Clear(); - options.RetryDelays.Add(TimeSpan.Zero); - options.RetryDelays.Add(TimeSpan.FromSeconds(3)); - options.RetryDelays.Add(TimeSpan.FromMilliseconds(-1)); - - options.Validate(); - - Assert.Equal(new[] { TimeSpan.FromSeconds(3) }, options.NormalizedRetryDelays); - Assert.Equal(options.NormalizedRetryDelays, options.RetryDelays); - } - - [Fact] - public void Validate_DisabledRetries_ProducesEmptyDelays() - { - var options = new StellaOpsAuthClientOptions - { - Authority = "https://authority.test", - EnableRetries = false - }; - - options.Validate(); - - Assert.Empty(options.NormalizedRetryDelays); - } - - [Fact] - public void Validate_Throws_When_OfflineToleranceNegative() - { - var options = new StellaOpsAuthClientOptions - { - Authority = "https://authority.test", - OfflineCacheTolerance = TimeSpan.FromSeconds(-1) - }; - - var exception = Assert.Throws(() => options.Validate()); - - Assert.Contains("Offline cache tolerance", exception.Message, StringComparison.OrdinalIgnoreCase); - } -} +using System; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class StellaOpsAuthClientOptionsTests +{ + [Fact] + public void Validate_NormalizesScopes() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + ClientId = "cli", + HttpTimeout = TimeSpan.FromSeconds(15) + }; + options.DefaultScopes.Add(" Concelier.Jobs.Trigger "); + options.DefaultScopes.Add("concelier.jobs.trigger"); + options.DefaultScopes.Add("AUTHORITY.USERS.MANAGE"); + + options.Validate(); + + Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes); + Assert.Equal(new Uri("https://authority.test"), options.AuthorityUri); + Assert.Equal(options.RetryDelays, options.NormalizedRetryDelays); + } + + [Fact] + public void Validate_Throws_When_AuthorityMissing() + { + var options = new StellaOpsAuthClientOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Validate_NormalizesRetryDelays() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test" + }; + options.RetryDelays.Clear(); + options.RetryDelays.Add(TimeSpan.Zero); + options.RetryDelays.Add(TimeSpan.FromSeconds(3)); + options.RetryDelays.Add(TimeSpan.FromMilliseconds(-1)); + + options.Validate(); + + Assert.Equal(new[] { TimeSpan.FromSeconds(3) }, options.NormalizedRetryDelays); + Assert.Equal(options.NormalizedRetryDelays, options.RetryDelays); + } + + [Fact] + public void Validate_DisabledRetries_ProducesEmptyDelays() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + EnableRetries = false + }; + + options.Validate(); + + Assert.Empty(options.NormalizedRetryDelays); + } + + [Fact] + public void Validate_Throws_When_OfflineToleranceNegative() + { + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + OfflineCacheTolerance = TimeSpan.FromSeconds(-1) + }; + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Offline cache tolerance", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs index 6dbe89ed..68a32c21 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.Client.Tests/StellaOpsTokenClientTests.cs @@ -1,111 +1,111 @@ -using System; -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using StellaOps.Auth.Client; -using Xunit; - -namespace StellaOps.Auth.Client.Tests; - -public class StellaOpsTokenClientTests -{ - [Fact] - public async Task RequestPasswordToken_ReturnsResultAndCaches() - { - var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-02-01T00:00:00Z")); - var responses = new Queue(); - responses.Enqueue(CreateJsonResponse("{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}")); - responses.Enqueue(CreateJsonResponse("{\"access_token\":\"abc\",\"token_type\":\"Bearer\",\"expires_in\":120,\"scope\":\"feedser.jobs.trigger\"}")); - responses.Enqueue(CreateJsonResponse("{\"keys\":[]}")); - - var handler = new StubHttpMessageHandler((request, cancellationToken) => - { - Assert.True(responses.Count > 0, $"Unexpected request {request.Method} {request.RequestUri}"); - return Task.FromResult(responses.Dequeue()); - }); - - var httpClient = new HttpClient(handler); - - var options = new StellaOpsAuthClientOptions - { - Authority = "https://authority.test", - ClientId = "cli" - }; - options.DefaultScopes.Add("feedser.jobs.trigger"); - options.Validate(); - - var optionsMonitor = new TestOptionsMonitor(options); - var cache = new InMemoryTokenCache(timeProvider, TimeSpan.FromSeconds(5)); - var discoveryCache = new StellaOpsDiscoveryCache(httpClient, optionsMonitor, timeProvider); - var jwksCache = new StellaOpsJwksCache(httpClient, discoveryCache, optionsMonitor, timeProvider); - var client = new StellaOpsTokenClient(httpClient, discoveryCache, jwksCache, optionsMonitor, cache, timeProvider, NullLogger.Instance); - - var result = await client.RequestPasswordTokenAsync("user", "pass"); - - Assert.Equal("abc", result.AccessToken); - Assert.Contains("feedser.jobs.trigger", result.Scopes); - - await client.CacheTokenAsync("key", result.ToCacheEntry()); - var cached = await client.GetCachedTokenAsync("key"); - Assert.NotNull(cached); - Assert.Equal("abc", cached!.AccessToken); - - var jwks = await client.GetJsonWebKeySetAsync(); - Assert.Empty(jwks.Keys); - } - - private static HttpResponseMessage CreateJsonResponse(string json) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(json) - { - Headers = { ContentType = new MediaTypeHeaderValue("application/json") } - } - }; - } - - private sealed class StubHttpMessageHandler : HttpMessageHandler - { - private readonly Func> responder; - - public StubHttpMessageHandler(Func> responder) - { - this.responder = responder; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - => responder(request, cancellationToken); - } - - private sealed class TestOptionsMonitor : IOptionsMonitor - where TOptions : class - { - private readonly TOptions value; - - public TestOptionsMonitor(TOptions value) - { - this.value = value; - } - - public TOptions CurrentValue => value; - - public TOptions Get(string? name) => value; - - public IDisposable OnChange(Action listener) => NullDisposable.Instance; - - private sealed class NullDisposable : IDisposable - { - public static NullDisposable Instance { get; } = new(); - public void Dispose() - { - } - } - } -} +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Auth.Client; +using Xunit; + +namespace StellaOps.Auth.Client.Tests; + +public class StellaOpsTokenClientTests +{ + [Fact] + public async Task RequestPasswordToken_ReturnsResultAndCaches() + { + var timeProvider = new FakeTimeProvider(DateTimeOffset.Parse("2025-02-01T00:00:00Z")); + var responses = new Queue(); + responses.Enqueue(CreateJsonResponse("{\"token_endpoint\":\"https://authority.test/connect/token\",\"jwks_uri\":\"https://authority.test/jwks\"}")); + responses.Enqueue(CreateJsonResponse("{\"access_token\":\"abc\",\"token_type\":\"Bearer\",\"expires_in\":120,\"scope\":\"concelier.jobs.trigger\"}")); + responses.Enqueue(CreateJsonResponse("{\"keys\":[]}")); + + var handler = new StubHttpMessageHandler((request, cancellationToken) => + { + Assert.True(responses.Count > 0, $"Unexpected request {request.Method} {request.RequestUri}"); + return Task.FromResult(responses.Dequeue()); + }); + + var httpClient = new HttpClient(handler); + + var options = new StellaOpsAuthClientOptions + { + Authority = "https://authority.test", + ClientId = "cli" + }; + options.DefaultScopes.Add("concelier.jobs.trigger"); + options.Validate(); + + var optionsMonitor = new TestOptionsMonitor(options); + var cache = new InMemoryTokenCache(timeProvider, TimeSpan.FromSeconds(5)); + var discoveryCache = new StellaOpsDiscoveryCache(httpClient, optionsMonitor, timeProvider); + var jwksCache = new StellaOpsJwksCache(httpClient, discoveryCache, optionsMonitor, timeProvider); + var client = new StellaOpsTokenClient(httpClient, discoveryCache, jwksCache, optionsMonitor, cache, timeProvider, NullLogger.Instance); + + var result = await client.RequestPasswordTokenAsync("user", "pass"); + + Assert.Equal("abc", result.AccessToken); + Assert.Contains("concelier.jobs.trigger", result.Scopes); + + await client.CacheTokenAsync("key", result.ToCacheEntry()); + var cached = await client.GetCachedTokenAsync("key"); + Assert.NotNull(cached); + Assert.Equal("abc", cached!.AccessToken); + + var jwks = await client.GetJsonWebKeySetAsync(); + Assert.Empty(jwks.Keys); + } + + private static HttpResponseMessage CreateJsonResponse(string json) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json) + { + Headers = { ContentType = new MediaTypeHeaderValue("application/json") } + } + }; + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Func> responder; + + public StubHttpMessageHandler(Func> responder) + { + this.responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + => responder(request, cancellationToken); + } + + private sealed class TestOptionsMonitor : IOptionsMonitor + where TOptions : class + { + private readonly TOptions value; + + public TestOptionsMonitor(TOptions value) + { + this.value = value; + } + + public TOptions CurrentValue => value; + + public TOptions Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs index c0477f2a..8f904ddf 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/ServiceCollectionExtensionsTests.cs @@ -1,44 +1,44 @@ -using System; -using System.Collections.Generic; -using Microsoft.AspNetCore.Authentication.JwtBearer; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.ServerIntegration; -using Xunit; - -namespace StellaOps.Auth.ServerIntegration.Tests; - -public class ServiceCollectionExtensionsTests -{ - [Fact] - public void AddStellaOpsResourceServerAuthentication_ConfiguresJwtBearer() - { - var configuration = new ConfigurationBuilder() - .AddInMemoryCollection(new Dictionary - { - ["Authority:ResourceServer:Authority"] = "https://authority.example", - ["Authority:ResourceServer:Audiences:0"] = "api://feedser", - ["Authority:ResourceServer:RequiredScopes:0"] = "feedser.jobs.trigger", - ["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32" - }) - .Build(); - - var services = new ServiceCollection(); - services.AddLogging(); - services.AddStellaOpsResourceServerAuthentication(configuration); - - using var provider = services.BuildServiceProvider(); - - var resourceOptions = provider.GetRequiredService>().CurrentValue; - var jwtOptions = provider.GetRequiredService>().Get(StellaOpsAuthenticationDefaults.AuthenticationScheme); - - Assert.NotNull(jwtOptions.Authority); - Assert.Equal(new Uri("https://authority.example/"), new Uri(jwtOptions.Authority!)); - Assert.True(jwtOptions.TokenValidationParameters.ValidateAudience); - Assert.Contains("api://feedser", jwtOptions.TokenValidationParameters.ValidAudiences); - Assert.Equal(TimeSpan.FromSeconds(60), jwtOptions.TokenValidationParameters.ClockSkew); - Assert.Equal(new[] { "feedser.jobs.trigger" }, resourceOptions.NormalizedScopes); - } -} +using System; +using System.Collections.Generic; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class ServiceCollectionExtensionsTests +{ + [Fact] + public void AddStellaOpsResourceServerAuthentication_ConfiguresJwtBearer() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["Authority:ResourceServer:Authority"] = "https://authority.example", + ["Authority:ResourceServer:Audiences:0"] = "api://concelier", + ["Authority:ResourceServer:RequiredScopes:0"] = "concelier.jobs.trigger", + ["Authority:ResourceServer:BypassNetworks:0"] = "127.0.0.1/32" + }) + .Build(); + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddStellaOpsResourceServerAuthentication(configuration); + + using var provider = services.BuildServiceProvider(); + + var resourceOptions = provider.GetRequiredService>().CurrentValue; + var jwtOptions = provider.GetRequiredService>().Get(StellaOpsAuthenticationDefaults.AuthenticationScheme); + + Assert.NotNull(jwtOptions.Authority); + Assert.Equal(new Uri("https://authority.example/"), new Uri(jwtOptions.Authority!)); + Assert.True(jwtOptions.TokenValidationParameters.ValidateAudience); + Assert.Contains("api://concelier", jwtOptions.TokenValidationParameters.ValidAudiences); + Assert.Equal(TimeSpan.FromSeconds(60), jwtOptions.TokenValidationParameters.ClockSkew); + Assert.Equal(new[] { "concelier.jobs.trigger" }, resourceOptions.NormalizedScopes); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs index 91454125..69731659 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsResourceServerOptionsTests.cs @@ -1,50 +1,50 @@ -using System; -using System.Net; -using StellaOps.Auth.ServerIntegration; -using Xunit; - -namespace StellaOps.Auth.ServerIntegration.Tests; - -public class StellaOpsResourceServerOptionsTests -{ - [Fact] - public void Validate_NormalisesCollections() - { - var options = new StellaOpsResourceServerOptions - { - Authority = "https://authority.stella-ops.test", - BackchannelTimeout = TimeSpan.FromSeconds(10), - TokenClockSkew = TimeSpan.FromSeconds(30) - }; - - options.Audiences.Add(" api://feedser "); - options.Audiences.Add("api://feedser"); - options.Audiences.Add("api://feedser-admin"); - - options.RequiredScopes.Add(" Feedser.Jobs.Trigger "); - options.RequiredScopes.Add("feedser.jobs.trigger"); - options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE"); - - options.BypassNetworks.Add("127.0.0.1/32"); - options.BypassNetworks.Add(" 127.0.0.1/32 "); - options.BypassNetworks.Add("::1/128"); - - options.Validate(); - - Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri); - Assert.Equal(new[] { "api://feedser", "api://feedser-admin" }, options.Audiences); - Assert.Equal(new[] { "authority.users.manage", "feedser.jobs.trigger" }, options.NormalizedScopes); - Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); - Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback)); - } - - [Fact] - public void Validate_Throws_When_AuthorityMissing() - { - var options = new StellaOpsResourceServerOptions(); - - var exception = Assert.Throws(() => options.Validate()); - - Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); - } -} +using System; +using System.Net; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsResourceServerOptionsTests +{ + [Fact] + public void Validate_NormalisesCollections() + { + var options = new StellaOpsResourceServerOptions + { + Authority = "https://authority.stella-ops.test", + BackchannelTimeout = TimeSpan.FromSeconds(10), + TokenClockSkew = TimeSpan.FromSeconds(30) + }; + + options.Audiences.Add(" api://concelier "); + options.Audiences.Add("api://concelier"); + options.Audiences.Add("api://concelier-admin"); + + options.RequiredScopes.Add(" Concelier.Jobs.Trigger "); + options.RequiredScopes.Add("concelier.jobs.trigger"); + options.RequiredScopes.Add("AUTHORITY.USERS.MANAGE"); + + options.BypassNetworks.Add("127.0.0.1/32"); + options.BypassNetworks.Add(" 127.0.0.1/32 "); + options.BypassNetworks.Add("::1/128"); + + options.Validate(); + + Assert.Equal(new Uri("https://authority.stella-ops.test"), options.AuthorityUri); + Assert.Equal(new[] { "api://concelier", "api://concelier-admin" }, options.Audiences); + Assert.Equal(new[] { "authority.users.manage", "concelier.jobs.trigger" }, options.NormalizedScopes); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.Parse("127.0.0.1"))); + Assert.True(options.BypassMatcher.IsAllowed(IPAddress.IPv6Loopback)); + } + + [Fact] + public void Validate_Throws_When_AuthorityMissing() + { + var options = new StellaOpsResourceServerOptions(); + + var exception = Assert.Throws(() => options.Validate()); + + Assert.Contains("Authority", exception.Message, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs index 534a6ef0..b2efc9ee 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration.Tests/StellaOpsScopeAuthorizationHandlerTests.cs @@ -1,123 +1,123 @@ -using System; -using System.Net; -using System.Security.Claims; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.ServerIntegration; -using Xunit; - -namespace StellaOps.Auth.ServerIntegration.Tests; - -public class StellaOpsScopeAuthorizationHandlerTests -{ - [Fact] - public async Task HandleRequirement_Succeeds_WhenScopePresent() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); - var principal = new StellaOpsPrincipalBuilder() - .WithSubject("user-1") - .WithScopes(new[] { StellaOpsScopes.FeedserJobsTrigger }) - .Build(); - - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.True(context.HasSucceeded); - } - - [Fact] - public async Task HandleRequirement_Succeeds_WhenBypassNetworkMatches() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.BypassNetworks.Add("127.0.0.1/32"); - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); - var principal = new ClaimsPrincipal(new ClaimsIdentity()); - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.True(context.HasSucceeded); - } - - [Fact] - public async Task HandleRequirement_Fails_WhenScopeMissingAndNoBypass() - { - var optionsMonitor = CreateOptionsMonitor(options => - { - options.Authority = "https://authority.example"; - options.Validate(); - }); - - var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10")); - var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.FeedserJobsTrigger }); - var principal = new ClaimsPrincipal(new ClaimsIdentity()); - var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); - - await handler.HandleAsync(context); - - Assert.False(context.HasSucceeded); - } - - private static (StellaOpsScopeAuthorizationHandler Handler, IHttpContextAccessor Accessor) CreateHandler(IOptionsMonitor optionsMonitor, IPAddress remoteAddress) - { - var accessor = new HttpContextAccessor(); - var httpContext = new DefaultHttpContext(); - httpContext.Connection.RemoteIpAddress = remoteAddress; - accessor.HttpContext = httpContext; - - var bypassEvaluator = new StellaOpsBypassEvaluator(optionsMonitor, NullLogger.Instance); - - var handler = new StellaOpsScopeAuthorizationHandler( - accessor, - bypassEvaluator, - NullLogger.Instance); - return (handler, accessor); - } - - private static IOptionsMonitor CreateOptionsMonitor(Action configure) - => new TestOptionsMonitor(configure); - - private sealed class TestOptionsMonitor : IOptionsMonitor - where TOptions : class, new() - { - private readonly TOptions value; - - public TestOptionsMonitor(Action configure) - { - value = new TOptions(); - configure(value); - } - - public TOptions CurrentValue => value; - - public TOptions Get(string? name) => value; - - public IDisposable OnChange(Action listener) => NullDisposable.Instance; - - private sealed class NullDisposable : IDisposable - { - public static NullDisposable Instance { get; } = new(); - public void Dispose() - { - } - } - } -} +using System; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.ServerIntegration; +using Xunit; + +namespace StellaOps.Auth.ServerIntegration.Tests; + +public class StellaOpsScopeAuthorizationHandlerTests +{ + [Fact] + public async Task HandleRequirement_Succeeds_WhenScopePresent() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("10.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new StellaOpsPrincipalBuilder() + .WithSubject("user-1") + .WithScopes(new[] { StellaOpsScopes.ConcelierJobsTrigger }) + .Build(); + + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Succeeds_WhenBypassNetworkMatches() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.BypassNetworks.Add("127.0.0.1/32"); + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("127.0.0.1")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.True(context.HasSucceeded); + } + + [Fact] + public async Task HandleRequirement_Fails_WhenScopeMissingAndNoBypass() + { + var optionsMonitor = CreateOptionsMonitor(options => + { + options.Authority = "https://authority.example"; + options.Validate(); + }); + + var (handler, accessor) = CreateHandler(optionsMonitor, remoteAddress: IPAddress.Parse("203.0.113.10")); + var requirement = new StellaOpsScopeRequirement(new[] { StellaOpsScopes.ConcelierJobsTrigger }); + var principal = new ClaimsPrincipal(new ClaimsIdentity()); + var context = new AuthorizationHandlerContext(new[] { requirement }, principal, accessor.HttpContext); + + await handler.HandleAsync(context); + + Assert.False(context.HasSucceeded); + } + + private static (StellaOpsScopeAuthorizationHandler Handler, IHttpContextAccessor Accessor) CreateHandler(IOptionsMonitor optionsMonitor, IPAddress remoteAddress) + { + var accessor = new HttpContextAccessor(); + var httpContext = new DefaultHttpContext(); + httpContext.Connection.RemoteIpAddress = remoteAddress; + accessor.HttpContext = httpContext; + + var bypassEvaluator = new StellaOpsBypassEvaluator(optionsMonitor, NullLogger.Instance); + + var handler = new StellaOpsScopeAuthorizationHandler( + accessor, + bypassEvaluator, + NullLogger.Instance); + return (handler, accessor); + } + + private static IOptionsMonitor CreateOptionsMonitor(Action configure) + => new TestOptionsMonitor(configure); + + private sealed class TestOptionsMonitor : IOptionsMonitor + where TOptions : class, new() + { + private readonly TOptions value; + + public TestOptionsMonitor(Action configure) + { + value = new TOptions(); + configure(value); + } + + public TOptions CurrentValue => value; + + public TOptions Get(string? name) => value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static NullDisposable Instance { get; } = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md index 1f7eba7f..3577521c 100644 --- a/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md +++ b/src/StellaOps.Authority/StellaOps.Auth.ServerIntegration/README.NuGet.md @@ -1,9 +1,9 @@ -# StellaOps.Auth.ServerIntegration - -ASP.NET Core helpers that enable resource servers to authenticate with **StellaOps Authority**: - -- `AddStellaOpsResourceServerAuthentication` extension for JWT bearer + scope policies. -- Network bypass mask evaluation for on-host automation. -- Consistent `ProblemDetails` responses and policy helpers shared with Feedser/Backend services. - -Pair this package with `StellaOps.Auth.Abstractions` and `StellaOps.Auth.Client` for end-to-end Authority integration. +# StellaOps.Auth.ServerIntegration + +ASP.NET Core helpers that enable resource servers to authenticate with **StellaOps Authority**: + +- `AddStellaOpsResourceServerAuthentication` extension for JWT bearer + scope policies. +- Network bypass mask evaluation for on-host automation. +- Consistent `ProblemDetails` responses and policy helpers shared with Concelier/Backend services. + +Pair this package with `StellaOps.Auth.Abstractions` and `StellaOps.Auth.Client` for end-to-end Authority integration. diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs index a0fb6f8d..a7017fef 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardClientProvisioningStoreTests.cs @@ -1,6 +1,9 @@ +using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using MongoDB.Driver; using StellaOps.Authority.Plugins.Abstractions; using StellaOps.Authority.Plugin.Standard.Storage; using StellaOps.Authority.Storage.Mongo.Documents; @@ -42,23 +45,91 @@ public class StandardClientProvisioningStoreTests Assert.Contains("scopeA", descriptor.AllowedScopes); } + [Fact] + public async Task CreateOrUpdateAsync_StoresAudiences() + { + var store = new TrackingClientStore(); + var revocations = new TrackingRevocationStore(); + var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System); + + var registration = new AuthorityClientRegistration( + clientId: "signer", + confidential: false, + displayName: "Signer", + clientSecret: null, + allowedGrantTypes: new[] { "client_credentials" }, + allowedScopes: new[] { "signer.sign" }, + allowedAudiences: new[] { "attestor", "signer" }); + + var result = await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); + + Assert.True(result.Succeeded); + var document = Assert.Contains("signer", store.Documents); + Assert.Equal("attestor signer", document.Value.Properties[AuthorityClientMetadataKeys.Audiences]); + + var descriptor = await provisioning.FindByClientIdAsync("signer", CancellationToken.None); + Assert.NotNull(descriptor); + Assert.Equal(new[] { "attestor", "signer" }, descriptor!.Audiences.OrderBy(value => value, StringComparer.Ordinal)); + } + + [Fact] + public async Task CreateOrUpdateAsync_MapsCertificateBindings() + { + var store = new TrackingClientStore(); + var revocations = new TrackingRevocationStore(); + var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System); + + var bindingRegistration = new AuthorityClientCertificateBindingRegistration( + thumbprint: "aa:bb:cc:dd", + serialNumber: "01ff", + subject: "CN=mtls-client", + issuer: "CN=test-ca", + subjectAlternativeNames: new[] { "client.mtls.test", "spiffe://client" }, + notBefore: DateTimeOffset.UtcNow.AddMinutes(-5), + notAfter: DateTimeOffset.UtcNow.AddHours(1), + label: "primary"); + + var registration = new AuthorityClientRegistration( + clientId: "mtls-client", + confidential: true, + displayName: "MTLS Client", + clientSecret: "secret", + allowedGrantTypes: new[] { "client_credentials" }, + allowedScopes: new[] { "signer.sign" }, + allowedAudiences: new[] { "signer" }, + certificateBindings: new[] { bindingRegistration }); + + await provisioning.CreateOrUpdateAsync(registration, CancellationToken.None); + + var document = Assert.Contains("mtls-client", store.Documents).Value; + var binding = Assert.Single(document.CertificateBindings); + Assert.Equal("AABBCCDD", binding.Thumbprint); + Assert.Equal("01ff", binding.SerialNumber); + Assert.Equal("CN=mtls-client", binding.Subject); + Assert.Equal("CN=test-ca", binding.Issuer); + Assert.Equal(new[] { "client.mtls.test", "spiffe://client" }, binding.SubjectAlternativeNames); + Assert.Equal(bindingRegistration.NotBefore, binding.NotBefore); + Assert.Equal(bindingRegistration.NotAfter, binding.NotAfter); + Assert.Equal("primary", binding.Label); + } + private sealed class TrackingClientStore : IAuthorityClientStore { public Dictionary Documents { get; } = new(StringComparer.OrdinalIgnoreCase); - public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { Documents.TryGetValue(clientId, out var document); return ValueTask.FromResult(document); } - public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { Documents[document.ClientId] = document; return ValueTask.CompletedTask; } - public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { var removed = Documents.Remove(clientId); return ValueTask.FromResult(removed); @@ -69,16 +140,16 @@ public class StandardClientProvisioningStoreTests { public List Upserts { get; } = new(); - public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken) + public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { Upserts.Add(document); return ValueTask.CompletedTask; } - public ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken) + public ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(true); - public ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken) + public ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult>(Array.Empty()); } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs index 5c95104c..edaad0a9 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StandardPluginRegistrarTests.cs @@ -319,13 +319,13 @@ internal sealed class CapturingLoggerProvider : ILoggerProvider internal sealed class StubRevocationStore : IAuthorityRevocationStore { - public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken) + public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.CompletedTask; - public ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken) + public ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(false); - public ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken) + public ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult>(Array.Empty()); } @@ -333,18 +333,18 @@ internal sealed class InMemoryClientStore : IAuthorityClientStore { private readonly Dictionary clients = new(StringComparer.OrdinalIgnoreCase); - public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { clients.TryGetValue(clientId, out var document); return ValueTask.FromResult(document); } - public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { clients[document.ClientId] = document; return ValueTask.CompletedTask; } - public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(clients.Remove(clientId)); } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj index e80ccc23..c69b0b99 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard.Tests/StellaOps.Authority.Plugin.Standard.Tests.csproj @@ -1,12 +1,15 @@ - - - net10.0 - enable - enable - false - - - - - - + + + net10.0 + enable + enable + false + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj index e122e8ae..84f73910 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/StellaOps.Authority.Plugin.Standard.csproj @@ -1,24 +1,24 @@ - - - net10.0 - preview - enable - enable - true - true - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + true + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs index 5a8ff2e1..8b879c68 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/Storage/StandardClientProvisioningStore.cs @@ -1,150 +1,235 @@ -using System.Collections.Generic; -using System.Linq; -using StellaOps.Authority.Plugins.Abstractions; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Stores; - -namespace StellaOps.Authority.Plugin.Standard.Storage; - -internal sealed class StandardClientProvisioningStore : IClientProvisioningStore -{ - private readonly string pluginName; - private readonly IAuthorityClientStore clientStore; - private readonly IAuthorityRevocationStore revocationStore; - private readonly TimeProvider clock; - - public StandardClientProvisioningStore( - string pluginName, - IAuthorityClientStore clientStore, - IAuthorityRevocationStore revocationStore, - TimeProvider clock) - { - this.pluginName = pluginName ?? throw new ArgumentNullException(nameof(pluginName)); - this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); - this.revocationStore = revocationStore ?? throw new ArgumentNullException(nameof(revocationStore)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - } - - public async ValueTask> CreateOrUpdateAsync( - AuthorityClientRegistration registration, - CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(registration); - - if (registration.Confidential && string.IsNullOrWhiteSpace(registration.ClientSecret)) - { - return AuthorityPluginOperationResult.Failure("secret_required", "Confidential clients require a client secret."); - } - - var document = await clientStore.FindByClientIdAsync(registration.ClientId, cancellationToken).ConfigureAwait(false) - ?? new AuthorityClientDocument { ClientId = registration.ClientId, CreatedAt = clock.GetUtcNow() }; - - document.Plugin = pluginName; - document.ClientType = registration.Confidential ? "confidential" : "public"; - document.DisplayName = registration.DisplayName; - document.SecretHash = registration.Confidential && registration.ClientSecret is not null - ? AuthoritySecretHasher.ComputeHash(registration.ClientSecret) - : null; - document.UpdatedAt = clock.GetUtcNow(); - - document.RedirectUris = registration.RedirectUris.Select(static uri => uri.ToString()).ToList(); - document.PostLogoutRedirectUris = registration.PostLogoutRedirectUris.Select(static uri => uri.ToString()).ToList(); - - document.Properties[AuthorityClientMetadataKeys.AllowedGrantTypes] = string.Join(" ", registration.AllowedGrantTypes); - document.Properties[AuthorityClientMetadataKeys.AllowedScopes] = string.Join(" ", registration.AllowedScopes); - document.Properties[AuthorityClientMetadataKeys.RedirectUris] = string.Join(" ", document.RedirectUris); - document.Properties[AuthorityClientMetadataKeys.PostLogoutRedirectUris] = string.Join(" ", document.PostLogoutRedirectUris); - - foreach (var (key, value) in registration.Properties) - { - document.Properties[key] = value; - } - - await clientStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false); - await revocationStore.RemoveAsync("client", registration.ClientId, cancellationToken).ConfigureAwait(false); - - return AuthorityPluginOperationResult.Success(ToDescriptor(document)); - } - - public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) - { - var document = await clientStore.FindByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); - return document is null ? null : ToDescriptor(document); - } - - public async ValueTask DeleteAsync(string clientId, CancellationToken cancellationToken) - { - var deleted = await clientStore.DeleteByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); - if (!deleted) - { - return AuthorityPluginOperationResult.Failure("not_found", "Client was not found."); - } - - var now = clock.GetUtcNow(); - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["plugin"] = pluginName - }; - - var revocation = new AuthorityRevocationDocument - { - Category = "client", - RevocationId = clientId, - ClientId = clientId, - Reason = "operator_request", - ReasonDescription = $"Client '{clientId}' deleted via plugin '{pluginName}'.", - RevokedAt = now, - EffectiveAt = now, - Metadata = metadata - }; - - try - { - await revocationStore.UpsertAsync(revocation, cancellationToken).ConfigureAwait(false); - } - catch - { - // Revocation export should proceed even if the metadata write fails. - } - - return AuthorityPluginOperationResult.Success(); - } - - private static AuthorityClientDescriptor ToDescriptor(AuthorityClientDocument document) - { - var allowedGrantTypes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); - var allowedScopes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); - - var redirectUris = document.RedirectUris - .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) - .Where(static uri => uri is not null) - .Cast() - .ToArray(); - - var postLogoutUris = document.PostLogoutRedirectUris - .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) - .Where(static uri => uri is not null) - .Cast() - .ToArray(); - - return new AuthorityClientDescriptor( - document.ClientId, - document.DisplayName, - string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase), - allowedGrantTypes, - allowedScopes, - redirectUris, - postLogoutUris, - document.Properties); - } - - private static IReadOnlyCollection Split(IReadOnlyDictionary properties, string key) - { - if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) - { - return Array.Empty(); - } - - return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - } -} +using System.Collections.Generic; +using System.Linq; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.Plugin.Standard.Storage; + +internal sealed class StandardClientProvisioningStore : IClientProvisioningStore +{ + private readonly string pluginName; + private readonly IAuthorityClientStore clientStore; + private readonly IAuthorityRevocationStore revocationStore; + private readonly TimeProvider clock; + + public StandardClientProvisioningStore( + string pluginName, + IAuthorityClientStore clientStore, + IAuthorityRevocationStore revocationStore, + TimeProvider clock) + { + this.pluginName = pluginName ?? throw new ArgumentNullException(nameof(pluginName)); + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.revocationStore = revocationStore ?? throw new ArgumentNullException(nameof(revocationStore)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + } + + public async ValueTask> CreateOrUpdateAsync( + AuthorityClientRegistration registration, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(registration); + + if (registration.Confidential && string.IsNullOrWhiteSpace(registration.ClientSecret)) + { + return AuthorityPluginOperationResult.Failure("secret_required", "Confidential clients require a client secret."); + } + + var document = await clientStore.FindByClientIdAsync(registration.ClientId, cancellationToken).ConfigureAwait(false) + ?? new AuthorityClientDocument { ClientId = registration.ClientId, CreatedAt = clock.GetUtcNow() }; + + document.Plugin = pluginName; + document.ClientType = registration.Confidential ? "confidential" : "public"; + document.DisplayName = registration.DisplayName; + document.SecretHash = registration.Confidential && registration.ClientSecret is not null + ? AuthoritySecretHasher.ComputeHash(registration.ClientSecret) + : null; + document.UpdatedAt = clock.GetUtcNow(); + + document.RedirectUris = registration.RedirectUris.Select(static uri => uri.ToString()).ToList(); + document.PostLogoutRedirectUris = registration.PostLogoutRedirectUris.Select(static uri => uri.ToString()).ToList(); + + document.Properties[AuthorityClientMetadataKeys.AllowedGrantTypes] = JoinValues(registration.AllowedGrantTypes); + document.Properties[AuthorityClientMetadataKeys.AllowedScopes] = JoinValues(registration.AllowedScopes); + document.Properties[AuthorityClientMetadataKeys.Audiences] = JoinValues(registration.AllowedAudiences); + document.Properties[AuthorityClientMetadataKeys.RedirectUris] = string.Join(" ", document.RedirectUris); + document.Properties[AuthorityClientMetadataKeys.PostLogoutRedirectUris] = string.Join(" ", document.PostLogoutRedirectUris); + + if (registration.CertificateBindings is not null) + { + var now = clock.GetUtcNow(); + document.CertificateBindings = registration.CertificateBindings + .Select(binding => MapCertificateBinding(binding, now)) + .OrderBy(binding => binding.Thumbprint, StringComparer.Ordinal) + .ToList(); + } + + foreach (var (key, value) in registration.Properties) + { + document.Properties[key] = value; + } + + if (registration.Properties.TryGetValue(AuthorityClientMetadataKeys.SenderConstraint, out var senderConstraintRaw)) + { + var normalizedConstraint = NormalizeSenderConstraint(senderConstraintRaw); + if (normalizedConstraint is not null) + { + document.SenderConstraint = normalizedConstraint; + document.Properties[AuthorityClientMetadataKeys.SenderConstraint] = normalizedConstraint; + } + else + { + document.SenderConstraint = null; + document.Properties.Remove(AuthorityClientMetadataKeys.SenderConstraint); + } + } + + await clientStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + await revocationStore.RemoveAsync("client", registration.ClientId, cancellationToken).ConfigureAwait(false); + + return AuthorityPluginOperationResult.Success(ToDescriptor(document)); + } + + public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + { + var document = await clientStore.FindByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); + return document is null ? null : ToDescriptor(document); + } + + public async ValueTask DeleteAsync(string clientId, CancellationToken cancellationToken) + { + var deleted = await clientStore.DeleteByClientIdAsync(clientId, cancellationToken).ConfigureAwait(false); + if (!deleted) + { + return AuthorityPluginOperationResult.Failure("not_found", "Client was not found."); + } + + var now = clock.GetUtcNow(); + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["plugin"] = pluginName + }; + + var revocation = new AuthorityRevocationDocument + { + Category = "client", + RevocationId = clientId, + ClientId = clientId, + Reason = "operator_request", + ReasonDescription = $"Client '{clientId}' deleted via plugin '{pluginName}'.", + RevokedAt = now, + EffectiveAt = now, + Metadata = metadata + }; + + try + { + await revocationStore.UpsertAsync(revocation, cancellationToken).ConfigureAwait(false); + } + catch + { + // Revocation export should proceed even if the metadata write fails. + } + + return AuthorityPluginOperationResult.Success(); + } + + private static AuthorityClientDescriptor ToDescriptor(AuthorityClientDocument document) + { + var allowedGrantTypes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); + var allowedScopes = Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + + var redirectUris = document.RedirectUris + .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null) + .Cast() + .ToArray(); + + var postLogoutUris = document.PostLogoutRedirectUris + .Select(static value => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null) + .Cast() + .ToArray(); + + var audiences = Split(document.Properties, AuthorityClientMetadataKeys.Audiences); + + return new AuthorityClientDescriptor( + document.ClientId, + document.DisplayName, + string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase), + allowedGrantTypes, + allowedScopes, + audiences, + redirectUris, + postLogoutUris, + document.Properties); + } + + private static IReadOnlyCollection Split(IReadOnlyDictionary properties, string key) + { + if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + private static string JoinValues(IReadOnlyCollection values) + { + if (values is null || values.Count == 0) + { + return string.Empty; + } + + return string.Join( + " ", + values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .OrderBy(static value => value, StringComparer.Ordinal)); + } + + private static AuthorityClientCertificateBinding MapCertificateBinding( + AuthorityClientCertificateBindingRegistration registration, + DateTimeOffset now) + { + var subjectAlternativeNames = registration.SubjectAlternativeNames.Count == 0 + ? new List() + : registration.SubjectAlternativeNames + .Select(name => name.Trim()) + .OrderBy(name => name, StringComparer.OrdinalIgnoreCase) + .ToList(); + + return new AuthorityClientCertificateBinding + { + Thumbprint = registration.Thumbprint, + SerialNumber = registration.SerialNumber, + Subject = registration.Subject, + Issuer = registration.Issuer, + SubjectAlternativeNames = subjectAlternativeNames, + NotBefore = registration.NotBefore, + NotAfter = registration.NotAfter, + Label = registration.Label, + CreatedAt = now, + UpdatedAt = now + }; + } + + private static string? NormalizeSenderConstraint(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim() switch + { + { Length: 0 } => null, + var constraint when string.Equals(constraint, "dpop", StringComparison.OrdinalIgnoreCase) => "dpop", + var constraint when string.Equals(constraint, "mtls", StringComparison.OrdinalIgnoreCase) => "mtls", + _ => null + }; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md index 6f7190fa..a4caeba9 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugin.Standard/TASKS.md @@ -5,10 +5,10 @@ | PLG6.DOC | DONE (2025-10-11) | BE-Auth Plugin, Docs Guild | PLG1–PLG5 | Final polish + diagrams for plugin developer guide (AUTHPLUG-DOCS-01-001). | Docs team delivers copy-edit + exported diagrams; PR merged. | | SEC1.PLG | DONE (2025-10-11) | Security Guild, BE-Auth Plugin | SEC1.A (StellaOps.Cryptography) | Swap Standard plugin hashing to Argon2id via `StellaOps.Cryptography` abstractions; keep PBKDF2 verification for legacy. | ✅ `StandardUserCredentialStore` uses `ICryptoProvider` to hash/check; ✅ Transparent rehash on success; ✅ Unit tests cover tamper + legacy rehash. | | SEC1.OPT | DONE (2025-10-11) | Security Guild | SEC1.PLG | Expose password hashing knobs in `StandardPluginOptions` (`memoryKiB`, `iterations`, `parallelism`, `algorithm`) with validation. | ✅ Options bound from YAML; ✅ Invalid configs throw; ✅ Docs include tuning guidance. | -| SEC2.PLG | DOING (2025-10-14) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. | -| SEC3.PLG | DOING (2025-10-14) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. | +| SEC2.PLG | DOING (2025-10-14) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`.
⏳ Awaiting AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 completion to unlock Wave 0B verification paths. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. | +| SEC3.PLG | DOING (2025-10-14) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after).
⏳ Pending AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 so limiter telemetry contract matches final authority surface. | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. | | SEC4.PLG | DONE (2025-10-12) | Security Guild | SEC4.A (revocation schema) | Provide plugin hooks so revoked users/clients write reasons for revocation bundle export. | ✅ Revocation exporter consumes plugin data; ✅ Tests cover revoked user/client output. | -| SEC5.PLG | DOING (2025-10-14) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. | +| SEC5.PLG | DOING (2025-10-14) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog.
⏳ Final documentation depends on AUTH-DPOP-11-001 / AUTH-MTLS-11-002 / PLUGIN-DI-08-001 outcomes. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. | | PLG4-6.CAPABILITIES | BLOCKED (2025-10-12) | BE-Auth Plugin, Docs Guild | PLG1–PLG3 | Finalise capability metadata exposure, config validation, and developer guide updates; remaining action is Docs polish/diagram export. | ✅ Capability metadata + validation merged; ✅ Plugin guide updated with final copy & diagrams; ✅ Release notes mention new toggles.
⛔ Blocked awaiting Authority rate-limiter stream (CORE8/SEC3) to resume so doc updates reflect final limiter behaviour. | | PLG7.RFC | REVIEW | BE-Auth Plugin, Security Guild | PLG4 | Socialize LDAP plugin RFC (`docs/rfcs/authority-plugin-ldap.md`) and capture guild feedback. | ✅ Guild review sign-off recorded; ✅ Follow-up issues filed in module boards. | | PLG6.DIAGRAM | TODO | Docs Guild | PLG6.DOC | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | ✅ Mermaid sources committed; ✅ Rendered SVG/PNG linked from Section 2 + Section 9; ✅ Docs build preview shared with Plugin + Docs guilds. | @@ -16,3 +16,5 @@ > Update statuses to DOING/DONE/BLOCKED as you make progress. Always run `dotnet test` for touched projects before marking DONE. > Remark (2025-10-13, PLG6.DOC/PLG6.DIAGRAM): Security Guild delivered `docs/security/rate-limits.md`; Docs team can lift Section 3 (tuning table + alerts) into the developer guide diagrams when rendering assets. + +> Check-in (2025-10-19): Wave 0A dependencies (AUTH-DPOP-11-001, AUTH-MTLS-11-002, PLUGIN-DI-08-001) still open, so SEC2/SEC3/SEC5 remain in progress without new scope until upstream limiter updates land. diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs index 96e4d631..edbdd5fa 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/AuthorityClientMetadataKeys.cs @@ -1,12 +1,14 @@ -namespace StellaOps.Authority.Plugins.Abstractions; - -/// -/// Well-known metadata keys persisted with Authority client registrations. -/// -public static class AuthorityClientMetadataKeys -{ - public const string AllowedGrantTypes = "allowedGrantTypes"; - public const string AllowedScopes = "allowedScopes"; - public const string RedirectUris = "redirectUris"; - public const string PostLogoutRedirectUris = "postLogoutRedirectUris"; -} +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Well-known metadata keys persisted with Authority client registrations. +/// +public static class AuthorityClientMetadataKeys +{ + public const string AllowedGrantTypes = "allowedGrantTypes"; + public const string AllowedScopes = "allowedScopes"; + public const string Audiences = "audiences"; + public const string RedirectUris = "redirectUris"; + public const string PostLogoutRedirectUris = "postLogoutRedirectUris"; + public const string SenderConstraint = "senderConstraint"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs index ae35bdc2..e2037fe6 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Plugins.Abstractions/IdentityProviderContracts.cs @@ -1,795 +1,800 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Cryptography.Audit; - -namespace StellaOps.Authority.Plugins.Abstractions; - -/// -/// Describes feature support advertised by an identity provider plugin. -/// -public sealed record AuthorityIdentityProviderCapabilities( - bool SupportsPassword, - bool SupportsMfa, - bool SupportsClientProvisioning) -{ - /// - /// Builds capabilities metadata from a list of capability identifiers. - /// - public static AuthorityIdentityProviderCapabilities FromCapabilities(IEnumerable capabilities) - { - if (capabilities is null) - { - return new AuthorityIdentityProviderCapabilities(false, false, false); - } - - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var entry in capabilities) - { - if (string.IsNullOrWhiteSpace(entry)) - { - continue; - } - - seen.Add(entry.Trim()); - } - - return new AuthorityIdentityProviderCapabilities( - SupportsPassword: seen.Contains(AuthorityPluginCapabilities.Password), - SupportsMfa: seen.Contains(AuthorityPluginCapabilities.Mfa), - SupportsClientProvisioning: seen.Contains(AuthorityPluginCapabilities.ClientProvisioning)); - } -} - -/// -/// Represents a loaded Authority identity provider plugin instance. -/// -public interface IIdentityProviderPlugin -{ - /// - /// Gets the logical name of the plugin instance (matches the manifest key). - /// - string Name { get; } - - /// - /// Gets the plugin type identifier (e.g. standard, ldap). - /// - string Type { get; } - - /// - /// Gets the plugin context comprising the manifest and bound configuration. - /// - AuthorityPluginContext Context { get; } - - /// - /// Gets the credential store responsible for authenticator validation and user provisioning. - /// - IUserCredentialStore Credentials { get; } - - /// - /// Gets the claims enricher applied to issued principals. - /// - IClaimsEnricher ClaimsEnricher { get; } - - /// - /// Gets the optional client provisioning store exposed by the plugin. - /// - IClientProvisioningStore? ClientProvisioning { get; } - - /// - /// Gets the capability metadata advertised by the plugin. - /// - AuthorityIdentityProviderCapabilities Capabilities { get; } - - /// - /// Evaluates the health of the plugin and backing data stores. - /// - /// Token used to cancel the operation. - /// Health result describing the plugin status. - ValueTask CheckHealthAsync(CancellationToken cancellationToken); -} - -/// -/// Supplies operations for validating credentials and managing user records. -/// -public interface IUserCredentialStore -{ - /// - /// Verifies the supplied username/password combination. - /// - ValueTask VerifyPasswordAsync( - string username, - string password, - CancellationToken cancellationToken); - - /// - /// Creates or updates a user record based on the supplied registration data. - /// - ValueTask> UpsertUserAsync( - AuthorityUserRegistration registration, - CancellationToken cancellationToken); - - /// - /// Attempts to resolve a user descriptor by its canonical subject identifier. - /// - ValueTask FindBySubjectAsync( - string subjectId, - CancellationToken cancellationToken); -} - -/// -/// Enriches issued principals with additional claims based on plugin-specific rules. -/// -public interface IClaimsEnricher -{ - /// - /// Adds or adjusts claims on the provided identity. - /// - ValueTask EnrichAsync( - ClaimsIdentity identity, - AuthorityClaimsEnrichmentContext context, - CancellationToken cancellationToken); -} - -/// -/// Manages client (machine-to-machine) provisioning for Authority. -/// -public interface IClientProvisioningStore -{ - /// - /// Creates or updates a client registration. - /// - ValueTask> CreateOrUpdateAsync( - AuthorityClientRegistration registration, - CancellationToken cancellationToken); - - /// - /// Attempts to resolve a client descriptor by its identifier. - /// - ValueTask FindByClientIdAsync( - string clientId, - CancellationToken cancellationToken); - - /// - /// Removes a client registration. - /// - ValueTask DeleteAsync( - string clientId, - CancellationToken cancellationToken); -} - -/// -/// Represents the health state of a plugin or backing store. -/// -public enum AuthorityPluginHealthStatus -{ - /// - /// Plugin is healthy and operational. - /// - Healthy, - - /// - /// Plugin is degraded but still usable (e.g. transient connectivity issues). - /// - Degraded, - - /// - /// Plugin is unavailable and cannot service requests. - /// - Unavailable -} - -/// -/// Result of a plugin health probe. -/// -public sealed record AuthorityPluginHealthResult -{ - private AuthorityPluginHealthResult( - AuthorityPluginHealthStatus status, - string? message, - IReadOnlyDictionary details) - { - Status = status; - Message = message; - Details = details; - } - - /// - /// Gets the overall status of the plugin. - /// - public AuthorityPluginHealthStatus Status { get; } - - /// - /// Gets an optional human-readable status description. - /// - public string? Message { get; } - - /// - /// Gets optional structured details for diagnostics. - /// - public IReadOnlyDictionary Details { get; } - - /// - /// Creates a healthy result. - /// - public static AuthorityPluginHealthResult Healthy( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Healthy, message, details ?? EmptyDetails); - - /// - /// Creates a degraded result. - /// - public static AuthorityPluginHealthResult Degraded( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Degraded, message, details ?? EmptyDetails); - - /// - /// Creates an unavailable result. - /// - public static AuthorityPluginHealthResult Unavailable( - string? message = null, - IReadOnlyDictionary? details = null) - => new(AuthorityPluginHealthStatus.Unavailable, message, details ?? EmptyDetails); - - private static readonly IReadOnlyDictionary EmptyDetails = - new Dictionary(StringComparer.OrdinalIgnoreCase); -} - -/// -/// Describes a canonical Authority user surfaced by a plugin. -/// -public sealed record AuthorityUserDescriptor -{ - /// - /// Initialises a new user descriptor. - /// - public AuthorityUserDescriptor( - string subjectId, - string username, - string? displayName, - bool requiresPasswordReset, - IReadOnlyCollection? roles = null, - IReadOnlyDictionary? attributes = null) - { - SubjectId = ValidateRequired(subjectId, nameof(subjectId)); - Username = ValidateRequired(username, nameof(username)); - DisplayName = displayName; - RequiresPasswordReset = requiresPasswordReset; - Roles = roles is null ? Array.Empty() : roles.ToArray(); - Attributes = attributes is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Stable subject identifier for token issuance. - /// - public string SubjectId { get; } - - /// - /// Canonical username (case-normalised). - /// - public string Username { get; } - - /// - /// Optional human-friendly display name. - /// - public string? DisplayName { get; } - - /// - /// Indicates whether the user must reset their password. - /// - public bool RequiresPasswordReset { get; } - - /// - /// Collection of role identifiers associated with the user. - /// - public IReadOnlyCollection Roles { get; } - - /// - /// Arbitrary plugin-defined attributes (used by claims enricher). - /// - public IReadOnlyDictionary Attributes { get; } - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -/// -/// Outcome of a credential verification attempt. -/// -public sealed record AuthorityCredentialVerificationResult -{ - private AuthorityCredentialVerificationResult( - bool succeeded, - AuthorityUserDescriptor? user, - AuthorityCredentialFailureCode? failureCode, - string? message, - TimeSpan? retryAfter, - IReadOnlyList auditProperties) - { - Succeeded = succeeded; - User = user; - FailureCode = failureCode; - Message = message; - RetryAfter = retryAfter; - AuditProperties = auditProperties ?? Array.Empty(); - } - - /// - /// Indicates whether the verification succeeded. - /// - public bool Succeeded { get; } - - /// - /// Resolved user descriptor when successful. - /// - public AuthorityUserDescriptor? User { get; } - - /// - /// Failure classification when unsuccessful. - /// - public AuthorityCredentialFailureCode? FailureCode { get; } - - /// - /// Optional message describing the outcome. - /// - public string? Message { get; } - - /// - /// Optional suggested retry interval (e.g. for lockouts). - /// - public TimeSpan? RetryAfter { get; } - - /// - /// Additional audit properties emitted by the credential store. - /// - public IReadOnlyList AuditProperties { get; } - - /// - /// Builds a successful verification result. - /// - public static AuthorityCredentialVerificationResult Success( - AuthorityUserDescriptor user, - string? message = null, - IReadOnlyList? auditProperties = null) - => new(true, user ?? throw new ArgumentNullException(nameof(user)), null, message, null, auditProperties ?? Array.Empty()); - - /// - /// Builds a failed verification result. - /// - public static AuthorityCredentialVerificationResult Failure( - AuthorityCredentialFailureCode failureCode, - string? message = null, - TimeSpan? retryAfter = null, - IReadOnlyList? auditProperties = null) - => new(false, null, failureCode, message, retryAfter, auditProperties ?? Array.Empty()); -} - -/// -/// Classifies credential verification failures. -/// -public enum AuthorityCredentialFailureCode -{ - /// - /// Username/password combination is invalid. - /// - InvalidCredentials, - - /// - /// Account is locked out (retry after a specified duration). - /// - LockedOut, - - /// - /// Password has expired and must be reset. - /// - PasswordExpired, - - /// - /// User must reset password before proceeding. - /// - RequiresPasswordReset, - - /// - /// Additional multi-factor authentication is required. - /// - RequiresMfa, - - /// - /// Unexpected failure occurred (see message for details). - /// - UnknownError -} - -/// -/// Represents a user provisioning request. -/// -public sealed record AuthorityUserRegistration -{ - /// - /// Initialises a new registration. - /// - public AuthorityUserRegistration( - string username, - string? password, - string? displayName, - string? email, - bool requirePasswordReset, - IReadOnlyCollection? roles = null, - IReadOnlyDictionary? attributes = null) - { - Username = ValidateRequired(username, nameof(username)); - Password = password; - DisplayName = displayName; - Email = email; - RequirePasswordReset = requirePasswordReset; - Roles = roles is null ? Array.Empty() : roles.ToArray(); - Attributes = attributes is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Canonical username (unique). - /// - public string Username { get; } - - /// - /// Optional raw password (hashed by plugin). - /// - public string? Password { get; init; } - - /// - /// Optional human-friendly display name. - /// - public string? DisplayName { get; } - - /// - /// Optional contact email. - /// - public string? Email { get; } - - /// - /// Indicates whether the user must reset their password at next login. - /// - public bool RequirePasswordReset { get; } - - /// - /// Associated roles. - /// - public IReadOnlyCollection Roles { get; } - - /// - /// Plugin-defined attributes. - /// - public IReadOnlyDictionary Attributes { get; } - - /// - /// Creates a copy with the provided password while preserving other fields. - /// - public AuthorityUserRegistration WithPassword(string? password) - => new(Username, password, DisplayName, Email, RequirePasswordReset, Roles, Attributes); - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -/// -/// Generic operation result utilised by plugins. -/// -public sealed record AuthorityPluginOperationResult -{ - private AuthorityPluginOperationResult(bool succeeded, string? errorCode, string? message) - { - Succeeded = succeeded; - ErrorCode = errorCode; - Message = message; - } - - /// - /// Indicates whether the operation succeeded. - /// - public bool Succeeded { get; } - - /// - /// Machine-readable error code (populated on failure). - /// - public string? ErrorCode { get; } - - /// - /// Optional human-readable message. - /// - public string? Message { get; } - - /// - /// Returns a successful result. - /// - public static AuthorityPluginOperationResult Success(string? message = null) - => new(true, null, message); - - /// - /// Returns a failed result with the supplied error code. - /// - public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) - => new(false, ValidateErrorCode(errorCode), message); - - internal static string ValidateErrorCode(string errorCode) - => string.IsNullOrWhiteSpace(errorCode) - ? throw new ArgumentException("Error code is required for failures.", nameof(errorCode)) - : errorCode; -} - -/// -/// Generic operation result that returns a value. -/// -public sealed record AuthorityPluginOperationResult -{ - private AuthorityPluginOperationResult( - bool succeeded, - TValue? value, - string? errorCode, - string? message) - { - Succeeded = succeeded; - Value = value; - ErrorCode = errorCode; - Message = message; - } - - /// - /// Indicates whether the operation succeeded. - /// - public bool Succeeded { get; } - - /// - /// Returned value when successful. - /// - public TValue? Value { get; } - - /// - /// Machine-readable error code (on failure). - /// - public string? ErrorCode { get; } - - /// - /// Optional human-readable message. - /// - public string? Message { get; } - - /// - /// Returns a successful result with the provided value. - /// - public static AuthorityPluginOperationResult Success(TValue value, string? message = null) - => new(true, value, null, message); - - /// - /// Returns a successful result without a value (defaults to default). - /// - public static AuthorityPluginOperationResult Success(string? message = null) - => new(true, default, null, message); - - /// - /// Returns a failed result with the supplied error code. - /// - public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) - => new(false, default, AuthorityPluginOperationResult.ValidateErrorCode(errorCode), message); -} - -/// -/// Context supplied to claims enrichment routines. -/// -public sealed class AuthorityClaimsEnrichmentContext -{ - private readonly Dictionary items; - - /// - /// Initialises a new context instance. - /// - public AuthorityClaimsEnrichmentContext( - AuthorityPluginContext plugin, - AuthorityUserDescriptor? user, - AuthorityClientDescriptor? client) - { - Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); - User = user; - Client = client; - items = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - - /// - /// Gets the plugin context associated with the principal. - /// - public AuthorityPluginContext Plugin { get; } - - /// - /// Gets the user descriptor when available. - /// - public AuthorityUserDescriptor? User { get; } - - /// - /// Gets the client descriptor when available. - /// - public AuthorityClientDescriptor? Client { get; } - - /// - /// Extensible bag for plugin-specific data passed between enrichment stages. - /// - public IDictionary Items => items; -} - -/// -/// Represents a registered OAuth/OpenID client. -/// -public sealed record AuthorityClientDescriptor -{ - /// - /// Initialises a new client descriptor. - /// - public AuthorityClientDescriptor( - string clientId, - string? displayName, - bool confidential, - IReadOnlyCollection? allowedGrantTypes = null, - IReadOnlyCollection? allowedScopes = null, - IReadOnlyCollection? redirectUris = null, - IReadOnlyCollection? postLogoutRedirectUris = null, - IReadOnlyDictionary? properties = null) - { - ClientId = ValidateRequired(clientId, nameof(clientId)); - DisplayName = displayName; - Confidential = confidential; - AllowedGrantTypes = allowedGrantTypes is null ? Array.Empty() : allowedGrantTypes.ToArray(); - AllowedScopes = allowedScopes is null ? Array.Empty() : allowedScopes.ToArray(); - RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); - PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); - Properties = properties is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Unique client identifier. - /// - public string ClientId { get; } - - /// - /// Optional display name. - /// - public string? DisplayName { get; } - - /// - /// Indicates whether the client is confidential (requires secret). - /// - public bool Confidential { get; } - - /// - /// Permitted OAuth grant types. - /// - public IReadOnlyCollection AllowedGrantTypes { get; } - - /// - /// Permitted scopes. - /// - public IReadOnlyCollection AllowedScopes { get; } - - /// - /// Registered redirect URIs. - /// - public IReadOnlyCollection RedirectUris { get; } - - /// - /// Registered post-logout redirect URIs. - /// - public IReadOnlyCollection PostLogoutRedirectUris { get; } - - /// - /// Additional plugin-defined metadata. - /// - public IReadOnlyDictionary Properties { get; } - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} - -/// -/// Client registration payload used when provisioning clients through plugins. -/// -public sealed record AuthorityClientRegistration -{ - /// - /// Initialises a new registration. - /// - public AuthorityClientRegistration( - string clientId, - bool confidential, - string? displayName, - string? clientSecret, - IReadOnlyCollection? allowedGrantTypes = null, - IReadOnlyCollection? allowedScopes = null, - IReadOnlyCollection? redirectUris = null, - IReadOnlyCollection? postLogoutRedirectUris = null, - IReadOnlyDictionary? properties = null) - { - ClientId = ValidateRequired(clientId, nameof(clientId)); - Confidential = confidential; - DisplayName = displayName; - ClientSecret = confidential - ? ValidateRequired(clientSecret ?? string.Empty, nameof(clientSecret)) - : clientSecret; - AllowedGrantTypes = allowedGrantTypes is null ? Array.Empty() : allowedGrantTypes.ToArray(); - AllowedScopes = allowedScopes is null ? Array.Empty() : allowedScopes.ToArray(); - RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); - PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); - Properties = properties is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) - : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); - } - - /// - /// Unique client identifier. - /// - public string ClientId { get; } - - /// - /// Indicates whether the client is confidential (requires secret handling). - /// - public bool Confidential { get; } - - /// - /// Optional display name. - /// - public string? DisplayName { get; } - - /// - /// Optional raw client secret (hashed by the plugin for storage). - /// - public string? ClientSecret { get; init; } - - /// - /// Grant types to enable. - /// - public IReadOnlyCollection AllowedGrantTypes { get; } - - /// - /// Scopes assigned to the client. - /// - public IReadOnlyCollection AllowedScopes { get; } - - /// - /// Redirect URIs permitted for the client. - /// - public IReadOnlyCollection RedirectUris { get; } - - /// - /// Post-logout redirect URIs. - /// - public IReadOnlyCollection PostLogoutRedirectUris { get; } - - /// - /// Additional metadata for the plugin. - /// - public IReadOnlyDictionary Properties { get; } - - /// - /// Creates a copy of the registration with the provided client secret. - /// - public AuthorityClientRegistration WithClientSecret(string? clientSecret) - => new(ClientId, Confidential, DisplayName, clientSecret, AllowedGrantTypes, AllowedScopes, RedirectUris, PostLogoutRedirectUris, Properties); - - private static string ValidateRequired(string value, string paramName) - => string.IsNullOrWhiteSpace(value) - ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) - : value; -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cryptography.Audit; + +namespace StellaOps.Authority.Plugins.Abstractions; + +/// +/// Describes feature support advertised by an identity provider plugin. +/// +public sealed record AuthorityIdentityProviderCapabilities( + bool SupportsPassword, + bool SupportsMfa, + bool SupportsClientProvisioning) +{ + /// + /// Builds capabilities metadata from a list of capability identifiers. + /// + public static AuthorityIdentityProviderCapabilities FromCapabilities(IEnumerable capabilities) + { + if (capabilities is null) + { + return new AuthorityIdentityProviderCapabilities(false, false, false); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in capabilities) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + seen.Add(entry.Trim()); + } + + return new AuthorityIdentityProviderCapabilities( + SupportsPassword: seen.Contains(AuthorityPluginCapabilities.Password), + SupportsMfa: seen.Contains(AuthorityPluginCapabilities.Mfa), + SupportsClientProvisioning: seen.Contains(AuthorityPluginCapabilities.ClientProvisioning)); + } +} + +/// +/// Represents a loaded Authority identity provider plugin instance. +/// +public interface IIdentityProviderPlugin +{ + /// + /// Gets the logical name of the plugin instance (matches the manifest key). + /// + string Name { get; } + + /// + /// Gets the plugin type identifier (e.g. standard, ldap). + /// + string Type { get; } + + /// + /// Gets the plugin context comprising the manifest and bound configuration. + /// + AuthorityPluginContext Context { get; } + + /// + /// Gets the credential store responsible for authenticator validation and user provisioning. + /// + IUserCredentialStore Credentials { get; } + + /// + /// Gets the claims enricher applied to issued principals. + /// + IClaimsEnricher ClaimsEnricher { get; } + + /// + /// Gets the optional client provisioning store exposed by the plugin. + /// + IClientProvisioningStore? ClientProvisioning { get; } + + /// + /// Gets the capability metadata advertised by the plugin. + /// + AuthorityIdentityProviderCapabilities Capabilities { get; } + + /// + /// Evaluates the health of the plugin and backing data stores. + /// + /// Token used to cancel the operation. + /// Health result describing the plugin status. + ValueTask CheckHealthAsync(CancellationToken cancellationToken); +} + +/// +/// Supplies operations for validating credentials and managing user records. +/// +public interface IUserCredentialStore +{ + /// + /// Verifies the supplied username/password combination. + /// + ValueTask VerifyPasswordAsync( + string username, + string password, + CancellationToken cancellationToken); + + /// + /// Creates or updates a user record based on the supplied registration data. + /// + ValueTask> UpsertUserAsync( + AuthorityUserRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a user descriptor by its canonical subject identifier. + /// + ValueTask FindBySubjectAsync( + string subjectId, + CancellationToken cancellationToken); +} + +/// +/// Enriches issued principals with additional claims based on plugin-specific rules. +/// +public interface IClaimsEnricher +{ + /// + /// Adds or adjusts claims on the provided identity. + /// + ValueTask EnrichAsync( + ClaimsIdentity identity, + AuthorityClaimsEnrichmentContext context, + CancellationToken cancellationToken); +} + +/// +/// Manages client (machine-to-machine) provisioning for Authority. +/// +public interface IClientProvisioningStore +{ + /// + /// Creates or updates a client registration. + /// + ValueTask> CreateOrUpdateAsync( + AuthorityClientRegistration registration, + CancellationToken cancellationToken); + + /// + /// Attempts to resolve a client descriptor by its identifier. + /// + ValueTask FindByClientIdAsync( + string clientId, + CancellationToken cancellationToken); + + /// + /// Removes a client registration. + /// + ValueTask DeleteAsync( + string clientId, + CancellationToken cancellationToken); +} + +/// +/// Represents the health state of a plugin or backing store. +/// +public enum AuthorityPluginHealthStatus +{ + /// + /// Plugin is healthy and operational. + /// + Healthy, + + /// + /// Plugin is degraded but still usable (e.g. transient connectivity issues). + /// + Degraded, + + /// + /// Plugin is unavailable and cannot service requests. + /// + Unavailable +} + +/// +/// Result of a plugin health probe. +/// +public sealed record AuthorityPluginHealthResult +{ + private AuthorityPluginHealthResult( + AuthorityPluginHealthStatus status, + string? message, + IReadOnlyDictionary details) + { + Status = status; + Message = message; + Details = details; + } + + /// + /// Gets the overall status of the plugin. + /// + public AuthorityPluginHealthStatus Status { get; } + + /// + /// Gets an optional human-readable status description. + /// + public string? Message { get; } + + /// + /// Gets optional structured details for diagnostics. + /// + public IReadOnlyDictionary Details { get; } + + /// + /// Creates a healthy result. + /// + public static AuthorityPluginHealthResult Healthy( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Healthy, message, details ?? EmptyDetails); + + /// + /// Creates a degraded result. + /// + public static AuthorityPluginHealthResult Degraded( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Degraded, message, details ?? EmptyDetails); + + /// + /// Creates an unavailable result. + /// + public static AuthorityPluginHealthResult Unavailable( + string? message = null, + IReadOnlyDictionary? details = null) + => new(AuthorityPluginHealthStatus.Unavailable, message, details ?? EmptyDetails); + + private static readonly IReadOnlyDictionary EmptyDetails = + new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +/// +/// Describes a canonical Authority user surfaced by a plugin. +/// +public sealed record AuthorityUserDescriptor +{ + /// + /// Initialises a new user descriptor. + /// + public AuthorityUserDescriptor( + string subjectId, + string username, + string? displayName, + bool requiresPasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + SubjectId = ValidateRequired(subjectId, nameof(subjectId)); + Username = ValidateRequired(username, nameof(username)); + DisplayName = displayName; + RequiresPasswordReset = requiresPasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Stable subject identifier for token issuance. + /// + public string SubjectId { get; } + + /// + /// Canonical username (case-normalised). + /// + public string Username { get; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Indicates whether the user must reset their password. + /// + public bool RequiresPasswordReset { get; } + + /// + /// Collection of role identifiers associated with the user. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Arbitrary plugin-defined attributes (used by claims enricher). + /// + public IReadOnlyDictionary Attributes { get; } + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Outcome of a credential verification attempt. +/// +public sealed record AuthorityCredentialVerificationResult +{ + private AuthorityCredentialVerificationResult( + bool succeeded, + AuthorityUserDescriptor? user, + AuthorityCredentialFailureCode? failureCode, + string? message, + TimeSpan? retryAfter, + IReadOnlyList auditProperties) + { + Succeeded = succeeded; + User = user; + FailureCode = failureCode; + Message = message; + RetryAfter = retryAfter; + AuditProperties = auditProperties ?? Array.Empty(); + } + + /// + /// Indicates whether the verification succeeded. + /// + public bool Succeeded { get; } + + /// + /// Resolved user descriptor when successful. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Failure classification when unsuccessful. + /// + public AuthorityCredentialFailureCode? FailureCode { get; } + + /// + /// Optional message describing the outcome. + /// + public string? Message { get; } + + /// + /// Optional suggested retry interval (e.g. for lockouts). + /// + public TimeSpan? RetryAfter { get; } + + /// + /// Additional audit properties emitted by the credential store. + /// + public IReadOnlyList AuditProperties { get; } + + /// + /// Builds a successful verification result. + /// + public static AuthorityCredentialVerificationResult Success( + AuthorityUserDescriptor user, + string? message = null, + IReadOnlyList? auditProperties = null) + => new(true, user ?? throw new ArgumentNullException(nameof(user)), null, message, null, auditProperties ?? Array.Empty()); + + /// + /// Builds a failed verification result. + /// + public static AuthorityCredentialVerificationResult Failure( + AuthorityCredentialFailureCode failureCode, + string? message = null, + TimeSpan? retryAfter = null, + IReadOnlyList? auditProperties = null) + => new(false, null, failureCode, message, retryAfter, auditProperties ?? Array.Empty()); +} + +/// +/// Classifies credential verification failures. +/// +public enum AuthorityCredentialFailureCode +{ + /// + /// Username/password combination is invalid. + /// + InvalidCredentials, + + /// + /// Account is locked out (retry after a specified duration). + /// + LockedOut, + + /// + /// Password has expired and must be reset. + /// + PasswordExpired, + + /// + /// User must reset password before proceeding. + /// + RequiresPasswordReset, + + /// + /// Additional multi-factor authentication is required. + /// + RequiresMfa, + + /// + /// Unexpected failure occurred (see message for details). + /// + UnknownError +} + +/// +/// Represents a user provisioning request. +/// +public sealed record AuthorityUserRegistration +{ + /// + /// Initialises a new registration. + /// + public AuthorityUserRegistration( + string username, + string? password, + string? displayName, + string? email, + bool requirePasswordReset, + IReadOnlyCollection? roles = null, + IReadOnlyDictionary? attributes = null) + { + Username = ValidateRequired(username, nameof(username)); + Password = password; + DisplayName = displayName; + Email = email; + RequirePasswordReset = requirePasswordReset; + Roles = roles is null ? Array.Empty() : roles.ToArray(); + Attributes = attributes is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(attributes, StringComparer.OrdinalIgnoreCase); + } + + /// + /// Canonical username (unique). + /// + public string Username { get; } + + /// + /// Optional raw password (hashed by plugin). + /// + public string? Password { get; init; } + + /// + /// Optional human-friendly display name. + /// + public string? DisplayName { get; } + + /// + /// Optional contact email. + /// + public string? Email { get; } + + /// + /// Indicates whether the user must reset their password at next login. + /// + public bool RequirePasswordReset { get; } + + /// + /// Associated roles. + /// + public IReadOnlyCollection Roles { get; } + + /// + /// Plugin-defined attributes. + /// + public IReadOnlyDictionary Attributes { get; } + + /// + /// Creates a copy with the provided password while preserving other fields. + /// + public AuthorityUserRegistration WithPassword(string? password) + => new(Username, password, DisplayName, Email, RequirePasswordReset, Roles, Attributes); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +/// +/// Generic operation result utilised by plugins. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult(bool succeeded, string? errorCode, string? message) + { + Succeeded = succeeded; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Machine-readable error code (populated on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result. + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, ValidateErrorCode(errorCode), message); + + internal static string ValidateErrorCode(string errorCode) + => string.IsNullOrWhiteSpace(errorCode) + ? throw new ArgumentException("Error code is required for failures.", nameof(errorCode)) + : errorCode; +} + +/// +/// Generic operation result that returns a value. +/// +public sealed record AuthorityPluginOperationResult +{ + private AuthorityPluginOperationResult( + bool succeeded, + TValue? value, + string? errorCode, + string? message) + { + Succeeded = succeeded; + Value = value; + ErrorCode = errorCode; + Message = message; + } + + /// + /// Indicates whether the operation succeeded. + /// + public bool Succeeded { get; } + + /// + /// Returned value when successful. + /// + public TValue? Value { get; } + + /// + /// Machine-readable error code (on failure). + /// + public string? ErrorCode { get; } + + /// + /// Optional human-readable message. + /// + public string? Message { get; } + + /// + /// Returns a successful result with the provided value. + /// + public static AuthorityPluginOperationResult Success(TValue value, string? message = null) + => new(true, value, null, message); + + /// + /// Returns a successful result without a value (defaults to default). + /// + public static AuthorityPluginOperationResult Success(string? message = null) + => new(true, default, null, message); + + /// + /// Returns a failed result with the supplied error code. + /// + public static AuthorityPluginOperationResult Failure(string errorCode, string? message = null) + => new(false, default, AuthorityPluginOperationResult.ValidateErrorCode(errorCode), message); +} + +/// +/// Context supplied to claims enrichment routines. +/// +public sealed class AuthorityClaimsEnrichmentContext +{ + private readonly Dictionary items; + + /// + /// Initialises a new context instance. + /// + public AuthorityClaimsEnrichmentContext( + AuthorityPluginContext plugin, + AuthorityUserDescriptor? user, + AuthorityClientDescriptor? client) + { + Plugin = plugin ?? throw new ArgumentNullException(nameof(plugin)); + User = user; + Client = client; + items = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + /// + /// Gets the plugin context associated with the principal. + /// + public AuthorityPluginContext Plugin { get; } + + /// + /// Gets the user descriptor when available. + /// + public AuthorityUserDescriptor? User { get; } + + /// + /// Gets the client descriptor when available. + /// + public AuthorityClientDescriptor? Client { get; } + + /// + /// Extensible bag for plugin-specific data passed between enrichment stages. + /// + public IDictionary Items => items; +} + +/// +/// Represents a registered OAuth/OpenID client. +/// +public sealed record AuthorityClientDescriptor +{ + public AuthorityClientDescriptor( + string clientId, + string? displayName, + bool confidential, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? allowedAudiences = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + IReadOnlyDictionary? properties = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + DisplayName = displayName; + Confidential = confidential; + AllowedGrantTypes = Normalize(allowedGrantTypes); + AllowedScopes = Normalize(allowedScopes); + AllowedAudiences = Normalize(allowedAudiences); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + } + + public string ClientId { get; } + public string? DisplayName { get; } + public bool Confidential { get; } + public IReadOnlyCollection AllowedGrantTypes { get; } + public IReadOnlyCollection AllowedScopes { get; } + public IReadOnlyCollection AllowedAudiences { get; } + public IReadOnlyCollection RedirectUris { get; } + public IReadOnlyCollection PostLogoutRedirectUris { get; } + public IReadOnlyDictionary Properties { get; } + + private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) + => values is null || values.Count == 0 + ? Array.Empty() + : values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} + +public sealed record AuthorityClientCertificateBindingRegistration +{ + public AuthorityClientCertificateBindingRegistration( + string thumbprint, + string? serialNumber = null, + string? subject = null, + string? issuer = null, + IReadOnlyCollection? subjectAlternativeNames = null, + DateTimeOffset? notBefore = null, + DateTimeOffset? notAfter = null, + string? label = null) + { + Thumbprint = NormalizeThumbprint(thumbprint); + SerialNumber = Normalize(serialNumber); + Subject = Normalize(subject); + Issuer = Normalize(issuer); + SubjectAlternativeNames = subjectAlternativeNames is null || subjectAlternativeNames.Count == 0 + ? Array.Empty() + : subjectAlternativeNames + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + NotBefore = notBefore; + NotAfter = notAfter; + Label = Normalize(label); + } + + public string Thumbprint { get; } + public string? SerialNumber { get; } + public string? Subject { get; } + public string? Issuer { get; } + public IReadOnlyCollection SubjectAlternativeNames { get; } + public DateTimeOffset? NotBefore { get; } + public DateTimeOffset? NotAfter { get; } + public string? Label { get; } + + private static string NormalizeThumbprint(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Thumbprint is required.", nameof(value)); + } + + return value + .Replace(":", string.Empty, StringComparison.Ordinal) + .Replace(" ", string.Empty, StringComparison.Ordinal) + .ToUpperInvariant(); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} + +public sealed record AuthorityClientRegistration +{ + public AuthorityClientRegistration( + string clientId, + bool confidential, + string? displayName, + string? clientSecret, + IReadOnlyCollection? allowedGrantTypes = null, + IReadOnlyCollection? allowedScopes = null, + IReadOnlyCollection? allowedAudiences = null, + IReadOnlyCollection? redirectUris = null, + IReadOnlyCollection? postLogoutRedirectUris = null, + IReadOnlyDictionary? properties = null, + IReadOnlyCollection? certificateBindings = null) + { + ClientId = ValidateRequired(clientId, nameof(clientId)); + Confidential = confidential; + DisplayName = displayName; + ClientSecret = confidential + ? ValidateRequired(clientSecret ?? string.Empty, nameof(clientSecret)) + : clientSecret; + AllowedGrantTypes = Normalize(allowedGrantTypes); + AllowedScopes = Normalize(allowedScopes); + AllowedAudiences = Normalize(allowedAudiences); + RedirectUris = redirectUris is null ? Array.Empty() : redirectUris.ToArray(); + PostLogoutRedirectUris = postLogoutRedirectUris is null ? Array.Empty() : postLogoutRedirectUris.ToArray(); + Properties = properties is null + ? new Dictionary(StringComparer.OrdinalIgnoreCase) + : new Dictionary(properties, StringComparer.OrdinalIgnoreCase); + CertificateBindings = certificateBindings is null + ? Array.Empty() + : certificateBindings.ToArray(); + } + + public string ClientId { get; } + public bool Confidential { get; } + public string? DisplayName { get; } + public string? ClientSecret { get; init; } + public IReadOnlyCollection AllowedGrantTypes { get; } + public IReadOnlyCollection AllowedScopes { get; } + public IReadOnlyCollection AllowedAudiences { get; } + public IReadOnlyCollection RedirectUris { get; } + public IReadOnlyCollection PostLogoutRedirectUris { get; } + public IReadOnlyDictionary Properties { get; } + public IReadOnlyCollection CertificateBindings { get; } + + public AuthorityClientRegistration WithClientSecret(string? clientSecret) + => new(ClientId, Confidential, DisplayName, clientSecret, AllowedGrantTypes, AllowedScopes, AllowedAudiences, RedirectUris, PostLogoutRedirectUris, Properties, CertificateBindings); + + private static IReadOnlyCollection Normalize(IReadOnlyCollection? values) + => values is null || values.Count == 0 + ? Array.Empty() + : values + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .ToArray(); + + private static string ValidateRequired(string value, string paramName) + => string.IsNullOrWhiteSpace(value) + ? throw new ArgumentException("Value cannot be null or whitespace.", paramName) + : value; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientCertificateBinding.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientCertificateBinding.cs new file mode 100644 index 00000000..ea86feea --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientCertificateBinding.cs @@ -0,0 +1,45 @@ +using MongoDB.Bson.Serialization.Attributes; +using System.Collections.Generic; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Captures certificate metadata associated with an mTLS-bound client. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityClientCertificateBinding +{ + [BsonElement("thumbprint")] + public string Thumbprint { get; set; } = string.Empty; + + [BsonElement("serialNumber")] + [BsonIgnoreIfNull] + public string? SerialNumber { get; set; } + + [BsonElement("subject")] + [BsonIgnoreIfNull] + public string? Subject { get; set; } + + [BsonElement("issuer")] + [BsonIgnoreIfNull] + public string? Issuer { get; set; } + + [BsonElement("notBefore")] + public DateTimeOffset? NotBefore { get; set; } + + [BsonElement("notAfter")] + public DateTimeOffset? NotAfter { get; set; } + + [BsonElement("subjectAlternativeNames")] + public List SubjectAlternativeNames { get; set; } = new(); + + [BsonElement("label")] + [BsonIgnoreIfNull] + public string? Label { get; set; } + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs index 42b8699b..facaba2f 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityClientDocument.cs @@ -1,61 +1,69 @@ -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; - -namespace StellaOps.Authority.Storage.Mongo.Documents; - -/// -/// Represents an OAuth client/application registered with Authority. -/// -[BsonIgnoreExtraElements] -public sealed class AuthorityClientDocument -{ - [BsonId] - [BsonRepresentation(BsonType.ObjectId)] - public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); - - [BsonElement("clientId")] - public string ClientId { get; set; } = string.Empty; - - [BsonElement("clientType")] - public string ClientType { get; set; } = "confidential"; - - [BsonElement("displayName")] - [BsonIgnoreIfNull] - public string? DisplayName { get; set; } - - [BsonElement("description")] - [BsonIgnoreIfNull] - public string? Description { get; set; } - - [BsonElement("secretHash")] - [BsonIgnoreIfNull] - public string? SecretHash { get; set; } - - [BsonElement("permissions")] - public List Permissions { get; set; } = new(); - - [BsonElement("requirements")] - public List Requirements { get; set; } = new(); - - [BsonElement("redirectUris")] - public List RedirectUris { get; set; } = new(); - - [BsonElement("postLogoutRedirectUris")] - public List PostLogoutRedirectUris { get; set; } = new(); - - [BsonElement("properties")] - public Dictionary Properties { get; set; } = new(StringComparer.OrdinalIgnoreCase); - - [BsonElement("plugin")] - [BsonIgnoreIfNull] - public string? Plugin { get; set; } - - [BsonElement("createdAt")] - public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; - - [BsonElement("updatedAt")] - public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; - - [BsonElement("disabled")] - public bool Disabled { get; set; } -} +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using System.Collections.Generic; + +namespace StellaOps.Authority.Storage.Mongo.Documents; + +/// +/// Represents an OAuth client/application registered with Authority. +/// +[BsonIgnoreExtraElements] +public sealed class AuthorityClientDocument +{ + [BsonId] + [BsonRepresentation(BsonType.ObjectId)] + public string Id { get; set; } = ObjectId.GenerateNewId().ToString(); + + [BsonElement("clientId")] + public string ClientId { get; set; } = string.Empty; + + [BsonElement("clientType")] + public string ClientType { get; set; } = "confidential"; + + [BsonElement("displayName")] + [BsonIgnoreIfNull] + public string? DisplayName { get; set; } + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + + [BsonElement("secretHash")] + [BsonIgnoreIfNull] + public string? SecretHash { get; set; } + + [BsonElement("permissions")] + public List Permissions { get; set; } = new(); + + [BsonElement("requirements")] + public List Requirements { get; set; } = new(); + + [BsonElement("redirectUris")] + public List RedirectUris { get; set; } = new(); + + [BsonElement("postLogoutRedirectUris")] + public List PostLogoutRedirectUris { get; set; } = new(); + + [BsonElement("properties")] + public Dictionary Properties { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + [BsonElement("plugin")] + [BsonIgnoreIfNull] + public string? Plugin { get; set; } + + [BsonElement("senderConstraint")] + [BsonIgnoreIfNull] + public string? SenderConstraint { get; set; } + + [BsonElement("certificateBindings")] + public List CertificateBindings { get; set; } = new(); + + [BsonElement("createdAt")] + public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("updatedAt")] + public DateTimeOffset UpdatedAt { get; set; } = DateTimeOffset.UtcNow; + + [BsonElement("disabled")] + public bool Disabled { get; set; } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs index fd5156b5..79df9ac0 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Documents/AuthorityTokenDocument.cs @@ -62,6 +62,18 @@ public sealed class AuthorityTokenDocument [BsonIgnoreIfNull] public string? RevokedReasonDescription { get; set; } + [BsonElement("senderConstraint")] + [BsonIgnoreIfNull] + public string? SenderConstraint { get; set; } + + [BsonElement("senderKeyThumbprint")] + [BsonIgnoreIfNull] + public string? SenderKeyThumbprint { get; set; } + + [BsonElement("senderNonce")] + [BsonIgnoreIfNull] + public string? SenderNonce { get; set; } + [BsonElement("devices")] [BsonIgnoreIfNull] diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs index 9b48024d..afd00d45 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Extensions/ServiceCollectionExtensions.cs @@ -1,126 +1,129 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Initialization; -using StellaOps.Authority.Storage.Mongo.Migrations; -using StellaOps.Authority.Storage.Mongo.Options; -using StellaOps.Authority.Storage.Mongo.Stores; - -namespace StellaOps.Authority.Storage.Mongo.Extensions; - -/// -/// Dependency injection helpers for wiring the Authority MongoDB storage layer. -/// -public static class ServiceCollectionExtensions -{ - public static IServiceCollection AddAuthorityMongoStorage( - this IServiceCollection services, - Action configureOptions) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configureOptions); - - services.AddOptions() - .Configure(configureOptions) - .PostConfigure(static options => options.EnsureValid()); - - services.TryAddSingleton(TimeProvider.System); - - services.AddSingleton(static sp => - { - var options = sp.GetRequiredService>().Value; - return new MongoClient(options.ConnectionString); - }); - - services.AddSingleton(static sp => - { - var options = sp.GetRequiredService>().Value; - var client = sp.GetRequiredService(); - - var settings = new MongoDatabaseSettings - { - ReadConcern = ReadConcern.Majority, - WriteConcern = WriteConcern.WMajority, - ReadPreference = ReadPreference.PrimaryPreferred - }; - - var database = client.GetDatabase(options.GetDatabaseName(), settings); - var writeConcern = database.Settings.WriteConcern.With(wTimeout: options.CommandTimeout); - return database.WithWriteConcern(writeConcern); - }); - - services.AddSingleton(); - services.AddSingleton(); - - services.TryAddEnumerable(ServiceDescriptor.Singleton()); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Users); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Clients); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Scopes); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.LoginAttempts); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Revocations); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.RevocationState); - }); - - services.AddSingleton(static sp => - { - var database = sp.GetRequiredService(); - return database.GetCollection(AuthorityMongoDefaults.Collections.Invites); - }); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Initialization; +using StellaOps.Authority.Storage.Mongo.Migrations; +using StellaOps.Authority.Storage.Mongo.Options; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Authority.Storage.Mongo.Sessions; + +namespace StellaOps.Authority.Storage.Mongo.Extensions; + +/// +/// Dependency injection helpers for wiring the Authority MongoDB storage layer. +/// +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddAuthorityMongoStorage( + this IServiceCollection services, + Action configureOptions) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureOptions); + + services.AddOptions() + .Configure(configureOptions) + .PostConfigure(static options => options.EnsureValid()); + + services.TryAddSingleton(TimeProvider.System); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + return new MongoClient(options.ConnectionString); + }); + + services.AddSingleton(static sp => + { + var options = sp.GetRequiredService>().Value; + var client = sp.GetRequiredService(); + + var settings = new MongoDatabaseSettings + { + ReadConcern = ReadConcern.Majority, + WriteConcern = WriteConcern.WMajority, + ReadPreference = ReadPreference.PrimaryPreferred + }; + + var database = client.GetDatabase(options.GetDatabaseName(), settings); + var writeConcern = database.Settings.WriteConcern.With(wTimeout: options.CommandTimeout); + return database.WithWriteConcern(writeConcern); + }); + + services.AddSingleton(); + services.AddSingleton(); + + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + + services.AddScoped(); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Users); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Clients); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Scopes); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.LoginAttempts); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Revocations); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.RevocationState); + }); + + services.AddSingleton(static sp => + { + var database = sp.GetRequiredService(); + return database.GetCollection(AuthorityMongoDefaults.Collections.Invites); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs index 2f21c239..3970e400 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityClientCollectionInitializer.cs @@ -1,24 +1,30 @@ -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Initialization; - -internal sealed class AuthorityClientCollectionInitializer : IAuthorityCollectionInitializer -{ - public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Clients); - - var indexModels = new[] - { - new CreateIndexModel( - Builders.IndexKeys.Ascending(c => c.ClientId), - new CreateIndexOptions { Name = "client_id_unique", Unique = true }), - new CreateIndexModel( - Builders.IndexKeys.Ascending(c => c.Disabled), - new CreateIndexOptions { Name = "client_disabled" }) - }; - - await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); - } -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityClientCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Clients); + + var indexModels = new[] + { + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.ClientId), + new CreateIndexOptions { Name = "client_id_unique", Unique = true }), + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.Disabled), + new CreateIndexOptions { Name = "client_disabled" }), + new CreateIndexModel( + Builders.IndexKeys.Ascending(c => c.SenderConstraint), + new CreateIndexOptions { Name = "client_sender_constraint" }), + new CreateIndexModel( + Builders.IndexKeys.Ascending("certificateBindings.thumbprint"), + new CreateIndexOptions { Name = "client_cert_thumbprints" }) + }; + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs index eca61f40..03806b84 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Initialization/AuthorityTokenCollectionInitializer.cs @@ -1,45 +1,51 @@ -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Initialization; - -internal sealed class AuthorityTokenCollectionInitializer : IAuthorityCollectionInitializer -{ - public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); - - var indexModels = new List> - { - new( - Builders.IndexKeys.Ascending(t => t.TokenId), - new CreateIndexOptions { Name = "token_id_unique", Unique = true }), - new( - Builders.IndexKeys.Ascending(t => t.ReferenceId), - new CreateIndexOptions { Name = "token_reference_unique", Unique = true, Sparse = true }), - new( - Builders.IndexKeys.Ascending(t => t.SubjectId), - new CreateIndexOptions { Name = "token_subject" }), - new( - Builders.IndexKeys.Ascending(t => t.ClientId), - new CreateIndexOptions { Name = "token_client" }), - new( - Builders.IndexKeys - .Ascending(t => t.Status) - .Ascending(t => t.RevokedAt), - new CreateIndexOptions { Name = "token_status_revokedAt" }) - }; - - var expirationFilter = Builders.Filter.Exists(t => t.ExpiresAt, true); - indexModels.Add(new CreateIndexModel( - Builders.IndexKeys.Ascending(t => t.ExpiresAt), - new CreateIndexOptions - { - Name = "token_expiry_ttl", - ExpireAfter = TimeSpan.Zero, - PartialFilterExpression = expirationFilter - })); - - await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); - } -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Initialization; + +internal sealed class AuthorityTokenCollectionInitializer : IAuthorityCollectionInitializer +{ + public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); + + var indexModels = new List> + { + new( + Builders.IndexKeys.Ascending(t => t.TokenId), + new CreateIndexOptions { Name = "token_id_unique", Unique = true }), + new( + Builders.IndexKeys.Ascending(t => t.ReferenceId), + new CreateIndexOptions { Name = "token_reference_unique", Unique = true, Sparse = true }), + new( + Builders.IndexKeys.Ascending(t => t.SubjectId), + new CreateIndexOptions { Name = "token_subject" }), + new( + Builders.IndexKeys.Ascending(t => t.ClientId), + new CreateIndexOptions { Name = "token_client" }), + new( + Builders.IndexKeys + .Ascending(t => t.Status) + .Ascending(t => t.RevokedAt), + new CreateIndexOptions { Name = "token_status_revokedAt" }), + new( + Builders.IndexKeys.Ascending(t => t.SenderConstraint), + new CreateIndexOptions { Name = "token_sender_constraint", Sparse = true }), + new( + Builders.IndexKeys.Ascending(t => t.SenderKeyThumbprint), + new CreateIndexOptions { Name = "token_sender_thumbprint", Sparse = true }) + }; + + var expirationFilter = Builders.Filter.Exists(t => t.ExpiresAt, true); + indexModels.Add(new CreateIndexModel( + Builders.IndexKeys.Ascending(t => t.ExpiresAt), + new CreateIndexOptions + { + Name = "token_expiry_ttl", + ExpireAfter = TimeSpan.Zero, + PartialFilterExpression = expirationFilter + })); + + await collection.Indexes.CreateManyAsync(indexModels, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Sessions/AuthorityMongoSessionAccessor.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Sessions/AuthorityMongoSessionAccessor.cs new file mode 100644 index 00000000..6c8f11ee --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Sessions/AuthorityMongoSessionAccessor.cs @@ -0,0 +1,128 @@ +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Storage.Mongo.Options; + +namespace StellaOps.Authority.Storage.Mongo.Sessions; + +public interface IAuthorityMongoSessionAccessor : IAsyncDisposable +{ + ValueTask GetSessionAsync(CancellationToken cancellationToken = default); +} + +internal sealed class AuthorityMongoSessionAccessor : IAuthorityMongoSessionAccessor +{ + private readonly IMongoClient client; + private readonly AuthorityMongoOptions options; + private readonly object gate = new(); + private Task? sessionTask; + private IClientSessionHandle? session; + private bool disposed; + + public AuthorityMongoSessionAccessor( + IMongoClient client, + IOptions options) + { + this.client = client ?? throw new ArgumentNullException(nameof(client)); + this.options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + } + + public async ValueTask GetSessionAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(disposed, this); + + var existing = Volatile.Read(ref session); + if (existing is not null) + { + return existing; + } + + Task startTask; + + lock (gate) + { + if (session is { } cached) + { + return cached; + } + + sessionTask ??= StartSessionInternalAsync(cancellationToken); + startTask = sessionTask; + } + + try + { + var handle = await startTask.WaitAsync(cancellationToken).ConfigureAwait(false); + + if (session is null) + { + lock (gate) + { + if (session is null) + { + session = handle; + sessionTask = Task.FromResult(handle); + } + } + } + + return handle; + } + catch + { + lock (gate) + { + if (ReferenceEquals(sessionTask, startTask)) + { + sessionTask = null; + } + } + + throw; + } + } + + private async Task StartSessionInternalAsync(CancellationToken cancellationToken) + { + var sessionOptions = new ClientSessionOptions + { + CausalConsistency = true, + DefaultTransactionOptions = new TransactionOptions( + readPreference: ReadPreference.Primary, + readConcern: ReadConcern.Majority, + writeConcern: WriteConcern.WMajority.With(wTimeout: options.CommandTimeout)) + }; + + var handle = await client.StartSessionAsync(sessionOptions, cancellationToken).ConfigureAwait(false); + return handle; + } + + public ValueTask DisposeAsync() + { + if (disposed) + { + return ValueTask.CompletedTask; + } + + disposed = true; + + IClientSessionHandle? handle; + + lock (gate) + { + handle = session; + session = null; + sessionTask = null; + } + + if (handle is not null) + { + handle.Dispose(); + } + + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj index 44d7ac06..344af1c5 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/StellaOps.Authority.Storage.Mongo.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityBootstrapInviteStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityBootstrapInviteStore.cs index 48c0629f..7f8d5c99 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityBootstrapInviteStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityBootstrapInviteStore.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using MongoDB.Driver; using StellaOps.Authority.Storage.Mongo.Documents; @@ -12,11 +14,19 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS public AuthorityBootstrapInviteStore(IMongoCollection collection) => this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - public async ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken) + public async ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { ArgumentNullException.ThrowIfNull(document); - await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + if (session is { }) + { + await collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + return document; } @@ -25,7 +35,8 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS string expectedType, DateTimeOffset now, string? reservedBy, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(token)) { @@ -33,8 +44,9 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS } var normalizedToken = token.Trim(); + var tokenFilter = Builders.Filter.Eq(i => i.Token, normalizedToken); var filter = Builders.Filter.And( - Builders.Filter.Eq(i => i.Token, normalizedToken), + tokenFilter, Builders.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Pending)); var update = Builders.Update @@ -47,14 +59,31 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS ReturnDocument = ReturnDocument.After }; - var invite = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); + AuthorityBootstrapInviteDocument? invite; + if (session is { }) + { + invite = await collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + } + else + { + invite = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); + } if (invite is null) { - var existing = await collection - .Find(i => i.Token == normalizedToken) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); + AuthorityBootstrapInviteDocument? existing; + if (session is { }) + { + existing = await collection.Find(session, tokenFilter) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + else + { + existing = await collection.Find(tokenFilter) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } if (existing is null) { @@ -76,60 +105,76 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS if (!string.Equals(invite.Type, expectedType, StringComparison.OrdinalIgnoreCase)) { - await ReleaseAsync(normalizedToken, cancellationToken).ConfigureAwait(false); + await ReleaseAsync(normalizedToken, cancellationToken, session).ConfigureAwait(false); return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, invite); } if (invite.ExpiresAt <= now) { - await MarkExpiredAsync(normalizedToken, cancellationToken).ConfigureAwait(false); + await MarkExpiredAsync(normalizedToken, cancellationToken, session).ConfigureAwait(false); return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Expired, invite); } return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Reserved, invite); } - public async ValueTask ReleaseAsync(string token, CancellationToken cancellationToken) + public async ValueTask ReleaseAsync(string token, CancellationToken cancellationToken, IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(token)) { return false; } - var result = await collection.UpdateOneAsync( - Builders.Filter.And( - Builders.Filter.Eq(i => i.Token, token.Trim()), - Builders.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)), - Builders.Update - .Set(i => i.Status, AuthorityBootstrapInviteStatuses.Pending) - .Set(i => i.ReservedAt, null) - .Set(i => i.ReservedBy, null), - cancellationToken: cancellationToken).ConfigureAwait(false); + var filter = Builders.Filter.And( + Builders.Filter.Eq(i => i.Token, token.Trim()), + Builders.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)); + var update = Builders.Update + .Set(i => i.Status, AuthorityBootstrapInviteStatuses.Pending) + .Set(i => i.ReservedAt, null) + .Set(i => i.ReservedBy, null); + + UpdateResult result; + if (session is { }) + { + result = await collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } return result.ModifiedCount > 0; } - public async ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken) + public async ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(token)) { return false; } - var result = await collection.UpdateOneAsync( - Builders.Filter.And( - Builders.Filter.Eq(i => i.Token, token.Trim()), - Builders.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)), - Builders.Update - .Set(i => i.Status, AuthorityBootstrapInviteStatuses.Consumed) - .Set(i => i.ConsumedAt, consumedAt) - .Set(i => i.ConsumedBy, consumedBy), - cancellationToken: cancellationToken).ConfigureAwait(false); + var filter = Builders.Filter.And( + Builders.Filter.Eq(i => i.Token, token.Trim()), + Builders.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)); + var update = Builders.Update + .Set(i => i.Status, AuthorityBootstrapInviteStatuses.Consumed) + .Set(i => i.ConsumedAt, consumedAt) + .Set(i => i.ConsumedBy, consumedBy); + + UpdateResult result; + if (session is { }) + { + result = await collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } return result.ModifiedCount > 0; } - public async ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken) + public async ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken, IClientSessionHandle? session = null) { var filter = Builders.Filter.And( Builders.Filter.Lte(i => i.ExpiresAt, now), @@ -142,25 +187,49 @@ internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteS .Set(i => i.ReservedAt, null) .Set(i => i.ReservedBy, null); - var expired = await collection.Find(filter) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); + List expired; + if (session is { }) + { + expired = await collection.Find(session, filter) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + } + else + { + expired = await collection.Find(filter) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + } if (expired.Count == 0) { return Array.Empty(); } - await collection.UpdateManyAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + if (session is { }) + { + await collection.UpdateManyAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.UpdateManyAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } return expired; } - private async Task MarkExpiredAsync(string token, CancellationToken cancellationToken) + private async Task MarkExpiredAsync(string token, CancellationToken cancellationToken, IClientSessionHandle? session) { - await collection.UpdateOneAsync( - Builders.Filter.Eq(i => i.Token, token), - Builders.Update.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired), - cancellationToken: cancellationToken).ConfigureAwait(false); + var filter = Builders.Filter.Eq(i => i.Token, token); + var update = Builders.Update.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired); + + if (session is { }) + { + await collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } } } diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs index b7f0bc2e..95063cc0 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityClientStore.cs @@ -1,64 +1,88 @@ -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityClientStore : IAuthorityClientStore -{ - private readonly IMongoCollection collection; - private readonly TimeProvider clock; - private readonly ILogger logger; - - public AuthorityClientStore( - IMongoCollection collection, - TimeProvider clock, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(clientId)) - { - return null; - } - - var id = clientId.Trim(); - return await collection.Find(c => c.ClientId == id) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - } - - public async ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - document.UpdatedAt = clock.GetUtcNow(); - - var filter = Builders.Filter.Eq(c => c.ClientId, document.ClientId); - var options = new ReplaceOptions { IsUpsert = true }; - - var result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); - - if (result.UpsertedId is not null) - { - logger.LogInformation("Inserted Authority client {ClientId}.", document.ClientId); - } - } - - public async ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(clientId)) - { - return false; - } - - var id = clientId.Trim(); - var result = await collection.DeleteOneAsync(c => c.ClientId == id, cancellationToken).ConfigureAwait(false); - return result.DeletedCount > 0; - } -} +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityClientStore : IAuthorityClientStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityClientStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(clientId)) + { + return null; + } + + var id = clientId.Trim(); + var filter = Builders.Filter.Eq(c => c.ClientId, id); + var cursor = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(c => c.ClientId, document.ClientId); + var options = new ReplaceOptions { IsUpsert = true }; + + ReplaceOneResult result; + if (session is { }) + { + result = await collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority client {ClientId}.", document.ClientId); + } + } + + public async ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(clientId)) + { + return false; + } + + var id = clientId.Trim(); + var filter = Builders.Filter.Eq(c => c.ClientId, id); + + DeleteResult result; + if (session is { }) + { + result = await collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); + } + + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs index 48442a6d..4e601465 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityLoginAttemptStore.cs @@ -1,52 +1,72 @@ -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityLoginAttemptStore : IAuthorityLoginAttemptStore -{ - private readonly IMongoCollection collection; - private readonly ILogger logger; - - public AuthorityLoginAttemptStore( - IMongoCollection collection, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); - logger.LogDebug( - "Recorded authority audit event {EventType} for subject '{SubjectId}' with outcome {Outcome}.", - document.EventType, - document.SubjectId ?? document.Username ?? "", - document.Outcome); - } - - public async ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(subjectId) || limit <= 0) - { - return Array.Empty(); - } - - var normalized = subjectId.Trim(); - - var cursor = await collection.FindAsync( - Builders.Filter.Eq(a => a.SubjectId, normalized), - new FindOptions - { - Sort = Builders.Sort.Descending(a => a.OccurredAt), - Limit = limit - }, - cancellationToken).ConfigureAwait(false); - - return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); - } -} +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityLoginAttemptStore : IAuthorityLoginAttemptStore +{ + private readonly IMongoCollection collection; + private readonly ILogger logger; + + public AuthorityLoginAttemptStore( + IMongoCollection collection, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + if (session is { }) + { + await collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + logger.LogDebug( + "Recorded authority audit event {EventType} for subject '{SubjectId}' with outcome {Outcome}.", + document.EventType, + document.SubjectId ?? document.Username ?? "", + document.Outcome); + } + + public async ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(subjectId) || limit <= 0) + { + return Array.Empty(); + } + + var normalized = subjectId.Trim(); + + var filter = Builders.Filter.Eq(a => a.SubjectId, normalized); + var options = new FindOptions + { + Sort = Builders.Sort.Descending(a => a.OccurredAt), + Limit = limit + }; + + IAsyncCursor cursor; + if (session is { }) + { + cursor = await collection.FindAsync(session, filter, options, cancellationToken).ConfigureAwait(false); + } + else + { + cursor = await collection.FindAsync(filter, options, cancellationToken).ConfigureAwait(false); + } + + return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationExportStateStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationExportStateStore.cs index 081c0a33..a01f329f 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationExportStateStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationExportStateStore.cs @@ -1,83 +1,97 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityRevocationExportStateStore : IAuthorityRevocationExportStateStore -{ - private const string StateId = "state"; - - private readonly IMongoCollection collection; - private readonly ILogger logger; - - public AuthorityRevocationExportStateStore( - IMongoCollection collection, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask GetAsync(CancellationToken cancellationToken) - { - var filter = Builders.Filter.Eq(d => d.Id, StateId); - return await collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - } - - public async ValueTask UpdateAsync( - long expectedSequence, - long newSequence, - string bundleId, - DateTimeOffset issuedAt, - CancellationToken cancellationToken) - { - if (newSequence <= 0) - { - throw new ArgumentOutOfRangeException(nameof(newSequence), "Sequence must be positive."); - } - - var filter = Builders.Filter.Eq(d => d.Id, StateId); - - if (expectedSequence > 0) - { - filter &= Builders.Filter.Eq(d => d.Sequence, expectedSequence); - } - else - { - filter &= Builders.Filter.Or( - Builders.Filter.Exists(d => d.Sequence, false), - Builders.Filter.Eq(d => d.Sequence, 0)); - } - - var update = Builders.Update - .Set(d => d.Sequence, newSequence) - .Set(d => d.LastBundleId, bundleId) - .Set(d => d.LastIssuedAt, issuedAt); - - var options = new FindOneAndUpdateOptions - { - IsUpsert = expectedSequence == 0, - ReturnDocument = ReturnDocument.After - }; - - try - { - var result = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); - if (result is null) - { - throw new InvalidOperationException("Revocation export state update conflict."); - } - - logger.LogDebug("Updated revocation export state to sequence {Sequence}.", result.Sequence); - return result; - } - catch (MongoCommandException ex) when (string.Equals(ex.CodeName, "DuplicateKey", StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException("Revocation export state update conflict due to concurrent writer.", ex); - } - } -} +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityRevocationExportStateStore : IAuthorityRevocationExportStateStore +{ + private const string StateId = "state"; + + private readonly IMongoCollection collection; + private readonly ILogger logger; + + public AuthorityRevocationExportStateStore( + IMongoCollection collection, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask GetAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Eq(d => d.Id, StateId); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask UpdateAsync( + long expectedSequence, + long newSequence, + string bundleId, + DateTimeOffset issuedAt, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) + { + if (newSequence <= 0) + { + throw new ArgumentOutOfRangeException(nameof(newSequence), "Sequence must be positive."); + } + + var filter = Builders.Filter.Eq(d => d.Id, StateId); + + if (expectedSequence > 0) + { + filter &= Builders.Filter.Eq(d => d.Sequence, expectedSequence); + } + else + { + filter &= Builders.Filter.Or( + Builders.Filter.Exists(d => d.Sequence, false), + Builders.Filter.Eq(d => d.Sequence, 0)); + } + + var update = Builders.Update + .Set(d => d.Sequence, newSequence) + .Set(d => d.LastBundleId, bundleId) + .Set(d => d.LastIssuedAt, issuedAt); + + var options = new FindOneAndUpdateOptions + { + IsUpsert = expectedSequence == 0, + ReturnDocument = ReturnDocument.After + }; + + try + { + AuthorityRevocationExportStateDocument? result; + if (session is { }) + { + result = await collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false); + } + + if (result is null) + { + throw new InvalidOperationException("Revocation export state update conflict."); + } + + logger.LogDebug("Updated revocation export state to sequence {Sequence}.", result.Sequence); + return result; + } + catch (MongoCommandException ex) when (string.Equals(ex.CodeName, "DuplicateKey", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Revocation export state update conflict due to concurrent writer.", ex); + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationStore.cs index 1e8d47ea..da13f174 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityRevocationStore.cs @@ -1,143 +1,162 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityRevocationStore : IAuthorityRevocationStore -{ - private readonly IMongoCollection collection; - private readonly ILogger logger; - - public AuthorityRevocationStore( - IMongoCollection collection, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - if (string.IsNullOrWhiteSpace(document.Category)) - { - throw new ArgumentException("Revocation category is required.", nameof(document)); - } - - if (string.IsNullOrWhiteSpace(document.RevocationId)) - { - throw new ArgumentException("Revocation identifier is required.", nameof(document)); - } - - document.Category = document.Category.Trim(); - document.RevocationId = document.RevocationId.Trim(); - document.Scopes = NormalizeScopes(document.Scopes); - document.Metadata = NormalizeMetadata(document.Metadata); - - var filter = Builders.Filter.And( - Builders.Filter.Eq(d => d.Category, document.Category), - Builders.Filter.Eq(d => d.RevocationId, document.RevocationId)); - - var now = DateTimeOffset.UtcNow; - document.UpdatedAt = now; - - var existing = await collection - .Find(filter) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - - if (existing is null) - { - document.CreatedAt = now; - } - else - { - document.Id = existing.Id; - document.CreatedAt = existing.CreatedAt; - } - - await collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - logger.LogDebug("Upserted Authority revocation entry {Category}:{RevocationId}.", document.Category, document.RevocationId); - } - - public async ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(category) || string.IsNullOrWhiteSpace(revocationId)) - { - return false; - } - - var filter = Builders.Filter.And( - Builders.Filter.Eq(d => d.Category, category.Trim()), - Builders.Filter.Eq(d => d.RevocationId, revocationId.Trim())); - - var result = await collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); - if (result.DeletedCount > 0) - { - logger.LogInformation("Removed Authority revocation entry {Category}:{RevocationId}.", category, revocationId); - return true; - } - - return false; - } - - public async ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken) - { - var filter = Builders.Filter.Or( - Builders.Filter.Eq(d => d.ExpiresAt, null), - Builders.Filter.Gt(d => d.ExpiresAt, asOf)); - - var documents = await collection - .Find(filter) - .Sort(Builders.Sort.Ascending(d => d.Category).Ascending(d => d.RevocationId)) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - return documents; - } - - private static List? NormalizeScopes(List? scopes) - { - if (scopes is null || scopes.Count == 0) - { - return null; - } - - var distinct = scopes - .Where(scope => !string.IsNullOrWhiteSpace(scope)) - .Select(scope => scope.Trim()) - .Distinct(StringComparer.Ordinal) - .OrderBy(scope => scope, StringComparer.Ordinal) - .ToList(); - - return distinct.Count == 0 ? null : distinct; - } - - private static Dictionary? NormalizeMetadata(Dictionary? metadata) - { - if (metadata is null || metadata.Count == 0) - { - return null; - } - - var result = new SortedDictionary(StringComparer.OrdinalIgnoreCase); - foreach (var pair in metadata) - { - if (string.IsNullOrWhiteSpace(pair.Key)) - { - continue; - } - - result[pair.Key.Trim()] = pair.Value; - } - - return result.Count == 0 ? null : new Dictionary(result, StringComparer.OrdinalIgnoreCase); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityRevocationStore : IAuthorityRevocationStore +{ + private readonly IMongoCollection collection; + private readonly ILogger logger; + + public AuthorityRevocationStore( + IMongoCollection collection, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + if (string.IsNullOrWhiteSpace(document.Category)) + { + throw new ArgumentException("Revocation category is required.", nameof(document)); + } + + if (string.IsNullOrWhiteSpace(document.RevocationId)) + { + throw new ArgumentException("Revocation identifier is required.", nameof(document)); + } + + document.Category = document.Category.Trim(); + document.RevocationId = document.RevocationId.Trim(); + document.Scopes = NormalizeScopes(document.Scopes); + document.Metadata = NormalizeMetadata(document.Metadata); + + var filter = Builders.Filter.And( + Builders.Filter.Eq(d => d.Category, document.Category), + Builders.Filter.Eq(d => d.RevocationId, document.RevocationId)); + + var now = DateTimeOffset.UtcNow; + document.UpdatedAt = now; + + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + var existing = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + if (existing is null) + { + document.CreatedAt = now; + } + else + { + document.Id = existing.Id; + document.CreatedAt = existing.CreatedAt; + } + + var options = new ReplaceOptions { IsUpsert = true }; + if (session is { }) + { + await collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + logger.LogDebug("Upserted Authority revocation entry {Category}:{RevocationId}.", document.Category, document.RevocationId); + } + + public async ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(category) || string.IsNullOrWhiteSpace(revocationId)) + { + return false; + } + + var filter = Builders.Filter.And( + Builders.Filter.Eq(d => d.Category, category.Trim()), + Builders.Filter.Eq(d => d.RevocationId, revocationId.Trim())); + + DeleteResult result; + if (session is { }) + { + result = await collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.DeleteOneAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + } + if (result.DeletedCount > 0) + { + logger.LogInformation("Removed Authority revocation entry {Category}:{RevocationId}.", category, revocationId); + return true; + } + + return false; + } + + public async ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Or( + Builders.Filter.Eq(d => d.ExpiresAt, null), + Builders.Filter.Gt(d => d.ExpiresAt, asOf)); + + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + var documents = await query + .Sort(Builders.Sort.Ascending(d => d.Category).Ascending(d => d.RevocationId)) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents; + } + + private static List? NormalizeScopes(List? scopes) + { + if (scopes is null || scopes.Count == 0) + { + return null; + } + + var distinct = scopes + .Where(scope => !string.IsNullOrWhiteSpace(scope)) + .Select(scope => scope.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(scope => scope, StringComparer.Ordinal) + .ToList(); + + return distinct.Count == 0 ? null : distinct; + } + + private static Dictionary? NormalizeMetadata(Dictionary? metadata) + { + if (metadata is null || metadata.Count == 0) + { + return null; + } + + var result = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in metadata) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + result[pair.Key.Trim()] = pair.Value; + } + + return result.Count == 0 ? null : new Dictionary(result, StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs index c8f52f38..c9eba284 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityScopeStore.cs @@ -1,69 +1,104 @@ -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityScopeStore : IAuthorityScopeStore -{ - private readonly IMongoCollection collection; - private readonly TimeProvider clock; - private readonly ILogger logger; - - public AuthorityScopeStore( - IMongoCollection collection, - TimeProvider clock, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask FindByNameAsync(string name, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(name)) - { - return null; - } - - var normalized = name.Trim(); - return await collection.Find(s => s.Name == normalized) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - } - - public async ValueTask> ListAsync(CancellationToken cancellationToken) - { - var cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); - return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); - } - - public async ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - document.UpdatedAt = clock.GetUtcNow(); - - var filter = Builders.Filter.Eq(s => s.Name, document.Name); - var options = new ReplaceOptions { IsUpsert = true }; - - var result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); - if (result.UpsertedId is not null) - { - logger.LogInformation("Inserted Authority scope {ScopeName}.", document.Name); - } - } - - public async ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(name)) - { - return false; - } - - var normalized = name.Trim(); - var result = await collection.DeleteOneAsync(s => s.Name == normalized, cancellationToken).ConfigureAwait(false); - return result.DeletedCount > 0; - } -} +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityScopeStore : IAuthorityScopeStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityScopeStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindByNameAsync(string name, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(name)) + { + return null; + } + + var normalized = name.Trim(); + var filter = Builders.Filter.Eq(s => s.Name, normalized); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + IAsyncCursor cursor; + if (session is { }) + { + cursor = await collection.FindAsync(session, FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + return await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(s => s.Name, document.Name); + var options = new ReplaceOptions { IsUpsert = true }; + + ReplaceOneResult result; + if (session is { }) + { + result = await collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority scope {ScopeName}.", document.Name); + } + } + + public async ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(name)) + { + return false; + } + + var normalized = name.Trim(); + var filter = Builders.Filter.Eq(s => s.Name, normalized); + + DeleteResult result; + if (session is { }) + { + result = await collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.DeleteOneAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs index da2c4477..11830324 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityTokenStore.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; -using System.Linq; -using System.Globalization; using StellaOps.Authority.Storage.Mongo.Documents; namespace StellaOps.Authority.Storage.Mongo.Stores; @@ -22,15 +24,23 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken) + public async ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { ArgumentNullException.ThrowIfNull(document); - await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + if (session is { }) + { + await collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + logger.LogDebug("Inserted Authority token {TokenId}.", document.TokenId); } - public async ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken) + public async ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(tokenId)) { @@ -38,12 +48,15 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore } var id = tokenId.Trim(); - return await collection.Find(t => t.TokenId == id) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); + var filter = Builders.Filter.Eq(t => t.TokenId, id); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); } - public async ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken) + public async ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(referenceId)) { @@ -51,9 +64,12 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore } var id = referenceId.Trim(); - return await collection.Find(t => t.ReferenceId == id) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); + var filter = Builders.Filter.Eq(t => t.ReferenceId, id); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); } public async ValueTask UpdateStatusAsync( @@ -63,7 +79,8 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore string? reason, string? reasonDescription, IReadOnlyDictionary? metadata, - CancellationToken cancellationToken) + CancellationToken cancellationToken, + IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(tokenId)) { @@ -82,16 +99,29 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore .Set(t => t.RevokedReasonDescription, reasonDescription) .Set(t => t.RevokedMetadata, metadata is null ? null : new Dictionary(metadata, StringComparer.OrdinalIgnoreCase)); - var result = await collection.UpdateOneAsync( - Builders.Filter.Eq(t => t.TokenId, tokenId.Trim()), - update, - cancellationToken: cancellationToken).ConfigureAwait(false); + var filter = Builders.Filter.Eq(t => t.TokenId, tokenId.Trim()); + + UpdateResult result; + if (session is { }) + { + result = await collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } logger.LogDebug("Updated token {TokenId} status to {Status} (matched {Matched}).", tokenId, status, result.MatchedCount); } - public async ValueTask RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken) + public async ValueTask RecordUsageAsync( + string tokenId, + string? remoteAddress, + string? userAgent, + DateTimeOffset observedAt, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) { if (string.IsNullOrWhiteSpace(tokenId)) { @@ -104,10 +134,11 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore } var id = tokenId.Trim(); - var token = await collection - .Find(t => t.TokenId == id) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); + var filter = Builders.Filter.Eq(t => t.TokenId, id); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + var token = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (token is null) { @@ -147,10 +178,14 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore } var update = Builders.Update.Set(t => t.Devices, token.Devices); - await collection.UpdateOneAsync( - Builders.Filter.Eq(t => t.TokenId, id), - update, - cancellationToken: cancellationToken).ConfigureAwait(false); + if (session is { }) + { + await collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } return new TokenUsageUpdateResult(suspicious ? TokenUsageUpdateStatus.SuspectedReplay : TokenUsageUpdateStatus.Recorded, normalizedAddress, normalizedAgent); } @@ -170,14 +205,22 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore }; } - public async ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken) + public async ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken, IClientSessionHandle? session = null) { var filter = Builders.Filter.And( Builders.Filter.Not( Builders.Filter.Eq(t => t.Status, "revoked")), Builders.Filter.Lt(t => t.ExpiresAt, threshold)); - var result = await collection.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false); + DeleteResult result; + if (session is { }) + { + result = await collection.DeleteManyAsync(session, filter, options: null, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.DeleteManyAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + } if (result.DeletedCount > 0) { logger.LogInformation("Deleted {Count} expired Authority tokens.", result.DeletedCount); @@ -186,7 +229,7 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore return result.DeletedCount; } - public async ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken) + public async ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken, IClientSessionHandle? session = null) { var filter = Builders.Filter.Eq(t => t.Status, "revoked"); @@ -197,8 +240,11 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore Builders.Filter.Gt(t => t.RevokedAt, threshold)); } - var documents = await collection - .Find(filter) + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + var documents = await query .Sort(Builders.Sort.Ascending(t => t.RevokedAt).Ascending(t => t.TokenId)) .ToListAsync(cancellationToken) .ConfigureAwait(false); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs index 03242665..a8887c8a 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/AuthorityUserStore.cs @@ -1,81 +1,105 @@ -using Microsoft.Extensions.Logging; -using MongoDB.Driver; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -internal sealed class AuthorityUserStore : IAuthorityUserStore -{ - private readonly IMongoCollection collection; - private readonly TimeProvider clock; - private readonly ILogger logger; - - public AuthorityUserStore( - IMongoCollection collection, - TimeProvider clock, - ILogger logger) - { - this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(subjectId)) - { - return null; - } - - return await collection - .Find(u => u.SubjectId == subjectId) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - } - - public async ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(normalizedUsername)) - { - return null; - } - - var normalised = normalizedUsername.Trim(); - - return await collection - .Find(u => u.NormalizedUsername == normalised) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - } - - public async ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - document.UpdatedAt = clock.GetUtcNow(); - - var filter = Builders.Filter.Eq(u => u.SubjectId, document.SubjectId); - var options = new ReplaceOptions { IsUpsert = true }; - - var result = await collection - .ReplaceOneAsync(filter, document, options, cancellationToken) - .ConfigureAwait(false); - - if (result.UpsertedId is not null) - { - logger.LogInformation("Inserted Authority user {SubjectId}.", document.SubjectId); - } - } - - public async ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(subjectId)) - { - return false; - } - - var normalised = subjectId.Trim(); - var result = await collection.DeleteOneAsync(u => u.SubjectId == normalised, cancellationToken).ConfigureAwait(false); - return result.DeletedCount > 0; - } -} +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +internal sealed class AuthorityUserStore : IAuthorityUserStore +{ + private readonly IMongoCollection collection; + private readonly TimeProvider clock; + private readonly ILogger logger; + + public AuthorityUserStore( + IMongoCollection collection, + TimeProvider clock, + ILogger logger) + { + this.collection = collection ?? throw new ArgumentNullException(nameof(collection)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(subjectId)) + { + return null; + } + + var normalized = subjectId.Trim(); + var filter = Builders.Filter.Eq(u => u.SubjectId, normalized); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(normalizedUsername)) + { + return null; + } + + var normalised = normalizedUsername.Trim(); + + var filter = Builders.Filter.Eq(u => u.NormalizedUsername, normalised); + var query = session is { } + ? collection.Find(session, filter) + : collection.Find(filter); + + return await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + document.UpdatedAt = clock.GetUtcNow(); + + var filter = Builders.Filter.Eq(u => u.SubjectId, document.SubjectId); + var options = new ReplaceOptions { IsUpsert = true }; + + ReplaceOneResult result; + if (session is { }) + { + result = await collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + + if (result.UpsertedId is not null) + { + logger.LogInformation("Inserted Authority user {SubjectId}.", document.SubjectId); + } + } + + public async ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(subjectId)) + { + return false; + } + + var normalised = subjectId.Trim(); + var filter = Builders.Filter.Eq(u => u.SubjectId, normalised); + + DeleteResult result; + if (session is { }) + { + result = await collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + else + { + result = await collection.DeleteOneAsync(filter, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + return result.DeletedCount > 0; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityBootstrapInviteStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityBootstrapInviteStore.cs index c0a51bc5..3c04c551 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityBootstrapInviteStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityBootstrapInviteStore.cs @@ -1,18 +1,19 @@ +using MongoDB.Driver; using StellaOps.Authority.Storage.Mongo.Documents; namespace StellaOps.Authority.Storage.Mongo.Stores; public interface IAuthorityBootstrapInviteStore { - ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken); + ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken); + ValueTask TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask ReleaseAsync(string token, CancellationToken cancellationToken); + ValueTask ReleaseAsync(string token, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken); + ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken); + ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken, IClientSessionHandle? session = null); } public enum BootstrapInviteReservationStatus diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs index 67778ab8..3fdb960b 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityClientStore.cs @@ -1,12 +1,13 @@ -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityClientStore -{ - ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken); - - ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken); - - ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken); -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityClientStore +{ + ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs index f97d884b..3bbd06c6 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityLoginAttemptStore.cs @@ -1,10 +1,11 @@ -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityLoginAttemptStore -{ - ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken); - - ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken); -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityLoginAttemptStore +{ + ValueTask InsertAsync(AuthorityLoginAttemptDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> ListRecentAsync(string subjectId, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationExportStateStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationExportStateStore.cs index ff023777..f2dd73f3 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationExportStateStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationExportStateStore.cs @@ -1,18 +1,20 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityRevocationExportStateStore -{ - ValueTask GetAsync(CancellationToken cancellationToken); - - ValueTask UpdateAsync( - long expectedSequence, - long newSequence, - string bundleId, - DateTimeOffset issuedAt, - CancellationToken cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityRevocationExportStateStore +{ + ValueTask GetAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask UpdateAsync( + long expectedSequence, + long newSequence, + string bundleId, + DateTimeOffset issuedAt, + CancellationToken cancellationToken, + IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationStore.cs index 57eda334..104512e0 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityRevocationStore.cs @@ -1,16 +1,17 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityRevocationStore -{ - ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken); - - ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken); - - ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken); -} +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityRevocationStore +{ + ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask RemoveAsync(string category, string revocationId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs index f51cdc87..f2fb87a2 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityScopeStore.cs @@ -1,14 +1,15 @@ -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityScopeStore -{ - ValueTask FindByNameAsync(string name, CancellationToken cancellationToken); - - ValueTask> ListAsync(CancellationToken cancellationToken); - - ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken); - - ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken); -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityScopeStore +{ + ValueTask FindByNameAsync(string name, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask UpsertAsync(AuthorityScopeDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask DeleteByNameAsync(string name, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs index f4bb918a..83ecbf45 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityTokenStore.cs @@ -1,16 +1,17 @@ using System; using System.Collections.Generic; +using MongoDB.Driver; using StellaOps.Authority.Storage.Mongo.Documents; namespace StellaOps.Authority.Storage.Mongo.Stores; public interface IAuthorityTokenStore { - ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken); + ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken); + ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken); + ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken, IClientSessionHandle? session = null); ValueTask UpdateStatusAsync( string tokenId, @@ -19,13 +20,14 @@ public interface IAuthorityTokenStore string? reason, string? reasonDescription, IReadOnlyDictionary? metadata, - CancellationToken cancellationToken); + CancellationToken cancellationToken, + IClientSessionHandle? session = null); - ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken); + ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken); + ValueTask RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null); - ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken); + ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken, IClientSessionHandle? session = null); } public enum TokenUsageUpdateStatus diff --git a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs index 6f7cdf55..8f0d6d98 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Storage.Mongo/Stores/IAuthorityUserStore.cs @@ -1,14 +1,15 @@ -using StellaOps.Authority.Storage.Mongo.Documents; - -namespace StellaOps.Authority.Storage.Mongo.Stores; - -public interface IAuthorityUserStore -{ - ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken); - - ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken); - - ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken); - - ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken); -} +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Storage.Mongo.Stores; + +public interface IAuthorityUserStore +{ + ValueTask FindBySubjectIdAsync(string subjectId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask FindByNormalizedUsernameAsync(string normalizedUsername, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask UpsertAsync(AuthorityUserDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask DeleteBySubjectIdAsync(string subjectId, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/Bootstrap/BootstrapInviteCleanupServiceTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/Bootstrap/BootstrapInviteCleanupServiceTests.cs index b707b251..a4aa4e81 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/Bootstrap/BootstrapInviteCleanupServiceTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/Bootstrap/BootstrapInviteCleanupServiceTests.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Time.Testing; using StellaOps.Authority.Bootstrap; using StellaOps.Authority.Storage.Mongo.Documents; using StellaOps.Authority.Storage.Mongo.Stores; +using MongoDB.Driver; using StellaOps.Cryptography.Audit; using Xunit; @@ -65,19 +66,19 @@ public sealed class BootstrapInviteCleanupServiceTests public bool ExpireCalled { get; private set; } - public ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken) + public ValueTask CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) => throw new NotImplementedException(); - public ValueTask TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken) + public ValueTask TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null)); - public ValueTask ReleaseAsync(string token, CancellationToken cancellationToken) + public ValueTask ReleaseAsync(string token, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(false); - public ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken) + public ValueTask MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(false); - public ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken) + public ValueTask> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken, IClientSessionHandle? session = null) { ExpireCalled = true; return ValueTask.FromResult(invites); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs index 25b3e059..03dd02f9 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/ClientCredentialsAndTokenHandlersTests.cs @@ -1,9 +1,21 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IdentityModel.Tokens.Jwt; using System.Security.Claims; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Extensions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Configuration; +using StellaOps.Authority.Security; +using StellaOps.Auth.Security.Dpop; using OpenIddict.Abstractions; using OpenIddict.Extensions; using OpenIddict.Server; @@ -13,11 +25,13 @@ using StellaOps.Authority.OpenIddict; using StellaOps.Authority.OpenIddict.Handlers; using StellaOps.Authority.Plugins.Abstractions; using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; using StellaOps.Authority.Storage.Mongo.Stores; using StellaOps.Authority.RateLimiting; using StellaOps.Cryptography.Audit; using Xunit; using MongoDB.Bson; +using MongoDB.Driver; using static StellaOps.Authority.Tests.OpenIddict.TestHelpers; namespace StellaOps.Authority.Tests.OpenIddict; @@ -42,6 +56,8 @@ public class ClientCredentialsHandlersTests new TestAuthEventSink(), new TestRateLimiterMetadataAccessor(), TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:write"); @@ -70,6 +86,8 @@ public class ClientCredentialsHandlersTests new TestAuthEventSink(), new TestRateLimiterMetadataAccessor(), TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -102,6 +120,8 @@ public class ClientCredentialsHandlersTests sink, new TestRateLimiterMetadataAccessor(), TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); @@ -115,6 +135,315 @@ public class ClientCredentialsHandlersTests string.Equals(property.Value.Value, "unexpected_param", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public async Task ValidateDpopProof_AllowsSenderConstrainedClient() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Dpop.Enabled = true; + options.Security.SenderConstraints.Dpop.Nonce.Enabled = false; + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read"); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Dpop; + clientDocument.Properties[AuthorityClientMetadataKeys.SenderConstraint] = AuthoritySenderConstraintKinds.Dpop; + + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var securityKey = new ECDsaSecurityKey(ecdsa) + { + KeyId = Guid.NewGuid().ToString("N") + }; + var jwk = JsonWebKeyConverter.ConvertFromECDsaSecurityKey(securityKey); + var expectedThumbprint = ConvertThumbprintToString(jwk.ComputeJwkThumbprint()); + + var clientStore = new TestClientStore(clientDocument); + var auditSink = new TestAuthEventSink(); + var rateMetadata = new TestRateLimiterMetadataAccessor(); + + var dpopValidator = new DpopProofValidator( + Options.Create(new DpopValidationOptions()), + new InMemoryDpopReplayCache(TimeProvider.System), + TimeProvider.System, + NullLogger.Instance); + + var nonceStore = new InMemoryDpopNonceStore(TimeProvider.System, NullLogger.Instance); + + var dpopHandler = new ValidateDpopProofHandler( + options, + clientStore, + dpopValidator, + nonceStore, + rateMetadata, + auditSink, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + transaction.Options = new OpenIddictServerOptions(); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Method = "POST"; + httpContext.Request.Scheme = "https"; + httpContext.Request.Host = new HostString("authority.test"); + httpContext.Request.Path = "/token"; + + var now = TimeProvider.System.GetUtcNow(); + var proof = TestHelpers.CreateDpopProof(securityKey, httpContext.Request.Method, httpContext.Request.GetDisplayUrl(), now.ToUnixTimeSeconds()); + httpContext.Request.Headers["DPoP"] = proof; + + transaction.Properties[typeof(HttpContext).FullName!] = httpContext; + + var validateContext = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + await dpopHandler.HandleAsync(validateContext); + + Assert.False(validateContext.IsRejected); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var validateHandler = new ValidateClientCredentialsHandler( + clientStore, + registry, + TestActivitySource, + auditSink, + rateMetadata, + TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), + NullLogger.Instance); + + await validateHandler.HandleAsync(validateContext); + Assert.False(validateContext.IsRejected); + + var tokenStore = new TestTokenStore(); + var sessionAccessor = new NullMongoSessionAccessor(); + var handleHandler = new HandleClientCredentialsHandler( + registry, + tokenStore, + sessionAccessor, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var handleContext = new OpenIddictServerEvents.HandleTokenRequestContext(transaction); + await handleHandler.HandleAsync(handleContext); + Assert.True(handleContext.IsRequestHandled); + + var persistHandler = new PersistTokensHandler( + tokenStore, + sessionAccessor, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var signInContext = new OpenIddictServerEvents.ProcessSignInContext(transaction) + { + Principal = handleContext.Principal, + AccessTokenPrincipal = handleContext.Principal + }; + + await persistHandler.HandleAsync(signInContext); + + var confirmationClaim = handleContext.Principal?.GetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType); + Assert.False(string.IsNullOrWhiteSpace(confirmationClaim)); + + using (var confirmationJson = JsonDocument.Parse(confirmationClaim!)) + { + Assert.Equal(expectedThumbprint, confirmationJson.RootElement.GetProperty("jkt").GetString()); + } + + Assert.NotNull(tokenStore.Inserted); + Assert.Equal(AuthoritySenderConstraintKinds.Dpop, tokenStore.Inserted!.SenderConstraint); + Assert.Equal(expectedThumbprint, tokenStore.Inserted!.SenderKeyThumbprint); + } + + [Fact] + public async Task ValidateDpopProof_IssuesNonceChallenge_WhenNonceMissing() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Dpop.Enabled = true; + options.Security.SenderConstraints.Dpop.Nonce.Enabled = true; + options.Security.SenderConstraints.Dpop.Nonce.RequiredAudiences.Clear(); + options.Security.SenderConstraints.Dpop.Nonce.RequiredAudiences.Add("signer"); + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + Assert.Contains("signer", options.Security.SenderConstraints.Dpop.Nonce.RequiredAudiences); + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read", + allowedAudiences: "signer"); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Dpop; + clientDocument.Properties[AuthorityClientMetadataKeys.SenderConstraint] = AuthoritySenderConstraintKinds.Dpop; + + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var securityKey = new ECDsaSecurityKey(ecdsa) + { + KeyId = Guid.NewGuid().ToString("N") + }; + + var clientStore = new TestClientStore(clientDocument); + var auditSink = new TestAuthEventSink(); + var rateMetadata = new TestRateLimiterMetadataAccessor(); + + var dpopValidator = new DpopProofValidator( + Options.Create(new DpopValidationOptions()), + new InMemoryDpopReplayCache(TimeProvider.System), + TimeProvider.System, + NullLogger.Instance); + + var nonceStore = new InMemoryDpopNonceStore(TimeProvider.System, NullLogger.Instance); + + var dpopHandler = new ValidateDpopProofHandler( + options, + clientStore, + dpopValidator, + nonceStore, + rateMetadata, + auditSink, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + transaction.Options = new OpenIddictServerOptions(); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Method = "POST"; + httpContext.Request.Scheme = "https"; + httpContext.Request.Host = new HostString("authority.test"); + httpContext.Request.Path = "/token"; + + var now = TimeProvider.System.GetUtcNow(); + var proof = TestHelpers.CreateDpopProof(securityKey, httpContext.Request.Method, httpContext.Request.GetDisplayUrl(), now.ToUnixTimeSeconds()); + httpContext.Request.Headers["DPoP"] = proof; + + transaction.Properties[typeof(HttpContext).FullName!] = httpContext; + + var validateContext = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + await dpopHandler.HandleAsync(validateContext); + + Assert.True(validateContext.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, validateContext.Error); + var authenticateHeader = Assert.Single(httpContext.Response.Headers.Select(header => header) + .Where(header => string.Equals(header.Key, "WWW-Authenticate", StringComparison.OrdinalIgnoreCase))).Value; + Assert.Contains("use_dpop_nonce", authenticateHeader.ToString()); + Assert.True(httpContext.Response.Headers.TryGetValue("DPoP-Nonce", out var nonceValues)); + Assert.False(StringValues.IsNullOrEmpty(nonceValues)); + Assert.Contains(auditSink.Events, record => record.EventType == "authority.dpop.proof.challenge"); + } + + [Fact] + public async Task ValidateClientCredentials_AllowsMtlsClient_WithValidCertificate() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read"); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.Properties[AuthorityClientMetadataKeys.SenderConstraint] = AuthoritySenderConstraintKinds.Mtls; + + using var rsa = RSA.Create(2048); + var certificateRequest = new CertificateRequest("CN=mtls-client", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = certificateRequest.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + var hexThumbprint = Convert.ToHexString(certificate.GetCertHash(HashAlgorithmName.SHA256)); + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = hexThumbprint + }); + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var auditSink = new TestAuthEventSink(); + var metadataAccessor = new TestRateLimiterMetadataAccessor(); + var httpContextAccessor = new HttpContextAccessor { HttpContext = new DefaultHttpContext() }; + httpContextAccessor.HttpContext!.Connection.ClientCertificate = certificate; + + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + auditSink, + metadataAccessor, + TimeProvider.System, + validator, + httpContextAccessor, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected); + Assert.Equal(AuthoritySenderConstraintKinds.Mtls, context.Transaction.Properties[AuthorityOpenIddictConstants.SenderConstraintProperty]); + + var expectedBase64 = Base64UrlEncoder.Encode(certificate.GetCertHash(HashAlgorithmName.SHA256)); + Assert.Equal(expectedBase64, context.Transaction.Properties[AuthorityOpenIddictConstants.MtlsCertificateThumbprintProperty]); + } + + [Fact] + public async Task ValidateClientCredentials_RejectsMtlsClient_WhenCertificateMissing() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + var clientDocument = CreateClient( + secret: "s3cr3t!", + allowedGrantTypes: "client_credentials", + allowedScopes: "jobs:read"); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.Properties[AuthorityClientMetadataKeys.SenderConstraint] = AuthoritySenderConstraintKinds.Mtls; + + var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument)); + var httpContextAccessor = new HttpContextAccessor { HttpContext = new DefaultHttpContext() }; + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + + var handler = new ValidateClientCredentialsHandler( + new TestClientStore(clientDocument), + registry, + TestActivitySource, + new TestAuthEventSink(), + new TestRateLimiterMetadataAccessor(), + TimeProvider.System, + validator, + httpContextAccessor, + NullLogger.Instance); + + var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read"); + var context = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction); + + await handler.HandleAsync(context); + + Assert.True(context.IsRejected); + Assert.Equal(OpenIddictConstants.Errors.InvalidClient, context.Error); + } + [Fact] public async Task HandleClientCredentials_PersistsTokenAndEnrichesClaims() { @@ -122,11 +451,13 @@ public class ClientCredentialsHandlersTests secret: null, clientType: "public", allowedGrantTypes: "client_credentials", - allowedScopes: "jobs:trigger"); + allowedScopes: "jobs:trigger", + allowedAudiences: "signer"); var descriptor = CreateDescriptor(clientDocument); var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: descriptor); var tokenStore = new TestTokenStore(); + var sessionAccessor = new NullMongoSessionAccessor(); var authSink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); var validateHandler = new ValidateClientCredentialsHandler( @@ -136,6 +467,8 @@ public class ClientCredentialsHandlersTests authSink, metadataAccessor, TimeProvider.System, + new NoopCertificateValidator(), + new HttpContextAccessor(), NullLogger.Instance); var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger"); @@ -148,10 +481,11 @@ public class ClientCredentialsHandlersTests var handler = new HandleClientCredentialsHandler( registry, tokenStore, + sessionAccessor, TimeProvider.System, TestActivitySource, NullLogger.Instance); - var persistHandler = new PersistTokensHandler(tokenStore, TimeProvider.System, TestActivitySource, NullLogger.Instance); + var persistHandler = new PersistTokensHandler(tokenStore, sessionAccessor, TimeProvider.System, TestActivitySource, NullLogger.Instance); var context = new OpenIddictServerEvents.HandleTokenRequestContext(transaction); @@ -159,6 +493,7 @@ public class ClientCredentialsHandlersTests Assert.True(context.IsRequestHandled); Assert.NotNull(context.Principal); + Assert.Contains("signer", context.Principal!.GetAudiences()); Assert.Contains(authSink.Events, record => record.EventType == "authority.client_credentials.grant" && record.Outcome == AuthEventOutcome.Success); @@ -197,13 +532,15 @@ public class TokenValidationHandlersTests { TokenId = "token-1", Status = "revoked", - ClientId = "feedser" + ClientId = "concelier" }; var metadataAccessor = new TestRateLimiterMetadataAccessor(); var auditSink = new TestAuthEventSink(); + var sessionAccessor = new NullMongoSessionAccessor(); var handler = new ValidateAccessTokenHandler( tokenStore, + sessionAccessor, new TestClientStore(CreateClient()), CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(CreateClient())), metadataAccessor, @@ -219,7 +556,7 @@ public class TokenValidationHandlersTests Request = new OpenIddictRequest() }; - var principal = CreatePrincipal("feedser", "token-1", "standard"); + var principal = CreatePrincipal("concelier", "token-1", "standard"); var context = new OpenIddictServerEvents.ValidateTokenContext(transaction) { Principal = principal, @@ -248,8 +585,10 @@ public class TokenValidationHandlersTests var metadataAccessorSuccess = new TestRateLimiterMetadataAccessor(); var auditSinkSuccess = new TestAuthEventSink(); + var sessionAccessor = new NullMongoSessionAccessor(); var handler = new ValidateAccessTokenHandler( new TestTokenStore(), + sessionAccessor, new TestClientStore(clientDocument), registry, metadataAccessorSuccess, @@ -277,6 +616,62 @@ public class TokenValidationHandlersTests Assert.Contains(principal.Claims, claim => claim.Type == "enriched" && claim.Value == "true"); } + [Fact] + public async Task ValidateAccessTokenHandler_AddsConfirmationClaim_ForMtlsToken() + { + var tokenDocument = new AuthorityTokenDocument + { + TokenId = "token-mtls", + Status = "valid", + ClientId = "mtls-client", + SenderConstraint = AuthoritySenderConstraintKinds.Mtls, + SenderKeyThumbprint = "thumb-print" + }; + + var tokenStore = new TestTokenStore + { + Inserted = tokenDocument + }; + + var clientDocument = CreateClient(); + var registry = CreateRegistry(withClientProvisioning: false, clientDescriptor: null); + var metadataAccessor = new TestRateLimiterMetadataAccessor(); + var auditSink = new TestAuthEventSink(); + var sessionAccessor = new NullMongoSessionAccessor(); + var handler = new ValidateAccessTokenHandler( + tokenStore, + sessionAccessor, + new TestClientStore(clientDocument), + registry, + metadataAccessor, + auditSink, + TimeProvider.System, + TestActivitySource, + NullLogger.Instance); + + var transaction = new OpenIddictServerTransaction + { + Options = new OpenIddictServerOptions(), + EndpointType = OpenIddictServerEndpointType.Introspection, + Request = new OpenIddictRequest() + }; + + var principal = CreatePrincipal(clientDocument.ClientId, tokenDocument.TokenId, clientDocument.Plugin); + var context = new OpenIddictServerEvents.ValidateTokenContext(transaction) + { + Principal = principal, + TokenId = tokenDocument.TokenId + }; + + await handler.HandleAsync(context); + + Assert.False(context.IsRejected); + var confirmation = context.Principal?.GetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType); + Assert.False(string.IsNullOrWhiteSpace(confirmation)); + using var json = JsonDocument.Parse(confirmation!); + Assert.Equal(tokenDocument.SenderKeyThumbprint, json.RootElement.GetProperty("x5t#S256").GetString()); + } + [Fact] public async Task ValidateAccessTokenHandler_EmitsReplayAudit_WhenStoreDetectsSuspectedReplay() { @@ -313,8 +708,10 @@ public class TokenValidationHandlersTests clientDocument.ClientId = "agent"; var auditSink = new TestAuthEventSink(); var registry = CreateRegistry(withClientProvisioning: false, clientDescriptor: null); + var sessionAccessorReplay = new NullMongoSessionAccessor(); var handler = new ValidateAccessTokenHandler( tokenStore, + sessionAccessorReplay, new TestClientStore(clientDocument), registry, metadataAccessor, @@ -348,6 +745,89 @@ public class TokenValidationHandlersTests } } +public class AuthorityClientCertificateValidatorTests +{ + [Fact] + public async Task ValidateAsync_Rejects_WhenSanTypeNotAllowed() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Security.SenderConstraints.Mtls.AllowedSanTypes.Clear(); + options.Security.SenderConstraints.Mtls.AllowedSanTypes.Add("uri"); + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=mtls-client", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName("client.mtls.test"); + request.CertificateExtensions.Add(sanBuilder.Build()); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddMinutes(5)); + + var clientDocument = CreateClient(); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = Convert.ToHexString(certificate.GetCertHash(HashAlgorithmName.SHA256)) + }); + + var httpContext = new DefaultHttpContext(); + httpContext.Connection.ClientCertificate = certificate; + + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + var result = await validator.ValidateAsync(httpContext, clientDocument, CancellationToken.None); + + Assert.False(result.Succeeded); + Assert.Equal("certificate_san_type", result.Error); + } + + [Fact] + public async Task ValidateAsync_AllowsBindingWithinRotationGrace() + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.test") + }; + options.Security.SenderConstraints.Mtls.Enabled = true; + options.Security.SenderConstraints.Mtls.RequireChainValidation = false; + options.Security.SenderConstraints.Mtls.RotationGrace = TimeSpan.FromMinutes(5); + options.Signing.ActiveKeyId = "test-key"; + options.Signing.KeyPath = "/tmp/test-key.pem"; + options.Storage.ConnectionString = "mongodb://localhost/test"; + + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=mtls-client", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + var sanBuilder = new SubjectAlternativeNameBuilder(); + sanBuilder.AddDnsName("client.mtls.test"); + request.CertificateExtensions.Add(sanBuilder.Build()); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddMinutes(10)); + + var thumbprint = Convert.ToHexString(certificate.GetCertHash(HashAlgorithmName.SHA256)); + + var clientDocument = CreateClient(); + clientDocument.SenderConstraint = AuthoritySenderConstraintKinds.Mtls; + clientDocument.CertificateBindings.Add(new AuthorityClientCertificateBinding + { + Thumbprint = thumbprint, + NotBefore = TimeProvider.System.GetUtcNow().AddMinutes(2) + }); + + var httpContext = new DefaultHttpContext(); + httpContext.Connection.ClientCertificate = certificate; + + var validator = new AuthorityClientCertificateValidator(options, TimeProvider.System, NullLogger.Instance); + var result = await validator.ValidateAsync(httpContext, clientDocument, CancellationToken.None); + + Assert.True(result.Succeeded); + Assert.Equal(thumbprint, result.HexThumbprint); + } +} + internal sealed class TestClientStore : IAuthorityClientStore { private readonly Dictionary clients = new(StringComparer.OrdinalIgnoreCase); @@ -360,19 +840,19 @@ internal sealed class TestClientStore : IAuthorityClientStore } } - public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask FindByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) { clients.TryGetValue(clientId, out var document); return ValueTask.FromResult(document); } - public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken) + public ValueTask UpsertAsync(AuthorityClientDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { clients[document.ClientId] = document; return ValueTask.CompletedTask; } - public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken) + public ValueTask DeleteByClientIdAsync(string clientId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(clients.Remove(clientId)); } @@ -382,28 +862,28 @@ internal sealed class TestTokenStore : IAuthorityTokenStore public Func? UsageCallback { get; set; } - public ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken) + public ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) { Inserted = document; return ValueTask.CompletedTask; } - public ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken) + public ValueTask FindByTokenIdAsync(string tokenId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(Inserted is not null && string.Equals(Inserted.TokenId, tokenId, StringComparison.OrdinalIgnoreCase) ? Inserted : null); - public ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken) + public ValueTask FindByReferenceIdAsync(string referenceId, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(null); - public ValueTask UpdateStatusAsync(string tokenId, string status, DateTimeOffset? revokedAt, string? reason, string? reasonDescription, IReadOnlyDictionary? metadata, CancellationToken cancellationToken) + public ValueTask UpdateStatusAsync(string tokenId, string status, DateTimeOffset? revokedAt, string? reason, string? reasonDescription, IReadOnlyDictionary? metadata, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.CompletedTask; - public ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken) + public ValueTask DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(0L); - public ValueTask RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken) + public ValueTask RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult(UsageCallback?.Invoke(remoteAddress, userAgent) ?? new TokenUsageUpdateResult(TokenUsageUpdateStatus.Recorded, remoteAddress, userAgent)); - public ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken) + public ValueTask> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken, IClientSessionHandle? session = null) => ValueTask.FromResult>(Array.Empty()); } @@ -516,17 +996,39 @@ internal sealed class TestRateLimiterMetadataAccessor : IAuthorityRateLimiterMet public void SetTag(string name, string? value) => metadata.SetTag(name, value); } +internal sealed class NoopCertificateValidator : IAuthorityClientCertificateValidator +{ + public ValueTask ValidateAsync(HttpContext httpContext, AuthorityClientDocument client, CancellationToken cancellationToken) + { + var binding = new AuthorityClientCertificateBinding + { + Thumbprint = "stub" + }; + + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Success("stub", "stub", binding)); + } +} + +internal sealed class NullMongoSessionAccessor : IAuthorityMongoSessionAccessor +{ + public ValueTask GetSessionAsync(CancellationToken cancellationToken = default) + => ValueTask.FromResult(null!); + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} + internal static class TestHelpers { public static AuthorityClientDocument CreateClient( string? secret = "s3cr3t!", string clientType = "confidential", string allowedGrantTypes = "client_credentials", - string allowedScopes = "jobs:read") + string allowedScopes = "jobs:read", + string allowedAudiences = "") { - return new AuthorityClientDocument + var document = new AuthorityClientDocument { - ClientId = "feedser", + ClientId = "concelier", ClientType = clientType, SecretHash = secret is null ? null : AuthoritySecretHasher.ComputeHash(secret), Plugin = "standard", @@ -536,12 +1038,20 @@ internal static class TestHelpers [AuthorityClientMetadataKeys.AllowedScopes] = allowedScopes } }; + + if (!string.IsNullOrWhiteSpace(allowedAudiences)) + { + document.Properties[AuthorityClientMetadataKeys.Audiences] = allowedAudiences; + } + + return document; } public static AuthorityClientDescriptor CreateDescriptor(AuthorityClientDocument document) { var allowedGrantTypes = document.Properties.TryGetValue(AuthorityClientMetadataKeys.AllowedGrantTypes, out var grants) ? grants?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); var allowedScopes = document.Properties.TryGetValue(AuthorityClientMetadataKeys.AllowedScopes, out var scopes) ? scopes?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); + var allowedAudiences = document.Properties.TryGetValue(AuthorityClientMetadataKeys.Audiences, out var audiences) ? audiences?.Split(' ', StringSplitOptions.RemoveEmptyEntries) : Array.Empty(); return new AuthorityClientDescriptor( document.ClientId, @@ -549,6 +1059,7 @@ internal static class TestHelpers confidential: string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase), allowedGrantTypes, allowedScopes, + allowedAudiences, redirectUris: Array.Empty(), postLogoutRedirectUris: Array.Empty(), properties: document.Properties); @@ -620,6 +1131,57 @@ internal static class TestHelpers }; } + public static string ConvertThumbprintToString(object thumbprint) + => thumbprint switch + { + string value => value, + byte[] bytes => Base64UrlEncoder.Encode(bytes), + _ => throw new InvalidOperationException("Unsupported thumbprint representation.") + }; + + public static string CreateDpopProof(ECDsaSecurityKey key, string method, string url, long issuedAt, string? nonce = null) + { + var jwk = JsonWebKeyConverter.ConvertFromECDsaSecurityKey(key); + jwk.KeyId ??= key.KeyId ?? Guid.NewGuid().ToString("N"); + + var signingCredentials = new SigningCredentials(key, SecurityAlgorithms.EcdsaSha256); + var header = new JwtHeader(signingCredentials) + { + ["typ"] = "dpop+jwt", + ["jwk"] = new Dictionary + { + ["kty"] = jwk.Kty, + ["crv"] = jwk.Crv, + ["x"] = jwk.X, + ["y"] = jwk.Y, + ["kid"] = jwk.Kid ?? jwk.KeyId + } + }; + + var payload = new JwtPayload + { + ["htm"] = method.ToUpperInvariant(), + ["htu"] = url, + ["iat"] = issuedAt, + ["jti"] = Guid.NewGuid().ToString("N") + }; + + if (!string.IsNullOrWhiteSpace(nonce)) + { + payload["nonce"] = nonce; + } + + var token = new JwtSecurityToken(header, payload); + return new JwtSecurityTokenHandler().WriteToken(token); + } + + public static X509Certificate2 CreateTestCertificate(string subjectName) + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest(subjectName, rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + return request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + } + public static ClaimsPrincipal CreatePrincipal(string clientId, string tokenId, string provider, string? subject = null) { var identity = new ClaimsIdentity(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs index bf7e8dc5..63722c92 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/OpenIddict/TokenPersistenceIntegrationTests.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Security.Claims; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; @@ -16,9 +17,11 @@ using StellaOps.Authority.Storage.Mongo; using StellaOps.Authority.Storage.Mongo.Documents; using StellaOps.Authority.Storage.Mongo.Extensions; using StellaOps.Authority.Storage.Mongo.Initialization; +using StellaOps.Authority.Storage.Mongo.Sessions; using StellaOps.Authority.Storage.Mongo.Stores; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Testing; using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Security; using StellaOps.Cryptography.Audit; using Xunit; @@ -59,9 +62,11 @@ public sealed class TokenPersistenceIntegrationTests var authSink = new TestAuthEventSink(); var metadataAccessor = new TestRateLimiterMetadataAccessor(); - var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, NullLogger.Instance); - var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, clock, TestActivitySource, NullLogger.Instance); - var persistHandler = new PersistTokensHandler(tokenStore, clock, TestActivitySource, NullLogger.Instance); + await using var scope = provider.CreateAsyncScope(); + var sessionAccessor = scope.ServiceProvider.GetRequiredService(); + var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, new NoopCertificateValidator(), new HttpContextAccessor(), NullLogger.Instance); + var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); + var persistHandler = new PersistTokensHandler(tokenStore, sessionAccessor, clock, TestActivitySource, NullLogger.Instance); var transaction = TestHelpers.CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:trigger"); transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(15); @@ -151,8 +156,11 @@ public sealed class TokenPersistenceIntegrationTests var metadataAccessor = new TestRateLimiterMetadataAccessor(); var auditSink = new TestAuthEventSink(); + await using var scope = provider.CreateAsyncScope(); + var sessionAccessor = scope.ServiceProvider.GetRequiredService(); var handler = new ValidateAccessTokenHandler( tokenStore, + sessionAccessor, clientStore, registry, metadataAccessor, @@ -249,6 +257,107 @@ public sealed class TokenPersistenceIntegrationTests }); } + [Fact] + public async Task MongoSessions_ProvideReadYourWriteAfterPrimaryElection() + { + await ResetCollectionsAsync(); + + var clock = new FakeTimeProvider(DateTimeOffset.UtcNow); + await using var provider = await BuildMongoProviderAsync(clock); + + var tokenStore = provider.GetRequiredService(); + + await using var scope = provider.CreateAsyncScope(); + var sessionAccessor = scope.ServiceProvider.GetRequiredService(); + var session = await sessionAccessor.GetSessionAsync(CancellationToken.None); + + var tokenId = $"election-token-{Guid.NewGuid():N}"; + var document = new AuthorityTokenDocument + { + TokenId = tokenId, + Type = OpenIddictConstants.TokenTypeHints.AccessToken, + SubjectId = "session-subject", + ClientId = "session-client", + Scope = new List { "jobs:read" }, + Status = "valid", + CreatedAt = clock.GetUtcNow(), + ExpiresAt = clock.GetUtcNow().AddMinutes(30) + }; + + await tokenStore.InsertAsync(document, CancellationToken.None, session); + + await StepDownPrimaryAsync(fixture.Client, CancellationToken.None); + + AuthorityTokenDocument? fetched = null; + for (var attempt = 0; attempt < 5; attempt++) + { + try + { + fetched = await tokenStore.FindByTokenIdAsync(tokenId, CancellationToken.None, session); + if (fetched is not null) + { + break; + } + } + catch (MongoException) + { + await Task.Delay(250); + } + } + + Assert.NotNull(fetched); + Assert.Equal(tokenId, fetched!.TokenId); + } + + private static async Task StepDownPrimaryAsync(IMongoClient client, CancellationToken cancellationToken) + { + var admin = client.GetDatabase("admin"); + try + { + var command = new BsonDocument + { + { "replSetStepDown", 5 }, + { "force", true } + }; + + await admin.RunCommandAsync(command, cancellationToken: cancellationToken); + } + catch (MongoCommandException) + { + // Expected when the current primary steps down. + } + catch (MongoConnectionException) + { + // Connection may drop during election; ignore and continue. + } + + await WaitForPrimaryAsync(admin, cancellationToken); + } + + private static async Task WaitForPrimaryAsync(IMongoDatabase adminDatabase, CancellationToken cancellationToken) + { + for (var attempt = 0; attempt < 40; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + var status = await adminDatabase.RunCommandAsync(new BsonDocument { { "replSetGetStatus", 1 } }, cancellationToken: cancellationToken); + if (status.TryGetValue("myState", out var state) && state.ToInt32() == 1) + { + return; + } + } + catch (MongoCommandException) + { + // Ignore intermediate states and retry. + } + + await Task.Delay(250, cancellationToken); + } + + throw new TimeoutException("Replica set primary election did not complete in time."); + } + private async Task ResetCollectionsAsync() { var tokens = fixture.Database.GetCollection(AuthorityMongoDefaults.Collections.Tokens); diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs index d7dc1f8c..8c480130 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/RateLimiting/AuthorityRateLimiterIntegrationTests.cs @@ -1,141 +1,141 @@ -using System; -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using Microsoft.AspNetCore.Builder; -using Microsoft.AspNetCore.Hosting; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.TestHost; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Authority.RateLimiting; -using StellaOps.Configuration; -using Xunit; - -namespace StellaOps.Authority.Tests.RateLimiting; - -public class AuthorityRateLimiterIntegrationTests -{ - [Fact] - public async Task TokenEndpoint_Returns429_WhenLimitExceeded() - { - using var server = CreateServer(options => - { - options.Security.RateLimiting.Token.PermitLimit = 1; - options.Security.RateLimiting.Token.QueueLimit = 0; - options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); - }); - - using var client = server.CreateClient(); - client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.50"); - - var firstResponse = await client.PostAsync("/token", CreateTokenForm("feedser")); - Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); - - var secondResponse = await client.PostAsync("/token", CreateTokenForm("feedser")); - Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); - Assert.NotNull(secondResponse.Headers.RetryAfter); - } - - - [Fact] - public async Task TokenEndpoint_AllowsDifferentClientIdsWithinWindow() - { - using var server = CreateServer(options => - { - options.Security.RateLimiting.Token.PermitLimit = 1; - options.Security.RateLimiting.Token.QueueLimit = 0; - options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); - }); - - using var client = server.CreateClient(); - client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.70"); - - var firstResponse = await client.PostAsync("/token", CreateTokenForm("alpha-client")); - Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); - - var secondResponse = await client.PostAsync("/token", CreateTokenForm("beta-client")); - Assert.Equal(HttpStatusCode.OK, secondResponse.StatusCode); - } - - [Fact] - public async Task InternalEndpoint_Returns429_WhenLimitExceeded() - { - using var server = CreateServer(options => - { - options.Security.RateLimiting.Internal.Enabled = true; - options.Security.RateLimiting.Internal.PermitLimit = 1; - options.Security.RateLimiting.Internal.QueueLimit = 0; - options.Security.RateLimiting.Internal.Window = TimeSpan.FromSeconds(15); - }); - - using var client = server.CreateClient(); - client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.60"); - - var firstResponse = await client.GetAsync("/internal/ping"); - Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); - - var secondResponse = await client.GetAsync("/internal/ping"); - Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); - } - - private static TestServer CreateServer(Action? configure) - { - var options = new StellaOpsAuthorityOptions - { - Issuer = new Uri("https://authority.integration.test"), - SchemaVersion = 1 - }; - options.Storage.ConnectionString = "mongodb://localhost/authority"; - - configure?.Invoke(options); - - var builder = new WebHostBuilder() - .ConfigureServices(services => - { - services.AddSingleton(options); - services.AddSingleton>(Options.Create(options)); - services.TryAddSingleton(_ => TimeProvider.System); - services.AddHttpContextAccessor(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.AddRateLimiter(rateLimiterOptions => - { - AuthorityRateLimiter.Configure(rateLimiterOptions, options); - }); - }) - .Configure(app => - { - app.UseAuthorityRateLimiterContext(); - app.UseRateLimiter(); - - app.Map("/token", builder => - { - builder.Run(async context => - { - context.Response.StatusCode = StatusCodes.Status200OK; - await context.Response.WriteAsync("token-ok"); - }); - }); - - app.Map("/internal/ping", builder => - { - builder.Run(async context => - { - context.Response.StatusCode = StatusCodes.Status200OK; - await context.Response.WriteAsync("internal-ok"); - }); - }); - }); - - return new TestServer(builder); - } - - private static FormUrlEncodedContent CreateTokenForm(string clientId) - => new(new Dictionary - { - ["grant_type"] = "client_credentials", - ["client_id"] = clientId - }); -} +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Authority.RateLimiting; +using StellaOps.Configuration; +using Xunit; + +namespace StellaOps.Authority.Tests.RateLimiting; + +public class AuthorityRateLimiterIntegrationTests +{ + [Fact] + public async Task TokenEndpoint_Returns429_WhenLimitExceeded() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Token.PermitLimit = 1; + options.Security.RateLimiting.Token.QueueLimit = 0; + options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.50"); + + var firstResponse = await client.PostAsync("/token", CreateTokenForm("concelier")); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.PostAsync("/token", CreateTokenForm("concelier")); + Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); + Assert.NotNull(secondResponse.Headers.RetryAfter); + } + + + [Fact] + public async Task TokenEndpoint_AllowsDifferentClientIdsWithinWindow() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Token.PermitLimit = 1; + options.Security.RateLimiting.Token.QueueLimit = 0; + options.Security.RateLimiting.Token.Window = TimeSpan.FromSeconds(30); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.70"); + + var firstResponse = await client.PostAsync("/token", CreateTokenForm("alpha-client")); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.PostAsync("/token", CreateTokenForm("beta-client")); + Assert.Equal(HttpStatusCode.OK, secondResponse.StatusCode); + } + + [Fact] + public async Task InternalEndpoint_Returns429_WhenLimitExceeded() + { + using var server = CreateServer(options => + { + options.Security.RateLimiting.Internal.Enabled = true; + options.Security.RateLimiting.Internal.PermitLimit = 1; + options.Security.RateLimiting.Internal.QueueLimit = 0; + options.Security.RateLimiting.Internal.Window = TimeSpan.FromSeconds(15); + }); + + using var client = server.CreateClient(); + client.DefaultRequestHeaders.Add("X-Forwarded-For", "198.51.100.60"); + + var firstResponse = await client.GetAsync("/internal/ping"); + Assert.Equal(HttpStatusCode.OK, firstResponse.StatusCode); + + var secondResponse = await client.GetAsync("/internal/ping"); + Assert.Equal(HttpStatusCode.TooManyRequests, secondResponse.StatusCode); + } + + private static TestServer CreateServer(Action? configure) + { + var options = new StellaOpsAuthorityOptions + { + Issuer = new Uri("https://authority.integration.test"), + SchemaVersion = 1 + }; + options.Storage.ConnectionString = "mongodb://localhost/authority"; + + configure?.Invoke(options); + + var builder = new WebHostBuilder() + .ConfigureServices(services => + { + services.AddSingleton(options); + services.AddSingleton>(Options.Create(options)); + services.TryAddSingleton(_ => TimeProvider.System); + services.AddHttpContextAccessor(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddRateLimiter(rateLimiterOptions => + { + AuthorityRateLimiter.Configure(rateLimiterOptions, options); + }); + }) + .Configure(app => + { + app.UseAuthorityRateLimiterContext(); + app.UseRateLimiter(); + + app.Map("/token", builder => + { + builder.Run(async context => + { + context.Response.StatusCode = StatusCodes.Status200OK; + await context.Response.WriteAsync("token-ok"); + }); + }); + + app.Map("/internal/ping", builder => + { + builder.Run(async context => + { + context.Response.StatusCode = StatusCodes.Status200OK; + await context.Response.WriteAsync("internal-ok"); + }); + }); + }); + + return new TestServer(builder); + } + + private static FormUrlEncodedContent CreateTokenForm(string clientId) + => new(new Dictionary + { + ["grant_type"] = "client_credentials", + ["client_id"] = clientId + }); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj b/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj index ce847ce2..bd00628f 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority.Tests/StellaOps.Authority.Tests.csproj @@ -1,12 +1,16 @@ - - - net10.0 - enable - enable - false - - - - - - + + + net10.0 + enable + enable + false + + + + + + + + + + diff --git a/src/StellaOps.Authority/StellaOps.Authority.sln b/src/StellaOps.Authority/StellaOps.Authority.sln index 69348851..08e7b319 100644 --- a/src/StellaOps.Authority/StellaOps.Authority.sln +++ b/src/StellaOps.Authority/StellaOps.Authority.sln @@ -25,17 +25,17 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", " EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions.Tests", "StellaOps.Authority.Plugins.Abstractions.Tests\StellaOps.Authority.Plugins.Abstractions.Tests.csproj", "{EE97137B-22AF-4A84-9F65-9B4C6468B3CF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "..\StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Testing", "..\StellaOps.Concelier.Testing\StellaOps.Concelier.Testing.csproj", "{D48E48BF-80C8-43DA-8BE6-E2B9E769C49E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "..\StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common", "..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj", "{E0B9CD7A-C4FF-44EB-BE04-9B998C1C4166}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "..\StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{67C85AC6-1670-4A0D-A81F-6015574F46C7}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{67C85AC6-1670-4A0D-A81F-6015574F46C7}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "..\StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{17829125-C0F5-47E6-A16C-EC142BD58220}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "..\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{17829125-C0F5-47E6-A16C-EC142BD58220}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "..\StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{9B4BA030-C979-4191-8B4F-7E2AD9F88A94}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{9B4BA030-C979-4191-8B4F-7E2AD9F88A94}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "..\StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{26B58A9B-DB0B-4E3D-9827-3722859E5FB4}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Normalization", "..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj", "{26B58A9B-DB0B-4E3D-9827-3722859E5FB4}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Tests", "StellaOps.Authority.Tests\StellaOps.Authority.Tests.csproj", "{D719B01C-2424-4DAB-94B9-C9B6004F450B}" EndProject @@ -55,6 +55,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Test EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.DependencyInjection", "..\StellaOps.Cryptography.DependencyInjection\StellaOps.Cryptography.DependencyInjection.csproj", "{159A9B4E-61F8-4A82-8F6E-D01E3FB7E18F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Security", "..\StellaOps.Auth.Security\StellaOps.Auth.Security.csproj", "{ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -377,6 +379,18 @@ Global {159A9B4E-61F8-4A82-8F6E-D01E3FB7E18F}.Release|x64.Build.0 = Release|Any CPU {159A9B4E-61F8-4A82-8F6E-D01E3FB7E18F}.Release|x86.ActiveCfg = Release|Any CPU {159A9B4E-61F8-4A82-8F6E-D01E3FB7E18F}.Release|x86.Build.0 = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|x64.ActiveCfg = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|x64.Build.0 = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|x86.ActiveCfg = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Debug|x86.Build.0 = Debug|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|Any CPU.Build.0 = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|x64.ActiveCfg = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|x64.Build.0 = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|x86.ActiveCfg = Release|Any CPU + {ACEFD2D2-D4B9-47FB-91F2-1EA94C28D93C}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs index a6524532..1bcfb431 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Bootstrap/BootstrapRequests.cs @@ -1,75 +1,121 @@ -using System.ComponentModel.DataAnnotations; - -namespace StellaOps.Authority.Bootstrap; - -internal sealed record BootstrapUserRequest -{ - public string? Provider { get; init; } - - public string? InviteToken { get; init; } - - [Required] - public string Username { get; init; } = string.Empty; - - [Required] - public string Password { get; init; } = string.Empty; - - public string? DisplayName { get; init; } - - public string? Email { get; init; } - - public bool RequirePasswordReset { get; init; } - - public IReadOnlyCollection? Roles { get; init; } - - public IReadOnlyDictionary? Attributes { get; init; } -} - -internal sealed record BootstrapClientRequest -{ - public string? Provider { get; init; } - - public string? InviteToken { get; init; } - - [Required] - public string ClientId { get; init; } = string.Empty; - - public bool Confidential { get; init; } = true; - - public string? DisplayName { get; init; } - - public string? ClientSecret { get; init; } - - public IReadOnlyCollection? AllowedGrantTypes { get; init; } - - public IReadOnlyCollection? AllowedScopes { get; init; } - - public IReadOnlyCollection? RedirectUris { get; init; } - - public IReadOnlyCollection? PostLogoutRedirectUris { get; init; } - - public IReadOnlyDictionary? Properties { get; init; } -} - -internal sealed record BootstrapInviteRequest -{ - public string Type { get; init; } = BootstrapInviteTypes.User; - - public string? Token { get; init; } - - public string? Provider { get; init; } - - public string? Target { get; init; } - - public DateTimeOffset? ExpiresAt { get; init; } - - public string? IssuedBy { get; init; } - - public IReadOnlyDictionary? Metadata { get; init; } -} - -internal static class BootstrapInviteTypes -{ - public const string User = "user"; - public const string Client = "client"; -} +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Authority.Bootstrap; + +internal sealed record BootstrapUserRequest +{ + public string? Provider { get; init; } + + public string? InviteToken { get; init; } + + [Required] + public string Username { get; init; } = string.Empty; + + [Required] + public string Password { get; init; } = string.Empty; + + public string? DisplayName { get; init; } + + public string? Email { get; init; } + + public bool RequirePasswordReset { get; init; } + + public IReadOnlyCollection? Roles { get; init; } + + public IReadOnlyDictionary? Attributes { get; init; } +} + +internal sealed record BootstrapClientRequest +{ + public string? Provider { get; init; } + + public string? InviteToken { get; init; } + + [Required] + public string ClientId { get; init; } = string.Empty; + + public bool Confidential { get; init; } = true; + + public string? DisplayName { get; init; } + + public string? ClientSecret { get; init; } + + public IReadOnlyCollection? AllowedGrantTypes { get; init; } + + public IReadOnlyCollection? AllowedScopes { get; init; } + + public IReadOnlyCollection? AllowedAudiences { get; init; } + + public IReadOnlyCollection? RedirectUris { get; init; } + + public IReadOnlyCollection? PostLogoutRedirectUris { get; init; } + + public IReadOnlyDictionary? Properties { get; init; } + + public IReadOnlyCollection? CertificateBindings { get; init; } +} + +internal sealed record BootstrapInviteRequest +{ + public string Type { get; init; } = BootstrapInviteTypes.User; + + public string? Token { get; init; } + + public string? Provider { get; init; } + + public string? Target { get; init; } + + public DateTimeOffset? ExpiresAt { get; init; } + + public string? IssuedBy { get; init; } + + public IReadOnlyDictionary? Metadata { get; init; } +} + +internal sealed record BootstrapClientCertificateBinding +{ + public string Thumbprint { get; init; } = string.Empty; + + public string? SerialNumber { get; init; } + + public string? Subject { get; init; } + + public string? Issuer { get; init; } + + public IReadOnlyCollection? SubjectAlternativeNames { get; init; } + + public DateTimeOffset? NotBefore { get; init; } + + public DateTimeOffset? NotAfter { get; init; } + + public string? Label { get; init; } +} + +internal static class BootstrapInviteTypes +{ + public const string User = "user"; + public const string Client = "client"; +} + +internal sealed record BootstrapInviteRequest +{ + public string Type { get; init; } = BootstrapInviteTypes.User; + + public string? Token { get; init; } + + public string? Provider { get; init; } + + public string? Target { get; init; } + + public DateTimeOffset? ExpiresAt { get; init; } + + public string? IssuedBy { get; init; } + + public IReadOnlyDictionary? Metadata { get; init; } +} + +internal static class BootstrapInviteTypes +{ + public const string User = "user"; + public const string Client = "client"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs index 94e39a4e..883ede91 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/AuthorityOpenIddictConstants.cs @@ -1,18 +1,28 @@ -namespace StellaOps.Authority.OpenIddict; - -internal static class AuthorityOpenIddictConstants -{ - internal const string ProviderParameterName = "authority_provider"; - internal const string ProviderTransactionProperty = "authority:identity_provider"; - internal const string ClientTransactionProperty = "authority:client"; - internal const string ClientProviderTransactionProperty = "authority:client_provider"; - internal const string ClientGrantedScopesProperty = "authority:client_granted_scopes"; - internal const string TokenTransactionProperty = "authority:token"; - internal const string AuditCorrelationProperty = "authority:audit_correlation_id"; - internal const string AuditClientIdProperty = "authority:audit_client_id"; - internal const string AuditProviderProperty = "authority:audit_provider"; - internal const string AuditConfidentialProperty = "authority:audit_confidential"; - internal const string AuditRequestedScopesProperty = "authority:audit_requested_scopes"; - internal const string AuditGrantedScopesProperty = "authority:audit_granted_scopes"; - internal const string AuditInvalidScopeProperty = "authority:audit_invalid_scope"; -} +namespace StellaOps.Authority.OpenIddict; + +internal static class AuthorityOpenIddictConstants +{ + internal const string ProviderParameterName = "authority_provider"; + internal const string ProviderTransactionProperty = "authority:identity_provider"; + internal const string ClientTransactionProperty = "authority:client"; + internal const string ClientProviderTransactionProperty = "authority:client_provider"; + internal const string ClientGrantedScopesProperty = "authority:client_granted_scopes"; + internal const string TokenTransactionProperty = "authority:token"; + internal const string AuditCorrelationProperty = "authority:audit_correlation_id"; + internal const string AuditClientIdProperty = "authority:audit_client_id"; + internal const string AuditProviderProperty = "authority:audit_provider"; + internal const string AuditConfidentialProperty = "authority:audit_confidential"; + internal const string AuditRequestedScopesProperty = "authority:audit_requested_scopes"; + internal const string AuditGrantedScopesProperty = "authority:audit_granted_scopes"; + internal const string AuditInvalidScopeProperty = "authority:audit_invalid_scope"; + internal const string ClientSenderConstraintProperty = "authority:client_sender_constraint"; + internal const string SenderConstraintProperty = "authority:sender_constraint"; + internal const string DpopKeyThumbprintProperty = "authority:dpop_thumbprint"; + internal const string DpopProofJwtIdProperty = "authority:dpop_jti"; + internal const string DpopIssuedAtProperty = "authority:dpop_iat"; + internal const string DpopConsumedNonceProperty = "authority:dpop_nonce"; + internal const string ConfirmationClaimType = "cnf"; + internal const string SenderConstraintClaimType = "authority_sender_constraint"; + internal const string MtlsCertificateThumbprintProperty = "authority:mtls_thumbprint"; + internal const string MtlsCertificateHexProperty = "authority:mtls_thumbprint_hex"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs index 8ee2ac5c..92c5acf5 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/ClientCredentialsHandlers.cs @@ -1,487 +1,674 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using System.Security.Claims; -using System.Security.Cryptography; -using Microsoft.Extensions.Logging; -using OpenIddict.Abstractions; -using OpenIddict.Extensions; -using OpenIddict.Server; -using OpenIddict.Server.AspNetCore; -using StellaOps.Auth.Abstractions; -using StellaOps.Authority.OpenIddict; -using StellaOps.Authority.Plugins.Abstractions; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Stores; -using StellaOps.Authority.RateLimiting; -using StellaOps.Cryptography.Audit; - -namespace StellaOps.Authority.OpenIddict.Handlers; - -internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandler -{ - private readonly IAuthorityClientStore clientStore; - private readonly IAuthorityIdentityProviderRegistry registry; - private readonly ActivitySource activitySource; - private readonly IAuthEventSink auditSink; - private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; - private readonly TimeProvider timeProvider; - private readonly ILogger logger; - - public ValidateClientCredentialsHandler( - IAuthorityClientStore clientStore, - IAuthorityIdentityProviderRegistry registry, - ActivitySource activitySource, - IAuthEventSink auditSink, - IAuthorityRateLimiterMetadataAccessor metadataAccessor, - TimeProvider timeProvider, - ILogger logger) - { - this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); - this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); - this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); - this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) - { - ArgumentNullException.ThrowIfNull(context); - - if (!context.Request.IsClientCredentialsGrantType()) - { - return; - } - - using var activity = activitySource.StartActivity("authority.token.validate_client_credentials", ActivityKind.Internal); - activity?.SetTag("authority.endpoint", "/token"); - activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); - activity?.SetTag("authority.client_id", context.ClientId ?? string.Empty); - - ClientCredentialsAuditHelper.EnsureCorrelationId(context.Transaction); - - var metadata = metadataAccessor.GetMetadata(); - var clientId = context.ClientId ?? context.Request.ClientId; - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditClientIdProperty] = clientId; - if (!string.IsNullOrWhiteSpace(clientId)) - { - metadataAccessor.SetClientId(clientId); - } - - var requestedScopeInput = context.Request.GetScopes(); - var requestedScopes = requestedScopeInput.IsDefaultOrEmpty ? Array.Empty() : requestedScopeInput.ToArray(); - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditRequestedScopesProperty] = requestedScopes; - - var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedClientCredentialsParameters(context.Request); - if (unexpectedParameters.Count > 0) - { - var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString(); - var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord( - timeProvider, - context.Transaction, - metadata, - clientId, - providerHint, - confidential: null, - unexpectedParameters); - - await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false); - } - - try - { - if (string.IsNullOrWhiteSpace(context.ClientId)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client identifier is required."); - logger.LogWarning("Client credentials validation failed: missing client identifier."); - return; - } - - var document = await clientStore.FindByClientIdAsync(context.ClientId, context.CancellationToken).ConfigureAwait(false); - if (document is null || document.Disabled) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Unknown or disabled client identifier."); - logger.LogWarning("Client credentials validation failed for {ClientId}: client not found or disabled.", context.ClientId); - return; - } - - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditConfidentialProperty] = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); - - IIdentityProviderPlugin? provider = null; - if (!string.IsNullOrWhiteSpace(document.Plugin)) - { - if (!registry.TryGet(document.Plugin, out provider)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); - logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} unavailable.", context.ClientId, document.Plugin); - return; - } - - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditProviderProperty] = provider.Name; - - if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) - { - context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Associated identity provider does not support client provisioning."); - logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} lacks client provisioning capabilities.", context.ClientId, provider.Name); - return; - } - } - - var allowedGrantTypes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); - if (allowedGrantTypes.Count > 0 && - !allowedGrantTypes.Any(static grant => string.Equals(grant, OpenIddictConstants.GrantTypes.ClientCredentials, StringComparison.Ordinal))) - { - context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Client credentials grant is not permitted for this client."); - logger.LogWarning("Client credentials validation failed for {ClientId}: grant type not allowed.", document.ClientId); - return; - } - - var requiresSecret = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); - if (requiresSecret) - { - if (string.IsNullOrWhiteSpace(document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client secret is not configured."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret not configured.", document.ClientId); - return; - } - - if (string.IsNullOrWhiteSpace(context.ClientSecret) || - !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); - return; - } - } - else if (!string.IsNullOrWhiteSpace(context.ClientSecret) && !string.IsNullOrWhiteSpace(document.SecretHash) && - !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); - logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); - return; - } - - var allowedScopes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); - var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes( - allowedScopes, - context.Request.GetScopes()); - - if (resolvedScopes.InvalidScope is not null) - { - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = resolvedScopes.InvalidScope; - context.Reject(OpenIddictConstants.Errors.InvalidScope, $"Scope '{resolvedScopes.InvalidScope}' is not allowed for this client."); - logger.LogWarning("Client credentials validation failed for {ClientId}: scope {Scope} not permitted.", document.ClientId, resolvedScopes.InvalidScope); - return; - } - - context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = resolvedScopes.Scopes; - - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; - if (provider is not null) - { - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = provider.Name; - activity?.SetTag("authority.identity_provider", provider.Name); - } - - context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; - logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); - } - finally - { - var outcome = context.IsRejected ? AuthEventOutcome.Failure : AuthEventOutcome.Success; - var reason = context.IsRejected ? context.ErrorDescription : null; - var auditClientId = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditClientIdProperty, out var clientValue) - ? clientValue as string - : clientId; - var providerName = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditProviderProperty, out var providerValue) - ? providerValue as string - : null; - var confidentialValue = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditConfidentialProperty, out var confidentialValueObj) && confidentialValueObj is bool conf - ? (bool?)conf - : null; - var requested = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditRequestedScopesProperty, out var requestedValue) && requestedValue is string[] requestedArray - ? (IReadOnlyList)requestedArray - : requestedScopes; - var granted = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditGrantedScopesProperty, out var grantedValue) && grantedValue is string[] grantedArray - ? (IReadOnlyList)grantedArray - : Array.Empty(); - var invalidScope = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditInvalidScopeProperty, out var invalidValue) - ? invalidValue as string - : null; - - var record = ClientCredentialsAuditHelper.CreateRecord( - timeProvider, - context.Transaction, - metadata, - null, - outcome, - reason, - auditClientId, - providerName, - confidentialValue, - requested, - granted, - invalidScope); - - await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); - } - } -} - -internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler -{ - private readonly IAuthorityIdentityProviderRegistry registry; - private readonly IAuthorityTokenStore tokenStore; - private readonly TimeProvider clock; - private readonly ActivitySource activitySource; - private readonly ILogger logger; - - public HandleClientCredentialsHandler( - IAuthorityIdentityProviderRegistry registry, - IAuthorityTokenStore tokenStore, - TimeProvider clock, - ActivitySource activitySource, - ILogger logger) - { - this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); - this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask HandleAsync(OpenIddictServerEvents.HandleTokenRequestContext context) - { - ArgumentNullException.ThrowIfNull(context); - - if (!context.Request.IsClientCredentialsGrantType()) - { - return; - } - - using var activity = activitySource.StartActivity("authority.token.handle_client_credentials", ActivityKind.Internal); - - if (!context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTransactionProperty, out var value) || - value is not AuthorityClientDocument document) - { - context.Reject(OpenIddictConstants.Errors.ServerError, "Client metadata not available."); - return; - } - - var identity = new ClaimsIdentity(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme); - identity.AddClaim(new Claim(OpenIddictConstants.Claims.Subject, document.ClientId)); - identity.AddClaim(new Claim(OpenIddictConstants.Claims.ClientId, document.ClientId)); - activity?.SetTag("authority.client_id", document.ClientId); - activity?.SetTag("authority.endpoint", "/token"); - activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); - - var tokenId = identity.GetClaim(OpenIddictConstants.Claims.JwtId); - if (string.IsNullOrEmpty(tokenId)) - { - tokenId = Guid.NewGuid().ToString("N"); - identity.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); - } - - identity.SetDestinations(static claim => claim.Type switch - { - OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken }, - OpenIddictConstants.Claims.ClientId => new[] { OpenIddictConstants.Destinations.AccessToken }, - OpenIddictConstants.Claims.JwtId => new[] { OpenIddictConstants.Destinations.AccessToken }, - StellaOpsClaimTypes.IdentityProvider => new[] { OpenIddictConstants.Destinations.AccessToken }, - _ => new[] { OpenIddictConstants.Destinations.AccessToken } - }); - - var (provider, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); - if (context.IsRejected) - { - logger.LogWarning("Client credentials request rejected for {ClientId} during provider resolution.", document.ClientId); - return; - } - - if (provider is null) - { - if (!string.IsNullOrWhiteSpace(document.Plugin)) - { - identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, document.Plugin); - activity?.SetTag("authority.identity_provider", document.Plugin); - } - } - else - { - identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, provider.Name); - activity?.SetTag("authority.identity_provider", provider.Name); - } - - var principal = new ClaimsPrincipal(identity); - - var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var scopesValue) && - scopesValue is IReadOnlyList resolvedScopes - ? resolvedScopes - : ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); - - if (grantedScopes.Count > 0) - { - principal.SetScopes(grantedScopes); - } - else - { - principal.SetScopes(Array.Empty()); - } - - if (provider is not null && descriptor is not null) - { - var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); - await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); - } - - await PersistTokenAsync(context, document, tokenId, grantedScopes, activity).ConfigureAwait(false); - - context.Principal = principal; - context.HandleRequest(); - logger.LogInformation("Issued client credentials access token for {ClientId} with scopes {Scopes}.", document.ClientId, grantedScopes); - } - - private async ValueTask<(IIdentityProviderPlugin? Provider, AuthorityClientDescriptor? Descriptor)> ResolveProviderAsync( - OpenIddictServerEvents.HandleTokenRequestContext context, - AuthorityClientDocument document) - { - string? providerName = null; - if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientProviderTransactionProperty, out var providerValue) && - providerValue is string storedProvider) - { - providerName = storedProvider; - } - else if (!string.IsNullOrWhiteSpace(document.Plugin)) - { - providerName = document.Plugin; - } - - if (string.IsNullOrWhiteSpace(providerName)) - { - return (null, null); - } - - if (!registry.TryGet(providerName, out var provider) || provider.ClientProvisioning is null) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); - return (null, null); - } - - var descriptor = await provider.ClientProvisioning.FindByClientIdAsync(document.ClientId, context.CancellationToken).ConfigureAwait(false); - - if (descriptor is null) - { - context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client registration was not found."); - return (null, null); - } - - return (provider, descriptor); - } - - private async ValueTask PersistTokenAsync( - OpenIddictServerEvents.HandleTokenRequestContext context, - AuthorityClientDocument document, - string tokenId, - IReadOnlyCollection scopes, - Activity? activity) - { - if (context.IsRejected) - { - return; - } - - var issuedAt = clock.GetUtcNow(); - var lifetime = context.Options?.AccessTokenLifetime; - var expiresAt = lifetime.HasValue && lifetime.Value > TimeSpan.Zero - ? issuedAt + lifetime.Value - : (DateTimeOffset?)null; - - var record = new AuthorityTokenDocument - { - TokenId = tokenId, - Type = OpenIddictConstants.TokenTypeHints.AccessToken, - SubjectId = document.ClientId, - ClientId = document.ClientId, - Scope = scopes.Count > 0 ? scopes.ToList() : new List(), - Status = "valid", - CreatedAt = issuedAt, - ExpiresAt = expiresAt - }; - - await tokenStore.InsertAsync(record, context.CancellationToken).ConfigureAwait(false); - context.Transaction.Properties[AuthorityOpenIddictConstants.TokenTransactionProperty] = record; - activity?.SetTag("authority.token_id", tokenId); - } -} - -internal static class ClientCredentialHandlerHelpers -{ - public static IReadOnlyList Split(IReadOnlyDictionary properties, string key) - { - if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) - { - return Array.Empty(); - } - - return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - } - - public static (string[] Scopes, string? InvalidScope) ResolveGrantedScopes( - IReadOnlyCollection allowedScopes, - IReadOnlyList requestedScopes) - { - if (allowedScopes.Count == 0) - { - return (requestedScopes.Count == 0 ? Array.Empty() : requestedScopes.ToArray(), null); - } - - var allowed = new HashSet(allowedScopes, StringComparer.Ordinal); - - if (requestedScopes.Count == 0) - { - return (allowedScopes.ToArray(), null); - } - - foreach (var scope in requestedScopes) - { - if (!allowed.Contains(scope)) - { - return (Array.Empty(), scope); - } - } - - return (requestedScopes.ToArray(), null); - } - - public static bool VerifySecret(string secret, string storedHash) - { - ArgumentException.ThrowIfNullOrWhiteSpace(secret); - - if (string.IsNullOrWhiteSpace(storedHash)) - { - return false; - } - - try - { - var computed = Convert.FromBase64String(AuthoritySecretHasher.ComputeHash(secret)); - var expected = Convert.FromBase64String(storedHash); - - if (computed.Length != expected.Length) - { - return false; - } - - return CryptographicOperations.FixedTimeEquals(computed, expected); - } - catch (FormatException) - { - return false; - } - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using MongoDB.Driver; +using StellaOps.Auth.Abstractions; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Security; +using StellaOps.Cryptography.Audit; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityClientStore clientStore; + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly ActivitySource activitySource; + private readonly IAuthEventSink auditSink; + private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; + private readonly TimeProvider timeProvider; + private readonly IAuthorityClientCertificateValidator certificateValidator; + private readonly IHttpContextAccessor httpContextAccessor; + private readonly ILogger logger; + + public ValidateClientCredentialsHandler( + IAuthorityClientStore clientStore, + IAuthorityIdentityProviderRegistry registry, + ActivitySource activitySource, + IAuthEventSink auditSink, + IAuthorityRateLimiterMetadataAccessor metadataAccessor, + TimeProvider timeProvider, + IAuthorityClientCertificateValidator certificateValidator, + IHttpContextAccessor httpContextAccessor, + ILogger logger) + { + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); + this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.certificateValidator = certificateValidator ?? throw new ArgumentNullException(nameof(certificateValidator)); + this.httpContextAccessor = httpContextAccessor ?? throw new ArgumentNullException(nameof(httpContextAccessor)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsClientCredentialsGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.validate_client_credentials", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); + activity?.SetTag("authority.client_id", context.ClientId ?? string.Empty); + + ClientCredentialsAuditHelper.EnsureCorrelationId(context.Transaction); + + var metadata = metadataAccessor.GetMetadata(); + var clientId = context.ClientId ?? context.Request.ClientId; + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditClientIdProperty] = clientId; + if (!string.IsNullOrWhiteSpace(clientId)) + { + metadataAccessor.SetClientId(clientId); + } + + var requestedScopeInput = context.Request.GetScopes(); + var requestedScopes = requestedScopeInput.IsDefaultOrEmpty ? Array.Empty() : requestedScopeInput.ToArray(); + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditRequestedScopesProperty] = requestedScopes; + + var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedClientCredentialsParameters(context.Request); + if (unexpectedParameters.Count > 0) + { + var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString(); + var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord( + timeProvider, + context.Transaction, + metadata, + clientId, + providerHint, + confidential: null, + unexpectedParameters); + + await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false); + } + + try + { + if (string.IsNullOrWhiteSpace(context.ClientId)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client identifier is required."); + logger.LogWarning("Client credentials validation failed: missing client identifier."); + return; + } + + var document = await clientStore.FindByClientIdAsync(context.ClientId, context.CancellationToken).ConfigureAwait(false); + if (document is null || document.Disabled) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Unknown or disabled client identifier."); + logger.LogWarning("Client credentials validation failed for {ClientId}: client not found or disabled.", context.ClientId); + return; + } + + var existingSenderConstraint = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.SenderConstraintProperty, out var senderConstraintValue) && senderConstraintValue is string existingConstraint + ? existingConstraint + : null; + + var normalizedSenderConstraint = !string.IsNullOrWhiteSpace(existingSenderConstraint) + ? existingSenderConstraint + : ClientCredentialHandlerHelpers.NormalizeSenderConstraint(document); + + if (!string.IsNullOrWhiteSpace(normalizedSenderConstraint)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientSenderConstraintProperty] = normalizedSenderConstraint; + } + + if (string.Equals(normalizedSenderConstraint, AuthoritySenderConstraintKinds.Mtls, StringComparison.Ordinal)) + { + var httpContext = httpContextAccessor.HttpContext; + if (httpContext is null) + { + context.Reject(OpenIddictConstants.Errors.ServerError, "HTTP context unavailable for mTLS validation."); + logger.LogWarning("Client credentials validation failed for {ClientId}: HTTP context unavailable for mTLS validation.", context.ClientId); + return; + } + + var validation = await certificateValidator.ValidateAsync(httpContext, document, context.CancellationToken).ConfigureAwait(false); + if (!validation.Succeeded) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, validation.Error ?? "Client certificate validation failed."); + logger.LogWarning("Client credentials validation failed for {ClientId}: {Reason}.", context.ClientId, validation.Error ?? "certificate_invalid"); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.SenderConstraintProperty] = AuthoritySenderConstraintKinds.Mtls; + context.Transaction.Properties[AuthorityOpenIddictConstants.MtlsCertificateThumbprintProperty] = validation.ConfirmationThumbprint; + context.Transaction.Properties[AuthorityOpenIddictConstants.MtlsCertificateHexProperty] = validation.HexThumbprint; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditConfidentialProperty] = + string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); + + IIdentityProviderPlugin? provider = null; + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + if (!registry.TryGet(document.Plugin, out provider)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} unavailable.", context.ClientId, document.Plugin); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditProviderProperty] = provider.Name; + + if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Associated identity provider does not support client provisioning."); + logger.LogWarning("Client credentials validation failed for {ClientId}: provider {Provider} lacks client provisioning capabilities.", context.ClientId, provider.Name); + return; + } + } + + var allowedGrantTypes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedGrantTypes); + if (allowedGrantTypes.Count > 0 && + !allowedGrantTypes.Any(static grant => string.Equals(grant, OpenIddictConstants.GrantTypes.ClientCredentials, StringComparison.Ordinal))) + { + context.Reject(OpenIddictConstants.Errors.UnauthorizedClient, "Client credentials grant is not permitted for this client."); + logger.LogWarning("Client credentials validation failed for {ClientId}: grant type not allowed.", document.ClientId); + return; + } + + var requiresSecret = string.Equals(document.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); + if (requiresSecret) + { + if (string.IsNullOrWhiteSpace(document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client secret is not configured."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret not configured.", document.ClientId); + return; + } + + if (string.IsNullOrWhiteSpace(context.ClientSecret) || + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } + } + else if (!string.IsNullOrWhiteSpace(context.ClientSecret) && !string.IsNullOrWhiteSpace(document.SecretHash) && + !ClientCredentialHandlerHelpers.VerifySecret(context.ClientSecret, document.SecretHash)) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Invalid client credentials."); + logger.LogWarning("Client credentials validation failed for {ClientId}: secret verification failed.", document.ClientId); + return; + } + + var allowedScopes = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + var resolvedScopes = ClientCredentialHandlerHelpers.ResolveGrantedScopes( + allowedScopes, + context.Request.GetScopes()); + + if (resolvedScopes.InvalidScope is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditInvalidScopeProperty] = resolvedScopes.InvalidScope; + context.Reject(OpenIddictConstants.Errors.InvalidScope, $"Scope '{resolvedScopes.InvalidScope}' is not allowed for this client."); + logger.LogWarning("Client credentials validation failed for {ClientId}: scope {Scope} not permitted.", document.ClientId, resolvedScopes.InvalidScope); + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditGrantedScopesProperty] = resolvedScopes.Scopes; + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; + if (provider is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = provider.Name; + activity?.SetTag("authority.identity_provider", provider.Name); + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = resolvedScopes.Scopes; + logger.LogInformation("Client credentials validated for {ClientId}.", document.ClientId); + } + finally + { + var outcome = context.IsRejected ? AuthEventOutcome.Failure : AuthEventOutcome.Success; + var reason = context.IsRejected ? context.ErrorDescription : null; + var auditClientId = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditClientIdProperty, out var clientValue) + ? clientValue as string + : clientId; + var providerName = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditProviderProperty, out var providerValue) + ? providerValue as string + : null; + var confidentialValue = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditConfidentialProperty, out var confidentialValueObj) && confidentialValueObj is bool conf + ? (bool?)conf + : null; + var requested = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditRequestedScopesProperty, out var requestedValue) && requestedValue is string[] requestedArray + ? (IReadOnlyList)requestedArray + : requestedScopes; + var granted = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditGrantedScopesProperty, out var grantedValue) && grantedValue is string[] grantedArray + ? (IReadOnlyList)grantedArray + : Array.Empty(); + var invalidScope = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditInvalidScopeProperty, out var invalidValue) + ? invalidValue as string + : null; + + var record = ClientCredentialsAuditHelper.CreateRecord( + timeProvider, + context.Transaction, + metadata, + null, + outcome, + reason, + auditClientId, + providerName, + confidentialValue, + requested, + granted, + invalidScope); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + } + } +} + +internal sealed class HandleClientCredentialsHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityIdentityProviderRegistry registry; + private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityMongoSessionAccessor sessionAccessor; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public HandleClientCredentialsHandler( + IAuthorityIdentityProviderRegistry registry, + IAuthorityTokenStore tokenStore, + IAuthorityMongoSessionAccessor sessionAccessor, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.HandleTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsClientCredentialsGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.handle_client_credentials", ActivityKind.Internal); + + if (!context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTransactionProperty, out var value) || + value is not AuthorityClientDocument document) + { + context.Reject(OpenIddictConstants.Errors.ServerError, "Client metadata not available."); + return; + } + + var configuredAudiences = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.Audiences); + if (configuredAudiences.Count > 0) + { + if (context.Request.Resources is ICollection resources && configuredAudiences.Count > 0) + { + foreach (var audience in configuredAudiences) + { + if (!resources.Contains(audience)) + { + resources.Add(audience); + } + } + } + + if (context.Request.Audiences is ICollection audiencesCollection) + { + foreach (var audience in configuredAudiences) + { + if (!audiencesCollection.Contains(audience)) + { + audiencesCollection.Add(audience); + } + } + } + } + + var identity = new ClaimsIdentity(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.Subject, document.ClientId)); + identity.AddClaim(new Claim(OpenIddictConstants.Claims.ClientId, document.ClientId)); + activity?.SetTag("authority.client_id", document.ClientId); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); + + var tokenId = identity.GetClaim(OpenIddictConstants.Claims.JwtId); + if (string.IsNullOrEmpty(tokenId)) + { + tokenId = Guid.NewGuid().ToString("N"); + identity.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); + } + + identity.SetDestinations(static claim => claim.Type switch + { + OpenIddictConstants.Claims.Subject => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.ClientId => new[] { OpenIddictConstants.Destinations.AccessToken }, + OpenIddictConstants.Claims.JwtId => new[] { OpenIddictConstants.Destinations.AccessToken }, + StellaOpsClaimTypes.IdentityProvider => new[] { OpenIddictConstants.Destinations.AccessToken }, + _ => new[] { OpenIddictConstants.Destinations.AccessToken } + }); + + var (provider, descriptor) = await ResolveProviderAsync(context, document).ConfigureAwait(false); + if (context.IsRejected) + { + logger.LogWarning("Client credentials request rejected for {ClientId} during provider resolution.", document.ClientId); + return; + } + + if (provider is null) + { + if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, document.Plugin); + activity?.SetTag("authority.identity_provider", document.Plugin); + } + } + else + { + identity.SetClaim(StellaOpsClaimTypes.IdentityProvider, provider.Name); + activity?.SetTag("authority.identity_provider", provider.Name); + } + + ApplySenderConstraintClaims(context, identity, document); + + var principal = new ClaimsPrincipal(identity); + + var grantedScopes = context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientGrantedScopesProperty, out var scopesValue) && + scopesValue is IReadOnlyList resolvedScopes + ? resolvedScopes + : ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.AllowedScopes); + + if (grantedScopes.Count > 0) + { + principal.SetScopes(grantedScopes); + } + else + { + principal.SetScopes(Array.Empty()); + } + + if (configuredAudiences.Count > 0) + { + principal.SetAudiences(configuredAudiences); + } + + if (provider is not null && descriptor is not null) + { + var enrichmentContext = new AuthorityClaimsEnrichmentContext(provider.Context, user: null, descriptor); + await provider.ClaimsEnricher.EnrichAsync(identity, enrichmentContext, context.CancellationToken).ConfigureAwait(false); + } + + var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); + await PersistTokenAsync(context, document, tokenId, grantedScopes, session, activity).ConfigureAwait(false); + + context.Principal = principal; + context.HandleRequest(); + logger.LogInformation("Issued client credentials access token for {ClientId} with scopes {Scopes}.", document.ClientId, grantedScopes); + } + + private async ValueTask<(IIdentityProviderPlugin? Provider, AuthorityClientDescriptor? Descriptor)> ResolveProviderAsync( + OpenIddictServerEvents.HandleTokenRequestContext context, + AuthorityClientDocument document) + { + string? providerName = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientProviderTransactionProperty, out var providerValue) && + providerValue is string storedProvider) + { + providerName = storedProvider; + } + else if (!string.IsNullOrWhiteSpace(document.Plugin)) + { + providerName = document.Plugin; + } + + if (string.IsNullOrWhiteSpace(providerName)) + { + return (null, null); + } + + if (!registry.TryGet(providerName, out var provider) || provider.ClientProvisioning is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Configured identity provider is unavailable."); + return (null, null); + } + + var descriptor = await provider.ClientProvisioning.FindByClientIdAsync(document.ClientId, context.CancellationToken).ConfigureAwait(false); + + if (descriptor is null) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, "Client registration was not found."); + return (null, null); + } + + return (provider, descriptor); + } + + private async ValueTask PersistTokenAsync( + OpenIddictServerEvents.HandleTokenRequestContext context, + AuthorityClientDocument document, + string tokenId, + IReadOnlyCollection scopes, + IClientSessionHandle session, + Activity? activity) + { + if (context.IsRejected) + { + return; + } + + var issuedAt = clock.GetUtcNow(); + var lifetime = context.Options?.AccessTokenLifetime; + var expiresAt = lifetime.HasValue && lifetime.Value > TimeSpan.Zero + ? issuedAt + lifetime.Value + : (DateTimeOffset?)null; + + var record = new AuthorityTokenDocument + { + TokenId = tokenId, + Type = OpenIddictConstants.TokenTypeHints.AccessToken, + SubjectId = document.ClientId, + ClientId = document.ClientId, + Scope = scopes.Count > 0 ? scopes.ToList() : new List(), + Status = "valid", + CreatedAt = issuedAt, + ExpiresAt = expiresAt + }; + + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.SenderConstraintProperty, out var constraintObj) && + constraintObj is string senderConstraint && + !string.IsNullOrWhiteSpace(senderConstraint)) + { + record.SenderConstraint = senderConstraint; + } + + string? senderThumbprint = null; + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.DpopKeyThumbprintProperty, out var dpopThumbprintObj) && + dpopThumbprintObj is string dpopThumbprint && + !string.IsNullOrWhiteSpace(dpopThumbprint)) + { + senderThumbprint = dpopThumbprint; + } + else if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.MtlsCertificateThumbprintProperty, out var mtlsThumbprintObj) && + mtlsThumbprintObj is string mtlsThumbprint && + !string.IsNullOrWhiteSpace(mtlsThumbprint)) + { + senderThumbprint = mtlsThumbprint; + } + + if (senderThumbprint is not null) + { + record.SenderKeyThumbprint = senderThumbprint; + } + + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.DpopConsumedNonceProperty, out var nonceObj) && + nonceObj is string nonce && + !string.IsNullOrWhiteSpace(nonce)) + { + record.SenderNonce = nonce; + } + + await tokenStore.InsertAsync(record, context.CancellationToken, session).ConfigureAwait(false); + context.Transaction.Properties[AuthorityOpenIddictConstants.TokenTransactionProperty] = record; + activity?.SetTag("authority.token_id", tokenId); + } + + private static void ApplySenderConstraintClaims( + OpenIddictServerEvents.HandleTokenRequestContext context, + ClaimsIdentity identity, + AuthorityClientDocument document) + { + _ = document; + + if (!context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.SenderConstraintProperty, out var constraintObj) || + constraintObj is not string senderConstraint || + string.IsNullOrWhiteSpace(senderConstraint)) + { + return; + } + + var normalized = senderConstraint.Trim().ToLowerInvariant(); + context.Transaction.Properties[AuthorityOpenIddictConstants.SenderConstraintProperty] = normalized; + identity.SetClaim(AuthorityOpenIddictConstants.SenderConstraintClaimType, normalized); + + switch (normalized) + { + case AuthoritySenderConstraintKinds.Dpop: + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.DpopKeyThumbprintProperty, out var thumbprintObj) && + thumbprintObj is string thumbprint && + !string.IsNullOrWhiteSpace(thumbprint)) + { + var confirmation = JsonSerializer.Serialize(new Dictionary + { + ["jkt"] = thumbprint + }); + + identity.SetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType, confirmation); + } + + break; + case AuthoritySenderConstraintKinds.Mtls: + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.MtlsCertificateThumbprintProperty, out var mtlsThumbprintObj) && + mtlsThumbprintObj is string mtlsThumbprint && + !string.IsNullOrWhiteSpace(mtlsThumbprint)) + { + var confirmation = JsonSerializer.Serialize(new Dictionary + { + ["x5t#S256"] = mtlsThumbprint + }); + + identity.SetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType, confirmation); + } + + break; + } + } + +} + +internal static class ClientCredentialHandlerHelpers +{ + public static IReadOnlyList Split(IReadOnlyDictionary properties, string key) + { + if (!properties.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + return value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + public static (string[] Scopes, string? InvalidScope) ResolveGrantedScopes( + IReadOnlyCollection allowedScopes, + IReadOnlyList requestedScopes) + { + if (allowedScopes.Count == 0) + { + return (requestedScopes.Count == 0 ? Array.Empty() : requestedScopes.ToArray(), null); + } + + var allowed = new HashSet(allowedScopes, StringComparer.Ordinal); + + if (requestedScopes.Count == 0) + { + return (allowedScopes.ToArray(), null); + } + + foreach (var scope in requestedScopes) + { + if (!allowed.Contains(scope)) + { + return (Array.Empty(), scope); + } + } + + return (requestedScopes.ToArray(), null); + } + + public static bool VerifySecret(string secret, string storedHash) + { + ArgumentException.ThrowIfNullOrWhiteSpace(secret); + + if (string.IsNullOrWhiteSpace(storedHash)) + { + return false; + } + + try + { + var computed = Convert.FromBase64String(AuthoritySecretHasher.ComputeHash(secret)); + var expected = Convert.FromBase64String(storedHash); + + if (computed.Length != expected.Length) + { + return false; + } + + return CryptographicOperations.FixedTimeEquals(computed, expected); + } + catch (FormatException) + { + return false; + } + } + + public static string? NormalizeSenderConstraint(AuthorityClientDocument document) + { + if (!string.IsNullOrWhiteSpace(document.SenderConstraint)) + { + return document.SenderConstraint.Trim().ToLowerInvariant(); + } + + if (document.Properties.TryGetValue(AuthorityClientMetadataKeys.SenderConstraint, out var value) && + !string.IsNullOrWhiteSpace(value)) + { + return value.Trim().ToLowerInvariant(); + } + + return null; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs new file mode 100644 index 00000000..c17e9349 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/DpopHandlers.cs @@ -0,0 +1,643 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using OpenIddict.Server.AspNetCore; +using StellaOps.Auth.Security.Dpop; +using StellaOps.Authority.OpenIddict; +using StellaOps.Authority.RateLimiting; +using StellaOps.Authority.Security; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Stores; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Configuration; +using StellaOps.Cryptography.Audit; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class ValidateDpopProofHandler : IOpenIddictServerHandler +{ + private readonly StellaOpsAuthorityOptions authorityOptions; + private readonly IAuthorityClientStore clientStore; + private readonly IDpopProofValidator proofValidator; + private readonly IDpopNonceStore nonceStore; + private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; + private readonly IAuthEventSink auditSink; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public ValidateDpopProofHandler( + StellaOpsAuthorityOptions authorityOptions, + IAuthorityClientStore clientStore, + IDpopProofValidator proofValidator, + IDpopNonceStore nonceStore, + IAuthorityRateLimiterMetadataAccessor metadataAccessor, + IAuthEventSink auditSink, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.authorityOptions = authorityOptions ?? throw new ArgumentNullException(nameof(authorityOptions)); + this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); + this.proofValidator = proofValidator ?? throw new ArgumentNullException(nameof(proofValidator)); + this.nonceStore = nonceStore ?? throw new ArgumentNullException(nameof(nonceStore)); + this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); + this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.ValidateTokenRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (!context.Request.IsClientCredentialsGrantType()) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.validate_dpop", ActivityKind.Internal); + activity?.SetTag("authority.endpoint", "/token"); + activity?.SetTag("authority.grant_type", OpenIddictConstants.GrantTypes.ClientCredentials); + + var clientId = context.ClientId ?? context.Request.ClientId; + if (string.IsNullOrWhiteSpace(clientId)) + { + return; + } + + context.Transaction.Properties[AuthorityOpenIddictConstants.AuditClientIdProperty] = clientId; + + var senderConstraintOptions = authorityOptions.Security.SenderConstraints; + AuthorityClientDocument? clientDocument = await ResolveClientAsync(context, clientId, activity, cancel: context.CancellationToken).ConfigureAwait(false); + if (clientDocument is null) + { + return; + } + + var senderConstraint = NormalizeSenderConstraint(clientDocument); + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientSenderConstraintProperty] = senderConstraint; + + if (!string.Equals(senderConstraint, AuthoritySenderConstraintKinds.Dpop, StringComparison.Ordinal)) + { + return; + } + + var configuredAudiences = EnsureRequestAudiences(context.Request, clientDocument); + + if (!senderConstraintOptions.Dpop.Enabled) + { + logger.LogError("Client {ClientId} requires DPoP but server-side configuration has DPoP disabled.", clientId); + context.Reject(OpenIddictConstants.Errors.ServerError, "DPoP authentication is not enabled."); + await WriteAuditAsync(context, clientDocument, AuthEventOutcome.Failure, "DPoP disabled server-side.", null, null, null, "authority.dpop.proof.disabled").ConfigureAwait(false); + return; + } + + metadataAccessor.SetTag("authority.sender_constraint", AuthoritySenderConstraintKinds.Dpop); + activity?.SetTag("authority.sender_constraint", AuthoritySenderConstraintKinds.Dpop); + + HttpRequest? httpRequest = null; + HttpResponse? httpResponse = null; + if (context.Transaction.Properties.TryGetValue(typeof(HttpContext).FullName!, out var httpContextProperty) && + httpContextProperty is HttpContext capturedContext) + { + httpRequest = capturedContext.Request; + httpResponse = capturedContext.Response; + } + if (httpRequest is null) + { + context.Reject(OpenIddictConstants.Errors.ServerError, "Unable to access HTTP context for DPoP validation."); + logger.LogError("DPoP validation aborted for {ClientId}: HTTP request not available via transaction.", clientId); + await WriteAuditAsync(context, clientDocument, AuthEventOutcome.Failure, "HTTP request unavailable for DPoP.", null, null, null, "authority.dpop.proof.error").ConfigureAwait(false); + return; + } + + if (!httpRequest.Headers.TryGetValue("DPoP", out StringValues proofHeader) || StringValues.IsNullOrEmpty(proofHeader)) + { + logger.LogWarning("Missing DPoP header for client credentials request from {ClientId}.", clientId); + await ChallengeNonceAsync( + context, + clientDocument, + audience: null, + thumbprint: null, + reasonCode: "missing_proof", + description: "DPoP proof is required.", + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + } + + var proof = proofHeader.ToString(); + var requestUri = BuildRequestUri(httpRequest); + + var validationResult = await proofValidator.ValidateAsync( + proof, + httpRequest.Method, + requestUri, + cancellationToken: context.CancellationToken).ConfigureAwait(false); + + if (!validationResult.IsValid) + { + var error = string.IsNullOrWhiteSpace(validationResult.ErrorDescription) + ? "DPoP proof validation failed." + : validationResult.ErrorDescription; + + logger.LogWarning("DPoP proof validation failed for client {ClientId}: {Reason}.", clientId, error); + await ChallengeNonceAsync( + context, + clientDocument, + audience: null, + thumbprint: null, + reasonCode: validationResult.ErrorCode ?? "invalid_proof", + description: error, + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + } + + if (validationResult.PublicKey is not Microsoft.IdentityModel.Tokens.JsonWebKey jwk) + { + logger.LogWarning("DPoP proof for {ClientId} did not expose a JSON Web Key.", clientId); + await ChallengeNonceAsync( + context, + clientDocument, + audience: null, + thumbprint: null, + reasonCode: "invalid_key", + description: "DPoP proof must embed a JSON Web Key.", + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + } + + object rawThumbprint = jwk.ComputeJwkThumbprint(); + string thumbprint; + if (rawThumbprint is string value && !string.IsNullOrWhiteSpace(value)) + { + thumbprint = value; + } + else if (rawThumbprint is byte[] bytes) + { + thumbprint = Base64UrlEncoder.Encode(bytes); + } + else + { + throw new InvalidOperationException("DPoP JWK thumbprint could not be computed."); + } + context.Transaction.Properties[AuthorityOpenIddictConstants.SenderConstraintProperty] = AuthoritySenderConstraintKinds.Dpop; + context.Transaction.Properties[AuthorityOpenIddictConstants.DpopKeyThumbprintProperty] = thumbprint; + if (!string.IsNullOrWhiteSpace(validationResult.JwtId)) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.DpopProofJwtIdProperty] = validationResult.JwtId; + } + + if (validationResult.IssuedAt is { } issuedAt) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.DpopIssuedAtProperty] = issuedAt; + } + + var nonceOptions = senderConstraintOptions.Dpop.Nonce; + var requiredAudience = ResolveNonceAudience(context.Request, nonceOptions, configuredAudiences); + + if (nonceOptions.Enabled && requiredAudience is not null) + { + activity?.SetTag("authority.dpop_nonce_audience", requiredAudience); + var suppliedNonce = validationResult.Nonce; + + if (string.IsNullOrWhiteSpace(suppliedNonce)) + { + logger.LogInformation("DPoP nonce challenge issued to {ClientId} for audience {Audience}: nonce missing.", clientId, requiredAudience); + await ChallengeNonceAsync( + context, + clientDocument, + requiredAudience, + thumbprint, + "nonce_missing", + "DPoP nonce is required for this audience.", + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + } + + var consumeResult = await nonceStore.TryConsumeAsync( + suppliedNonce, + requiredAudience, + clientDocument.ClientId, + thumbprint, + context.CancellationToken).ConfigureAwait(false); + + switch (consumeResult.Status) + { + case DpopNonceConsumeStatus.Success: + context.Transaction.Properties[AuthorityOpenIddictConstants.DpopConsumedNonceProperty] = suppliedNonce; + break; + case DpopNonceConsumeStatus.Expired: + logger.LogInformation("DPoP nonce expired for {ClientId} and audience {Audience}.", clientId, requiredAudience); + await ChallengeNonceAsync( + context, + clientDocument, + requiredAudience, + thumbprint, + "nonce_expired", + "DPoP nonce has expired. Retry with a fresh nonce.", + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + default: + logger.LogInformation("DPoP nonce invalid for {ClientId} and audience {Audience}.", clientId, requiredAudience); + await ChallengeNonceAsync( + context, + clientDocument, + requiredAudience, + thumbprint, + "nonce_invalid", + "DPoP nonce is invalid. Request a new nonce and retry.", + senderConstraintOptions, + httpResponse).ConfigureAwait(false); + return; + } + } + + await WriteAuditAsync( + context, + clientDocument, + AuthEventOutcome.Success, + "DPoP proof validated.", + thumbprint, + validationResult, + requiredAudience, + "authority.dpop.proof.valid") + .ConfigureAwait(false); + logger.LogInformation("DPoP proof validated for client {ClientId}.", clientId); + } + + private async ValueTask ResolveClientAsync( + OpenIddictServerEvents.ValidateTokenRequestContext context, + string clientId, + Activity? activity, + CancellationToken cancel) + { + if (context.Transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.ClientTransactionProperty, out var value) && + value is AuthorityClientDocument cached) + { + activity?.SetTag("authority.client_id", cached.ClientId); + return cached; + } + + var document = await clientStore.FindByClientIdAsync(clientId, cancel).ConfigureAwait(false); + if (document is not null) + { + context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = document; + activity?.SetTag("authority.client_id", document.ClientId); + } + + return document; + } + + private static string? NormalizeSenderConstraint(AuthorityClientDocument document) + { + if (!string.IsNullOrWhiteSpace(document.SenderConstraint)) + { + return document.SenderConstraint.Trim().ToLowerInvariant(); + } + + if (document.Properties.TryGetValue(AuthorityClientMetadataKeys.SenderConstraint, out var value) && + !string.IsNullOrWhiteSpace(value)) + { + return value.Trim().ToLowerInvariant(); + } + + return null; + } + + private static IReadOnlyList EnsureRequestAudiences(OpenIddictRequest? request, AuthorityClientDocument document) + { + if (request is null) + { + return Array.Empty(); + } + + var configuredAudiences = ClientCredentialHandlerHelpers.Split(document.Properties, AuthorityClientMetadataKeys.Audiences); + if (configuredAudiences.Count == 0) + { + return configuredAudiences; + } + + if (request.Resources is ICollection resources) + { + foreach (var audience in configuredAudiences) + { + if (!resources.Contains(audience)) + { + resources.Add(audience); + } + } + } + + if (request.Audiences is ICollection audiencesCollection) + { + foreach (var audience in configuredAudiences) + { + if (!audiencesCollection.Contains(audience)) + { + audiencesCollection.Add(audience); + } + } + } + + return configuredAudiences; + } + + private static Uri BuildRequestUri(HttpRequest request) + { + ArgumentNullException.ThrowIfNull(request); + var url = request.GetDisplayUrl(); + return new Uri(url, UriKind.Absolute); + } + +private static string? ResolveNonceAudience(OpenIddictRequest request, AuthorityDpopNonceOptions nonceOptions, IReadOnlyList configuredAudiences) + { + if (!nonceOptions.Enabled || request is null) + { + return null; + } + + if (request.Resources is not null) + { + foreach (var resource in request.Resources) + { + if (string.IsNullOrWhiteSpace(resource)) + { + continue; + } + + var normalized = resource.Trim(); + if (nonceOptions.RequiredAudiences.Contains(normalized)) + { + return normalized; + } + } + } + + if (request.Audiences is not null) + { + foreach (var audience in request.Audiences) + { + if (string.IsNullOrWhiteSpace(audience)) + { + continue; + } + + var normalized = audience.Trim(); + if (nonceOptions.RequiredAudiences.Contains(normalized)) + { + return normalized; + } + } + } + + if (configuredAudiences is { Count: > 0 }) + { + foreach (var audience in configuredAudiences) + { + if (string.IsNullOrWhiteSpace(audience)) + { + continue; + } + + var normalized = audience.Trim(); + if (nonceOptions.RequiredAudiences.Contains(normalized)) + { + return normalized; + } + } + } + + return null; + } + + private async ValueTask ChallengeNonceAsync( + OpenIddictServerEvents.ValidateTokenRequestContext context, + AuthorityClientDocument clientDocument, + string? audience, + string? thumbprint, + string reasonCode, + string description, + AuthoritySenderConstraintOptions senderConstraintOptions, + HttpResponse? httpResponse) + { + context.Reject(OpenIddictConstants.Errors.InvalidClient, description); + metadataAccessor.SetTag("authority.dpop_result", reasonCode); + + string? issuedNonce = null; + DateTimeOffset? expiresAt = null; + if (audience is not null && thumbprint is not null && senderConstraintOptions.Dpop.Nonce.Enabled) + { + var issuance = await nonceStore.IssueAsync( + audience, + clientDocument.ClientId, + thumbprint, + senderConstraintOptions.Dpop.Nonce.Ttl, + senderConstraintOptions.Dpop.Nonce.MaxIssuancePerMinute, + context.CancellationToken).ConfigureAwait(false); + + if (issuance.Status == DpopNonceIssueStatus.Success) + { + issuedNonce = issuance.Nonce; + expiresAt = issuance.ExpiresAt; + } + else + { + logger.LogWarning("Unable to issue DPoP nonce for {ClientId} (audience {Audience}): {Status}.", clientDocument.ClientId, audience, issuance.Status); + } + } + + if (httpResponse is not null) + { + httpResponse.Headers["WWW-Authenticate"] = BuildAuthenticateHeader(reasonCode, description, issuedNonce); + + if (!string.IsNullOrWhiteSpace(issuedNonce)) + { + httpResponse.Headers["DPoP-Nonce"] = issuedNonce; + } + } + + await WriteAuditAsync( + context, + clientDocument, + AuthEventOutcome.Failure, + description, + thumbprint, + validationResult: null, + audience, + "authority.dpop.proof.challenge", + reasonCode, + issuedNonce, + expiresAt) + .ConfigureAwait(false); + } + + private static string BuildAuthenticateHeader(string reasonCode, string description, string? nonce) + { + var parameters = new Dictionary + { + ["error"] = string.Equals(reasonCode, "nonce_missing", StringComparison.OrdinalIgnoreCase) + ? "use_dpop_nonce" + : "invalid_dpop_proof", + ["error_description"] = description + }; + + if (!string.IsNullOrWhiteSpace(nonce)) + { + parameters["dpop-nonce"] = nonce; + } + + var segments = new List(); + foreach (var kvp in parameters) + { + if (kvp.Value is null) + { + continue; + } + + segments.Add($"{kvp.Key}=\"{EscapeHeaderValue(kvp.Value)}\""); + } + + return segments.Count > 0 + ? $"DPoP {string.Join(", ", segments)}" + : "DPoP"; + + static string EscapeHeaderValue(string value) + => value + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal); + } + + private async ValueTask WriteAuditAsync( + OpenIddictServerEvents.ValidateTokenRequestContext context, + AuthorityClientDocument clientDocument, + AuthEventOutcome outcome, + string reason, + string? thumbprint, + DpopValidationResult? validationResult, + string? audience, + string eventType, + string? reasonCode = null, + string? issuedNonce = null, + DateTimeOffset? nonceExpiresAt = null) + { + var metadata = metadataAccessor.GetMetadata(); + var properties = new List + { + new() + { + Name = "sender.constraint", + Value = ClassifiedString.Public(AuthoritySenderConstraintKinds.Dpop) + } + }; + + if (!string.IsNullOrWhiteSpace(reasonCode)) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.reason_code", + Value = ClassifiedString.Public(reasonCode) + }); + } + + if (!string.IsNullOrWhiteSpace(thumbprint)) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.jkt", + Value = ClassifiedString.Public(thumbprint) + }); + } + + if (validationResult?.JwtId is not null) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.jti", + Value = ClassifiedString.Public(validationResult.JwtId) + }); + } + + if (validationResult?.IssuedAt is { } issuedAt) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.issued_at", + Value = ClassifiedString.Public(issuedAt.ToString("O", CultureInfo.InvariantCulture)) + }); + } + + if (audience is not null) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.audience", + Value = ClassifiedString.Public(audience) + }); + } + + if (!string.IsNullOrWhiteSpace(validationResult?.Nonce)) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.nonce.presented", + Value = ClassifiedString.Sensitive(validationResult.Nonce) + }); + } + + if (!string.IsNullOrWhiteSpace(issuedNonce)) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.nonce.issued", + Value = ClassifiedString.Sensitive(issuedNonce) + }); + } + + if (nonceExpiresAt is { } expiresAt) + { + properties.Add(new AuthEventProperty + { + Name = "dpop.nonce.expires_at", + Value = ClassifiedString.Public(expiresAt.ToString("O", CultureInfo.InvariantCulture)) + }); + } + + var confidential = string.Equals(clientDocument.ClientType, "confidential", StringComparison.OrdinalIgnoreCase); + + var record = ClientCredentialsAuditHelper.CreateRecord( + clock, + context.Transaction, + metadata, + clientSecret: null, + outcome, + reason, + clientDocument.ClientId, + providerName: clientDocument.Plugin, + confidential, + requestedScopes: Array.Empty(), + grantedScopes: Array.Empty(), + invalidScope: null, + extraProperties: properties, + eventType: eventType); + + await auditSink.WriteAsync(record, context.CancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/RevocationHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/RevocationHandlers.cs index 95efe539..5b42ae80 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/RevocationHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/RevocationHandlers.cs @@ -1,136 +1,142 @@ -using System; -using System.Diagnostics; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using OpenIddict.Abstractions; -using OpenIddict.Server; -using StellaOps.Authority.Storage.Mongo.Stores; - -namespace StellaOps.Authority.OpenIddict.Handlers; - -internal sealed class HandleRevocationRequestHandler : IOpenIddictServerHandler -{ - private readonly IAuthorityTokenStore tokenStore; - private readonly TimeProvider clock; - private readonly ILogger logger; - private readonly ActivitySource activitySource; - - public HandleRevocationRequestHandler( - IAuthorityTokenStore tokenStore, - TimeProvider clock, - ActivitySource activitySource, - ILogger logger) - { - this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask HandleAsync(OpenIddictServerEvents.HandleRevocationRequestContext context) - { - ArgumentNullException.ThrowIfNull(context); - - using var activity = activitySource.StartActivity("authority.token.revoke", ActivityKind.Internal); - - var request = context.Request; - if (request is null || string.IsNullOrWhiteSpace(request.Token)) - { - context.Reject(OpenIddictConstants.Errors.InvalidRequest, "The revocation request is missing the token parameter."); - return; - } - - var token = request.Token.Trim(); - var document = await tokenStore.FindByTokenIdAsync(token, context.CancellationToken).ConfigureAwait(false); - - if (document is null) - { - var tokenId = TryExtractTokenId(token); - if (!string.IsNullOrWhiteSpace(tokenId)) - { - document = await tokenStore.FindByTokenIdAsync(tokenId!, context.CancellationToken).ConfigureAwait(false); - } - } - - if (document is null) - { - logger.LogDebug("Revocation request for unknown token ignored."); - context.HandleRequest(); - return; - } - - if (!string.Equals(document.Status, "revoked", StringComparison.OrdinalIgnoreCase)) - { - await tokenStore.UpdateStatusAsync( - document.TokenId, - "revoked", - clock.GetUtcNow(), - "client_request", - null, - null, - context.CancellationToken).ConfigureAwait(false); - - logger.LogInformation("Token {TokenId} revoked via revocation endpoint.", document.TokenId); - activity?.SetTag("authority.token_id", document.TokenId); - } - - context.HandleRequest(); - } - - private static string? TryExtractTokenId(string token) - { - var parts = token.Split('.'); - if (parts.Length < 2) - { - return null; - } - - try - { - var payload = Base64UrlDecode(parts[1]); - using var document = JsonDocument.Parse(payload); - if (document.RootElement.TryGetProperty("jti", out var jti) && jti.ValueKind == JsonValueKind.String) - { - var value = jti.GetString(); - return string.IsNullOrWhiteSpace(value) ? null : value; - } - } - catch (JsonException) - { - return null; - } - catch (FormatException) - { - return null; - } - - return null; - } - - private static byte[] Base64UrlDecode(string value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return Array.Empty(); - } - - var remainder = value.Length % 4; - if (remainder == 2) - { - value += "=="; - } - else if (remainder == 3) - { - value += "="; - } - else if (remainder != 0) - { - value += new string('=', 4 - remainder); - } - - var padded = value.Replace('-', '+').Replace('_', '/'); - return Convert.FromBase64String(padded); - } -} +using System; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Server; +using StellaOps.Authority.Storage.Mongo.Sessions; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class HandleRevocationRequestHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityMongoSessionAccessor sessionAccessor; + private readonly TimeProvider clock; + private readonly ILogger logger; + private readonly ActivitySource activitySource; + + public HandleRevocationRequestHandler( + IAuthorityTokenStore tokenStore, + IAuthorityMongoSessionAccessor sessionAccessor, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.HandleRevocationRequestContext context) + { + ArgumentNullException.ThrowIfNull(context); + + using var activity = activitySource.StartActivity("authority.token.revoke", ActivityKind.Internal); + + var request = context.Request; + if (request is null || string.IsNullOrWhiteSpace(request.Token)) + { + context.Reject(OpenIddictConstants.Errors.InvalidRequest, "The revocation request is missing the token parameter."); + return; + } + + var token = request.Token.Trim(); + var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); + var document = await tokenStore.FindByTokenIdAsync(token, context.CancellationToken, session).ConfigureAwait(false); + + if (document is null) + { + var tokenId = TryExtractTokenId(token); + if (!string.IsNullOrWhiteSpace(tokenId)) + { + document = await tokenStore.FindByTokenIdAsync(tokenId!, context.CancellationToken, session).ConfigureAwait(false); + } + } + + if (document is null) + { + logger.LogDebug("Revocation request for unknown token ignored."); + context.HandleRequest(); + return; + } + + if (!string.Equals(document.Status, "revoked", StringComparison.OrdinalIgnoreCase)) + { + await tokenStore.UpdateStatusAsync( + document.TokenId, + "revoked", + clock.GetUtcNow(), + "client_request", + null, + null, + context.CancellationToken, + session).ConfigureAwait(false); + + logger.LogInformation("Token {TokenId} revoked via revocation endpoint.", document.TokenId); + activity?.SetTag("authority.token_id", document.TokenId); + } + + context.HandleRequest(); + } + + private static string? TryExtractTokenId(string token) + { + var parts = token.Split('.'); + if (parts.Length < 2) + { + return null; + } + + try + { + var payload = Base64UrlDecode(parts[1]); + using var document = JsonDocument.Parse(payload); + if (document.RootElement.TryGetProperty("jti", out var jti) && jti.ValueKind == JsonValueKind.String) + { + var value = jti.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + } + catch (JsonException) + { + return null; + } + catch (FormatException) + { + return null; + } + + return null; + } + + private static byte[] Base64UrlDecode(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + var remainder = value.Length % 4; + if (remainder == 2) + { + value += "=="; + } + else if (remainder == 3) + { + value += "="; + } + else if (remainder != 0) + { + value += new string('=', 4 - remainder); + } + + var padded = value.Replace('-', '+').Replace('_', '/'); + return Convert.FromBase64String(padded); + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs index ee161b56..ffc79e54 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenPersistenceHandlers.cs @@ -1,135 +1,169 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.Linq; -using System.Security.Claims; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using OpenIddict.Abstractions; -using OpenIddict.Extensions; -using OpenIddict.Server; -using StellaOps.Authority.Storage.Mongo.Documents; -using StellaOps.Authority.Storage.Mongo.Stores; - -namespace StellaOps.Authority.OpenIddict.Handlers; - -internal sealed class PersistTokensHandler : IOpenIddictServerHandler -{ - private readonly IAuthorityTokenStore tokenStore; - private readonly TimeProvider clock; - private readonly ActivitySource activitySource; - private readonly ILogger logger; - - public PersistTokensHandler( - IAuthorityTokenStore tokenStore, - TimeProvider clock, - ActivitySource activitySource, - ILogger logger) - { - this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); - this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); - this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); - this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask HandleAsync(OpenIddictServerEvents.ProcessSignInContext context) - { - ArgumentNullException.ThrowIfNull(context); - - if (context.AccessTokenPrincipal is null && - context.RefreshTokenPrincipal is null && - context.AuthorizationCodePrincipal is null && - context.DeviceCodePrincipal is null) - { - return; - } - - using var activity = activitySource.StartActivity("authority.token.persist", ActivityKind.Internal); - var issuedAt = clock.GetUtcNow(); - - if (context.AccessTokenPrincipal is ClaimsPrincipal accessPrincipal) - { - await PersistAsync(accessPrincipal, OpenIddictConstants.TokenTypeHints.AccessToken, issuedAt, context.CancellationToken).ConfigureAwait(false); - } - - if (context.RefreshTokenPrincipal is ClaimsPrincipal refreshPrincipal) - { - await PersistAsync(refreshPrincipal, OpenIddictConstants.TokenTypeHints.RefreshToken, issuedAt, context.CancellationToken).ConfigureAwait(false); - } - - if (context.AuthorizationCodePrincipal is ClaimsPrincipal authorizationPrincipal) - { - await PersistAsync(authorizationPrincipal, OpenIddictConstants.TokenTypeHints.AuthorizationCode, issuedAt, context.CancellationToken).ConfigureAwait(false); - } - - if (context.DeviceCodePrincipal is ClaimsPrincipal devicePrincipal) - { - await PersistAsync(devicePrincipal, OpenIddictConstants.TokenTypeHints.DeviceCode, issuedAt, context.CancellationToken).ConfigureAwait(false); - } - } - - private async ValueTask PersistAsync(ClaimsPrincipal principal, string tokenType, DateTimeOffset issuedAt, CancellationToken cancellationToken) - { - var tokenId = EnsureTokenId(principal); - var scopes = ExtractScopes(principal); - var document = new AuthorityTokenDocument - { - TokenId = tokenId, - Type = tokenType, - SubjectId = principal.GetClaim(OpenIddictConstants.Claims.Subject), - ClientId = principal.GetClaim(OpenIddictConstants.Claims.ClientId), - Scope = scopes, - Status = "valid", - CreatedAt = issuedAt, - ExpiresAt = TryGetExpiration(principal) - }; - - try - { - await tokenStore.InsertAsync(document, cancellationToken).ConfigureAwait(false); - logger.LogDebug("Persisted {Type} token {TokenId} for client {ClientId}.", tokenType, tokenId, document.ClientId ?? ""); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to persist {Type} token {TokenId}.", tokenType, tokenId); - } - } - - private static string EnsureTokenId(ClaimsPrincipal principal) - { - var tokenId = principal.GetClaim(OpenIddictConstants.Claims.JwtId); - if (string.IsNullOrWhiteSpace(tokenId)) - { - tokenId = Guid.NewGuid().ToString("N"); - principal.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); - } - - return tokenId; - } - - private static List ExtractScopes(ClaimsPrincipal principal) - => principal.GetScopes() - .Where(scope => !string.IsNullOrWhiteSpace(scope)) - .Select(scope => scope.Trim()) - .Distinct(StringComparer.Ordinal) - .OrderBy(scope => scope, StringComparer.Ordinal) - .ToList(); - - private static DateTimeOffset? TryGetExpiration(ClaimsPrincipal principal) - { - var value = principal.GetClaim("exp"); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) - { - return DateTimeOffset.FromUnixTimeSeconds(seconds); - } - - return null; - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using System.Security.Claims; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenIddict.Abstractions; +using OpenIddict.Extensions; +using OpenIddict.Server; +using MongoDB.Driver; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; +using StellaOps.Authority.Storage.Mongo.Stores; + +namespace StellaOps.Authority.OpenIddict.Handlers; + +internal sealed class PersistTokensHandler : IOpenIddictServerHandler +{ + private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityMongoSessionAccessor sessionAccessor; + private readonly TimeProvider clock; + private readonly ActivitySource activitySource; + private readonly ILogger logger; + + public PersistTokensHandler( + IAuthorityTokenStore tokenStore, + IAuthorityMongoSessionAccessor sessionAccessor, + TimeProvider clock, + ActivitySource activitySource, + ILogger logger) + { + this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); + this.clock = clock ?? throw new ArgumentNullException(nameof(clock)); + this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask HandleAsync(OpenIddictServerEvents.ProcessSignInContext context) + { + ArgumentNullException.ThrowIfNull(context); + + if (context.AccessTokenPrincipal is null && + context.RefreshTokenPrincipal is null && + context.AuthorizationCodePrincipal is null && + context.DeviceCodePrincipal is null) + { + return; + } + + using var activity = activitySource.StartActivity("authority.token.persist", ActivityKind.Internal); + var session = await sessionAccessor.GetSessionAsync(context.CancellationToken).ConfigureAwait(false); + var issuedAt = clock.GetUtcNow(); + + if (context.AccessTokenPrincipal is ClaimsPrincipal accessPrincipal) + { + await PersistAsync(accessPrincipal, OpenIddictConstants.TokenTypeHints.AccessToken, issuedAt, session, context.CancellationToken).ConfigureAwait(false); + } + + if (context.RefreshTokenPrincipal is ClaimsPrincipal refreshPrincipal) + { + await PersistAsync(refreshPrincipal, OpenIddictConstants.TokenTypeHints.RefreshToken, issuedAt, session, context.CancellationToken).ConfigureAwait(false); + } + + if (context.AuthorizationCodePrincipal is ClaimsPrincipal authorizationPrincipal) + { + await PersistAsync(authorizationPrincipal, OpenIddictConstants.TokenTypeHints.AuthorizationCode, issuedAt, session, context.CancellationToken).ConfigureAwait(false); + } + + if (context.DeviceCodePrincipal is ClaimsPrincipal devicePrincipal) + { + await PersistAsync(devicePrincipal, OpenIddictConstants.TokenTypeHints.DeviceCode, issuedAt, session, context.CancellationToken).ConfigureAwait(false); + } + } + + private async ValueTask PersistAsync(ClaimsPrincipal principal, string tokenType, DateTimeOffset issuedAt, IClientSessionHandle session, CancellationToken cancellationToken) + { + var tokenId = EnsureTokenId(principal); + var scopes = ExtractScopes(principal); + var document = new AuthorityTokenDocument + { + TokenId = tokenId, + Type = tokenType, + SubjectId = principal.GetClaim(OpenIddictConstants.Claims.Subject), + ClientId = principal.GetClaim(OpenIddictConstants.Claims.ClientId), + Scope = scopes, + Status = "valid", + CreatedAt = issuedAt, + ExpiresAt = TryGetExpiration(principal) + }; + + var senderConstraint = principal.GetClaim(AuthorityOpenIddictConstants.SenderConstraintClaimType); + if (!string.IsNullOrWhiteSpace(senderConstraint)) + { + document.SenderConstraint = senderConstraint; + } + + var confirmation = principal.GetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType); + if (!string.IsNullOrWhiteSpace(confirmation)) + { + try + { + using var json = JsonDocument.Parse(confirmation); + if (json.RootElement.TryGetProperty("jkt", out var thumbprintElement)) + { + document.SenderKeyThumbprint = thumbprintElement.GetString(); + } + else if (json.RootElement.TryGetProperty("x5t#S256", out var certificateThumbprintElement)) + { + document.SenderKeyThumbprint = certificateThumbprintElement.GetString(); + } + } + catch (JsonException) + { + // Ignore malformed confirmation claims in persistence layer. + } + } + + try + { + await tokenStore.InsertAsync(document, cancellationToken, session).ConfigureAwait(false); + logger.LogDebug("Persisted {Type} token {TokenId} for client {ClientId}.", tokenType, tokenId, document.ClientId ?? ""); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to persist {Type} token {TokenId}.", tokenType, tokenId); + } + } + + private static string EnsureTokenId(ClaimsPrincipal principal) + { + var tokenId = principal.GetClaim(OpenIddictConstants.Claims.JwtId); + if (string.IsNullOrWhiteSpace(tokenId)) + { + tokenId = Guid.NewGuid().ToString("N"); + principal.SetClaim(OpenIddictConstants.Claims.JwtId, tokenId); + } + + return tokenId; + } + + private static List ExtractScopes(ClaimsPrincipal principal) + => principal.GetScopes() + .Where(scope => !string.IsNullOrWhiteSpace(scope)) + .Select(scope => scope.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(scope => scope, StringComparer.Ordinal) + .ToList(); + + private static DateTimeOffset? TryGetExpiration(ClaimsPrincipal principal) + { + var value = principal.GetClaim("exp"); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) + { + return DateTimeOffset.FromUnixTimeSeconds(seconds); + } + + return null; + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs index eefab879..6a6fac9a 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/OpenIddict/Handlers/TokenValidationHandlers.cs @@ -3,23 +3,28 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Security.Claims; +using System.Text.Json; using Microsoft.Extensions.Logging; using OpenIddict.Abstractions; using OpenIddict.Extensions; using OpenIddict.Server; using StellaOps.Auth.Abstractions; +using MongoDB.Driver; using StellaOps.Authority.OpenIddict; using StellaOps.Authority.Plugins.Abstractions; using StellaOps.Authority.RateLimiting; using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Storage.Mongo.Sessions; using StellaOps.Authority.Storage.Mongo.Stores; using StellaOps.Cryptography.Audit; +using StellaOps.Authority.Security; namespace StellaOps.Authority.OpenIddict.Handlers; internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler { private readonly IAuthorityTokenStore tokenStore; + private readonly IAuthorityMongoSessionAccessor sessionAccessor; private readonly IAuthorityClientStore clientStore; private readonly IAuthorityIdentityProviderRegistry registry; private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor; @@ -30,6 +35,7 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler logger) { this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore)); + this.sessionAccessor = sessionAccessor ?? throw new ArgumentNullException(nameof(sessionAccessor)); this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore)); this.registry = registry ?? throw new ArgumentNullException(nameof(registry)); this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor)); @@ -74,10 +81,12 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler claim.Type == AuthorityOpenIddictConstants.SenderConstraintClaimType)) + { + identity.SetClaim(AuthorityOpenIddictConstants.SenderConstraintClaimType, tokenDocument.SenderConstraint); + } + + if (identity.HasClaim(claim => claim.Type == AuthorityOpenIddictConstants.ConfirmationClaimType)) + { + return; + } + + if (string.IsNullOrWhiteSpace(tokenDocument.SenderConstraint) || string.IsNullOrWhiteSpace(tokenDocument.SenderKeyThumbprint)) + { + return; + } + + string confirmation = tokenDocument.SenderConstraint switch + { + AuthoritySenderConstraintKinds.Dpop => JsonSerializer.Serialize(new Dictionary + { + ["jkt"] = tokenDocument.SenderKeyThumbprint + }), + AuthoritySenderConstraintKinds.Mtls => JsonSerializer.Serialize(new Dictionary + { + ["x5t#S256"] = tokenDocument.SenderKeyThumbprint + }), + _ => string.Empty + }; + + if (!string.IsNullOrEmpty(confirmation)) + { + identity.SetClaim(AuthorityOpenIddictConstants.ConfirmationClaimType, confirmation); + } + } } diff --git a/src/StellaOps.Authority/StellaOps.Authority/Program.cs b/src/StellaOps.Authority/StellaOps.Authority/Program.cs index 8eb43fbd..4d032efe 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/Program.cs +++ b/src/StellaOps.Authority/StellaOps.Authority/Program.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Microsoft.AspNetCore.RateLimiting; +using Microsoft.AspNetCore.Server.Kestrel.Https; using Microsoft.Extensions.Logging.Abstractions; using OpenIddict.Abstractions; using OpenIddict.Server; @@ -37,6 +38,11 @@ using StellaOps.Authority.Revocation; using StellaOps.Authority.Signing; using StellaOps.Cryptography; using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Authority.Security; +#if STELLAOPS_AUTH_SECURITY +using StellaOps.Auth.Security.Dpop; +using StackExchange.Redis; +#endif var builder = WebApplication.CreateBuilder(args); @@ -66,6 +72,15 @@ var authorityConfiguration = StellaOpsAuthorityConfiguration.Build(options => }; }); +builder.WebHost.ConfigureKestrel(options => +{ + options.ConfigureHttpsDefaults(https => + { + https.ClientCertificateMode = ClientCertificateMode.AllowCertificate; + https.CheckCertificateRevocation = true; + }); +}); + builder.Configuration.AddConfiguration(authorityConfiguration.Configuration); builder.Host.UseSerilog((context, _, loggerConfiguration) => @@ -85,6 +100,52 @@ builder.Services.AddHttpContextAccessor(); builder.Services.TryAddSingleton(_ => TimeProvider.System); builder.Services.TryAddSingleton(); builder.Services.TryAddSingleton(); +builder.Services.AddSingleton(); + +#if STELLAOPS_AUTH_SECURITY +var senderConstraints = authorityOptions.Security.SenderConstraints; + +builder.Services.AddOptions() + .Configure(options => + { + options.ProofLifetime = senderConstraints.Dpop.ProofLifetime; + options.AllowedClockSkew = senderConstraints.Dpop.AllowedClockSkew; + options.ReplayWindow = senderConstraints.Dpop.ReplayWindow; + + options.AllowedAlgorithms.Clear(); + foreach (var algorithm in senderConstraints.Dpop.NormalizedAlgorithms) + { + options.AllowedAlgorithms.Add(algorithm); + } + }) + .PostConfigure(static options => options.Validate()); + +builder.Services.TryAddSingleton(provider => new InMemoryDpopReplayCache(provider.GetService())); +builder.Services.TryAddSingleton(); +if (string.Equals(senderConstraints.Dpop.Nonce.Store, "redis", StringComparison.OrdinalIgnoreCase)) +{ + builder.Services.TryAddSingleton(_ => + ConnectionMultiplexer.Connect(senderConstraints.Dpop.Nonce.RedisConnectionString!)); + + builder.Services.TryAddSingleton(provider => + { + var multiplexer = provider.GetRequiredService(); + var timeProvider = provider.GetService(); + return new RedisDpopNonceStore(multiplexer, timeProvider); + }); +} +else +{ + builder.Services.TryAddSingleton(provider => + { + var timeProvider = provider.GetService(); + var nonceLogger = provider.GetService>(); + return new InMemoryDpopNonceStore(timeProvider, nonceLogger); + }); +} + +builder.Services.AddScoped(); +#endif builder.Services.AddRateLimiter(rateLimiterOptions => { @@ -184,6 +245,13 @@ builder.Services.AddOpenIddict() aspNetCoreBuilder.DisableTransportSecurityRequirement(); } +#if STELLAOPS_AUTH_SECURITY + options.AddEventHandler(descriptor => + { + descriptor.UseScopedHandler(); + }); +#endif + options.AddEventHandler(descriptor => { descriptor.UseScopedHandler(); @@ -688,6 +756,33 @@ if (authorityOptions.Bootstrap.Enabled) ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary(request.Properties, StringComparer.OrdinalIgnoreCase); + IReadOnlyCollection? certificateBindings = null; + if (request.CertificateBindings is not null) + { + var bindingRegistrations = new List(request.CertificateBindings.Count); + foreach (var binding in request.CertificateBindings) + { + if (binding is null || string.IsNullOrWhiteSpace(binding.Thumbprint)) + { + await ReleaseInviteAsync("Certificate binding thumbprint is required."); + await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Certificate binding thumbprint is required.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty(), request.Confidential, inviteToken).ConfigureAwait(false); + return Results.BadRequest(new { error = "invalid_request", message = "Certificate binding thumbprint is required." }); + } + + bindingRegistrations.Add(new AuthorityClientCertificateBindingRegistration( + binding.Thumbprint, + binding.SerialNumber, + binding.Subject, + binding.Issuer, + binding.SubjectAlternativeNames, + binding.NotBefore, + binding.NotAfter, + binding.Label)); + } + + certificateBindings = bindingRegistrations; + } + var registration = new AuthorityClientRegistration( request.ClientId, request.Confidential, @@ -695,9 +790,11 @@ if (authorityOptions.Bootstrap.Enabled) request.ClientSecret, request.AllowedGrantTypes ?? Array.Empty(), request.AllowedScopes ?? Array.Empty(), + request.AllowedAudiences ?? Array.Empty(), redirectUris, postLogoutUris, - properties); + properties, + certificateBindings); var result = await provider.ClientProvisioning.CreateOrUpdateAsync(registration, cancellationToken).ConfigureAwait(false); @@ -1114,7 +1211,7 @@ static PluginHostOptions BuildPluginHostOptions(StellaOpsAuthorityOptions option { BaseDirectory = basePath, PluginsDirectory = string.IsNullOrWhiteSpace(pluginDirectory) - ? Path.Combine("PluginBinaries", "Authority") + ? "StellaOps.Authority.PluginBinaries" : pluginDirectory, PrimaryPrefix = "StellaOps.Authority" }; diff --git a/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidationResult.cs b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidationResult.cs new file mode 100644 index 00000000..3f7ea122 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidationResult.cs @@ -0,0 +1,32 @@ +using System; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Security; + +internal sealed class AuthorityClientCertificateValidationResult +{ + private AuthorityClientCertificateValidationResult(bool succeeded, string? confirmationThumbprint, string? hexThumbprint, AuthorityClientCertificateBinding? binding, string? error) + { + Succeeded = succeeded; + ConfirmationThumbprint = confirmationThumbprint; + HexThumbprint = hexThumbprint; + Binding = binding; + Error = error; + } + + public bool Succeeded { get; } + + public string? ConfirmationThumbprint { get; } + + public string? HexThumbprint { get; } + + public AuthorityClientCertificateBinding? Binding { get; } + + public string? Error { get; } + + public static AuthorityClientCertificateValidationResult Success(string confirmationThumbprint, string hexThumbprint, AuthorityClientCertificateBinding binding) + => new(true, confirmationThumbprint, hexThumbprint, binding, null); + + public static AuthorityClientCertificateValidationResult Failure(string error) + => new(false, null, null, null, error); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs new file mode 100644 index 00000000..bfe0ec17 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthorityClientCertificateValidator.cs @@ -0,0 +1,283 @@ +using System; +using System.Linq; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Asn1; +using System.Net; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using StellaOps.Authority.Storage.Mongo.Documents; +using StellaOps.Configuration; +using Microsoft.IdentityModel.Tokens; + +namespace StellaOps.Authority.Security; + +internal sealed class AuthorityClientCertificateValidator : IAuthorityClientCertificateValidator +{ + private readonly StellaOpsAuthorityOptions authorityOptions; + private readonly TimeProvider timeProvider; + private readonly ILogger logger; + + public AuthorityClientCertificateValidator( + StellaOpsAuthorityOptions authorityOptions, + TimeProvider timeProvider, + ILogger logger) + { + this.authorityOptions = authorityOptions ?? throw new ArgumentNullException(nameof(authorityOptions)); + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ValueTask ValidateAsync(HttpContext httpContext, AuthorityClientDocument client, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(httpContext); + ArgumentNullException.ThrowIfNull(client); + + var certificate = httpContext.Connection.ClientCertificate; + if (certificate is null) + { + logger.LogWarning("mTLS validation failed for {ClientId}: no client certificate present.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("client_certificate_required")); + } + + var mtlsOptions = authorityOptions.Security.SenderConstraints.Mtls; + var requiresChain = mtlsOptions.RequireChainValidation || mtlsOptions.AllowedCertificateAuthorities.Count > 0; + + X509Chain? chain = null; + var chainBuilt = false; + try + { + if (requiresChain) + { + chain = CreateChain(); + chainBuilt = TryBuildChain(chain, certificate); + if (mtlsOptions.RequireChainValidation && !chainBuilt) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate chain validation failed.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_chain_invalid")); + } + } + + var now = timeProvider.GetUtcNow(); + if (now < certificate.NotBefore || now > certificate.NotAfter) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate outside validity window (notBefore={NotBefore:o}, notAfter={NotAfter:o}).", client.ClientId, certificate.NotBefore, certificate.NotAfter); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_expired")); + } + + if (mtlsOptions.NormalizedSubjectPatterns.Count > 0 && + !mtlsOptions.NormalizedSubjectPatterns.Any(pattern => pattern.IsMatch(certificate.Subject))) + { + logger.LogWarning("mTLS validation failed for {ClientId}: subject {Subject} did not match allowed patterns.", client.ClientId, certificate.Subject); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_subject_mismatch")); + } + + var subjectAlternativeNames = GetSubjectAlternativeNames(certificate); + if (mtlsOptions.AllowedSanTypes.Count > 0) + { + if (subjectAlternativeNames.Count == 0) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate does not contain subject alternative names.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_san_missing")); + } + + if (subjectAlternativeNames.Any(san => !mtlsOptions.AllowedSanTypes.Contains(san.Type))) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate SAN types [{Types}] not allowed.", client.ClientId, string.Join(",", subjectAlternativeNames.Select(san => san.Type))); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_san_type")); + } + + if (!subjectAlternativeNames.Any(san => mtlsOptions.AllowedSanTypes.Contains(san.Type))) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate SANs did not include any of the required types.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_san_missing_required")); + } + } + + if (mtlsOptions.AllowedCertificateAuthorities.Count > 0) + { + var allowedCas = mtlsOptions.AllowedCertificateAuthorities + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Select(value => value.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var matchedCa = false; + if (chainBuilt && chain is not null) + { + foreach (var element in chain.ChainElements.Cast().Skip(1)) + { + if (allowedCas.Contains(element.Certificate.Subject)) + { + matchedCa = true; + break; + } + } + } + + if (!matchedCa && allowedCas.Contains(certificate.Issuer)) + { + matchedCa = true; + } + + if (!matchedCa) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate issuer {Issuer} is not allow-listed.", client.ClientId, certificate.Issuer); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_ca_untrusted")); + } + } + + if (client.CertificateBindings.Count == 0) + { + logger.LogWarning("mTLS validation failed for {ClientId}: no certificate bindings registered for client.", client.ClientId); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_missing")); + } + + var certificateHash = certificate.GetCertHash(HashAlgorithmName.SHA256); + var hexThumbprint = Convert.ToHexString(certificateHash); + var base64Thumbprint = Base64UrlEncoder.Encode(certificateHash); + + var binding = client.CertificateBindings.FirstOrDefault(b => string.Equals(b.Thumbprint, hexThumbprint, StringComparison.OrdinalIgnoreCase)); + if (binding is null) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate thumbprint {Thumbprint} not registered.", client.ClientId, hexThumbprint); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_unbound")); + } + + if (binding.NotBefore is { } bindingNotBefore) + { + var effectiveNotBefore = bindingNotBefore - mtlsOptions.RotationGrace; + if (now < effectiveNotBefore) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate binding not active until {NotBefore:o} (grace applied).", client.ClientId, bindingNotBefore); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_inactive")); + } + } + + if (binding.NotAfter is { } bindingNotAfter) + { + var effectiveNotAfter = bindingNotAfter + mtlsOptions.RotationGrace; + if (now > effectiveNotAfter) + { + logger.LogWarning("mTLS validation failed for {ClientId}: certificate binding expired at {NotAfter:o} (grace applied).", client.ClientId, bindingNotAfter); + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Failure("certificate_binding_expired")); + } + } + + return ValueTask.FromResult(AuthorityClientCertificateValidationResult.Success(base64Thumbprint, hexThumbprint, binding)); + } + finally + { + chain?.Dispose(); + } + } + + private static X509Chain CreateChain() + => new() + { + ChainPolicy = + { + RevocationMode = X509RevocationMode.NoCheck, + RevocationFlag = X509RevocationFlag.ExcludeRoot, + VerificationFlags = X509VerificationFlags.IgnoreWrongUsage + } + }; + + private bool TryBuildChain(X509Chain chain, X509Certificate2 certificate) + { + try + { + return chain.Build(certificate); + } + catch (Exception ex) + { + logger.LogWarning(ex, "mTLS chain validation threw an exception."); + return false; + } + } + + private static IReadOnlyList<(string Type, string Value)> GetSubjectAlternativeNames(X509Certificate2 certificate) + { + foreach (var extension in certificate.Extensions) + { + if (!string.Equals(extension.Oid?.Value, "2.5.29.17", StringComparison.Ordinal)) + { + continue; + } + + try + { + var reader = new AsnReader(extension.RawData, AsnEncodingRules.DER); + var sequence = reader.ReadSequence(); + var results = new List<(string, string)>(); + + while (sequence.HasData) + { + var tag = sequence.PeekTag(); + if (tag.TagClass != TagClass.ContextSpecific) + { + sequence.ReadEncodedValue(); + continue; + } + + switch (tag.TagValue) + { + case 2: + { + var dns = sequence.ReadCharacterString(UniversalTagNumber.IA5String, new Asn1Tag(TagClass.ContextSpecific, 2)); + results.Add(("dns", dns)); + break; + } + case 6: + { + var uri = sequence.ReadCharacterString(UniversalTagNumber.IA5String, new Asn1Tag(TagClass.ContextSpecific, 6)); + results.Add(("uri", uri)); + break; + } + case 7: + { + var bytes = sequence.ReadOctetString(new Asn1Tag(TagClass.ContextSpecific, 7)); + var ip = new IPAddress(bytes).ToString(); + results.Add(("ip", ip)); + break; + } + default: + sequence.ReadEncodedValue(); + break; + } + } + + return results; + } + catch + { + return Array.Empty<(string, string)>(); + } + } + + return Array.Empty<(string, string)>(); + } + private bool ValidateCertificateChain(X509Certificate2 certificate) + { + using var chain = new X509Chain + { + ChainPolicy = + { + RevocationMode = X509RevocationMode.NoCheck, + RevocationFlag = X509RevocationFlag.ExcludeRoot, + VerificationFlags = X509VerificationFlags.IgnoreWrongUsage + } + }; + + try + { + return chain.Build(certificate); + } + catch (Exception ex) + { + logger.LogWarning(ex, "mTLS chain validation threw an exception."); + return false; + } + } +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Security/AuthoritySenderConstraintKinds.cs b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthoritySenderConstraintKinds.cs new file mode 100644 index 00000000..9d046d5b --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Security/AuthoritySenderConstraintKinds.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Authority.Security; + +/// +/// Canonical string identifiers for Authority sender-constraint policies. +/// +internal static class AuthoritySenderConstraintKinds +{ + internal const string Dpop = "dpop"; + internal const string Mtls = "mtls"; +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/Security/IAuthorityClientCertificateValidator.cs b/src/StellaOps.Authority/StellaOps.Authority/Security/IAuthorityClientCertificateValidator.cs new file mode 100644 index 00000000..35ce20a3 --- /dev/null +++ b/src/StellaOps.Authority/StellaOps.Authority/Security/IAuthorityClientCertificateValidator.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using StellaOps.Authority.Storage.Mongo.Documents; + +namespace StellaOps.Authority.Security; + +internal interface IAuthorityClientCertificateValidator +{ + ValueTask ValidateAsync(HttpContext httpContext, AuthorityClientDocument client, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj b/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj index 8b65f5b3..fd5a79b5 100644 --- a/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj +++ b/src/StellaOps.Authority/StellaOps.Authority/StellaOps.Authority.csproj @@ -1,29 +1,32 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + $(DefineConstants);STELLAOPS_AUTH_SECURITY + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Authority/TASKS.md b/src/StellaOps.Authority/TASKS.md index c7d7db2a..e3879c0d 100644 --- a/src/StellaOps.Authority/TASKS.md +++ b/src/StellaOps.Authority/TASKS.md @@ -15,10 +15,18 @@ > Remark (2025-10-14): Background sweep emits invite expiry audits; integration test added. | SEC5.HOST-REPLAY | DONE (2025-10-14) | Security Guild, Zastava | SEC5.E | Persist token usage metadata and surface suspected replay heuristics. | ✅ Validation handlers record device metadata; ✅ Suspected replay flagged via audit/logs; ✅ Tests cover regression cases. | > Remark (2025-10-14): Token validation handler logs suspected replay audits with device metadata; coverage via unit/integration tests. -| SEC3.BUILD | DONE (2025-10-11) | Authority Core, Security Guild | SEC3.HOST, FEEDMERGE-COORD-02-900 | Track normalized-range dependency fallout and restore full test matrix once Feedser range primitives land. | ✅ Feedser normalized range libraries merged; ✅ Authority + Configuration test suites (`dotnet test src/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) pass without Feedser compile failures; ✅ Status recorded here/Sprints (authority-core broadcast not available). | +| SEC3.BUILD | DONE (2025-10-11) | Authority Core, Security Guild | SEC3.HOST, FEEDMERGE-COORD-02-900 | Track normalized-range dependency fallout and restore full test matrix once Concelier range primitives land. | ✅ Concelier normalized range libraries merged; ✅ Authority + Configuration test suites (`dotnet test src/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) pass without Concelier compile failures; ✅ Status recorded here/Sprints (authority-core broadcast not available). | | AUTHCORE-BUILD-OPENIDDICT | DONE (2025-10-14) | Authority Core | SEC2.HOST | Adapt host/audit handlers for OpenIddict 6.4 API surface (no `OpenIddictServerTransaction`) and restore Authority solution build. | ✅ Build `dotnet build src/StellaOps.Authority.sln` succeeds; ✅ Audit correlation + tamper logging verified under new abstractions; ✅ Tests updated. | | AUTHCORE-STORAGE-DEVICE-TOKENS | DONE (2025-10-14) | Authority Core, Storage Guild | AUTHCORE-BUILD-OPENIDDICT | Reintroduce `AuthorityTokenDeviceDocument` + projections removed during refactor so storage layer compiles. | ✅ Document type restored with mappings/migrations; ✅ Storage tests cover device artifacts; ✅ Authority solution build green. | | AUTHCORE-BOOTSTRAP-INVITES | DONE (2025-10-14) | Authority Core, DevOps | AUTHCORE-STORAGE-DEVICE-TOKENS | Wire bootstrap invite cleanup service against restored document schema and re-enable lifecycle tests. | ✅ `BootstrapInviteCleanupService` passes integration tests; ✅ Operator guide updated if behavior changes; ✅ Build/test matrices green. | -| AUTHSTORAGE-MONGO-08-001 | TODO | Authority Core & Storage Guild | — | Harden Mongo session usage with causal consistency for mutations and follow-up reads. | • Scoped middleware/service creates `IClientSessionHandle` with causal consistency + majority read/write concerns
• Stores accept optional session parameter and reuse it for write + immediate reads
• GraphQL/HTTP pipelines updated to flow session through post-mutation queries
• Replica-set integration test exercises primary election and verifies read-your-write guarantees | +| AUTHSTORAGE-MONGO-08-001 | DONE (2025-10-19) | Authority Core & Storage Guild | — | Harden Mongo session usage with causal consistency for mutations and follow-up reads. | • Scoped middleware/service creates `IClientSessionHandle` with causal consistency + majority read/write concerns
• Stores accept optional session parameter and reuse it for write + immediate reads
• GraphQL/HTTP pipelines updated to flow session through post-mutation queries
• Replica-set integration test exercises primary election and verifies read-your-write guarantees | +| AUTH-PLUGIN-COORD-08-002 | DOING (2025-10-19) | Authority Core, Plugin Platform Guild | PLUGIN-DI-08-001 | Coordinate scoped-service adoption for Authority plug-in registrars and background jobs ahead of PLUGIN-DI-08-002 implementation. | ✅ Workshop locked for 2025-10-20 15:00–16:00 UTC; ✅ Pre-read checklist in `docs/dev/authority-plugin-di-coordination.md`; ✅ Follow-up tasks captured in module backlogs before code changes begin. | +| AUTH-DPOP-11-001 | DOING (2025-10-19) | Authority Core & Security Guild | — | Implement DPoP proof validation + nonce handling for high-value audiences per architecture. | • Proof handler validates method/uri/hash + replay; nonce issuing/consumption implemented for in-memory + Redis stores
• Client credential path stamps `cnf.jkt` and persists sender metadata
• Remaining: finalize Redis configuration surface (docs/sample config), unskip nonce-challenge regression once HTTP pipeline emits high-value audiences, refresh operator docs | +> Remark (2025-10-19): DPoP handler now seeds request resources/audiences from client metadata; nonce challenge integration test re-enabled (still requires full suite once Concelier build restored). +| AUTH-MTLS-11-002 | DOING (2025-10-19) | Authority Core & Security Guild | — | Add OAuth mTLS client credential support with certificate-bound tokens and introspection updates. | • Certificate validator scaffold plus cnf stamping present; tokens persist sender thumbprints
• Remaining: provisioning/storage for certificate bindings, SAN/CA validation, introspection propagation, integration tests/docs before marking DONE | +> Remark (2025-10-19): Client provisioning accepts certificate bindings; validator enforces SAN types/CA allow-list with rotation grace; mtls integration tests updated (full suite still blocked by upstream build). +> Remark (2025-10-19, AUTHSTORAGE-MONGO-08-001): Prerequisites re-checked (none outstanding). Session accessor wired through Authority pipeline; stores accept optional sessions; added replica-set election regression test for read-your-write. +> Remark (2025-10-19, AUTH-DPOP-11-001): Handler, nonce store, and persistence hooks merged; Redis-backed configuration + end-to-end nonce enforcement still open. Full solution test blocked by `StellaOps.Concelier.Storage.Mongo` compile errors. +> Remark (2025-10-19, AUTH-MTLS-11-002): Certificate validator + cnf stamping delivered; binding storage, CA/SAN validation, integration suites outstanding before status can move to DONE. > Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic. diff --git a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs index 31545959..7f08abb9 100644 --- a/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs +++ b/src/StellaOps.Cli.Tests/Commands/CommandHandlersTests.cs @@ -1,12 +1,16 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.IO; +using System.Linq; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; +using System.Globalization; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.IdentityModel.Tokens; @@ -19,20 +23,22 @@ using StellaOps.Cli.Services.Models; using StellaOps.Cli.Telemetry; using StellaOps.Cli.Tests.Testing; using StellaOps.Cryptography; +using Spectre.Console; +using Spectre.Console.Testing; namespace StellaOps.Cli.Tests.Commands; - -public sealed class CommandHandlersTests -{ - [Fact] - public async Task HandleExportJobAsync_SetsExitCodeZeroOnSuccess() - { - var original = Environment.ExitCode; - try - { - var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", "/jobs/export:json/1", null)); - var provider = BuildServiceProvider(backend); - + +public sealed class CommandHandlersTests +{ + [Fact] + public async Task HandleExportJobAsync_SetsExitCodeZeroOnSuccess() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", "/jobs/export:json/1", null)); + var provider = BuildServiceProvider(backend); + await CommandHandlers.HandleExportJobAsync( provider, format: "json", @@ -43,36 +49,36 @@ public sealed class CommandHandlersTests includeDelta: null, verbose: false, cancellationToken: CancellationToken.None); - - Assert.Equal(0, Environment.ExitCode); - Assert.Equal("export:json", backend.LastJobKind); - } - finally - { - Environment.ExitCode = original; - } - } - - [Fact] - public async Task HandleMergeJobAsync_SetsExitCodeOnFailure() - { - var original = Environment.ExitCode; - try - { - var backend = new StubBackendClient(new JobTriggerResult(false, "Job already running", null, null)); - var provider = BuildServiceProvider(backend); - - await CommandHandlers.HandleMergeJobAsync(provider, verbose: false, CancellationToken.None); - - Assert.Equal(1, Environment.ExitCode); - Assert.Equal("merge:reconcile", backend.LastJobKind); - } - finally - { - Environment.ExitCode = original; - } - } - + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal("export:json", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleMergeJobAsync_SetsExitCodeOnFailure() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(false, "Job already running", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleMergeJobAsync(provider, verbose: false, CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + Assert.Equal("merge:reconcile", backend.LastJobKind); + } + finally + { + Environment.ExitCode = original; + } + } + [Fact] public async Task HandleScannerRunAsync_AutomaticallyUploadsResults() { @@ -81,34 +87,34 @@ public sealed class CommandHandlersTests var backend = new StubBackendClient(new JobTriggerResult(true, "Accepted", null, null)); var metadataFile = Path.Combine(tempDir.Path, "results", "scan-run.json"); var executor = new StubExecutor(new ScannerExecutionResult(0, resultsFile, metadataFile)); - var options = new StellaOpsCliOptions - { - ResultsDirectory = Path.Combine(tempDir.Path, "results") - }; - - var provider = BuildServiceProvider(backend, executor, new StubInstaller(), options); - - Directory.CreateDirectory(Path.Combine(tempDir.Path, "target")); - - var original = Environment.ExitCode; - try - { - await CommandHandlers.HandleScannerRunAsync( - provider, - runner: "docker", - entry: "scanner-image", - targetDirectory: Path.Combine(tempDir.Path, "target"), - arguments: Array.Empty(), - verbose: false, - cancellationToken: CancellationToken.None); - - Assert.Equal(0, Environment.ExitCode); + var options = new StellaOpsCliOptions + { + ResultsDirectory = Path.Combine(tempDir.Path, "results") + }; + + var provider = BuildServiceProvider(backend, executor, new StubInstaller(), options); + + Directory.CreateDirectory(Path.Combine(tempDir.Path, "target")); + + var original = Environment.ExitCode; + try + { + await CommandHandlers.HandleScannerRunAsync( + provider, + runner: "docker", + entry: "scanner-image", + targetDirectory: Path.Combine(tempDir.Path, "target"), + arguments: Array.Empty(), + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); Assert.Equal(resultsFile, backend.LastUploadPath); Assert.True(File.Exists(metadataFile)); - } - finally - { - Environment.ExitCode = original; + } + finally + { + Environment.ExitCode = original; } } @@ -128,7 +134,7 @@ public sealed class CommandHandlersTests Url = "https://authority.example", ClientId = "cli", ClientSecret = "secret", - Scope = "feedser.jobs.trigger", + Scope = "concelier.jobs.trigger", TokenCacheDirectory = tempDir.Path } }; @@ -211,6 +217,168 @@ public sealed class CommandHandlersTests } } + [Fact] + public async Task HandleExcititorInitAsync_CallsBackend() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "accepted", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleExcititorInitAsync( + provider, + new[] { "redhat" }, + resume: true, + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal("init", backend.LastExcititorRoute); + Assert.Equal(HttpMethod.Post, backend.LastExcititorMethod); + var payload = Assert.IsAssignableFrom>(backend.LastExcititorPayload); + Assert.Equal(true, payload["resume"]); + var providers = Assert.IsAssignableFrom>(payload["providers"]!); + Assert.Contains("redhat", providers, StringComparer.OrdinalIgnoreCase); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleExcititorListProvidersAsync_WritesOutput() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)) + { + ProviderSummaries = new[] + { + new ExcititorProviderSummary("redhat", "distro", "Red Hat", "vendor", true, DateTimeOffset.UtcNow) + } + }; + + var provider = BuildServiceProvider(backend); + await CommandHandlers.HandleExcititorListProvidersAsync(provider, includeDisabled: false, verbose: false, cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleExcititorVerifyAsync_FailsWithoutArguments() + { + var original = Environment.ExitCode; + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleExcititorVerifyAsync(provider, null, null, null, verbose: false, cancellationToken: CancellationToken.None); + + Assert.Equal(1, Environment.ExitCode); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleExcititorVerifyAsync_AttachesAttestationFile() + { + var original = Environment.ExitCode; + using var tempFile = new TempFile("attestation.json", Encoding.UTF8.GetBytes("{\"ok\":true}")); + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)); + var provider = BuildServiceProvider(backend); + + await CommandHandlers.HandleExcititorVerifyAsync( + provider, + exportId: "export-123", + digest: "sha256:abc", + attestationPath: tempFile.Path, + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Equal("verify", backend.LastExcititorRoute); + var payload = Assert.IsAssignableFrom>(backend.LastExcititorPayload); + Assert.Equal("export-123", payload["exportId"]); + Assert.Equal("sha256:abc", payload["digest"]); + var attestation = Assert.IsAssignableFrom>(payload["attestation"]!); + Assert.Equal(Path.GetFileName(tempFile.Path), attestation["fileName"]); + Assert.NotNull(attestation["base64"]); + } + finally + { + Environment.ExitCode = original; + } + } + + [Fact] + public async Task HandleExcititorExportAsync_DownloadsWhenOutputProvided() + { + var original = Environment.ExitCode; + using var tempDir = new TempDirectory(); + + try + { + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)); + const string manifestJson = """ + { + "exportId": "exports/20251019T101530Z/abcdef1234567890", + "format": "openvex", + "createdAt": "2025-10-19T10:15:30Z", + "artifact": { "algorithm": "sha256", "digest": "abcdef1234567890" }, + "fromCache": false, + "sizeBytes": 2048, + "attestation": { + "rekor": { + "location": "https://rekor.example/api/v1/log/entries/123", + "logIndex": "123" + } + } + } + """; + + backend.ExcititorResult = new ExcititorOperationResult(true, "ok", null, JsonDocument.Parse(manifestJson).RootElement.Clone()); + var provider = BuildServiceProvider(backend); + var outputPath = Path.Combine(tempDir.Path, "export.json"); + + await CommandHandlers.HandleExcititorExportAsync( + provider, + format: "openvex", + delta: false, + scope: null, + since: null, + provider: null, + outputPath: outputPath, + verbose: false, + cancellationToken: CancellationToken.None); + + Assert.Equal(0, Environment.ExitCode); + Assert.Single(backend.ExportDownloads); + var request = backend.ExportDownloads[0]; + Assert.Equal("exports/20251019T101530Z/abcdef1234567890", request.ExportId); + Assert.Equal(Path.GetFullPath(outputPath), request.DestinationPath); + Assert.Equal("sha256", request.Algorithm); + Assert.Equal("abcdef1234567890", request.Digest); + } + finally + { + Environment.ExitCode = original; + } + } + [Theory] [InlineData(null)] [InlineData("default")] @@ -263,7 +431,7 @@ public sealed class CommandHandlersTests "token", "Bearer", DateTimeOffset.UtcNow.AddMinutes(30), - new[] { StellaOpsScopes.FeedserJobsTrigger }); + new[] { StellaOpsScopes.ConcelierJobsTrigger }); var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); @@ -331,13 +499,13 @@ public sealed class CommandHandlersTests tokenClient.CachedEntry = new StellaOpsTokenCacheEntry( CreateUnsignedJwt( ("sub", "cli-user"), - ("aud", "feedser"), + ("aud", "concelier"), ("iss", "https://authority.example"), ("iat", 1_700_000_000), ("nbf", 1_700_000_000)), "Bearer", DateTimeOffset.UtcNow.AddMinutes(30), - new[] { StellaOpsScopes.FeedserJobsTrigger }); + new[] { StellaOpsScopes.ConcelierJobsTrigger }); var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); @@ -375,7 +543,7 @@ public sealed class CommandHandlersTests "token", "Bearer", DateTimeOffset.UtcNow.AddMinutes(5), - new[] { StellaOpsScopes.FeedserJobsTrigger }); + new[] { StellaOpsScopes.ConcelierJobsTrigger }); var provider = BuildServiceProvider(new StubBackendClient(new JobTriggerResult(true, "ok", null, null)), options: options, tokenClient: tokenClient); @@ -390,7 +558,219 @@ public sealed class CommandHandlersTests Environment.ExitCode = original; } } - + + [Fact] + public async Task HandleRuntimePolicyTestAsync_WritesInteractiveTable() + { + var originalExit = Environment.ExitCode; + var originalConsole = AnsiConsole.Console; + + var console = new TestConsole(); + console.Width(120); + console.Interactive(); + console.EmitAnsiSequences(); + + AnsiConsole.Console = console; + + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)); + + var decisions = new Dictionary(StringComparer.Ordinal) + { + ["sha256:aaa"] = new RuntimePolicyImageDecision( + "allow", + true, + true, + Array.AsReadOnly(new[] { "trusted baseline" }), + new RuntimePolicyRekorReference("uuid-allow", "https://rekor.example/entries/uuid-allow", true), + new ReadOnlyDictionary(new Dictionary(StringComparer.Ordinal) + { + ["source"] = "baseline", + ["quieted"] = false, + ["confidence"] = 0.97, + ["confidenceBand"] = "high" + })), + ["sha256:bbb"] = new RuntimePolicyImageDecision( + "block", + false, + false, + Array.AsReadOnly(new[] { "missing attestation" }), + new RuntimePolicyRekorReference("uuid-block", "https://rekor.example/entries/uuid-block", false), + new ReadOnlyDictionary(new Dictionary(StringComparer.Ordinal) + { + ["source"] = "policy", + ["quieted"] = false, + ["confidence"] = 0.12, + ["confidenceBand"] = "low" + })), + ["sha256:ccc"] = new RuntimePolicyImageDecision( + "audit", + true, + false, + Array.AsReadOnly(new[] { "pending sbom sync" }), + new RuntimePolicyRekorReference(null, null, null), + new ReadOnlyDictionary(new Dictionary(StringComparer.Ordinal) + { + ["source"] = "mirror", + ["quieted"] = true, + ["quietedBy"] = "allow-temporary", + ["confidence"] = 0.42, + ["confidenceBand"] = "medium" + })) + }; + + backend.RuntimePolicyResult = new RuntimePolicyEvaluationResult( + 300, + DateTimeOffset.Parse("2025-10-19T12:00:00Z", CultureInfo.InvariantCulture), + "rev-42", + new ReadOnlyDictionary(decisions)); + + var provider = BuildServiceProvider(backend); + + try + { + await CommandHandlers.HandleRuntimePolicyTestAsync( + provider, + namespaceValue: "prod", + imageArguments: new[] { "sha256:aaa", "sha256:bbb" }, + filePath: null, + labelArguments: new[] { "app=frontend" }, + outputJson: false, + verbose: false, + cancellationToken: CancellationToken.None); + + var output = console.Output; + + Assert.Equal(0, Environment.ExitCode); + Assert.Contains("Image", output, StringComparison.Ordinal); + Assert.Contains("Verdict", output, StringComparison.Ordinal); + Assert.Contains("SBOM Ref", output, StringComparison.Ordinal); + Assert.Contains("Quieted", output, StringComparison.Ordinal); + Assert.Contains("Confidence", output, StringComparison.Ordinal); + Assert.Contains("sha256:aaa", output, StringComparison.Ordinal); + Assert.Contains("uuid-allow", output, StringComparison.Ordinal); + Assert.Contains("(verified)", output, StringComparison.Ordinal); + Assert.Contains("0.97 (high)", output, StringComparison.Ordinal); + Assert.Contains("sha256:bbb", output, StringComparison.Ordinal); + Assert.Contains("uuid-block", output, StringComparison.Ordinal); + Assert.Contains("(unverified)", output, StringComparison.Ordinal); + Assert.Contains("sha256:ccc", output, StringComparison.Ordinal); + Assert.Contains("yes", output, StringComparison.Ordinal); + Assert.Contains("allow-temporary", output, StringComparison.Ordinal); + Assert.True( + output.IndexOf("sha256:aaa", StringComparison.Ordinal) < + output.IndexOf("sha256:ccc", StringComparison.Ordinal)); + } + finally + { + Environment.ExitCode = originalExit; + AnsiConsole.Console = originalConsole; + } + } + + [Fact] + public async Task HandleRuntimePolicyTestAsync_WritesDeterministicJson() + { + var originalExit = Environment.ExitCode; + var originalOut = Console.Out; + + var backend = new StubBackendClient(new JobTriggerResult(true, "ok", null, null)); + + var decisions = new Dictionary(StringComparer.Ordinal) + { + ["sha256:json-a"] = new RuntimePolicyImageDecision( + "allow", + true, + true, + Array.AsReadOnly(new[] { "baseline allow" }), + new RuntimePolicyRekorReference("uuid-json-allow", "https://rekor.example/entries/uuid-json-allow", true), + new ReadOnlyDictionary(new Dictionary(StringComparer.Ordinal) + { + ["source"] = "baseline", + ["confidence"] = 0.66 + })), + ["sha256:json-b"] = new RuntimePolicyImageDecision( + "audit", + true, + false, + Array.AsReadOnly(Array.Empty()), + new RuntimePolicyRekorReference(null, null, null), + new ReadOnlyDictionary(new Dictionary(StringComparer.Ordinal) + { + ["source"] = "mirror", + ["quieted"] = true, + ["quietedBy"] = "risk-accepted" + })) + }; + + backend.RuntimePolicyResult = new RuntimePolicyEvaluationResult( + 600, + DateTimeOffset.Parse("2025-10-20T00:00:00Z", CultureInfo.InvariantCulture), + "rev-json-7", + new ReadOnlyDictionary(decisions)); + + var provider = BuildServiceProvider(backend); + + using var writer = new StringWriter(); + Console.SetOut(writer); + + try + { + await CommandHandlers.HandleRuntimePolicyTestAsync( + provider, + namespaceValue: "staging", + imageArguments: new[] { "sha256:json-a", "sha256:json-b" }, + filePath: null, + labelArguments: Array.Empty(), + outputJson: true, + verbose: false, + cancellationToken: CancellationToken.None); + + var output = writer.ToString().Trim(); + + Assert.Equal(0, Environment.ExitCode); + Assert.False(string.IsNullOrWhiteSpace(output)); + + using var document = JsonDocument.Parse(output); + var root = document.RootElement; + + Assert.Equal(600, root.GetProperty("ttlSeconds").GetInt32()); + Assert.Equal("rev-json-7", root.GetProperty("policyRevision").GetString()); + var expiresAt = root.GetProperty("expiresAtUtc").GetString(); + Assert.NotNull(expiresAt); + Assert.Equal( + DateTimeOffset.Parse("2025-10-20T00:00:00Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal), + DateTimeOffset.Parse(expiresAt!, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal)); + + var results = root.GetProperty("results"); + var keys = results.EnumerateObject().Select(p => p.Name).ToArray(); + Assert.Equal(new[] { "sha256:json-a", "sha256:json-b" }, keys); + + var first = results.GetProperty("sha256:json-a"); + Assert.Equal("allow", first.GetProperty("policyVerdict").GetString()); + Assert.True(first.GetProperty("signed").GetBoolean()); + Assert.True(first.GetProperty("hasSbomReferrers").GetBoolean()); + var rekor = first.GetProperty("rekor"); + Assert.Equal("uuid-json-allow", rekor.GetProperty("uuid").GetString()); + Assert.True(rekor.GetProperty("verified").GetBoolean()); + Assert.Equal("baseline", first.GetProperty("source").GetString()); + Assert.Equal(0.66, first.GetProperty("confidence").GetDouble(), 3); + + var second = results.GetProperty("sha256:json-b"); + Assert.Equal("audit", second.GetProperty("policyVerdict").GetString()); + Assert.True(second.GetProperty("signed").GetBoolean()); + Assert.False(second.GetProperty("hasSbomReferrers").GetBoolean()); + Assert.Equal("mirror", second.GetProperty("source").GetString()); + Assert.True(second.GetProperty("quieted").GetBoolean()); + Assert.Equal("risk-accepted", second.GetProperty("quietedBy").GetString()); + Assert.False(second.TryGetProperty("rekor", out _)); + } + finally + { + Console.SetOut(originalOut); + Environment.ExitCode = originalExit; + } + } + private static async Task WriteRevocationArtifactsAsync(TempDirectory temp, string? providerHint) { var (bundleBytes, signature, keyPem) = await BuildRevocationArtifactsAsync(providerHint); @@ -501,44 +881,87 @@ public sealed class CommandHandlersTests $"{Path.GetFileNameWithoutExtension(tempResultsFile)}-run.json"); return new StubExecutor(new ScannerExecutionResult(0, tempResultsFile, tempMetadataFile)); } - - private sealed class StubBackendClient : IBackendOperationsClient - { - private readonly JobTriggerResult _result; - - public StubBackendClient(JobTriggerResult result) - { - _result = result; - } - - public string? LastJobKind { get; private set; } - public string? LastUploadPath { get; private set; } - - public Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) - { - LastUploadPath = filePath; - return Task.CompletedTask; - } - - public Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) - { - LastJobKind = jobKind; - return Task.FromResult(_result); - } - } - - private sealed class StubExecutor : IScannerExecutor - { - private readonly ScannerExecutionResult _result; - - public StubExecutor(ScannerExecutionResult result) - { - _result = result; - } - + + private sealed class StubBackendClient : IBackendOperationsClient + { + private readonly JobTriggerResult _jobResult; + private static readonly RuntimePolicyEvaluationResult DefaultRuntimePolicyResult = + new RuntimePolicyEvaluationResult( + 0, + null, + null, + new ReadOnlyDictionary( + new Dictionary())); + + public StubBackendClient(JobTriggerResult result) + { + _jobResult = result; + } + + public string? LastJobKind { get; private set; } + public string? LastUploadPath { get; private set; } + public string? LastExcititorRoute { get; private set; } + public HttpMethod? LastExcititorMethod { get; private set; } + public object? LastExcititorPayload { get; private set; } + public List<(string ExportId, string DestinationPath, string? Algorithm, string? Digest)> ExportDownloads { get; } = new(); + public ExcititorOperationResult? ExcititorResult { get; set; } = new ExcititorOperationResult(true, "ok", null, null); + public IReadOnlyList ProviderSummaries { get; set; } = Array.Empty(); + public RuntimePolicyEvaluationResult RuntimePolicyResult { get; set; } = DefaultRuntimePolicyResult; + + public Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken) + => throw new NotImplementedException(); + + public Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + LastUploadPath = filePath; + return Task.CompletedTask; + } + + public Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) + { + LastJobKind = jobKind; + return Task.FromResult(_jobResult); + } + + public Task ExecuteExcititorOperationAsync(string route, HttpMethod method, object? payload, CancellationToken cancellationToken) + { + LastExcititorRoute = route; + LastExcititorMethod = method; + LastExcititorPayload = payload; + return Task.FromResult(ExcititorResult ?? new ExcititorOperationResult(true, "ok", null, null)); + } + + public Task DownloadExcititorExportAsync(string exportId, string destinationPath, string? expectedDigestAlgorithm, string? expectedDigest, CancellationToken cancellationToken) + { + var fullPath = Path.GetFullPath(destinationPath); + var directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + File.WriteAllText(fullPath, "{}"); + var info = new FileInfo(fullPath); + ExportDownloads.Add((exportId, fullPath, expectedDigestAlgorithm, expectedDigest)); + return Task.FromResult(new ExcititorExportDownloadResult(fullPath, info.Length, false)); + } + + public Task> GetExcititorProvidersAsync(bool includeDisabled, CancellationToken cancellationToken) + => Task.FromResult(ProviderSummaries); + + public Task EvaluateRuntimePolicyAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken) + => Task.FromResult(RuntimePolicyResult); + } + + private sealed class StubExecutor : IScannerExecutor + { + private readonly ScannerExecutionResult _result; + + public StubExecutor(ScannerExecutionResult result) + { + _result = result; + } + public Task RunAsync(string runner, string entry, string targetDirectory, string resultsDirectory, IReadOnlyList arguments, bool verbose, CancellationToken cancellationToken) { Directory.CreateDirectory(Path.GetDirectoryName(_result.ResultsPath)!); @@ -555,8 +978,8 @@ public sealed class CommandHandlersTests return Task.FromResult(_result); } - } - + } + private sealed class StubInstaller : IScannerInstaller { public Task InstallAsync(string artifactPath, bool verbose, CancellationToken cancellationToken) @@ -573,7 +996,7 @@ public sealed class CommandHandlersTests "token-123", "Bearer", DateTimeOffset.UtcNow.AddMinutes(30), - new[] { StellaOpsScopes.FeedserJobsTrigger }); + new[] { StellaOpsScopes.ConcelierJobsTrigger }); } public int ClientCredentialRequests { get; private set; } diff --git a/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs index d19deb4f..38562646 100644 --- a/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs +++ b/src/StellaOps.Cli.Tests/Configuration/CliBootstrapperTests.cs @@ -24,7 +24,7 @@ public sealed class CliBootstrapperTests : IDisposable Environment.SetEnvironmentVariable("STELLAOPS_BACKEND_URL", "https://env-backend.example"); Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_URL", "https://authority.env"); Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_CLIENT_ID", "cli-env"); - Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_SCOPE", "feedser.jobs.trigger"); + Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_SCOPE", "concelier.jobs.trigger"); Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ENABLE_RETRIES", "false"); Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_RETRY_DELAYS", "00:00:02,00:00:05"); Environment.SetEnvironmentVariable("STELLAOPS_AUTHORITY_ALLOW_OFFLINE_CACHE_FALLBACK", "false"); @@ -38,7 +38,7 @@ public sealed class CliBootstrapperTests : IDisposable Assert.Equal("https://env-backend.example", options.BackendUrl); Assert.Equal("https://authority.env", options.Authority.Url); Assert.Equal("cli-env", options.Authority.ClientId); - Assert.Equal("feedser.jobs.trigger", options.Authority.Scope); + Assert.Equal("concelier.jobs.trigger", options.Authority.Scope); Assert.NotNull(options.Authority.Resilience); Assert.False(options.Authority.Resilience.EnableRetries); @@ -73,7 +73,7 @@ public sealed class CliBootstrapperTests : IDisposable { Url = "https://authority.file", ClientId = "cli-file", - Scope = "feedser.jobs.trigger" + Scope = "concelier.jobs.trigger" } } }); diff --git a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs index 14587016..541b2273 100644 --- a/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs +++ b/src/StellaOps.Cli.Tests/Services/BackendOperationsClientTests.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.ObjectModel; +using System.Globalization; using System.IO; using System.Net; using System.Net.Http; @@ -8,10 +10,10 @@ using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.IdentityModel.Tokens; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.Client; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; using StellaOps.Cli.Configuration; using StellaOps.Cli.Services; using StellaOps.Cli.Services.Models; @@ -46,12 +48,12 @@ public sealed class BackendOperationsClientTests var httpClient = new HttpClient(handler) { - BaseAddress = new Uri("https://feedser.example") + BaseAddress = new Uri("https://concelier.example") }; var options = new StellaOpsCliOptions { - BackendUrl = "https://feedser.example", + BackendUrl = "https://concelier.example", ScannerCacheDirectory = temp.Path, ScannerDownloadAttempts = 1 }; @@ -92,12 +94,12 @@ public sealed class BackendOperationsClientTests var httpClient = new HttpClient(handler) { - BaseAddress = new Uri("https://feedser.example") + BaseAddress = new Uri("https://concelier.example") }; var options = new StellaOpsCliOptions { - BackendUrl = "https://feedser.example", + BackendUrl = "https://concelier.example", ScannerCacheDirectory = temp.Path, ScannerDownloadAttempts = 1 }; @@ -111,11 +113,11 @@ public sealed class BackendOperationsClientTests Assert.False(File.Exists(targetPath)); } - [Fact] - public async Task DownloadScannerAsync_RetriesOnFailure() - { - using var temp = new TempDirectory(); - + [Fact] + public async Task DownloadScannerAsync_RetriesOnFailure() + { + using var temp = new TempDirectory(); + var successBytes = Encoding.UTF8.GetBytes("success"); var digestHex = Convert.ToHexString(SHA256.HashData(successBytes)).ToLowerInvariant(); var attempts = 0; @@ -144,12 +146,12 @@ public sealed class BackendOperationsClientTests var httpClient = new HttpClient(handler) { - BaseAddress = new Uri("https://feedser.example") + BaseAddress = new Uri("https://concelier.example") }; var options = new StellaOpsCliOptions { - BackendUrl = "https://feedser.example", + BackendUrl = "https://concelier.example", ScannerCacheDirectory = temp.Path, ScannerDownloadAttempts = 3 }; @@ -161,94 +163,94 @@ public sealed class BackendOperationsClientTests var result = await client.DownloadScannerAsync("stable", targetPath, overwrite: false, verbose: false, CancellationToken.None); Assert.Equal(2, attempts); - Assert.False(result.FromCache); - Assert.True(File.Exists(targetPath)); - } - - [Fact] - public async Task UploadScanResultsAsync_RetriesOnRetryAfter() - { - using var temp = new TempDirectory(); - var filePath = Path.Combine(temp.Path, "scan.json"); - await File.WriteAllTextAsync(filePath, "{}"); - - var attempts = 0; - var handler = new StubHttpMessageHandler( - (request, _) => - { - attempts++; - var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests) - { - RequestMessage = request, - Content = new StringContent("busy") - }; - response.Headers.Add("Retry-After", "1"); - return response; - }, - (request, _) => - { - attempts++; - return new HttpResponseMessage(HttpStatusCode.OK) - { - RequestMessage = request - }; - }); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://feedser.example") - }; - - var options = new StellaOpsCliOptions - { - BackendUrl = "https://feedser.example", - ScanUploadAttempts = 3 - }; - - var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); - var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); - - await client.UploadScanResultsAsync(filePath, CancellationToken.None); - - Assert.Equal(2, attempts); - } - - [Fact] - public async Task UploadScanResultsAsync_ThrowsAfterMaxAttempts() - { - using var temp = new TempDirectory(); - var filePath = Path.Combine(temp.Path, "scan.json"); - await File.WriteAllTextAsync(filePath, "{}"); - - var attempts = 0; - var handler = new StubHttpMessageHandler( - (request, _) => - { - attempts++; - return new HttpResponseMessage(HttpStatusCode.BadGateway) - { - RequestMessage = request, - Content = new StringContent("bad gateway") - }; - }); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://feedser.example") - }; - - var options = new StellaOpsCliOptions - { - BackendUrl = "https://feedser.example", - ScanUploadAttempts = 2 - }; - - var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); - var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); - - await Assert.ThrowsAsync(() => client.UploadScanResultsAsync(filePath, CancellationToken.None)); - Assert.Equal(2, attempts); - } + Assert.False(result.FromCache); + Assert.True(File.Exists(targetPath)); + } + + [Fact] + public async Task UploadScanResultsAsync_RetriesOnRetryAfter() + { + using var temp = new TempDirectory(); + var filePath = Path.Combine(temp.Path, "scan.json"); + await File.WriteAllTextAsync(filePath, "{}"); + + var attempts = 0; + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + var response = new HttpResponseMessage(HttpStatusCode.TooManyRequests) + { + RequestMessage = request, + Content = new StringContent("busy") + }; + response.Headers.Add("Retry-After", "1"); + return response; + }, + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.OK) + { + RequestMessage = request + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://concelier.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://concelier.example", + ScanUploadAttempts = 3 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + await client.UploadScanResultsAsync(filePath, CancellationToken.None); + + Assert.Equal(2, attempts); + } + + [Fact] + public async Task UploadScanResultsAsync_ThrowsAfterMaxAttempts() + { + using var temp = new TempDirectory(); + var filePath = Path.Combine(temp.Path, "scan.json"); + await File.WriteAllTextAsync(filePath, "{}"); + + var attempts = 0; + var handler = new StubHttpMessageHandler( + (request, _) => + { + attempts++; + return new HttpResponseMessage(HttpStatusCode.BadGateway) + { + RequestMessage = request, + Content = new StringContent("bad gateway") + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://concelier.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://concelier.example", + ScanUploadAttempts = 2 + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + await Assert.ThrowsAsync(() => client.UploadScanResultsAsync(filePath, CancellationToken.None)); + Assert.Equal(2, attempts); + } [Fact] public async Task TriggerJobAsync_ReturnsAcceptedResult() @@ -273,10 +275,10 @@ public sealed class BackendOperationsClientTests var httpClient = new HttpClient(handler) { - BaseAddress = new Uri("https://feedser.example") + BaseAddress = new Uri("https://concelier.example") }; - var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var options = new StellaOpsCliOptions { BackendUrl = "https://concelier.example" }; var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); @@ -288,11 +290,11 @@ public sealed class BackendOperationsClientTests } [Fact] - public async Task TriggerJobAsync_ReturnsFailureMessage() - { - var handler = new StubHttpMessageHandler((request, _) => - { - var problem = new + public async Task TriggerJobAsync_ReturnsFailureMessage() + { + var handler = new StubHttpMessageHandler((request, _) => + { + var problem = new { title = "Job already running", detail = "export job active" @@ -308,110 +310,214 @@ public sealed class BackendOperationsClientTests var httpClient = new HttpClient(handler) { - BaseAddress = new Uri("https://feedser.example") + BaseAddress = new Uri("https://concelier.example") }; - var options = new StellaOpsCliOptions { BackendUrl = "https://feedser.example" }; + var options = new StellaOpsCliOptions { BackendUrl = "https://concelier.example" }; var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); var result = await client.TriggerJobAsync("export:json", new Dictionary(), CancellationToken.None); - - Assert.False(result.Success); - Assert.Contains("Job already running", result.Message); - } - - [Fact] - public async Task TriggerJobAsync_UsesAuthorityTokenWhenConfigured() - { - using var temp = new TempDirectory(); - - var handler = new StubHttpMessageHandler((request, _) => - { - Assert.NotNull(request.Headers.Authorization); - Assert.Equal("Bearer", request.Headers.Authorization!.Scheme); - Assert.Equal("token-123", request.Headers.Authorization.Parameter); - - return new HttpResponseMessage(HttpStatusCode.Accepted) - { - RequestMessage = request, - Content = JsonContent.Create(new JobRunResponse - { - RunId = Guid.NewGuid(), - Kind = "test", - Status = "Pending", - Trigger = "cli", - CreatedAt = DateTimeOffset.UtcNow - }) - }; - }); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://feedser.example") - }; - - var options = new StellaOpsCliOptions - { - BackendUrl = "https://feedser.example", - Authority = - { - Url = "https://authority.example", - ClientId = "cli", - ClientSecret = "secret", - Scope = "feedser.jobs.trigger", - TokenCacheDirectory = temp.Path - } - }; - - var tokenClient = new StubTokenClient(); - var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); - var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger(), tokenClient); - - var result = await client.TriggerJobAsync("test", new Dictionary(), CancellationToken.None); - - Assert.True(result.Success); - Assert.Equal("Accepted", result.Message); - Assert.True(tokenClient.Requests > 0); - } - - private sealed class StubTokenClient : IStellaOpsTokenClient - { - private readonly StellaOpsTokenResult _tokenResult; - - public int Requests { get; private set; } - - public StubTokenClient() - { - _tokenResult = new StellaOpsTokenResult( - "token-123", - "Bearer", - DateTimeOffset.UtcNow.AddMinutes(5), - new[] { StellaOpsScopes.FeedserJobsTrigger }); - } - - public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) - => ValueTask.CompletedTask; - - public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) - => ValueTask.CompletedTask; - - public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) - => Task.FromResult(new JsonWebKeySet("{\"keys\":[]}")); - - public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) - => ValueTask.FromResult(null); - - public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) - { - Requests++; - return Task.FromResult(_tokenResult); - } - - public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) - { - Requests++; - return Task.FromResult(_tokenResult); - } - } -} + + Assert.False(result.Success); + Assert.Contains("Job already running", result.Message); + } + + [Fact] + public async Task TriggerJobAsync_UsesAuthorityTokenWhenConfigured() + { + using var temp = new TempDirectory(); + + var handler = new StubHttpMessageHandler((request, _) => + { + Assert.NotNull(request.Headers.Authorization); + Assert.Equal("Bearer", request.Headers.Authorization!.Scheme); + Assert.Equal("token-123", request.Headers.Authorization.Parameter); + + return new HttpResponseMessage(HttpStatusCode.Accepted) + { + RequestMessage = request, + Content = JsonContent.Create(new JobRunResponse + { + RunId = Guid.NewGuid(), + Kind = "test", + Status = "Pending", + Trigger = "cli", + CreatedAt = DateTimeOffset.UtcNow + }) + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://concelier.example") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://concelier.example", + Authority = + { + Url = "https://authority.example", + ClientId = "cli", + ClientSecret = "secret", + Scope = "concelier.jobs.trigger", + TokenCacheDirectory = temp.Path + } + }; + + var tokenClient = new StubTokenClient(); + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger(), tokenClient); + + var result = await client.TriggerJobAsync("test", new Dictionary(), CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal("Accepted", result.Message); + Assert.True(tokenClient.Requests > 0); + } + + [Fact] + public async Task EvaluateRuntimePolicyAsync_ParsesDecisionPayload() + { + var handler = new StubHttpMessageHandler((request, _) => + { + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal("/api/scanner/policy/runtime", request.RequestUri!.AbsolutePath); + + var body = request.Content!.ReadAsStringAsync().GetAwaiter().GetResult(); + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + Assert.Equal("prod", root.GetProperty("namespace").GetString()); + Assert.Equal("payments", root.GetProperty("labels").GetProperty("app").GetString()); + var images = root.GetProperty("images"); + Assert.Equal(2, images.GetArrayLength()); + Assert.Equal("ghcr.io/app@sha256:abc", images[0].GetString()); + Assert.Equal("ghcr.io/api@sha256:def", images[1].GetString()); + + var responseJson = @"{ + ""ttlSeconds"": 120, + ""policyRevision"": ""rev-123"", + ""expiresAtUtc"": ""2025-10-19T12:34:56Z"", + ""results"": { + ""ghcr.io/app@sha256:abc"": { + ""policyVerdict"": ""pass"", + ""signed"": true, + ""hasSbomReferrers"": true, + ""reasons"": [], + ""rekor"": { ""uuid"": ""uuid-1"", ""url"": ""https://rekor.example/uuid-1"", ""verified"": true }, + ""confidence"": 0.87, + ""quieted"": false, + ""metadata"": { ""note"": ""cached"" } + }, + ""ghcr.io/api@sha256:def"": { + ""policyVerdict"": ""fail"", + ""signed"": false, + ""hasSbomReferrers"": false, + ""reasons"": [""unsigned"", ""missing sbom""], + ""quietedBy"": ""manual-override"" + } + } +}"; + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(responseJson, Encoding.UTF8, "application/json"), + RequestMessage = request + }; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://scanner.example/") + }; + + var options = new StellaOpsCliOptions + { + BackendUrl = "https://scanner.example/" + }; + + var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Debug)); + var client = new BackendOperationsClient(httpClient, options, loggerFactory.CreateLogger()); + + var labels = new ReadOnlyDictionary(new Dictionary { ["app"] = "payments" }); + var imagesList = new ReadOnlyCollection(new List + { + "ghcr.io/app@sha256:abc", + "ghcr.io/app@sha256:abc", + "ghcr.io/api@sha256:def" + }); + var requestModel = new RuntimePolicyEvaluationRequest("prod", labels, imagesList); + + var result = await client.EvaluateRuntimePolicyAsync(requestModel, CancellationToken.None); + + Assert.Equal(120, result.TtlSeconds); + Assert.Equal("rev-123", result.PolicyRevision); + Assert.Equal(DateTimeOffset.Parse("2025-10-19T12:34:56Z", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal), result.ExpiresAtUtc); + Assert.Equal(2, result.Decisions.Count); + + var primary = result.Decisions["ghcr.io/app@sha256:abc"]; + Assert.Equal("pass", primary.PolicyVerdict); + Assert.True(primary.Signed); + Assert.True(primary.HasSbomReferrers); + Assert.Empty(primary.Reasons); + Assert.NotNull(primary.Rekor); + Assert.Equal("uuid-1", primary.Rekor!.Uuid); + Assert.Equal("https://rekor.example/uuid-1", primary.Rekor.Url); + Assert.True(primary.Rekor.Verified); + Assert.Equal(0.87, Assert.IsType(primary.AdditionalProperties["confidence"]), 3); + Assert.False(Assert.IsType(primary.AdditionalProperties["quieted"])); + var metadataJson = Assert.IsType(primary.AdditionalProperties["metadata"]); + using var metadataDocument = JsonDocument.Parse(metadataJson); + Assert.Equal("cached", metadataDocument.RootElement.GetProperty("note").GetString()); + + var secondary = result.Decisions["ghcr.io/api@sha256:def"]; + Assert.Equal("fail", secondary.PolicyVerdict); + Assert.False(secondary.Signed); + Assert.False(secondary.HasSbomReferrers); + Assert.Collection(secondary.Reasons, + item => Assert.Equal("unsigned", item), + item => Assert.Equal("missing sbom", item)); + Assert.Equal("manual-override", Assert.IsType(secondary.AdditionalProperties["quietedBy"])); + } + + private sealed class StubTokenClient : IStellaOpsTokenClient + { + private readonly StellaOpsTokenResult _tokenResult; + + public int Requests { get; private set; } + + public StubTokenClient() + { + _tokenResult = new StellaOpsTokenResult( + "token-123", + "Bearer", + DateTimeOffset.UtcNow.AddMinutes(5), + new[] { StellaOpsScopes.ConcelierJobsTrigger }); + } + + public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) + => Task.FromResult(new JsonWebKeySet("{\"keys\":[]}")); + + public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + + public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) + { + Requests++; + return Task.FromResult(_tokenResult); + } + + public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) + { + Requests++; + return Task.FromResult(_tokenResult); + } + } +} diff --git a/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj index ab4b3308..ba54eb41 100644 --- a/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj +++ b/src/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj @@ -16,13 +16,14 @@ - - - - - - - - - - + + + + + + + + + + + diff --git a/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs index 561e27da..28c35051 100644 --- a/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs +++ b/src/StellaOps.Cli.Tests/Testing/TestHelpers.cs @@ -1,14 +1,15 @@ using System; -using System.Collections.Generic; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Text; namespace StellaOps.Cli.Tests.Testing; -internal sealed class TempDirectory : IDisposable -{ +internal sealed class TempDirectory : IDisposable +{ public TempDirectory() { Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-cli-tests-{Guid.NewGuid():N}"); @@ -31,7 +32,41 @@ internal sealed class TempDirectory : IDisposable // ignored } } -} +} + +internal sealed class TempFile : IDisposable +{ + public TempFile(string fileName, byte[] contents) + { + var directory = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-cli-file-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + Path = System.IO.Path.Combine(directory, fileName); + File.WriteAllBytes(Path, contents); + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (File.Exists(Path)) + { + File.Delete(Path); + } + + var directory = System.IO.Path.GetDirectoryName(Path); + if (!string.IsNullOrEmpty(directory) && Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // ignored intentionally + } + } +} internal sealed class StubHttpMessageHandler : HttpMessageHandler { diff --git a/src/StellaOps.Cli/AGENTS.md b/src/StellaOps.Cli/AGENTS.md index 77dc3e3a..e72b6563 100644 --- a/src/StellaOps.Cli/AGENTS.md +++ b/src/StellaOps.Cli/AGENTS.md @@ -1,14 +1,14 @@ # StellaOps.Cli — Agent Brief ## Mission -- Deliver an offline-capable command-line interface that drives StellaOps back-end operations: scanner distribution, scan execution, result uploads, and Feedser database lifecycle calls (init/resume/export). +- Deliver an offline-capable command-line interface that drives StellaOps back-end operations: scanner distribution, scan execution, result uploads, and Concelier database lifecycle calls (init/resume/export). - Honour StellaOps principles of determinism, observability, and offline-first behaviour while providing a polished operator experience. ## Role Charter | Role | Mandate | Collaboration | | --- | --- | --- | | **DevEx/CLI** | Own CLI UX, command routing, and configuration model. Ensure commands work with empty/default config and document overrides. | Coordinate with Backend/WebService for API contracts and with Docs for operator workflows. | -| **Ops Integrator** | Maintain integration paths for shell/dotnet/docker tooling. Validate that air-gapped runners can bootstrap required binaries. | Work with Feedser/Agent teams to mirror packaging and signing requirements. | +| **Ops Integrator** | Maintain integration paths for shell/dotnet/docker tooling. Validate that air-gapped runners can bootstrap required binaries. | Work with Concelier/Agent teams to mirror packaging and signing requirements. | | **QA** | Provide command-level fixtures, golden outputs, and regression coverage (unit & smoke). Ensure commands respect cancellation and deterministic logging. | Partner with QA guild for shared harnesses and test data. | ## Working Agreements @@ -21,7 +21,7 @@ - Update `TASKS.md` as states change (TODO → DOING → DONE/BLOCKED) and record added tests/fixtures alongside implementation notes. ## Reference Materials -- `docs/ARCHITECTURE_FEEDSER.md` for database operations surface area. +- `docs/ARCHITECTURE_CONCELIER.md` for database operations surface area. - Backend OpenAPI/contract docs (once available) for job triggers and scanner endpoints. - Existing module AGENTS/TASKS files for style and coordination cues. - `docs/09_API_CLI_REFERENCE.md` (section 3) for the user-facing synopsis of the CLI verbs and flags. diff --git a/src/StellaOps.Cli/Commands/CommandFactory.cs b/src/StellaOps.Cli/Commands/CommandFactory.cs index cc1b5873..aa0230a5 100644 --- a/src/StellaOps.Cli/Commands/CommandFactory.cs +++ b/src/StellaOps.Cli/Commands/CommandFactory.cs @@ -24,6 +24,8 @@ internal static class CommandFactory root.Add(BuildScannerCommand(services, verboseOption, cancellationToken)); root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken)); root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken)); + root.Add(BuildExcititorCommand(services, verboseOption, cancellationToken)); + root.Add(BuildRuntimeCommand(services, verboseOption, cancellationToken)); root.Add(BuildAuthCommand(services, options, verboseOption, cancellationToken)); root.Add(BuildConfigCommand(options)); @@ -137,7 +139,7 @@ internal static class CommandFactory private static Command BuildDatabaseCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) { - var db = new Command("db", "Trigger Feedser database operations via backend jobs."); + var db = new Command("db", "Trigger Concelier database operations via backend jobs."); var fetch = new Command("fetch", "Trigger connector fetch/parse/map stages."); var sourceOption = new Option("--source") @@ -174,7 +176,7 @@ internal static class CommandFactory return CommandHandlers.HandleMergeJobAsync(services, verbose, cancellationToken); }); - var export = new Command("export", "Run Feedser export jobs."); + var export = new Command("export", "Run Concelier export jobs."); var formatOption = new Option("--format") { Description = "Export format: json or trivy-db." @@ -220,10 +222,304 @@ internal static class CommandFactory db.Add(fetch); db.Add(merge); - db.Add(export); + db.Add(export); return db; } + private static Command BuildExcititorCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var excititor = new Command("excititor", "Manage Excititor ingest, exports, and reconciliation workflows."); + + var init = new Command("init", "Initialize Excititor ingest state."); + var initProviders = new Option("--provider", new[] { "-p" }) + { + Description = "Optional provider identifier(s) to initialize.", + Arity = ArgumentArity.ZeroOrMore + }; + var resumeOption = new Option("--resume") + { + Description = "Resume ingest from the last persisted checkpoint instead of starting fresh." + }; + init.Add(initProviders); + init.Add(resumeOption); + init.SetAction((parseResult, _) => + { + var providers = parseResult.GetValue(initProviders) ?? Array.Empty(); + var resume = parseResult.GetValue(resumeOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorInitAsync(services, providers, resume, verbose, cancellationToken); + }); + + var pull = new Command("pull", "Trigger Excititor ingest for configured providers."); + var pullProviders = new Option("--provider", new[] { "-p" }) + { + Description = "Optional provider identifier(s) to ingest.", + Arity = ArgumentArity.ZeroOrMore + }; + var sinceOption = new Option("--since") + { + Description = "Optional ISO-8601 timestamp to begin the ingest window." + }; + var windowOption = new Option("--window") + { + Description = "Optional window duration (e.g. 24:00:00)." + }; + var forceOption = new Option("--force") + { + Description = "Force ingestion even if the backend reports no pending work." + }; + pull.Add(pullProviders); + pull.Add(sinceOption); + pull.Add(windowOption); + pull.Add(forceOption); + pull.SetAction((parseResult, _) => + { + var providers = parseResult.GetValue(pullProviders) ?? Array.Empty(); + var since = parseResult.GetValue(sinceOption); + var window = parseResult.GetValue(windowOption); + var force = parseResult.GetValue(forceOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorPullAsync(services, providers, since, window, force, verbose, cancellationToken); + }); + + var resume = new Command("resume", "Resume Excititor ingest using a checkpoint token."); + var resumeProviders = new Option("--provider", new[] { "-p" }) + { + Description = "Optional provider identifier(s) to resume.", + Arity = ArgumentArity.ZeroOrMore + }; + var checkpointOption = new Option("--checkpoint") + { + Description = "Optional checkpoint identifier to resume from." + }; + resume.Add(resumeProviders); + resume.Add(checkpointOption); + resume.SetAction((parseResult, _) => + { + var providers = parseResult.GetValue(resumeProviders) ?? Array.Empty(); + var checkpoint = parseResult.GetValue(checkpointOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorResumeAsync(services, providers, checkpoint, verbose, cancellationToken); + }); + + var list = new Command("list-providers", "List Excititor providers and their ingest status."); + var includeDisabledOption = new Option("--include-disabled") + { + Description = "Include disabled providers in the listing." + }; + list.Add(includeDisabledOption); + list.SetAction((parseResult, _) => + { + var includeDisabled = parseResult.GetValue(includeDisabledOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorListProvidersAsync(services, includeDisabled, verbose, cancellationToken); + }); + + var export = new Command("export", "Trigger Excititor export generation."); + var formatOption = new Option("--format") + { + Description = "Export format (e.g. openvex, json)." + }; + var exportDeltaOption = new Option("--delta") + { + Description = "Request a delta export when supported." + }; + var exportScopeOption = new Option("--scope") + { + Description = "Optional policy scope or tenant identifier." + }; + var exportSinceOption = new Option("--since") + { + Description = "Optional ISO-8601 timestamp to restrict export contents." + }; + var exportProviderOption = new Option("--provider") + { + Description = "Optional provider identifier when requesting targeted exports." + }; + var exportOutputOption = new Option("--output") + { + Description = "Optional path to download the export artifact." + }; + export.Add(formatOption); + export.Add(exportDeltaOption); + export.Add(exportScopeOption); + export.Add(exportSinceOption); + export.Add(exportProviderOption); + export.Add(exportOutputOption); + export.SetAction((parseResult, _) => + { + var format = parseResult.GetValue(formatOption) ?? "openvex"; + var delta = parseResult.GetValue(exportDeltaOption); + var scope = parseResult.GetValue(exportScopeOption); + var since = parseResult.GetValue(exportSinceOption); + var provider = parseResult.GetValue(exportProviderOption); + var output = parseResult.GetValue(exportOutputOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorExportAsync(services, format, delta, scope, since, provider, output, verbose, cancellationToken); + }); + + var backfill = new Command("backfill-statements", "Replay historical raw documents into Excititor statements."); + var backfillRetrievedSinceOption = new Option("--retrieved-since") + { + Description = "Only process raw documents retrieved on or after the provided ISO-8601 timestamp." + }; + var backfillForceOption = new Option("--force") + { + Description = "Reprocess documents even if statements already exist." + }; + var backfillBatchSizeOption = new Option("--batch-size") + { + Description = "Number of raw documents to fetch per batch (default 100)." + }; + var backfillMaxDocumentsOption = new Option("--max-documents") + { + Description = "Optional maximum number of raw documents to process." + }; + backfill.Add(backfillRetrievedSinceOption); + backfill.Add(backfillForceOption); + backfill.Add(backfillBatchSizeOption); + backfill.Add(backfillMaxDocumentsOption); + backfill.SetAction((parseResult, _) => + { + var retrievedSince = parseResult.GetValue(backfillRetrievedSinceOption); + var force = parseResult.GetValue(backfillForceOption); + var batchSize = parseResult.GetValue(backfillBatchSizeOption); + if (batchSize <= 0) + { + batchSize = 100; + } + var maxDocuments = parseResult.GetValue(backfillMaxDocumentsOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorBackfillStatementsAsync( + services, + retrievedSince, + force, + batchSize, + maxDocuments, + verbose, + cancellationToken); + }); + + var verify = new Command("verify", "Verify Excititor exports or attestations."); + var exportIdOption = new Option("--export-id") + { + Description = "Export identifier to verify." + }; + var digestOption = new Option("--digest") + { + Description = "Expected digest for the export or attestation." + }; + var attestationOption = new Option("--attestation") + { + Description = "Path to a local attestation file to verify (base64 content will be uploaded)." + }; + verify.Add(exportIdOption); + verify.Add(digestOption); + verify.Add(attestationOption); + verify.SetAction((parseResult, _) => + { + var exportId = parseResult.GetValue(exportIdOption); + var digest = parseResult.GetValue(digestOption); + var attestation = parseResult.GetValue(attestationOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorVerifyAsync(services, exportId, digest, attestation, verbose, cancellationToken); + }); + + var reconcile = new Command("reconcile", "Trigger Excititor reconciliation against canonical advisories."); + var reconcileProviders = new Option("--provider", new[] { "-p" }) + { + Description = "Optional provider identifier(s) to reconcile.", + Arity = ArgumentArity.ZeroOrMore + }; + var maxAgeOption = new Option("--max-age") + { + Description = "Optional maximum age window (e.g. 7.00:00:00)." + }; + reconcile.Add(reconcileProviders); + reconcile.Add(maxAgeOption); + reconcile.SetAction((parseResult, _) => + { + var providers = parseResult.GetValue(reconcileProviders) ?? Array.Empty(); + var maxAge = parseResult.GetValue(maxAgeOption); + var verbose = parseResult.GetValue(verboseOption); + return CommandHandlers.HandleExcititorReconcileAsync(services, providers, maxAge, verbose, cancellationToken); + }); + + excititor.Add(init); + excititor.Add(pull); + excititor.Add(resume); + excititor.Add(list); + excititor.Add(export); + excititor.Add(backfill); + excititor.Add(verify); + excititor.Add(reconcile); + return excititor; + } + + private static Command BuildRuntimeCommand(IServiceProvider services, Option verboseOption, CancellationToken cancellationToken) + { + var runtime = new Command("runtime", "Interact with runtime admission policy APIs."); + var policy = new Command("policy", "Runtime policy operations."); + + var test = new Command("test", "Evaluate runtime policy decisions for image digests."); + var namespaceOption = new Option("--namespace", new[] { "--ns" }) + { + Description = "Namespace or logical scope for the evaluation." + }; + + var imageOption = new Option("--image", new[] { "-i", "--images" }) + { + Description = "Image digests to evaluate (repeatable).", + Arity = ArgumentArity.ZeroOrMore + }; + + var fileOption = new Option("--file", new[] { "-f" }) + { + Description = "Path to a file containing image digests (one per line)." + }; + + var labelOption = new Option("--label", new[] { "-l", "--labels" }) + { + Description = "Pod labels in key=value format (repeatable).", + Arity = ArgumentArity.ZeroOrMore + }; + + var jsonOption = new Option("--json") + { + Description = "Emit the raw JSON response." + }; + + test.Add(namespaceOption); + test.Add(imageOption); + test.Add(fileOption); + test.Add(labelOption); + test.Add(jsonOption); + + test.SetAction((parseResult, _) => + { + var nsValue = parseResult.GetValue(namespaceOption); + var images = parseResult.GetValue(imageOption) ?? Array.Empty(); + var file = parseResult.GetValue(fileOption); + var labels = parseResult.GetValue(labelOption) ?? Array.Empty(); + var outputJson = parseResult.GetValue(jsonOption); + var verbose = parseResult.GetValue(verboseOption); + + return CommandHandlers.HandleRuntimePolicyTestAsync( + services, + nsValue, + images, + file, + labels, + outputJson, + verbose, + cancellationToken); + }); + + policy.Add(test); + runtime.Add(policy); + return runtime; + } + private static Command BuildAuthCommand(IServiceProvider services, StellaOpsCliOptions options, Option verboseOption, CancellationToken cancellationToken) { var auth = new Command("auth", "Manage authentication with StellaOps Authority."); diff --git a/src/StellaOps.Cli/Commands/CommandHandlers.cs b/src/StellaOps.Cli/Commands/CommandHandlers.cs index 3fcb77a8..7ac40593 100644 --- a/src/StellaOps.Cli/Commands/CommandHandlers.cs +++ b/src/StellaOps.Cli/Commands/CommandHandlers.cs @@ -1,26 +1,30 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Spectre.Console; -using StellaOps.Auth.Client; -using StellaOps.Cli.Configuration; -using StellaOps.Cli.Prompts; -using StellaOps.Cli.Services; -using StellaOps.Cli.Services.Models; -using StellaOps.Cli.Telemetry; -using StellaOps.Cryptography; - -namespace StellaOps.Cli.Commands; +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Spectre.Console; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Prompts; +using StellaOps.Cli.Services; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Telemetry; +using StellaOps.Cryptography; + +namespace StellaOps.Cli.Commands; internal static class CommandHandlers { @@ -118,22 +122,22 @@ internal static class CommandHandlers Environment.ExitCode = executionResult.ExitCode; CliMetrics.RecordScanRun(runner, executionResult.ExitCode); - if (executionResult.ExitCode == 0) - { - var backend = scope.ServiceProvider.GetRequiredService(); - logger.LogInformation("Uploading scan artefact {Path}...", executionResult.ResultsPath); - await backend.UploadScanResultsAsync(executionResult.ResultsPath, cancellationToken).ConfigureAwait(false); - logger.LogInformation("Scan artefact uploaded."); - activity?.SetTag("stellaops.cli.results", executionResult.ResultsPath); - } - else - { - logger.LogWarning("Skipping automatic upload because scan exited with code {Code}.", executionResult.ExitCode); - } - - logger.LogInformation("Run metadata written to {Path}.", executionResult.RunMetadataPath); - activity?.SetTag("stellaops.cli.run_metadata", executionResult.RunMetadataPath); - } + if (executionResult.ExitCode == 0) + { + var backend = scope.ServiceProvider.GetRequiredService(); + logger.LogInformation("Uploading scan artefact {Path}...", executionResult.ResultsPath); + await backend.UploadScanResultsAsync(executionResult.ResultsPath, cancellationToken).ConfigureAwait(false); + logger.LogInformation("Scan artefact uploaded."); + activity?.SetTag("stellaops.cli.results", executionResult.ResultsPath); + } + else + { + logger.LogWarning("Skipping automatic upload because scan exited with code {Code}.", executionResult.ExitCode); + } + + logger.LogInformation("Run metadata written to {Path}.", executionResult.RunMetadataPath); + activity?.SetTag("stellaops.cli.run_metadata", executionResult.RunMetadataPath); + } catch (Exception ex) { logger.LogError(ex, "Scanner execution failed."); @@ -256,17 +260,17 @@ internal static class CommandHandlers } } - public static async Task HandleExportJobAsync( - IServiceProvider services, - string format, - bool delta, - bool? publishFull, - bool? publishDelta, - bool? includeFull, - bool? includeDelta, - bool verbose, - CancellationToken cancellationToken) - { + public static async Task HandleExportJobAsync( + IServiceProvider services, + string format, + bool delta, + bool? publishFull, + bool? publishDelta, + bool? includeFull, + bool? includeDelta, + bool verbose, + CancellationToken cancellationToken) + { await using var scope = services.CreateAsyncScope(); var client = scope.ServiceProvider.GetRequiredService(); var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("db-export"); @@ -277,55 +281,55 @@ internal static class CommandHandlers activity?.SetTag("stellaops.cli.command", "db export"); activity?.SetTag("stellaops.cli.format", format); activity?.SetTag("stellaops.cli.delta", delta); - using var duration = CliMetrics.MeasureCommandDuration("db export"); - activity?.SetTag("stellaops.cli.publish_full", publishFull); - activity?.SetTag("stellaops.cli.publish_delta", publishDelta); - activity?.SetTag("stellaops.cli.include_full", includeFull); - activity?.SetTag("stellaops.cli.include_delta", includeDelta); - - try - { - var jobKind = format switch - { - "trivy-db" or "trivy" => "export:trivy-db", - _ => "export:json" - }; - - var isTrivy = jobKind == "export:trivy-db"; - if (isTrivy - && !publishFull.HasValue - && !publishDelta.HasValue - && !includeFull.HasValue - && !includeDelta.HasValue - && AnsiConsole.Profile.Capabilities.Interactive) - { - var overrides = TrivyDbExportPrompt.PromptOverrides(); - publishFull = overrides.publishFull; - publishDelta = overrides.publishDelta; - includeFull = overrides.includeFull; - includeDelta = overrides.includeDelta; - } - - var parameters = new Dictionary(StringComparer.Ordinal) - { - ["delta"] = delta - }; - if (publishFull.HasValue) - { - parameters["publishFull"] = publishFull.Value; - } - if (publishDelta.HasValue) - { - parameters["publishDelta"] = publishDelta.Value; - } - if (includeFull.HasValue) - { - parameters["includeFull"] = includeFull.Value; - } - if (includeDelta.HasValue) - { - parameters["includeDelta"] = includeDelta.Value; - } + using var duration = CliMetrics.MeasureCommandDuration("db export"); + activity?.SetTag("stellaops.cli.publish_full", publishFull); + activity?.SetTag("stellaops.cli.publish_delta", publishDelta); + activity?.SetTag("stellaops.cli.include_full", includeFull); + activity?.SetTag("stellaops.cli.include_delta", includeDelta); + + try + { + var jobKind = format switch + { + "trivy-db" or "trivy" => "export:trivy-db", + _ => "export:json" + }; + + var isTrivy = jobKind == "export:trivy-db"; + if (isTrivy + && !publishFull.HasValue + && !publishDelta.HasValue + && !includeFull.HasValue + && !includeDelta.HasValue + && AnsiConsole.Profile.Capabilities.Interactive) + { + var overrides = TrivyDbExportPrompt.PromptOverrides(); + publishFull = overrides.publishFull; + publishDelta = overrides.publishDelta; + includeFull = overrides.includeFull; + includeDelta = overrides.includeDelta; + } + + var parameters = new Dictionary(StringComparer.Ordinal) + { + ["delta"] = delta + }; + if (publishFull.HasValue) + { + parameters["publishFull"] = publishFull.Value; + } + if (publishDelta.HasValue) + { + parameters["publishDelta"] = publishDelta.Value; + } + if (includeFull.HasValue) + { + parameters["includeFull"] = includeFull.Value; + } + if (includeDelta.HasValue) + { + parameters["includeDelta"] = includeDelta.Value; + } await TriggerJobAsync(client, logger, jobKind, parameters, cancellationToken).ConfigureAwait(false); } @@ -336,785 +340,2273 @@ internal static class CommandHandlers } finally { - verbosity.MinimumLevel = previousLevel; - } - } - - public static async Task HandleAuthLoginAsync( - IServiceProvider services, - StellaOpsCliOptions options, - bool verbose, - bool force, - CancellationToken cancellationToken) - { - await using var scope = services.CreateAsyncScope(); - var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-login"); - Environment.ExitCode = 0; - - if (string.IsNullOrWhiteSpace(options.Authority?.Url)) - { - logger.LogError("Authority URL is not configured. Set STELLAOPS_AUTHORITY_URL or update your configuration."); - Environment.ExitCode = 1; - return; - } - - var tokenClient = scope.ServiceProvider.GetService(); - if (tokenClient is null) - { - logger.LogError("Authority client is not available. Ensure AddStellaOpsAuthClient is registered in Program.cs."); - Environment.ExitCode = 1; - return; - } - - var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); - if (string.IsNullOrWhiteSpace(cacheKey)) - { - logger.LogError("Authority configuration is incomplete; unable to determine cache key."); - Environment.ExitCode = 1; - return; - } - - try - { - if (force) - { - await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); - } - - var scopeName = AuthorityTokenUtilities.ResolveScope(options); - StellaOpsTokenResult token; - - if (!string.IsNullOrWhiteSpace(options.Authority.Username)) - { - if (string.IsNullOrWhiteSpace(options.Authority.Password)) - { - logger.LogError("Authority password must be provided when username is configured."); - Environment.ExitCode = 1; - return; - } - - token = await tokenClient.RequestPasswordTokenAsync( - options.Authority.Username, - options.Authority.Password!, - scopeName, - cancellationToken).ConfigureAwait(false); - } - else - { - token = await tokenClient.RequestClientCredentialsTokenAsync(scopeName, cancellationToken).ConfigureAwait(false); - } - - await tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); - - if (verbose) - { - logger.LogInformation("Authenticated with {Authority} (scopes: {Scopes}).", options.Authority.Url, string.Join(", ", token.Scopes)); - } - - logger.LogInformation("Login successful. Access token expires at {Expires}.", token.ExpiresAtUtc.ToString("u")); - } - catch (Exception ex) - { - logger.LogError(ex, "Authentication failed: {Message}", ex.Message); - Environment.ExitCode = 1; - } - } - - public static async Task HandleAuthLogoutAsync( - IServiceProvider services, - StellaOpsCliOptions options, - bool verbose, - CancellationToken cancellationToken) - { - await using var scope = services.CreateAsyncScope(); - var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-logout"); - Environment.ExitCode = 0; - - var tokenClient = scope.ServiceProvider.GetService(); - if (tokenClient is null) - { - logger.LogInformation("No authority client registered; nothing to remove."); - return; - } - - var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); - if (string.IsNullOrWhiteSpace(cacheKey)) - { - logger.LogInformation("Authority configuration missing; no cached tokens to remove."); - return; - } - - await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (verbose) - { - logger.LogInformation("Cleared cached token for {Authority}.", options.Authority?.Url ?? "authority"); - } - } - - public static async Task HandleAuthStatusAsync( - IServiceProvider services, - StellaOpsCliOptions options, - bool verbose, - CancellationToken cancellationToken) - { - await using var scope = services.CreateAsyncScope(); - var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-status"); - Environment.ExitCode = 0; - - if (string.IsNullOrWhiteSpace(options.Authority?.Url)) - { - logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); - Environment.ExitCode = 1; - return; - } - - var tokenClient = scope.ServiceProvider.GetService(); - if (tokenClient is null) - { - logger.LogInformation("Authority client not registered; no cached tokens available."); - Environment.ExitCode = 1; - return; - } - - var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); - if (string.IsNullOrWhiteSpace(cacheKey)) - { - logger.LogInformation("Authority configuration incomplete; no cached tokens available."); - Environment.ExitCode = 1; - return; - } - - var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (entry is null) - { - logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); - Environment.ExitCode = 1; - return; - } - - logger.LogInformation("Cached token for {Authority} expires at {Expires}.", options.Authority.Url, entry.ExpiresAtUtc.ToString("u")); - if (verbose) - { - logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); - } - } - - public static async Task HandleAuthWhoAmIAsync( - IServiceProvider services, - StellaOpsCliOptions options, - bool verbose, - CancellationToken cancellationToken) - { - await using var scope = services.CreateAsyncScope(); - var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-whoami"); - Environment.ExitCode = 0; - - if (string.IsNullOrWhiteSpace(options.Authority?.Url)) - { - logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); - Environment.ExitCode = 1; - return; - } - - var tokenClient = scope.ServiceProvider.GetService(); - if (tokenClient is null) - { - logger.LogInformation("Authority client not registered; no cached tokens available."); - Environment.ExitCode = 1; - return; - } - - var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); - if (string.IsNullOrWhiteSpace(cacheKey)) - { - logger.LogInformation("Authority configuration incomplete; no cached tokens available."); - Environment.ExitCode = 1; - return; - } - - var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (entry is null) - { - logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); - Environment.ExitCode = 1; - return; - } - - var grantType = string.IsNullOrWhiteSpace(options.Authority.Username) ? "client_credentials" : "password"; - var now = DateTimeOffset.UtcNow; - var remaining = entry.ExpiresAtUtc - now; - if (remaining < TimeSpan.Zero) - { - remaining = TimeSpan.Zero; - } - - logger.LogInformation("Authority: {Authority}", options.Authority.Url); - logger.LogInformation("Grant type: {GrantType}", grantType); - logger.LogInformation("Token type: {TokenType}", entry.TokenType); - logger.LogInformation("Expires: {Expires} ({Remaining})", entry.ExpiresAtUtc.ToString("u"), FormatDuration(remaining)); - - if (entry.Scopes.Count > 0) - { - logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); - } - - if (TryExtractJwtClaims(entry.AccessToken, out var claims, out var issuedAt, out var notBefore)) - { - if (claims.TryGetValue("sub", out var subject) && !string.IsNullOrWhiteSpace(subject)) - { - logger.LogInformation("Subject: {Subject}", subject); - } - - if (claims.TryGetValue("client_id", out var clientId) && !string.IsNullOrWhiteSpace(clientId)) - { - logger.LogInformation("Client ID (token): {ClientId}", clientId); - } - - if (claims.TryGetValue("aud", out var audience) && !string.IsNullOrWhiteSpace(audience)) - { - logger.LogInformation("Audience: {Audience}", audience); - } - - if (claims.TryGetValue("iss", out var issuer) && !string.IsNullOrWhiteSpace(issuer)) - { - logger.LogInformation("Issuer: {Issuer}", issuer); - } - - if (issuedAt is not null) - { - logger.LogInformation("Issued at: {IssuedAt}", issuedAt.Value.ToString("u")); - } - - if (notBefore is not null) - { - logger.LogInformation("Not before: {NotBefore}", notBefore.Value.ToString("u")); - } - - var extraClaims = CollectAdditionalClaims(claims); - if (extraClaims.Count > 0 && verbose) - { - logger.LogInformation("Additional claims: {Claims}", string.Join(", ", extraClaims)); - } - } - else - { - logger.LogInformation("Access token appears opaque; claims are unavailable."); - } - } - - public static async Task HandleAuthRevokeExportAsync( - IServiceProvider services, - StellaOpsCliOptions options, - string? outputDirectory, - bool verbose, - CancellationToken cancellationToken) - { - await using var scope = services.CreateAsyncScope(); - var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-revoke-export"); - Environment.ExitCode = 0; - - try - { - var client = scope.ServiceProvider.GetRequiredService(); - var result = await client.ExportAsync(verbose, cancellationToken).ConfigureAwait(false); - - var directory = string.IsNullOrWhiteSpace(outputDirectory) - ? Directory.GetCurrentDirectory() - : Path.GetFullPath(outputDirectory); - - Directory.CreateDirectory(directory); - - var bundlePath = Path.Combine(directory, "revocation-bundle.json"); - var signaturePath = Path.Combine(directory, "revocation-bundle.json.jws"); - var digestPath = Path.Combine(directory, "revocation-bundle.json.sha256"); - - await File.WriteAllBytesAsync(bundlePath, result.BundleBytes, cancellationToken).ConfigureAwait(false); - await File.WriteAllTextAsync(signaturePath, result.Signature, cancellationToken).ConfigureAwait(false); - await File.WriteAllTextAsync(digestPath, $"sha256:{result.Digest}", cancellationToken).ConfigureAwait(false); - - var computedDigest = Convert.ToHexString(SHA256.HashData(result.BundleBytes)).ToLowerInvariant(); - if (!string.Equals(computedDigest, result.Digest, StringComparison.OrdinalIgnoreCase)) - { - logger.LogError("Digest mismatch. Expected {Expected} but computed {Actual}.", result.Digest, computedDigest); - Environment.ExitCode = 1; - return; - } - - logger.LogInformation( - "Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}, provider {Provider}).", - directory, - result.Sequence, - result.IssuedAt, - string.IsNullOrWhiteSpace(result.SigningKeyId) ? "" : result.SigningKeyId, - string.IsNullOrWhiteSpace(result.SigningProvider) ? "default" : result.SigningProvider); - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to export revocation bundle."); - Environment.ExitCode = 1; - } - } - - public static async Task HandleAuthRevokeVerifyAsync( - string bundlePath, - string signaturePath, - string keyPath, - bool verbose, - CancellationToken cancellationToken) - { - var loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "HH:mm:ss "; - })); - var logger = loggerFactory.CreateLogger("auth-revoke-verify"); - Environment.ExitCode = 0; - - try - { - if (string.IsNullOrWhiteSpace(bundlePath) || string.IsNullOrWhiteSpace(signaturePath) || string.IsNullOrWhiteSpace(keyPath)) - { - logger.LogError("Arguments --bundle, --signature, and --key are required."); - Environment.ExitCode = 1; - return; - } - - var bundleBytes = await File.ReadAllBytesAsync(bundlePath, cancellationToken).ConfigureAwait(false); - var signatureContent = (await File.ReadAllTextAsync(signaturePath, cancellationToken).ConfigureAwait(false)).Trim(); - var keyPem = await File.ReadAllTextAsync(keyPath, cancellationToken).ConfigureAwait(false); - - var digest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant(); - logger.LogInformation("Bundle digest sha256:{Digest}", digest); - - if (!TryParseDetachedJws(signatureContent, out var encodedHeader, out var encodedSignature)) - { - logger.LogError("Signature is not in detached JWS format."); - Environment.ExitCode = 1; - return; - } - - var headerJson = Encoding.UTF8.GetString(Base64UrlDecode(encodedHeader)); - using var headerDocument = JsonDocument.Parse(headerJson); - var header = headerDocument.RootElement; - - if (!header.TryGetProperty("b64", out var b64Element) || b64Element.GetBoolean()) - { - logger.LogError("Detached JWS header must include '\"b64\": false'."); - Environment.ExitCode = 1; - return; - } - - var algorithm = header.TryGetProperty("alg", out var algElement) ? algElement.GetString() : SignatureAlgorithms.Es256; - if (string.IsNullOrWhiteSpace(algorithm)) - { - algorithm = SignatureAlgorithms.Es256; - } - - var providerHint = header.TryGetProperty("provider", out var providerElement) - ? providerElement.GetString() - : null; - - var keyId = header.TryGetProperty("kid", out var kidElement) ? kidElement.GetString() : null; - if (string.IsNullOrWhiteSpace(keyId)) - { - keyId = Path.GetFileNameWithoutExtension(keyPath); - logger.LogWarning("JWS header missing 'kid'; using fallback key id {KeyId}.", keyId); - } - - CryptoSigningKey signingKey; - try - { - signingKey = CreateVerificationSigningKey(keyId!, algorithm!, providerHint, keyPem, keyPath); - } - catch (Exception ex) when (ex is InvalidOperationException or CryptographicException) - { - logger.LogError(ex, "Failed to load verification key material."); - Environment.ExitCode = 1; - return; - } - - var providers = new List - { - new DefaultCryptoProvider() - }; - -#if STELLAOPS_CRYPTO_SODIUM - providers.Add(new LibsodiumCryptoProvider()); -#endif - - foreach (var provider in providers) - { - if (provider.Supports(CryptoCapability.Verification, algorithm!)) - { - provider.UpsertSigningKey(signingKey); - } - } - - var preferredOrder = !string.IsNullOrWhiteSpace(providerHint) - ? new[] { providerHint! } - : Array.Empty(); - var registry = new CryptoProviderRegistry(providers, preferredOrder); - CryptoSignerResolution resolution; - try - { - resolution = registry.ResolveSigner( - CryptoCapability.Verification, - algorithm!, - signingKey.Reference, - providerHint); - } - catch (Exception ex) - { - logger.LogError(ex, "No crypto provider available for verification (algorithm {Algorithm}).", algorithm); - Environment.ExitCode = 1; - return; - } - - var signingInputLength = encodedHeader.Length + 1 + bundleBytes.Length; - var buffer = ArrayPool.Shared.Rent(signingInputLength); - try - { - var headerBytes = Encoding.ASCII.GetBytes(encodedHeader); - Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length); - buffer[headerBytes.Length] = (byte)'.'; - Buffer.BlockCopy(bundleBytes, 0, buffer, headerBytes.Length + 1, bundleBytes.Length); - - var signatureBytes = Base64UrlDecode(encodedSignature); - var verified = await resolution.Signer.VerifyAsync( - new ReadOnlyMemory(buffer, 0, signingInputLength), - signatureBytes, - cancellationToken).ConfigureAwait(false); - - if (!verified) - { - logger.LogError("Signature verification failed."); - Environment.ExitCode = 1; - return; - } - } - finally - { - ArrayPool.Shared.Return(buffer); - } - - if (!string.IsNullOrWhiteSpace(providerHint) && !string.Equals(providerHint, resolution.ProviderName, StringComparison.OrdinalIgnoreCase)) - { - logger.LogWarning( - "Preferred provider '{Preferred}' unavailable; verification used '{Provider}'.", - providerHint, - resolution.ProviderName); - } - - logger.LogInformation( - "Signature verified using algorithm {Algorithm} via provider {Provider} (kid {KeyId}).", - algorithm, - resolution.ProviderName, - signingKey.Reference.KeyId); - - if (verbose) - { - logger.LogInformation("JWS header: {Header}", headerJson); - } - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to verify revocation bundle."); - Environment.ExitCode = 1; - } - finally - { - loggerFactory.Dispose(); - } - } - - private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) - { - encodedHeader = string.Empty; - encodedSignature = string.Empty; - - if (string.IsNullOrWhiteSpace(value)) - { - return false; - } - - var parts = value.Split('.'); - if (parts.Length != 3) - { - return false; - } - - encodedHeader = parts[0]; - encodedSignature = parts[2]; - return parts[1].Length == 0; - } - - private static byte[] Base64UrlDecode(string value) - { - var normalized = value.Replace('-', '+').Replace('_', '/'); - var padding = normalized.Length % 4; - if (padding == 2) - { - normalized += "=="; - } - else if (padding == 3) - { - normalized += "="; - } - else if (padding == 1) - { - throw new FormatException("Invalid Base64Url value."); - } - - return Convert.FromBase64String(normalized); - } - - private static CryptoSigningKey CreateVerificationSigningKey( - string keyId, - string algorithm, - string? providerHint, - string keyPem, - string keyPath) - { - if (string.IsNullOrWhiteSpace(keyPem)) - { - throw new InvalidOperationException("Verification key PEM content is empty."); - } - - using var ecdsa = ECDsa.Create(); - ecdsa.ImportFromPem(keyPem); - - var parameters = ecdsa.ExportParameters(includePrivateParameters: false); - if (parameters.D is null || parameters.D.Length == 0) - { - parameters.D = new byte[] { 0x01 }; - } - - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["source"] = Path.GetFullPath(keyPath), - ["verificationOnly"] = "true" - }; - - return new CryptoSigningKey( - new CryptoKeyReference(keyId, providerHint), - algorithm, - in parameters, - DateTimeOffset.UtcNow, - metadata: metadata); - } - - private static string FormatDuration(TimeSpan duration) - { - if (duration <= TimeSpan.Zero) - { - return "expired"; - } - - if (duration.TotalDays >= 1) - { - var days = (int)duration.TotalDays; - var hours = duration.Hours; - return hours > 0 - ? FormattableString.Invariant($"{days}d {hours}h") - : FormattableString.Invariant($"{days}d"); - } - - if (duration.TotalHours >= 1) - { - return FormattableString.Invariant($"{(int)duration.TotalHours}h {duration.Minutes}m"); - } - - if (duration.TotalMinutes >= 1) - { - return FormattableString.Invariant($"{(int)duration.TotalMinutes}m {duration.Seconds}s"); - } - - return FormattableString.Invariant($"{duration.Seconds}s"); - } - - private static bool TryExtractJwtClaims( - string accessToken, - out Dictionary claims, - out DateTimeOffset? issuedAt, - out DateTimeOffset? notBefore) - { - claims = new Dictionary(StringComparer.OrdinalIgnoreCase); - issuedAt = null; - notBefore = null; - - if (string.IsNullOrWhiteSpace(accessToken)) - { - return false; - } - - var parts = accessToken.Split('.'); - if (parts.Length < 2) - { - return false; - } - - if (!TryDecodeBase64Url(parts[1], out var payloadBytes)) - { - return false; - } - - try - { - using var document = JsonDocument.Parse(payloadBytes); - foreach (var property in document.RootElement.EnumerateObject()) - { - var value = FormatJsonValue(property.Value); - claims[property.Name] = value; - - if (issuedAt is null && property.NameEquals("iat") && TryParseUnixSeconds(property.Value, out var parsedIat)) - { - issuedAt = parsedIat; - } - - if (notBefore is null && property.NameEquals("nbf") && TryParseUnixSeconds(property.Value, out var parsedNbf)) - { - notBefore = parsedNbf; - } - } - - return true; - } - catch (JsonException) - { - claims.Clear(); - issuedAt = null; - notBefore = null; - return false; - } - } - - private static bool TryDecodeBase64Url(string value, out byte[] bytes) - { - bytes = Array.Empty(); - - if (string.IsNullOrWhiteSpace(value)) - { - return false; - } - - var normalized = value.Replace('-', '+').Replace('_', '/'); - var padding = normalized.Length % 4; - if (padding is 2 or 3) - { - normalized = normalized.PadRight(normalized.Length + (4 - padding), '='); - } - else if (padding == 1) - { - return false; - } - - try - { - bytes = Convert.FromBase64String(normalized); - return true; - } - catch (FormatException) - { - return false; - } - } - - private static string FormatJsonValue(JsonElement element) - { - return element.ValueKind switch - { - JsonValueKind.String => element.GetString() ?? string.Empty, - JsonValueKind.Number => element.TryGetInt64(out var longValue) - ? longValue.ToString(CultureInfo.InvariantCulture) - : element.GetDouble().ToString(CultureInfo.InvariantCulture), - JsonValueKind.True => "true", - JsonValueKind.False => "false", - JsonValueKind.Null => "null", - JsonValueKind.Array => FormatArray(element), - JsonValueKind.Object => element.GetRawText(), - _ => element.GetRawText() - }; - } - - private static string FormatArray(JsonElement array) - { - var values = new List(); - foreach (var item in array.EnumerateArray()) - { - values.Add(FormatJsonValue(item)); - } - - return string.Join(", ", values); - } - - private static bool TryParseUnixSeconds(JsonElement element, out DateTimeOffset value) - { - value = default; - - if (element.ValueKind == JsonValueKind.Number) - { - if (element.TryGetInt64(out var seconds)) - { - value = DateTimeOffset.FromUnixTimeSeconds(seconds); - return true; - } - - if (element.TryGetDouble(out var doubleValue)) - { - value = DateTimeOffset.FromUnixTimeSeconds((long)doubleValue); - return true; - } - } - - if (element.ValueKind == JsonValueKind.String) - { - var text = element.GetString(); - if (!string.IsNullOrWhiteSpace(text) && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) - { - value = DateTimeOffset.FromUnixTimeSeconds(seconds); - return true; - } - } - - return false; - } - - private static List CollectAdditionalClaims(Dictionary claims) - { - var result = new List(); - foreach (var pair in claims) - { - if (CommonClaimNames.Contains(pair.Key)) - { - continue; - } - - result.Add(FormattableString.Invariant($"{pair.Key}={pair.Value}")); - } - - result.Sort(StringComparer.OrdinalIgnoreCase); - return result; - } - - private static readonly HashSet CommonClaimNames = new(StringComparer.OrdinalIgnoreCase) - { - "aud", - "client_id", - "exp", - "iat", - "iss", - "nbf", - "scope", - "scopes", - "sub", - "token_type", - "jti" - }; - - private static async Task TriggerJobAsync( - IBackendOperationsClient client, - ILogger logger, - string jobKind, + verbosity.MinimumLevel = previousLevel; + } + } + + public static Task HandleExcititorInitAsync( + IServiceProvider services, + IReadOnlyList providers, + bool resume, + bool verbose, + CancellationToken cancellationToken) + { + var normalizedProviders = NormalizeProviders(providers); + var payload = new Dictionary(StringComparer.Ordinal); + if (normalizedProviders.Count > 0) + { + payload["providers"] = normalizedProviders; + } + if (resume) + { + payload["resume"] = true; + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor init", + verbose, + new Dictionary + { + ["providers"] = normalizedProviders.Count, + ["resume"] = resume + }, + client => client.ExecuteExcititorOperationAsync("init", HttpMethod.Post, RemoveNullValues(payload), cancellationToken), + cancellationToken); + } + + public static Task HandleExcititorPullAsync( + IServiceProvider services, + IReadOnlyList providers, + DateTimeOffset? since, + TimeSpan? window, + bool force, + bool verbose, + CancellationToken cancellationToken) + { + var normalizedProviders = NormalizeProviders(providers); + var payload = new Dictionary(StringComparer.Ordinal); + if (normalizedProviders.Count > 0) + { + payload["providers"] = normalizedProviders; + } + if (since.HasValue) + { + payload["since"] = since.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + } + if (window.HasValue) + { + payload["window"] = window.Value.ToString("c", CultureInfo.InvariantCulture); + } + if (force) + { + payload["force"] = true; + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor pull", + verbose, + new Dictionary + { + ["providers"] = normalizedProviders.Count, + ["force"] = force, + ["since"] = since?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture), + ["window"] = window?.ToString("c", CultureInfo.InvariantCulture) + }, + client => client.ExecuteExcititorOperationAsync("ingest/run", HttpMethod.Post, RemoveNullValues(payload), cancellationToken), + cancellationToken); + } + + public static Task HandleExcititorResumeAsync( + IServiceProvider services, + IReadOnlyList providers, + string? checkpoint, + bool verbose, + CancellationToken cancellationToken) + { + var normalizedProviders = NormalizeProviders(providers); + var payload = new Dictionary(StringComparer.Ordinal); + if (normalizedProviders.Count > 0) + { + payload["providers"] = normalizedProviders; + } + if (!string.IsNullOrWhiteSpace(checkpoint)) + { + payload["checkpoint"] = checkpoint.Trim(); + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor resume", + verbose, + new Dictionary + { + ["providers"] = normalizedProviders.Count, + ["checkpoint"] = checkpoint + }, + client => client.ExecuteExcititorOperationAsync("ingest/resume", HttpMethod.Post, RemoveNullValues(payload), cancellationToken), + cancellationToken); + } + + public static async Task HandleExcititorListProvidersAsync( + IServiceProvider services, + bool includeDisabled, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("excititor-list-providers"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.excititor.list-providers", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "excititor list-providers"); + activity?.SetTag("stellaops.cli.include_disabled", includeDisabled); + using var duration = CliMetrics.MeasureCommandDuration("excititor list-providers"); + + try + { + var providers = await client.GetExcititorProvidersAsync(includeDisabled, cancellationToken).ConfigureAwait(false); + Environment.ExitCode = 0; + logger.LogInformation("Providers returned: {Count}", providers.Count); + + if (providers.Count > 0) + { + if (AnsiConsole.Profile.Capabilities.Interactive) + { + var table = new Table().Border(TableBorder.Rounded).AddColumns("Provider", "Kind", "Trust", "Enabled", "Last Ingested"); + foreach (var provider in providers) + { + table.AddRow( + provider.Id, + provider.Kind, + string.IsNullOrWhiteSpace(provider.TrustTier) ? "-" : provider.TrustTier, + provider.Enabled ? "yes" : "no", + provider.LastIngestedAt?.ToString("yyyy-MM-dd HH:mm:ss 'UTC'", CultureInfo.InvariantCulture) ?? "unknown"); + } + + AnsiConsole.Write(table); + } + else + { + foreach (var provider in providers) + { + logger.LogInformation("{ProviderId} [{Kind}] Enabled={Enabled} Trust={Trust} LastIngested={LastIngested}", + provider.Id, + provider.Kind, + provider.Enabled ? "yes" : "no", + string.IsNullOrWhiteSpace(provider.TrustTier) ? "-" : provider.TrustTier, + provider.LastIngestedAt?.ToString("O", CultureInfo.InvariantCulture) ?? "unknown"); + } + } + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to list Excititor providers."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleExcititorExportAsync( + IServiceProvider services, + string format, + bool delta, + string? scope, + DateTimeOffset? since, + string? provider, + string? outputPath, + bool verbose, + CancellationToken cancellationToken) + { + await using var scopeHandle = services.CreateAsyncScope(); + var client = scopeHandle.ServiceProvider.GetRequiredService(); + var logger = scopeHandle.ServiceProvider.GetRequiredService().CreateLogger("excititor-export"); + var options = scopeHandle.ServiceProvider.GetRequiredService(); + var verbosity = scopeHandle.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.excititor.export", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "excititor export"); + activity?.SetTag("stellaops.cli.format", format); + activity?.SetTag("stellaops.cli.delta", delta); + if (!string.IsNullOrWhiteSpace(scope)) + { + activity?.SetTag("stellaops.cli.scope", scope); + } + if (since.HasValue) + { + activity?.SetTag("stellaops.cli.since", since.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)); + } + if (!string.IsNullOrWhiteSpace(provider)) + { + activity?.SetTag("stellaops.cli.provider", provider); + } + if (!string.IsNullOrWhiteSpace(outputPath)) + { + activity?.SetTag("stellaops.cli.output", outputPath); + } + using var duration = CliMetrics.MeasureCommandDuration("excititor export"); + + try + { + var payload = new Dictionary(StringComparer.Ordinal) + { + ["format"] = string.IsNullOrWhiteSpace(format) ? "openvex" : format.Trim(), + ["delta"] = delta + }; + + if (!string.IsNullOrWhiteSpace(scope)) + { + payload["scope"] = scope.Trim(); + } + if (since.HasValue) + { + payload["since"] = since.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + } + if (!string.IsNullOrWhiteSpace(provider)) + { + payload["provider"] = provider.Trim(); + } + + var result = await client.ExecuteExcititorOperationAsync( + "export", + HttpMethod.Post, + RemoveNullValues(payload), + cancellationToken).ConfigureAwait(false); + + if (!result.Success) + { + logger.LogError(string.IsNullOrWhiteSpace(result.Message) ? "Excititor export failed." : result.Message); + Environment.ExitCode = 1; + return; + } + + Environment.ExitCode = 0; + + var manifest = TryParseExportManifest(result.Payload); + if (!string.IsNullOrWhiteSpace(result.Message) + && (manifest is null || !string.Equals(result.Message, "ok", StringComparison.OrdinalIgnoreCase))) + { + logger.LogInformation(result.Message); + } + + if (manifest is not null) + { + activity?.SetTag("stellaops.cli.export_id", manifest.ExportId); + if (!string.IsNullOrWhiteSpace(manifest.Format)) + { + activity?.SetTag("stellaops.cli.export_format", manifest.Format); + } + if (manifest.FromCache.HasValue) + { + activity?.SetTag("stellaops.cli.export_cached", manifest.FromCache.Value); + } + if (manifest.SizeBytes.HasValue) + { + activity?.SetTag("stellaops.cli.export_size", manifest.SizeBytes.Value); + } + + if (manifest.FromCache == true) + { + logger.LogInformation("Reusing cached export {ExportId} ({Format}).", manifest.ExportId, manifest.Format ?? "unknown"); + } + else + { + logger.LogInformation("Export ready: {ExportId} ({Format}).", manifest.ExportId, manifest.Format ?? "unknown"); + } + + if (manifest.CreatedAt.HasValue) + { + logger.LogInformation("Created at {CreatedAt}.", manifest.CreatedAt.Value.ToString("u", CultureInfo.InvariantCulture)); + } + + if (!string.IsNullOrWhiteSpace(manifest.Digest)) + { + var digestDisplay = BuildDigestDisplay(manifest.Algorithm, manifest.Digest); + if (manifest.SizeBytes.HasValue) + { + logger.LogInformation("Digest {Digest} ({Size}).", digestDisplay, FormatSize(manifest.SizeBytes.Value)); + } + else + { + logger.LogInformation("Digest {Digest}.", digestDisplay); + } + } + + if (!string.IsNullOrWhiteSpace(manifest.RekorLocation)) + { + if (!string.IsNullOrWhiteSpace(manifest.RekorIndex)) + { + logger.LogInformation("Rekor entry: {Location} (index {Index}).", manifest.RekorLocation, manifest.RekorIndex); + } + else + { + logger.LogInformation("Rekor entry: {Location}.", manifest.RekorLocation); + } + } + + if (!string.IsNullOrWhiteSpace(manifest.RekorInclusionUrl) + && !string.Equals(manifest.RekorInclusionUrl, manifest.RekorLocation, StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Rekor inclusion proof: {Url}.", manifest.RekorInclusionUrl); + } + + if (!string.IsNullOrWhiteSpace(outputPath)) + { + var resolvedPath = ResolveExportOutputPath(outputPath!, manifest); + var download = await client.DownloadExcititorExportAsync( + manifest.ExportId, + resolvedPath, + manifest.Algorithm, + manifest.Digest, + cancellationToken).ConfigureAwait(false); + + activity?.SetTag("stellaops.cli.export_path", download.Path); + + if (download.FromCache) + { + logger.LogInformation("Export already cached at {Path} ({Size}).", download.Path, FormatSize(download.SizeBytes)); + } + else + { + logger.LogInformation("Export saved to {Path} ({Size}).", download.Path, FormatSize(download.SizeBytes)); + } + } + else if (!string.IsNullOrWhiteSpace(result.Location)) + { + var downloadUrl = ResolveLocationUrl(options, result.Location); + if (!string.IsNullOrWhiteSpace(downloadUrl)) + { + logger.LogInformation("Download URL: {Url}", downloadUrl); + } + else + { + logger.LogInformation("Download location: {Location}", result.Location); + } + } + } + else + { + if (!string.IsNullOrWhiteSpace(result.Location)) + { + var downloadUrl = ResolveLocationUrl(options, result.Location); + if (!string.IsNullOrWhiteSpace(downloadUrl)) + { + logger.LogInformation("Download URL: {Url}", downloadUrl); + } + else + { + logger.LogInformation("Location: {Location}", result.Location); + } + } + else if (string.IsNullOrWhiteSpace(result.Message)) + { + logger.LogInformation("Export request accepted."); + } + } + } + catch (Exception ex) + { + logger.LogError(ex, "Excititor export failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static Task HandleExcititorBackfillStatementsAsync( + IServiceProvider services, + DateTimeOffset? retrievedSince, + bool force, + int batchSize, + int? maxDocuments, + bool verbose, + CancellationToken cancellationToken) + { + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), "Batch size must be greater than zero."); + } + + if (maxDocuments.HasValue && maxDocuments.Value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(maxDocuments), "Max documents must be greater than zero when specified."); + } + + var payload = new Dictionary(StringComparer.Ordinal) + { + ["force"] = force, + ["batchSize"] = batchSize, + ["maxDocuments"] = maxDocuments + }; + + if (retrievedSince.HasValue) + { + payload["retrievedSince"] = retrievedSince.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + } + + var activityTags = new Dictionary(StringComparer.Ordinal) + { + ["stellaops.cli.force"] = force, + ["stellaops.cli.batch_size"] = batchSize, + ["stellaops.cli.max_documents"] = maxDocuments + }; + + if (retrievedSince.HasValue) + { + activityTags["stellaops.cli.retrieved_since"] = retrievedSince.Value.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor backfill-statements", + verbose, + activityTags, + client => client.ExecuteExcititorOperationAsync( + "admin/backfill-statements", + HttpMethod.Post, + RemoveNullValues(payload), + cancellationToken), + cancellationToken); + } + + public static Task HandleExcititorVerifyAsync( + IServiceProvider services, + string? exportId, + string? digest, + string? attestationPath, + bool verbose, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(exportId) && string.IsNullOrWhiteSpace(digest) && string.IsNullOrWhiteSpace(attestationPath)) + { + var logger = services.GetRequiredService().CreateLogger("excititor-verify"); + logger.LogError("At least one of --export-id, --digest, or --attestation must be provided."); + Environment.ExitCode = 1; + return Task.CompletedTask; + } + + var payload = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(exportId)) + { + payload["exportId"] = exportId.Trim(); + } + if (!string.IsNullOrWhiteSpace(digest)) + { + payload["digest"] = digest.Trim(); + } + if (!string.IsNullOrWhiteSpace(attestationPath)) + { + var fullPath = Path.GetFullPath(attestationPath); + if (!File.Exists(fullPath)) + { + var logger = services.GetRequiredService().CreateLogger("excititor-verify"); + logger.LogError("Attestation file not found at {Path}.", fullPath); + Environment.ExitCode = 1; + return Task.CompletedTask; + } + + var bytes = File.ReadAllBytes(fullPath); + payload["attestation"] = new Dictionary(StringComparer.Ordinal) + { + ["fileName"] = Path.GetFileName(fullPath), + ["base64"] = Convert.ToBase64String(bytes) + }; + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor verify", + verbose, + new Dictionary + { + ["export_id"] = exportId, + ["digest"] = digest, + ["attestation_path"] = attestationPath + }, + client => client.ExecuteExcititorOperationAsync("verify", HttpMethod.Post, RemoveNullValues(payload), cancellationToken), + cancellationToken); + } + + public static Task HandleExcititorReconcileAsync( + IServiceProvider services, + IReadOnlyList providers, + TimeSpan? maxAge, + bool verbose, + CancellationToken cancellationToken) + { + var normalizedProviders = NormalizeProviders(providers); + var payload = new Dictionary(StringComparer.Ordinal); + if (normalizedProviders.Count > 0) + { + payload["providers"] = normalizedProviders; + } + if (maxAge.HasValue) + { + payload["maxAge"] = maxAge.Value.ToString("c", CultureInfo.InvariantCulture); + } + + return ExecuteExcititorCommandAsync( + services, + commandName: "excititor reconcile", + verbose, + new Dictionary + { + ["providers"] = normalizedProviders.Count, + ["max_age"] = maxAge?.ToString("c", CultureInfo.InvariantCulture) + }, + client => client.ExecuteExcititorOperationAsync("reconcile", HttpMethod.Post, RemoveNullValues(payload), cancellationToken), + cancellationToken); + } + + public static async Task HandleRuntimePolicyTestAsync( + IServiceProvider services, + string? namespaceValue, + IReadOnlyList imageArguments, + string? filePath, + IReadOnlyList labelArguments, + bool outputJson, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("runtime-policy-test"); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity("cli.runtime.policy.test", ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", "runtime policy test"); + if (!string.IsNullOrWhiteSpace(namespaceValue)) + { + activity?.SetTag("stellaops.cli.namespace", namespaceValue); + } + using var duration = CliMetrics.MeasureCommandDuration("runtime policy test"); + + try + { + IReadOnlyList images; + try + { + images = await GatherImageDigestsAsync(imageArguments, filePath, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException or FileNotFoundException) + { + logger.LogError(ex, "Failed to gather image digests: {Message}", ex.Message); + Environment.ExitCode = 9; + return; + } + + if (images.Count == 0) + { + logger.LogError("No image digests provided. Use --image, --file, or pipe digests via stdin."); + Environment.ExitCode = 9; + return; + } + + IReadOnlyDictionary labels; + try + { + labels = ParseLabelSelectors(labelArguments); + } + catch (ArgumentException ex) + { + logger.LogError(ex.Message); + Environment.ExitCode = 9; + return; + } + + activity?.SetTag("stellaops.cli.images", images.Count); + activity?.SetTag("stellaops.cli.labels", labels.Count); + + var request = new RuntimePolicyEvaluationRequest(namespaceValue, labels, images); + var result = await client.EvaluateRuntimePolicyAsync(request, cancellationToken).ConfigureAwait(false); + + activity?.SetTag("stellaops.cli.ttl_seconds", result.TtlSeconds); + Environment.ExitCode = 0; + + if (outputJson) + { + var json = BuildRuntimePolicyJson(result, images); + Console.WriteLine(json); + return; + } + + if (result.ExpiresAtUtc.HasValue) + { + logger.LogInformation("Decision TTL: {TtlSeconds}s (expires {ExpiresAt})", result.TtlSeconds, result.ExpiresAtUtc.Value.ToString("u", CultureInfo.InvariantCulture)); + } + else + { + logger.LogInformation("Decision TTL: {TtlSeconds}s", result.TtlSeconds); + } + + if (!string.IsNullOrWhiteSpace(result.PolicyRevision)) + { + logger.LogInformation("Policy revision: {Revision}", result.PolicyRevision); + } + + DisplayRuntimePolicyResults(logger, result, images); + } + catch (Exception ex) + { + logger.LogError(ex, "Runtime policy evaluation failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + public static async Task HandleAuthLoginAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + bool force, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-login"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogError("Authority URL is not configured. Set STELLAOPS_AUTHORITY_URL or update your configuration."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogError("Authority client is not available. Ensure AddStellaOpsAuthClient is registered in Program.cs."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogError("Authority configuration is incomplete; unable to determine cache key."); + Environment.ExitCode = 1; + return; + } + + try + { + if (force) + { + await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + } + + var scopeName = AuthorityTokenUtilities.ResolveScope(options); + StellaOpsTokenResult token; + + if (!string.IsNullOrWhiteSpace(options.Authority.Username)) + { + if (string.IsNullOrWhiteSpace(options.Authority.Password)) + { + logger.LogError("Authority password must be provided when username is configured."); + Environment.ExitCode = 1; + return; + } + + token = await tokenClient.RequestPasswordTokenAsync( + options.Authority.Username, + options.Authority.Password!, + scopeName, + cancellationToken).ConfigureAwait(false); + } + else + { + token = await tokenClient.RequestClientCredentialsTokenAsync(scopeName, cancellationToken).ConfigureAwait(false); + } + + await tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); + + if (verbose) + { + logger.LogInformation("Authenticated with {Authority} (scopes: {Scopes}).", options.Authority.Url, string.Join(", ", token.Scopes)); + } + + logger.LogInformation("Login successful. Access token expires at {Expires}.", token.ExpiresAtUtc.ToString("u")); + } + catch (Exception ex) + { + logger.LogError(ex, "Authentication failed: {Message}", ex.Message); + Environment.ExitCode = 1; + } + } + + public static async Task HandleAuthLogoutAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-logout"); + Environment.ExitCode = 0; + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("No authority client registered; nothing to remove."); + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration missing; no cached tokens to remove."); + return; + } + + await tokenClient.ClearCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (verbose) + { + logger.LogInformation("Cleared cached token for {Authority}.", options.Authority?.Url ?? "authority"); + } + } + + public static async Task HandleAuthStatusAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-status"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("Authority client not registered; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration incomplete; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); + Environment.ExitCode = 1; + return; + } + + logger.LogInformation("Cached token for {Authority} expires at {Expires}.", options.Authority.Url, entry.ExpiresAtUtc.ToString("u")); + if (verbose) + { + logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); + } + } + + public static async Task HandleAuthWhoAmIAsync( + IServiceProvider services, + StellaOpsCliOptions options, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-whoami"); + Environment.ExitCode = 0; + + if (string.IsNullOrWhiteSpace(options.Authority?.Url)) + { + logger.LogInformation("Authority URL not configured. Set STELLAOPS_AUTHORITY_URL and run 'auth login'."); + Environment.ExitCode = 1; + return; + } + + var tokenClient = scope.ServiceProvider.GetService(); + if (tokenClient is null) + { + logger.LogInformation("Authority client not registered; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(options); + if (string.IsNullOrWhiteSpace(cacheKey)) + { + logger.LogInformation("Authority configuration incomplete; no cached tokens available."); + Environment.ExitCode = 1; + return; + } + + var entry = await tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + logger.LogInformation("No cached token for {Authority}. Run 'auth login' to authenticate.", options.Authority.Url); + Environment.ExitCode = 1; + return; + } + + var grantType = string.IsNullOrWhiteSpace(options.Authority.Username) ? "client_credentials" : "password"; + var now = DateTimeOffset.UtcNow; + var remaining = entry.ExpiresAtUtc - now; + if (remaining < TimeSpan.Zero) + { + remaining = TimeSpan.Zero; + } + + logger.LogInformation("Authority: {Authority}", options.Authority.Url); + logger.LogInformation("Grant type: {GrantType}", grantType); + logger.LogInformation("Token type: {TokenType}", entry.TokenType); + logger.LogInformation("Expires: {Expires} ({Remaining})", entry.ExpiresAtUtc.ToString("u"), FormatDuration(remaining)); + + if (entry.Scopes.Count > 0) + { + logger.LogInformation("Scopes: {Scopes}", string.Join(", ", entry.Scopes)); + } + + if (TryExtractJwtClaims(entry.AccessToken, out var claims, out var issuedAt, out var notBefore)) + { + if (claims.TryGetValue("sub", out var subject) && !string.IsNullOrWhiteSpace(subject)) + { + logger.LogInformation("Subject: {Subject}", subject); + } + + if (claims.TryGetValue("client_id", out var clientId) && !string.IsNullOrWhiteSpace(clientId)) + { + logger.LogInformation("Client ID (token): {ClientId}", clientId); + } + + if (claims.TryGetValue("aud", out var audience) && !string.IsNullOrWhiteSpace(audience)) + { + logger.LogInformation("Audience: {Audience}", audience); + } + + if (claims.TryGetValue("iss", out var issuer) && !string.IsNullOrWhiteSpace(issuer)) + { + logger.LogInformation("Issuer: {Issuer}", issuer); + } + + if (issuedAt is not null) + { + logger.LogInformation("Issued at: {IssuedAt}", issuedAt.Value.ToString("u")); + } + + if (notBefore is not null) + { + logger.LogInformation("Not before: {NotBefore}", notBefore.Value.ToString("u")); + } + + var extraClaims = CollectAdditionalClaims(claims); + if (extraClaims.Count > 0 && verbose) + { + logger.LogInformation("Additional claims: {Claims}", string.Join(", ", extraClaims)); + } + } + else + { + logger.LogInformation("Access token appears opaque; claims are unavailable."); + } + } + + public static async Task HandleAuthRevokeExportAsync( + IServiceProvider services, + StellaOpsCliOptions options, + string? outputDirectory, + bool verbose, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger("auth-revoke-export"); + Environment.ExitCode = 0; + + try + { + var client = scope.ServiceProvider.GetRequiredService(); + var result = await client.ExportAsync(verbose, cancellationToken).ConfigureAwait(false); + + var directory = string.IsNullOrWhiteSpace(outputDirectory) + ? Directory.GetCurrentDirectory() + : Path.GetFullPath(outputDirectory); + + Directory.CreateDirectory(directory); + + var bundlePath = Path.Combine(directory, "revocation-bundle.json"); + var signaturePath = Path.Combine(directory, "revocation-bundle.json.jws"); + var digestPath = Path.Combine(directory, "revocation-bundle.json.sha256"); + + await File.WriteAllBytesAsync(bundlePath, result.BundleBytes, cancellationToken).ConfigureAwait(false); + await File.WriteAllTextAsync(signaturePath, result.Signature, cancellationToken).ConfigureAwait(false); + await File.WriteAllTextAsync(digestPath, $"sha256:{result.Digest}", cancellationToken).ConfigureAwait(false); + + var computedDigest = Convert.ToHexString(SHA256.HashData(result.BundleBytes)).ToLowerInvariant(); + if (!string.Equals(computedDigest, result.Digest, StringComparison.OrdinalIgnoreCase)) + { + logger.LogError("Digest mismatch. Expected {Expected} but computed {Actual}.", result.Digest, computedDigest); + Environment.ExitCode = 1; + return; + } + + logger.LogInformation( + "Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}, provider {Provider}).", + directory, + result.Sequence, + result.IssuedAt, + string.IsNullOrWhiteSpace(result.SigningKeyId) ? "" : result.SigningKeyId, + string.IsNullOrWhiteSpace(result.SigningProvider) ? "default" : result.SigningProvider); + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to export revocation bundle."); + Environment.ExitCode = 1; + } + } + + public static async Task HandleAuthRevokeVerifyAsync( + string bundlePath, + string signaturePath, + string keyPath, + bool verbose, + CancellationToken cancellationToken) + { + var loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(options => + { + options.SingleLine = true; + options.TimestampFormat = "HH:mm:ss "; + })); + var logger = loggerFactory.CreateLogger("auth-revoke-verify"); + Environment.ExitCode = 0; + + try + { + if (string.IsNullOrWhiteSpace(bundlePath) || string.IsNullOrWhiteSpace(signaturePath) || string.IsNullOrWhiteSpace(keyPath)) + { + logger.LogError("Arguments --bundle, --signature, and --key are required."); + Environment.ExitCode = 1; + return; + } + + var bundleBytes = await File.ReadAllBytesAsync(bundlePath, cancellationToken).ConfigureAwait(false); + var signatureContent = (await File.ReadAllTextAsync(signaturePath, cancellationToken).ConfigureAwait(false)).Trim(); + var keyPem = await File.ReadAllTextAsync(keyPath, cancellationToken).ConfigureAwait(false); + + var digest = Convert.ToHexString(SHA256.HashData(bundleBytes)).ToLowerInvariant(); + logger.LogInformation("Bundle digest sha256:{Digest}", digest); + + if (!TryParseDetachedJws(signatureContent, out var encodedHeader, out var encodedSignature)) + { + logger.LogError("Signature is not in detached JWS format."); + Environment.ExitCode = 1; + return; + } + + var headerJson = Encoding.UTF8.GetString(Base64UrlDecode(encodedHeader)); + using var headerDocument = JsonDocument.Parse(headerJson); + var header = headerDocument.RootElement; + + if (!header.TryGetProperty("b64", out var b64Element) || b64Element.GetBoolean()) + { + logger.LogError("Detached JWS header must include '\"b64\": false'."); + Environment.ExitCode = 1; + return; + } + + var algorithm = header.TryGetProperty("alg", out var algElement) ? algElement.GetString() : SignatureAlgorithms.Es256; + if (string.IsNullOrWhiteSpace(algorithm)) + { + algorithm = SignatureAlgorithms.Es256; + } + + var providerHint = header.TryGetProperty("provider", out var providerElement) + ? providerElement.GetString() + : null; + + var keyId = header.TryGetProperty("kid", out var kidElement) ? kidElement.GetString() : null; + if (string.IsNullOrWhiteSpace(keyId)) + { + keyId = Path.GetFileNameWithoutExtension(keyPath); + logger.LogWarning("JWS header missing 'kid'; using fallback key id {KeyId}.", keyId); + } + + CryptoSigningKey signingKey; + try + { + signingKey = CreateVerificationSigningKey(keyId!, algorithm!, providerHint, keyPem, keyPath); + } + catch (Exception ex) when (ex is InvalidOperationException or CryptographicException) + { + logger.LogError(ex, "Failed to load verification key material."); + Environment.ExitCode = 1; + return; + } + + var providers = new List + { + new DefaultCryptoProvider() + }; + +#if STELLAOPS_CRYPTO_SODIUM + providers.Add(new LibsodiumCryptoProvider()); +#endif + + foreach (var provider in providers) + { + if (provider.Supports(CryptoCapability.Verification, algorithm!)) + { + provider.UpsertSigningKey(signingKey); + } + } + + var preferredOrder = !string.IsNullOrWhiteSpace(providerHint) + ? new[] { providerHint! } + : Array.Empty(); + var registry = new CryptoProviderRegistry(providers, preferredOrder); + CryptoSignerResolution resolution; + try + { + resolution = registry.ResolveSigner( + CryptoCapability.Verification, + algorithm!, + signingKey.Reference, + providerHint); + } + catch (Exception ex) + { + logger.LogError(ex, "No crypto provider available for verification (algorithm {Algorithm}).", algorithm); + Environment.ExitCode = 1; + return; + } + + var signingInputLength = encodedHeader.Length + 1 + bundleBytes.Length; + var buffer = ArrayPool.Shared.Rent(signingInputLength); + try + { + var headerBytes = Encoding.ASCII.GetBytes(encodedHeader); + Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length); + buffer[headerBytes.Length] = (byte)'.'; + Buffer.BlockCopy(bundleBytes, 0, buffer, headerBytes.Length + 1, bundleBytes.Length); + + var signatureBytes = Base64UrlDecode(encodedSignature); + var verified = await resolution.Signer.VerifyAsync( + new ReadOnlyMemory(buffer, 0, signingInputLength), + signatureBytes, + cancellationToken).ConfigureAwait(false); + + if (!verified) + { + logger.LogError("Signature verification failed."); + Environment.ExitCode = 1; + return; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + if (!string.IsNullOrWhiteSpace(providerHint) && !string.Equals(providerHint, resolution.ProviderName, StringComparison.OrdinalIgnoreCase)) + { + logger.LogWarning( + "Preferred provider '{Preferred}' unavailable; verification used '{Provider}'.", + providerHint, + resolution.ProviderName); + } + + logger.LogInformation( + "Signature verified using algorithm {Algorithm} via provider {Provider} (kid {KeyId}).", + algorithm, + resolution.ProviderName, + signingKey.Reference.KeyId); + + if (verbose) + { + logger.LogInformation("JWS header: {Header}", headerJson); + } + } + catch (Exception ex) + { + logger.LogError(ex, "Failed to verify revocation bundle."); + Environment.ExitCode = 1; + } + finally + { + loggerFactory.Dispose(); + } + } + + private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) + { + encodedHeader = string.Empty; + encodedSignature = string.Empty; + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var parts = value.Split('.'); + if (parts.Length != 3) + { + return false; + } + + encodedHeader = parts[0]; + encodedSignature = parts[2]; + return parts[1].Length == 0; + } + + private static byte[] Base64UrlDecode(string value) + { + var normalized = value.Replace('-', '+').Replace('_', '/'); + var padding = normalized.Length % 4; + if (padding == 2) + { + normalized += "=="; + } + else if (padding == 3) + { + normalized += "="; + } + else if (padding == 1) + { + throw new FormatException("Invalid Base64Url value."); + } + + return Convert.FromBase64String(normalized); + } + + private static CryptoSigningKey CreateVerificationSigningKey( + string keyId, + string algorithm, + string? providerHint, + string keyPem, + string keyPath) + { + if (string.IsNullOrWhiteSpace(keyPem)) + { + throw new InvalidOperationException("Verification key PEM content is empty."); + } + + using var ecdsa = ECDsa.Create(); + ecdsa.ImportFromPem(keyPem); + + var parameters = ecdsa.ExportParameters(includePrivateParameters: false); + if (parameters.D is null || parameters.D.Length == 0) + { + parameters.D = new byte[] { 0x01 }; + } + + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["source"] = Path.GetFullPath(keyPath), + ["verificationOnly"] = "true" + }; + + return new CryptoSigningKey( + new CryptoKeyReference(keyId, providerHint), + algorithm, + in parameters, + DateTimeOffset.UtcNow, + metadata: metadata); + } + + private static string FormatDuration(TimeSpan duration) + { + if (duration <= TimeSpan.Zero) + { + return "expired"; + } + + if (duration.TotalDays >= 1) + { + var days = (int)duration.TotalDays; + var hours = duration.Hours; + return hours > 0 + ? FormattableString.Invariant($"{days}d {hours}h") + : FormattableString.Invariant($"{days}d"); + } + + if (duration.TotalHours >= 1) + { + return FormattableString.Invariant($"{(int)duration.TotalHours}h {duration.Minutes}m"); + } + + if (duration.TotalMinutes >= 1) + { + return FormattableString.Invariant($"{(int)duration.TotalMinutes}m {duration.Seconds}s"); + } + + return FormattableString.Invariant($"{duration.Seconds}s"); + } + + private static bool TryExtractJwtClaims( + string accessToken, + out Dictionary claims, + out DateTimeOffset? issuedAt, + out DateTimeOffset? notBefore) + { + claims = new Dictionary(StringComparer.OrdinalIgnoreCase); + issuedAt = null; + notBefore = null; + + if (string.IsNullOrWhiteSpace(accessToken)) + { + return false; + } + + var parts = accessToken.Split('.'); + if (parts.Length < 2) + { + return false; + } + + if (!TryDecodeBase64Url(parts[1], out var payloadBytes)) + { + return false; + } + + try + { + using var document = JsonDocument.Parse(payloadBytes); + foreach (var property in document.RootElement.EnumerateObject()) + { + var value = FormatJsonValue(property.Value); + claims[property.Name] = value; + + if (issuedAt is null && property.NameEquals("iat") && TryParseUnixSeconds(property.Value, out var parsedIat)) + { + issuedAt = parsedIat; + } + + if (notBefore is null && property.NameEquals("nbf") && TryParseUnixSeconds(property.Value, out var parsedNbf)) + { + notBefore = parsedNbf; + } + } + + return true; + } + catch (JsonException) + { + claims.Clear(); + issuedAt = null; + notBefore = null; + return false; + } + } + + private static bool TryDecodeBase64Url(string value, out byte[] bytes) + { + bytes = Array.Empty(); + + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var normalized = value.Replace('-', '+').Replace('_', '/'); + var padding = normalized.Length % 4; + if (padding is 2 or 3) + { + normalized = normalized.PadRight(normalized.Length + (4 - padding), '='); + } + else if (padding == 1) + { + return false; + } + + try + { + bytes = Convert.FromBase64String(normalized); + return true; + } + catch (FormatException) + { + return false; + } + } + + private static string FormatJsonValue(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString() ?? string.Empty, + JsonValueKind.Number => element.TryGetInt64(out var longValue) + ? longValue.ToString(CultureInfo.InvariantCulture) + : element.GetDouble().ToString(CultureInfo.InvariantCulture), + JsonValueKind.True => "true", + JsonValueKind.False => "false", + JsonValueKind.Null => "null", + JsonValueKind.Array => FormatArray(element), + JsonValueKind.Object => element.GetRawText(), + _ => element.GetRawText() + }; + } + + private static string FormatArray(JsonElement array) + { + var values = new List(); + foreach (var item in array.EnumerateArray()) + { + values.Add(FormatJsonValue(item)); + } + + return string.Join(", ", values); + } + + private static bool TryParseUnixSeconds(JsonElement element, out DateTimeOffset value) + { + value = default; + + if (element.ValueKind == JsonValueKind.Number) + { + if (element.TryGetInt64(out var seconds)) + { + value = DateTimeOffset.FromUnixTimeSeconds(seconds); + return true; + } + + if (element.TryGetDouble(out var doubleValue)) + { + value = DateTimeOffset.FromUnixTimeSeconds((long)doubleValue); + return true; + } + } + + if (element.ValueKind == JsonValueKind.String) + { + var text = element.GetString(); + if (!string.IsNullOrWhiteSpace(text) && long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds)) + { + value = DateTimeOffset.FromUnixTimeSeconds(seconds); + return true; + } + } + + return false; + } + + private static List CollectAdditionalClaims(Dictionary claims) + { + var result = new List(); + foreach (var pair in claims) + { + if (CommonClaimNames.Contains(pair.Key)) + { + continue; + } + + result.Add(FormattableString.Invariant($"{pair.Key}={pair.Value}")); + } + + result.Sort(StringComparer.OrdinalIgnoreCase); + return result; + } + + private static readonly HashSet CommonClaimNames = new(StringComparer.OrdinalIgnoreCase) + { + "aud", + "client_id", + "exp", + "iat", + "iss", + "nbf", + "scope", + "scopes", + "sub", + "token_type", + "jti" + }; + + private static async Task ExecuteExcititorCommandAsync( + IServiceProvider services, + string commandName, + bool verbose, + IDictionary? activityTags, + Func> operation, + CancellationToken cancellationToken) + { + await using var scope = services.CreateAsyncScope(); + var client = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService().CreateLogger(commandName.Replace(' ', '-')); + var verbosity = scope.ServiceProvider.GetRequiredService(); + var previousLevel = verbosity.MinimumLevel; + verbosity.MinimumLevel = verbose ? LogLevel.Debug : LogLevel.Information; + using var activity = CliActivitySource.Instance.StartActivity($"cli.{commandName.Replace(' ', '.')}" , ActivityKind.Client); + activity?.SetTag("stellaops.cli.command", commandName); + if (activityTags is not null) + { + foreach (var tag in activityTags) + { + activity?.SetTag(tag.Key, tag.Value); + } + } + using var duration = CliMetrics.MeasureCommandDuration(commandName); + + try + { + var result = await operation(client).ConfigureAwait(false); + if (result.Success) + { + if (!string.IsNullOrWhiteSpace(result.Message)) + { + logger.LogInformation(result.Message); + } + else + { + logger.LogInformation("Operation completed successfully."); + } + + if (!string.IsNullOrWhiteSpace(result.Location)) + { + logger.LogInformation("Location: {Location}", result.Location); + } + + if (result.Payload is JsonElement payload && payload.ValueKind is not JsonValueKind.Undefined and not JsonValueKind.Null) + { + logger.LogDebug("Response payload: {Payload}", payload.ToString()); + } + + Environment.ExitCode = 0; + } + else + { + logger.LogError(string.IsNullOrWhiteSpace(result.Message) ? "Operation failed." : result.Message); + Environment.ExitCode = 1; + } + } + catch (Exception ex) + { + logger.LogError(ex, "Excititor operation failed."); + Environment.ExitCode = 1; + } + finally + { + verbosity.MinimumLevel = previousLevel; + } + } + + private static async Task> GatherImageDigestsAsync( + IReadOnlyList inline, + string? filePath, + CancellationToken cancellationToken) + { + var results = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + void AddCandidates(string? candidate) + { + foreach (var image in SplitImageCandidates(candidate)) + { + if (seen.Add(image)) + { + results.Add(image); + } + } + } + + if (inline is not null) + { + foreach (var entry in inline) + { + AddCandidates(entry); + } + } + + if (!string.IsNullOrWhiteSpace(filePath)) + { + var path = Path.GetFullPath(filePath); + if (!File.Exists(path)) + { + throw new FileNotFoundException("Input file not found.", path); + } + + foreach (var line in File.ReadLines(path)) + { + cancellationToken.ThrowIfCancellationRequested(); + AddCandidates(line); + } + } + + if (Console.IsInputRedirected) + { + while (!cancellationToken.IsCancellationRequested) + { + var line = await Console.In.ReadLineAsync().ConfigureAwait(false); + if (line is null) + { + break; + } + + AddCandidates(line); + } + } + + return new ReadOnlyCollection(results); + } + + private static IEnumerable SplitImageCandidates(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + { + yield break; + } + + var candidate = raw.Trim(); + var commentIndex = candidate.IndexOf('#'); + if (commentIndex >= 0) + { + candidate = candidate[..commentIndex].Trim(); + } + + if (candidate.Length == 0) + { + yield break; + } + + var tokens = candidate.Split(new[] { ',', ' ', '\t' }, StringSplitOptions.RemoveEmptyEntries); + foreach (var token in tokens) + { + var trimmed = token.Trim(); + if (trimmed.Length > 0) + { + yield return trimmed; + } + } + } + + private static IReadOnlyDictionary ParseLabelSelectors(IReadOnlyList labelArguments) + { + if (labelArguments is null || labelArguments.Count == 0) + { + return EmptyLabelSelectors; + } + + var labels = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var raw in labelArguments) + { + if (string.IsNullOrWhiteSpace(raw)) + { + continue; + } + + var trimmed = raw.Trim(); + var delimiter = trimmed.IndexOf('='); + if (delimiter <= 0 || delimiter == trimmed.Length - 1) + { + throw new ArgumentException($"Invalid label '{raw}'. Expected key=value format."); + } + + var key = trimmed[..delimiter].Trim(); + var value = trimmed[(delimiter + 1)..].Trim(); + if (key.Length == 0) + { + throw new ArgumentException($"Invalid label '{raw}'. Label key cannot be empty."); + } + + labels[key] = value; + } + + return labels.Count == 0 ? EmptyLabelSelectors : new ReadOnlyDictionary(labels); + } + + private sealed record ExcititorExportManifestSummary( + string ExportId, + string? Format, + string? Algorithm, + string? Digest, + long? SizeBytes, + bool? FromCache, + DateTimeOffset? CreatedAt, + string? RekorLocation, + string? RekorIndex, + string? RekorInclusionUrl); + + private static ExcititorExportManifestSummary? TryParseExportManifest(JsonElement? payload) + { + if (payload is null || payload.Value.ValueKind is JsonValueKind.Undefined or JsonValueKind.Null) + { + return null; + } + + var element = payload.Value; + var exportId = GetStringProperty(element, "exportId"); + if (string.IsNullOrWhiteSpace(exportId)) + { + return null; + } + + var format = GetStringProperty(element, "format"); + var algorithm = default(string?); + var digest = default(string?); + + if (TryGetPropertyCaseInsensitive(element, "artifact", out var artifact) && artifact.ValueKind == JsonValueKind.Object) + { + algorithm = GetStringProperty(artifact, "algorithm"); + digest = GetStringProperty(artifact, "digest"); + } + + var sizeBytes = GetInt64Property(element, "sizeBytes"); + var fromCache = GetBooleanProperty(element, "fromCache"); + var createdAt = GetDateTimeOffsetProperty(element, "createdAt"); + + string? rekorLocation = null; + string? rekorIndex = null; + string? rekorInclusion = null; + + if (TryGetPropertyCaseInsensitive(element, "attestation", out var attestation) && attestation.ValueKind == JsonValueKind.Object) + { + if (TryGetPropertyCaseInsensitive(attestation, "rekor", out var rekor) && rekor.ValueKind == JsonValueKind.Object) + { + rekorLocation = GetStringProperty(rekor, "location"); + rekorIndex = GetStringProperty(rekor, "logIndex"); + var inclusion = GetStringProperty(rekor, "inclusionProofUri"); + if (!string.IsNullOrWhiteSpace(inclusion)) + { + rekorInclusion = inclusion; + } + } + } + + return new ExcititorExportManifestSummary( + exportId.Trim(), + format, + algorithm, + digest, + sizeBytes, + fromCache, + createdAt, + rekorLocation, + rekorIndex, + rekorInclusion); + } + + private static bool TryGetPropertyCaseInsensitive(JsonElement element, string propertyName, out JsonElement property) + { + if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out property)) + { + return true; + } + + if (element.ValueKind == JsonValueKind.Object) + { + foreach (var candidate in element.EnumerateObject()) + { + if (string.Equals(candidate.Name, propertyName, StringComparison.OrdinalIgnoreCase)) + { + property = candidate.Value; + return true; + } + } + } + + property = default; + return false; + } + + private static string? GetStringProperty(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property)) + { + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number => property.ToString(), + _ => null + }; + } + + return null; + } + + private static bool? GetBooleanProperty(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property)) + { + return property.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed, + _ => null + }; + } + + return null; + } + + private static long? GetInt64Property(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property)) + { + if (property.ValueKind == JsonValueKind.Number && property.TryGetInt64(out var value)) + { + return value; + } + + if (property.ValueKind == JsonValueKind.String + && long.TryParse(property.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + } + + return null; + } + + private static DateTimeOffset? GetDateTimeOffsetProperty(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property) + && property.ValueKind == JsonValueKind.String + && DateTimeOffset.TryParse(property.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var value)) + { + return value.ToUniversalTime(); + } + + return null; + } + + private static string BuildDigestDisplay(string? algorithm, string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return string.Empty; + } + + if (digest.Contains(':', StringComparison.Ordinal)) + { + return digest; + } + + if (string.IsNullOrWhiteSpace(algorithm) || algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + return $"sha256:{digest}"; + } + + return $"{algorithm}:{digest}"; + } + + private static string FormatSize(long sizeBytes) + { + if (sizeBytes < 0) + { + return $"{sizeBytes} bytes"; + } + + string[] units = { "bytes", "KB", "MB", "GB", "TB" }; + double size = sizeBytes; + var unit = 0; + + while (size >= 1024 && unit < units.Length - 1) + { + size /= 1024; + unit++; + } + + return unit == 0 ? $"{sizeBytes} bytes" : $"{size:0.##} {units[unit]}"; + } + + private static string ResolveExportOutputPath(string outputPath, ExcititorExportManifestSummary manifest) + { + if (string.IsNullOrWhiteSpace(outputPath)) + { + throw new ArgumentException("Output path must be provided.", nameof(outputPath)); + } + + var fullPath = Path.GetFullPath(outputPath); + if (Directory.Exists(fullPath) + || outputPath.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal) + || outputPath.EndsWith(Path.AltDirectorySeparatorChar.ToString(), StringComparison.Ordinal)) + { + return Path.Combine(fullPath, BuildExportFileName(manifest)); + } + + var directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + return fullPath; + } + + private static string BuildExportFileName(ExcititorExportManifestSummary manifest) + { + var token = !string.IsNullOrWhiteSpace(manifest.Digest) + ? manifest.Digest! + : manifest.ExportId; + + token = SanitizeToken(token); + if (token.Length > 40) + { + token = token[..40]; + } + + var extension = DetermineExportExtension(manifest.Format); + return $"stellaops-excititor-{token}{extension}"; + } + + private static string DetermineExportExtension(string? format) + { + if (string.IsNullOrWhiteSpace(format)) + { + return ".bin"; + } + + return format switch + { + not null when format.Equals("jsonl", StringComparison.OrdinalIgnoreCase) => ".jsonl", + not null when format.Equals("json", StringComparison.OrdinalIgnoreCase) => ".json", + not null when format.Equals("openvex", StringComparison.OrdinalIgnoreCase) => ".json", + not null when format.Equals("csaf", StringComparison.OrdinalIgnoreCase) => ".json", + _ => ".bin" + }; + } + + private static string SanitizeToken(string token) + { + var builder = new StringBuilder(token.Length); + foreach (var ch in token) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + } + } + + if (builder.Length == 0) + { + builder.Append("export"); + } + + return builder.ToString(); + } + + private static string? ResolveLocationUrl(StellaOpsCliOptions options, string location) + { + if (string.IsNullOrWhiteSpace(location)) + { + return null; + } + + if (Uri.TryCreate(location, UriKind.Absolute, out var absolute)) + { + return absolute.ToString(); + } + + if (!string.IsNullOrWhiteSpace(options?.BackendUrl) && Uri.TryCreate(options.BackendUrl, UriKind.Absolute, out var baseUri)) + { + if (!location.StartsWith("/", StringComparison.Ordinal)) + { + location = "/" + location; + } + + return new Uri(baseUri, location).ToString(); + } + + return location; + } + + private static string BuildRuntimePolicyJson(RuntimePolicyEvaluationResult result, IReadOnlyList requestedImages) + { + var orderedImages = BuildImageOrder(requestedImages, result.Decisions.Keys); + var results = new Dictionary(StringComparer.Ordinal); + + foreach (var image in orderedImages) + { + if (result.Decisions.TryGetValue(image, out var decision)) + { + results[image] = BuildDecisionMap(decision); + } + } + + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + var payload = new Dictionary(StringComparer.Ordinal) + { + ["ttlSeconds"] = result.TtlSeconds, + ["expiresAtUtc"] = result.ExpiresAtUtc?.ToString("O", CultureInfo.InvariantCulture), + ["policyRevision"] = result.PolicyRevision, + ["results"] = results + }; + + return JsonSerializer.Serialize(payload, options); + } + + private static IDictionary BuildDecisionMap(RuntimePolicyImageDecision decision) + { + var map = new Dictionary(StringComparer.Ordinal) + { + ["policyVerdict"] = decision.PolicyVerdict, + ["signed"] = decision.Signed, + ["hasSbomReferrers"] = decision.HasSbomReferrers + }; + + if (decision.Reasons.Count > 0) + { + map["reasons"] = decision.Reasons; + } + + if (decision.Rekor is not null) + { + var rekorMap = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(decision.Rekor.Uuid)) + { + rekorMap["uuid"] = decision.Rekor.Uuid; + } + + if (!string.IsNullOrWhiteSpace(decision.Rekor.Url)) + { + rekorMap["url"] = decision.Rekor.Url; + } + + if (decision.Rekor.Verified.HasValue) + { + rekorMap["verified"] = decision.Rekor.Verified; + } + + if (rekorMap.Count > 0) + { + map["rekor"] = rekorMap; + } + } + + foreach (var kvp in decision.AdditionalProperties) + { + map[kvp.Key] = kvp.Value; + } + + return map; + } + + private static void DisplayRuntimePolicyResults(ILogger logger, RuntimePolicyEvaluationResult result, IReadOnlyList requestedImages) + { + var orderedImages = BuildImageOrder(requestedImages, result.Decisions.Keys); + var summary = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (AnsiConsole.Profile.Capabilities.Interactive) + { + var table = new Table().Border(TableBorder.Rounded) + .AddColumns("Image", "Verdict", "Signed", "SBOM Ref", "Quieted", "Confidence", "Reasons", "Attestation"); + + foreach (var image in orderedImages) + { + if (result.Decisions.TryGetValue(image, out var decision)) + { + table.AddRow( + image, + decision.PolicyVerdict, + FormatBoolean(decision.Signed), + FormatBoolean(decision.HasSbomReferrers), + FormatQuietedDisplay(decision.AdditionalProperties), + FormatConfidenceDisplay(decision.AdditionalProperties), + decision.Reasons.Count > 0 ? string.Join(Environment.NewLine, decision.Reasons) : "-", + FormatAttestation(decision.Rekor)); + + summary[decision.PolicyVerdict] = summary.TryGetValue(decision.PolicyVerdict, out var count) ? count + 1 : 1; + + if (decision.AdditionalProperties.Count > 0) + { + var metadata = string.Join(", ", decision.AdditionalProperties.Select(kvp => $"{kvp.Key}={FormatAdditionalValue(kvp.Value)}")); + logger.LogDebug("Metadata for {Image}: {Metadata}", image, metadata); + } + } + else + { + table.AddRow(image, "", "-", "-", "-", "-", "-", "-"); + } + } + + AnsiConsole.Write(table); + } + else + { + foreach (var image in orderedImages) + { + if (result.Decisions.TryGetValue(image, out var decision)) + { + var reasons = decision.Reasons.Count > 0 ? string.Join(", ", decision.Reasons) : "none"; + logger.LogInformation( + "{Image} -> verdict={Verdict} signed={Signed} sbomRef={Sbom} quieted={Quieted} confidence={Confidence} attestation={Attestation} reasons={Reasons}", + image, + decision.PolicyVerdict, + FormatBoolean(decision.Signed), + FormatBoolean(decision.HasSbomReferrers), + FormatQuietedDisplay(decision.AdditionalProperties), + FormatConfidenceDisplay(decision.AdditionalProperties), + FormatAttestation(decision.Rekor), + reasons); + + summary[decision.PolicyVerdict] = summary.TryGetValue(decision.PolicyVerdict, out var count) ? count + 1 : 1; + + if (decision.AdditionalProperties.Count > 0) + { + var metadata = string.Join(", ", decision.AdditionalProperties.Select(kvp => $"{kvp.Key}={FormatAdditionalValue(kvp.Value)}")); + logger.LogDebug("Metadata for {Image}: {Metadata}", image, metadata); + } + } + else + { + logger.LogWarning("{Image} -> no decision returned by backend.", image); + } + } + } + + if (summary.Count > 0) + { + var summaryText = string.Join(", ", summary.Select(kvp => $"{kvp.Key}:{kvp.Value}")); + logger.LogInformation("Verdict summary: {Summary}", summaryText); + } + } + + private static IReadOnlyList BuildImageOrder(IReadOnlyList requestedImages, IEnumerable actual) + { + var order = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + if (requestedImages is not null) + { + foreach (var image in requestedImages) + { + if (!string.IsNullOrWhiteSpace(image)) + { + var trimmed = image.Trim(); + if (seen.Add(trimmed)) + { + order.Add(trimmed); + } + } + } + } + + foreach (var image in actual) + { + if (!string.IsNullOrWhiteSpace(image)) + { + var trimmed = image.Trim(); + if (seen.Add(trimmed)) + { + order.Add(trimmed); + } + } + } + + return new ReadOnlyCollection(order); + } + + private static string FormatBoolean(bool? value) + => value is null ? "unknown" : value.Value ? "yes" : "no"; + + private static string FormatQuietedDisplay(IReadOnlyDictionary metadata) + { + var quieted = GetMetadataBoolean(metadata, "quieted", "quiet"); + var quietedBy = GetMetadataString(metadata, "quietedBy", "quietedReason"); + + if (quieted is true) + { + return string.IsNullOrWhiteSpace(quietedBy) ? "yes" : $"yes ({quietedBy})"; + } + + if (quieted is false) + { + return "no"; + } + + return string.IsNullOrWhiteSpace(quietedBy) ? "-" : $"? ({quietedBy})"; + } + + private static string FormatConfidenceDisplay(IReadOnlyDictionary metadata) + { + var confidence = GetMetadataDouble(metadata, "confidence"); + var confidenceBand = GetMetadataString(metadata, "confidenceBand", "confidenceTier"); + + if (confidence.HasValue && !string.IsNullOrWhiteSpace(confidenceBand)) + { + return string.Format(CultureInfo.InvariantCulture, "{0:0.###} ({1})", confidence.Value, confidenceBand); + } + + if (confidence.HasValue) + { + return confidence.Value.ToString("0.###", CultureInfo.InvariantCulture); + } + + if (!string.IsNullOrWhiteSpace(confidenceBand)) + { + return confidenceBand!; + } + + return "-"; + } + + private static string FormatAttestation(RuntimePolicyRekorReference? rekor) + { + if (rekor is null) + { + return "-"; + } + + var uuid = string.IsNullOrWhiteSpace(rekor.Uuid) ? null : rekor.Uuid; + var url = string.IsNullOrWhiteSpace(rekor.Url) ? null : rekor.Url; + var verified = rekor.Verified; + + var core = uuid ?? url; + if (!string.IsNullOrEmpty(core)) + { + if (verified.HasValue) + { + var suffix = verified.Value ? " (verified)" : " (unverified)"; + return core + suffix; + } + + return core!; + } + + if (verified.HasValue) + { + return verified.Value ? "verified" : "unverified"; + } + + return "-"; + } + + private static bool? GetMetadataBoolean(IReadOnlyDictionary metadata, params string[] keys) + { + foreach (var key in keys) + { + if (metadata.TryGetValue(key, out var value) && value is not null) + { + switch (value) + { + case bool b: + return b; + case string s when bool.TryParse(s, out var parsed): + return parsed; + } + } + } + + return null; + } + + private static string? GetMetadataString(IReadOnlyDictionary metadata, params string[] keys) + { + foreach (var key in keys) + { + if (metadata.TryGetValue(key, out var value) && value is not null) + { + if (value is string s) + { + return string.IsNullOrWhiteSpace(s) ? null : s; + } + } + } + + return null; + } + + private static double? GetMetadataDouble(IReadOnlyDictionary metadata, params string[] keys) + { + foreach (var key in keys) + { + if (metadata.TryGetValue(key, out var value) && value is not null) + { + switch (value) + { + case double d: + return d; + case float f: + return f; + case decimal m: + return (double)m; + case long l: + return l; + case int i: + return i; + case string s when double.TryParse(s, NumberStyles.Float | NumberStyles.AllowThousands, CultureInfo.InvariantCulture, out var parsed): + return parsed; + } + } + } + + return null; + } + + private static readonly IReadOnlyDictionary EmptyLabelSelectors = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.OrdinalIgnoreCase)); + + + private static string FormatAdditionalValue(object? value) + { + return value switch + { + null => "null", + bool b => b ? "true" : "false", + double d => d.ToString("G17", CultureInfo.InvariantCulture), + float f => f.ToString("G9", CultureInfo.InvariantCulture), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString() ?? string.Empty + }; + } + + + private static IReadOnlyList NormalizeProviders(IReadOnlyList providers) + { + if (providers is null || providers.Count == 0) + { + return Array.Empty(); + } + + var list = new List(); + foreach (var provider in providers) + { + if (!string.IsNullOrWhiteSpace(provider)) + { + list.Add(provider.Trim()); + } + } + + return list.Count == 0 ? Array.Empty() : list; + } + + private static IDictionary RemoveNullValues(Dictionary source) + { + foreach (var key in source.Where(kvp => kvp.Value is null).Select(kvp => kvp.Key).ToList()) + { + source.Remove(key); + } + + return source; + } + + private static async Task TriggerJobAsync( + IBackendOperationsClient client, + ILogger logger, + string jobKind, IDictionary parameters, CancellationToken cancellationToken) { @@ -1140,6 +2632,6 @@ internal static class CommandHandlers { logger.LogError("Job '{JobKind}' failed: {Message}", jobKind, result.Message); Environment.ExitCode = 1; - } - } -} + } + } +} diff --git a/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs b/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs index 8694a4c7..6e102185 100644 --- a/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs +++ b/src/StellaOps.Cli/Configuration/AuthorityTokenUtilities.cs @@ -1,34 +1,34 @@ -using System; -using StellaOps.Auth.Abstractions; - -namespace StellaOps.Cli.Configuration; - -internal static class AuthorityTokenUtilities -{ - public static string ResolveScope(StellaOpsCliOptions options) - { - ArgumentNullException.ThrowIfNull(options); - - var scope = options.Authority?.Scope; - return string.IsNullOrWhiteSpace(scope) - ? StellaOpsScopes.FeedserJobsTrigger - : scope.Trim(); - } - - public static string BuildCacheKey(StellaOpsCliOptions options) - { - ArgumentNullException.ThrowIfNull(options); - - if (options.Authority is null) - { - return string.Empty; - } - - var scope = ResolveScope(options); - var credential = !string.IsNullOrWhiteSpace(options.Authority.Username) - ? $"user:{options.Authority.Username}" - : $"client:{options.Authority.ClientId}"; - - return $"{options.Authority.Url}|{credential}|{scope}"; - } -} +using System; +using StellaOps.Auth.Abstractions; + +namespace StellaOps.Cli.Configuration; + +internal static class AuthorityTokenUtilities +{ + public static string ResolveScope(StellaOpsCliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var scope = options.Authority?.Scope; + return string.IsNullOrWhiteSpace(scope) + ? StellaOpsScopes.ConcelierJobsTrigger + : scope.Trim(); + } + + public static string BuildCacheKey(StellaOpsCliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.Authority is null) + { + return string.Empty; + } + + var scope = ResolveScope(options); + var credential = !string.IsNullOrWhiteSpace(options.Authority.Username) + ? $"user:{options.Authority.Username}" + : $"client:{options.Authority.ClientId}"; + + return $"{options.Authority.Url}|{credential}|{scope}"; + } +} diff --git a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs index a84f5d3c..ffba6a75 100644 --- a/src/StellaOps.Cli/Configuration/CliBootstrapper.cs +++ b/src/StellaOps.Cli/Configuration/CliBootstrapper.cs @@ -113,7 +113,7 @@ public static class CliBootstrapper authority.ClientSecret = string.IsNullOrWhiteSpace(authority.ClientSecret) ? null : authority.ClientSecret.Trim(); authority.Username = authority.Username?.Trim() ?? string.Empty; authority.Password = string.IsNullOrWhiteSpace(authority.Password) ? null : authority.Password.Trim(); - authority.Scope = string.IsNullOrWhiteSpace(authority.Scope) ? StellaOpsScopes.FeedserJobsTrigger : authority.Scope.Trim(); + authority.Scope = string.IsNullOrWhiteSpace(authority.Scope) ? StellaOpsScopes.ConcelierJobsTrigger : authority.Scope.Trim(); authority.Resilience ??= new StellaOpsCliAuthorityResilienceOptions(); authority.Resilience.RetryDelays ??= new List(); diff --git a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs index 0cacb7cb..54355bbc 100644 --- a/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs +++ b/src/StellaOps.Cli/Configuration/StellaOpsCliOptions.cs @@ -37,7 +37,7 @@ public sealed class StellaOpsCliAuthorityOptions public string? Password { get; set; } - public string Scope { get; set; } = StellaOpsScopes.FeedserJobsTrigger; + public string Scope { get; set; } = StellaOpsScopes.ConcelierJobsTrigger; public string TokenCacheDirectory { get; set; } = string.Empty; diff --git a/src/StellaOps.Cli/Program.cs b/src/StellaOps.Cli/Program.cs index ad93a34c..a533a5b3 100644 --- a/src/StellaOps.Cli/Program.cs +++ b/src/StellaOps.Cli/Program.cs @@ -46,7 +46,7 @@ internal static class Program clientOptions.ClientSecret = options.Authority.ClientSecret; clientOptions.DefaultScopes.Clear(); clientOptions.DefaultScopes.Add(string.IsNullOrWhiteSpace(options.Authority.Scope) - ? StellaOps.Auth.Abstractions.StellaOpsScopes.FeedserJobsTrigger + ? StellaOps.Auth.Abstractions.StellaOpsScopes.ConcelierJobsTrigger : options.Authority.Scope); var resilience = options.Authority.Resilience ?? new StellaOpsCliAuthorityResilienceOptions(); diff --git a/src/StellaOps.Cli/Services/BackendOperationsClient.cs b/src/StellaOps.Cli/Services/BackendOperationsClient.cs index 6e4df175..0e387083 100644 --- a/src/StellaOps.Cli/Services/BackendOperationsClient.cs +++ b/src/StellaOps.Cli/Services/BackendOperationsClient.cs @@ -1,50 +1,53 @@ using System; using System.Collections.Generic; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Linq; -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Globalization; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Auth.Abstractions; -using StellaOps.Auth.Client; -using StellaOps.Cli.Configuration; -using StellaOps.Cli.Services.Models; -using StellaOps.Cli.Services.Models.Transport; +using System.Collections.ObjectModel; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Linq; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; +using StellaOps.Auth.Client; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; +using StellaOps.Cli.Services.Models.Transport; namespace StellaOps.Cli.Services; -internal sealed class BackendOperationsClient : IBackendOperationsClient -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); - private static readonly TimeSpan TokenRefreshSkew = TimeSpan.FromSeconds(30); - - private readonly HttpClient _httpClient; - private readonly StellaOpsCliOptions _options; - private readonly ILogger _logger; - private readonly IStellaOpsTokenClient? _tokenClient; - private readonly object _tokenSync = new(); - private string? _cachedAccessToken; - private DateTimeOffset _cachedAccessTokenExpiresAt = DateTimeOffset.MinValue; - - public BackendOperationsClient(HttpClient httpClient, StellaOpsCliOptions options, ILogger logger, IStellaOpsTokenClient? tokenClient = null) - { - _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _tokenClient = tokenClient; - - if (!string.IsNullOrWhiteSpace(_options.BackendUrl) && httpClient.BaseAddress is null) - { - if (Uri.TryCreate(_options.BackendUrl, UriKind.Absolute, out var baseUri)) - { +internal sealed class BackendOperationsClient : IBackendOperationsClient +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private static readonly TimeSpan TokenRefreshSkew = TimeSpan.FromSeconds(30); + private static readonly IReadOnlyDictionary EmptyMetadata = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.OrdinalIgnoreCase)); + + private readonly HttpClient _httpClient; + private readonly StellaOpsCliOptions _options; + private readonly ILogger _logger; + private readonly IStellaOpsTokenClient? _tokenClient; + private readonly object _tokenSync = new(); + private string? _cachedAccessToken; + private DateTimeOffset _cachedAccessTokenExpiresAt = DateTimeOffset.MinValue; + + public BackendOperationsClient(HttpClient httpClient, StellaOpsCliOptions options, ILogger logger, IStellaOpsTokenClient? tokenClient = null) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _tokenClient = tokenClient; + + if (!string.IsNullOrWhiteSpace(_options.BackendUrl) && httpClient.BaseAddress is null) + { + if (Uri.TryCreate(_options.BackendUrl, UriKind.Absolute, out var baseUri)) + { httpClient.BaseAddress = baseUri; } } @@ -73,13 +76,13 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient attempt++; try { - using var request = CreateRequest(HttpMethod.Get, $"api/scanner/artifacts/{channel}"); - await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); - using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); - throw new InvalidOperationException(failure); + using var request = CreateRequest(HttpMethod.Get, $"api/scanner/artifacts/{channel}"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); } return await ProcessScannerResponseAsync(response, outputPath, channel, verbose, cancellationToken).ConfigureAwait(false); @@ -129,68 +132,68 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient return new ScannerArtifactResult(outputPath, downloaded.Length, false); } - public async Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) - { - EnsureBackendConfigured(); - - if (!File.Exists(filePath)) - { - throw new FileNotFoundException("Scan result file not found.", filePath); - } - - var maxAttempts = Math.Max(1, _options.ScanUploadAttempts); - var attempt = 0; - - while (true) - { - attempt++; - try - { - using var content = new MultipartFormDataContent(); - await using var fileStream = File.OpenRead(filePath); - var streamContent = new StreamContent(fileStream); - streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - content.Add(streamContent, "file", Path.GetFileName(filePath)); - - using var request = CreateRequest(HttpMethod.Post, "api/scanner/results"); - await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); - request.Content = content; - - using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - _logger.LogInformation("Scan results uploaded from {Path}.", filePath); - return; - } - - var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); - if (attempt >= maxAttempts) - { - throw new InvalidOperationException(failure); - } - - var delay = GetRetryDelay(response, attempt); - _logger.LogWarning( - "Scan upload attempt {Attempt}/{MaxAttempts} failed ({Reason}). Retrying in {Delay:F1}s...", - attempt, - maxAttempts, - failure, - delay.TotalSeconds); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (attempt < maxAttempts) - { - var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)); - _logger.LogWarning( - ex, - "Scan upload attempt {Attempt}/{MaxAttempts} threw an exception. Retrying in {Delay:F1}s...", - attempt, - maxAttempts, - delay.TotalSeconds); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); - } - } - } + public async Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException("Scan result file not found.", filePath); + } + + var maxAttempts = Math.Max(1, _options.ScanUploadAttempts); + var attempt = 0; + + while (true) + { + attempt++; + try + { + using var content = new MultipartFormDataContent(); + await using var fileStream = File.OpenRead(filePath); + var streamContent = new StreamContent(fileStream); + streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + content.Add(streamContent, "file", Path.GetFileName(filePath)); + + using var request = CreateRequest(HttpMethod.Post, "api/scanner/results"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + request.Content = content; + + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + _logger.LogInformation("Scan results uploaded from {Path}.", filePath); + return; + } + + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + if (attempt >= maxAttempts) + { + throw new InvalidOperationException(failure); + } + + var delay = GetRetryDelay(response, attempt); + _logger.LogWarning( + "Scan upload attempt {Attempt}/{MaxAttempts} failed ({Reason}). Retrying in {Delay:F1}s...", + attempt, + maxAttempts, + failure, + delay.TotalSeconds); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < maxAttempts) + { + var delay = TimeSpan.FromSeconds(Math.Pow(2, attempt)); + _logger.LogWarning( + ex, + "Scan upload attempt {Attempt}/{MaxAttempts} threw an exception. Retrying in {Delay:F1}s...", + attempt, + maxAttempts, + delay.TotalSeconds); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + } public async Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken) { @@ -207,9 +210,9 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient Parameters = parameters is null ? new Dictionary(StringComparer.Ordinal) : new Dictionary(parameters, StringComparer.Ordinal) }; - var request = CreateRequest(HttpMethod.Post, $"jobs/{jobKind}"); - await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); - request.Content = JsonContent.Create(requestBody, options: SerializerOptions); + var request = CreateRequest(HttpMethod.Post, $"jobs/{jobKind}"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + request.Content = JsonContent.Create(requestBody, options: SerializerOptions); using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.Accepted) @@ -235,11 +238,397 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient return new JobTriggerResult(false, failureMessage, null, null); } - private HttpRequestMessage CreateRequest(HttpMethod method, string relativeUri) - { - if (!Uri.TryCreate(relativeUri, UriKind.RelativeOrAbsolute, out var requestUri)) - { - throw new InvalidOperationException($"Invalid request URI '{relativeUri}'."); + public async Task ExecuteExcititorOperationAsync(string route, HttpMethod method, object? payload, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (string.IsNullOrWhiteSpace(route)) + { + throw new ArgumentException("Route must be provided.", nameof(route)); + } + + var relative = route.TrimStart('/'); + using var request = CreateRequest(method, $"excititor/{relative}"); + + if (payload is not null && method != HttpMethod.Get && method != HttpMethod.Delete) + { + request.Content = JsonContent.Create(payload, options: SerializerOptions); + } + + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + var (message, payloadElement) = await ExtractExcititorResponseAsync(response, cancellationToken).ConfigureAwait(false); + var location = response.Headers.Location?.ToString(); + return new ExcititorOperationResult(true, message, location, payloadElement); + } + + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + return new ExcititorOperationResult(false, failure, null, null); + } + + public async Task DownloadExcititorExportAsync(string exportId, string destinationPath, string? expectedDigestAlgorithm, string? expectedDigest, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (string.IsNullOrWhiteSpace(exportId)) + { + throw new ArgumentException("Export id must be provided.", nameof(exportId)); + } + + if (string.IsNullOrWhiteSpace(destinationPath)) + { + throw new ArgumentException("Destination path must be provided.", nameof(destinationPath)); + } + + var fullPath = Path.GetFullPath(destinationPath); + var directory = Path.GetDirectoryName(fullPath); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + var normalizedAlgorithm = string.IsNullOrWhiteSpace(expectedDigestAlgorithm) + ? null + : expectedDigestAlgorithm.Trim(); + var normalizedDigest = NormalizeExpectedDigest(expectedDigest); + + if (File.Exists(fullPath) + && string.Equals(normalizedAlgorithm, "sha256", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(normalizedDigest)) + { + var existingDigest = await ComputeSha256Async(fullPath, cancellationToken).ConfigureAwait(false); + if (string.Equals(existingDigest, normalizedDigest, StringComparison.OrdinalIgnoreCase)) + { + var info = new FileInfo(fullPath); + _logger.LogDebug("Export {ExportId} already present at {Path}; digest matches.", exportId, fullPath); + return new ExcititorExportDownloadResult(fullPath, info.Length, true); + } + } + + var encodedId = Uri.EscapeDataString(exportId); + using var request = CreateRequest(HttpMethod.Get, $"excititor/export/{encodedId}/download"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + + var tempPath = fullPath + ".tmp"; + if (File.Exists(tempPath)) + { + File.Delete(tempPath); + } + + using (var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) + { + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + await using (var fileStream = File.Create(tempPath)) + { + await stream.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false); + } + } + + if (!string.IsNullOrWhiteSpace(normalizedAlgorithm) && !string.IsNullOrWhiteSpace(normalizedDigest)) + { + if (string.Equals(normalizedAlgorithm, "sha256", StringComparison.OrdinalIgnoreCase)) + { + var computed = await ComputeSha256Async(tempPath, cancellationToken).ConfigureAwait(false); + if (!string.Equals(computed, normalizedDigest, StringComparison.OrdinalIgnoreCase)) + { + File.Delete(tempPath); + throw new InvalidOperationException($"Export digest mismatch. Expected sha256:{normalizedDigest}, computed sha256:{computed}."); + } + } + else + { + _logger.LogWarning("Export digest verification skipped. Unsupported algorithm {Algorithm}.", normalizedAlgorithm); + } + } + + if (File.Exists(fullPath)) + { + File.Delete(fullPath); + } + + File.Move(tempPath, fullPath); + + var downloaded = new FileInfo(fullPath); + return new ExcititorExportDownloadResult(fullPath, downloaded.Length, false); + } + + public async Task EvaluateRuntimePolicyAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + var images = NormalizeImages(request.Images); + if (images.Count == 0) + { + throw new ArgumentException("At least one image digest must be provided.", nameof(request)); + } + + var payload = new RuntimePolicyEvaluationRequestDocument + { + Namespace = string.IsNullOrWhiteSpace(request.Namespace) ? null : request.Namespace.Trim(), + Images = images + }; + + if (request.Labels.Count > 0) + { + payload.Labels = new Dictionary(StringComparer.Ordinal); + foreach (var label in request.Labels) + { + if (!string.IsNullOrWhiteSpace(label.Key)) + { + payload.Labels[label.Key] = label.Value ?? string.Empty; + } + } + } + + using var message = CreateRequest(HttpMethod.Post, "api/scanner/policy/runtime"); + await AuthorizeRequestAsync(message, cancellationToken).ConfigureAwait(false); + message.Content = JsonContent.Create(payload, options: SerializerOptions); + + using var response = await _httpClient.SendAsync(message, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + RuntimePolicyEvaluationResponseDocument? document; + try + { + document = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false); + } + catch (JsonException ex) + { + var raw = response.Content is null ? string.Empty : await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to parse runtime policy response. {ex.Message}", ex) + { + Data = { ["payload"] = raw } + }; + } + + if (document is null) + { + throw new InvalidOperationException("Runtime policy response was empty."); + } + + var decisions = new Dictionary(StringComparer.Ordinal); + if (document.Results is not null) + { + foreach (var kvp in document.Results) + { + var image = kvp.Key; + var decision = kvp.Value; + if (string.IsNullOrWhiteSpace(image) || decision is null) + { + continue; + } + + var verdict = string.IsNullOrWhiteSpace(decision.PolicyVerdict) + ? "unknown" + : decision.PolicyVerdict!.Trim(); + + var reasons = ExtractReasons(decision.Reasons); + var metadata = ExtractExtensionMetadata(decision.ExtensionData); + + var hasSbom = decision.HasSbomReferrers ?? decision.HasSbomLegacy; + + RuntimePolicyRekorReference? rekor = null; + if (decision.Rekor is not null && + (!string.IsNullOrWhiteSpace(decision.Rekor.Uuid) || + !string.IsNullOrWhiteSpace(decision.Rekor.Url) || + decision.Rekor.Verified.HasValue)) + { + rekor = new RuntimePolicyRekorReference( + NormalizeOptionalString(decision.Rekor.Uuid), + NormalizeOptionalString(decision.Rekor.Url), + decision.Rekor.Verified); + } + + decisions[image] = new RuntimePolicyImageDecision( + verdict, + decision.Signed, + hasSbom, + reasons, + rekor, + metadata); + } + } + + var decisionsView = new ReadOnlyDictionary(decisions); + + return new RuntimePolicyEvaluationResult( + document.TtlSeconds ?? 0, + document.ExpiresAtUtc?.ToUniversalTime(), + string.IsNullOrWhiteSpace(document.PolicyRevision) ? null : document.PolicyRevision, + decisionsView); + } + + public async Task> GetExcititorProvidersAsync(bool includeDisabled, CancellationToken cancellationToken) + { + EnsureBackendConfigured(); + + var query = includeDisabled ? "?includeDisabled=true" : string.Empty; + using var request = CreateRequest(HttpMethod.Get, $"excititor/providers{query}"); + await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false); + using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(failure); + } + + if (response.Content is null || response.Content.Headers.ContentLength is 0) + { + return Array.Empty(); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + if (stream is null || stream.Length == 0) + { + return Array.Empty(); + } + + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("providers", out var providersProperty)) + { + root = providersProperty; + } + + if (root.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var list = new List(); + foreach (var item in root.EnumerateArray()) + { + var id = GetStringProperty(item, "id") ?? string.Empty; + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + var kind = GetStringProperty(item, "kind") ?? "unknown"; + var displayName = GetStringProperty(item, "displayName") ?? id; + var trustTier = GetStringProperty(item, "trustTier") ?? string.Empty; + var enabled = GetBooleanProperty(item, "enabled", defaultValue: true); + var lastIngested = GetDateTimeOffsetProperty(item, "lastIngestedAt"); + + list.Add(new ExcititorProviderSummary(id, kind, displayName, trustTier, enabled, lastIngested)); + } + + return list; + } + + private static List NormalizeImages(IReadOnlyList images) + { + var normalized = new List(); + if (images is null) + { + return normalized; + } + + var seen = new HashSet(StringComparer.Ordinal); + foreach (var entry in images) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var trimmed = entry.Trim(); + if (seen.Add(trimmed)) + { + normalized.Add(trimmed); + } + } + + return normalized; + } + + private static IReadOnlyList ExtractReasons(List? reasons) + { + if (reasons is null || reasons.Count == 0) + { + return Array.Empty(); + } + + var list = new List(); + foreach (var reason in reasons) + { + if (!string.IsNullOrWhiteSpace(reason)) + { + list.Add(reason.Trim()); + } + } + + return list.Count == 0 ? Array.Empty() : list; + } + + private static IReadOnlyDictionary ExtractExtensionMetadata(Dictionary? extensionData) + { + if (extensionData is null || extensionData.Count == 0) + { + return EmptyMetadata; + } + + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in extensionData) + { + var value = ConvertJsonElementToObject(kvp.Value); + if (value is not null) + { + metadata[kvp.Key] = value; + } + } + + if (metadata.Count == 0) + { + return EmptyMetadata; + } + + return new ReadOnlyDictionary(metadata); + } + + private static object? ConvertJsonElementToObject(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Number when element.TryGetInt64(out var integer) => integer, + JsonValueKind.Number when element.TryGetDouble(out var @double) => @double, + JsonValueKind.Null or JsonValueKind.Undefined => null, + _ => element.GetRawText() + }; + } + + private static string? NormalizeOptionalString(string? value) + { + return string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + } + + private HttpRequestMessage CreateRequest(HttpMethod method, string relativeUri) + { + if (!Uri.TryCreate(relativeUri, UriKind.RelativeOrAbsolute, out var requestUri)) + { + throw new InvalidOperationException($"Invalid request URI '{relativeUri}'."); } if (requestUri.IsAbsoluteUri) @@ -249,88 +638,192 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient else { requestUri = new Uri(relativeUri.TrimStart('/'), UriKind.Relative); - } - - return new HttpRequestMessage(method, requestUri); - } - - private async Task AuthorizeRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - var token = await ResolveAccessTokenAsync(cancellationToken).ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(token)) - { - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); - } - } - - private async Task ResolveAccessTokenAsync(CancellationToken cancellationToken) - { - if (!string.IsNullOrWhiteSpace(_options.ApiKey)) - { - return _options.ApiKey; - } - - if (_tokenClient is null || string.IsNullOrWhiteSpace(_options.Authority.Url)) - { - return null; - } - - var now = DateTimeOffset.UtcNow; - - lock (_tokenSync) - { - if (!string.IsNullOrEmpty(_cachedAccessToken) && now < _cachedAccessTokenExpiresAt - TokenRefreshSkew) - { - return _cachedAccessToken; - } - } - - var cacheKey = AuthorityTokenUtilities.BuildCacheKey(_options); - var cachedEntry = await _tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); - if (cachedEntry is not null && now < cachedEntry.ExpiresAtUtc - TokenRefreshSkew) - { - lock (_tokenSync) - { - _cachedAccessToken = cachedEntry.AccessToken; - _cachedAccessTokenExpiresAt = cachedEntry.ExpiresAtUtc; - return _cachedAccessToken; - } - } - - var scope = AuthorityTokenUtilities.ResolveScope(_options); - - StellaOpsTokenResult token; - if (!string.IsNullOrWhiteSpace(_options.Authority.Username)) - { - if (string.IsNullOrWhiteSpace(_options.Authority.Password)) - { - throw new InvalidOperationException("Authority password must be configured when username is provided."); - } - - token = await _tokenClient.RequestPasswordTokenAsync( - _options.Authority.Username, - _options.Authority.Password!, - scope, - cancellationToken).ConfigureAwait(false); - } - else - { - token = await _tokenClient.RequestClientCredentialsTokenAsync(scope, cancellationToken).ConfigureAwait(false); - } - - await _tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); - - lock (_tokenSync) - { - _cachedAccessToken = token.AccessToken; - _cachedAccessTokenExpiresAt = token.ExpiresAtUtc; - return _cachedAccessToken; - } - } - - private void EnsureBackendConfigured() - { - if (_httpClient.BaseAddress is null) + } + + return new HttpRequestMessage(method, requestUri); + } + + private async Task AuthorizeRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var token = await ResolveAccessTokenAsync(cancellationToken).ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(token)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + } + } + + private async Task ResolveAccessTokenAsync(CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(_options.ApiKey)) + { + return _options.ApiKey; + } + + if (_tokenClient is null || string.IsNullOrWhiteSpace(_options.Authority.Url)) + { + return null; + } + + var now = DateTimeOffset.UtcNow; + + lock (_tokenSync) + { + if (!string.IsNullOrEmpty(_cachedAccessToken) && now < _cachedAccessTokenExpiresAt - TokenRefreshSkew) + { + return _cachedAccessToken; + } + } + + var cacheKey = AuthorityTokenUtilities.BuildCacheKey(_options); + var cachedEntry = await _tokenClient.GetCachedTokenAsync(cacheKey, cancellationToken).ConfigureAwait(false); + if (cachedEntry is not null && now < cachedEntry.ExpiresAtUtc - TokenRefreshSkew) + { + lock (_tokenSync) + { + _cachedAccessToken = cachedEntry.AccessToken; + _cachedAccessTokenExpiresAt = cachedEntry.ExpiresAtUtc; + return _cachedAccessToken; + } + } + + var scope = AuthorityTokenUtilities.ResolveScope(_options); + + StellaOpsTokenResult token; + if (!string.IsNullOrWhiteSpace(_options.Authority.Username)) + { + if (string.IsNullOrWhiteSpace(_options.Authority.Password)) + { + throw new InvalidOperationException("Authority password must be configured when username is provided."); + } + + token = await _tokenClient.RequestPasswordTokenAsync( + _options.Authority.Username, + _options.Authority.Password!, + scope, + cancellationToken).ConfigureAwait(false); + } + else + { + token = await _tokenClient.RequestClientCredentialsTokenAsync(scope, cancellationToken).ConfigureAwait(false); + } + + await _tokenClient.CacheTokenAsync(cacheKey, token.ToCacheEntry(), cancellationToken).ConfigureAwait(false); + + lock (_tokenSync) + { + _cachedAccessToken = token.AccessToken; + _cachedAccessTokenExpiresAt = token.ExpiresAtUtc; + return _cachedAccessToken; + } + } + + private async Task<(string Message, JsonElement? Payload)> ExtractExcititorResponseAsync(HttpResponseMessage response, CancellationToken cancellationToken) + { + if (response.Content is null || response.Content.Headers.ContentLength is 0) + { + return ($"HTTP {(int)response.StatusCode}", null); + } + + try + { + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + if (stream is null || stream.Length == 0) + { + return ($"HTTP {(int)response.StatusCode}", null); + } + + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement.Clone(); + string? message = null; + if (root.ValueKind == JsonValueKind.Object) + { + message = GetStringProperty(root, "message") ?? GetStringProperty(root, "status"); + } + + if (string.IsNullOrWhiteSpace(message)) + { + message = root.ValueKind == JsonValueKind.Object || root.ValueKind == JsonValueKind.Array + ? root.ToString() + : root.GetRawText(); + } + + return (message ?? $"HTTP {(int)response.StatusCode}", root); + } + catch (JsonException) + { + var text = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return (string.IsNullOrWhiteSpace(text) ? $"HTTP {(int)response.StatusCode}" : text.Trim(), null); + } + } + + private static bool TryGetPropertyCaseInsensitive(JsonElement element, string propertyName, out JsonElement property) + { + if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out property)) + { + return true; + } + + if (element.ValueKind == JsonValueKind.Object) + { + foreach (var candidate in element.EnumerateObject()) + { + if (string.Equals(candidate.Name, propertyName, StringComparison.OrdinalIgnoreCase)) + { + property = candidate.Value; + return true; + } + } + } + + property = default; + return false; + } + + private static string? GetStringProperty(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property)) + { + if (property.ValueKind == JsonValueKind.String) + { + return property.GetString(); + } + } + + return null; + } + + private static bool GetBooleanProperty(JsonElement element, string propertyName, bool defaultValue) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property)) + { + return property.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed, + _ => defaultValue + }; + } + + return defaultValue; + } + + private static DateTimeOffset? GetDateTimeOffsetProperty(JsonElement element, string propertyName) + { + if (TryGetPropertyCaseInsensitive(element, propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + if (DateTimeOffset.TryParse(property.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + } + + return null; + } + + private void EnsureBackendConfigured() + { + if (_httpClient.BaseAddress is null) { throw new InvalidOperationException("Backend URL is not configured. Provide STELLAOPS_BACKEND_URL or configure appsettings."); } @@ -402,6 +895,19 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient return null; } + private static string? NormalizeExpectedDigest(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var trimmed = digest.Trim(); + return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) + ? trimmed[7..] + : trimmed; + } + private async Task ValidateDigestAsync(string filePath, string? expectedDigest, CancellationToken cancellationToken) { string digestHex; @@ -438,6 +944,13 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient return digest; } + private static async Task ComputeSha256Async(string filePath, CancellationToken cancellationToken) + { + await using var stream = File.OpenRead(filePath); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + private async Task ValidateSignatureAsync(string? signatureHeader, string digestHex, bool verbose, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(_options.ScannerSignaturePublicKeyPath)) @@ -479,11 +992,11 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient } } - private void PersistMetadata(string outputPath, string channel, string digestHex, string? signatureHeader, HttpResponseMessage response) - { - var metadata = new - { - channel, + private void PersistMetadata(string outputPath, string channel, string digestHex, string? signatureHeader, HttpResponseMessage response) + { + var metadata = new + { + channel, digest = $"sha256:{digestHex}", signature = signatureHeader, downloadedAt = DateTimeOffset.UtcNow, @@ -502,34 +1015,34 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient { WriteIndented = true }); - - File.WriteAllText(metadataPath, json); - } - - private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) - { - if (response.Headers.TryGetValues("Retry-After", out var retryValues)) - { - var value = retryValues.FirstOrDefault(); - if (!string.IsNullOrWhiteSpace(value)) - { - if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds) && seconds >= 0) - { - return TimeSpan.FromSeconds(Math.Min(seconds, 300)); - } - - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var when)) - { - var delta = when - DateTimeOffset.UtcNow; - if (delta > TimeSpan.Zero) - { - return delta < TimeSpan.FromMinutes(5) ? delta : TimeSpan.FromMinutes(5); - } - } - } - } - - var fallbackSeconds = Math.Min(60, Math.Pow(2, attempt)); - return TimeSpan.FromSeconds(fallbackSeconds); - } -} + + File.WriteAllText(metadataPath, json); + } + + private static TimeSpan GetRetryDelay(HttpResponseMessage response, int attempt) + { + if (response.Headers.TryGetValues("Retry-After", out var retryValues)) + { + var value = retryValues.FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(value)) + { + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var seconds) && seconds >= 0) + { + return TimeSpan.FromSeconds(Math.Min(seconds, 300)); + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var when)) + { + var delta = when - DateTimeOffset.UtcNow; + if (delta > TimeSpan.Zero) + { + return delta < TimeSpan.FromMinutes(5) ? delta : TimeSpan.FromMinutes(5); + } + } + } + } + + var fallbackSeconds = Math.Min(60, Math.Pow(2, attempt)); + return TimeSpan.FromSeconds(fallbackSeconds); + } +} diff --git a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs index b593524b..1c44f132 100644 --- a/src/StellaOps.Cli/Services/IBackendOperationsClient.cs +++ b/src/StellaOps.Cli/Services/IBackendOperationsClient.cs @@ -1,16 +1,25 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Cli.Configuration; -using StellaOps.Cli.Services.Models; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Cli.Configuration; +using StellaOps.Cli.Services.Models; namespace StellaOps.Cli.Services; internal interface IBackendOperationsClient { - Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken); - - Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken); - - Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken); -} + Task DownloadScannerAsync(string channel, string outputPath, bool overwrite, bool verbose, CancellationToken cancellationToken); + + Task UploadScanResultsAsync(string filePath, CancellationToken cancellationToken); + + Task TriggerJobAsync(string jobKind, IDictionary parameters, CancellationToken cancellationToken); + + Task ExecuteExcititorOperationAsync(string route, HttpMethod method, object? payload, CancellationToken cancellationToken); + + Task DownloadExcititorExportAsync(string exportId, string destinationPath, string? expectedDigestAlgorithm, string? expectedDigest, CancellationToken cancellationToken); + + Task> GetExcititorProvidersAsync(bool includeDisabled, CancellationToken cancellationToken); + + Task EvaluateRuntimePolicyAsync(RuntimePolicyEvaluationRequest request, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Cli/Services/Models/ExcititorExportDownloadResult.cs b/src/StellaOps.Cli/Services/Models/ExcititorExportDownloadResult.cs new file mode 100644 index 00000000..b1364345 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/ExcititorExportDownloadResult.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Cli.Services.Models; + +internal sealed record ExcititorExportDownloadResult( + string Path, + long SizeBytes, + bool FromCache); diff --git a/src/StellaOps.Cli/Services/Models/ExcititorOperationResult.cs b/src/StellaOps.Cli/Services/Models/ExcititorOperationResult.cs new file mode 100644 index 00000000..05fdeaa4 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/ExcititorOperationResult.cs @@ -0,0 +1,9 @@ +using System.Text.Json; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record ExcititorOperationResult( + bool Success, + string Message, + string? Location, + JsonElement? Payload); diff --git a/src/StellaOps.Cli/Services/Models/ExcititorProviderSummary.cs b/src/StellaOps.Cli/Services/Models/ExcititorProviderSummary.cs new file mode 100644 index 00000000..ff490c04 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/ExcititorProviderSummary.cs @@ -0,0 +1,11 @@ +using System; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record ExcititorProviderSummary( + string Id, + string Kind, + string DisplayName, + string TrustTier, + bool Enabled, + DateTimeOffset? LastIngestedAt); diff --git a/src/StellaOps.Cli/Services/Models/RuntimePolicyEvaluationModels.cs b/src/StellaOps.Cli/Services/Models/RuntimePolicyEvaluationModels.cs new file mode 100644 index 00000000..e60294f7 --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/RuntimePolicyEvaluationModels.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Cli.Services.Models; + +internal sealed record RuntimePolicyEvaluationRequest( + string? Namespace, + IReadOnlyDictionary Labels, + IReadOnlyList Images); + +internal sealed record RuntimePolicyEvaluationResult( + int TtlSeconds, + DateTimeOffset? ExpiresAtUtc, + string? PolicyRevision, + IReadOnlyDictionary Decisions); + +internal sealed record RuntimePolicyImageDecision( + string PolicyVerdict, + bool? Signed, + bool? HasSbomReferrers, + IReadOnlyList Reasons, + RuntimePolicyRekorReference? Rekor, + IReadOnlyDictionary AdditionalProperties); + +internal sealed record RuntimePolicyRekorReference(string? Uuid, string? Url, bool? Verified); diff --git a/src/StellaOps.Cli/Services/Models/Transport/RuntimePolicyEvaluationTransport.cs b/src/StellaOps.Cli/Services/Models/Transport/RuntimePolicyEvaluationTransport.cs new file mode 100644 index 00000000..3dc20cfb --- /dev/null +++ b/src/StellaOps.Cli/Services/Models/Transport/RuntimePolicyEvaluationTransport.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Cli.Services.Models.Transport; + +internal sealed class RuntimePolicyEvaluationRequestDocument +{ + [JsonPropertyName("namespace")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Namespace { get; set; } + + [JsonPropertyName("labels")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Dictionary? Labels { get; set; } + + [JsonPropertyName("images")] + public List Images { get; set; } = new(); +} + +internal sealed class RuntimePolicyEvaluationResponseDocument +{ + [JsonPropertyName("ttlSeconds")] + public int? TtlSeconds { get; set; } + + [JsonPropertyName("expiresAtUtc")] + public DateTimeOffset? ExpiresAtUtc { get; set; } + + [JsonPropertyName("policyRevision")] + public string? PolicyRevision { get; set; } + + [JsonPropertyName("results")] + public Dictionary? Results { get; set; } +} + +internal sealed class RuntimePolicyEvaluationImageDocument +{ + [JsonPropertyName("policyVerdict")] + public string? PolicyVerdict { get; set; } + + [JsonPropertyName("signed")] + public bool? Signed { get; set; } + + [JsonPropertyName("hasSbomReferrers")] + public bool? HasSbomReferrers { get; set; } + + // Legacy field kept for pre-contract-sync services. + [JsonPropertyName("hasSbom")] + public bool? HasSbomLegacy { get; set; } + + [JsonPropertyName("reasons")] + public List? Reasons { get; set; } + + [JsonPropertyName("rekor")] + public RuntimePolicyRekorDocument? Rekor { get; set; } + + [JsonExtensionData] + public Dictionary? ExtensionData { get; set; } +} + +internal sealed class RuntimePolicyRekorDocument +{ + [JsonPropertyName("uuid")] + public string? Uuid { get; set; } + + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("verified")] + public bool? Verified { get; set; } +} diff --git a/src/StellaOps.Cli/TASKS.md b/src/StellaOps.Cli/TASKS.md index 1b7fc261..fbf604b8 100644 --- a/src/StellaOps.Cli/TASKS.md +++ b/src/StellaOps.Cli/TASKS.md @@ -1,4 +1,4 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). # TASKS | Task | Owner(s) | Depends on | Notes | |---|---|---|---| @@ -6,7 +6,7 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and |Introduce command host & routing skeleton|DevEx/CLI|Configuration|**DONE** – System.CommandLine (v2.0.0-beta5) router stitched with `scanner`, `scan`, `db`, and `config` verbs.| |Scanner artifact download/install commands|Ops Integrator|Backend contracts|**DONE** – `scanner download` caches bundles, validates SHA-256 (plus optional RSA signature), installs via `docker load`, persists metadata, and retries with exponential backoff.| |Scan execution & result upload workflow|Ops Integrator, QA|Scanner cmd|**DONE** – `scan run` drives container scans against directories, emits artefacts in `ResultsDirectory`, auto-uploads on success, and `scan upload` covers manual retries.| -|Feedser DB operations passthrough|DevEx/CLI|Backend, Feedser APIs|**DONE** – `db fetch|merge|export` trigger `/jobs/*` endpoints with parameter binding and consistent exit codes.| +|Concelier DB operations passthrough|DevEx/CLI|Backend, Concelier APIs|**DONE** – `db fetch|merge|export` trigger `/jobs/*` endpoints with parameter binding and consistent exit codes.| |CLI observability & tests|QA|Command host|**DONE** – Added console logging defaults & configuration bootstrap tests; future metrics hooks tracked separately.| |Authority auth commands|DevEx/CLI|Auth libraries|**DONE** – `auth login/logout/status` wrap the shared auth client, manage token cache, and surface status messages.| |Document authority workflow in CLI help & quickstart|Docs/CLI|Authority auth commands|**DONE (2025-10-10)** – CLI help now surfaces Authority config fields and docs/09 + docs/10 describe env vars, auth login/status flow, and cache location.| @@ -14,6 +14,11 @@ If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and |Expose auth client resilience settings|DevEx/CLI|Auth libraries LIB5|**DONE (2025-10-10)** – CLI options now bind resilience knobs, `AddStellaOpsAuthClient` honours them, and tests cover env overrides.| |Document advanced Authority tuning|Docs/CLI|Expose auth client resilience settings|**DONE (2025-10-10)** – docs/09 and docs/10 describe retry/offline settings with env examples and point to the integration guide.| |Surface password policy diagnostics in CLI output|DevEx/CLI, Security Guild|AUTHSEC-CRYPTO-02-004|**DONE (2025-10-15)** – CLI startup runs the Authority plug-in analyzer, logs weakened password policy warnings with manifest paths, added unit tests (`dotnet test src/StellaOps.Cli.Tests`) and updated docs/09 with remediation guidance.| -|VEXER-CLI-01-001 – Add `vexer` command group|DevEx/CLI|VEXER-WEB-01-001|TODO – Introduce `vexer` verb hierarchy (init/pull/resume/list-providers/export/verify/reconcile) forwarding to WebService with token auth and consistent exit codes.| -|VEXER-CLI-01-002 – Export download & attestation UX|DevEx/CLI|VEXER-CLI-01-001, VEXER-EXPORT-01-001|TODO – Display export metadata (sha256, size, Rekor link), support optional artifact download path, and handle cache hits gracefully.| -|VEXER-CLI-01-003 – CLI docs & examples for Vexer|Docs/CLI|VEXER-CLI-01-001|TODO – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Vexer verbs, offline guidance, and attestation verification workflow.| +|EXCITITOR-CLI-01-001 – Add `excititor` command group|DevEx/CLI|EXCITITOR-WEB-01-001|DONE (2025-10-18) – Introduced `excititor` verbs (init/pull/resume/list-providers/export/verify/reconcile) with token-auth backend calls, provenance-friendly logging, and regression coverage.| +|EXCITITOR-CLI-01-002 – Export download & attestation UX|DevEx/CLI|EXCITITOR-CLI-01-001, EXCITITOR-EXPORT-01-001|DONE (2025-10-19) – CLI export prints digest/size/Rekor metadata, `--output` downloads with SHA-256 verification + cache reuse, and unit coverage validated via `dotnet test src/StellaOps.Cli.Tests`.| +|EXCITITOR-CLI-01-003 – CLI docs & examples for Excititor|Docs/CLI|EXCITITOR-CLI-01-001|**DOING (2025-10-19)** – Update docs/09_API_CLI_REFERENCE.md and quickstart snippets to cover Excititor verbs, offline guidance, and attestation verification workflow.| +|CLI-RUNTIME-13-005 – Runtime policy test verbs|DevEx/CLI|SCANNER-RUNTIME-12-302, ZASTAVA-WEBHOOK-12-102|**DONE (2025-10-19)** – Added `runtime policy test` command (stdin/file support, JSON output), backend client method + typed models, verdict table output, docs/tests updated (`dotnet test src/StellaOps.Cli.Tests`).| +|CLI-OFFLINE-13-006 – Offline kit workflows|DevEx/CLI|DEVOPS-OFFLINE-14-002|TODO – Implement `offline kit pull/import/status` commands with integrity checks, resumable downloads, and doc updates.| +|CLI-PLUGIN-13-007 – Plugin packaging|DevEx/CLI|CLI-RUNTIME-13-005, CLI-OFFLINE-13-006|TODO – Package non-core verbs as restart-time plug-ins (manifest + loader updates, tests ensuring no hot reload).| +|CLI-RUNTIME-13-008 – Runtime policy contract sync|DevEx/CLI, Scanner WebService Guild|SCANNER-RUNTIME-12-302|**DONE (2025-10-19)** – CLI runtime table/JSON now align with SCANNER-RUNTIME-12-302 (SBOM referrers, quieted provenance, confidence, verified Rekor); docs/09 updated with joint sign-off note.| +|CLI-RUNTIME-13-009 – Runtime policy smoke fixture|DevEx/CLI, QA Guild|CLI-RUNTIME-13-005|**DONE (2025-10-19)** – Spectre console harness + regression tests cover table and `--json` output paths for `runtime policy test`, using stubbed backend and integrated into `dotnet test` suite.| diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs rename to src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs index b5e29013..656875df 100644 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorFetchTests.cs @@ -1,215 +1,215 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Net; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Acsc; -using StellaOps.Feedser.Source.Acsc.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Acsc.Tests.Acsc; - -[Collection("mongo-fixture")] -public sealed class AcscConnectorFetchTests : IAsyncLifetime -{ - private static readonly Uri BaseEndpoint = new("https://origin.example/"); - private static readonly Uri RelayEndpoint = new("https://relay.example/"); - private static readonly Uri AlertsDirectUri = new(BaseEndpoint, "/feeds/alerts/rss"); - private static readonly Uri AlertsRelayUri = new(RelayEndpoint, "/feeds/alerts/rss"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - - public AcscConnectorFetchTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchAsync_DirectSuccessAdvancesCursor() - { - await using var provider = await BuildProviderAsync(preferRelay: false); - - var connector = provider.GetRequiredService(); - SeedRssResponse(AlertsDirectUri, "direct", DateTimeOffset.Parse("2025-10-10T02:15:00Z"), DateTimeOffset.Parse("2025-10-11T05:30:00Z")); - - await connector.FetchAsync(provider, CancellationToken.None); - _handler.AssertNoPendingResponses(); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.Equal("Direct", state!.Cursor.GetValue("preferredEndpoint").AsString); - - var feeds = state.Cursor.GetValue("feeds").AsBsonDocument; - Assert.True(feeds.TryGetValue("alerts", out var published)); - Assert.Equal(DateTime.Parse("2025-10-11T05:30:00Z").ToUniversalTime(), published.ToUniversalTime()); - - var pendingDocuments = state.Cursor.GetValue("pendingDocuments").AsBsonArray; - Assert.Single(pendingDocuments); - - var documentStore = provider.GetRequiredService(); - var documentId = Guid.Parse(pendingDocuments[0]!.AsString); - var document = await documentStore.FindAsync(documentId, CancellationToken.None); - Assert.NotNull(document); - Assert.Equal(DocumentStatuses.PendingParse, document!.Status); - var directMetadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); - Assert.True(directMetadata.TryGetValue("acsc.fetch.mode", out var mode)); - Assert.Equal("direct", mode); - } - - [Fact] - public async Task FetchAsync_DirectFailureFallsBackToRelay() - { - await using var provider = await BuildProviderAsync(preferRelay: false); - - var connector = provider.GetRequiredService(); - _handler.AddException(HttpMethod.Get, AlertsDirectUri, new HttpRequestException("HTTP/2 reset")); - SeedRssResponse(AlertsRelayUri, "relay", DateTimeOffset.Parse("2025-10-09T10:00:00Z"), DateTimeOffset.Parse("2025-10-11T00:00:00Z")); - - await connector.FetchAsync(provider, CancellationToken.None); - _handler.AssertNoPendingResponses(); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.Equal("Relay", state!.Cursor.GetValue("preferredEndpoint").AsString); - - var feeds = state.Cursor.GetValue("feeds").AsBsonDocument; - Assert.True(feeds.TryGetValue("alerts", out var published)); - Assert.Equal(DateTime.Parse("2025-10-11T00:00:00Z").ToUniversalTime(), published.ToUniversalTime()); - - var pendingDocuments = state.Cursor.GetValue("pendingDocuments").AsBsonArray; - Assert.Single(pendingDocuments); - - var documentStore = provider.GetRequiredService(); - var documentId = Guid.Parse(pendingDocuments[0]!.AsString); - var document = await documentStore.FindAsync(documentId, CancellationToken.None); - Assert.NotNull(document); - Assert.Equal(DocumentStatuses.PendingParse, document!.Status); - var metadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); - Assert.True(metadata.TryGetValue("acsc.fetch.mode", out var mode)); - Assert.Equal("relay", mode); - - Assert.Collection(_handler.Requests, - request => - { - Assert.Equal(HttpMethod.Get, request.Method); - Assert.Equal(AlertsDirectUri, request.Uri); - }, - request => - { - Assert.Equal(HttpMethod.Get, request.Method); - Assert.Equal(AlertsRelayUri, request.Uri); - }); - } - - public async Task InitializeAsync() => await Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; - - private async Task BuildProviderAsync(bool preferRelay) - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddAcscConnector(options => - { - options.BaseEndpoint = BaseEndpoint; - options.RelayEndpoint = RelayEndpoint; - options.EnableRelayFallback = true; - options.PreferRelayByDefault = preferRelay; - options.ForceRelay = false; - options.RequestTimeout = TimeSpan.FromSeconds(10); - options.Feeds.Clear(); - options.Feeds.Add(new AcscFeedOptions - { - Slug = "alerts", - RelativePath = "/feeds/alerts/rss", - Enabled = true, - }); - }); - - services.Configure(AcscOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); - }); - services.Configure(AcscOptions.HttpClientName, options => - { - options.MaxAttempts = 1; - options.BaseDelay = TimeSpan.Zero; - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedRssResponse(Uri uri, string mode, DateTimeOffset first, DateTimeOffset second) - { - var payload = CreateRssPayload(first, second); - _handler.AddResponse(HttpMethod.Get, uri, _ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), - }; - - response.Headers.ETag = new System.Net.Http.Headers.EntityTagHeaderValue($"\"{mode}-etag\""); - response.Content.Headers.LastModified = second; - return response; - }); - } - - private static string CreateRssPayload(DateTimeOffset first, DateTimeOffset second) - { - return $$""" - - - - Alerts - https://origin.example/feeds/alerts - - First - https://origin.example/alerts/first - {{first.ToString("r", CultureInfo.InvariantCulture)}} - - - Second - https://origin.example/alerts/second - {{second.ToString("r", CultureInfo.InvariantCulture)}} - - - - """; - } -} +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Acsc; +using StellaOps.Concelier.Connector.Acsc.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Acsc.Tests.Acsc; + +[Collection("mongo-fixture")] +public sealed class AcscConnectorFetchTests : IAsyncLifetime +{ + private static readonly Uri BaseEndpoint = new("https://origin.example/"); + private static readonly Uri RelayEndpoint = new("https://relay.example/"); + private static readonly Uri AlertsDirectUri = new(BaseEndpoint, "/feeds/alerts/rss"); + private static readonly Uri AlertsRelayUri = new(RelayEndpoint, "/feeds/alerts/rss"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public AcscConnectorFetchTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchAsync_DirectSuccessAdvancesCursor() + { + await using var provider = await BuildProviderAsync(preferRelay: false); + + var connector = provider.GetRequiredService(); + SeedRssResponse(AlertsDirectUri, "direct", DateTimeOffset.Parse("2025-10-10T02:15:00Z"), DateTimeOffset.Parse("2025-10-11T05:30:00Z")); + + await connector.FetchAsync(provider, CancellationToken.None); + _handler.AssertNoPendingResponses(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal("Direct", state!.Cursor.GetValue("preferredEndpoint").AsString); + + var feeds = state.Cursor.GetValue("feeds").AsBsonDocument; + Assert.True(feeds.TryGetValue("alerts", out var published)); + Assert.Equal(DateTime.Parse("2025-10-11T05:30:00Z").ToUniversalTime(), published.ToUniversalTime()); + + var pendingDocuments = state.Cursor.GetValue("pendingDocuments").AsBsonArray; + Assert.Single(pendingDocuments); + + var documentStore = provider.GetRequiredService(); + var documentId = Guid.Parse(pendingDocuments[0]!.AsString); + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.PendingParse, document!.Status); + var directMetadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); + Assert.True(directMetadata.TryGetValue("acsc.fetch.mode", out var mode)); + Assert.Equal("direct", mode); + } + + [Fact] + public async Task FetchAsync_DirectFailureFallsBackToRelay() + { + await using var provider = await BuildProviderAsync(preferRelay: false); + + var connector = provider.GetRequiredService(); + _handler.AddException(HttpMethod.Get, AlertsDirectUri, new HttpRequestException("HTTP/2 reset")); + SeedRssResponse(AlertsRelayUri, "relay", DateTimeOffset.Parse("2025-10-09T10:00:00Z"), DateTimeOffset.Parse("2025-10-11T00:00:00Z")); + + await connector.FetchAsync(provider, CancellationToken.None); + _handler.AssertNoPendingResponses(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal("Relay", state!.Cursor.GetValue("preferredEndpoint").AsString); + + var feeds = state.Cursor.GetValue("feeds").AsBsonDocument; + Assert.True(feeds.TryGetValue("alerts", out var published)); + Assert.Equal(DateTime.Parse("2025-10-11T00:00:00Z").ToUniversalTime(), published.ToUniversalTime()); + + var pendingDocuments = state.Cursor.GetValue("pendingDocuments").AsBsonArray; + Assert.Single(pendingDocuments); + + var documentStore = provider.GetRequiredService(); + var documentId = Guid.Parse(pendingDocuments[0]!.AsString); + var document = await documentStore.FindAsync(documentId, CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.PendingParse, document!.Status); + var metadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); + Assert.True(metadata.TryGetValue("acsc.fetch.mode", out var mode)); + Assert.Equal("relay", mode); + + Assert.Collection(_handler.Requests, + request => + { + Assert.Equal(HttpMethod.Get, request.Method); + Assert.Equal(AlertsDirectUri, request.Uri); + }, + request => + { + Assert.Equal(HttpMethod.Get, request.Method); + Assert.Equal(AlertsRelayUri, request.Uri); + }); + } + + public async Task InitializeAsync() => await Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task BuildProviderAsync(bool preferRelay) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAcscConnector(options => + { + options.BaseEndpoint = BaseEndpoint; + options.RelayEndpoint = RelayEndpoint; + options.EnableRelayFallback = true; + options.PreferRelayByDefault = preferRelay; + options.ForceRelay = false; + options.RequestTimeout = TimeSpan.FromSeconds(10); + options.Feeds.Clear(); + options.Feeds.Add(new AcscFeedOptions + { + Slug = "alerts", + RelativePath = "/feeds/alerts/rss", + Enabled = true, + }); + }); + + services.Configure(AcscOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + services.Configure(AcscOptions.HttpClientName, options => + { + options.MaxAttempts = 1; + options.BaseDelay = TimeSpan.Zero; + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedRssResponse(Uri uri, string mode, DateTimeOffset first, DateTimeOffset second) + { + var payload = CreateRssPayload(first, second); + _handler.AddResponse(HttpMethod.Get, uri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), + }; + + response.Headers.ETag = new System.Net.Http.Headers.EntityTagHeaderValue($"\"{mode}-etag\""); + response.Content.Headers.LastModified = second; + return response; + }); + } + + private static string CreateRssPayload(DateTimeOffset first, DateTimeOffset second) + { + return $$""" + + + + Alerts + https://origin.example/feeds/alerts + + First + https://origin.example/alerts/first + {{first.ToString("r", CultureInfo.InvariantCulture)}} + + + Second + https://origin.example/alerts/second + {{second.ToString("r", CultureInfo.InvariantCulture)}} + + + + """; + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorParseTests.cs b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorParseTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorParseTests.cs rename to src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorParseTests.cs index a54ac2ae..5f18b328 100644 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscConnectorParseTests.cs +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscConnectorParseTests.cs @@ -1,371 +1,371 @@ -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Acsc; -using StellaOps.Feedser.Source.Acsc.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Acsc.Tests.Acsc; - -[Collection("mongo-fixture")] -public sealed class AcscConnectorParseTests : IAsyncLifetime -{ - private static readonly Uri BaseEndpoint = new("https://origin.example/"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - - public AcscConnectorParseTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task ParseAsync_PersistsDtoAndAdvancesCursor() - { - await using var provider = await BuildProviderAsync(); - var connector = provider.GetRequiredService(); - - var feedUri = new Uri(BaseEndpoint, "/feeds/alerts/rss"); - SeedRssResponse(feedUri); - - await connector.FetchAsync(provider, CancellationToken.None); - _handler.AssertNoPendingResponses(); - - var documentStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(AcscConnectorPlugin.SourceName, feedUri.ToString(), CancellationToken.None); - Assert.NotNull(document); - - await connector.ParseAsync(provider, CancellationToken.None); - - var refreshed = await documentStore.FindAsync(document!.Id, CancellationToken.None); - Assert.NotNull(refreshed); - Assert.Equal(DocumentStatuses.PendingMap, refreshed!.Status); - - var dtoStore = provider.GetRequiredService(); - var dtoRecord = await dtoStore.FindByDocumentIdAsync(document.Id, CancellationToken.None); - Assert.NotNull(dtoRecord); - Assert.Equal("acsc.feed.v1", dtoRecord!.SchemaVersion); - - var payload = dtoRecord.Payload; - Assert.NotNull(payload); - Assert.Equal("alerts", payload.GetValue("feedSlug").AsString); - Assert.Single(payload.GetValue("entries").AsBsonArray); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.DoesNotContain(document.Id.ToString(), state!.Cursor.GetValue("pendingDocuments").AsBsonArray.Select(v => v.AsString)); - Assert.Contains(document.Id.ToString(), state.Cursor.GetValue("pendingMappings").AsBsonArray.Select(v => v.AsString)); - - await connector.MapAsync(provider, CancellationToken.None); - - var advisoriesStore = provider.GetRequiredService(); - var advisories = await advisoriesStore.GetRecentAsync(10, CancellationToken.None); - Assert.Single(advisories); - - var ordered = advisories - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - WriteOrAssertSnapshot( - SnapshotSerializer.ToSnapshot(ordered), - "acsc-advisories.snapshot.json"); - - var mappedDocument = await documentStore.FindAsync(document.Id, CancellationToken.None); - Assert.NotNull(mappedDocument); - Assert.Equal(DocumentStatuses.Mapped, mappedDocument!.Status); - - state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.True(state!.Cursor.GetValue("pendingMappings").AsBsonArray.Count == 0); - } - - [Fact] - public async Task MapAsync_MultiEntryFeedProducesExpectedSnapshot() - { - await using var provider = await BuildProviderAsync(options => - { - options.Feeds.Clear(); - options.Feeds.Add(new AcscFeedOptions - { - Slug = "multi", - RelativePath = "/feeds/multi/rss", - Enabled = true, - }); - }); - var connector = provider.GetRequiredService(); - - var feedUri = new Uri(BaseEndpoint, "/feeds/multi/rss"); - SeedMultiEntryResponse(feedUri); - - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - - var documentStore = provider.GetRequiredService(); - var dtoStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(AcscConnectorPlugin.SourceName, feedUri.ToString(), CancellationToken.None); - Assert.NotNull(document); - var dtoRecord = await dtoStore.FindByDocumentIdAsync(document!.Id, CancellationToken.None); - Assert.NotNull(dtoRecord); - var payload = dtoRecord!.Payload; - Assert.NotNull(payload); - var entries = payload.GetValue("entries").AsBsonArray; - Assert.Equal(2, entries.Count); - var fields = entries[0].AsBsonDocument.GetValue("fields").AsBsonDocument; - Assert.Equal("Critical", fields.GetValue("severity").AsString); - Assert.Equal("ExampleCo Router X, ExampleCo Router Y", fields.GetValue("systemsAffected").AsString); - - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(2, advisories.Count); - - var ordered = advisories - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - WriteOrAssertSnapshot( - SnapshotSerializer.ToSnapshot(ordered), - "acsc-advisories-multi.snapshot.json"); - - var affected = ordered.First(advisory => advisory.AffectedPackages.Any()); - Assert.Contains("ExampleCo Router X", affected.AffectedPackages[0].Identifier); - Assert.Equal("critical", ordered.First(a => a.Severity is not null).Severity, StringComparer.OrdinalIgnoreCase); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; - - private async Task BuildProviderAsync(Action? configure = null) - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddAcscConnector(options => - { - options.BaseEndpoint = BaseEndpoint; - options.RelayEndpoint = null; - options.PreferRelayByDefault = false; - options.ForceRelay = false; - options.EnableRelayFallback = false; - options.RequestTimeout = TimeSpan.FromSeconds(10); - options.Feeds.Clear(); - options.Feeds.Add(new AcscFeedOptions - { - Slug = "alerts", - RelativePath = "/feeds/alerts/rss", - Enabled = true, - }); - configure?.Invoke(options); - }); - - services.Configure(AcscOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); - }); - - services.Configure(AcscOptions.HttpClientName, options => - { - options.MaxAttempts = 1; - options.BaseDelay = TimeSpan.Zero; - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedRssResponse(Uri uri) - { - const string payload = """ - - - - ACSC Alerts - https://origin.example/feeds/alerts - Sun, 12 Oct 2025 04:20:00 GMT - - ACSC-2025-001 Example Advisory - https://origin.example/advisories/example - https://origin.example/advisories/example - Sun, 12 Oct 2025 03:00:00 GMT - Serial number: ACSC-2025-001

-

Advisory type: Alert

-

First paragraph describing issue.

-

Second paragraph with Vendor patch.

- ]]> - - - - """; - - _handler.AddResponse(HttpMethod.Get, uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), - }; - response.Headers.ETag = new EntityTagHeaderValue("\"parse-etag\""); - response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 12, 4, 20, 0, TimeSpan.Zero); - return response; - }); - } - - private void SeedMultiEntryResponse(Uri uri) - { - const string payload = """ - - - - ACSC Advisories - https://origin.example/feeds/advisories - Sun, 12 Oct 2025 05:00:00 GMT - - Critical router vulnerability - https://origin.example/advisories/router-critical - https://origin.example/advisories/router-critical - Sun, 12 Oct 2025 04:45:00 GMT - Serial number: ACSC-2025-010

-

Severity: Critical

-

Systems affected: ExampleCo Router X, ExampleCo Router Y

-

Remote code execution on ExampleCo routers. See vendor patch.

-

CVE references: CVE-2025-0001

- ]]> - - - Information bulletin - https://origin.example/advisories/info-bulletin - https://origin.example/advisories/info-bulletin - Sun, 12 Oct 2025 02:30:00 GMT - Serial number: ACSC-2025-011

-

Advisory type: Bulletin

-

General guidance bulletin.

- ]]> - - - - """; - - _handler.AddResponse(HttpMethod.Get, uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), - }; - response.Headers.ETag = new EntityTagHeaderValue("\"multi-etag\""); - response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 12, 5, 0, 0, TimeSpan.Zero); - return response; - }); - } - - private static void WriteOrAssertSnapshot(string snapshot, string filename) - { - if (ShouldUpdateFixtures() || !FixtureExists(filename)) - { - var writable = GetWritableFixturePath(filename); - Directory.CreateDirectory(Path.GetDirectoryName(writable)!); - File.WriteAllText(writable, Normalize(snapshot)); - return; - } - - var expected = Normalize(File.ReadAllText(GetExistingFixturePath(filename))); - var actual = Normalize(snapshot); - - if (!string.Equals(expected, actual, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); - File.WriteAllText(actualPath, actual); - } - - Assert.Equal(expected, actual); - } - - private static bool ShouldUpdateFixtures() - { - var value = Environment.GetEnvironmentVariable("UPDATE_ACSC_FIXTURES"); - return string.Equals(value, "1", StringComparison.Ordinal) - || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); - } - - private static string GetExistingFixturePath(string filename) - { - var baseDir = AppContext.BaseDirectory; - var primary = Path.Combine(baseDir, "Acsc", "Fixtures", filename); - if (File.Exists(primary)) - { - return primary; - } - - var secondary = Path.Combine(baseDir, "Fixtures", filename); - if (File.Exists(secondary)) - { - return secondary; - } - - var projectRelative = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Acsc", "Fixtures", filename); - if (File.Exists(projectRelative)) - { - return Path.GetFullPath(projectRelative); - } - - throw new FileNotFoundException($"Fixture '{filename}' not found.", filename); - } - - private static string GetWritableFixturePath(string filename) - => Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Acsc", "Fixtures", filename); - - private static string Normalize(string value) - => value.Replace("\r\n", "\n", StringComparison.Ordinal).Trim(); - - private static bool FixtureExists(string filename) - { - try - { - _ = GetExistingFixturePath(filename); - return true; - } - catch (FileNotFoundException) - { - return false; - } - } -} +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Acsc; +using StellaOps.Concelier.Connector.Acsc.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Acsc.Tests.Acsc; + +[Collection("mongo-fixture")] +public sealed class AcscConnectorParseTests : IAsyncLifetime +{ + private static readonly Uri BaseEndpoint = new("https://origin.example/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public AcscConnectorParseTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task ParseAsync_PersistsDtoAndAdvancesCursor() + { + await using var provider = await BuildProviderAsync(); + var connector = provider.GetRequiredService(); + + var feedUri = new Uri(BaseEndpoint, "/feeds/alerts/rss"); + SeedRssResponse(feedUri); + + await connector.FetchAsync(provider, CancellationToken.None); + _handler.AssertNoPendingResponses(); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(AcscConnectorPlugin.SourceName, feedUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + + await connector.ParseAsync(provider, CancellationToken.None); + + var refreshed = await documentStore.FindAsync(document!.Id, CancellationToken.None); + Assert.NotNull(refreshed); + Assert.Equal(DocumentStatuses.PendingMap, refreshed!.Status); + + var dtoStore = provider.GetRequiredService(); + var dtoRecord = await dtoStore.FindByDocumentIdAsync(document.Id, CancellationToken.None); + Assert.NotNull(dtoRecord); + Assert.Equal("acsc.feed.v1", dtoRecord!.SchemaVersion); + + var payload = dtoRecord.Payload; + Assert.NotNull(payload); + Assert.Equal("alerts", payload.GetValue("feedSlug").AsString); + Assert.Single(payload.GetValue("entries").AsBsonArray); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.DoesNotContain(document.Id.ToString(), state!.Cursor.GetValue("pendingDocuments").AsBsonArray.Select(v => v.AsString)); + Assert.Contains(document.Id.ToString(), state.Cursor.GetValue("pendingMappings").AsBsonArray.Select(v => v.AsString)); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoriesStore = provider.GetRequiredService(); + var advisories = await advisoriesStore.GetRecentAsync(10, CancellationToken.None); + Assert.Single(advisories); + + var ordered = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + WriteOrAssertSnapshot( + SnapshotSerializer.ToSnapshot(ordered), + "acsc-advisories.snapshot.json"); + + var mappedDocument = await documentStore.FindAsync(document.Id, CancellationToken.None); + Assert.NotNull(mappedDocument); + Assert.Equal(DocumentStatuses.Mapped, mappedDocument!.Status); + + state = await stateRepository.TryGetAsync(AcscConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.Cursor.GetValue("pendingMappings").AsBsonArray.Count == 0); + } + + [Fact] + public async Task MapAsync_MultiEntryFeedProducesExpectedSnapshot() + { + await using var provider = await BuildProviderAsync(options => + { + options.Feeds.Clear(); + options.Feeds.Add(new AcscFeedOptions + { + Slug = "multi", + RelativePath = "/feeds/multi/rss", + Enabled = true, + }); + }); + var connector = provider.GetRequiredService(); + + var feedUri = new Uri(BaseEndpoint, "/feeds/multi/rss"); + SeedMultiEntryResponse(feedUri); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var dtoStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(AcscConnectorPlugin.SourceName, feedUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + var dtoRecord = await dtoStore.FindByDocumentIdAsync(document!.Id, CancellationToken.None); + Assert.NotNull(dtoRecord); + var payload = dtoRecord!.Payload; + Assert.NotNull(payload); + var entries = payload.GetValue("entries").AsBsonArray; + Assert.Equal(2, entries.Count); + var fields = entries[0].AsBsonDocument.GetValue("fields").AsBsonDocument; + Assert.Equal("Critical", fields.GetValue("severity").AsString); + Assert.Equal("ExampleCo Router X, ExampleCo Router Y", fields.GetValue("systemsAffected").AsString); + + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var ordered = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + WriteOrAssertSnapshot( + SnapshotSerializer.ToSnapshot(ordered), + "acsc-advisories-multi.snapshot.json"); + + var affected = ordered.First(advisory => advisory.AffectedPackages.Any()); + Assert.Contains("ExampleCo Router X", affected.AffectedPackages[0].Identifier); + Assert.Equal("critical", ordered.First(a => a.Severity is not null).Severity, StringComparer.OrdinalIgnoreCase); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task BuildProviderAsync(Action? configure = null) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAcscConnector(options => + { + options.BaseEndpoint = BaseEndpoint; + options.RelayEndpoint = null; + options.PreferRelayByDefault = false; + options.ForceRelay = false; + options.EnableRelayFallback = false; + options.RequestTimeout = TimeSpan.FromSeconds(10); + options.Feeds.Clear(); + options.Feeds.Add(new AcscFeedOptions + { + Slug = "alerts", + RelativePath = "/feeds/alerts/rss", + Enabled = true, + }); + configure?.Invoke(options); + }); + + services.Configure(AcscOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + services.Configure(AcscOptions.HttpClientName, options => + { + options.MaxAttempts = 1; + options.BaseDelay = TimeSpan.Zero; + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedRssResponse(Uri uri) + { + const string payload = """ + + + + ACSC Alerts + https://origin.example/feeds/alerts + Sun, 12 Oct 2025 04:20:00 GMT + + ACSC-2025-001 Example Advisory + https://origin.example/advisories/example + https://origin.example/advisories/example + Sun, 12 Oct 2025 03:00:00 GMT + Serial number: ACSC-2025-001

+

Advisory type: Alert

+

First paragraph describing issue.

+

Second paragraph with Vendor patch.

+ ]]> + + + + """; + + _handler.AddResponse(HttpMethod.Get, uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"parse-etag\""); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 12, 4, 20, 0, TimeSpan.Zero); + return response; + }); + } + + private void SeedMultiEntryResponse(Uri uri) + { + const string payload = """ + + + + ACSC Advisories + https://origin.example/feeds/advisories + Sun, 12 Oct 2025 05:00:00 GMT + + Critical router vulnerability + https://origin.example/advisories/router-critical + https://origin.example/advisories/router-critical + Sun, 12 Oct 2025 04:45:00 GMT + Serial number: ACSC-2025-010

+

Severity: Critical

+

Systems affected: ExampleCo Router X, ExampleCo Router Y

+

Remote code execution on ExampleCo routers. See vendor patch.

+

CVE references: CVE-2025-0001

+ ]]> + + + Information bulletin + https://origin.example/advisories/info-bulletin + https://origin.example/advisories/info-bulletin + Sun, 12 Oct 2025 02:30:00 GMT + Serial number: ACSC-2025-011

+

Advisory type: Bulletin

+

General guidance bulletin.

+ ]]> + + + + """; + + _handler.AddResponse(HttpMethod.Get, uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/rss+xml"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"multi-etag\""); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 12, 5, 0, 0, TimeSpan.Zero); + return response; + }); + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + if (ShouldUpdateFixtures() || !FixtureExists(filename)) + { + var writable = GetWritableFixturePath(filename); + Directory.CreateDirectory(Path.GetDirectoryName(writable)!); + File.WriteAllText(writable, Normalize(snapshot)); + return; + } + + var expected = Normalize(File.ReadAllText(GetExistingFixturePath(filename))); + var actual = Normalize(snapshot); + + if (!string.Equals(expected, actual, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); + File.WriteAllText(actualPath, actual); + } + + Assert.Equal(expected, actual); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable("UPDATE_ACSC_FIXTURES"); + return string.Equals(value, "1", StringComparison.Ordinal) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string GetExistingFixturePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Acsc", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + var secondary = Path.Combine(baseDir, "Fixtures", filename); + if (File.Exists(secondary)) + { + return secondary; + } + + var projectRelative = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Acsc", "Fixtures", filename); + if (File.Exists(projectRelative)) + { + return Path.GetFullPath(projectRelative); + } + + throw new FileNotFoundException($"Fixture '{filename}' not found.", filename); + } + + private static string GetWritableFixturePath(string filename) + => Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Acsc", "Fixtures", filename); + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal).Trim(); + + private static bool FixtureExists(string filename) + { + try + { + _ = GetExistingFixturePath(filename); + return true; + } + catch (FileNotFoundException) + { + return false; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs rename to src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs index 45750968..52dc647b 100644 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/AcscHttpClientConfigurationTests.cs @@ -1,43 +1,43 @@ -using System.Net; -using System.Net.Http; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Acsc.Configuration; -using StellaOps.Feedser.Source.Common.Http; -using Xunit; - -namespace StellaOps.Feedser.Source.Acsc.Tests.Acsc; - -public sealed class AcscHttpClientConfigurationTests -{ - [Fact] - public void AddAcscConnector_ConfiguresHttpClientOptions() - { - var services = new ServiceCollection(); - services.AddAcscConnector(options => - { - options.BaseEndpoint = new Uri("https://origin.example/"); - options.RelayEndpoint = new Uri("https://relay.example/"); - options.RequestTimeout = TimeSpan.FromSeconds(42); - options.Feeds.Clear(); - options.Feeds.Add(new AcscFeedOptions - { - Slug = "alerts", - RelativePath = "/feeds/alerts/rss", - Enabled = true, - }); - }); - - var provider = services.BuildServiceProvider(); - var monitor = provider.GetRequiredService>(); - var options = monitor.Get(AcscOptions.HttpClientName); - - Assert.Equal("StellaOps/Feedser (+https://stella-ops.org)", options.UserAgent); - Assert.Equal(HttpVersion.Version20, options.RequestVersion); - Assert.Equal(HttpVersionPolicy.RequestVersionOrLower, options.VersionPolicy); - Assert.Equal(TimeSpan.FromSeconds(42), options.Timeout); - Assert.Contains("origin.example", options.AllowedHosts, StringComparer.OrdinalIgnoreCase); - Assert.Contains("relay.example", options.AllowedHosts, StringComparer.OrdinalIgnoreCase); - Assert.Equal("application/rss+xml, application/atom+xml;q=0.9, application/xml;q=0.8, text/xml;q=0.7", options.DefaultRequestHeaders["Accept"]); - } -} +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Acsc.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using Xunit; + +namespace StellaOps.Concelier.Connector.Acsc.Tests.Acsc; + +public sealed class AcscHttpClientConfigurationTests +{ + [Fact] + public void AddAcscConnector_ConfiguresHttpClientOptions() + { + var services = new ServiceCollection(); + services.AddAcscConnector(options => + { + options.BaseEndpoint = new Uri("https://origin.example/"); + options.RelayEndpoint = new Uri("https://relay.example/"); + options.RequestTimeout = TimeSpan.FromSeconds(42); + options.Feeds.Clear(); + options.Feeds.Add(new AcscFeedOptions + { + Slug = "alerts", + RelativePath = "/feeds/alerts/rss", + Enabled = true, + }); + }); + + var provider = services.BuildServiceProvider(); + var monitor = provider.GetRequiredService>(); + var options = monitor.Get(AcscOptions.HttpClientName); + + Assert.Equal("StellaOps/Concelier (+https://stella-ops.org)", options.UserAgent); + Assert.Equal(HttpVersion.Version20, options.RequestVersion); + Assert.Equal(HttpVersionPolicy.RequestVersionOrLower, options.VersionPolicy); + Assert.Equal(TimeSpan.FromSeconds(42), options.Timeout); + Assert.Contains("origin.example", options.AllowedHosts, StringComparer.OrdinalIgnoreCase); + Assert.Contains("relay.example", options.AllowedHosts, StringComparer.OrdinalIgnoreCase); + Assert.Equal("application/rss+xml, application/atom+xml;q=0.9, application/xml;q=0.8, text/xml;q=0.7", options.DefaultRequestHeaders["Accept"]); + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json rename to src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json index bba48d75..68cb04dd 100644 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories-multi.snapshot.json @@ -1,201 +1,201 @@ -[ - { - "advisoryKey": "acsc/multi/https-origin-example-advisories-info-bulletin", - "affectedPackages": [], - "aliases": [ - "ACSC-2025-011", - "Bulletin", - "https://origin.example/advisories/info-bulletin" - ], - "credits": [], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": null, - "provenance": [ - { - "source": "acsc", - "kind": "document", - "value": "https://origin.example/feeds/multi/rss", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - }, - { - "source": "acsc", - "kind": "feed", - "value": "multi", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "summary" - ] - }, - { - "source": "acsc", - "kind": "mapping", - "value": "https://origin.example/advisories/info-bulletin", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - } - ], - "published": "2025-10-12T02:30:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "acsc", - "kind": "reference", - "value": "https://origin.example/advisories/info-bulletin", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "multi", - "summary": "Information bulletin", - "url": "https://origin.example/advisories/info-bulletin" - } - ], - "severity": null, - "summary": "Serial number: ACSC-2025-011\n\nAdvisory type: Bulletin\n\nGeneral guidance bulletin.", - "title": "Information bulletin" - }, - { - "advisoryKey": "acsc/multi/https-origin-example-advisories-router-critical", - "affectedPackages": [ - { - "type": "vendor", - "identifier": "ExampleCo Router X", - "platform": null, - "versionRanges": [], - "normalizedVersions": [], - "statuses": [], - "provenance": [ - { - "source": "acsc", - "kind": "affected", - "value": "ExampleCo Router X", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages" - ] - } - ] - }, - { - "type": "vendor", - "identifier": "ExampleCo Router Y", - "platform": null, - "versionRanges": [], - "normalizedVersions": [], - "statuses": [], - "provenance": [ - { - "source": "acsc", - "kind": "affected", - "value": "ExampleCo Router Y", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages" - ] - } - ] - } - ], - "aliases": [ - "ACSC-2025-010", - "CVE-2025-0001", - "https://origin.example/advisories/router-critical" - ], - "credits": [], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": null, - "provenance": [ - { - "source": "acsc", - "kind": "document", - "value": "https://origin.example/feeds/multi/rss", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - }, - { - "source": "acsc", - "kind": "feed", - "value": "multi", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "summary" - ] - }, - { - "source": "acsc", - "kind": "mapping", - "value": "https://origin.example/advisories/router-critical", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - } - ], - "published": "2025-10-12T04:45:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "acsc", - "kind": "reference", - "value": "https://origin.example/advisories/router-critical", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "multi", - "summary": "Critical router vulnerability", - "url": "https://origin.example/advisories/router-critical" - }, - { - "kind": "reference", - "provenance": { - "source": "acsc", - "kind": "reference", - "value": "https://vendor.example/router/patch", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": null, - "summary": "vendor patch", - "url": "https://vendor.example/router/patch" - } - ], - "severity": "critical", - "summary": "Serial number: ACSC-2025-010\n\nSeverity: Critical\n\nSystems affected: ExampleCo Router X, ExampleCo Router Y\n\nRemote code execution on ExampleCo routers. See vendor patch.\n\nCVE references: CVE-2025-0001", - "title": "Critical router vulnerability" - } +[ + { + "advisoryKey": "acsc/multi/https-origin-example-advisories-info-bulletin", + "affectedPackages": [], + "aliases": [ + "ACSC-2025-011", + "Bulletin", + "https://origin.example/advisories/info-bulletin" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "source": "acsc", + "kind": "document", + "value": "https://origin.example/feeds/multi/rss", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + }, + { + "source": "acsc", + "kind": "feed", + "value": "multi", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "summary" + ] + }, + { + "source": "acsc", + "kind": "mapping", + "value": "https://origin.example/advisories/info-bulletin", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + } + ], + "published": "2025-10-12T02:30:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "acsc", + "kind": "reference", + "value": "https://origin.example/advisories/info-bulletin", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "multi", + "summary": "Information bulletin", + "url": "https://origin.example/advisories/info-bulletin" + } + ], + "severity": null, + "summary": "Serial number: ACSC-2025-011\n\nAdvisory type: Bulletin\n\nGeneral guidance bulletin.", + "title": "Information bulletin" + }, + { + "advisoryKey": "acsc/multi/https-origin-example-advisories-router-critical", + "affectedPackages": [ + { + "type": "vendor", + "identifier": "ExampleCo Router X", + "platform": null, + "versionRanges": [], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "acsc", + "kind": "affected", + "value": "ExampleCo Router X", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages" + ] + } + ] + }, + { + "type": "vendor", + "identifier": "ExampleCo Router Y", + "platform": null, + "versionRanges": [], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "acsc", + "kind": "affected", + "value": "ExampleCo Router Y", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages" + ] + } + ] + } + ], + "aliases": [ + "ACSC-2025-010", + "CVE-2025-0001", + "https://origin.example/advisories/router-critical" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "source": "acsc", + "kind": "document", + "value": "https://origin.example/feeds/multi/rss", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + }, + { + "source": "acsc", + "kind": "feed", + "value": "multi", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "summary" + ] + }, + { + "source": "acsc", + "kind": "mapping", + "value": "https://origin.example/advisories/router-critical", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + } + ], + "published": "2025-10-12T04:45:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "acsc", + "kind": "reference", + "value": "https://origin.example/advisories/router-critical", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "multi", + "summary": "Critical router vulnerability", + "url": "https://origin.example/advisories/router-critical" + }, + { + "kind": "reference", + "provenance": { + "source": "acsc", + "kind": "reference", + "value": "https://vendor.example/router/patch", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "vendor patch", + "url": "https://vendor.example/router/patch" + } + ], + "severity": "critical", + "summary": "Serial number: ACSC-2025-010\n\nSeverity: Critical\n\nSystems affected: ExampleCo Router X, ExampleCo Router Y\n\nRemote code execution on ExampleCo routers. See vendor patch.\n\nCVE references: CVE-2025-0001", + "title": "Critical router vulnerability" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json index cd348048..b33b9dd0 100644 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/Acsc/Fixtures/acsc-advisories.snapshot.json @@ -1,88 +1,88 @@ -[ - { - "advisoryKey": "acsc/alerts/https-origin-example-advisories-example", - "affectedPackages": [], - "aliases": [ - "ACSC-2025-001", - "Alert", - "https://origin.example/advisories/example" - ], - "credits": [], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": null, - "provenance": [ - { - "source": "acsc", - "kind": "document", - "value": "https://origin.example/feeds/alerts/rss", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - }, - { - "source": "acsc", - "kind": "feed", - "value": "alerts", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "summary" - ] - }, - { - "source": "acsc", - "kind": "mapping", - "value": "https://origin.example/advisories/example", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [ - "affectedpackages", - "aliases", - "references", - "summary" - ] - } - ], - "published": "2025-10-12T03:00:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "acsc", - "kind": "reference", - "value": "https://origin.example/advisories/example", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "alerts", - "summary": "ACSC-2025-001 Example Advisory", - "url": "https://origin.example/advisories/example" - }, - { - "kind": "reference", - "provenance": { - "source": "acsc", - "kind": "reference", - "value": "https://vendor.example/patch", - "decisionReason": null, - "recordedAt": "2025-10-12T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": null, - "summary": "Vendor patch", - "url": "https://vendor.example/patch" - } - ], - "severity": null, - "summary": "Serial number: ACSC-2025-001\n\nAdvisory type: Alert\n\nFirst paragraph describing issue.\n\nSecond paragraph with Vendor patch.", - "title": "ACSC-2025-001 Example Advisory" - } +[ + { + "advisoryKey": "acsc/alerts/https-origin-example-advisories-example", + "affectedPackages": [], + "aliases": [ + "ACSC-2025-001", + "Alert", + "https://origin.example/advisories/example" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": null, + "provenance": [ + { + "source": "acsc", + "kind": "document", + "value": "https://origin.example/feeds/alerts/rss", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + }, + { + "source": "acsc", + "kind": "feed", + "value": "alerts", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "summary" + ] + }, + { + "source": "acsc", + "kind": "mapping", + "value": "https://origin.example/advisories/example", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [ + "affectedpackages", + "aliases", + "references", + "summary" + ] + } + ], + "published": "2025-10-12T03:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "acsc", + "kind": "reference", + "value": "https://origin.example/advisories/example", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "alerts", + "summary": "ACSC-2025-001 Example Advisory", + "url": "https://origin.example/advisories/example" + }, + { + "kind": "reference", + "provenance": { + "source": "acsc", + "kind": "reference", + "value": "https://vendor.example/patch", + "decisionReason": null, + "recordedAt": "2025-10-12T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "Vendor patch", + "url": "https://vendor.example/patch" + } + ], + "severity": null, + "summary": "Serial number: ACSC-2025-001\n\nAdvisory type: Alert\n\nFirst paragraph describing issue.\n\nSecond paragraph with Vendor patch.", + "title": "ACSC-2025-001 Example Advisory" + } ] \ No newline at end of file diff --git a/src/StellaOps.Concelier.Connector.Acsc.Tests/StellaOps.Concelier.Connector.Acsc.Tests.csproj b/src/StellaOps.Concelier.Connector.Acsc.Tests/StellaOps.Concelier.Connector.Acsc.Tests.csproj new file mode 100644 index 00000000..d843b146 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Acsc.Tests/StellaOps.Concelier.Connector.Acsc.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Acsc/AGENTS.md b/src/StellaOps.Concelier.Connector.Acsc/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.Acsc/AGENTS.md rename to src/StellaOps.Concelier.Connector.Acsc/AGENTS.md index c090d238..257b7a98 100644 --- a/src/StellaOps.Feedser.Source.Acsc/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Acsc/AGENTS.md @@ -1,40 +1,40 @@ -# AGENTS -## Role -Bootstrap the ACSC (Australian Cyber Security Centre) advisories connector so the Feedser pipeline can ingest, normalise, and enrich ACSC security bulletins. - -## Scope -- Research the authoritative ACSC advisory feed (RSS/Atom, JSON API, or HTML). -- Implement fetch windowing, cursor persistence, and retry strategy consistent with other external connectors. -- Parse advisory content (summary, affected products, mitigation guidance, references). -- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. -- Provide deterministic fixtures and regression tests that cover fetch/parse/map flows. - -## Participants -- `Source.Common` for HTTP client creation, fetch service, and DTO persistence helpers. -- `Storage.Mongo` for raw/document/DTO/advisory storage plus cursor management. -- `Feedser.Models` for canonical advisory structures and provenance utilities. -- `Feedser.Testing` for integration harnesses and snapshot helpers. - -## Interfaces & Contracts -- Job kinds should follow the pattern `acsc:fetch`, `acsc:parse`, `acsc:map`. -- Documents persisted to Mongo must include ETag/Last-Modified metadata when the source exposes it. -- Canonical advisories must emit aliases (ACSC ID + CVE IDs) and references (official bulletin + vendor notices). - -## In/Out of scope -In scope: -- Initial end-to-end connector implementation with tests, fixtures, and range primitive coverage. -- Minimal telemetry (logging + diagnostics counters) consistent with other connectors. - -Out of scope: -- Upstream remediation automation or vendor-specific enrichment beyond ACSC data. -- Export-related changes (handled by exporter teams). - -## Observability & Security Expectations -- Log key lifecycle events (fetch/page processed, parse success/error counts, mapping stats). -- Sanitise HTML safely and avoid persisting external scripts or embedded media. -- Handle transient fetch failures gracefully with exponential backoff and mark failures in source state. - -## Tests -- Add integration-style tests under `StellaOps.Feedser.Source.Acsc.Tests` covering fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; provide UPDATE flag flow for regeneration. -- Validate determinism (ordering, casing, timestamps) to satisfy pipeline reproducibility requirements. +# AGENTS +## Role +Bootstrap the ACSC (Australian Cyber Security Centre) advisories connector so the Concelier pipeline can ingest, normalise, and enrich ACSC security bulletins. + +## Scope +- Research the authoritative ACSC advisory feed (RSS/Atom, JSON API, or HTML). +- Implement fetch windowing, cursor persistence, and retry strategy consistent with other external connectors. +- Parse advisory content (summary, affected products, mitigation guidance, references). +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. +- Provide deterministic fixtures and regression tests that cover fetch/parse/map flows. + +## Participants +- `Source.Common` for HTTP client creation, fetch service, and DTO persistence helpers. +- `Storage.Mongo` for raw/document/DTO/advisory storage plus cursor management. +- `Concelier.Models` for canonical advisory structures and provenance utilities. +- `Concelier.Testing` for integration harnesses and snapshot helpers. + +## Interfaces & Contracts +- Job kinds should follow the pattern `acsc:fetch`, `acsc:parse`, `acsc:map`. +- Documents persisted to Mongo must include ETag/Last-Modified metadata when the source exposes it. +- Canonical advisories must emit aliases (ACSC ID + CVE IDs) and references (official bulletin + vendor notices). + +## In/Out of scope +In scope: +- Initial end-to-end connector implementation with tests, fixtures, and range primitive coverage. +- Minimal telemetry (logging + diagnostics counters) consistent with other connectors. + +Out of scope: +- Upstream remediation automation or vendor-specific enrichment beyond ACSC data. +- Export-related changes (handled by exporter teams). + +## Observability & Security Expectations +- Log key lifecycle events (fetch/page processed, parse success/error counts, mapping stats). +- Sanitise HTML safely and avoid persisting external scripts or embedded media. +- Handle transient fetch failures gracefully with exponential backoff and mark failures in source state. + +## Tests +- Add integration-style tests under `StellaOps.Concelier.Connector.Acsc.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; provide UPDATE flag flow for regeneration. +- Validate determinism (ordering, casing, timestamps) to satisfy pipeline reproducibility requirements. diff --git a/src/StellaOps.Feedser.Source.Acsc/AcscConnector.cs b/src/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Acsc/AcscConnector.cs rename to src/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs index 59b61f6f..a82ef18f 100644 --- a/src/StellaOps.Feedser.Source.Acsc/AcscConnector.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/AcscConnector.cs @@ -1,699 +1,699 @@ -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Xml.Linq; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Bson.IO; -using StellaOps.Feedser.Source.Acsc.Configuration; -using StellaOps.Feedser.Source.Acsc.Internal; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Acsc; - -public sealed class AcscConnector : IFeedConnector -{ - private static readonly string[] AcceptHeaders = - { - "application/rss+xml", - "application/atom+xml;q=0.9", - "application/xml;q=0.8", - "text/xml;q=0.7", - }; - - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - WriteIndented = false, - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly IHttpClientFactory _httpClientFactory; - private readonly AcscOptions _options; - private readonly AcscDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly HtmlContentSanitizer _htmlSanitizer = new(); - - public AcscConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IHttpClientFactory httpClientFactory, - IOptions options, - AcscDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => AcscConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var now = _timeProvider.GetUtcNow(); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - - var lastPublished = new Dictionary(cursor.LastPublishedByFeed, StringComparer.OrdinalIgnoreCase); - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var failures = new List<(AcscFeedOptions Feed, Exception Error)>(); - - var preferredEndpoint = ResolveInitialPreference(cursor); - AcscEndpointPreference? successPreference = null; - - foreach (var feed in GetEnabledFeeds()) - { - cancellationToken.ThrowIfCancellationRequested(); - - Exception? lastError = null; - bool handled = false; - - foreach (var mode in BuildFetchOrder(preferredEndpoint)) - { - cancellationToken.ThrowIfCancellationRequested(); - if (mode == AcscFetchMode.Relay && !IsRelayConfigured) - { - continue; - } - - var modeName = ModeName(mode); - var targetUri = BuildFeedUri(feed, mode); - - var metadata = CreateMetadata(feed, cursor, modeName); - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, targetUri.ToString(), cancellationToken).ConfigureAwait(false); - - var request = new SourceFetchRequest(AcscOptions.HttpClientName, SourceName, targetUri) - { - Metadata = metadata, - ETag = existing?.Etag, - LastModified = existing?.LastModified, - AcceptHeaders = AcceptHeaders, - TimeoutOverride = _options.RequestTimeout, - }; - - try - { - _diagnostics.FetchAttempt(feed.Slug, modeName); - var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); - - if (result.IsNotModified) - { - _diagnostics.FetchUnchanged(feed.Slug, modeName); - successPreference ??= mode switch - { - AcscFetchMode.Relay => AcscEndpointPreference.Relay, - _ => AcscEndpointPreference.Direct, - }; - handled = true; - _logger.LogDebug("ACSC feed {Feed} returned 304 via {Mode}", feed.Slug, modeName); - break; - } - - if (!result.IsSuccess || result.Document is null) - { - _diagnostics.FetchFailure(feed.Slug, modeName); - lastError = new InvalidOperationException($"Fetch returned no document for {targetUri}"); - continue; - } - - pendingDocuments.Add(result.Document.Id); - successPreference = mode switch - { - AcscFetchMode.Relay => AcscEndpointPreference.Relay, - _ => AcscEndpointPreference.Direct, - }; - handled = true; - _diagnostics.FetchSuccess(feed.Slug, modeName); - _logger.LogInformation("ACSC fetched {Feed} via {Mode} (documentId={DocumentId})", feed.Slug, modeName, result.Document.Id); - - var latestPublished = await TryComputeLatestPublishedAsync(result.Document, cancellationToken).ConfigureAwait(false); - if (latestPublished.HasValue) - { - if (!lastPublished.TryGetValue(feed.Slug, out var existingPublished) || latestPublished.Value > existingPublished) - { - lastPublished[feed.Slug] = latestPublished.Value; - _diagnostics.CursorUpdated(feed.Slug); - _logger.LogDebug("ACSC feed {Feed} advanced published cursor to {Timestamp:O}", feed.Slug, latestPublished.Value); - } - } - - break; - } - catch (HttpRequestException ex) when (ShouldRetryWithRelay(mode)) - { - lastError = ex; - _diagnostics.FetchFallback(feed.Slug, modeName, "http-request"); - _logger.LogWarning(ex, "ACSC fetch via {Mode} failed for {Feed}; attempting relay fallback.", modeName, feed.Slug); - continue; - } - catch (TaskCanceledException ex) when (ShouldRetryWithRelay(mode)) - { - lastError = ex; - _diagnostics.FetchFallback(feed.Slug, modeName, "timeout"); - _logger.LogWarning(ex, "ACSC fetch via {Mode} timed out for {Feed}; attempting relay fallback.", modeName, feed.Slug); - continue; - } - catch (Exception ex) - { - lastError = ex; - _diagnostics.FetchFailure(feed.Slug, modeName); - _logger.LogError(ex, "ACSC fetch failed for {Feed} via {Mode}", feed.Slug, modeName); - break; - } - } - - if (!handled && lastError is not null) - { - failures.Add((feed, lastError)); - } - } - - if (failures.Count > 0) - { - var failureReason = string.Join("; ", failures.Select(f => $"{f.Feed.Slug}: {f.Error.Message}")); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, failureReason, cancellationToken).ConfigureAwait(false); - throw new AggregateException($"ACSC fetch failed for {failures.Count} feed(s): {failureReason}", failures.Select(f => f.Error)); - } - - var updatedPreference = successPreference ?? preferredEndpoint; - if (_options.ForceRelay) - { - updatedPreference = AcscEndpointPreference.Relay; - } - else if (!IsRelayConfigured) - { - updatedPreference = AcscEndpointPreference.Direct; - } - - var updatedCursor = cursor - .WithPreferredEndpoint(updatedPreference) - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastPublished(lastPublished); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var metadata = AcscDocumentMetadata.FromDocument(document); - var feedTag = string.IsNullOrWhiteSpace(metadata.FeedSlug) ? "(unknown)" : metadata.FeedSlug; - - _diagnostics.ParseAttempt(feedTag); - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(feedTag, "missingPayload"); - _logger.LogWarning("ACSC document {DocumentId} missing GridFS payload (feed={Feed})", document.Id, feedTag); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - byte[] rawBytes; - try - { - rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(feedTag, "download"); - _logger.LogError(ex, "ACSC failed to download payload for document {DocumentId} (feed={Feed})", document.Id, feedTag); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - try - { - var parsedAt = _timeProvider.GetUtcNow(); - var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer); - - var json = JsonSerializer.Serialize(dto, SerializerOptions); - var payload = BsonDocument.Parse(json); - - var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); - var dtoRecord = existingDto is null - ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "acsc.feed.v1", payload, parsedAt) - : existingDto with - { - Payload = payload, - SchemaVersion = "acsc.feed.v1", - ValidatedAt = parsedAt, - }; - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - pendingDocuments.Remove(documentId); - pendingMappings.Add(document.Id); - - _diagnostics.ParseSuccess(feedTag); - _logger.LogInformation("ACSC parsed document {DocumentId} (feed={Feed}, entries={EntryCount})", document.Id, feedTag, dto.Entries.Count); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(feedTag, "parse"); - _logger.LogError(ex, "ACSC parse failed for document {DocumentId} (feed={Feed})", document.Id, feedTag); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var documentIds = cursor.PendingMappings.ToList(); - - foreach (var documentId in documentIds) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - - if (dtoRecord is null || document is null) - { - pendingMappings.Remove(documentId); - continue; - } - - AcscFeedDto? feed; - try - { - var dtoJson = dtoRecord.Payload.ToJson(new JsonWriterSettings - { - OutputMode = JsonOutputMode.RelaxedExtendedJson, - }); - - feed = JsonSerializer.Deserialize(dtoJson, SerializerOptions); - } - catch (Exception ex) - { - _logger.LogError(ex, "ACSC mapping failed to deserialize DTO for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - if (feed is null) - { - _logger.LogWarning("ACSC mapping encountered null DTO payload for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - var mappedAt = _timeProvider.GetUtcNow(); - var advisories = AcscMapper.Map(feed, document, dtoRecord, SourceName, mappedAt); - - if (advisories.Count > 0) - { - foreach (var advisory in advisories) - { - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - } - - _diagnostics.MapSuccess(advisories.Count); - _logger.LogInformation( - "ACSC mapped {Count} advisories from document {DocumentId} (feed={Feed})", - advisories.Count, - document.Id, - feed.FeedSlug ?? "(unknown)"); - } - else - { - _logger.LogInformation( - "ACSC mapping produced no advisories for document {DocumentId} (feed={Feed})", - document.Id, - feed.FeedSlug ?? "(unknown)"); - } - - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ProbeAsync(CancellationToken cancellationToken) - { - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - - if (_options.ForceRelay) - { - if (cursor.PreferredEndpoint != AcscEndpointPreference.Relay) - { - await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Relay), cancellationToken).ConfigureAwait(false); - } - return; - } - - if (!IsRelayConfigured) - { - if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) - { - await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); - } - return; - } - - var feed = GetEnabledFeeds().FirstOrDefault(); - if (feed is null) - { - return; - } - - var httpClient = _httpClientFactory.CreateClient(AcscOptions.HttpClientName); - httpClient.Timeout = TimeSpan.FromSeconds(15); - - var directUri = BuildFeedUri(feed, AcscFetchMode.Direct); - - try - { - using var headRequest = new HttpRequestMessage(HttpMethod.Head, directUri); - using var response = await httpClient.SendAsync(headRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) - { - await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); - _logger.LogInformation("ACSC probe succeeded via direct endpoint ({StatusCode}); relay preference cleared.", (int)response.StatusCode); - } - return; - } - - if (response.StatusCode == HttpStatusCode.MethodNotAllowed) - { - using var probeRequest = new HttpRequestMessage(HttpMethod.Get, directUri); - using var probeResponse = await httpClient.SendAsync(probeRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - if (probeResponse.IsSuccessStatusCode) - { - if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) - { - await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); - _logger.LogInformation("ACSC probe succeeded via direct endpoint after GET fallback ({StatusCode}).", (int)probeResponse.StatusCode); - } - return; - } - } - - _logger.LogWarning("ACSC direct probe returned HTTP {StatusCode}; relay preference enabled.", (int)response.StatusCode); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "ACSC direct probe failed; relay preference will be enabled."); - } - - if (cursor.PreferredEndpoint != AcscEndpointPreference.Relay) - { - await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Relay), cancellationToken).ConfigureAwait(false); - } - } - - private bool ShouldRetryWithRelay(AcscFetchMode mode) - => mode == AcscFetchMode.Direct && _options.EnableRelayFallback && IsRelayConfigured && !_options.ForceRelay; - - private IEnumerable BuildFetchOrder(AcscEndpointPreference preference) - { - if (_options.ForceRelay) - { - if (IsRelayConfigured) - { - yield return AcscFetchMode.Relay; - } - yield break; - } - - if (!IsRelayConfigured) - { - yield return AcscFetchMode.Direct; - yield break; - } - - var preferRelay = preference == AcscEndpointPreference.Relay; - if (preference == AcscEndpointPreference.Auto) - { - preferRelay = _options.PreferRelayByDefault; - } - - if (preferRelay) - { - yield return AcscFetchMode.Relay; - if (_options.EnableRelayFallback) - { - yield return AcscFetchMode.Direct; - } - } - else - { - yield return AcscFetchMode.Direct; - if (_options.EnableRelayFallback) - { - yield return AcscFetchMode.Relay; - } - } - } - - private AcscEndpointPreference ResolveInitialPreference(AcscCursor cursor) - { - if (_options.ForceRelay) - { - return AcscEndpointPreference.Relay; - } - - if (!IsRelayConfigured) - { - return AcscEndpointPreference.Direct; - } - - if (cursor.PreferredEndpoint != AcscEndpointPreference.Auto) - { - return cursor.PreferredEndpoint; - } - - return _options.PreferRelayByDefault ? AcscEndpointPreference.Relay : AcscEndpointPreference.Direct; - } - - private async Task TryComputeLatestPublishedAsync(DocumentRecord document, CancellationToken cancellationToken) - { - if (!document.GridFsId.HasValue) - { - return null; - } - - var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - if (rawBytes.Length == 0) - { - return null; - } - - try - { - using var memoryStream = new MemoryStream(rawBytes, writable: false); - var xml = XDocument.Load(memoryStream, LoadOptions.None); - - DateTimeOffset? latest = null; - foreach (var element in xml.Descendants()) - { - if (!IsEntryElement(element.Name.LocalName)) - { - continue; - } - - var published = ExtractPublished(element); - if (!published.HasValue) - { - continue; - } - - if (latest is null || published.Value > latest.Value) - { - latest = published; - } - } - - return latest; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "ACSC failed to derive published cursor for document {DocumentId} ({Uri})", document.Id, document.Uri); - return null; - } - } - - private static bool IsEntryElement(string localName) - => string.Equals(localName, "item", StringComparison.OrdinalIgnoreCase) - || string.Equals(localName, "entry", StringComparison.OrdinalIgnoreCase); - - private static DateTimeOffset? ExtractPublished(XElement element) - { - foreach (var name in EnumerateTimestampNames(element)) - { - if (DateTimeOffset.TryParse( - name.Value, - CultureInfo.InvariantCulture, - DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal, - out var parsed)) - { - return parsed.ToUniversalTime(); - } - } - - return null; - } - - private static IEnumerable EnumerateTimestampNames(XElement element) - { - foreach (var child in element.Elements()) - { - var localName = child.Name.LocalName; - if (string.Equals(localName, "pubDate", StringComparison.OrdinalIgnoreCase) || - string.Equals(localName, "published", StringComparison.OrdinalIgnoreCase) || - string.Equals(localName, "updated", StringComparison.OrdinalIgnoreCase) || - string.Equals(localName, "date", StringComparison.OrdinalIgnoreCase)) - { - yield return child; - } - } - } - - private Dictionary CreateMetadata(AcscFeedOptions feed, AcscCursor cursor, string mode) - { - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["acsc.feed.slug"] = feed.Slug, - ["acsc.fetch.mode"] = mode, - }; - - if (cursor.LastPublishedByFeed.TryGetValue(feed.Slug, out var published) && published.HasValue) - { - metadata["acsc.cursor.lastPublished"] = published.Value.ToString("O"); - } - - return metadata; - } - - private Uri BuildFeedUri(AcscFeedOptions feed, AcscFetchMode mode) - { - var baseUri = mode switch - { - AcscFetchMode.Relay when IsRelayConfigured => _options.RelayEndpoint!, - _ => _options.BaseEndpoint, - }; - - return new Uri(baseUri, feed.RelativePath); - } - - private IEnumerable GetEnabledFeeds() - => _options.Feeds.Where(feed => feed is { Enabled: true }); - - private Task GetCursorAsync(CancellationToken cancellationToken) - => GetCursorCoreAsync(cancellationToken); - - private async Task GetCursorCoreAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? AcscCursor.Empty : AcscCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBsonDocument(); - var completedAt = _timeProvider.GetUtcNow(); - return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); - } - - private bool IsRelayConfigured => _options.RelayEndpoint is not null; - - private static string ModeName(AcscFetchMode mode) => mode switch - { - AcscFetchMode.Relay => "relay", - _ => "direct", - }; - - private enum AcscFetchMode - { - Direct = 0, - Relay = 1, - } -} +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Concelier.Connector.Acsc.Configuration; +using StellaOps.Concelier.Connector.Acsc.Internal; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Acsc; + +public sealed class AcscConnector : IFeedConnector +{ + private static readonly string[] AcceptHeaders = + { + "application/rss+xml", + "application/atom+xml;q=0.9", + "application/xml;q=0.8", + "text/xml;q=0.7", + }; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IHttpClientFactory _httpClientFactory; + private readonly AcscOptions _options; + private readonly AcscDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly HtmlContentSanitizer _htmlSanitizer = new(); + + public AcscConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IHttpClientFactory httpClientFactory, + IOptions options, + AcscDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => AcscConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + var lastPublished = new Dictionary(cursor.LastPublishedByFeed, StringComparer.OrdinalIgnoreCase); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var failures = new List<(AcscFeedOptions Feed, Exception Error)>(); + + var preferredEndpoint = ResolveInitialPreference(cursor); + AcscEndpointPreference? successPreference = null; + + foreach (var feed in GetEnabledFeeds()) + { + cancellationToken.ThrowIfCancellationRequested(); + + Exception? lastError = null; + bool handled = false; + + foreach (var mode in BuildFetchOrder(preferredEndpoint)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (mode == AcscFetchMode.Relay && !IsRelayConfigured) + { + continue; + } + + var modeName = ModeName(mode); + var targetUri = BuildFeedUri(feed, mode); + + var metadata = CreateMetadata(feed, cursor, modeName); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, targetUri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(AcscOptions.HttpClientName, SourceName, targetUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = AcceptHeaders, + TimeoutOverride = _options.RequestTimeout, + }; + + try + { + _diagnostics.FetchAttempt(feed.Slug, modeName); + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + + if (result.IsNotModified) + { + _diagnostics.FetchUnchanged(feed.Slug, modeName); + successPreference ??= mode switch + { + AcscFetchMode.Relay => AcscEndpointPreference.Relay, + _ => AcscEndpointPreference.Direct, + }; + handled = true; + _logger.LogDebug("ACSC feed {Feed} returned 304 via {Mode}", feed.Slug, modeName); + break; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.FetchFailure(feed.Slug, modeName); + lastError = new InvalidOperationException($"Fetch returned no document for {targetUri}"); + continue; + } + + pendingDocuments.Add(result.Document.Id); + successPreference = mode switch + { + AcscFetchMode.Relay => AcscEndpointPreference.Relay, + _ => AcscEndpointPreference.Direct, + }; + handled = true; + _diagnostics.FetchSuccess(feed.Slug, modeName); + _logger.LogInformation("ACSC fetched {Feed} via {Mode} (documentId={DocumentId})", feed.Slug, modeName, result.Document.Id); + + var latestPublished = await TryComputeLatestPublishedAsync(result.Document, cancellationToken).ConfigureAwait(false); + if (latestPublished.HasValue) + { + if (!lastPublished.TryGetValue(feed.Slug, out var existingPublished) || latestPublished.Value > existingPublished) + { + lastPublished[feed.Slug] = latestPublished.Value; + _diagnostics.CursorUpdated(feed.Slug); + _logger.LogDebug("ACSC feed {Feed} advanced published cursor to {Timestamp:O}", feed.Slug, latestPublished.Value); + } + } + + break; + } + catch (HttpRequestException ex) when (ShouldRetryWithRelay(mode)) + { + lastError = ex; + _diagnostics.FetchFallback(feed.Slug, modeName, "http-request"); + _logger.LogWarning(ex, "ACSC fetch via {Mode} failed for {Feed}; attempting relay fallback.", modeName, feed.Slug); + continue; + } + catch (TaskCanceledException ex) when (ShouldRetryWithRelay(mode)) + { + lastError = ex; + _diagnostics.FetchFallback(feed.Slug, modeName, "timeout"); + _logger.LogWarning(ex, "ACSC fetch via {Mode} timed out for {Feed}; attempting relay fallback.", modeName, feed.Slug); + continue; + } + catch (Exception ex) + { + lastError = ex; + _diagnostics.FetchFailure(feed.Slug, modeName); + _logger.LogError(ex, "ACSC fetch failed for {Feed} via {Mode}", feed.Slug, modeName); + break; + } + } + + if (!handled && lastError is not null) + { + failures.Add((feed, lastError)); + } + } + + if (failures.Count > 0) + { + var failureReason = string.Join("; ", failures.Select(f => $"{f.Feed.Slug}: {f.Error.Message}")); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, failureReason, cancellationToken).ConfigureAwait(false); + throw new AggregateException($"ACSC fetch failed for {failures.Count} feed(s): {failureReason}", failures.Select(f => f.Error)); + } + + var updatedPreference = successPreference ?? preferredEndpoint; + if (_options.ForceRelay) + { + updatedPreference = AcscEndpointPreference.Relay; + } + else if (!IsRelayConfigured) + { + updatedPreference = AcscEndpointPreference.Direct; + } + + var updatedCursor = cursor + .WithPreferredEndpoint(updatedPreference) + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(lastPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var metadata = AcscDocumentMetadata.FromDocument(document); + var feedTag = string.IsNullOrWhiteSpace(metadata.FeedSlug) ? "(unknown)" : metadata.FeedSlug; + + _diagnostics.ParseAttempt(feedTag); + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(feedTag, "missingPayload"); + _logger.LogWarning("ACSC document {DocumentId} missing GridFS payload (feed={Feed})", document.Id, feedTag); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(feedTag, "download"); + _logger.LogError(ex, "ACSC failed to download payload for document {DocumentId} (feed={Feed})", document.Id, feedTag); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + try + { + var parsedAt = _timeProvider.GetUtcNow(); + var dto = AcscFeedParser.Parse(rawBytes, metadata.FeedSlug, parsedAt, _htmlSanitizer); + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "acsc.feed.v1", payload, parsedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "acsc.feed.v1", + ValidatedAt = parsedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + + _diagnostics.ParseSuccess(feedTag); + _logger.LogInformation("ACSC parsed document {DocumentId} (feed={Feed}, entries={EntryCount})", document.Id, feedTag, dto.Entries.Count); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(feedTag, "parse"); + _logger.LogError(ex, "ACSC parse failed for document {DocumentId} (feed={Feed})", document.Id, feedTag); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var documentIds = cursor.PendingMappings.ToList(); + + foreach (var documentId in documentIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + AcscFeedDto? feed; + try + { + var dtoJson = dtoRecord.Payload.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + }); + + feed = JsonSerializer.Deserialize(dtoJson, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "ACSC mapping failed to deserialize DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (feed is null) + { + _logger.LogWarning("ACSC mapping encountered null DTO payload for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var mappedAt = _timeProvider.GetUtcNow(); + var advisories = AcscMapper.Map(feed, document, dtoRecord, SourceName, mappedAt); + + if (advisories.Count > 0) + { + foreach (var advisory in advisories) + { + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + } + + _diagnostics.MapSuccess(advisories.Count); + _logger.LogInformation( + "ACSC mapped {Count} advisories from document {DocumentId} (feed={Feed})", + advisories.Count, + document.Id, + feed.FeedSlug ?? "(unknown)"); + } + else + { + _logger.LogInformation( + "ACSC mapping produced no advisories for document {DocumentId} (feed={Feed})", + document.Id, + feed.FeedSlug ?? "(unknown)"); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ProbeAsync(CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + if (_options.ForceRelay) + { + if (cursor.PreferredEndpoint != AcscEndpointPreference.Relay) + { + await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Relay), cancellationToken).ConfigureAwait(false); + } + return; + } + + if (!IsRelayConfigured) + { + if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) + { + await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); + } + return; + } + + var feed = GetEnabledFeeds().FirstOrDefault(); + if (feed is null) + { + return; + } + + var httpClient = _httpClientFactory.CreateClient(AcscOptions.HttpClientName); + httpClient.Timeout = TimeSpan.FromSeconds(15); + + var directUri = BuildFeedUri(feed, AcscFetchMode.Direct); + + try + { + using var headRequest = new HttpRequestMessage(HttpMethod.Head, directUri); + using var response = await httpClient.SendAsync(headRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) + { + await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); + _logger.LogInformation("ACSC probe succeeded via direct endpoint ({StatusCode}); relay preference cleared.", (int)response.StatusCode); + } + return; + } + + if (response.StatusCode == HttpStatusCode.MethodNotAllowed) + { + using var probeRequest = new HttpRequestMessage(HttpMethod.Get, directUri); + using var probeResponse = await httpClient.SendAsync(probeRequest, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (probeResponse.IsSuccessStatusCode) + { + if (cursor.PreferredEndpoint != AcscEndpointPreference.Direct) + { + await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Direct), cancellationToken).ConfigureAwait(false); + _logger.LogInformation("ACSC probe succeeded via direct endpoint after GET fallback ({StatusCode}).", (int)probeResponse.StatusCode); + } + return; + } + } + + _logger.LogWarning("ACSC direct probe returned HTTP {StatusCode}; relay preference enabled.", (int)response.StatusCode); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "ACSC direct probe failed; relay preference will be enabled."); + } + + if (cursor.PreferredEndpoint != AcscEndpointPreference.Relay) + { + await UpdateCursorAsync(cursor.WithPreferredEndpoint(AcscEndpointPreference.Relay), cancellationToken).ConfigureAwait(false); + } + } + + private bool ShouldRetryWithRelay(AcscFetchMode mode) + => mode == AcscFetchMode.Direct && _options.EnableRelayFallback && IsRelayConfigured && !_options.ForceRelay; + + private IEnumerable BuildFetchOrder(AcscEndpointPreference preference) + { + if (_options.ForceRelay) + { + if (IsRelayConfigured) + { + yield return AcscFetchMode.Relay; + } + yield break; + } + + if (!IsRelayConfigured) + { + yield return AcscFetchMode.Direct; + yield break; + } + + var preferRelay = preference == AcscEndpointPreference.Relay; + if (preference == AcscEndpointPreference.Auto) + { + preferRelay = _options.PreferRelayByDefault; + } + + if (preferRelay) + { + yield return AcscFetchMode.Relay; + if (_options.EnableRelayFallback) + { + yield return AcscFetchMode.Direct; + } + } + else + { + yield return AcscFetchMode.Direct; + if (_options.EnableRelayFallback) + { + yield return AcscFetchMode.Relay; + } + } + } + + private AcscEndpointPreference ResolveInitialPreference(AcscCursor cursor) + { + if (_options.ForceRelay) + { + return AcscEndpointPreference.Relay; + } + + if (!IsRelayConfigured) + { + return AcscEndpointPreference.Direct; + } + + if (cursor.PreferredEndpoint != AcscEndpointPreference.Auto) + { + return cursor.PreferredEndpoint; + } + + return _options.PreferRelayByDefault ? AcscEndpointPreference.Relay : AcscEndpointPreference.Direct; + } + + private async Task TryComputeLatestPublishedAsync(DocumentRecord document, CancellationToken cancellationToken) + { + if (!document.GridFsId.HasValue) + { + return null; + } + + var rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + if (rawBytes.Length == 0) + { + return null; + } + + try + { + using var memoryStream = new MemoryStream(rawBytes, writable: false); + var xml = XDocument.Load(memoryStream, LoadOptions.None); + + DateTimeOffset? latest = null; + foreach (var element in xml.Descendants()) + { + if (!IsEntryElement(element.Name.LocalName)) + { + continue; + } + + var published = ExtractPublished(element); + if (!published.HasValue) + { + continue; + } + + if (latest is null || published.Value > latest.Value) + { + latest = published; + } + } + + return latest; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "ACSC failed to derive published cursor for document {DocumentId} ({Uri})", document.Id, document.Uri); + return null; + } + } + + private static bool IsEntryElement(string localName) + => string.Equals(localName, "item", StringComparison.OrdinalIgnoreCase) + || string.Equals(localName, "entry", StringComparison.OrdinalIgnoreCase); + + private static DateTimeOffset? ExtractPublished(XElement element) + { + foreach (var name in EnumerateTimestampNames(element)) + { + if (DateTimeOffset.TryParse( + name.Value, + CultureInfo.InvariantCulture, + DateTimeStyles.AllowWhiteSpaces | DateTimeStyles.AssumeUniversal, + out var parsed)) + { + return parsed.ToUniversalTime(); + } + } + + return null; + } + + private static IEnumerable EnumerateTimestampNames(XElement element) + { + foreach (var child in element.Elements()) + { + var localName = child.Name.LocalName; + if (string.Equals(localName, "pubDate", StringComparison.OrdinalIgnoreCase) || + string.Equals(localName, "published", StringComparison.OrdinalIgnoreCase) || + string.Equals(localName, "updated", StringComparison.OrdinalIgnoreCase) || + string.Equals(localName, "date", StringComparison.OrdinalIgnoreCase)) + { + yield return child; + } + } + } + + private Dictionary CreateMetadata(AcscFeedOptions feed, AcscCursor cursor, string mode) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["acsc.feed.slug"] = feed.Slug, + ["acsc.fetch.mode"] = mode, + }; + + if (cursor.LastPublishedByFeed.TryGetValue(feed.Slug, out var published) && published.HasValue) + { + metadata["acsc.cursor.lastPublished"] = published.Value.ToString("O"); + } + + return metadata; + } + + private Uri BuildFeedUri(AcscFeedOptions feed, AcscFetchMode mode) + { + var baseUri = mode switch + { + AcscFetchMode.Relay when IsRelayConfigured => _options.RelayEndpoint!, + _ => _options.BaseEndpoint, + }; + + return new Uri(baseUri, feed.RelativePath); + } + + private IEnumerable GetEnabledFeeds() + => _options.Feeds.Where(feed => feed is { Enabled: true }); + + private Task GetCursorAsync(CancellationToken cancellationToken) + => GetCursorCoreAsync(cancellationToken); + + private async Task GetCursorCoreAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? AcscCursor.Empty : AcscCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(AcscCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var completedAt = _timeProvider.GetUtcNow(); + return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); + } + + private bool IsRelayConfigured => _options.RelayEndpoint is not null; + + private static string ModeName(AcscFetchMode mode) => mode switch + { + AcscFetchMode.Relay => "relay", + _ => "direct", + }; + + private enum AcscFetchMode + { + Direct = 0, + Relay = 1, + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/AcscConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Acsc/AcscConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs index 7eb57d24..8d43f1bb 100644 --- a/src/StellaOps.Feedser.Source.Acsc/AcscConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/AcscConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Acsc; - -public sealed class AcscConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "acsc"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Acsc; + +public sealed class AcscConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "acsc"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/AcscDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Acsc/AcscDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs index bb99dfb5..20211c12 100644 --- a/src/StellaOps.Feedser.Source.Acsc/AcscDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/AcscDependencyInjectionRoutine.cs @@ -1,44 +1,44 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Acsc.Configuration; - -namespace StellaOps.Feedser.Source.Acsc; - -public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:acsc"; - - private const string FetchCron = "7,37 * * * *"; - private const string ParseCron = "12,42 * * * *"; - private const string MapCron = "17,47 * * * *"; - private const string ProbeCron = "25,55 * * * *"; - - private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4); - private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3); - private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3); - private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1); - private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3); - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddAcscConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - var scheduler = new JobSchedulerBuilder(services); - scheduler - .AddJob(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration) - .AddJob(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration) - .AddJob(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration) - .AddJob(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration); - - return services; - } -} +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Acsc.Configuration; + +namespace StellaOps.Concelier.Connector.Acsc; + +public sealed class AcscDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:acsc"; + + private const string FetchCron = "7,37 * * * *"; + private const string ParseCron = "12,42 * * * *"; + private const string MapCron = "17,47 * * * *"; + private const string ProbeCron = "25,55 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(4); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(3); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(3); + private static readonly TimeSpan ProbeTimeout = TimeSpan.FromMinutes(1); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(3); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddAcscConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob(AcscJobKinds.Fetch, FetchCron, FetchTimeout, LeaseDuration) + .AddJob(AcscJobKinds.Parse, ParseCron, ParseTimeout, LeaseDuration) + .AddJob(AcscJobKinds.Map, MapCron, MapTimeout, LeaseDuration) + .AddJob(AcscJobKinds.Probe, ProbeCron, ProbeTimeout, LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/AcscServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Acsc/AcscServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs index 048e1d28..7e6ef895 100644 --- a/src/StellaOps.Feedser.Source.Acsc/AcscServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/AcscServiceCollectionExtensions.cs @@ -1,56 +1,56 @@ -using System.Net; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Acsc.Configuration; -using StellaOps.Feedser.Source.Acsc.Internal; -using StellaOps.Feedser.Source.Common.Http; - -namespace StellaOps.Feedser.Source.Acsc; - -public static class AcscServiceCollectionExtensions -{ - public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); - - services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.Timeout = options.RequestTimeout; - clientOptions.UserAgent = options.UserAgent; - clientOptions.RequestVersion = options.RequestVersion; - clientOptions.VersionPolicy = options.VersionPolicy; - clientOptions.AllowAutoRedirect = true; - clientOptions.ConfigureHandler = handler => - { - handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; - handler.AllowAutoRedirect = true; - }; - - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); - if (options.RelayEndpoint is not null) - { - clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host); - } - - clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[] - { - "application/rss+xml", - "application/atom+xml;q=0.9", - "application/xml;q=0.8", - "text/xml;q=0.7", - }); - }); - - services.AddSingleton(); - services.AddTransient(); - - return services; - } -} +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Acsc.Configuration; +using StellaOps.Concelier.Connector.Acsc.Internal; +using StellaOps.Concelier.Connector.Common.Http; + +namespace StellaOps.Concelier.Connector.Acsc; + +public static class AcscServiceCollectionExtensions +{ + public static IServiceCollection AddAcscConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(AcscOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = options.RequestTimeout; + clientOptions.UserAgent = options.UserAgent; + clientOptions.RequestVersion = options.RequestVersion; + clientOptions.VersionPolicy = options.VersionPolicy; + clientOptions.AllowAutoRedirect = true; + clientOptions.ConfigureHandler = handler => + { + handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; + handler.AllowAutoRedirect = true; + }; + + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + if (options.RelayEndpoint is not null) + { + clientOptions.AllowedHosts.Add(options.RelayEndpoint.Host); + } + + clientOptions.DefaultRequestHeaders["Accept"] = string.Join(", ", new[] + { + "application/rss+xml", + "application/atom+xml;q=0.9", + "application/xml;q=0.8", + "text/xml;q=0.7", + }); + }); + + services.AddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Configuration/AcscFeedOptions.cs b/src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscFeedOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Acsc/Configuration/AcscFeedOptions.cs rename to src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscFeedOptions.cs index f44e7380..0f9f84b1 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Configuration/AcscFeedOptions.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscFeedOptions.cs @@ -1,54 +1,54 @@ -using System.Text.RegularExpressions; - -namespace StellaOps.Feedser.Source.Acsc.Configuration; - -/// -/// Defines a single ACSC RSS feed endpoint. -/// -public sealed class AcscFeedOptions -{ - private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - - /// - /// Logical slug for the feed (alerts, advisories, threats, etc.). - /// - public string Slug { get; set; } = "alerts"; - - /// - /// Relative path (under ) for the RSS feed. - /// - public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss"; - - /// - /// Indicates whether the feed is active. - /// - public bool Enabled { get; set; } = true; - - /// - /// Optional display name for logging. - /// - public string? DisplayName { get; set; } - - internal void Validate(int index) - { - if (string.IsNullOrWhiteSpace(Slug)) - { - throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug."); - } - - if (!SlugPattern.IsMatch(Slug)) - { - throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators."); - } - - if (string.IsNullOrWhiteSpace(RelativePath)) - { - throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path."); - } - - if (!RelativePath.StartsWith("/", StringComparison.Ordinal)) - { - throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}')."); - } - } -} +using System.Text.RegularExpressions; + +namespace StellaOps.Concelier.Connector.Acsc.Configuration; + +/// +/// Defines a single ACSC RSS feed endpoint. +/// +public sealed class AcscFeedOptions +{ + private static readonly Regex SlugPattern = new("^[a-z0-9][a-z0-9\\-]*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + /// + /// Logical slug for the feed (alerts, advisories, threats, etc.). + /// + public string Slug { get; set; } = "alerts"; + + /// + /// Relative path (under ) for the RSS feed. + /// + public string RelativePath { get; set; } = "/acsc/view-all-content/alerts/rss"; + + /// + /// Indicates whether the feed is active. + /// + public bool Enabled { get; set; } = true; + + /// + /// Optional display name for logging. + /// + public string? DisplayName { get; set; } + + internal void Validate(int index) + { + if (string.IsNullOrWhiteSpace(Slug)) + { + throw new InvalidOperationException($"ACSC feed entry #{index} must define a slug."); + } + + if (!SlugPattern.IsMatch(Slug)) + { + throw new InvalidOperationException($"ACSC feed slug '{Slug}' is invalid. Slugs must be lower-case alphanumeric with optional hyphen separators."); + } + + if (string.IsNullOrWhiteSpace(RelativePath)) + { + throw new InvalidOperationException($"ACSC feed '{Slug}' must specify a relative path."); + } + + if (!RelativePath.StartsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"ACSC feed '{Slug}' relative path must begin with '/' (value: '{RelativePath}')."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Configuration/AcscOptions.cs b/src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Acsc/Configuration/AcscOptions.cs rename to src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs index d6ea8ba9..4492f02d 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Configuration/AcscOptions.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Configuration/AcscOptions.cs @@ -1,153 +1,153 @@ -using System.Net; -using System.Net.Http; - -namespace StellaOps.Feedser.Source.Acsc.Configuration; - -/// -/// Connector options governing ACSC feed access and retry behaviour. -/// -public sealed class AcscOptions -{ - public const string HttpClientName = "acsc"; - - private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45); - private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5); - private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120); - - public AcscOptions() - { - Feeds = new List - { - new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" }, - new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" }, - new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false }, - new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false }, - new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false }, - }; - } - - /// - /// Base endpoint for direct ACSC fetches. - /// - public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute); - - /// - /// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections. - /// - public Uri? RelayEndpoint { get; set; } - - /// - /// Default mode when no preference has been captured in connector state. When true, the relay will be preferred for initial fetches. - /// - public bool PreferRelayByDefault { get; set; } - - /// - /// If enabled, the connector may switch to the relay endpoint when direct fetches fail. - /// - public bool EnableRelayFallback { get; set; } = true; - - /// - /// If set, the connector will always use the relay endpoint and skip direct attempts. - /// - public bool ForceRelay { get; set; } - - /// - /// Timeout applied to fetch requests (overrides HttpClient default). - /// - public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; - - /// - /// Backoff applied when marking fetch failures. - /// - public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; - - /// - /// Look-back period used when deriving initial published cursors. - /// - public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill; - - /// - /// User-agent header sent with outbound requests. - /// - public string UserAgent { get; set; } = "StellaOps/Feedser (+https://stella-ops.org)"; - - /// - /// RSS feeds requested during fetch. - /// - public IList Feeds { get; } - - /// - /// HTTP version policy requested for outbound requests. - /// - public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower; - - /// - /// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade). - /// - public Version RequestVersion { get; set; } = HttpVersion.Version20; - - public void Validate() - { - if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI."); - } - - if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) - { - throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash."); - } - - if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified."); - } - - if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) - { - throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("ACSC RequestTimeout must be positive."); - } - - if (FailureBackoff < TimeSpan.Zero) - { - throw new InvalidOperationException("ACSC FailureBackoff cannot be negative."); - } - - if (InitialBackfill <= TimeSpan.Zero) - { - throw new InvalidOperationException("ACSC InitialBackfill must be positive."); - } - - if (string.IsNullOrWhiteSpace(UserAgent)) - { - throw new InvalidOperationException("ACSC UserAgent cannot be empty."); - } - - if (Feeds.Count == 0) - { - throw new InvalidOperationException("At least one ACSC feed must be configured."); - } - - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - for (var i = 0; i < Feeds.Count; i++) - { - var feed = Feeds[i]; - feed.Validate(i); - - if (!feed.Enabled) - { - continue; - } - - if (!seen.Add(feed.Slug)) - { - throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive)."); - } - } - } -} +using System.Net; +using System.Net.Http; + +namespace StellaOps.Concelier.Connector.Acsc.Configuration; + +/// +/// Connector options governing ACSC feed access and retry behaviour. +/// +public sealed class AcscOptions +{ + public const string HttpClientName = "acsc"; + + private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(45); + private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(5); + private static readonly TimeSpan DefaultInitialBackfill = TimeSpan.FromDays(120); + + public AcscOptions() + { + Feeds = new List + { + new() { Slug = "alerts", RelativePath = "/acsc/view-all-content/alerts/rss" }, + new() { Slug = "advisories", RelativePath = "/acsc/view-all-content/advisories/rss" }, + new() { Slug = "news", RelativePath = "/acsc/view-all-content/news/rss", Enabled = false }, + new() { Slug = "publications", RelativePath = "/acsc/view-all-content/publications/rss", Enabled = false }, + new() { Slug = "threats", RelativePath = "/acsc/view-all-content/threats/rss", Enabled = false }, + }; + } + + /// + /// Base endpoint for direct ACSC fetches. + /// + public Uri BaseEndpoint { get; set; } = new("https://www.cyber.gov.au/", UriKind.Absolute); + + /// + /// Optional relay endpoint used when Akamai terminates direct HTTP/2 connections. + /// + public Uri? RelayEndpoint { get; set; } + + /// + /// Default mode when no preference has been captured in connector state. When true, the relay will be preferred for initial fetches. + /// + public bool PreferRelayByDefault { get; set; } + + /// + /// If enabled, the connector may switch to the relay endpoint when direct fetches fail. + /// + public bool EnableRelayFallback { get; set; } = true; + + /// + /// If set, the connector will always use the relay endpoint and skip direct attempts. + /// + public bool ForceRelay { get; set; } + + /// + /// Timeout applied to fetch requests (overrides HttpClient default). + /// + public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; + + /// + /// Backoff applied when marking fetch failures. + /// + public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; + + /// + /// Look-back period used when deriving initial published cursors. + /// + public TimeSpan InitialBackfill { get; set; } = DefaultInitialBackfill; + + /// + /// User-agent header sent with outbound requests. + /// + public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)"; + + /// + /// RSS feeds requested during fetch. + /// + public IList Feeds { get; } + + /// + /// HTTP version policy requested for outbound requests. + /// + public HttpVersionPolicy VersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower; + + /// + /// Default HTTP version requested when connecting to ACSC (defaults to HTTP/2 but allows downgrade). + /// + public Version RequestVersion { get; set; } = HttpVersion.Version20; + + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("ACSC BaseEndpoint must be an absolute URI."); + } + + if (!BaseEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("ACSC BaseEndpoint must include a trailing slash."); + } + + if (RelayEndpoint is not null && !RelayEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("ACSC RelayEndpoint must be an absolute URI when specified."); + } + + if (RelayEndpoint is not null && !RelayEndpoint.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("ACSC RelayEndpoint must include a trailing slash when specified."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("ACSC RequestTimeout must be positive."); + } + + if (FailureBackoff < TimeSpan.Zero) + { + throw new InvalidOperationException("ACSC FailureBackoff cannot be negative."); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + throw new InvalidOperationException("ACSC InitialBackfill must be positive."); + } + + if (string.IsNullOrWhiteSpace(UserAgent)) + { + throw new InvalidOperationException("ACSC UserAgent cannot be empty."); + } + + if (Feeds.Count == 0) + { + throw new InvalidOperationException("At least one ACSC feed must be configured."); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < Feeds.Count; i++) + { + var feed = Feeds[i]; + feed.Validate(i); + + if (!feed.Enabled) + { + continue; + } + + if (!seen.Add(feed.Slug)) + { + throw new InvalidOperationException($"Duplicate ACSC feed slug '{feed.Slug}' detected. Slugs must be unique (case-insensitive)."); + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscCursor.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscCursor.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscCursor.cs index 29f3d196..c01f1113 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscCursor.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscCursor.cs @@ -1,141 +1,141 @@ -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -internal enum AcscEndpointPreference -{ - Auto = 0, - Direct = 1, - Relay = 2, -} - -internal sealed record AcscCursor( - AcscEndpointPreference PreferredEndpoint, - IReadOnlyDictionary LastPublishedByFeed, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); - private static readonly IReadOnlyDictionary EmptyFeedDictionary = - new Dictionary(StringComparer.OrdinalIgnoreCase); - - public static AcscCursor Empty { get; } = new( - AcscEndpointPreference.Auto, - EmptyFeedDictionary, - EmptyGuidList, - EmptyGuidList); - - public AcscCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList }; - - public AcscCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList }; - - public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference) - => this with { PreferredEndpoint = preference }; - - public AcscCursor WithLastPublished(IDictionary values) - { - var snapshot = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (values is not null) - { - foreach (var kvp in values) - { - snapshot[kvp.Key] = kvp.Value; - } - } - - return this with { LastPublishedByFeed = snapshot }; - } - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["preferredEndpoint"] = PreferredEndpoint.ToString(), - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - var feedsDocument = new BsonDocument(); - foreach (var kvp in LastPublishedByFeed) - { - if (kvp.Value.HasValue) - { - feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime; - } - } - - document["feeds"] = feedsDocument; - return document; - } - - public static AcscCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue) - ? ParseEndpointPreference(endpointValue.AsString) - : AcscEndpointPreference.Auto; - - var feeds = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument) - { - foreach (var element in feedsDocument.Elements) - { - feeds[element.Name] = ParseDate(element.Value); - } - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new AcscCursor( - preferredEndpoint, - feeds, - pendingDocuments, - pendingMappings); - } - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuidList; - } - - var list = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var guid)) - { - list.Add(guid); - } - } - - return list; - } - - private static DateTimeOffset? ParseDate(BsonValue value) - { - return value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - private static AcscEndpointPreference ParseEndpointPreference(string? value) - { - if (Enum.TryParse(value, ignoreCase: true, out var parsed)) - { - return parsed; - } - - return AcscEndpointPreference.Auto; - } -} +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +internal enum AcscEndpointPreference +{ + Auto = 0, + Direct = 1, + Relay = 2, +} + +internal sealed record AcscCursor( + AcscEndpointPreference PreferredEndpoint, + IReadOnlyDictionary LastPublishedByFeed, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidList = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyFeedDictionary = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + public static AcscCursor Empty { get; } = new( + AcscEndpointPreference.Auto, + EmptyFeedDictionary, + EmptyGuidList, + EmptyGuidList); + + public AcscCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidList }; + + public AcscCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidList }; + + public AcscCursor WithPreferredEndpoint(AcscEndpointPreference preference) + => this with { PreferredEndpoint = preference }; + + public AcscCursor WithLastPublished(IDictionary values) + { + var snapshot = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (values is not null) + { + foreach (var kvp in values) + { + snapshot[kvp.Key] = kvp.Value; + } + } + + return this with { LastPublishedByFeed = snapshot }; + } + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["preferredEndpoint"] = PreferredEndpoint.ToString(), + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + var feedsDocument = new BsonDocument(); + foreach (var kvp in LastPublishedByFeed) + { + if (kvp.Value.HasValue) + { + feedsDocument[kvp.Key] = kvp.Value.Value.UtcDateTime; + } + } + + document["feeds"] = feedsDocument; + return document; + } + + public static AcscCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var preferredEndpoint = document.TryGetValue("preferredEndpoint", out var endpointValue) + ? ParseEndpointPreference(endpointValue.AsString) + : AcscEndpointPreference.Auto; + + var feeds = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (document.TryGetValue("feeds", out var feedsValue) && feedsValue is BsonDocument feedsDocument) + { + foreach (var element in feedsDocument.Elements) + { + feeds[element.Name] = ParseDate(element.Value); + } + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new AcscCursor( + preferredEndpoint, + feeds, + pendingDocuments, + pendingMappings); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var list = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var guid)) + { + list.Add(guid); + } + } + + return list; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static AcscEndpointPreference ParseEndpointPreference(string? value) + { + if (Enum.TryParse(value, ignoreCase: true, out var parsed)) + { + return parsed; + } + + return AcscEndpointPreference.Auto; + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDiagnostics.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDiagnostics.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDiagnostics.cs index e5bea7b2..644a66a8 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDiagnostics.cs @@ -1,97 +1,97 @@ -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -public sealed class AcscDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.Acsc"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _fetchAttempts; - private readonly Counter _fetchSuccess; - private readonly Counter _fetchFailures; - private readonly Counter _fetchUnchanged; - private readonly Counter _fetchFallbacks; - private readonly Counter _cursorUpdates; - private readonly Counter _parseAttempts; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Counter _mapSuccess; - - public AcscDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchAttempts = _meter.CreateCounter("acsc.fetch.attempts", unit: "operations"); - _fetchSuccess = _meter.CreateCounter("acsc.fetch.success", unit: "operations"); - _fetchFailures = _meter.CreateCounter("acsc.fetch.failures", unit: "operations"); - _fetchUnchanged = _meter.CreateCounter("acsc.fetch.unchanged", unit: "operations"); - _fetchFallbacks = _meter.CreateCounter("acsc.fetch.fallbacks", unit: "operations"); - _cursorUpdates = _meter.CreateCounter("acsc.cursor.published_updates", unit: "feeds"); - _parseAttempts = _meter.CreateCounter("acsc.parse.attempts", unit: "documents"); - _parseSuccess = _meter.CreateCounter("acsc.parse.success", unit: "documents"); - _parseFailures = _meter.CreateCounter("acsc.parse.failures", unit: "documents"); - _mapSuccess = _meter.CreateCounter("acsc.map.success", unit: "advisories"); - } - - public void FetchAttempt(string feed, string mode) - => _fetchAttempts.Add(1, GetTags(feed, mode)); - - public void FetchSuccess(string feed, string mode) - => _fetchSuccess.Add(1, GetTags(feed, mode)); - - public void FetchFailure(string feed, string mode) - => _fetchFailures.Add(1, GetTags(feed, mode)); - - public void FetchUnchanged(string feed, string mode) - => _fetchUnchanged.Add(1, GetTags(feed, mode)); - - public void FetchFallback(string feed, string mode, string reason) - => _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair("reason", reason))); - - public void CursorUpdated(string feed) - => _cursorUpdates.Add(1, new KeyValuePair("feed", feed)); - - public void ParseAttempt(string feed) - => _parseAttempts.Add(1, new KeyValuePair("feed", feed)); - - public void ParseSuccess(string feed) - => _parseSuccess.Add(1, new KeyValuePair("feed", feed)); - - public void ParseFailure(string feed, string reason) - => _parseFailures.Add(1, new KeyValuePair[] - { - new("feed", feed), - new("reason", reason), - }); - - public void MapSuccess(int advisoryCount) - { - if (advisoryCount <= 0) - { - return; - } - - _mapSuccess.Add(advisoryCount); - } - - private static KeyValuePair[] GetTags(string feed, string mode) - => new[] - { - new KeyValuePair("feed", feed), - new KeyValuePair("mode", mode), - }; - - private static KeyValuePair[] GetTags(string feed, string mode, KeyValuePair extra) - => new[] - { - new KeyValuePair("feed", feed), - new KeyValuePair("mode", mode), - extra, - }; - - public void Dispose() - { - _meter.Dispose(); - } -} +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +public sealed class AcscDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.Acsc"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchSuccess; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _fetchFallbacks; + private readonly Counter _cursorUpdates; + private readonly Counter _parseAttempts; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _mapSuccess; + + public AcscDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter("acsc.fetch.attempts", unit: "operations"); + _fetchSuccess = _meter.CreateCounter("acsc.fetch.success", unit: "operations"); + _fetchFailures = _meter.CreateCounter("acsc.fetch.failures", unit: "operations"); + _fetchUnchanged = _meter.CreateCounter("acsc.fetch.unchanged", unit: "operations"); + _fetchFallbacks = _meter.CreateCounter("acsc.fetch.fallbacks", unit: "operations"); + _cursorUpdates = _meter.CreateCounter("acsc.cursor.published_updates", unit: "feeds"); + _parseAttempts = _meter.CreateCounter("acsc.parse.attempts", unit: "documents"); + _parseSuccess = _meter.CreateCounter("acsc.parse.success", unit: "documents"); + _parseFailures = _meter.CreateCounter("acsc.parse.failures", unit: "documents"); + _mapSuccess = _meter.CreateCounter("acsc.map.success", unit: "advisories"); + } + + public void FetchAttempt(string feed, string mode) + => _fetchAttempts.Add(1, GetTags(feed, mode)); + + public void FetchSuccess(string feed, string mode) + => _fetchSuccess.Add(1, GetTags(feed, mode)); + + public void FetchFailure(string feed, string mode) + => _fetchFailures.Add(1, GetTags(feed, mode)); + + public void FetchUnchanged(string feed, string mode) + => _fetchUnchanged.Add(1, GetTags(feed, mode)); + + public void FetchFallback(string feed, string mode, string reason) + => _fetchFallbacks.Add(1, GetTags(feed, mode, new KeyValuePair("reason", reason))); + + public void CursorUpdated(string feed) + => _cursorUpdates.Add(1, new KeyValuePair("feed", feed)); + + public void ParseAttempt(string feed) + => _parseAttempts.Add(1, new KeyValuePair("feed", feed)); + + public void ParseSuccess(string feed) + => _parseSuccess.Add(1, new KeyValuePair("feed", feed)); + + public void ParseFailure(string feed, string reason) + => _parseFailures.Add(1, new KeyValuePair[] + { + new("feed", feed), + new("reason", reason), + }); + + public void MapSuccess(int advisoryCount) + { + if (advisoryCount <= 0) + { + return; + } + + _mapSuccess.Add(advisoryCount); + } + + private static KeyValuePair[] GetTags(string feed, string mode) + => new[] + { + new KeyValuePair("feed", feed), + new KeyValuePair("mode", mode), + }; + + private static KeyValuePair[] GetTags(string feed, string mode, KeyValuePair extra) + => new[] + { + new KeyValuePair("feed", feed), + new KeyValuePair("mode", mode), + extra, + }; + + public void Dispose() + { + _meter.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDocumentMetadata.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDocumentMetadata.cs index bfb12acb..757acd53 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDocumentMetadata.cs @@ -1,20 +1,20 @@ -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode) -{ - public static AcscDocumentMetadata FromDocument(DocumentRecord document) - { - if (document.Metadata is null) - { - return new AcscDocumentMetadata(string.Empty, string.Empty); - } - - document.Metadata.TryGetValue("acsc.feed.slug", out var slug); - document.Metadata.TryGetValue("acsc.fetch.mode", out var mode); - return new AcscDocumentMetadata( - string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(), - string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim()); - } -} +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +internal readonly record struct AcscDocumentMetadata(string FeedSlug, string FetchMode) +{ + public static AcscDocumentMetadata FromDocument(DocumentRecord document) + { + if (document.Metadata is null) + { + return new AcscDocumentMetadata(string.Empty, string.Empty); + } + + document.Metadata.TryGetValue("acsc.feed.slug", out var slug); + document.Metadata.TryGetValue("acsc.fetch.mode", out var mode); + return new AcscDocumentMetadata( + string.IsNullOrWhiteSpace(slug) ? string.Empty : slug.Trim(), + string.IsNullOrWhiteSpace(mode) ? string.Empty : mode.Trim()); + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDto.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDto.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscDto.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDto.cs index 7ad11068..0df201d6 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscDto.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscDto.cs @@ -1,58 +1,58 @@ -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -internal sealed record AcscFeedDto( - [property: JsonPropertyName("feedSlug")] string FeedSlug, - [property: JsonPropertyName("feedTitle")] string? FeedTitle, - [property: JsonPropertyName("feedLink")] string? FeedLink, - [property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated, - [property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt, - [property: JsonPropertyName("entries")] IReadOnlyList Entries) -{ - public static AcscFeedDto Empty { get; } = new( - FeedSlug: string.Empty, - FeedTitle: null, - FeedLink: null, - FeedUpdated: null, - ParsedAt: DateTimeOffset.UnixEpoch, - Entries: Array.Empty()); -} - -internal sealed record AcscEntryDto( - [property: JsonPropertyName("entryId")] string EntryId, - [property: JsonPropertyName("title")] string Title, - [property: JsonPropertyName("link")] string? Link, - [property: JsonPropertyName("feedSlug")] string FeedSlug, - [property: JsonPropertyName("published")] DateTimeOffset? Published, - [property: JsonPropertyName("updated")] DateTimeOffset? Updated, - [property: JsonPropertyName("summary")] string Summary, - [property: JsonPropertyName("contentHtml")] string ContentHtml, - [property: JsonPropertyName("contentText")] string ContentText, - [property: JsonPropertyName("references")] IReadOnlyList References, - [property: JsonPropertyName("aliases")] IReadOnlyList Aliases, - [property: JsonPropertyName("fields")] IReadOnlyDictionary Fields) -{ - public static AcscEntryDto Empty { get; } = new( - EntryId: string.Empty, - Title: string.Empty, - Link: null, - FeedSlug: string.Empty, - Published: null, - Updated: null, - Summary: string.Empty, - ContentHtml: string.Empty, - ContentText: string.Empty, - References: Array.Empty(), - Aliases: Array.Empty(), - Fields: new Dictionary(StringComparer.OrdinalIgnoreCase)); -} - -internal sealed record AcscReferenceDto( - [property: JsonPropertyName("title")] string Title, - [property: JsonPropertyName("url")] string Url) -{ - public static AcscReferenceDto Empty { get; } = new( - Title: string.Empty, - Url: string.Empty); -} +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +internal sealed record AcscFeedDto( + [property: JsonPropertyName("feedSlug")] string FeedSlug, + [property: JsonPropertyName("feedTitle")] string? FeedTitle, + [property: JsonPropertyName("feedLink")] string? FeedLink, + [property: JsonPropertyName("feedUpdated")] DateTimeOffset? FeedUpdated, + [property: JsonPropertyName("parsedAt")] DateTimeOffset ParsedAt, + [property: JsonPropertyName("entries")] IReadOnlyList Entries) +{ + public static AcscFeedDto Empty { get; } = new( + FeedSlug: string.Empty, + FeedTitle: null, + FeedLink: null, + FeedUpdated: null, + ParsedAt: DateTimeOffset.UnixEpoch, + Entries: Array.Empty()); +} + +internal sealed record AcscEntryDto( + [property: JsonPropertyName("entryId")] string EntryId, + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("link")] string? Link, + [property: JsonPropertyName("feedSlug")] string FeedSlug, + [property: JsonPropertyName("published")] DateTimeOffset? Published, + [property: JsonPropertyName("updated")] DateTimeOffset? Updated, + [property: JsonPropertyName("summary")] string Summary, + [property: JsonPropertyName("contentHtml")] string ContentHtml, + [property: JsonPropertyName("contentText")] string ContentText, + [property: JsonPropertyName("references")] IReadOnlyList References, + [property: JsonPropertyName("aliases")] IReadOnlyList Aliases, + [property: JsonPropertyName("fields")] IReadOnlyDictionary Fields) +{ + public static AcscEntryDto Empty { get; } = new( + EntryId: string.Empty, + Title: string.Empty, + Link: null, + FeedSlug: string.Empty, + Published: null, + Updated: null, + Summary: string.Empty, + ContentHtml: string.Empty, + ContentText: string.Empty, + References: Array.Empty(), + Aliases: Array.Empty(), + Fields: new Dictionary(StringComparer.OrdinalIgnoreCase)); +} + +internal sealed record AcscReferenceDto( + [property: JsonPropertyName("title")] string Title, + [property: JsonPropertyName("url")] string Url) +{ + public static AcscReferenceDto Empty { get; } = new( + Title: string.Empty, + Url: string.Empty); +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscFeedParser.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscFeedParser.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs index e813907e..dc188c82 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscFeedParser.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscFeedParser.cs @@ -1,594 +1,594 @@ -using System.Globalization; -using System.Text; -using System.Xml.Linq; -using AngleSharp.Dom; -using AngleSharp.Html.Parser; -using System.Security.Cryptography; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -internal static class AcscFeedParser -{ - private static readonly XNamespace AtomNamespace = "http://www.w3.org/2005/Atom"; - private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/"; - public static AcscFeedDto Parse(byte[] payload, string feedSlug, DateTimeOffset parsedAt, HtmlContentSanitizer sanitizer) - { - ArgumentNullException.ThrowIfNull(payload); - ArgumentNullException.ThrowIfNull(sanitizer); - - if (payload.Length == 0) - { - return AcscFeedDto.Empty with - { - FeedSlug = feedSlug ?? string.Empty, - ParsedAt = parsedAt, - Entries = Array.Empty(), - }; - } - - var xml = XDocument.Parse(Encoding.UTF8.GetString(payload)); - - var (feedTitle, feedLink, feedUpdated) = ExtractFeedMetadata(xml); - var items = ExtractEntries(xml).ToArray(); - - var entries = new List(items.Length); - foreach (var item in items) - { - var entryId = ExtractEntryId(item); - if (string.IsNullOrWhiteSpace(entryId)) - { - // Fall back to hash of title + link to avoid duplicates. - entryId = GenerateFallbackId(item); - } - - var title = ExtractTitle(item); - var link = ExtractLink(item); - var published = ExtractDate(item, "pubDate") ?? ExtractAtomDate(item, "published") ?? ExtractDcDate(item); - var updated = ExtractAtomDate(item, "updated"); - - var rawHtml = ExtractContent(item); - var baseUri = TryCreateUri(link); - var sanitizedHtml = sanitizer.Sanitize(rawHtml, baseUri); - var htmlFragment = ParseHtmlFragment(sanitizedHtml); - - var summary = BuildSummary(htmlFragment) ?? string.Empty; - var contentText = NormalizeWhitespace(htmlFragment?.TextContent ?? string.Empty); - - var references = ExtractReferences(htmlFragment); - var fields = ExtractFields(htmlFragment, out var serialNumber, out var advisoryType); - var aliases = BuildAliases(serialNumber, advisoryType); - - var entry = new AcscEntryDto( - EntryId: entryId, - Title: title, - Link: link, - FeedSlug: feedSlug ?? string.Empty, - Published: published, - Updated: updated, - Summary: summary, - ContentHtml: sanitizedHtml, - ContentText: contentText, - References: references, - Aliases: aliases, - Fields: fields); - - entries.Add(entry); - } - - return new AcscFeedDto( - FeedSlug: feedSlug ?? string.Empty, - FeedTitle: feedTitle, - FeedLink: feedLink, - FeedUpdated: feedUpdated, - ParsedAt: parsedAt, - Entries: entries); - } - - private static (string? Title, string? Link, DateTimeOffset? Updated) ExtractFeedMetadata(XDocument xml) - { - var root = xml.Root; - if (root is null) - { - return (null, null, null); - } - - if (string.Equals(root.Name.LocalName, "rss", StringComparison.OrdinalIgnoreCase)) - { - var channel = root.Element("channel"); - var title = channel?.Element("title")?.Value?.Trim(); - var link = channel?.Element("link")?.Value?.Trim(); - var updated = TryParseDate(channel?.Element("lastBuildDate")?.Value); - return (title, link, updated); - } - - if (root.Name == AtomNamespace + "feed") - { - var title = root.Element(AtomNamespace + "title")?.Value?.Trim(); - var link = root.Elements(AtomNamespace + "link") - .FirstOrDefault(static element => - string.Equals(element.Attribute("rel")?.Value, "alternate", StringComparison.OrdinalIgnoreCase)) - ?.Attribute("href")?.Value?.Trim() - ?? root.Element(AtomNamespace + "link")?.Attribute("href")?.Value?.Trim(); - var updated = TryParseDate(root.Element(AtomNamespace + "updated")?.Value); - return (title, link, updated); - } - - return (null, null, null); - } - - private static IEnumerable ExtractEntries(XDocument xml) - { - var root = xml.Root; - if (root is null) - { - yield break; - } - - if (string.Equals(root.Name.LocalName, "rss", StringComparison.OrdinalIgnoreCase)) - { - var channel = root.Element("channel"); - if (channel is null) - { - yield break; - } - - foreach (var item in channel.Elements("item")) - { - yield return item; - } - yield break; - } - - if (root.Name == AtomNamespace + "feed") - { - foreach (var entry in root.Elements(AtomNamespace + "entry")) - { - yield return entry; - } - } - } - - private static string ExtractTitle(XElement element) - { - var title = element.Element("title")?.Value - ?? element.Element(AtomNamespace + "title")?.Value - ?? string.Empty; - return title.Trim(); - } - - private static string? ExtractLink(XElement element) - { - var linkValue = element.Element("link")?.Value; - if (!string.IsNullOrWhiteSpace(linkValue)) - { - return linkValue.Trim(); - } - - var atomLink = element.Elements(AtomNamespace + "link") - .FirstOrDefault(static el => - string.Equals(el.Attribute("rel")?.Value, "alternate", StringComparison.OrdinalIgnoreCase)) - ?? element.Element(AtomNamespace + "link"); - - if (atomLink is not null) - { - var href = atomLink.Attribute("href")?.Value; - if (!string.IsNullOrWhiteSpace(href)) - { - return href.Trim(); - } - } - - return null; - } - - private static string ExtractEntryId(XElement element) - { - var guid = element.Element("guid")?.Value; - if (!string.IsNullOrWhiteSpace(guid)) - { - return guid.Trim(); - } - - var atomId = element.Element(AtomNamespace + "id")?.Value; - if (!string.IsNullOrWhiteSpace(atomId)) - { - return atomId.Trim(); - } - - if (!string.IsNullOrWhiteSpace(element.Element("link")?.Value)) - { - return element.Element("link")!.Value.Trim(); - } - - if (!string.IsNullOrWhiteSpace(element.Element("title")?.Value)) - { - return GenerateStableKey(element.Element("title")!.Value); - } - - return string.Empty; - } - - private static string GenerateFallbackId(XElement element) - { - var builder = new StringBuilder(); - var title = element.Element("title")?.Value; - if (!string.IsNullOrWhiteSpace(title)) - { - builder.Append(title.Trim()); - } - - var link = ExtractLink(element); - if (!string.IsNullOrWhiteSpace(link)) - { - if (builder.Length > 0) - { - builder.Append("::"); - } - builder.Append(link); - } - - if (builder.Length == 0) - { - return Guid.NewGuid().ToString("n"); - } - - return GenerateStableKey(builder.ToString()); - } - - private static string GenerateStableKey(string value) - { - using var sha = SHA256.Create(); - var bytes = Encoding.UTF8.GetBytes(value); - var hash = sha.ComputeHash(bytes); - return Convert.ToHexString(hash).ToLowerInvariant(); - } - - private static string ExtractContent(XElement element) - { - var encoded = element.Element(ContentNamespace + "encoded")?.Value; - if (!string.IsNullOrWhiteSpace(encoded)) - { - return encoded; - } - - var description = element.Element("description")?.Value; - if (!string.IsNullOrWhiteSpace(description)) - { - return description; - } - - var summary = element.Element(AtomNamespace + "summary")?.Value; - if (!string.IsNullOrWhiteSpace(summary)) - { - return summary; - } - - return string.Empty; - } - - private static DateTimeOffset? ExtractDate(XElement element, string name) - { - var value = element.Element(name)?.Value; - return TryParseDate(value); - } - - private static DateTimeOffset? ExtractAtomDate(XElement element, string name) - { - var value = element.Element(AtomNamespace + name)?.Value; - return TryParseDate(value); - } - - private static DateTimeOffset? ExtractDcDate(XElement element) - { - var value = element.Element(XName.Get("date", "http://purl.org/dc/elements/1.1/"))?.Value; - return TryParseDate(value); - } - - private static DateTimeOffset? TryParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out var result)) - { - return result.ToUniversalTime(); - } - - if (DateTimeOffset.TryParse(value, CultureInfo.CurrentCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out result)) - { - return result.ToUniversalTime(); - } - - return null; - } - - private static Uri? TryCreateUri(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; - } - - private static IElement? ParseHtmlFragment(string html) - { - if (string.IsNullOrWhiteSpace(html)) - { - return null; - } - - var parser = new HtmlParser(new HtmlParserOptions - { - IsKeepingSourceReferences = false, - }); - var document = parser.ParseDocument($"{html}"); - return document.Body; - } - - private static string? BuildSummary(IElement? root) - { - if (root is null || !root.HasChildNodes) - { - return root?.TextContent is { Length: > 0 } text - ? NormalizeWhitespace(text) - : string.Empty; - } - - var segments = new List(); - foreach (var child in root.Children) - { - var text = NormalizeWhitespace(child.TextContent); - if (string.IsNullOrEmpty(text)) - { - continue; - } - - if (string.Equals(child.NodeName, "LI", StringComparison.OrdinalIgnoreCase)) - { - segments.Add($"- {text}"); - continue; - } - - segments.Add(text); - } - - if (segments.Count == 0) - { - var fallback = NormalizeWhitespace(root.TextContent); - return fallback; - } - - return string.Join("\n\n", segments); - } - - private static IReadOnlyList ExtractReferences(IElement? root) - { - if (root is null) - { - return Array.Empty(); - } - - var anchors = root.QuerySelectorAll("a"); - if (anchors.Length == 0) - { - return Array.Empty(); - } - - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - var references = new List(anchors.Length); - - foreach (var anchor in anchors) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!seen.Add(href)) - { - continue; - } - - var text = NormalizeWhitespace(anchor.TextContent); - if (string.IsNullOrEmpty(text)) - { - text = href; - } - - references.Add(new AcscReferenceDto(text, href)); - } - - return references; - } - - private static IReadOnlyDictionary ExtractFields(IElement? root, out string? serialNumber, out string? advisoryType) - { - serialNumber = null; - advisoryType = null; - - if (root is null) - { - return EmptyFields; - } - - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var element in root.QuerySelectorAll("strong")) - { - var labelRaw = NormalizeWhitespace(element.TextContent); - if (string.IsNullOrEmpty(labelRaw)) - { - continue; - } - - var label = labelRaw.TrimEnd(':').Trim(); - if (string.IsNullOrEmpty(label)) - { - continue; - } - - var key = NormalizeFieldKey(label); - if (string.IsNullOrEmpty(key)) - { - continue; - } - - var value = ExtractFieldValue(element); - if (string.IsNullOrEmpty(value)) - { - continue; - } - - if (!map.ContainsKey(key)) - { - map[key] = value; - } - - if (string.Equals(key, "serialNumber", StringComparison.OrdinalIgnoreCase)) - { - serialNumber ??= value; - } - else if (string.Equals(key, "advisoryType", StringComparison.OrdinalIgnoreCase)) - { - advisoryType ??= value; - } - } - - return map.Count == 0 - ? EmptyFields - : map; - } - - private static string? ExtractFieldValue(IElement strongElement) - { - var builder = new StringBuilder(); - var node = strongElement.NextSibling; - - while (node is not null) - { - if (node.NodeType == NodeType.Text) - { - builder.Append(node.TextContent); - } - else if (node is IElement element) - { - builder.Append(element.TextContent); - } - - node = node.NextSibling; - } - - var value = builder.ToString(); - if (string.IsNullOrWhiteSpace(value)) - { - var parent = strongElement.ParentElement; - if (parent is not null) - { - var parentText = parent.TextContent ?? string.Empty; - var trimmed = parentText.Replace(strongElement.TextContent ?? string.Empty, string.Empty, StringComparison.OrdinalIgnoreCase); - value = trimmed; - } - } - - value = NormalizeWhitespace(value); - if (string.IsNullOrEmpty(value)) - { - return null; - } - - value = value.TrimStart(':', '-', '–', '—', ' '); - return value.Trim(); - } - - private static IReadOnlyList BuildAliases(string? serialNumber, string? advisoryType) - { - var aliases = new List(capacity: 2); - if (!string.IsNullOrWhiteSpace(serialNumber)) - { - aliases.Add(serialNumber.Trim()); - } - - if (!string.IsNullOrWhiteSpace(advisoryType)) - { - aliases.Add(advisoryType.Trim()); - } - - return aliases.Count == 0 ? Array.Empty() : aliases; - } - - private static string NormalizeFieldKey(string label) - { - if (string.IsNullOrWhiteSpace(label)) - { - return string.Empty; - } - - var builder = new StringBuilder(label.Length); - var upperNext = false; - - foreach (var c in label) - { - if (char.IsLetterOrDigit(c)) - { - if (builder.Length == 0) - { - builder.Append(char.ToLowerInvariant(c)); - } - else if (upperNext) - { - builder.Append(char.ToUpperInvariant(c)); - upperNext = false; - } - else - { - builder.Append(char.ToLowerInvariant(c)); - } - } - else - { - if (builder.Length > 0) - { - upperNext = true; - } - } - } - - return builder.Length == 0 ? label.Trim() : builder.ToString(); - } - - private static string NormalizeWhitespace(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var builder = new StringBuilder(value.Length); - var previousIsWhitespace = false; - - foreach (var ch in value) - { - if (char.IsWhiteSpace(ch)) - { - if (!previousIsWhitespace) - { - builder.Append(' '); - previousIsWhitespace = true; - } - continue; - } - - builder.Append(ch); - previousIsWhitespace = false; - } - - return builder.ToString().Trim(); - } - private static readonly IReadOnlyDictionary EmptyFields = new Dictionary(StringComparer.OrdinalIgnoreCase); -} +using System.Globalization; +using System.Text; +using System.Xml.Linq; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using System.Security.Cryptography; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +internal static class AcscFeedParser +{ + private static readonly XNamespace AtomNamespace = "http://www.w3.org/2005/Atom"; + private static readonly XNamespace ContentNamespace = "http://purl.org/rss/1.0/modules/content/"; + public static AcscFeedDto Parse(byte[] payload, string feedSlug, DateTimeOffset parsedAt, HtmlContentSanitizer sanitizer) + { + ArgumentNullException.ThrowIfNull(payload); + ArgumentNullException.ThrowIfNull(sanitizer); + + if (payload.Length == 0) + { + return AcscFeedDto.Empty with + { + FeedSlug = feedSlug ?? string.Empty, + ParsedAt = parsedAt, + Entries = Array.Empty(), + }; + } + + var xml = XDocument.Parse(Encoding.UTF8.GetString(payload)); + + var (feedTitle, feedLink, feedUpdated) = ExtractFeedMetadata(xml); + var items = ExtractEntries(xml).ToArray(); + + var entries = new List(items.Length); + foreach (var item in items) + { + var entryId = ExtractEntryId(item); + if (string.IsNullOrWhiteSpace(entryId)) + { + // Fall back to hash of title + link to avoid duplicates. + entryId = GenerateFallbackId(item); + } + + var title = ExtractTitle(item); + var link = ExtractLink(item); + var published = ExtractDate(item, "pubDate") ?? ExtractAtomDate(item, "published") ?? ExtractDcDate(item); + var updated = ExtractAtomDate(item, "updated"); + + var rawHtml = ExtractContent(item); + var baseUri = TryCreateUri(link); + var sanitizedHtml = sanitizer.Sanitize(rawHtml, baseUri); + var htmlFragment = ParseHtmlFragment(sanitizedHtml); + + var summary = BuildSummary(htmlFragment) ?? string.Empty; + var contentText = NormalizeWhitespace(htmlFragment?.TextContent ?? string.Empty); + + var references = ExtractReferences(htmlFragment); + var fields = ExtractFields(htmlFragment, out var serialNumber, out var advisoryType); + var aliases = BuildAliases(serialNumber, advisoryType); + + var entry = new AcscEntryDto( + EntryId: entryId, + Title: title, + Link: link, + FeedSlug: feedSlug ?? string.Empty, + Published: published, + Updated: updated, + Summary: summary, + ContentHtml: sanitizedHtml, + ContentText: contentText, + References: references, + Aliases: aliases, + Fields: fields); + + entries.Add(entry); + } + + return new AcscFeedDto( + FeedSlug: feedSlug ?? string.Empty, + FeedTitle: feedTitle, + FeedLink: feedLink, + FeedUpdated: feedUpdated, + ParsedAt: parsedAt, + Entries: entries); + } + + private static (string? Title, string? Link, DateTimeOffset? Updated) ExtractFeedMetadata(XDocument xml) + { + var root = xml.Root; + if (root is null) + { + return (null, null, null); + } + + if (string.Equals(root.Name.LocalName, "rss", StringComparison.OrdinalIgnoreCase)) + { + var channel = root.Element("channel"); + var title = channel?.Element("title")?.Value?.Trim(); + var link = channel?.Element("link")?.Value?.Trim(); + var updated = TryParseDate(channel?.Element("lastBuildDate")?.Value); + return (title, link, updated); + } + + if (root.Name == AtomNamespace + "feed") + { + var title = root.Element(AtomNamespace + "title")?.Value?.Trim(); + var link = root.Elements(AtomNamespace + "link") + .FirstOrDefault(static element => + string.Equals(element.Attribute("rel")?.Value, "alternate", StringComparison.OrdinalIgnoreCase)) + ?.Attribute("href")?.Value?.Trim() + ?? root.Element(AtomNamespace + "link")?.Attribute("href")?.Value?.Trim(); + var updated = TryParseDate(root.Element(AtomNamespace + "updated")?.Value); + return (title, link, updated); + } + + return (null, null, null); + } + + private static IEnumerable ExtractEntries(XDocument xml) + { + var root = xml.Root; + if (root is null) + { + yield break; + } + + if (string.Equals(root.Name.LocalName, "rss", StringComparison.OrdinalIgnoreCase)) + { + var channel = root.Element("channel"); + if (channel is null) + { + yield break; + } + + foreach (var item in channel.Elements("item")) + { + yield return item; + } + yield break; + } + + if (root.Name == AtomNamespace + "feed") + { + foreach (var entry in root.Elements(AtomNamespace + "entry")) + { + yield return entry; + } + } + } + + private static string ExtractTitle(XElement element) + { + var title = element.Element("title")?.Value + ?? element.Element(AtomNamespace + "title")?.Value + ?? string.Empty; + return title.Trim(); + } + + private static string? ExtractLink(XElement element) + { + var linkValue = element.Element("link")?.Value; + if (!string.IsNullOrWhiteSpace(linkValue)) + { + return linkValue.Trim(); + } + + var atomLink = element.Elements(AtomNamespace + "link") + .FirstOrDefault(static el => + string.Equals(el.Attribute("rel")?.Value, "alternate", StringComparison.OrdinalIgnoreCase)) + ?? element.Element(AtomNamespace + "link"); + + if (atomLink is not null) + { + var href = atomLink.Attribute("href")?.Value; + if (!string.IsNullOrWhiteSpace(href)) + { + return href.Trim(); + } + } + + return null; + } + + private static string ExtractEntryId(XElement element) + { + var guid = element.Element("guid")?.Value; + if (!string.IsNullOrWhiteSpace(guid)) + { + return guid.Trim(); + } + + var atomId = element.Element(AtomNamespace + "id")?.Value; + if (!string.IsNullOrWhiteSpace(atomId)) + { + return atomId.Trim(); + } + + if (!string.IsNullOrWhiteSpace(element.Element("link")?.Value)) + { + return element.Element("link")!.Value.Trim(); + } + + if (!string.IsNullOrWhiteSpace(element.Element("title")?.Value)) + { + return GenerateStableKey(element.Element("title")!.Value); + } + + return string.Empty; + } + + private static string GenerateFallbackId(XElement element) + { + var builder = new StringBuilder(); + var title = element.Element("title")?.Value; + if (!string.IsNullOrWhiteSpace(title)) + { + builder.Append(title.Trim()); + } + + var link = ExtractLink(element); + if (!string.IsNullOrWhiteSpace(link)) + { + if (builder.Length > 0) + { + builder.Append("::"); + } + builder.Append(link); + } + + if (builder.Length == 0) + { + return Guid.NewGuid().ToString("n"); + } + + return GenerateStableKey(builder.ToString()); + } + + private static string GenerateStableKey(string value) + { + using var sha = SHA256.Create(); + var bytes = Encoding.UTF8.GetBytes(value); + var hash = sha.ComputeHash(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string ExtractContent(XElement element) + { + var encoded = element.Element(ContentNamespace + "encoded")?.Value; + if (!string.IsNullOrWhiteSpace(encoded)) + { + return encoded; + } + + var description = element.Element("description")?.Value; + if (!string.IsNullOrWhiteSpace(description)) + { + return description; + } + + var summary = element.Element(AtomNamespace + "summary")?.Value; + if (!string.IsNullOrWhiteSpace(summary)) + { + return summary; + } + + return string.Empty; + } + + private static DateTimeOffset? ExtractDate(XElement element, string name) + { + var value = element.Element(name)?.Value; + return TryParseDate(value); + } + + private static DateTimeOffset? ExtractAtomDate(XElement element, string name) + { + var value = element.Element(AtomNamespace + name)?.Value; + return TryParseDate(value); + } + + private static DateTimeOffset? ExtractDcDate(XElement element) + { + var value = element.Element(XName.Get("date", "http://purl.org/dc/elements/1.1/"))?.Value; + return TryParseDate(value); + } + + private static DateTimeOffset? TryParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out var result)) + { + return result.ToUniversalTime(); + } + + if (DateTimeOffset.TryParse(value, CultureInfo.CurrentCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AllowWhiteSpaces, out result)) + { + return result.ToUniversalTime(); + } + + return null; + } + + private static Uri? TryCreateUri(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; + } + + private static IElement? ParseHtmlFragment(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var parser = new HtmlParser(new HtmlParserOptions + { + IsKeepingSourceReferences = false, + }); + var document = parser.ParseDocument($"{html}"); + return document.Body; + } + + private static string? BuildSummary(IElement? root) + { + if (root is null || !root.HasChildNodes) + { + return root?.TextContent is { Length: > 0 } text + ? NormalizeWhitespace(text) + : string.Empty; + } + + var segments = new List(); + foreach (var child in root.Children) + { + var text = NormalizeWhitespace(child.TextContent); + if (string.IsNullOrEmpty(text)) + { + continue; + } + + if (string.Equals(child.NodeName, "LI", StringComparison.OrdinalIgnoreCase)) + { + segments.Add($"- {text}"); + continue; + } + + segments.Add(text); + } + + if (segments.Count == 0) + { + var fallback = NormalizeWhitespace(root.TextContent); + return fallback; + } + + return string.Join("\n\n", segments); + } + + private static IReadOnlyList ExtractReferences(IElement? root) + { + if (root is null) + { + return Array.Empty(); + } + + var anchors = root.QuerySelectorAll("a"); + if (anchors.Length == 0) + { + return Array.Empty(); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var references = new List(anchors.Length); + + foreach (var anchor in anchors) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!seen.Add(href)) + { + continue; + } + + var text = NormalizeWhitespace(anchor.TextContent); + if (string.IsNullOrEmpty(text)) + { + text = href; + } + + references.Add(new AcscReferenceDto(text, href)); + } + + return references; + } + + private static IReadOnlyDictionary ExtractFields(IElement? root, out string? serialNumber, out string? advisoryType) + { + serialNumber = null; + advisoryType = null; + + if (root is null) + { + return EmptyFields; + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var element in root.QuerySelectorAll("strong")) + { + var labelRaw = NormalizeWhitespace(element.TextContent); + if (string.IsNullOrEmpty(labelRaw)) + { + continue; + } + + var label = labelRaw.TrimEnd(':').Trim(); + if (string.IsNullOrEmpty(label)) + { + continue; + } + + var key = NormalizeFieldKey(label); + if (string.IsNullOrEmpty(key)) + { + continue; + } + + var value = ExtractFieldValue(element); + if (string.IsNullOrEmpty(value)) + { + continue; + } + + if (!map.ContainsKey(key)) + { + map[key] = value; + } + + if (string.Equals(key, "serialNumber", StringComparison.OrdinalIgnoreCase)) + { + serialNumber ??= value; + } + else if (string.Equals(key, "advisoryType", StringComparison.OrdinalIgnoreCase)) + { + advisoryType ??= value; + } + } + + return map.Count == 0 + ? EmptyFields + : map; + } + + private static string? ExtractFieldValue(IElement strongElement) + { + var builder = new StringBuilder(); + var node = strongElement.NextSibling; + + while (node is not null) + { + if (node.NodeType == NodeType.Text) + { + builder.Append(node.TextContent); + } + else if (node is IElement element) + { + builder.Append(element.TextContent); + } + + node = node.NextSibling; + } + + var value = builder.ToString(); + if (string.IsNullOrWhiteSpace(value)) + { + var parent = strongElement.ParentElement; + if (parent is not null) + { + var parentText = parent.TextContent ?? string.Empty; + var trimmed = parentText.Replace(strongElement.TextContent ?? string.Empty, string.Empty, StringComparison.OrdinalIgnoreCase); + value = trimmed; + } + } + + value = NormalizeWhitespace(value); + if (string.IsNullOrEmpty(value)) + { + return null; + } + + value = value.TrimStart(':', '-', '–', '—', ' '); + return value.Trim(); + } + + private static IReadOnlyList BuildAliases(string? serialNumber, string? advisoryType) + { + var aliases = new List(capacity: 2); + if (!string.IsNullOrWhiteSpace(serialNumber)) + { + aliases.Add(serialNumber.Trim()); + } + + if (!string.IsNullOrWhiteSpace(advisoryType)) + { + aliases.Add(advisoryType.Trim()); + } + + return aliases.Count == 0 ? Array.Empty() : aliases; + } + + private static string NormalizeFieldKey(string label) + { + if (string.IsNullOrWhiteSpace(label)) + { + return string.Empty; + } + + var builder = new StringBuilder(label.Length); + var upperNext = false; + + foreach (var c in label) + { + if (char.IsLetterOrDigit(c)) + { + if (builder.Length == 0) + { + builder.Append(char.ToLowerInvariant(c)); + } + else if (upperNext) + { + builder.Append(char.ToUpperInvariant(c)); + upperNext = false; + } + else + { + builder.Append(char.ToLowerInvariant(c)); + } + } + else + { + if (builder.Length > 0) + { + upperNext = true; + } + } + } + + return builder.Length == 0 ? label.Trim() : builder.ToString(); + } + + private static string NormalizeWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var builder = new StringBuilder(value.Length); + var previousIsWhitespace = false; + + foreach (var ch in value) + { + if (char.IsWhiteSpace(ch)) + { + if (!previousIsWhitespace) + { + builder.Append(' '); + previousIsWhitespace = true; + } + continue; + } + + builder.Append(ch); + previousIsWhitespace = false; + } + + return builder.ToString().Trim(); + } + private static readonly IReadOnlyDictionary EmptyFields = new Dictionary(StringComparer.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscMapper.cs b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Acsc/Internal/AcscMapper.cs rename to src/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs index d8d36a68..e5a207f6 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Internal/AcscMapper.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Internal/AcscMapper.cs @@ -1,312 +1,312 @@ -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; - -namespace StellaOps.Feedser.Source.Acsc.Internal; - -internal static class AcscMapper -{ - private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); - - public static IReadOnlyList Map( - AcscFeedDto feed, - DocumentRecord document, - DtoRecord dtoRecord, - string sourceName, - DateTimeOffset mappedAt) - { - ArgumentNullException.ThrowIfNull(feed); - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(dtoRecord); - ArgumentException.ThrowIfNullOrEmpty(sourceName); - - if (feed.Entries is null || feed.Entries.Count == 0) - { - return Array.Empty(); - } - - var advisories = new List(feed.Entries.Count); - foreach (var entry in feed.Entries) - { - if (entry is null) - { - continue; - } - - var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry); - var fetchProvenance = new AdvisoryProvenance( - sourceName, - "document", - document.Uri, - document.FetchedAt.ToUniversalTime(), - fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" }); - - var feedProvenance = new AdvisoryProvenance( - sourceName, - "feed", - feed.FeedSlug ?? string.Empty, - feed.ParsedAt.ToUniversalTime(), - fieldMask: new[] { "summary" }); - - var mappingProvenance = new AdvisoryProvenance( - sourceName, - "mapping", - entry.EntryId ?? entry.Link ?? advisoryKey, - mappedAt.ToUniversalTime(), - fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" }); - - var provenance = new[] - { - fetchProvenance, - feedProvenance, - mappingProvenance, - }; - - var aliases = BuildAliases(entry); - var severity = TryGetSeverity(entry.Fields); - var references = BuildReferences(entry, sourceName, mappedAt); - var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt); - - var advisory = new Advisory( - advisoryKey, - string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title, - string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary, - language: "en", - published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(), - modified: entry.Updated?.ToUniversalTime(), - severity: severity, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affectedPackages, - cvssMetrics: Array.Empty(), - provenance: provenance); - - advisories.Add(advisory); - } - - return advisories; - } - - private static IReadOnlyList BuildAliases(AcscEntryDto entry) - { - var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); - - if (!string.IsNullOrWhiteSpace(entry.EntryId)) - { - aliases.Add(entry.EntryId.Trim()); - } - - foreach (var alias in entry.Aliases ?? Array.Empty()) - { - if (!string.IsNullOrWhiteSpace(alias)) - { - aliases.Add(alias.Trim()); - } - } - - foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast()) - { - var value = match.Value.ToUpperInvariant(); - aliases.Add(value); - } - - foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast()) - { - var value = match.Value.ToUpperInvariant(); - aliases.Add(value); - } - - return aliases.Count == 0 - ? Array.Empty() - : aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(); - } - - private static IReadOnlyList BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt) - { - var references = new List(); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - void AddReference(string? url, string? kind, string? sourceTag, string? summary) - { - if (string.IsNullOrWhiteSpace(url)) - { - return; - } - - if (!Validation.LooksLikeHttpUrl(url)) - { - return; - } - - if (!seen.Add(url)) - { - return; - } - - references.Add(new AdvisoryReference( - url, - kind, - sourceTag, - summary, - new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime()))); - } - - AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title); - - foreach (var reference in entry.References ?? Array.Empty()) - { - if (reference is null) - { - continue; - } - - AddReference(reference.Url, "reference", null, reference.Title); - } - - return references.Count == 0 - ? Array.Empty() - : references - .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt) - { - if (entry.Fields is null || entry.Fields.Count == 0) - { - return Array.Empty(); - } - - if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected)) - { - return Array.Empty(); - } - - if (string.IsNullOrWhiteSpace(systemsAffected)) - { - return Array.Empty(); - } - - var identifiers = systemsAffected - .Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Select(static value => value.Trim()) - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - if (identifiers.Length == 0) - { - return Array.Empty(); - } - - var packages = new List(identifiers.Length); - foreach (var identifier in identifiers) - { - var provenance = new[] - { - new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }), - }; - - packages.Add(new AffectedPackage( - AffectedPackageTypes.Vendor, - identifier, - platform: null, - versionRanges: Array.Empty(), - statuses: Array.Empty(), - provenance: provenance, - normalizedVersions: Array.Empty())); - } - - return packages - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static string? TryGetSeverity(IReadOnlyDictionary fields) - { - if (fields is null || fields.Count == 0) - { - return null; - } - - var keys = new[] - { - "severity", - "riskLevel", - "threatLevel", - "impact", - }; - - foreach (var key in keys) - { - if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) - { - return value.Trim(); - } - } - - return null; - } - - private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry) - { - var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug); - var candidate = !string.IsNullOrWhiteSpace(entry.EntryId) - ? entry.EntryId - : !string.IsNullOrWhiteSpace(entry.Link) - ? entry.Link - : entry.Title; - - var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null; - if (string.IsNullOrEmpty(identifier)) - { - identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString()); - } - - return $"{sourceName}/{slug}/{identifier}"; - } - - private static string ToSlug(string value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return "unknown"; - } - - var builder = new StringBuilder(value.Length); - var previousDash = false; - - foreach (var ch in value) - { - if (char.IsLetterOrDigit(ch)) - { - builder.Append(char.ToLowerInvariant(ch)); - previousDash = false; - } - else if (!previousDash) - { - builder.Append('-'); - previousDash = true; - } - } - - var slug = builder.ToString().Trim('-'); - if (string.IsNullOrEmpty(slug)) - { - slug = CreateHash(value); - } - - return slug.Length <= 64 ? slug : slug[..64]; - } - - private static string CreateHash(string value) - { - var bytes = Encoding.UTF8.GetBytes(value); - var hash = SHA256.HashData(bytes); - return Convert.ToHexString(hash).ToLowerInvariant()[..16]; - } -} +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; + +namespace StellaOps.Concelier.Connector.Acsc.Internal; + +internal static class AcscMapper +{ + private static readonly Regex CveRegex = new("CVE-\\d{4}-\\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static IReadOnlyList Map( + AcscFeedDto feed, + DocumentRecord document, + DtoRecord dtoRecord, + string sourceName, + DateTimeOffset mappedAt) + { + ArgumentNullException.ThrowIfNull(feed); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + if (feed.Entries is null || feed.Entries.Count == 0) + { + return Array.Empty(); + } + + var advisories = new List(feed.Entries.Count); + foreach (var entry in feed.Entries) + { + if (entry is null) + { + continue; + } + + var advisoryKey = CreateAdvisoryKey(sourceName, feed.FeedSlug, entry); + var fetchProvenance = new AdvisoryProvenance( + sourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime(), + fieldMask: new[] { "summary", "aliases", "references", "affectedPackages" }); + + var feedProvenance = new AdvisoryProvenance( + sourceName, + "feed", + feed.FeedSlug ?? string.Empty, + feed.ParsedAt.ToUniversalTime(), + fieldMask: new[] { "summary" }); + + var mappingProvenance = new AdvisoryProvenance( + sourceName, + "mapping", + entry.EntryId ?? entry.Link ?? advisoryKey, + mappedAt.ToUniversalTime(), + fieldMask: new[] { "summary", "aliases", "references", "affectedpackages" }); + + var provenance = new[] + { + fetchProvenance, + feedProvenance, + mappingProvenance, + }; + + var aliases = BuildAliases(entry); + var severity = TryGetSeverity(entry.Fields); + var references = BuildReferences(entry, sourceName, mappedAt); + var affectedPackages = BuildAffectedPackages(entry, sourceName, mappedAt); + + var advisory = new Advisory( + advisoryKey, + string.IsNullOrWhiteSpace(entry.Title) ? $"ACSC Advisory {entry.EntryId}" : entry.Title, + string.IsNullOrWhiteSpace(entry.Summary) ? null : entry.Summary, + language: "en", + published: entry.Published?.ToUniversalTime() ?? feed.FeedUpdated?.ToUniversalTime() ?? document.FetchedAt.ToUniversalTime(), + modified: entry.Updated?.ToUniversalTime(), + severity: severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: provenance); + + advisories.Add(advisory); + } + + return advisories; + } + + private static IReadOnlyList BuildAliases(AcscEntryDto entry) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(entry.EntryId)) + { + aliases.Add(entry.EntryId.Trim()); + } + + foreach (var alias in entry.Aliases ?? Array.Empty()) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + aliases.Add(alias.Trim()); + } + } + + foreach (var match in CveRegex.Matches(entry.Summary ?? string.Empty).Cast()) + { + var value = match.Value.ToUpperInvariant(); + aliases.Add(value); + } + + foreach (var match in CveRegex.Matches(entry.ContentText ?? string.Empty).Cast()) + { + var value = match.Value.ToUpperInvariant(); + aliases.Add(value); + } + + return aliases.Count == 0 + ? Array.Empty() + : aliases.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(); + } + + private static IReadOnlyList BuildReferences(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt) + { + var references = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + void AddReference(string? url, string? kind, string? sourceTag, string? summary) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + if (!Validation.LooksLikeHttpUrl(url)) + { + return; + } + + if (!seen.Add(url)) + { + return; + } + + references.Add(new AdvisoryReference( + url, + kind, + sourceTag, + summary, + new AdvisoryProvenance(sourceName, "reference", url, recordedAt.ToUniversalTime()))); + } + + AddReference(entry.Link, "advisory", entry.FeedSlug, entry.Title); + + foreach (var reference in entry.References ?? Array.Empty()) + { + if (reference is null) + { + continue; + } + + AddReference(reference.Url, "reference", null, reference.Title); + } + + return references.Count == 0 + ? Array.Empty() + : references + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildAffectedPackages(AcscEntryDto entry, string sourceName, DateTimeOffset recordedAt) + { + if (entry.Fields is null || entry.Fields.Count == 0) + { + return Array.Empty(); + } + + if (!entry.Fields.TryGetValue("systemsAffected", out var systemsAffected) && !entry.Fields.TryGetValue("productsAffected", out systemsAffected)) + { + return Array.Empty(); + } + + if (string.IsNullOrWhiteSpace(systemsAffected)) + { + return Array.Empty(); + } + + var identifiers = systemsAffected + .Split(new[] { ',', ';', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(static value => value.Trim()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (identifiers.Length == 0) + { + return Array.Empty(); + } + + var packages = new List(identifiers.Length); + foreach (var identifier in identifiers) + { + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "affected", identifier, recordedAt.ToUniversalTime(), fieldMask: new[] { "affectedpackages" }), + }; + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: provenance, + normalizedVersions: Array.Empty())); + } + + return packages + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? TryGetSeverity(IReadOnlyDictionary fields) + { + if (fields is null || fields.Count == 0) + { + return null; + } + + var keys = new[] + { + "severity", + "riskLevel", + "threatLevel", + "impact", + }; + + foreach (var key in keys) + { + if (fields.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + + return null; + } + + private static string CreateAdvisoryKey(string sourceName, string? feedSlug, AcscEntryDto entry) + { + var slug = string.IsNullOrWhiteSpace(feedSlug) ? "general" : ToSlug(feedSlug); + var candidate = !string.IsNullOrWhiteSpace(entry.EntryId) + ? entry.EntryId + : !string.IsNullOrWhiteSpace(entry.Link) + ? entry.Link + : entry.Title; + + var identifier = !string.IsNullOrWhiteSpace(candidate) ? ToSlug(candidate!) : null; + if (string.IsNullOrEmpty(identifier)) + { + identifier = CreateHash(entry.Title ?? Guid.NewGuid().ToString()); + } + + return $"{sourceName}/{slug}/{identifier}"; + } + + private static string ToSlug(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "unknown"; + } + + var builder = new StringBuilder(value.Length); + var previousDash = false; + + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + previousDash = false; + } + else if (!previousDash) + { + builder.Append('-'); + previousDash = true; + } + } + + var slug = builder.ToString().Trim('-'); + if (string.IsNullOrEmpty(slug)) + { + slug = CreateHash(value); + } + + return slug.Length <= 64 ? slug : slug[..64]; + } + + private static string CreateHash(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant()[..16]; + } +} diff --git a/src/StellaOps.Feedser.Source.Acsc/Jobs.cs b/src/StellaOps.Concelier.Connector.Acsc/Jobs.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Acsc/Jobs.cs rename to src/StellaOps.Concelier.Connector.Acsc/Jobs.cs index 8596b78d..a600a6e6 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Jobs.cs @@ -1,55 +1,55 @@ -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Acsc; - -internal static class AcscJobKinds -{ - public const string Fetch = "source:acsc:fetch"; - public const string Parse = "source:acsc:parse"; - public const string Map = "source:acsc:map"; - public const string Probe = "source:acsc:probe"; -} - -internal sealed class AcscFetchJob : IJob -{ - private readonly AcscConnector _connector; - - public AcscFetchJob(AcscConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class AcscParseJob : IJob -{ - private readonly AcscConnector _connector; - - public AcscParseJob(AcscConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class AcscMapJob : IJob -{ - private readonly AcscConnector _connector; - - public AcscMapJob(AcscConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} - -internal sealed class AcscProbeJob : IJob -{ - private readonly AcscConnector _connector; - - public AcscProbeJob(AcscConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ProbeAsync(cancellationToken); -} +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Acsc; + +internal static class AcscJobKinds +{ + public const string Fetch = "source:acsc:fetch"; + public const string Parse = "source:acsc:parse"; + public const string Map = "source:acsc:map"; + public const string Probe = "source:acsc:probe"; +} + +internal sealed class AcscFetchJob : IJob +{ + private readonly AcscConnector _connector; + + public AcscFetchJob(AcscConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class AcscParseJob : IJob +{ + private readonly AcscConnector _connector; + + public AcscParseJob(AcscConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class AcscMapJob : IJob +{ + private readonly AcscConnector _connector; + + public AcscMapJob(AcscConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} + +internal sealed class AcscProbeJob : IJob +{ + private readonly AcscConnector _connector; + + public AcscProbeJob(AcscConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ProbeAsync(cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Acsc/Properties/AssemblyInfo.cs similarity index 52% rename from src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs rename to src/StellaOps.Concelier.Connector.Acsc/Properties/AssemblyInfo.cs index a2fc89fa..2e0a5406 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Properties/AssemblyInfo.cs +++ b/src/StellaOps.Concelier.Connector.Acsc/Properties/AssemblyInfo.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("FixtureUpdater")] -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Ghsa.Tests")] +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("FixtureUpdater")] +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Acsc.Tests")] diff --git a/src/StellaOps.Feedser.Source.Acsc/README.md b/src/StellaOps.Concelier.Connector.Acsc/README.md similarity index 81% rename from src/StellaOps.Feedser.Source.Acsc/README.md rename to src/StellaOps.Concelier.Connector.Acsc/README.md index 5e2f02a3..469b2b06 100644 --- a/src/StellaOps.Feedser.Source.Acsc/README.md +++ b/src/StellaOps.Concelier.Connector.Acsc/README.md @@ -1,68 +1,68 @@ -## StellaOps.Feedser.Source.Acsc - -Australian Cyber Security Centre (ACSC) connector that ingests RSS/Atom advisories, sanitises embedded HTML, and maps entries into canonical `Advisory` records for Feedser. - -### Configuration -Settings live under `feedser:sources:acsc` (see `AcscOptions`): - -| Setting | Description | Default | -| --- | --- | --- | -| `baseEndpoint` | Base URI for direct ACSC requests (trailing slash required). | `https://www.cyber.gov.au/` | -| `relayEndpoint` | Optional relay host to fall back to when Akamai refuses HTTP/2. | empty | -| `preferRelayByDefault` | Default endpoint preference when no cursor state exists. | `false` | -| `enableRelayFallback` | Allows automatic relay fallback when direct fetch fails. | `true` | -| `forceRelay` | Forces all fetches through the relay (skips direct attempts). | `false` | -| `feeds` | Array of feed descriptors (`slug`, `relativePath`, `enabled`). | alerts/advisories enabled | -| `requestTimeout` | Per-request timeout override. | 45 seconds | -| `failureBackoff` | Backoff window when fetch fails. | 5 minutes | -| `initialBackfill` | Sliding window used to seed published cursors. | 120 days | -| `userAgent` | Outbound `User-Agent` header. | `StellaOps/Feedser (+https://stella-ops.org)` | -| `requestVersion`/`versionPolicy` | HTTP version negotiation knobs. | HTTP/2 with downgrade | - -The dependency injection routine registers the connector plus scheduled jobs: - -| Job | Cron | Purpose | -| --- | --- | --- | -| `source:acsc:fetch` | `7,37 * * * *` | Fetch RSS/Atom feeds (direct + relay fallback). | -| `source:acsc:parse` | `12,42 * * * *` | Persist sanitised DTOs (`acsc.feed.v1`). | -| `source:acsc:map` | `17,47 * * * *` | Map DTO entries into canonical advisories. | -| `source:acsc:probe` | `25,55 * * * *` | Verify direct endpoint health and adjust cursor preference. | - -### Metrics -Emitted via `AcscDiagnostics` (`Meter` = `StellaOps.Feedser.Source.Acsc`): - -| Instrument | Unit | Description | -| --- | --- | --- | -| `acsc.fetch.attempts` | operations | Feed fetch attempts (tags: `feed`, `mode`). | -| `acsc.fetch.success` | operations | Successful fetches. | -| `acsc.fetch.failures` | operations | Failed fetches before retry backoff. | -| `acsc.fetch.unchanged` | operations | 304 Not Modified responses. | -| `acsc.fetch.fallbacks` | operations | Relay fallbacks triggered (`reason` tag). | -| `acsc.cursor.published_updates` | feeds | Published cursor updates per feed slug. | -| `acsc.parse.attempts` | documents | Parse attempts per feed. | -| `acsc.parse.success` | documents | Successful RSS → DTO conversions. | -| `acsc.parse.failures` | documents | Parse failures (tags: `feed`, `reason`). | -| `acsc.map.success` | advisories | Advisories emitted from a mapping pass. | - -### Logging -Key log messages include: -- Fetch successes/failures, HTTP status codes, and relay fallbacks. -- Parse failures with reasons (download, schema, sanitisation). -- Mapping summaries showing advisory counts per document. -- Probe results toggling relay usage. - -Logs include feed slug metadata for troubleshooting parallel ingestion. - -### Tests & fixtures -`StellaOps.Feedser.Source.Acsc.Tests` exercises the fetch→parse→map pipeline using canned RSS content. Deterministic snapshots live in `Acsc/Fixtures`. To refresh them after intentional behavioural changes: - -```bash -UPDATE_ACSC_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.Acsc.Tests/StellaOps.Feedser.Source.Acsc.Tests.csproj -``` - -Remember to review the generated `.actual.json` files when assertions fail without fixture updates. - -### Operational notes -- Keep the relay endpoint allowlisted for air-gapped deployments; the probe job will automatically switch back to direct fetching when Akamai stabilises. -- Mapping currently emits vendor `affectedPackages` from “Systems/Products affected” fields; expand range primitives once structured version data appears in ACSC feeds. -- The connector is offline-friendly—no outbound calls beyond the configured feeds. +## StellaOps.Concelier.Connector.Acsc + +Australian Cyber Security Centre (ACSC) connector that ingests RSS/Atom advisories, sanitises embedded HTML, and maps entries into canonical `Advisory` records for Concelier. + +### Configuration +Settings live under `concelier:sources:acsc` (see `AcscOptions`): + +| Setting | Description | Default | +| --- | --- | --- | +| `baseEndpoint` | Base URI for direct ACSC requests (trailing slash required). | `https://www.cyber.gov.au/` | +| `relayEndpoint` | Optional relay host to fall back to when Akamai refuses HTTP/2. | empty | +| `preferRelayByDefault` | Default endpoint preference when no cursor state exists. | `false` | +| `enableRelayFallback` | Allows automatic relay fallback when direct fetch fails. | `true` | +| `forceRelay` | Forces all fetches through the relay (skips direct attempts). | `false` | +| `feeds` | Array of feed descriptors (`slug`, `relativePath`, `enabled`). | alerts/advisories enabled | +| `requestTimeout` | Per-request timeout override. | 45 seconds | +| `failureBackoff` | Backoff window when fetch fails. | 5 minutes | +| `initialBackfill` | Sliding window used to seed published cursors. | 120 days | +| `userAgent` | Outbound `User-Agent` header. | `StellaOps/Concelier (+https://stella-ops.org)` | +| `requestVersion`/`versionPolicy` | HTTP version negotiation knobs. | HTTP/2 with downgrade | + +The dependency injection routine registers the connector plus scheduled jobs: + +| Job | Cron | Purpose | +| --- | --- | --- | +| `source:acsc:fetch` | `7,37 * * * *` | Fetch RSS/Atom feeds (direct + relay fallback). | +| `source:acsc:parse` | `12,42 * * * *` | Persist sanitised DTOs (`acsc.feed.v1`). | +| `source:acsc:map` | `17,47 * * * *` | Map DTO entries into canonical advisories. | +| `source:acsc:probe` | `25,55 * * * *` | Verify direct endpoint health and adjust cursor preference. | + +### Metrics +Emitted via `AcscDiagnostics` (`Meter` = `StellaOps.Concelier.Connector.Acsc`): + +| Instrument | Unit | Description | +| --- | --- | --- | +| `acsc.fetch.attempts` | operations | Feed fetch attempts (tags: `feed`, `mode`). | +| `acsc.fetch.success` | operations | Successful fetches. | +| `acsc.fetch.failures` | operations | Failed fetches before retry backoff. | +| `acsc.fetch.unchanged` | operations | 304 Not Modified responses. | +| `acsc.fetch.fallbacks` | operations | Relay fallbacks triggered (`reason` tag). | +| `acsc.cursor.published_updates` | feeds | Published cursor updates per feed slug. | +| `acsc.parse.attempts` | documents | Parse attempts per feed. | +| `acsc.parse.success` | documents | Successful RSS → DTO conversions. | +| `acsc.parse.failures` | documents | Parse failures (tags: `feed`, `reason`). | +| `acsc.map.success` | advisories | Advisories emitted from a mapping pass. | + +### Logging +Key log messages include: +- Fetch successes/failures, HTTP status codes, and relay fallbacks. +- Parse failures with reasons (download, schema, sanitisation). +- Mapping summaries showing advisory counts per document. +- Probe results toggling relay usage. + +Logs include feed slug metadata for troubleshooting parallel ingestion. + +### Tests & fixtures +`StellaOps.Concelier.Connector.Acsc.Tests` exercises the fetch→parse→map pipeline using canned RSS content. Deterministic snapshots live in `Acsc/Fixtures`. To refresh them after intentional behavioural changes: + +```bash +UPDATE_ACSC_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.Acsc.Tests/StellaOps.Concelier.Connector.Acsc.Tests.csproj +``` + +Remember to review the generated `.actual.json` files when assertions fail without fixture updates. + +### Operational notes +- Keep the relay endpoint allowlisted for air-gapped deployments; the probe job will automatically switch back to direct fetching when Akamai stabilises. +- Mapping currently emits vendor `affectedPackages` from “Systems/Products affected” fields; expand range primitives once structured version data appears in ACSC feeds. +- The connector is offline-friendly—no outbound calls beyond the configured feeds. diff --git a/src/StellaOps.Concelier.Connector.Acsc/StellaOps.Concelier.Connector.Acsc.csproj b/src/StellaOps.Concelier.Connector.Acsc/StellaOps.Concelier.Connector.Acsc.csproj new file mode 100644 index 00000000..cfae3203 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Acsc/StellaOps.Concelier.Connector.Acsc.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Acsc/TASKS.md b/src/StellaOps.Concelier.Connector.Acsc/TASKS.md similarity index 80% rename from src/StellaOps.Feedser.Source.Acsc/TASKS.md rename to src/StellaOps.Concelier.Connector.Acsc/TASKS.md index 3e70e703..cacd3a5b 100644 --- a/src/StellaOps.Feedser.Source.Acsc/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Acsc/TASKS.md @@ -1,11 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-ACSC-02-001 Source discovery & feed contract|BE-Conn-ACSC|Research|**DONE (2025-10-11)** – Catalogued feed slugs `/acsc/view-all-content/{alerts,advisories,news,publications,threats}/rss`; every endpoint currently negotiates HTTP/2 then aborts with `INTERNAL_ERROR` (curl exit 92) and hanging >600 s when forcing `--http1.1`. Documented traces + mitigations in `docs/feedser-connector-research-20251011.md` and opened `FEEDCONN-SHARED-HTTP2-001` for shared handler tweaks (force `RequestVersionOrLower`, jittered retries, relay option).| -|FEEDCONN-ACSC-02-002 Fetch pipeline & cursor persistence|BE-Conn-ACSC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – HTTP client now pins `HttpRequestMessage.VersionPolicy = RequestVersionOrLower`, forces `AutomaticDecompression = GZip | Deflate`, and sends `User-Agent: StellaOps/Feedser (+https://stella-ops.org)` via `AddAcscConnector`. Fetch pipeline implemented in `AcscConnector` with relay-aware fallback (`AcscProbeJob` seeds preference), deterministic cursor updates (`preferredEndpoint`, published timestamp per feed), and metadata-deduped documents. Unit tests `AcscConnectorFetchTests` + `AcscHttpClientConfigurationTests` cover direct/relay flows and client wiring.| -|FEEDCONN-ACSC-02-003 Parser & DTO sanitiser|BE-Conn-ACSC|Source.Common|**DONE (2025-10-12)** – Added `AcscFeedParser` to sanitise RSS payloads, collapse multi-paragraph summaries, dedupe references, and surface `serialNumber`/`advisoryType` fields as structured metadata + alias candidates. `ParseAsync` now materialises `acsc.feed.v1` DTOs, promotes documents to `pending-map`, and advances cursor state. Covered by `AcscConnectorParseTests`.| -|FEEDCONN-ACSC-02-004 Canonical mapper + range primitives|BE-Conn-ACSC|Models|**DONE (2025-10-12)** – Introduced `AcscMapper` and wired `MapAsync` to emit canonical advisories with normalized aliases, source-tagged references, and optional vendor `affectedPackages` derived from “Systems/Products affected” fields. Documents transition to `mapped`, advisories persist via `IAdvisoryStore`, and metrics/logging capture mapped counts. `AcscConnectorParseTests` exercise fetch→parse→map flow.| -|FEEDCONN-ACSC-02-005 Deterministic fixtures & regression tests|QA|Testing|**DONE (2025-10-12)** – `AcscConnectorParseTests` now snapshots fetch→parse→map output via `Acsc/Fixtures/acsc-advisories.snapshot.json`; set `UPDATE_ACSC_FIXTURES=1` to regenerate. Tests assert DTO status transitions, advisory persistence, and state cleanup.| -|FEEDCONN-ACSC-02-006 Diagnostics & documentation|DevEx|Docs|**DONE (2025-10-12)** – Added module README describing configuration, job schedules, metrics (including new `acsc.map.success` counter), relay behaviour, and fixture workflow. Diagnostics updated to count map successes alongside existing fetch/parse metrics.| -|FEEDCONN-ACSC-02-007 Feed retention & pagination validation|BE-Conn-ACSC|Research|**DONE (2025-10-11)** – Relay sampling shows retention ≥ July 2025; need to re-run once direct HTTP/2 path is stable to see if feed caps at ~50 items and whether `?page=` exists. Pending action tracked in shared HTTP downgrade task.| -|FEEDCONN-ACSC-02-008 HTTP client compatibility plan|BE-Conn-ACSC|Source.Common|**DONE (2025-10-11)** – Reproduced Akamai resets, drafted downgrade plan (two-stage HTTP/2 retry + relay fallback), and filed `FEEDCONN-SHARED-HTTP2-001`; module README TODO will host the per-environment knob matrix.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-ACSC-02-001 Source discovery & feed contract|BE-Conn-ACSC|Research|**DONE (2025-10-11)** – Catalogued feed slugs `/acsc/view-all-content/{alerts,advisories,news,publications,threats}/rss`; every endpoint currently negotiates HTTP/2 then aborts with `INTERNAL_ERROR` (curl exit 92) and hanging >600 s when forcing `--http1.1`. Documented traces + mitigations in `docs/concelier-connector-research-20251011.md` and opened `FEEDCONN-SHARED-HTTP2-001` for shared handler tweaks (force `RequestVersionOrLower`, jittered retries, relay option).| +|FEEDCONN-ACSC-02-002 Fetch pipeline & cursor persistence|BE-Conn-ACSC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – HTTP client now pins `HttpRequestMessage.VersionPolicy = RequestVersionOrLower`, forces `AutomaticDecompression = GZip | Deflate`, and sends `User-Agent: StellaOps/Concelier (+https://stella-ops.org)` via `AddAcscConnector`. Fetch pipeline implemented in `AcscConnector` with relay-aware fallback (`AcscProbeJob` seeds preference), deterministic cursor updates (`preferredEndpoint`, published timestamp per feed), and metadata-deduped documents. Unit tests `AcscConnectorFetchTests` + `AcscHttpClientConfigurationTests` cover direct/relay flows and client wiring.| +|FEEDCONN-ACSC-02-003 Parser & DTO sanitiser|BE-Conn-ACSC|Source.Common|**DONE (2025-10-12)** – Added `AcscFeedParser` to sanitise RSS payloads, collapse multi-paragraph summaries, dedupe references, and surface `serialNumber`/`advisoryType` fields as structured metadata + alias candidates. `ParseAsync` now materialises `acsc.feed.v1` DTOs, promotes documents to `pending-map`, and advances cursor state. Covered by `AcscConnectorParseTests`.| +|FEEDCONN-ACSC-02-004 Canonical mapper + range primitives|BE-Conn-ACSC|Models|**DONE (2025-10-12)** – Introduced `AcscMapper` and wired `MapAsync` to emit canonical advisories with normalized aliases, source-tagged references, and optional vendor `affectedPackages` derived from “Systems/Products affected” fields. Documents transition to `mapped`, advisories persist via `IAdvisoryStore`, and metrics/logging capture mapped counts. `AcscConnectorParseTests` exercise fetch→parse→map flow.| +|FEEDCONN-ACSC-02-005 Deterministic fixtures & regression tests|QA|Testing|**DONE (2025-10-12)** – `AcscConnectorParseTests` now snapshots fetch→parse→map output via `Acsc/Fixtures/acsc-advisories.snapshot.json`; set `UPDATE_ACSC_FIXTURES=1` to regenerate. Tests assert DTO status transitions, advisory persistence, and state cleanup.| +|FEEDCONN-ACSC-02-006 Diagnostics & documentation|DevEx|Docs|**DONE (2025-10-12)** – Added module README describing configuration, job schedules, metrics (including new `acsc.map.success` counter), relay behaviour, and fixture workflow. Diagnostics updated to count map successes alongside existing fetch/parse metrics.| +|FEEDCONN-ACSC-02-007 Feed retention & pagination validation|BE-Conn-ACSC|Research|**DONE (2025-10-11)** – Relay sampling shows retention ≥ July 2025; need to re-run once direct HTTP/2 path is stable to see if feed caps at ~50 items and whether `?page=` exists. Pending action tracked in shared HTTP downgrade task.| +|FEEDCONN-ACSC-02-008 HTTP client compatibility plan|BE-Conn-ACSC|Source.Common|**DONE (2025-10-11)** – Reproduced Akamai resets, drafted downgrade plan (two-stage HTTP/2 retry + relay fallback), and filed `FEEDCONN-SHARED-HTTP2-001`; module README TODO will host the per-environment knob matrix.| diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/CccsConnectorTests.cs b/src/StellaOps.Concelier.Connector.Cccs.Tests/CccsConnectorTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Cccs.Tests/CccsConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Cccs.Tests/CccsConnectorTests.cs index 1a82676f..7d06d55f 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/CccsConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/CccsConnectorTests.cs @@ -1,163 +1,163 @@ -using System; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Cccs; -using StellaOps.Feedser.Source.Cccs.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Cccs.Tests; - -[Collection("mongo-fixture")] -public sealed class CccsConnectorTests : IAsyncLifetime -{ - private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"); - private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type"); - - private readonly MongoIntegrationFixture _fixture; - private readonly CannedHttpMessageHandler _handler; - - public CccsConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesCanonicalAdvisory() - { - await using var provider = await BuildServiceProviderAsync(); - SeedFeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - advisories.Should().HaveCount(1); - - var advisory = advisories[0]; - advisory.AdvisoryKey.Should().Be("TEST-001"); - advisory.Title.Should().Be("Test Advisory Title"); - advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" }); - advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details"); - advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"); - advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0"); - advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0"); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.Should().NotBeNull(); - state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); - pendingDocs!.AsBsonArray.Should().BeEmpty(); - state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); - pendingMappings!.AsBsonArray.Should().BeEmpty(); - } - - [Fact] - public async Task Fetch_PersistsRawDocumentWithMetadata() - { - await using var provider = await BuildServiceProviderAsync(); - SeedFeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - - var documentStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None); - document.Should().NotBeNull(); - document!.Status.Should().Be(DocumentStatuses.PendingParse); - document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en"); - document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001"); - document.ContentType.Should().Be("application/json"); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddCccsConnector(options => - { - options.Feeds.Clear(); - options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri)); - options.RequestDelay = TimeSpan.Zero; - options.MaxEntriesPerFetch = 10; - options.MaxKnownEntries = 32; - }); - - services.Configure(CccsOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedFeedResponses() - { - AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json")); - AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json")); - } - - private void AddJsonResponse(Uri uri, string json, string? etag = null) - { - _handler.AddResponse(uri, () => - { - var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(json, Encoding.UTF8, "application/json"), - }; - if (!string.IsNullOrWhiteSpace(etag)) - { - response.Headers.ETag = new EntityTagHeaderValue(etag); - } - - return response; - }); - } - - private static string ReadFixture(string fileName) - => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; -} +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Cccs; +using StellaOps.Concelier.Connector.Cccs.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Cccs.Tests; + +[Collection("mongo-fixture")] +public sealed class CccsConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"); + private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type"); + + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler; + + public CccsConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().HaveCount(1); + + var advisory = advisories[0]; + advisory.AdvisoryKey.Should().Be("TEST-001"); + advisory.Title.Should().Be("Test Advisory Title"); + advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" }); + advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details"); + advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"); + advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0"); + advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0"); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); + pendingDocs!.AsBsonArray.Should().BeEmpty(); + state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); + pendingMappings!.AsBsonArray.Should().BeEmpty(); + } + + [Fact] + public async Task Fetch_PersistsRawDocumentWithMetadata() + { + await using var provider = await BuildServiceProviderAsync(); + SeedFeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None); + document.Should().NotBeNull(); + document!.Status.Should().Be(DocumentStatuses.PendingParse); + document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en"); + document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001"); + document.ContentType.Should().Be("application/json"); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCccsConnector(options => + { + options.Feeds.Clear(); + options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri)); + options.RequestDelay = TimeSpan.Zero; + options.MaxEntriesPerFetch = 10; + options.MaxKnownEntries = 32; + }); + + services.Configure(CccsOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedFeedResponses() + { + AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json")); + AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json")); + } + + private void AddJsonResponse(Uri uri, string json, string? etag = null) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + if (!string.IsNullOrWhiteSpace(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private static string ReadFixture(string fileName) + => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-feed-en.json b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-feed-en.json similarity index 97% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-feed-en.json rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-feed-en.json index 2d701af6..38a7f4ed 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-feed-en.json +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-feed-en.json @@ -1,25 +1,25 @@ -{ - "ERROR": false, - "response": [ - { - "nid": 1001, - "title": "Test Advisory Title", - "uuid": "uuid-test-001", - "banner": null, - "lang": "en", - "date_modified": "2025-08-11", - "date_modified_ts": "2025-08-11T12:00:00Z", - "date_created": "2025-08-10T15:30:00Z", - "summary": "Summary of advisory.", - "body": [ - "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

" - ], - "url": "/en/alerts-advisories/test-advisory", - "alert_type": 397, - "serial_number": "TEST-001", - "subject": "Infrastructure", - "moderation_state": "published", - "external_url": "https://example.com/external/advisory" - } - ] -} +{ + "ERROR": false, + "response": [ + { + "nid": 1001, + "title": "Test Advisory Title", + "uuid": "uuid-test-001", + "banner": null, + "lang": "en", + "date_modified": "2025-08-11", + "date_modified_ts": "2025-08-11T12:00:00Z", + "date_created": "2025-08-10T15:30:00Z", + "summary": "Summary of advisory.", + "body": [ + "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

" + ], + "url": "/en/alerts-advisories/test-advisory", + "alert_type": 397, + "serial_number": "TEST-001", + "subject": "Infrastructure", + "moderation_state": "published", + "external_url": "https://example.com/external/advisory" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json similarity index 98% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json index 3fc2df85..505baa3c 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory-fr.json @@ -1,21 +1,21 @@ -{ - "sourceId": "TEST-002-FR", - "serialNumber": "TEST-002-FR", - "uuid": "uuid-test-002", - "language": "fr", - "title": "Avis de sécurité – Mise à jour urgente", - "summary": "Résumé de l'avis en français.", - "canonicalUrl": "https://www.cyber.gc.ca/fr/alertes-avis/test-avis", - "externalUrl": "https://exemple.ca/avis", - "bodyHtml": "

Numéro : TEST-002-FR
Date : 15 août 2025

Produits touchés

  • Produit Exemple 3.1
  • Produit Exemple 3.2
    • Variante 3.2.1

Voir Lien de détails.

Lien interne Contactez-nous.

Correctifs pour CVE-2024-1111.

", - "bodySegments": [ - "

Numéro : TEST-002-FR
Date : 15 août 2025

Produits touchés

  • Produit Exemple 3.1
  • Produit Exemple 3.2
    • Variante 3.2.1

Voir Lien de détails.

Lien interne Contactez-nous.

Correctifs pour CVE-2024-1111.

" - ], - "alertType": "Alerte", - "subject": "Infrastructure critique", - "banner": null, - "published": "2025-08-15T13:45:00Z", - "modified": "2025-08-16T09:15:00Z", - "rawCreated": "15 août 2025", - "rawModified": "2025-08-16T09:15:00Z" -} +{ + "sourceId": "TEST-002-FR", + "serialNumber": "TEST-002-FR", + "uuid": "uuid-test-002", + "language": "fr", + "title": "Avis de sécurité – Mise à jour urgente", + "summary": "Résumé de l'avis en français.", + "canonicalUrl": "https://www.cyber.gc.ca/fr/alertes-avis/test-avis", + "externalUrl": "https://exemple.ca/avis", + "bodyHtml": "

Numéro : TEST-002-FR
Date : 15 août 2025

Produits touchés

  • Produit Exemple 3.1
  • Produit Exemple 3.2
    • Variante 3.2.1

Voir Lien de détails.

Lien interne Contactez-nous.

Correctifs pour CVE-2024-1111.

", + "bodySegments": [ + "

Numéro : TEST-002-FR
Date : 15 août 2025

Produits touchés

  • Produit Exemple 3.1
  • Produit Exemple 3.2
    • Variante 3.2.1

Voir Lien de détails.

Lien interne Contactez-nous.

Correctifs pour CVE-2024-1111.

" + ], + "alertType": "Alerte", + "subject": "Infrastructure critique", + "banner": null, + "published": "2025-08-15T13:45:00Z", + "modified": "2025-08-16T09:15:00Z", + "rawCreated": "15 août 2025", + "rawModified": "2025-08-16T09:15:00Z" +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory.json b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory.json similarity index 98% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory.json rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory.json index 7ff94196..74f2e99c 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-raw-advisory.json +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-raw-advisory.json @@ -1,21 +1,21 @@ -{ - "sourceId": "TEST-001", - "serialNumber": "TEST-001", - "uuid": "uuid-test-001", - "language": "en", - "title": "Test Advisory Title", - "summary": "Summary of advisory.", - "canonicalUrl": "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", - "externalUrl": "https://example.com/external/advisory", - "bodyHtml": "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

", - "bodySegments": [ - "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

" - ], - "alertType": "Advisory", - "subject": "Infrastructure", - "banner": null, - "published": "2025-08-10T15:30:00Z", - "modified": "2025-08-11T12:00:00Z", - "rawCreated": "August 10, 2025", - "rawModified": "2025-08-11T12:00:00Z" -} +{ + "sourceId": "TEST-001", + "serialNumber": "TEST-001", + "uuid": "uuid-test-001", + "language": "en", + "title": "Test Advisory Title", + "summary": "Summary of advisory.", + "canonicalUrl": "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", + "externalUrl": "https://example.com/external/advisory", + "bodyHtml": "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

", + "bodySegments": [ + "

Number: TEST-001
Date: 14 April 2018

Affected Products

  • Vendor Widget 1.0
  • Vendor Widget 2.0

See Details Link.

Internal link Contact.

Mitigation for CVE-2020-1234 and CVE-2021-9999.

" + ], + "alertType": "Advisory", + "subject": "Infrastructure", + "banner": null, + "published": "2025-08-10T15:30:00Z", + "modified": "2025-08-11T12:00:00Z", + "rawCreated": "August 10, 2025", + "rawModified": "2025-08-11T12:00:00Z" +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-taxonomy-en.json b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-taxonomy-en.json similarity index 92% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-taxonomy-en.json rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-taxonomy-en.json index 88a77fad..0d6d5208 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Fixtures/cccs-taxonomy-en.json +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Fixtures/cccs-taxonomy-en.json @@ -1,13 +1,13 @@ -{ - "ERROR": false, - "response": [ - { - "id": 396, - "title": "Advisory" - }, - { - "id": 397, - "title": "Alert" - } - ] -} +{ + "ERROR": false, + "response": [ + { + "id": 396, + "title": "Advisory" + }, + { + "id": 397, + "title": "Alert" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsHtmlParserTests.cs b/src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsHtmlParserTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsHtmlParserTests.cs rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsHtmlParserTests.cs index 20e14885..852685e8 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsHtmlParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsHtmlParserTests.cs @@ -1,92 +1,92 @@ -using System; -using System.IO; -using System.Linq; -using System.Text.Json; -using FluentAssertions; -using StellaOps.Feedser.Source.Cccs.Internal; -using StellaOps.Feedser.Source.Common.Html; -using Xunit; -using Xunit.Abstractions; - -namespace StellaOps.Feedser.Source.Cccs.Tests.Internal; - -public sealed class CccsHtmlParserTests -{ - private readonly ITestOutputHelper _output; - private static readonly HtmlContentSanitizer Sanitizer = new(); - private static readonly CccsHtmlParser Parser = new(Sanitizer); - - public CccsHtmlParserTests(ITestOutputHelper output) - { - _output = output ?? throw new ArgumentNullException(nameof(output)); - } - - public static IEnumerable ParserCases() - { - yield return new object[] - { - "cccs-raw-advisory.json", - "TEST-001", - "en", - new[] { "Vendor Widget 1.0", "Vendor Widget 2.0" }, - new[] - { - "https://example.com/details", - "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en" - }, - new[] { "CVE-2020-1234", "CVE-2021-9999" } - }; - - yield return new object[] - { - "cccs-raw-advisory-fr.json", - "TEST-002-FR", - "fr", - new[] { "Produit Exemple 3.1", "Produit Exemple 3.2", "Variante 3.2.1" }, - new[] - { - "https://exemple.ca/details", - "https://www.cyber.gc.ca/fr/contact-centre-cyber" - }, - new[] { "CVE-2024-1111" } - }; - } - - [Theory] - [MemberData(nameof(ParserCases))] - public void Parse_ExtractsExpectedFields( - string fixtureName, - string expectedSerial, - string expectedLanguage, - string[] expectedProducts, - string[] expectedReferenceUrls, - string[] expectedCves) - { - var raw = LoadFixture(fixtureName); - - var dto = Parser.Parse(raw); - - _output.WriteLine("Products: {0}", string.Join("|", dto.Products)); - _output.WriteLine("References: {0}", string.Join("|", dto.References.Select(r => $"{r.Url} ({r.Label})"))); - _output.WriteLine("CVEs: {0}", string.Join("|", dto.CveIds)); - - dto.SerialNumber.Should().Be(expectedSerial); - dto.Language.Should().Be(expectedLanguage); - dto.Products.Should().BeEquivalentTo(expectedProducts); - foreach (var url in expectedReferenceUrls) - { - dto.References.Should().Contain(reference => reference.Url == url); - } - - dto.CveIds.Should().BeEquivalentTo(expectedCves); - dto.ContentHtml.Should().Contain("
    ").And.Contain("
  • "); - dto.ContentHtml.Should().Contain("(string fileName) - { - var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName); - var json = File.ReadAllText(path); - return JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; - } -} +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using FluentAssertions; +using StellaOps.Concelier.Connector.Cccs.Internal; +using StellaOps.Concelier.Connector.Common.Html; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Concelier.Connector.Cccs.Tests.Internal; + +public sealed class CccsHtmlParserTests +{ + private readonly ITestOutputHelper _output; + private static readonly HtmlContentSanitizer Sanitizer = new(); + private static readonly CccsHtmlParser Parser = new(Sanitizer); + + public CccsHtmlParserTests(ITestOutputHelper output) + { + _output = output ?? throw new ArgumentNullException(nameof(output)); + } + + public static IEnumerable ParserCases() + { + yield return new object[] + { + "cccs-raw-advisory.json", + "TEST-001", + "en", + new[] { "Vendor Widget 1.0", "Vendor Widget 2.0" }, + new[] + { + "https://example.com/details", + "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en" + }, + new[] { "CVE-2020-1234", "CVE-2021-9999" } + }; + + yield return new object[] + { + "cccs-raw-advisory-fr.json", + "TEST-002-FR", + "fr", + new[] { "Produit Exemple 3.1", "Produit Exemple 3.2", "Variante 3.2.1" }, + new[] + { + "https://exemple.ca/details", + "https://www.cyber.gc.ca/fr/contact-centre-cyber" + }, + new[] { "CVE-2024-1111" } + }; + } + + [Theory] + [MemberData(nameof(ParserCases))] + public void Parse_ExtractsExpectedFields( + string fixtureName, + string expectedSerial, + string expectedLanguage, + string[] expectedProducts, + string[] expectedReferenceUrls, + string[] expectedCves) + { + var raw = LoadFixture(fixtureName); + + var dto = Parser.Parse(raw); + + _output.WriteLine("Products: {0}", string.Join("|", dto.Products)); + _output.WriteLine("References: {0}", string.Join("|", dto.References.Select(r => $"{r.Url} ({r.Label})"))); + _output.WriteLine("CVEs: {0}", string.Join("|", dto.CveIds)); + + dto.SerialNumber.Should().Be(expectedSerial); + dto.Language.Should().Be(expectedLanguage); + dto.Products.Should().BeEquivalentTo(expectedProducts); + foreach (var url in expectedReferenceUrls) + { + dto.References.Should().Contain(reference => reference.Url == url); + } + + dto.CveIds.Should().BeEquivalentTo(expectedCves); + dto.ContentHtml.Should().Contain("
      ").And.Contain("
    • "); + dto.ContentHtml.Should().Contain("(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName); + var json = File.ReadAllText(path); + return JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsMapperTests.cs b/src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsMapperTests.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsMapperTests.cs rename to src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsMapperTests.cs index 54d5d898..8dca2117 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/Internal/CccsMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/Internal/CccsMapperTests.cs @@ -1,43 +1,43 @@ -using System; -using FluentAssertions; -using StellaOps.Feedser.Source.Cccs.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Storage.Mongo.Documents; -using Xunit; - -namespace StellaOps.Feedser.Source.Cccs.Tests.Internal; - -public sealed class CccsMapperTests -{ - [Fact] - public void Map_CreatesCanonicalAdvisory() - { - var raw = CccsHtmlParserTests.LoadFixture("cccs-raw-advisory.json"); - var dto = new CccsHtmlParser(new HtmlContentSanitizer()).Parse(raw); - var document = new DocumentRecord( - Guid.NewGuid(), - CccsConnectorPlugin.SourceName, - dto.CanonicalUrl, - DateTimeOffset.UtcNow, - "sha-test", - DocumentStatuses.PendingMap, - "application/json", - Headers: null, - Metadata: null, - Etag: null, - LastModified: dto.Modified, - GridFsId: null); - - var recordedAt = DateTimeOffset.Parse("2025-08-12T00:00:00Z"); - var advisory = CccsMapper.Map(dto, document, recordedAt); - - advisory.AdvisoryKey.Should().Be("TEST-001"); - advisory.Title.Should().Be(dto.Title); - advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" }); - advisory.References.Should().Contain(reference => reference.Url == dto.CanonicalUrl && reference.Kind == "details"); - advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details"); - advisory.AffectedPackages.Should().HaveCount(2); - advisory.Provenance.Should().ContainSingle(p => p.Source == CccsConnectorPlugin.SourceName && p.Kind == "advisory"); - } -} +using System; +using FluentAssertions; +using StellaOps.Concelier.Connector.Cccs.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Storage.Mongo.Documents; +using Xunit; + +namespace StellaOps.Concelier.Connector.Cccs.Tests.Internal; + +public sealed class CccsMapperTests +{ + [Fact] + public void Map_CreatesCanonicalAdvisory() + { + var raw = CccsHtmlParserTests.LoadFixture("cccs-raw-advisory.json"); + var dto = new CccsHtmlParser(new HtmlContentSanitizer()).Parse(raw); + var document = new DocumentRecord( + Guid.NewGuid(), + CccsConnectorPlugin.SourceName, + dto.CanonicalUrl, + DateTimeOffset.UtcNow, + "sha-test", + DocumentStatuses.PendingMap, + "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: dto.Modified, + GridFsId: null); + + var recordedAt = DateTimeOffset.Parse("2025-08-12T00:00:00Z"); + var advisory = CccsMapper.Map(dto, document, recordedAt); + + advisory.AdvisoryKey.Should().Be("TEST-001"); + advisory.Title.Should().Be(dto.Title); + advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" }); + advisory.References.Should().Contain(reference => reference.Url == dto.CanonicalUrl && reference.Kind == "details"); + advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details"); + advisory.AffectedPackages.Should().HaveCount(2); + advisory.Provenance.Should().ContainSingle(p => p.Source == CccsConnectorPlugin.SourceName && p.Kind == "advisory"); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj b/src/StellaOps.Concelier.Connector.Cccs.Tests/StellaOps.Concelier.Connector.Cccs.Tests.csproj similarity index 65% rename from src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj rename to src/StellaOps.Concelier.Connector.Cccs.Tests/StellaOps.Concelier.Connector.Cccs.Tests.csproj index f6da87b5..65052fa2 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/StellaOps.Feedser.Source.CertCc.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.Cccs.Tests/StellaOps.Concelier.Connector.Cccs.Tests.csproj @@ -1,19 +1,19 @@ - - - net10.0 - enable - enable - - - - - - - - - - - PreserveNewest - - - + + + net10.0 + enable + enable + + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Cccs/AGENTS.md b/src/StellaOps.Concelier.Connector.Cccs/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.Cccs/AGENTS.md rename to src/StellaOps.Concelier.Connector.Cccs/AGENTS.md index f64aaef7..a5c48edf 100644 --- a/src/StellaOps.Feedser.Source.Cccs/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Cccs/AGENTS.md @@ -1,40 +1,40 @@ -# AGENTS -## Role -Build the CCCS (Canadian Centre for Cyber Security) advisories connector so Feedser can ingest national cyber bulletins alongside other vendor/regional sources. - -## Scope -- Research CCCS advisory feeds (RSS/Atom, JSON API, or HTML listings) and define the canonical fetch workflow. -- Implement fetch, parse, and mapping stages with deterministic cursoring and retry/backoff behaviour. -- Normalise advisory content (summary, affected vendors/products, mitigation guidance, references, CVE IDs). -- Emit canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. -- Provide fixtures and regression tests to keep the connector deterministic. - -## Participants -- `Source.Common` (HTTP clients, fetch service, DTO storage helpers). -- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). -- `Feedser.Models` (canonical advisory data structures). -- `Feedser.Testing` (integration fixtures and snapshot utilities). - -## Interfaces & Contracts -- Job kinds: `cccs:fetch`, `cccs:parse`, `cccs:map`. -- Persist ETag/Last-Modified metadata when the upstream supports it. -- Include alias entries for CCCS advisory IDs plus referenced CVE IDs. - -## In/Out of scope -In scope: -- End-to-end connector implementation with range primitive coverage for affected packages. -- Minimal telemetry logging/counters matching other connectors. - -Out of scope: -- Automated remediation actions or vendor-specific enrichment beyond CCCS published data. -- Export or downstream pipeline changes. - -## Observability & Security Expectations -- Log fetch attempts, success/failure counts, and mapping statistics. -- Sanitize HTML safely, dropping scripts/styles before storing DTOs. -- Respect upstream rate limits; mark failures in source state with backoff. - -## Tests -- Add `StellaOps.Feedser.Source.Cccs.Tests` covering fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Validate deterministic ordering and timestamps to maintain reproducibility. +# AGENTS +## Role +Build the CCCS (Canadian Centre for Cyber Security) advisories connector so Concelier can ingest national cyber bulletins alongside other vendor/regional sources. + +## Scope +- Research CCCS advisory feeds (RSS/Atom, JSON API, or HTML listings) and define the canonical fetch workflow. +- Implement fetch, parse, and mapping stages with deterministic cursoring and retry/backoff behaviour. +- Normalise advisory content (summary, affected vendors/products, mitigation guidance, references, CVE IDs). +- Emit canonical `Advisory` records with aliases, references, affected packages, and provenance metadata. +- Provide fixtures and regression tests to keep the connector deterministic. + +## Participants +- `Source.Common` (HTTP clients, fetch service, DTO storage helpers). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Concelier.Models` (canonical advisory data structures). +- `Concelier.Testing` (integration fixtures and snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `cccs:fetch`, `cccs:parse`, `cccs:map`. +- Persist ETag/Last-Modified metadata when the upstream supports it. +- Include alias entries for CCCS advisory IDs plus referenced CVE IDs. + +## In/Out of scope +In scope: +- End-to-end connector implementation with range primitive coverage for affected packages. +- Minimal telemetry logging/counters matching other connectors. + +Out of scope: +- Automated remediation actions or vendor-specific enrichment beyond CCCS published data. +- Export or downstream pipeline changes. + +## Observability & Security Expectations +- Log fetch attempts, success/failure counts, and mapping statistics. +- Sanitize HTML safely, dropping scripts/styles before storing DTOs. +- Respect upstream rate limits; mark failures in source state with backoff. + +## Tests +- Add `StellaOps.Concelier.Connector.Cccs.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Validate deterministic ordering and timestamps to maintain reproducibility. diff --git a/src/StellaOps.Feedser.Source.Cccs/CccsConnector.cs b/src/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Cccs/CccsConnector.cs rename to src/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs index 02595be9..39c4ef2e 100644 --- a/src/StellaOps.Feedser.Source.Cccs/CccsConnector.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/CccsConnector.cs @@ -1,606 +1,606 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using System.Globalization; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Cccs.Configuration; -using StellaOps.Feedser.Source.Cccs.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Cccs; - -public sealed class CccsConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private const string DtoSchemaVersion = "cccs.dto.v1"; - - private readonly CccsFeedClient _feedClient; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly CccsHtmlParser _htmlParser; - private readonly CccsDiagnostics _diagnostics; - private readonly CccsOptions _options; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - - public CccsConnector( - CccsFeedClient feedClient, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - CccsHtmlParser htmlParser, - CccsDiagnostics diagnostics, - IOptions options, - TimeProvider? timeProvider, - ILogger logger) - { - _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _htmlParser = htmlParser ?? throw new ArgumentNullException(nameof(htmlParser)); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => CccsConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var now = _timeProvider.GetUtcNow(); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var pendingDocuments = new HashSet(cursor.PendingDocuments); - var pendingMappings = new HashSet(cursor.PendingMappings); - var knownHashes = new Dictionary(cursor.KnownEntryHashes, StringComparer.Ordinal); - var feedsProcessed = 0; - var totalItems = 0; - var added = 0; - var unchanged = 0; - - try - { - foreach (var feed in _options.Feeds) - { - cancellationToken.ThrowIfCancellationRequested(); - - _diagnostics.FetchAttempt(); - var result = await _feedClient.FetchAsync(feed, _options.RequestTimeout, cancellationToken).ConfigureAwait(false); - feedsProcessed++; - totalItems += result.Items.Count; - - if (result.Items.Count == 0) - { - _diagnostics.FetchSuccess(); - await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false); - continue; - } - - var items = result.Items - .Where(static item => !string.IsNullOrWhiteSpace(item.Title)) - .OrderByDescending(item => ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified) ?? DateTimeOffset.MinValue) - .ThenByDescending(item => ParseDate(item.DateCreated) ?? DateTimeOffset.MinValue) - .ToList(); - - foreach (var item in items) - { - cancellationToken.ThrowIfCancellationRequested(); - - var documentUri = BuildDocumentUri(item, feed); - var rawDocument = CreateRawDocument(item, feed, result.AlertTypes); - var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions); - var sha = ComputeSha256(payload); - - if (knownHashes.TryGetValue(documentUri, out var existingHash) - && string.Equals(existingHash, sha, StringComparison.Ordinal)) - { - unchanged++; - _diagnostics.FetchUnchanged(); - continue; - } - - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); - if (existing is not null - && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase) - && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) - { - knownHashes[documentUri] = sha; - unchanged++; - _diagnostics.FetchUnchanged(); - continue; - } - - var gridFsId = await _rawDocumentStorage.UploadAsync( - SourceName, - documentUri, - payload, - "application/json", - expiresAt: null, - cancellationToken).ConfigureAwait(false); - - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["cccs.language"] = rawDocument.Language, - ["cccs.sourceId"] = rawDocument.SourceId, - }; - - if (!string.IsNullOrWhiteSpace(rawDocument.SerialNumber)) - { - metadata["cccs.serialNumber"] = rawDocument.SerialNumber!; - } - - if (!string.IsNullOrWhiteSpace(rawDocument.AlertType)) - { - metadata["cccs.alertType"] = rawDocument.AlertType!; - } - - var recordId = existing?.Id ?? Guid.NewGuid(); - var record = new DocumentRecord( - recordId, - SourceName, - documentUri, - now, - sha, - DocumentStatuses.PendingParse, - "application/json", - Headers: null, - Metadata: metadata, - Etag: null, - LastModified: rawDocument.Modified ?? rawDocument.Published ?? result.LastModifiedUtc, - GridFsId: gridFsId, - ExpiresAt: null); - - var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); - pendingDocuments.Add(upserted.Id); - pendingMappings.Remove(upserted.Id); - knownHashes[documentUri] = sha; - added++; - _diagnostics.FetchDocument(); - - if (added >= _options.MaxEntriesPerFetch) - { - break; - } - } - - _diagnostics.FetchSuccess(); - await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false); - - if (added >= _options.MaxEntriesPerFetch) - { - break; - } - } - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException) - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "CCCS fetch failed"); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - var trimmedHashes = TrimKnownHashes(knownHashes, _options.MaxKnownEntries); - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownEntryHashes(trimmedHashes) - .WithLastFetch(now); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - _logger.LogInformation( - "CCCS fetch completed feeds={Feeds} items={Items} newDocuments={Added} unchanged={Unchanged} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}", - feedsProcessed, - totalItems, - added, - unchanged, - pendingDocuments.Count, - pendingMappings.Count); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToList(); - var now = _timeProvider.GetUtcNow(); - var parsed = 0; - var parseFailures = 0; - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(); - parseFailures++; - continue; - } - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(); - _logger.LogWarning("CCCS document {DocumentId} missing GridFS payload", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - parseFailures++; - continue; - } - - byte[] payload; - try - { - payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogError(ex, "CCCS unable to download raw document {DocumentId}", documentId); - throw; - } - - CccsRawAdvisoryDocument? raw; - try - { - raw = JsonSerializer.Deserialize(payload, RawSerializerOptions); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogWarning(ex, "CCCS failed to deserialize raw document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - parseFailures++; - continue; - } - - if (raw is null) - { - _diagnostics.ParseFailure(); - _logger.LogWarning("CCCS raw document {DocumentId} produced null payload", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - parseFailures++; - continue; - } - - CccsAdvisoryDto dto; - try - { - dto = _htmlParser.Parse(raw); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogWarning(ex, "CCCS failed to parse advisory DTO for {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - parseFailures++; - continue; - } - - var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions); - var dtoBson = BsonDocument.Parse(dtoJson); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now); - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - pendingDocuments.Remove(documentId); - if (!pendingMappings.Contains(documentId)) - { - pendingMappings.Add(documentId); - } - _diagnostics.ParseSuccess(); - parsed++; - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - if (parsed > 0 || parseFailures > 0) - { - _logger.LogInformation( - "CCCS parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}", - parsed, - parseFailures, - pendingDocuments.Count, - pendingMappings.Count); - } - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToList(); - var mapped = 0; - var mappingFailures = 0; - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingMappings.Remove(documentId); - _diagnostics.MapFailure(); - mappingFailures++; - continue; - } - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - _diagnostics.MapFailure(); - _logger.LogWarning("CCCS document {DocumentId} missing DTO payload", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - mappingFailures++; - continue; - } - - CccsAdvisoryDto? dto; - try - { - var json = dtoRecord.Payload.ToJson(); - dto = JsonSerializer.Deserialize(json, DtoSerializerOptions); - } - catch (Exception ex) - { - _diagnostics.MapFailure(); - _logger.LogWarning(ex, "CCCS failed to deserialize DTO for document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - mappingFailures++; - continue; - } - - if (dto is null) - { - _diagnostics.MapFailure(); - _logger.LogWarning("CCCS DTO for document {DocumentId} evaluated to null", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - mappingFailures++; - continue; - } - - try - { - var advisory = CccsMapper.Map(dto, document, dtoRecord.ValidatedAt); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - _diagnostics.MapSuccess(); - mapped++; - } - catch (Exception ex) - { - _diagnostics.MapFailure(); - _logger.LogError(ex, "CCCS mapping failed for document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - mappingFailures++; - } - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - if (mapped > 0 || mappingFailures > 0) - { - _logger.LogInformation( - "CCCS map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}", - mapped, - mappingFailures, - pendingMappings.Count); - } - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? CccsCursor.Empty : CccsCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBsonDocument(); - var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow(); - return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); - } - - private async Task DelayBetweenRequestsAsync(CancellationToken cancellationToken) - { - if (_options.RequestDelay <= TimeSpan.Zero) - { - return; - } - - try - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - // Ignore cancellation during delay; caller handles. - } - } - - private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed) - { - if (!string.IsNullOrWhiteSpace(item.Url)) - { - if (Uri.TryCreate(item.Url, UriKind.Absolute, out var absolute)) - { - return absolute.ToString(); - } - - var baseUri = new Uri("https://www.cyber.gc.ca", UriKind.Absolute); - if (Uri.TryCreate(baseUri, item.Url, out var combined)) - { - return combined.ToString(); - } - } - - return $"https://www.cyber.gc.ca/api/cccs/threats/{feed.Language}/{item.Nid}"; - } - - private static CccsRawAdvisoryDocument CreateRawDocument(CccsFeedItem item, CccsFeedEndpoint feed, IReadOnlyDictionary taxonomy) - { - var language = string.IsNullOrWhiteSpace(item.Language) ? feed.Language : item.Language!.Trim(); - var identifier = !string.IsNullOrWhiteSpace(item.SerialNumber) - ? item.SerialNumber!.Trim() - : !string.IsNullOrWhiteSpace(item.Uuid) - ? item.Uuid!.Trim() - : $"nid-{item.Nid}"; - - var canonicalUrl = BuildDocumentUri(item, feed); - var bodySegments = item.Body ?? Array.Empty(); - var bodyHtml = string.Join(Environment.NewLine, bodySegments); - var published = ParseDate(item.DateCreated); - var modified = ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified); - var alertType = ResolveAlertType(item, taxonomy); - - return new CccsRawAdvisoryDocument - { - SourceId = identifier, - SerialNumber = item.SerialNumber?.Trim(), - Uuid = item.Uuid, - Language = language.ToLowerInvariant(), - Title = item.Title?.Trim() ?? identifier, - Summary = item.Summary?.Trim(), - CanonicalUrl = canonicalUrl, - ExternalUrl = item.ExternalUrl, - BodyHtml = bodyHtml, - BodySegments = bodySegments, - AlertType = alertType, - Subject = item.Subject, - Banner = item.Banner, - Published = published, - Modified = modified, - RawDateCreated = item.DateCreated, - RawDateModified = item.DateModifiedTimestamp ?? item.DateModified, - }; - } - - private static string? ResolveAlertType(CccsFeedItem item, IReadOnlyDictionary taxonomy) - { - if (item.AlertType.ValueKind == JsonValueKind.Number) - { - var id = item.AlertType.GetInt32(); - return taxonomy.TryGetValue(id, out var label) ? label : id.ToString(CultureInfo.InvariantCulture); - } - - if (item.AlertType.ValueKind == JsonValueKind.String) - { - return item.AlertType.GetString(); - } - - if (item.AlertType.ValueKind == JsonValueKind.Array) - { - foreach (var element in item.AlertType.EnumerateArray()) - { - if (element.ValueKind == JsonValueKind.Number) - { - var id = element.GetInt32(); - if (taxonomy.TryGetValue(id, out var label)) - { - return label; - } - } - else if (element.ValueKind == JsonValueKind.String) - { - var label = element.GetString(); - if (!string.IsNullOrWhiteSpace(label)) - { - return label; - } - } - } - } - - return null; - } - - private static Dictionary TrimKnownHashes(Dictionary hashes, int maxEntries) - { - if (hashes.Count <= maxEntries) - { - return hashes; - } - - var overflow = hashes.Count - maxEntries; - foreach (var key in hashes.Keys.Take(overflow).ToList()) - { - hashes.Remove(key); - } - - return hashes; - } - - private static DateTimeOffset? ParseDate(string? value) - => string.IsNullOrWhiteSpace(value) - ? null - : DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) - ? parsed - : null; - - private static string ComputeSha256(byte[] payload) - => Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using System.Globalization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Cccs.Configuration; +using StellaOps.Concelier.Connector.Cccs.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Cccs; + +public sealed class CccsConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private const string DtoSchemaVersion = "cccs.dto.v1"; + + private readonly CccsFeedClient _feedClient; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CccsHtmlParser _htmlParser; + private readonly CccsDiagnostics _diagnostics; + private readonly CccsOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public CccsConnector( + CccsFeedClient feedClient, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + CccsHtmlParser htmlParser, + CccsDiagnostics diagnostics, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _htmlParser = htmlParser ?? throw new ArgumentNullException(nameof(htmlParser)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CccsConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = new HashSet(cursor.PendingDocuments); + var pendingMappings = new HashSet(cursor.PendingMappings); + var knownHashes = new Dictionary(cursor.KnownEntryHashes, StringComparer.Ordinal); + var feedsProcessed = 0; + var totalItems = 0; + var added = 0; + var unchanged = 0; + + try + { + foreach (var feed in _options.Feeds) + { + cancellationToken.ThrowIfCancellationRequested(); + + _diagnostics.FetchAttempt(); + var result = await _feedClient.FetchAsync(feed, _options.RequestTimeout, cancellationToken).ConfigureAwait(false); + feedsProcessed++; + totalItems += result.Items.Count; + + if (result.Items.Count == 0) + { + _diagnostics.FetchSuccess(); + await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false); + continue; + } + + var items = result.Items + .Where(static item => !string.IsNullOrWhiteSpace(item.Title)) + .OrderByDescending(item => ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified) ?? DateTimeOffset.MinValue) + .ThenByDescending(item => ParseDate(item.DateCreated) ?? DateTimeOffset.MinValue) + .ToList(); + + foreach (var item in items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var documentUri = BuildDocumentUri(item, feed); + var rawDocument = CreateRawDocument(item, feed, result.AlertTypes); + var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions); + var sha = ComputeSha256(payload); + + if (knownHashes.TryGetValue(documentUri, out var existingHash) + && string.Equals(existingHash, sha, StringComparison.Ordinal)) + { + unchanged++; + _diagnostics.FetchUnchanged(); + continue; + } + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); + if (existing is not null + && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase) + && string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal)) + { + knownHashes[documentUri] = sha; + unchanged++; + _diagnostics.FetchUnchanged(); + continue; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync( + SourceName, + documentUri, + payload, + "application/json", + expiresAt: null, + cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["cccs.language"] = rawDocument.Language, + ["cccs.sourceId"] = rawDocument.SourceId, + }; + + if (!string.IsNullOrWhiteSpace(rawDocument.SerialNumber)) + { + metadata["cccs.serialNumber"] = rawDocument.SerialNumber!; + } + + if (!string.IsNullOrWhiteSpace(rawDocument.AlertType)) + { + metadata["cccs.alertType"] = rawDocument.AlertType!; + } + + var recordId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + recordId, + SourceName, + documentUri, + now, + sha, + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: rawDocument.Modified ?? rawDocument.Published ?? result.LastModifiedUtc, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + pendingDocuments.Add(upserted.Id); + pendingMappings.Remove(upserted.Id); + knownHashes[documentUri] = sha; + added++; + _diagnostics.FetchDocument(); + + if (added >= _options.MaxEntriesPerFetch) + { + break; + } + } + + _diagnostics.FetchSuccess(); + await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false); + + if (added >= _options.MaxEntriesPerFetch) + { + break; + } + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "CCCS fetch failed"); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var trimmedHashes = TrimKnownHashes(knownHashes, _options.MaxKnownEntries); + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownEntryHashes(trimmedHashes) + .WithLastFetch(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + _logger.LogInformation( + "CCCS fetch completed feeds={Feeds} items={Items} newDocuments={Added} unchanged={Unchanged} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}", + feedsProcessed, + totalItems, + added, + unchanged, + pendingDocuments.Count, + pendingMappings.Count); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + var now = _timeProvider.GetUtcNow(); + var parsed = 0; + var parseFailures = 0; + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(); + parseFailures++; + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("CCCS document {DocumentId} missing GridFS payload", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + parseFailures++; + continue; + } + + byte[] payload; + try + { + payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "CCCS unable to download raw document {DocumentId}", documentId); + throw; + } + + CccsRawAdvisoryDocument? raw; + try + { + raw = JsonSerializer.Deserialize(payload, RawSerializerOptions); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogWarning(ex, "CCCS failed to deserialize raw document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + parseFailures++; + continue; + } + + if (raw is null) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("CCCS raw document {DocumentId} produced null payload", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + parseFailures++; + continue; + } + + CccsAdvisoryDto dto; + try + { + dto = _htmlParser.Parse(raw); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogWarning(ex, "CCCS failed to parse advisory DTO for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + parseFailures++; + continue; + } + + var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions); + var dtoBson = BsonDocument.Parse(dtoJson); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + _diagnostics.ParseSuccess(); + parsed++; + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + if (parsed > 0 || parseFailures > 0) + { + _logger.LogInformation( + "CCCS parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}", + parsed, + parseFailures, + pendingDocuments.Count, + pendingMappings.Count); + } + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + var mapped = 0; + var mappingFailures = 0; + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingMappings.Remove(documentId); + _diagnostics.MapFailure(); + mappingFailures++; + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + _diagnostics.MapFailure(); + _logger.LogWarning("CCCS document {DocumentId} missing DTO payload", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + mappingFailures++; + continue; + } + + CccsAdvisoryDto? dto; + try + { + var json = dtoRecord.Payload.ToJson(); + dto = JsonSerializer.Deserialize(json, DtoSerializerOptions); + } + catch (Exception ex) + { + _diagnostics.MapFailure(); + _logger.LogWarning(ex, "CCCS failed to deserialize DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + mappingFailures++; + continue; + } + + if (dto is null) + { + _diagnostics.MapFailure(); + _logger.LogWarning("CCCS DTO for document {DocumentId} evaluated to null", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + mappingFailures++; + continue; + } + + try + { + var advisory = CccsMapper.Map(dto, document, dtoRecord.ValidatedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(); + mapped++; + } + catch (Exception ex) + { + _diagnostics.MapFailure(); + _logger.LogError(ex, "CCCS mapping failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + mappingFailures++; + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + if (mapped > 0 || mappingFailures > 0) + { + _logger.LogInformation( + "CCCS map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}", + mapped, + mappingFailures, + pendingMappings.Count); + } + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? CccsCursor.Empty : CccsCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow(); + return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); + } + + private async Task DelayBetweenRequestsAsync(CancellationToken cancellationToken) + { + if (_options.RequestDelay <= TimeSpan.Zero) + { + return; + } + + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + // Ignore cancellation during delay; caller handles. + } + } + + private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed) + { + if (!string.IsNullOrWhiteSpace(item.Url)) + { + if (Uri.TryCreate(item.Url, UriKind.Absolute, out var absolute)) + { + return absolute.ToString(); + } + + var baseUri = new Uri("https://www.cyber.gc.ca", UriKind.Absolute); + if (Uri.TryCreate(baseUri, item.Url, out var combined)) + { + return combined.ToString(); + } + } + + return $"https://www.cyber.gc.ca/api/cccs/threats/{feed.Language}/{item.Nid}"; + } + + private static CccsRawAdvisoryDocument CreateRawDocument(CccsFeedItem item, CccsFeedEndpoint feed, IReadOnlyDictionary taxonomy) + { + var language = string.IsNullOrWhiteSpace(item.Language) ? feed.Language : item.Language!.Trim(); + var identifier = !string.IsNullOrWhiteSpace(item.SerialNumber) + ? item.SerialNumber!.Trim() + : !string.IsNullOrWhiteSpace(item.Uuid) + ? item.Uuid!.Trim() + : $"nid-{item.Nid}"; + + var canonicalUrl = BuildDocumentUri(item, feed); + var bodySegments = item.Body ?? Array.Empty(); + var bodyHtml = string.Join(Environment.NewLine, bodySegments); + var published = ParseDate(item.DateCreated); + var modified = ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified); + var alertType = ResolveAlertType(item, taxonomy); + + return new CccsRawAdvisoryDocument + { + SourceId = identifier, + SerialNumber = item.SerialNumber?.Trim(), + Uuid = item.Uuid, + Language = language.ToLowerInvariant(), + Title = item.Title?.Trim() ?? identifier, + Summary = item.Summary?.Trim(), + CanonicalUrl = canonicalUrl, + ExternalUrl = item.ExternalUrl, + BodyHtml = bodyHtml, + BodySegments = bodySegments, + AlertType = alertType, + Subject = item.Subject, + Banner = item.Banner, + Published = published, + Modified = modified, + RawDateCreated = item.DateCreated, + RawDateModified = item.DateModifiedTimestamp ?? item.DateModified, + }; + } + + private static string? ResolveAlertType(CccsFeedItem item, IReadOnlyDictionary taxonomy) + { + if (item.AlertType.ValueKind == JsonValueKind.Number) + { + var id = item.AlertType.GetInt32(); + return taxonomy.TryGetValue(id, out var label) ? label : id.ToString(CultureInfo.InvariantCulture); + } + + if (item.AlertType.ValueKind == JsonValueKind.String) + { + return item.AlertType.GetString(); + } + + if (item.AlertType.ValueKind == JsonValueKind.Array) + { + foreach (var element in item.AlertType.EnumerateArray()) + { + if (element.ValueKind == JsonValueKind.Number) + { + var id = element.GetInt32(); + if (taxonomy.TryGetValue(id, out var label)) + { + return label; + } + } + else if (element.ValueKind == JsonValueKind.String) + { + var label = element.GetString(); + if (!string.IsNullOrWhiteSpace(label)) + { + return label; + } + } + } + } + + return null; + } + + private static Dictionary TrimKnownHashes(Dictionary hashes, int maxEntries) + { + if (hashes.Count <= maxEntries) + { + return hashes; + } + + var overflow = hashes.Count - maxEntries; + foreach (var key in hashes.Keys.Take(overflow).ToList()) + { + hashes.Remove(key); + } + + return hashes; + } + + private static DateTimeOffset? ParseDate(string? value) + => string.IsNullOrWhiteSpace(value) + ? null + : DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed + : null; + + private static string ComputeSha256(byte[] payload) + => Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); +} diff --git a/src/StellaOps.Feedser.Source.Cccs/CccsConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Cccs/CccsConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Cccs/CccsConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Cccs/CccsConnectorPlugin.cs index 2bba9dc4..02998126 100644 --- a/src/StellaOps.Feedser.Source.Cccs/CccsConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/CccsConnectorPlugin.cs @@ -1,21 +1,21 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Cccs; - -public sealed class CccsConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "cccs"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) - => services.GetService() is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetRequiredService(); - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Cccs; + +public sealed class CccsConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cccs"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/CccsDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Cccs/CccsDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs index a6812e0f..e3f243b6 100644 --- a/src/StellaOps.Feedser.Source.Cccs/CccsDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/CccsDependencyInjectionRoutine.cs @@ -1,50 +1,50 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Cccs.Configuration; - -namespace StellaOps.Feedser.Source.Cccs; - -public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:cccs"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddCccsConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Cccs.Configuration; + +namespace StellaOps.Concelier.Connector.Cccs; + +public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:cccs"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCccsConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/CccsServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Cccs/CccsServiceCollectionExtensions.cs similarity index 80% rename from src/StellaOps.Feedser.Source.Cccs/CccsServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Cccs/CccsServiceCollectionExtensions.cs index adb3c7d3..b042d2fe 100644 --- a/src/StellaOps.Feedser.Source.Cccs/CccsServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/CccsServiceCollectionExtensions.cs @@ -1,47 +1,47 @@ -using System; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Cccs.Configuration; -using StellaOps.Feedser.Source.Cccs.Internal; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.Cccs; - -public static class CccsServiceCollectionExtensions -{ - public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); - - services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.UserAgent = "StellaOps.Feedser.Cccs/1.0"; - clientOptions.Timeout = options.RequestTimeout; - clientOptions.AllowedHosts.Clear(); - - foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null)) - { - clientOptions.AllowedHosts.Add(feed.Uri!.Host); - } - - clientOptions.AllowedHosts.Add("www.cyber.gc.ca"); - clientOptions.AllowedHosts.Add("cyber.gc.ca"); - }); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.AddTransient(); - return services; - } -} +using System; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Cccs.Configuration; +using StellaOps.Concelier.Connector.Cccs.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.Cccs; + +public static class CccsServiceCollectionExtensions +{ + public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.UserAgent = "StellaOps.Concelier.Cccs/1.0"; + clientOptions.Timeout = options.RequestTimeout; + clientOptions.AllowedHosts.Clear(); + + foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null)) + { + clientOptions.AllowedHosts.Add(feed.Uri!.Host); + } + + clientOptions.AllowedHosts.Add("www.cyber.gc.ca"); + clientOptions.AllowedHosts.Add("cyber.gc.ca"); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Configuration/CccsOptions.cs b/src/StellaOps.Concelier.Connector.Cccs/Configuration/CccsOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Cccs/Configuration/CccsOptions.cs rename to src/StellaOps.Concelier.Connector.Cccs/Configuration/CccsOptions.cs index 16c2fa8f..321af8f4 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Configuration/CccsOptions.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Configuration/CccsOptions.cs @@ -1,175 +1,175 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Cccs.Configuration; - -public sealed class CccsOptions -{ - public const string HttpClientName = "feedser.source.cccs"; - - private readonly List _feeds = new(); - - public CccsOptions() - { - _feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"))); - _feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat"))); - } - - /// - /// Feed endpoints to poll; configure per language or content category. - /// - public IList Feeds => _feeds; - - /// - /// Maximum number of entries to enqueue per fetch cycle. - /// - public int MaxEntriesPerFetch { get; set; } = 80; - - /// - /// Maximum remembered entries (URI+hash) for deduplication. - /// - public int MaxKnownEntries { get; set; } = 512; - - /// - /// Timeout applied to feed and taxonomy requests. - /// - public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); - - /// - /// Delay between successive feed requests to respect upstream throttling. - /// - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - /// - /// Backoff recorded in source state when fetch fails. - /// - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1); - - public void Validate() - { - if (_feeds.Count == 0) - { - throw new InvalidOperationException("At least one CCCS feed endpoint must be configured."); - } - - var seenLanguages = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var feed in _feeds) - { - feed.Validate(); - if (!seenLanguages.Add(feed.Language)) - { - throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion."); - } - } - - if (MaxEntriesPerFetch <= 0) - { - throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero."); - } - - if (MaxKnownEntries <= 0) - { - throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); - } - } -} - -public sealed class CccsFeedEndpoint -{ - public CccsFeedEndpoint() - { - } - - public CccsFeedEndpoint(string language, Uri uri) - { - Language = language; - Uri = uri; - } - - public string Language { get; set; } = "en"; - - public Uri? Uri { get; set; } - - public void Validate() - { - if (string.IsNullOrWhiteSpace(Language)) - { - throw new InvalidOperationException("CCCS feed language must be specified."); - } - - if (Uri is null || !Uri.IsAbsoluteUri) - { - throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}')."); - } - } - - public Uri BuildTaxonomyUri() - { - if (Uri is null) - { - throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI."); - } - - var language = Uri.GetQueryParameterValueOrDefault("lang", Language); - var builder = $"https://www.cyber.gc.ca/api/cccs/taxonomy/v1/get?lang={language}&vocabulary=cccs_alert_type"; - return new Uri(builder, UriKind.Absolute); - } -} - -internal static class CccsUriExtensions -{ - public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback) - { - if (uri is null) - { - return fallback; - } - - var query = uri.Query; - if (string.IsNullOrEmpty(query)) - { - return fallback; - } - - var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query; - foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries)) - { - var separatorIndex = pair.IndexOf('='); - if (separatorIndex < 0) - { - continue; - } - - var left = pair[..separatorIndex].Trim(); - if (!left.Equals(key, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var right = pair[(separatorIndex + 1)..].Trim(); - if (right.Length == 0) - { - continue; - } - - return Uri.UnescapeDataString(right); - } - - return fallback; - } -} +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Cccs.Configuration; + +public sealed class CccsOptions +{ + public const string HttpClientName = "concelier.source.cccs"; + + private readonly List _feeds = new(); + + public CccsOptions() + { + _feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat"))); + _feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat"))); + } + + /// + /// Feed endpoints to poll; configure per language or content category. + /// + public IList Feeds => _feeds; + + /// + /// Maximum number of entries to enqueue per fetch cycle. + /// + public int MaxEntriesPerFetch { get; set; } = 80; + + /// + /// Maximum remembered entries (URI+hash) for deduplication. + /// + public int MaxKnownEntries { get; set; } = 512; + + /// + /// Timeout applied to feed and taxonomy requests. + /// + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Delay between successive feed requests to respect upstream throttling. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Backoff recorded in source state when fetch fails. + /// + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1); + + public void Validate() + { + if (_feeds.Count == 0) + { + throw new InvalidOperationException("At least one CCCS feed endpoint must be configured."); + } + + var seenLanguages = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var feed in _feeds) + { + feed.Validate(); + if (!seenLanguages.Add(feed.Language)) + { + throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion."); + } + } + + if (MaxEntriesPerFetch <= 0) + { + throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero."); + } + + if (MaxKnownEntries <= 0) + { + throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); + } + } +} + +public sealed class CccsFeedEndpoint +{ + public CccsFeedEndpoint() + { + } + + public CccsFeedEndpoint(string language, Uri uri) + { + Language = language; + Uri = uri; + } + + public string Language { get; set; } = "en"; + + public Uri? Uri { get; set; } + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Language)) + { + throw new InvalidOperationException("CCCS feed language must be specified."); + } + + if (Uri is null || !Uri.IsAbsoluteUri) + { + throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}')."); + } + } + + public Uri BuildTaxonomyUri() + { + if (Uri is null) + { + throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI."); + } + + var language = Uri.GetQueryParameterValueOrDefault("lang", Language); + var builder = $"https://www.cyber.gc.ca/api/cccs/taxonomy/v1/get?lang={language}&vocabulary=cccs_alert_type"; + return new Uri(builder, UriKind.Absolute); + } +} + +internal static class CccsUriExtensions +{ + public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback) + { + if (uri is null) + { + return fallback; + } + + var query = uri.Query; + if (string.IsNullOrEmpty(query)) + { + return fallback; + } + + var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query; + foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries)) + { + var separatorIndex = pair.IndexOf('='); + if (separatorIndex < 0) + { + continue; + } + + var left = pair[..separatorIndex].Trim(); + if (!left.Equals(key, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var right = pair[(separatorIndex + 1)..].Trim(); + if (right.Length == 0) + { + continue; + } + + return Uri.UnescapeDataString(right); + } + + return fallback; + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsAdvisoryDto.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsAdvisoryDto.cs index 2f31e3aa..b86b6bed 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsAdvisoryDto.cs @@ -1,54 +1,54 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -internal sealed record CccsAdvisoryDto -{ - [JsonPropertyName("sourceId")] - public string SourceId { get; init; } = string.Empty; - - [JsonPropertyName("serialNumber")] - public string SerialNumber { get; init; } = string.Empty; - - [JsonPropertyName("language")] - public string Language { get; init; } = "en"; - - [JsonPropertyName("title")] - public string Title { get; init; } = string.Empty; - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("canonicalUrl")] - public string CanonicalUrl { get; init; } = string.Empty; - - [JsonPropertyName("contentHtml")] - public string ContentHtml { get; init; } = string.Empty; - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("modified")] - public DateTimeOffset? Modified { get; init; } - - [JsonPropertyName("alertType")] - public string? AlertType { get; init; } - - [JsonPropertyName("subject")] - public string? Subject { get; init; } - - [JsonPropertyName("products")] - public IReadOnlyList Products { get; init; } = Array.Empty(); - - [JsonPropertyName("references")] - public IReadOnlyList References { get; init; } = Array.Empty(); - - [JsonPropertyName("cveIds")] - public IReadOnlyList CveIds { get; init; } = Array.Empty(); -} - -internal sealed record CccsReferenceDto( - [property: JsonPropertyName("url")] string Url, - [property: JsonPropertyName("label")] string? Label); +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +internal sealed record CccsAdvisoryDto +{ + [JsonPropertyName("sourceId")] + public string SourceId { get; init; } = string.Empty; + + [JsonPropertyName("serialNumber")] + public string SerialNumber { get; init; } = string.Empty; + + [JsonPropertyName("language")] + public string Language { get; init; } = "en"; + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("canonicalUrl")] + public string CanonicalUrl { get; init; } = string.Empty; + + [JsonPropertyName("contentHtml")] + public string ContentHtml { get; init; } = string.Empty; + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("alertType")] + public string? AlertType { get; init; } + + [JsonPropertyName("subject")] + public string? Subject { get; init; } + + [JsonPropertyName("products")] + public IReadOnlyList Products { get; init; } = Array.Empty(); + + [JsonPropertyName("references")] + public IReadOnlyList References { get; init; } = Array.Empty(); + + [JsonPropertyName("cveIds")] + public IReadOnlyList CveIds { get; init; } = Array.Empty(); +} + +internal sealed record CccsReferenceDto( + [property: JsonPropertyName("url")] string Url, + [property: JsonPropertyName("label")] string? Label); diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsCursor.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsCursor.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs index 7e50faac..e5e17211 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsCursor.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsCursor.cs @@ -1,145 +1,145 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -internal sealed record CccsCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - IReadOnlyDictionary KnownEntryHashes, - DateTimeOffset? LastFetchAt) -{ - private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); - private static readonly IReadOnlyDictionary EmptyHashes = new Dictionary(StringComparer.Ordinal); - - public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null); - - public CccsCursor WithPendingDocuments(IEnumerable documents) - { - var distinct = (documents ?? Enumerable.Empty()).Distinct().ToArray(); - return this with { PendingDocuments = distinct }; - } - - public CccsCursor WithPendingMappings(IEnumerable mappings) - { - var distinct = (mappings ?? Enumerable.Empty()).Distinct().ToArray(); - return this with { PendingMappings = distinct }; - } - - public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary hashes) - { - var map = hashes is null || hashes.Count == 0 - ? EmptyHashes - : new Dictionary(hashes, StringComparer.Ordinal); - return this with { KnownEntryHashes = map }; - } - - public CccsCursor WithLastFetch(DateTimeOffset? timestamp) - => this with { LastFetchAt = timestamp }; - - public BsonDocument ToBsonDocument() - { - var doc = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (KnownEntryHashes.Count > 0) - { - var hashes = new BsonArray(); - foreach (var kvp in KnownEntryHashes) - { - hashes.Add(new BsonDocument - { - ["uri"] = kvp.Key, - ["hash"] = kvp.Value, - }); - } - - doc["knownEntryHashes"] = hashes; - } - - if (LastFetchAt.HasValue) - { - doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; - } - - return doc; - } - - public static CccsCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var hashes = ReadHashMap(document); - var lastFetch = document.TryGetValue("lastFetchAt", out var value) - ? ParseDateTime(value) - : null; - - return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch); - } - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuidCollection; - } - - var items = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var guid)) - { - items.Add(guid); - } - } - - return items; - } - - private static IReadOnlyDictionary ReadHashMap(BsonDocument document) - { - if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0) - { - return EmptyHashes; - } - - var map = new Dictionary(array.Count, StringComparer.Ordinal); - foreach (var element in array) - { - if (element is not BsonDocument entry) - { - continue; - } - - if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString)) - { - continue; - } - - var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull - ? hashValue.AsString - : string.Empty; - map[uriValue.AsString] = hash; - } - - return map; - } - - private static DateTimeOffset? ParseDateTime(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +internal sealed record CccsCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyDictionary KnownEntryHashes, + DateTimeOffset? LastFetchAt) +{ + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + private static readonly IReadOnlyDictionary EmptyHashes = new Dictionary(StringComparer.Ordinal); + + public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null); + + public CccsCursor WithPendingDocuments(IEnumerable documents) + { + var distinct = (documents ?? Enumerable.Empty()).Distinct().ToArray(); + return this with { PendingDocuments = distinct }; + } + + public CccsCursor WithPendingMappings(IEnumerable mappings) + { + var distinct = (mappings ?? Enumerable.Empty()).Distinct().ToArray(); + return this with { PendingMappings = distinct }; + } + + public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary hashes) + { + var map = hashes is null || hashes.Count == 0 + ? EmptyHashes + : new Dictionary(hashes, StringComparer.Ordinal); + return this with { KnownEntryHashes = map }; + } + + public CccsCursor WithLastFetch(DateTimeOffset? timestamp) + => this with { LastFetchAt = timestamp }; + + public BsonDocument ToBsonDocument() + { + var doc = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (KnownEntryHashes.Count > 0) + { + var hashes = new BsonArray(); + foreach (var kvp in KnownEntryHashes) + { + hashes.Add(new BsonDocument + { + ["uri"] = kvp.Key, + ["hash"] = kvp.Value, + }); + } + + doc["knownEntryHashes"] = hashes; + } + + if (LastFetchAt.HasValue) + { + doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; + } + + return doc; + } + + public static CccsCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var hashes = ReadHashMap(document); + var lastFetch = document.TryGetValue("lastFetchAt", out var value) + ? ParseDateTime(value) + : null; + + return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidCollection; + } + + var items = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var guid)) + { + items.Add(guid); + } + } + + return items; + } + + private static IReadOnlyDictionary ReadHashMap(BsonDocument document) + { + if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyHashes; + } + + var map = new Dictionary(array.Count, StringComparer.Ordinal); + foreach (var element in array) + { + if (element is not BsonDocument entry) + { + continue; + } + + if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString)) + { + continue; + } + + var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull + ? hashValue.AsString + : string.Empty; + map[uriValue.AsString] = hash; + } + + return map; + } + + private static DateTimeOffset? ParseDateTime(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsDiagnostics.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs index 0dc1ae4f..ffe39888 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsDiagnostics.cs @@ -1,58 +1,58 @@ -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -public sealed class CccsDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.Cccs"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _fetchAttempts; - private readonly Counter _fetchSuccess; - private readonly Counter _fetchDocuments; - private readonly Counter _fetchUnchanged; - private readonly Counter _fetchFailures; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Counter _parseQuarantine; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - - public CccsDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchAttempts = _meter.CreateCounter("cccs.fetch.attempts", unit: "operations"); - _fetchSuccess = _meter.CreateCounter("cccs.fetch.success", unit: "operations"); - _fetchDocuments = _meter.CreateCounter("cccs.fetch.documents", unit: "documents"); - _fetchUnchanged = _meter.CreateCounter("cccs.fetch.unchanged", unit: "documents"); - _fetchFailures = _meter.CreateCounter("cccs.fetch.failures", unit: "operations"); - _parseSuccess = _meter.CreateCounter("cccs.parse.success", unit: "documents"); - _parseFailures = _meter.CreateCounter("cccs.parse.failures", unit: "documents"); - _parseQuarantine = _meter.CreateCounter("cccs.parse.quarantine", unit: "documents"); - _mapSuccess = _meter.CreateCounter("cccs.map.success", unit: "advisories"); - _mapFailures = _meter.CreateCounter("cccs.map.failures", unit: "advisories"); - } - - public void FetchAttempt() => _fetchAttempts.Add(1); - - public void FetchSuccess() => _fetchSuccess.Add(1); - - public void FetchDocument() => _fetchDocuments.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void ParseSuccess() => _parseSuccess.Add(1); - - public void ParseFailure() => _parseFailures.Add(1); - - public void ParseQuarantine() => _parseQuarantine.Add(1); - - public void MapSuccess() => _mapSuccess.Add(1); - - public void MapFailure() => _mapFailures.Add(1); - - public void Dispose() => _meter.Dispose(); -} +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +public sealed class CccsDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.Cccs"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchSuccess; + private readonly Counter _fetchDocuments; + private readonly Counter _fetchUnchanged; + private readonly Counter _fetchFailures; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _parseQuarantine; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + + public CccsDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter("cccs.fetch.attempts", unit: "operations"); + _fetchSuccess = _meter.CreateCounter("cccs.fetch.success", unit: "operations"); + _fetchDocuments = _meter.CreateCounter("cccs.fetch.documents", unit: "documents"); + _fetchUnchanged = _meter.CreateCounter("cccs.fetch.unchanged", unit: "documents"); + _fetchFailures = _meter.CreateCounter("cccs.fetch.failures", unit: "operations"); + _parseSuccess = _meter.CreateCounter("cccs.parse.success", unit: "documents"); + _parseFailures = _meter.CreateCounter("cccs.parse.failures", unit: "documents"); + _parseQuarantine = _meter.CreateCounter("cccs.parse.quarantine", unit: "documents"); + _mapSuccess = _meter.CreateCounter("cccs.map.success", unit: "advisories"); + _mapFailures = _meter.CreateCounter("cccs.map.failures", unit: "advisories"); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchSuccess() => _fetchSuccess.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess() => _mapSuccess.Add(1); + + public void MapFailure() => _mapFailures.Add(1); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedClient.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedClient.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs index b6914cc6..d8bf841b 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedClient.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedClient.cs @@ -1,146 +1,146 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Source.Cccs.Configuration; -using StellaOps.Feedser.Source.Common.Fetch; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -public sealed class CccsFeedClient -{ - private static readonly string[] AcceptHeaders = - { - "application/json", - "application/vnd.api+json;q=0.9", - "text/json;q=0.8", - "application/*+json;q=0.7", - }; - - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private readonly SourceFetchService _fetchService; - private readonly ILogger _logger; - - public CccsFeedClient(SourceFetchService fetchService, ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - internal async Task FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(endpoint); - if (endpoint.Uri is null) - { - throw new InvalidOperationException("Feed endpoint URI must be configured."); - } - - var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri) - { - AcceptHeaders = AcceptHeaders, - TimeoutOverride = requestTimeout, - Metadata = new Dictionary(StringComparer.Ordinal) - { - ["cccs.language"] = endpoint.Language, - ["cccs.feedUri"] = endpoint.Uri.ToString(), - }, - }; - - try - { - var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - - if (!result.IsSuccess || result.Content is null) - { - _logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode); - return CccsFeedResult.Empty; - } - - var feedResponse = Deserialize(result.Content); - if (feedResponse is null || feedResponse.Error) - { - _logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri); - return CccsFeedResult.Empty; - } - - var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false); - var items = (IReadOnlyList)feedResponse.Response ?? Array.Empty(); - return new CccsFeedResult(items, taxonomy, result.LastModified); - } - catch (Exception ex) when (ex is JsonException or InvalidOperationException) - { - _logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri); - throw; - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - _logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri); - throw; - } - } - - private async Task> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken) - { - var taxonomyUri = endpoint.BuildTaxonomyUri(); - var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri) - { - AcceptHeaders = AcceptHeaders, - TimeoutOverride = timeout, - Metadata = new Dictionary(StringComparer.Ordinal) - { - ["cccs.language"] = endpoint.Language, - ["cccs.taxonomyUri"] = taxonomyUri.ToString(), - }, - }; - - try - { - var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - if (!result.IsSuccess || result.Content is null) - { - _logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri); - return new Dictionary(0); - } - - var taxonomyResponse = Deserialize(result.Content); - if (taxonomyResponse is null || taxonomyResponse.Error) - { - _logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri); - return new Dictionary(0); - } - - var map = new Dictionary(taxonomyResponse.Response.Count); - foreach (var item in taxonomyResponse.Response) - { - if (!string.IsNullOrWhiteSpace(item.Title)) - { - map[item.Id] = item.Title!; - } - } - - return map; - } - catch (Exception ex) when (ex is JsonException or InvalidOperationException) - { - _logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri); - return new Dictionary(0); - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - _logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri); - return new Dictionary(0); - } - } - - private static T? Deserialize(byte[] content) - => JsonSerializer.Deserialize(content, SerializerOptions); -} +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Connector.Cccs.Configuration; +using StellaOps.Concelier.Connector.Common.Fetch; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +public sealed class CccsFeedClient +{ + private static readonly string[] AcceptHeaders = + { + "application/json", + "application/vnd.api+json;q=0.9", + "text/json;q=0.8", + "application/*+json;q=0.7", + }; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly SourceFetchService _fetchService; + private readonly ILogger _logger; + + public CccsFeedClient(SourceFetchService fetchService, ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + internal async Task FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(endpoint); + if (endpoint.Uri is null) + { + throw new InvalidOperationException("Feed endpoint URI must be configured."); + } + + var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri) + { + AcceptHeaders = AcceptHeaders, + TimeoutOverride = requestTimeout, + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["cccs.language"] = endpoint.Language, + ["cccs.feedUri"] = endpoint.Uri.ToString(), + }, + }; + + try + { + var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + + if (!result.IsSuccess || result.Content is null) + { + _logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode); + return CccsFeedResult.Empty; + } + + var feedResponse = Deserialize(result.Content); + if (feedResponse is null || feedResponse.Error) + { + _logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri); + return CccsFeedResult.Empty; + } + + var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false); + var items = (IReadOnlyList)feedResponse.Response ?? Array.Empty(); + return new CccsFeedResult(items, taxonomy, result.LastModified); + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { + _logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri); + throw; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri); + throw; + } + } + + private async Task> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken) + { + var taxonomyUri = endpoint.BuildTaxonomyUri(); + var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri) + { + AcceptHeaders = AcceptHeaders, + TimeoutOverride = timeout, + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["cccs.language"] = endpoint.Language, + ["cccs.taxonomyUri"] = taxonomyUri.ToString(), + }, + }; + + try + { + var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Content is null) + { + _logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri); + return new Dictionary(0); + } + + var taxonomyResponse = Deserialize(result.Content); + if (taxonomyResponse is null || taxonomyResponse.Error) + { + _logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri); + return new Dictionary(0); + } + + var map = new Dictionary(taxonomyResponse.Response.Count); + foreach (var item in taxonomyResponse.Response) + { + if (!string.IsNullOrWhiteSpace(item.Title)) + { + map[item.Id] = item.Title!; + } + } + + return map; + } + catch (Exception ex) when (ex is JsonException or InvalidOperationException) + { + _logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri); + return new Dictionary(0); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri); + return new Dictionary(0); + } + } + + private static T? Deserialize(byte[] content) + => JsonSerializer.Deserialize(content, SerializerOptions); +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedModels.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedModels.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedModels.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedModels.cs index b3c44711..b59c4864 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedModels.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsFeedModels.cs @@ -1,101 +1,101 @@ -using System; -using System.Collections.Generic; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -internal sealed class CccsFeedResponse -{ - [JsonPropertyName("ERROR")] - public bool Error { get; init; } - - [JsonPropertyName("response")] - public List Response { get; init; } = new(); -} - -internal sealed class CccsFeedItem -{ - [JsonPropertyName("nid")] - public int Nid { get; init; } - - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("uuid")] - public string? Uuid { get; init; } - - [JsonPropertyName("banner")] - public string? Banner { get; init; } - - [JsonPropertyName("lang")] - public string? Language { get; init; } - - [JsonPropertyName("date_modified")] - public string? DateModified { get; init; } - - [JsonPropertyName("date_modified_ts")] - public string? DateModifiedTimestamp { get; init; } - - [JsonPropertyName("date_created")] - public string? DateCreated { get; init; } - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("body")] - public string[] Body { get; init; } = Array.Empty(); - - [JsonPropertyName("url")] - public string? Url { get; init; } - - [JsonPropertyName("alert_type")] - public JsonElement AlertType { get; init; } - - [JsonPropertyName("serial_number")] - public string? SerialNumber { get; init; } - - [JsonPropertyName("subject")] - public string? Subject { get; init; } - - [JsonPropertyName("moderation_state")] - public string? ModerationState { get; init; } - - [JsonPropertyName("external_url")] - public string? ExternalUrl { get; init; } -} - -internal sealed class CccsTaxonomyResponse -{ - [JsonPropertyName("ERROR")] - public bool Error { get; init; } - - [JsonPropertyName("response")] - public List Response { get; init; } = new(); -} - -internal sealed class CccsTaxonomyItem -{ - [JsonPropertyName("id")] - public int Id { get; init; } - - [JsonPropertyName("title")] - public string? Title { get; init; } -} - -internal sealed record CccsFeedResult( - IReadOnlyList Items, - IReadOnlyDictionary AlertTypes, - DateTimeOffset? LastModifiedUtc) -{ - public static CccsFeedResult Empty { get; } = new( - Array.Empty(), - new Dictionary(0), - null); -} - -internal static class CccsFeedResultExtensions -{ - public static CccsFeedResult ToResult(this IReadOnlyList items, DateTimeOffset? lastModified, IReadOnlyDictionary alertTypes) - => new(items, alertTypes, lastModified); -} +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +internal sealed class CccsFeedResponse +{ + [JsonPropertyName("ERROR")] + public bool Error { get; init; } + + [JsonPropertyName("response")] + public List Response { get; init; } = new(); +} + +internal sealed class CccsFeedItem +{ + [JsonPropertyName("nid")] + public int Nid { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("uuid")] + public string? Uuid { get; init; } + + [JsonPropertyName("banner")] + public string? Banner { get; init; } + + [JsonPropertyName("lang")] + public string? Language { get; init; } + + [JsonPropertyName("date_modified")] + public string? DateModified { get; init; } + + [JsonPropertyName("date_modified_ts")] + public string? DateModifiedTimestamp { get; init; } + + [JsonPropertyName("date_created")] + public string? DateCreated { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("body")] + public string[] Body { get; init; } = Array.Empty(); + + [JsonPropertyName("url")] + public string? Url { get; init; } + + [JsonPropertyName("alert_type")] + public JsonElement AlertType { get; init; } + + [JsonPropertyName("serial_number")] + public string? SerialNumber { get; init; } + + [JsonPropertyName("subject")] + public string? Subject { get; init; } + + [JsonPropertyName("moderation_state")] + public string? ModerationState { get; init; } + + [JsonPropertyName("external_url")] + public string? ExternalUrl { get; init; } +} + +internal sealed class CccsTaxonomyResponse +{ + [JsonPropertyName("ERROR")] + public bool Error { get; init; } + + [JsonPropertyName("response")] + public List Response { get; init; } = new(); +} + +internal sealed class CccsTaxonomyItem +{ + [JsonPropertyName("id")] + public int Id { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } +} + +internal sealed record CccsFeedResult( + IReadOnlyList Items, + IReadOnlyDictionary AlertTypes, + DateTimeOffset? LastModifiedUtc) +{ + public static CccsFeedResult Empty { get; } = new( + Array.Empty(), + new Dictionary(0), + null); +} + +internal static class CccsFeedResultExtensions +{ + public static CccsFeedResult ToResult(this IReadOnlyList items, DateTimeOffset? lastModified, IReadOnlyDictionary alertTypes) + => new(items, alertTypes, lastModified); +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsHtmlParser.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsHtmlParser.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs index 6b7236be..82f948c8 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsHtmlParser.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsHtmlParser.cs @@ -1,449 +1,449 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text.RegularExpressions; -using AngleSharp.Dom; -using AngleSharp.Html.Dom; -using AngleSharp.Html.Parser; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -public sealed class CccsHtmlParser -{ - private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled); - - private static readonly CultureInfo[] EnglishCultures = - { - CultureInfo.GetCultureInfo("en-CA"), - CultureInfo.GetCultureInfo("en-US"), - CultureInfo.InvariantCulture, - }; - - private static readonly CultureInfo[] FrenchCultures = - { - CultureInfo.GetCultureInfo("fr-CA"), - CultureInfo.GetCultureInfo("fr-FR"), - CultureInfo.InvariantCulture, - }; - - private static readonly string[] ProductHeadingKeywords = - { - "affected", - "produit", - "produits", - "produits touch", - "produits concern", - "mesures recommand", - }; - - private static readonly string[] TrackingParameterPrefixes = - { - "utm_", - "mc_", - "mkt_", - "elq", - }; - - private readonly HtmlContentSanitizer _sanitizer; - private readonly HtmlParser _parser; - - public CccsHtmlParser(HtmlContentSanitizer sanitizer) - { - _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); - _parser = new HtmlParser(new HtmlParserOptions - { - IsScripting = false, - IsKeepingSourceReferences = false, - }); - } - - internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw) - { - ArgumentNullException.ThrowIfNull(raw); - - var baseUri = TryCreateUri(raw.CanonicalUrl); - var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty); - var body = document.Body ?? document.DocumentElement; - var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri); - var contentRoot = body ?? document.DocumentElement; - - var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber) - ? raw.SerialNumber!.Trim() - : ExtractSerialNumber(document) ?? raw.SourceId; - - var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified; - var references = ExtractReferences(contentRoot, baseUri, raw.Language); - var products = ExtractProducts(contentRoot); - var cveIds = ExtractCveIds(document); - - return new CccsAdvisoryDto - { - SourceId = raw.SourceId, - SerialNumber = serialNumber, - Language = raw.Language, - Title = raw.Title, - Summary = CollapseWhitespace(raw.Summary), - CanonicalUrl = raw.CanonicalUrl, - ContentHtml = sanitized, - Published = published, - Modified = raw.Modified ?? published, - AlertType = raw.AlertType, - Subject = raw.Subject, - Products = products, - References = references, - CveIds = cveIds, - }; - } - - private static Uri? TryCreateUri(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null; - } - - private static string? ExtractSerialNumber(IDocument document) - { - if (document.Body is null) - { - return null; - } - - foreach (var element in document.QuerySelectorAll("strong, p, div")) - { - var text = element.TextContent; - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - var match = SerialRegex.Match(text); - if (match.Success && match.Groups["id"].Success) - { - var value = match.Groups["id"].Value.Trim(); - if (!string.IsNullOrWhiteSpace(value)) - { - return value; - } - } - } - - var bodyText = document.Body.TextContent; - var fallback = SerialRegex.Match(bodyText ?? string.Empty); - return fallback.Success && fallback.Groups["id"].Success - ? fallback.Groups["id"].Value.Trim() - : null; - } - - private static DateTimeOffset? ExtractDate(IDocument document, string language) - { - if (document.Body is null) - { - return null; - } - - var textSegments = new List(); - foreach (var element in document.QuerySelectorAll("strong, p, div")) - { - var text = element.TextContent; - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - var match = DateRegex.Match(text); - if (match.Success && match.Groups["date"].Success) - { - textSegments.Add(match.Groups["date"].Value.Trim()); - } - } - - if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent)) - { - textSegments.Add(document.Body.TextContent); - } - - var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures; - - foreach (var segment in textSegments) - { - foreach (var culture in cultures) - { - if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) - { - return new DateTimeOffset(parsed.ToUniversalTime()); - } - } - } - - return null; - } - - private static IReadOnlyList ExtractProducts(IElement? root) - { - if (root is null) - { - return Array.Empty(); - } - - var results = new List(); - - foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6")) - { - var text = heading.TextContent?.Trim(); - if (!IsProductHeading(text)) - { - continue; - } - - var sibling = heading.NextElementSibling; - while (sibling is not null) - { - if (IsHeading(sibling)) - { - break; - } - - if (IsListElement(sibling)) - { - AppendListItems(sibling, results); - if (results.Count > 0) - { - break; - } - } - else if (IsContentContainer(sibling)) - { - foreach (var list in sibling.QuerySelectorAll("ul,ol")) - { - AppendListItems(list, results); - } - - if (results.Count > 0) - { - break; - } - } - - sibling = sibling.NextElementSibling; - } - - if (results.Count > 0) - { - break; - } - } - - if (results.Count == 0) - { - foreach (var li in root.QuerySelectorAll("ul li,ol li")) - { - var itemText = CollapseWhitespace(li.TextContent); - if (!string.IsNullOrWhiteSpace(itemText)) - { - results.Add(itemText); - } - } - } - - return results.Count == 0 - ? Array.Empty() - : results - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static bool IsProductHeading(string? heading) - { - if (string.IsNullOrWhiteSpace(heading)) - { - return false; - } - - var lowered = heading.ToLowerInvariant(); - return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase)); - } - - private static bool IsHeading(IElement element) - => element.LocalName.Length == 2 - && element.LocalName[0] == 'h' - && char.IsDigit(element.LocalName[1]); - - private static bool IsListElement(IElement element) - => string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase) - || string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase); - - private static bool IsContentContainer(IElement element) - => string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase) - || string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase) - || string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase); - - private static void AppendListItems(IElement listElement, ICollection buffer) - { - foreach (var li in listElement.QuerySelectorAll("li")) - { - if (li is null) - { - continue; - } - - var clone = li.Clone(true) as IElement; - if (clone is null) - { - continue; - } - - foreach (var nested in clone.QuerySelectorAll("ul,ol")) - { - nested.Remove(); - } - - var itemText = CollapseWhitespace(clone.TextContent); - if (!string.IsNullOrWhiteSpace(itemText)) - { - buffer.Add(itemText); - } - } - } - - private static IReadOnlyList ExtractReferences(IElement? root, Uri? baseUri, string language) - { - if (root is null) - { - return Array.Empty(); - } - - var references = new List(); - foreach (var anchor in root.QuerySelectorAll("a[href]")) - { - var href = anchor.GetAttribute("href"); - var normalized = NormalizeReferenceUrl(href, baseUri, language); - if (normalized is null) - { - continue; - } - - var label = CollapseWhitespace(anchor.TextContent); - references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label)); - } - - return references.Count == 0 - ? Array.Empty() - : references - .GroupBy(reference => reference.Url, StringComparer.Ordinal) - .Select(group => group.First()) - .OrderBy(reference => reference.Url, StringComparer.Ordinal) - .ToArray(); - } - - private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language) - { - if (string.IsNullOrWhiteSpace(href)) - { - return null; - } - - if (!Uri.TryCreate(href, UriKind.Absolute, out var absolute)) - { - if (baseUri is null || !Uri.TryCreate(baseUri, href, out absolute)) - { - return null; - } - } - - var builder = new UriBuilder(absolute) - { - Fragment = string.Empty, - }; - - var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language); - builder.Query = filteredQuery; - - return builder.Uri.ToString(); - } - - private static string FilterTrackingParameters(string query, Uri uri, string language) - { - if (string.IsNullOrWhiteSpace(query)) - { - return string.Empty; - } - - var trimmed = query.TrimStart('?'); - if (string.IsNullOrWhiteSpace(trimmed)) - { - return string.Empty; - } - - var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries); - var kept = new List(); - - foreach (var parameter in parameters) - { - var separatorIndex = parameter.IndexOf('='); - var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter; - if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) - { - continue; - } - - if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase) - && key.Equals("lang", StringComparison.OrdinalIgnoreCase)) - { - kept.Add($"lang={language}"); - continue; - } - - kept.Add(parameter); - } - - if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase) - && kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase))) - { - kept.Add($"lang={language}"); - } - - return kept.Count == 0 ? string.Empty : string.Join("&", kept); - } - - private static IReadOnlyList ExtractCveIds(IDocument document) - { - if (document.Body is null) - { - return Array.Empty(); - } - - var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty); - if (matches.Count == 0) - { - return Array.Empty(); - } - - return matches - .Select(match => match.Value.ToUpperInvariant()) - .Distinct(StringComparer.Ordinal) - .OrderBy(value => value, StringComparer.Ordinal) - .ToArray(); - } - - private static string? CollapseWhitespace(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim(); - return collapsed.Length == 0 ? null : collapsed; - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +public sealed class CccsHtmlParser +{ + private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[:：]\s*)(?[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[:：]\s*)(?[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled); + + private static readonly CultureInfo[] EnglishCultures = + { + CultureInfo.GetCultureInfo("en-CA"), + CultureInfo.GetCultureInfo("en-US"), + CultureInfo.InvariantCulture, + }; + + private static readonly CultureInfo[] FrenchCultures = + { + CultureInfo.GetCultureInfo("fr-CA"), + CultureInfo.GetCultureInfo("fr-FR"), + CultureInfo.InvariantCulture, + }; + + private static readonly string[] ProductHeadingKeywords = + { + "affected", + "produit", + "produits", + "produits touch", + "produits concern", + "mesures recommand", + }; + + private static readonly string[] TrackingParameterPrefixes = + { + "utm_", + "mc_", + "mkt_", + "elq", + }; + + private readonly HtmlContentSanitizer _sanitizer; + private readonly HtmlParser _parser; + + public CccsHtmlParser(HtmlContentSanitizer sanitizer) + { + _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); + _parser = new HtmlParser(new HtmlParserOptions + { + IsScripting = false, + IsKeepingSourceReferences = false, + }); + } + + internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw) + { + ArgumentNullException.ThrowIfNull(raw); + + var baseUri = TryCreateUri(raw.CanonicalUrl); + var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty); + var body = document.Body ?? document.DocumentElement; + var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri); + var contentRoot = body ?? document.DocumentElement; + + var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber) + ? raw.SerialNumber!.Trim() + : ExtractSerialNumber(document) ?? raw.SourceId; + + var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified; + var references = ExtractReferences(contentRoot, baseUri, raw.Language); + var products = ExtractProducts(contentRoot); + var cveIds = ExtractCveIds(document); + + return new CccsAdvisoryDto + { + SourceId = raw.SourceId, + SerialNumber = serialNumber, + Language = raw.Language, + Title = raw.Title, + Summary = CollapseWhitespace(raw.Summary), + CanonicalUrl = raw.CanonicalUrl, + ContentHtml = sanitized, + Published = published, + Modified = raw.Modified ?? published, + AlertType = raw.AlertType, + Subject = raw.Subject, + Products = products, + References = references, + CveIds = cveIds, + }; + } + + private static Uri? TryCreateUri(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null; + } + + private static string? ExtractSerialNumber(IDocument document) + { + if (document.Body is null) + { + return null; + } + + foreach (var element in document.QuerySelectorAll("strong, p, div")) + { + var text = element.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var match = SerialRegex.Match(text); + if (match.Success && match.Groups["id"].Success) + { + var value = match.Groups["id"].Value.Trim(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + } + + var bodyText = document.Body.TextContent; + var fallback = SerialRegex.Match(bodyText ?? string.Empty); + return fallback.Success && fallback.Groups["id"].Success + ? fallback.Groups["id"].Value.Trim() + : null; + } + + private static DateTimeOffset? ExtractDate(IDocument document, string language) + { + if (document.Body is null) + { + return null; + } + + var textSegments = new List(); + foreach (var element in document.QuerySelectorAll("strong, p, div")) + { + var text = element.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var match = DateRegex.Match(text); + if (match.Success && match.Groups["date"].Success) + { + textSegments.Add(match.Groups["date"].Value.Trim()); + } + } + + if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent)) + { + textSegments.Add(document.Body.TextContent); + } + + var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures; + + foreach (var segment in textSegments) + { + foreach (var culture in cultures) + { + if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return new DateTimeOffset(parsed.ToUniversalTime()); + } + } + } + + return null; + } + + private static IReadOnlyList ExtractProducts(IElement? root) + { + if (root is null) + { + return Array.Empty(); + } + + var results = new List(); + + foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6")) + { + var text = heading.TextContent?.Trim(); + if (!IsProductHeading(text)) + { + continue; + } + + var sibling = heading.NextElementSibling; + while (sibling is not null) + { + if (IsHeading(sibling)) + { + break; + } + + if (IsListElement(sibling)) + { + AppendListItems(sibling, results); + if (results.Count > 0) + { + break; + } + } + else if (IsContentContainer(sibling)) + { + foreach (var list in sibling.QuerySelectorAll("ul,ol")) + { + AppendListItems(list, results); + } + + if (results.Count > 0) + { + break; + } + } + + sibling = sibling.NextElementSibling; + } + + if (results.Count > 0) + { + break; + } + } + + if (results.Count == 0) + { + foreach (var li in root.QuerySelectorAll("ul li,ol li")) + { + var itemText = CollapseWhitespace(li.TextContent); + if (!string.IsNullOrWhiteSpace(itemText)) + { + results.Add(itemText); + } + } + } + + return results.Count == 0 + ? Array.Empty() + : results + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static bool IsProductHeading(string? heading) + { + if (string.IsNullOrWhiteSpace(heading)) + { + return false; + } + + var lowered = heading.ToLowerInvariant(); + return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase)); + } + + private static bool IsHeading(IElement element) + => element.LocalName.Length == 2 + && element.LocalName[0] == 'h' + && char.IsDigit(element.LocalName[1]); + + private static bool IsListElement(IElement element) + => string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase); + + private static bool IsContentContainer(IElement element) + => string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase) + || string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase); + + private static void AppendListItems(IElement listElement, ICollection buffer) + { + foreach (var li in listElement.QuerySelectorAll("li")) + { + if (li is null) + { + continue; + } + + var clone = li.Clone(true) as IElement; + if (clone is null) + { + continue; + } + + foreach (var nested in clone.QuerySelectorAll("ul,ol")) + { + nested.Remove(); + } + + var itemText = CollapseWhitespace(clone.TextContent); + if (!string.IsNullOrWhiteSpace(itemText)) + { + buffer.Add(itemText); + } + } + } + + private static IReadOnlyList ExtractReferences(IElement? root, Uri? baseUri, string language) + { + if (root is null) + { + return Array.Empty(); + } + + var references = new List(); + foreach (var anchor in root.QuerySelectorAll("a[href]")) + { + var href = anchor.GetAttribute("href"); + var normalized = NormalizeReferenceUrl(href, baseUri, language); + if (normalized is null) + { + continue; + } + + var label = CollapseWhitespace(anchor.TextContent); + references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label)); + } + + return references.Count == 0 + ? Array.Empty() + : references + .GroupBy(reference => reference.Url, StringComparer.Ordinal) + .Select(group => group.First()) + .OrderBy(reference => reference.Url, StringComparer.Ordinal) + .ToArray(); + } + + private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language) + { + if (string.IsNullOrWhiteSpace(href)) + { + return null; + } + + if (!Uri.TryCreate(href, UriKind.Absolute, out var absolute)) + { + if (baseUri is null || !Uri.TryCreate(baseUri, href, out absolute)) + { + return null; + } + } + + var builder = new UriBuilder(absolute) + { + Fragment = string.Empty, + }; + + var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language); + builder.Query = filteredQuery; + + return builder.Uri.ToString(); + } + + private static string FilterTrackingParameters(string query, Uri uri, string language) + { + if (string.IsNullOrWhiteSpace(query)) + { + return string.Empty; + } + + var trimmed = query.TrimStart('?'); + if (string.IsNullOrWhiteSpace(trimmed)) + { + return string.Empty; + } + + var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries); + var kept = new List(); + + foreach (var parameter in parameters) + { + var separatorIndex = parameter.IndexOf('='); + var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter; + if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase) + && key.Equals("lang", StringComparison.OrdinalIgnoreCase)) + { + kept.Add($"lang={language}"); + continue; + } + + kept.Add(parameter); + } + + if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase) + && kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase))) + { + kept.Add($"lang={language}"); + } + + return kept.Count == 0 ? string.Empty : string.Join("&", kept); + } + + private static IReadOnlyList ExtractCveIds(IDocument document) + { + if (document.Body is null) + { + return Array.Empty(); + } + + var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty); + if (matches.Count == 0) + { + return Array.Empty(); + } + + return matches + .Select(match => match.Value.ToUpperInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(value => value, StringComparer.Ordinal) + .ToArray(); + } + + private static string? CollapseWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim(); + return collapsed.Length == 0 ? null : collapsed; + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsMapper.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsMapper.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsMapper.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsMapper.cs index 779f841a..a23cadc5 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsMapper.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsMapper.cs @@ -1,151 +1,151 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -internal static class CccsMapper -{ - public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var packages = BuildPackages(dto, recordedAt); - var provenance = new[] - { - new AdvisoryProvenance( - CccsConnectorPlugin.SourceName, - "advisory", - dto.AlertType ?? dto.SerialNumber, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }) - }; - - return new Advisory( - advisoryKey: dto.SerialNumber, - title: dto.Title, - summary: dto.Summary, - language: dto.Language, - published: dto.Published ?? dto.Modified, - modified: dto.Modified ?? dto.Published, - severity: null, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: packages, - cvssMetrics: Array.Empty(), - provenance: provenance); - } - - private static IReadOnlyList BuildAliases(CccsAdvisoryDto dto) - { - var aliases = new List(capacity: 4) - { - dto.SerialNumber, - }; - - if (!string.IsNullOrWhiteSpace(dto.SourceId) - && !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase)) - { - aliases.Add(dto.SourceId); - } - - foreach (var cve in dto.CveIds) - { - if (!string.IsNullOrWhiteSpace(cve)) - { - aliases.Add(cve); - } - } - - return aliases - .Where(static alias => !string.IsNullOrWhiteSpace(alias)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt) - { - var references = new List - { - new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance( - CccsConnectorPlugin.SourceName, - "reference", - dto.CanonicalUrl, - recordedAt, - new[] { ProvenanceFieldMasks.References })) - }; - - foreach (var reference in dto.References) - { - if (string.IsNullOrWhiteSpace(reference.Url)) - { - continue; - } - - references.Add(new AdvisoryReference( - reference.Url, - "reference", - "cccs", - reference.Label, - new AdvisoryProvenance( - CccsConnectorPlugin.SourceName, - "reference", - reference.Url, - recordedAt, - new[] { ProvenanceFieldMasks.References }))); - } - - return references - .DistinctBy(static reference => reference.Url, StringComparer.Ordinal) - .OrderBy(static reference => reference.Url, StringComparer.Ordinal) - .ToArray(); - } - - private static IReadOnlyList BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt) - { - if (dto.Products.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Products.Count); - foreach (var product in dto.Products) - { - if (string.IsNullOrWhiteSpace(product)) - { - continue; - } - - var identifier = product.Trim(); - var provenance = new AdvisoryProvenance( - CccsConnectorPlugin.SourceName, - "package", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.AffectedPackages }); - - packages.Add(new AffectedPackage( - AffectedPackageTypes.Vendor, - identifier, - platform: null, - versionRanges: Array.Empty(), - statuses: Array.Empty(), - provenance: new[] { provenance }, - normalizedVersions: Array.Empty())); - } - - return packages.Count == 0 - ? Array.Empty() - : packages - .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +internal static class CccsMapper +{ + public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + var provenance = new[] + { + new AdvisoryProvenance( + CccsConnectorPlugin.SourceName, + "advisory", + dto.AlertType ?? dto.SerialNumber, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }) + }; + + return new Advisory( + advisoryKey: dto.SerialNumber, + title: dto.Title, + summary: dto.Summary, + language: dto.Language, + published: dto.Published ?? dto.Modified, + modified: dto.Modified ?? dto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty(), + provenance: provenance); + } + + private static IReadOnlyList BuildAliases(CccsAdvisoryDto dto) + { + var aliases = new List(capacity: 4) + { + dto.SerialNumber, + }; + + if (!string.IsNullOrWhiteSpace(dto.SourceId) + && !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase)) + { + aliases.Add(dto.SourceId); + } + + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + aliases.Add(cve); + } + } + + return aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt) + { + var references = new List + { + new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance( + CccsConnectorPlugin.SourceName, + "reference", + dto.CanonicalUrl, + recordedAt, + new[] { ProvenanceFieldMasks.References })) + }; + + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + references.Add(new AdvisoryReference( + reference.Url, + "reference", + "cccs", + reference.Label, + new AdvisoryProvenance( + CccsConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + return references + .DistinctBy(static reference => reference.Url, StringComparer.Ordinal) + .OrderBy(static reference => reference.Url, StringComparer.Ordinal) + .ToArray(); + } + + private static IReadOnlyList BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt) + { + if (dto.Products.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Products.Count); + foreach (var product in dto.Products) + { + if (string.IsNullOrWhiteSpace(product)) + { + continue; + } + + var identifier = product.Trim(); + var provenance = new AdvisoryProvenance( + CccsConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + provenance: new[] { provenance }, + normalizedVersions: Array.Empty())); + } + + return packages.Count == 0 + ? Array.Empty() + : packages + .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsRawAdvisoryDocument.cs b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsRawAdvisoryDocument.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Cccs/Internal/CccsRawAdvisoryDocument.cs rename to src/StellaOps.Concelier.Connector.Cccs/Internal/CccsRawAdvisoryDocument.cs index c5111d93..74ab840e 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Internal/CccsRawAdvisoryDocument.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Internal/CccsRawAdvisoryDocument.cs @@ -1,58 +1,58 @@ -using System; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Cccs.Internal; - -internal sealed record CccsRawAdvisoryDocument -{ - [JsonPropertyName("sourceId")] - public string SourceId { get; init; } = string.Empty; - - [JsonPropertyName("serialNumber")] - public string? SerialNumber { get; init; } - - [JsonPropertyName("uuid")] - public string? Uuid { get; init; } - - [JsonPropertyName("language")] - public string Language { get; init; } = "en"; - - [JsonPropertyName("title")] - public string Title { get; init; } = string.Empty; - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("canonicalUrl")] - public string CanonicalUrl { get; init; } = string.Empty; - - [JsonPropertyName("externalUrl")] - public string? ExternalUrl { get; init; } - - [JsonPropertyName("bodyHtml")] - public string BodyHtml { get; init; } = string.Empty; - - [JsonPropertyName("bodySegments")] - public string[] BodySegments { get; init; } = Array.Empty(); - - [JsonPropertyName("alertType")] - public string? AlertType { get; init; } - - [JsonPropertyName("subject")] - public string? Subject { get; init; } - - [JsonPropertyName("banner")] - public string? Banner { get; init; } - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("modified")] - public DateTimeOffset? Modified { get; init; } - - [JsonPropertyName("rawCreated")] - public string? RawDateCreated { get; init; } - - [JsonPropertyName("rawModified")] - public string? RawDateModified { get; init; } -} +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Cccs.Internal; + +internal sealed record CccsRawAdvisoryDocument +{ + [JsonPropertyName("sourceId")] + public string SourceId { get; init; } = string.Empty; + + [JsonPropertyName("serialNumber")] + public string? SerialNumber { get; init; } + + [JsonPropertyName("uuid")] + public string? Uuid { get; init; } + + [JsonPropertyName("language")] + public string Language { get; init; } = "en"; + + [JsonPropertyName("title")] + public string Title { get; init; } = string.Empty; + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("canonicalUrl")] + public string CanonicalUrl { get; init; } = string.Empty; + + [JsonPropertyName("externalUrl")] + public string? ExternalUrl { get; init; } + + [JsonPropertyName("bodyHtml")] + public string BodyHtml { get; init; } = string.Empty; + + [JsonPropertyName("bodySegments")] + public string[] BodySegments { get; init; } = Array.Empty(); + + [JsonPropertyName("alertType")] + public string? AlertType { get; init; } + + [JsonPropertyName("subject")] + public string? Subject { get; init; } + + [JsonPropertyName("banner")] + public string? Banner { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("rawCreated")] + public string? RawDateCreated { get; init; } + + [JsonPropertyName("rawModified")] + public string? RawDateModified { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Cccs/Jobs.cs b/src/StellaOps.Concelier.Connector.Cccs/Jobs.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Cccs/Jobs.cs rename to src/StellaOps.Concelier.Connector.Cccs/Jobs.cs index 8110431d..fe861c82 100644 --- a/src/StellaOps.Feedser.Source.Cccs/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Cccs/Jobs.cs @@ -1,22 +1,22 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Cccs; - -internal static class CccsJobKinds -{ - public const string Fetch = "source:cccs:fetch"; -} - -internal sealed class CccsFetchJob : IJob -{ - private readonly CccsConnector _connector; - - public CccsFetchJob(CccsConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Cccs; + +internal static class CccsJobKinds +{ + public const string Fetch = "source:cccs:fetch"; +} + +internal sealed class CccsFetchJob : IJob +{ + private readonly CccsConnector _connector; + + public CccsFetchJob(CccsConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.Cccs/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Cccs/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..5592945c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Cccs/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Cccs.Tests")] diff --git a/src/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj b/src/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj new file mode 100644 index 00000000..45e65329 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Cccs/StellaOps.Concelier.Connector.Cccs.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Cccs/TASKS.md b/src/StellaOps.Concelier.Connector.Cccs/TASKS.md similarity index 71% rename from src/StellaOps.Feedser.Source.Cccs/TASKS.md rename to src/StellaOps.Concelier.Connector.Cccs/TASKS.md index 36c25954..de8a2afe 100644 --- a/src/StellaOps.Feedser.Source.Cccs/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Cccs/TASKS.md @@ -1,11 +1,12 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-CCCS-02-001 Catalogue official CCCS advisory feeds|BE-Conn-CCCS|Research|**DONE (2025-10-11)** – Resolved RSS→Atom redirects (`/api/cccs/rss/v1/get?...` → `/api/cccs/atom/v1/get?...`), confirmed feed caps at 50 entries with inline HTML bodies, no `Last-Modified`/`ETag`, and `updated` timestamps in UTC. Findings and packet captures parked in `docs/feedser-connector-research-20251011.md`; retention sweep follow-up tracked in 02-007.| -|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CccsConnector.FetchAsync` now hydrates feeds via `CccsFeedClient`, persists per-entry JSON payloads with SHA256 dedupe and cursor state, throttles requests, and records taxonomy + language metadata in document state.| -|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**DONE (2025-10-14)** – Added `CccsHtmlParser` to sanitize Atom body HTML, extract serial/date/product bullets, collapse whitespace, and emit normalized reference URLs; `ParseAsync` now persists DTO records under schema `cccs.dto.v1`.| -|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**DONE (2025-10-14)** – `CccsMapper` now materializes canonical advisories (aliases from serial/source/CVEs, references incl. canonical URL, vendor package records) with provenance masks; `MapAsync` stores results in `AdvisoryStore`.| -|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added English/French fixtures plus parser + connector end-to-end tests (`StellaOps.Feedser.Source.Cccs.Tests`). Canned HTTP handler + Mongo fixture enables fetch→parse→map regression; fixtures refresh via `UPDATE_CCCS_FIXTURES=1`.| -|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CccsDiagnostics` meter (fetch/parse/map counters), enriched connector logs with document counts, and published `docs/ops/feedser-cccs-operations.md` covering config, telemetry, and sanitiser guidance.| -|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**DONE (2025-10-15)** – Measured `/api/cccs/threats/v1/get` inventory (~5.1k rows/lang; earliest 2018-06-08), documented backfill workflow + language split strategy, and linked the runbook for Offline Kit execution.| -|FEEDCONN-CCCS-02-008 Raw DOM parsing refinement|BE-Conn-CCCS|Source.Common|**DONE (2025-10-15)** – Parser now walks unsanitised DOM (heading + nested list coverage), sanitizer keeps ``/`section` nodes, and regression fixtures/tests assert EN/FR list handling + preserved HTML structure.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CCCS-02-001 Catalogue official CCCS advisory feeds|BE-Conn-CCCS|Research|**DONE (2025-10-11)** – Resolved RSS→Atom redirects (`/api/cccs/rss/v1/get?...` → `/api/cccs/atom/v1/get?...`), confirmed feed caps at 50 entries with inline HTML bodies, no `Last-Modified`/`ETag`, and `updated` timestamps in UTC. Findings and packet captures parked in `docs/concelier-connector-research-20251011.md`; retention sweep follow-up tracked in 02-007.| +|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CccsConnector.FetchAsync` now hydrates feeds via `CccsFeedClient`, persists per-entry JSON payloads with SHA256 dedupe and cursor state, throttles requests, and records taxonomy + language metadata in document state.| +|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**DONE (2025-10-14)** – Added `CccsHtmlParser` to sanitize Atom body HTML, extract serial/date/product bullets, collapse whitespace, and emit normalized reference URLs; `ParseAsync` now persists DTO records under schema `cccs.dto.v1`.| +|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**DONE (2025-10-14)** – `CccsMapper` now materializes canonical advisories (aliases from serial/source/CVEs, references incl. canonical URL, vendor package records) with provenance masks; `MapAsync` stores results in `AdvisoryStore`.| +|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added English/French fixtures plus parser + connector end-to-end tests (`StellaOps.Concelier.Connector.Cccs.Tests`). Canned HTTP handler + Mongo fixture enables fetch→parse→map regression; fixtures refresh via `UPDATE_CCCS_FIXTURES=1`.| +|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CccsDiagnostics` meter (fetch/parse/map counters), enriched connector logs with document counts, and published `docs/ops/concelier-cccs-operations.md` covering config, telemetry, and sanitiser guidance.| +|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**DONE (2025-10-15)** – Measured `/api/cccs/threats/v1/get` inventory (~5.1k rows/lang; earliest 2018-06-08), documented backfill workflow + language split strategy, and linked the runbook for Offline Kit execution.| +|FEEDCONN-CCCS-02-008 Raw DOM parsing refinement|BE-Conn-CCCS|Source.Common|**DONE (2025-10-15)** – Parser now walks unsanitised DOM (heading + nested list coverage), sanitizer keeps ``/`section` nodes, and regression fixtures/tests assert EN/FR list handling + preserved HTML structure.| +|FEEDCONN-CCCS-02-009 Normalized versions rollout (Oct 2025)|BE-Conn-CCCS|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** – Implement trailing-version split helper per Merge guidance (see `../Merge/RANGE_PRIMITIVES_COORDINATION.md` “Helper snippets”) to emit `NormalizedVersions` via `SemVerRangeRuleBuilder`; refresh mapper tests/fixtures to assert provenance notes (`cccs:{serial}:{index}`) and confirm merge counters drop.| diff --git a/src/StellaOps.Feedser.Source.CertBund.Tests/CertBundConnectorTests.cs b/src/StellaOps.Concelier.Connector.CertBund.Tests/CertBundConnectorTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.CertBund.Tests/CertBundConnectorTests.cs rename to src/StellaOps.Concelier.Connector.CertBund.Tests/CertBundConnectorTests.cs index 504fe6c9..93dc5fd7 100644 --- a/src/StellaOps.Feedser.Source.CertBund.Tests/CertBundConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.CertBund.Tests/CertBundConnectorTests.cs @@ -1,188 +1,188 @@ -using System; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.CertBund.Configuration; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.CertBund.Tests; - -[Collection("mongo-fixture")] -public sealed class CertBundConnectorTests : IAsyncLifetime -{ - private static readonly Uri FeedUri = new("https://test.local/content/public/securityAdvisory/rss"); - private static readonly Uri PortalUri = new("https://test.local/portal/"); - private static readonly Uri DetailUri = new("https://test.local/portal/api/securityadvisory?name=WID-SEC-2025-2264"); - - private readonly MongoIntegrationFixture _fixture; - private readonly CannedHttpMessageHandler _handler; - - public CertBundConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesCanonicalAdvisory() - { - await using var provider = await BuildServiceProviderAsync(); - SeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); - advisories.Should().HaveCount(1); - - var advisory = advisories[0]; - advisory.AdvisoryKey.Should().Be("WID-SEC-2025-2264"); - advisory.Aliases.Should().Contain("CVE-2025-1234"); - advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("Ivanti")); - advisory.References.Should().Contain(reference => reference.Url == DetailUri.ToString()); - advisory.Language.Should().Be("de"); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.Should().NotBeNull(); - state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); - pendingDocs!.AsBsonArray.Should().BeEmpty(); - state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); - pendingMappings!.AsBsonArray.Should().BeEmpty(); - } - - [Fact] - public async Task Fetch_PersistsDocumentWithMetadata() - { - await using var provider = await BuildServiceProviderAsync(); - SeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - - var documentStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(CertBundConnectorPlugin.SourceName, DetailUri.ToString(), CancellationToken.None); - document.Should().NotBeNull(); - document!.Metadata.Should().ContainKey("certbund.advisoryId").WhoseValue.Should().Be("WID-SEC-2025-2264"); - document.Metadata.Should().ContainKey("certbund.category"); - document.Metadata.Should().ContainKey("certbund.published"); - document.Status.Should().Be(DocumentStatuses.PendingParse); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.Should().NotBeNull(); - state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); - pendingDocs!.AsBsonArray.Should().HaveCount(1); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddCertBundConnector(options => - { - options.FeedUri = FeedUri; - options.PortalBootstrapUri = PortalUri; - options.DetailApiUri = new Uri("https://test.local/portal/api/securityadvisory"); - options.RequestDelay = TimeSpan.Zero; - options.MaxAdvisoriesPerFetch = 10; - options.MaxKnownAdvisories = 32; - }); - - services.Configure(CertBundOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedResponses() - { - AddJsonResponse(DetailUri, ReadFixture("certbund-detail.json")); - AddXmlResponse(FeedUri, ReadFixture("certbund-feed.xml"), "application/rss+xml"); - AddHtmlResponse(PortalUri, "OK"); - } - - private void AddJsonResponse(Uri uri, string json, string? etag = null) - { - _handler.AddResponse(uri, () => - { - var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(json, Encoding.UTF8, "application/json"), - }; - if (!string.IsNullOrWhiteSpace(etag)) - { - response.Headers.ETag = new EntityTagHeaderValue(etag); - } - - return response; - }); - } - - private void AddXmlResponse(Uri uri, string xml, string contentType) - { - _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(xml, Encoding.UTF8, contentType), - }); - } - - private void AddHtmlResponse(Uri uri, string html) - { - _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(html, Encoding.UTF8, "text/html"), - }); - } - - private static string ReadFixture(string fileName) - => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; -} +using System; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.CertBund.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertBund.Tests; + +[Collection("mongo-fixture")] +public sealed class CertBundConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://test.local/content/public/securityAdvisory/rss"); + private static readonly Uri PortalUri = new("https://test.local/portal/"); + private static readonly Uri DetailUri = new("https://test.local/portal/api/securityadvisory?name=WID-SEC-2025-2264"); + + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler; + + public CertBundConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + advisories.Should().HaveCount(1); + + var advisory = advisories[0]; + advisory.AdvisoryKey.Should().Be("WID-SEC-2025-2264"); + advisory.Aliases.Should().Contain("CVE-2025-1234"); + advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("Ivanti")); + advisory.References.Should().Contain(reference => reference.Url == DetailUri.ToString()); + advisory.Language.Should().Be("de"); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); + pendingDocs!.AsBsonArray.Should().BeEmpty(); + state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); + pendingMappings!.AsBsonArray.Should().BeEmpty(); + } + + [Fact] + public async Task Fetch_PersistsDocumentWithMetadata() + { + await using var provider = await BuildServiceProviderAsync(); + SeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(CertBundConnectorPlugin.SourceName, DetailUri.ToString(), CancellationToken.None); + document.Should().NotBeNull(); + document!.Metadata.Should().ContainKey("certbund.advisoryId").WhoseValue.Should().Be("WID-SEC-2025-2264"); + document.Metadata.Should().ContainKey("certbund.category"); + document.Metadata.Should().ContainKey("certbund.published"); + document.Status.Should().Be(DocumentStatuses.PendingParse); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); + pendingDocs!.AsBsonArray.Should().HaveCount(1); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertBundConnector(options => + { + options.FeedUri = FeedUri; + options.PortalBootstrapUri = PortalUri; + options.DetailApiUri = new Uri("https://test.local/portal/api/securityadvisory"); + options.RequestDelay = TimeSpan.Zero; + options.MaxAdvisoriesPerFetch = 10; + options.MaxKnownAdvisories = 32; + }); + + services.Configure(CertBundOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedResponses() + { + AddJsonResponse(DetailUri, ReadFixture("certbund-detail.json")); + AddXmlResponse(FeedUri, ReadFixture("certbund-feed.xml"), "application/rss+xml"); + AddHtmlResponse(PortalUri, "OK"); + } + + private void AddJsonResponse(Uri uri, string json, string? etag = null) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + if (!string.IsNullOrWhiteSpace(etag)) + { + response.Headers.ETag = new EntityTagHeaderValue(etag); + } + + return response; + }); + } + + private void AddXmlResponse(Uri uri, string xml, string contentType) + { + _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(xml, Encoding.UTF8, contentType), + }); + } + + private void AddHtmlResponse(Uri uri, string html) + { + _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(html, Encoding.UTF8, "text/html"), + }); + } + + private static string ReadFixture(string fileName) + => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-detail.json b/src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-detail.json similarity index 96% rename from src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-detail.json rename to src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-detail.json index 9c545914..0a4c823c 100644 --- a/src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-detail.json +++ b/src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-detail.json @@ -1,36 +1,36 @@ -{ - "name": "WID-SEC-2025-2264", - "title": "Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung", - "summary": "Ein entfernter, anonymer Angreifer kann mehrere Schwachstellen in Ivanti Endpoint Manager ausnutzen.", - "description": "

      Ivanti Endpoint Manager weist mehrere Schwachstellen auf.

      Ein Angreifer kann beliebigen Code ausführen.

      ", - "severity": "hoch", - "language": "de", - "published": "2025-10-14T06:24:49Z", - "updated": "2025-10-14T07:00:00Z", - "cveIds": [ - "CVE-2025-1234", - "CVE-2025-5678" - ], - "references": [ - { - "url": "https://example.com/vendor/advisory", - "label": "Vendor Advisory" - }, - { - "url": "https://example.com/mitre", - "label": "MITRE" - } - ], - "products": [ - { - "vendor": "Ivanti", - "name": "Endpoint Manager", - "versions": "2023.1 bis 2024.2" - }, - { - "vendor": "Ivanti", - "name": "Endpoint Manager Cloud", - "versions": "alle" - } - ] -} +{ + "name": "WID-SEC-2025-2264", + "title": "Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung", + "summary": "Ein entfernter, anonymer Angreifer kann mehrere Schwachstellen in Ivanti Endpoint Manager ausnutzen.", + "description": "

      Ivanti Endpoint Manager weist mehrere Schwachstellen auf.

      Ein Angreifer kann beliebigen Code ausführen.

      ", + "severity": "hoch", + "language": "de", + "published": "2025-10-14T06:24:49Z", + "updated": "2025-10-14T07:00:00Z", + "cveIds": [ + "CVE-2025-1234", + "CVE-2025-5678" + ], + "references": [ + { + "url": "https://example.com/vendor/advisory", + "label": "Vendor Advisory" + }, + { + "url": "https://example.com/mitre", + "label": "MITRE" + } + ], + "products": [ + { + "vendor": "Ivanti", + "name": "Endpoint Manager", + "versions": "2023.1 bis 2024.2" + }, + { + "vendor": "Ivanti", + "name": "Endpoint Manager Cloud", + "versions": "alle" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-feed.xml b/src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-feed.xml similarity index 97% rename from src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-feed.xml rename to src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-feed.xml index 67f3ff2d..e735a970 100644 --- a/src/StellaOps.Feedser.Source.CertBund.Tests/Fixtures/certbund-feed.xml +++ b/src/StellaOps.Concelier.Connector.CertBund.Tests/Fixtures/certbund-feed.xml @@ -1,15 +1,15 @@ - - - - BSI Warn- und Informationsdienst - https://wid.cert-bund.de/portal/wid/securityadvisory - Test feed - Tue, 14 Oct 2025 07:06:21 GMT - - [hoch] Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung - https://wid.cert-bund.de/portal/wid/securityadvisory?name=WID-SEC-2025-2264 - hoch - Tue, 14 Oct 2025 06:24:49 GMT - - - + + + + BSI Warn- und Informationsdienst + https://wid.cert-bund.de/portal/wid/securityadvisory + Test feed + Tue, 14 Oct 2025 07:06:21 GMT + + [hoch] Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung + https://wid.cert-bund.de/portal/wid/securityadvisory?name=WID-SEC-2025-2264 + hoch + Tue, 14 Oct 2025 06:24:49 GMT + + + diff --git a/src/StellaOps.Feedser.Source.CertBund.Tests/StellaOps.Feedser.Source.CertBund.Tests.csproj b/src/StellaOps.Concelier.Connector.CertBund.Tests/StellaOps.Concelier.Connector.CertBund.Tests.csproj similarity index 69% rename from src/StellaOps.Feedser.Source.CertBund.Tests/StellaOps.Feedser.Source.CertBund.Tests.csproj rename to src/StellaOps.Concelier.Connector.CertBund.Tests/StellaOps.Concelier.Connector.CertBund.Tests.csproj index d31cb11a..df6e25a9 100644 --- a/src/StellaOps.Feedser.Source.CertBund.Tests/StellaOps.Feedser.Source.CertBund.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.CertBund.Tests/StellaOps.Concelier.Connector.CertBund.Tests.csproj @@ -1,22 +1,22 @@ - - - net10.0 - enable - enable - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - + + + net10.0 + enable + enable + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.CertBund/AGENTS.md b/src/StellaOps.Concelier.Connector.CertBund/AGENTS.md similarity index 79% rename from src/StellaOps.Feedser.Source.CertBund/AGENTS.md rename to src/StellaOps.Concelier.Connector.CertBund/AGENTS.md index 724b2fd5..cb4debae 100644 --- a/src/StellaOps.Feedser.Source.CertBund/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.CertBund/AGENTS.md @@ -1,40 +1,40 @@ -# AGENTS -## Role -Deliver a connector for Germany’s CERT-Bund advisories so Feedser can ingest, normalise, and enrich BSI alerts alongside other national feeds. - -## Scope -- Identify the authoritative CERT-Bund advisory feed(s) (RSS/Atom, JSON, CSV, or HTML). -- Implement fetch/cursor logic with proper windowing, dedupe, and failure backoff. -- Parse advisory detail pages for summary, affected products/vendors, mitigation, and references. -- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and provenance/range primitives. -- Provide deterministic fixtures and regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). -- `Feedser.Models` (canonical data model). -- `Feedser.Testing` (integration harness, snapshot utilities). - -## Interfaces & Contracts -- Job kinds: `certbund:fetch`, `certbund:parse`, `certbund:map`. -- Persist upstream metadata (ETag/Last-Modified) if provided. -- Alias set should include CERT-Bund ID and referenced CVE entries. - -## In/Out of scope -In scope: -- End-to-end connector implementation with deterministic tests and range primitive coverage. -- Baseline logging/metrics for pipeline observability. - -Out of scope: -- Non-advisory CERT-Bund digests or newsletters. -- Downstream exporter changes. - -## Observability & Security Expectations -- Log fetch attempts, item counts, and mapping metrics. -- Sanitize HTML thoroughly before persistence. -- Handle transient failures gracefully with exponential backoff and failure records in source state. - -## Tests -- Add `StellaOps.Feedser.Source.CertBund.Tests` covering fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; support regeneration via environment flag. -- Ensure deterministic ordering, casing, and timestamps. +# AGENTS +## Role +Deliver a connector for Germany’s CERT-Bund advisories so Concelier can ingest, normalise, and enrich BSI alerts alongside other national feeds. + +## Scope +- Identify the authoritative CERT-Bund advisory feed(s) (RSS/Atom, JSON, CSV, or HTML). +- Implement fetch/cursor logic with proper windowing, dedupe, and failure backoff. +- Parse advisory detail pages for summary, affected products/vendors, mitigation, and references. +- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and provenance/range primitives. +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Concelier.Models` (canonical data model). +- `Concelier.Testing` (integration harness, snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `certbund:fetch`, `certbund:parse`, `certbund:map`. +- Persist upstream metadata (ETag/Last-Modified) if provided. +- Alias set should include CERT-Bund ID and referenced CVE entries. + +## In/Out of scope +In scope: +- End-to-end connector implementation with deterministic tests and range primitive coverage. +- Baseline logging/metrics for pipeline observability. + +Out of scope: +- Non-advisory CERT-Bund digests or newsletters. +- Downstream exporter changes. + +## Observability & Security Expectations +- Log fetch attempts, item counts, and mapping metrics. +- Sanitize HTML thoroughly before persistence. +- Handle transient failures gracefully with exponential backoff and failure records in source state. + +## Tests +- Add `StellaOps.Concelier.Connector.CertBund.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support regeneration via environment flag. +- Ensure deterministic ordering, casing, and timestamps. diff --git a/src/StellaOps.Feedser.Source.CertBund/CertBundConnector.cs b/src/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs similarity index 97% rename from src/StellaOps.Feedser.Source.CertBund/CertBundConnector.cs rename to src/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs index e56b6623..2c19b78b 100644 --- a/src/StellaOps.Feedser.Source.CertBund/CertBundConnector.cs +++ b/src/StellaOps.Concelier.Connector.CertBund/CertBundConnector.cs @@ -7,18 +7,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Source.CertBund.Configuration; -using StellaOps.Feedser.Source.CertBund.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Connector.CertBund.Configuration; +using StellaOps.Concelier.Connector.CertBund.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertBund; +namespace StellaOps.Concelier.Connector.CertBund; public sealed class CertBundConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.CertBund/CertBundConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs similarity index 91% rename from src/StellaOps.Feedser.Source.CertBund/CertBundConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs index 00b3be7f..be2b681e 100644 --- a/src/StellaOps.Feedser.Source.CertBund/CertBundConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.CertBund/CertBundConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertBund; +namespace StellaOps.Concelier.Connector.CertBund; public sealed class CertBundConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.CertBund/CertBundDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.CertBund/CertBundDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs index 7bba7602..1fd39094 100644 --- a/src/StellaOps.Feedser.Source.CertBund/CertBundDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.CertBund/CertBundDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.CertBund.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.CertBund.Configuration; -namespace StellaOps.Feedser.Source.CertBund; +namespace StellaOps.Concelier.Connector.CertBund; public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:cert-bund"; + private const string ConfigurationSection = "concelier:sources:cert-bund"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.CertBund/CertBundServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.CertBund/CertBundServiceCollectionExtensions.cs similarity index 83% rename from src/StellaOps.Feedser.Source.CertBund/CertBundServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.CertBund/CertBundServiceCollectionExtensions.cs index 0385bfc4..0c055615 100644 --- a/src/StellaOps.Feedser.Source.CertBund/CertBundServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.CertBund/CertBundServiceCollectionExtensions.cs @@ -3,12 +3,12 @@ using System.Net; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertBund.Configuration; -using StellaOps.Feedser.Source.CertBund.Internal; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Concelier.Connector.CertBund.Configuration; +using StellaOps.Concelier.Connector.CertBund.Internal; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Connector.Common.Http; -namespace StellaOps.Feedser.Source.CertBund; +namespace StellaOps.Concelier.Connector.CertBund; public static class CertBundServiceCollectionExtensions { @@ -25,7 +25,7 @@ public static class CertBundServiceCollectionExtensions { var options = sp.GetRequiredService>().Value; clientOptions.Timeout = options.RequestTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.CertBund/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.CertBund/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.FeedUri.Host); clientOptions.AllowedHosts.Add(options.DetailApiUri.Host); diff --git a/src/StellaOps.Feedser.Source.CertBund/Jobs.cs b/src/StellaOps.Concelier.Connector.CertBund/Jobs.cs similarity index 87% rename from src/StellaOps.Feedser.Source.CertBund/Jobs.cs rename to src/StellaOps.Concelier.Connector.CertBund/Jobs.cs index cedb948c..bc4fd0c9 100644 --- a/src/StellaOps.Feedser.Source.CertBund/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.CertBund/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.CertBund; +namespace StellaOps.Concelier.Connector.CertBund; internal static class CertBundJobKinds { diff --git a/src/StellaOps.Feedser.Source.CertBund/README.md b/src/StellaOps.Concelier.Connector.CertBund/README.md similarity index 94% rename from src/StellaOps.Feedser.Source.CertBund/README.md rename to src/StellaOps.Concelier.Connector.CertBund/README.md index faefeda6..4553f4b2 100644 --- a/src/StellaOps.Feedser.Source.CertBund/README.md +++ b/src/StellaOps.Concelier.Connector.CertBund/README.md @@ -6,7 +6,7 @@ - **Detail API** – `https://wid.cert-bund.de/portal/api/securityadvisory?name=`. The connector reuses the bootstrapped `SocketsHttpHandler` so cookies and headers match the Angular SPA. Manual reproduction requires the same cookie container; otherwise the endpoint responds with the shell HTML document. ## Telemetry -The OpenTelemetry meter is `StellaOps.Feedser.Source.CertBund`. Key instruments: +The OpenTelemetry meter is `StellaOps.Concelier.Connector.CertBund`. Key instruments: | Metric | Type | Notes | | --- | --- | --- | @@ -32,7 +32,7 @@ Dashboards should chart coverage days and enqueued counts alongside fetch failur ## Locale & translation stance - CERT-Bund publishes advisory titles and summaries **only in German** (language tag `de`). The connector preserves original casing/content and sets `Advisory.Language = "de"`. - Operator guidance: - 1. Front-line analysts consuming Feedser data should maintain German literacy or rely on approved machine-translation pipelines. + 1. Front-line analysts consuming Concelier data should maintain German literacy or rely on approved machine-translation pipelines. 2. When mirroring advisories into English dashboards, store translations outside the canonical advisory payload to keep determinism. Suggested approach: create an auxiliary collection keyed by advisory ID with timestamped translated snippets. 3. Offline Kit bundles must document that CERT-Bund content is untranslated to avoid surprise during audits. diff --git a/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj b/src/StellaOps.Concelier.Connector.CertBund/StellaOps.Concelier.Connector.CertBund.csproj similarity index 58% rename from src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj rename to src/StellaOps.Concelier.Connector.CertBund/StellaOps.Concelier.Connector.CertBund.csproj index f857e20d..6ba1e732 100644 --- a/src/StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj +++ b/src/StellaOps.Concelier.Connector.CertBund/StellaOps.Concelier.Connector.CertBund.csproj @@ -9,7 +9,7 @@ - - + + diff --git a/src/StellaOps.Feedser.Source.CertBund/TASKS.md b/src/StellaOps.Concelier.Connector.CertBund/TASKS.md similarity index 70% rename from src/StellaOps.Feedser.Source.CertBund/TASKS.md rename to src/StellaOps.Concelier.Connector.CertBund/TASKS.md index 48c152f6..3228d705 100644 --- a/src/StellaOps.Feedser.Source.CertBund/TASKS.md +++ b/src/StellaOps.Concelier.Connector.CertBund/TASKS.md @@ -1,12 +1,13 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-CERTBUND-02-001 Research CERT-Bund advisory endpoints|BE-Conn-CERTBUND|Research|**DONE (2025-10-11)** – Confirmed public RSS at `https://wid.cert-bund.de/content/public/securityAdvisory/rss` (HTTP 200 w/out cookies), 250-item window, German titles/categories, and detail links pointing to Angular SPA. Captured header profile (no cache hints) and logged open item to discover the JSON API used by `portal` frontend.| -|FEEDCONN-CERTBUND-02-002 Fetch job & state persistence|BE-Conn-CERTBUND|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CertBundConnector.FetchAsync` consumes RSS via session-bootstrapped client, stores per-advisory JSON documents with metadata + SHA, throttles detail requests, and maintains cursor state (pending docs/mappings, known advisory IDs, last published).| -|FEEDCONN-CERTBUND-02-003 Parser/DTO implementation|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Detail JSON piped through `CertBundDetailParser` (raw DOM sanitised to HTML), capturing severity, CVEs, product list, and references into DTO records (`cert-bund.detail.v1`).| -|FEEDCONN-CERTBUND-02-004 Canonical mapping & range primitives|BE-Conn-CERTBUND|Models|**DONE (2025-10-14)** – `CertBundMapper` emits canonical advisories (aliases, references, vendor package ranges, provenance) with severity normalisation and deterministic ordering.| -|FEEDCONN-CERTBUND-02-005 Regression fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Feedser.Source.CertBund.Tests` covering fetch→parse→map against canned RSS/JSON fixtures; integration harness uses Mongo2Go + canned HTTP handler; fixtures regenerate via `UPDATE_CERTBUND_FIXTURES=1`.| -|FEEDCONN-CERTBUND-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CertBundDiagnostics` (meter `StellaOps.Feedser.Source.CertBund`) with fetch/parse/map counters + histograms, recorded coverage days, wired stage summary logs, and published the ops runbook (`docs/ops/feedser-certbund-operations.md`).| -|FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**DONE (2025-10-15)** – Measured RSS retention (~6 days/≈250 items), captured connector-driven backfill guidance in the runbook, and aligned locale guidance (preserve `language=de`, Docs glossary follow-up). **Next:** coordinate with Tools to land the state-seeding helper so scripted backfills replace manual Mongo tweaks.| -|FEEDCONN-CERTBUND-02-008 Session bootstrap & cookie strategy|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Feed client primes the portal session (cookie container via `SocketsHttpHandler`), shares cookies across detail requests, and documents bootstrap behaviour in options (`PortalBootstrapUri`).| -|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**DONE (2025-10-17)** – Added `tools/certbund_offline_snapshot.py` to capture search/export JSON, emit deterministic manifests + SHA files, and refreshed docs (`docs/ops/feedser-certbund-operations.md`, `docs/24_OFFLINE_KIT.md`) with offline-kit instructions and manifest layout guidance. Seed data README/ignore rules cover local snapshot hygiene.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CERTBUND-02-001 Research CERT-Bund advisory endpoints|BE-Conn-CERTBUND|Research|**DONE (2025-10-11)** – Confirmed public RSS at `https://wid.cert-bund.de/content/public/securityAdvisory/rss` (HTTP 200 w/out cookies), 250-item window, German titles/categories, and detail links pointing to Angular SPA. Captured header profile (no cache hints) and logged open item to discover the JSON API used by `portal` frontend.| +|FEEDCONN-CERTBUND-02-002 Fetch job & state persistence|BE-Conn-CERTBUND|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CertBundConnector.FetchAsync` consumes RSS via session-bootstrapped client, stores per-advisory JSON documents with metadata + SHA, throttles detail requests, and maintains cursor state (pending docs/mappings, known advisory IDs, last published).| +|FEEDCONN-CERTBUND-02-003 Parser/DTO implementation|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Detail JSON piped through `CertBundDetailParser` (raw DOM sanitised to HTML), capturing severity, CVEs, product list, and references into DTO records (`cert-bund.detail.v1`).| +|FEEDCONN-CERTBUND-02-004 Canonical mapping & range primitives|BE-Conn-CERTBUND|Models|**DONE (2025-10-14)** – `CertBundMapper` emits canonical advisories (aliases, references, vendor package ranges, provenance) with severity normalisation and deterministic ordering.| +|FEEDCONN-CERTBUND-02-005 Regression fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Concelier.Connector.CertBund.Tests` covering fetch→parse→map against canned RSS/JSON fixtures; integration harness uses Mongo2Go + canned HTTP handler; fixtures regenerate via `UPDATE_CERTBUND_FIXTURES=1`.| +|FEEDCONN-CERTBUND-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CertBundDiagnostics` (meter `StellaOps.Concelier.Connector.CertBund`) with fetch/parse/map counters + histograms, recorded coverage days, wired stage summary logs, and published the ops runbook (`docs/ops/concelier-certbund-operations.md`).| +|FEEDCONN-CERTBUND-02-007 Feed history & locale assessment|BE-Conn-CERTBUND|Research|**DONE (2025-10-15)** – Measured RSS retention (~6 days/≈250 items), captured connector-driven backfill guidance in the runbook, and aligned locale guidance (preserve `language=de`, Docs glossary follow-up). **Next:** coordinate with Tools to land the state-seeding helper so scripted backfills replace manual Mongo tweaks.| +|FEEDCONN-CERTBUND-02-008 Session bootstrap & cookie strategy|BE-Conn-CERTBUND|Source.Common|**DONE (2025-10-14)** – Feed client primes the portal session (cookie container via `SocketsHttpHandler`), shares cookies across detail requests, and documents bootstrap behaviour in options (`PortalBootstrapUri`).| +|FEEDCONN-CERTBUND-02-009 Offline Kit export packaging|BE-Conn-CERTBUND, Docs|Offline Kit|**DONE (2025-10-17)** – Added `tools/certbund_offline_snapshot.py` to capture search/export JSON, emit deterministic manifests + SHA files, and refreshed docs (`docs/ops/concelier-certbund-operations.md`, `docs/24_OFFLINE_KIT.md`) with offline-kit instructions and manifest layout guidance. Seed data README/ignore rules cover local snapshot hygiene.| +|FEEDCONN-CERTBUND-02-010 Normalized range translator|BE-Conn-CERTBUND|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-22)** – Translate `product.Versions` phrases (e.g., `2023.1 bis 2024.2`, `alle`) into comparator strings for `SemVerRangeRuleBuilder`, emit `NormalizedVersions` with `certbund:{advisoryId}:{vendor}` provenance, and extend tests/README with localisation notes.| diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs index aa6b2ade..1c908ec6 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorFetchTests.cs @@ -1,263 +1,263 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Source.CertCc; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.CertCc.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Cursors; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; - -[Collection("mongo-fixture")] -public sealed class CertCcConnectorFetchTests : IAsyncLifetime -{ - private const string TestNoteId = "294418"; - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - private ServiceProvider? _serviceProvider; - - public CertCcConnectorFetchTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")] - public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor() - { - var template = new CertCcOptions - { - BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute), - SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = TimeSpan.FromDays(30), - Overlap = TimeSpan.FromDays(5), - InitialBackfill = TimeSpan.FromDays(60), - MinimumWindowSize = TimeSpan.FromDays(1), - }, - MaxMonthlySummaries = 3, - MaxNotesPerFetch = 3, - DetailRequestDelay = TimeSpan.Zero, - }; - - await EnsureServiceProviderAsync(template); - var provider = _serviceProvider!; - - _handler.Clear(); - - var planner = provider.GetRequiredService(); - var plan = planner.CreatePlan(state: null); - Assert.NotEmpty(plan.Requests); - - foreach (var request in plan.Requests) - { - _handler.AddJsonResponse(request.Uri, BuildSummaryPayload()); - } - - RegisterDetailResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - - var documentStore = provider.GetRequiredService(); - foreach (var request in plan.Requests) - { - var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None); - Assert.NotNull(record); - Assert.Equal(DocumentStatuses.PendingParse, record!.Status); - Assert.NotNull(record.Metadata); - Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]); - Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]); - if (request.Month.HasValue) - { - Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]); - } - else - { - Assert.False(record.Metadata.ContainsKey("certcc.month")); - } - } - - foreach (var uri in EnumerateDetailUris()) - { - var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); - Assert.NotNull(record); - Assert.Equal(DocumentStatuses.PendingParse, record!.Status); - Assert.NotNull(record.Metadata); - Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]); - } - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - - BsonValue summaryValue; - Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue)); - var summaryDocument = Assert.IsType(summaryValue); - Assert.True(summaryDocument.TryGetValue("start", out _)); - Assert.True(summaryDocument.TryGetValue("end", out _)); - - var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) - ? pendingNotesValue.AsBsonArray.Count - : 0; - Assert.Equal(0, pendingNotesCount); - - var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) - ? pendingSummariesValue.AsBsonArray.Count - : 0; - Assert.Equal(0, pendingSummariesCount); - - Assert.True(state.Cursor.TryGetValue("lastRun", out _)); - - Assert.True(_handler.Requests.Count >= plan.Requests.Count); - foreach (var request in _handler.Requests) - { - if (request.Headers.TryGetValue("Accept", out var accept)) - { - Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase); - } - } - } - - private static string BuildSummaryPayload() - { - return $$""" - { - "count": 1, - "notes": [ - "VU#{TestNoteId}" - ] - } - """; - } - - private void RegisterDetailResponses() - { - foreach (var uri in EnumerateDetailUris()) - { - var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase) - ? "vu-294418-vendors.json" - : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase) - ? "vu-294418-vuls.json" - : "vu-294418.json"; - - _handler.AddJsonResponse(uri, ReadFixture(fixtureName)); - } - } - - private static IEnumerable EnumerateDetailUris() - { - var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); - yield return new Uri(baseUri, $"{TestNoteId}/"); - yield return new Uri(baseUri, $"{TestNoteId}/vendors/"); - yield return new Uri(baseUri, $"{TestNoteId}/vuls/"); - } - - private async Task EnsureServiceProviderAsync(CertCcOptions template) - { - await DisposeServiceProviderAsync(); - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - options.RawDocumentRetention = TimeSpan.Zero; - options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5); - options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1); - }); - - services.AddSourceCommon(); - services.AddCertCcConnector(options => - { - options.BaseApiUri = template.BaseApiUri; - options.SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = template.SummaryWindow.WindowSize, - Overlap = template.SummaryWindow.Overlap, - InitialBackfill = template.SummaryWindow.InitialBackfill, - MinimumWindowSize = template.SummaryWindow.MinimumWindowSize, - }; - options.MaxMonthlySummaries = template.MaxMonthlySummaries; - options.MaxNotesPerFetch = template.MaxNotesPerFetch; - options.DetailRequestDelay = template.DetailRequestDelay; - options.EnableDetailMapping = template.EnableDetailMapping; - }); - - services.Configure(CertCcOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); - }); - - _serviceProvider = services.BuildServiceProvider(); - var bootstrapper = _serviceProvider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - } - - private async Task DisposeServiceProviderAsync() - { - if (_serviceProvider is null) - { - return; - } - - if (_serviceProvider is IAsyncDisposable asyncDisposable) - { - await asyncDisposable.DisposeAsync(); - } - else - { - _serviceProvider.Dispose(); - } - - _serviceProvider = null; - } - - private static string ReadFixture(string filename) - { - var baseDirectory = AppContext.BaseDirectory; - var primary = Path.Combine(baseDirectory, "Fixtures", filename); - if (File.Exists(primary)) - { - return File.ReadAllText(primary); - } - - return File.ReadAllText(Path.Combine(baseDirectory, filename)); - } - - public Task InitializeAsync() - { - _handler.Clear(); - return Task.CompletedTask; - } - - public async Task DisposeAsync() - { - await DisposeServiceProviderAsync(); - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.CertCc; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.CertCc.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Cursors; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorFetchTests : IAsyncLifetime +{ + private const string TestNoteId = "294418"; + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private ServiceProvider? _serviceProvider; + + public CertCcConnectorFetchTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 8, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact(Skip = "Superseded by snapshot regression coverage (FEEDCONN-CERTCC-02-005).")] + public async Task FetchAsync_PersistsSummaryAndDetailDocumentsAndUpdatesCursor() + { + var template = new CertCcOptions + { + BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute), + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(5), + InitialBackfill = TimeSpan.FromDays(60), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + MaxMonthlySummaries = 3, + MaxNotesPerFetch = 3, + DetailRequestDelay = TimeSpan.Zero, + }; + + await EnsureServiceProviderAsync(template); + var provider = _serviceProvider!; + + _handler.Clear(); + + var planner = provider.GetRequiredService(); + var plan = planner.CreatePlan(state: null); + Assert.NotEmpty(plan.Requests); + + foreach (var request in plan.Requests) + { + _handler.AddJsonResponse(request.Uri, BuildSummaryPayload()); + } + + RegisterDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + foreach (var request in plan.Requests) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, request.Uri.ToString(), CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(DocumentStatuses.PendingParse, record!.Status); + Assert.NotNull(record.Metadata); + Assert.Equal(request.Scope.ToString().ToLowerInvariant(), record.Metadata!["certcc.scope"]); + Assert.Equal(request.Year.ToString("D4"), record.Metadata["certcc.year"]); + if (request.Month.HasValue) + { + Assert.Equal(request.Month.Value.ToString("D2"), record.Metadata["certcc.month"]); + } + else + { + Assert.False(record.Metadata.ContainsKey("certcc.month")); + } + } + + foreach (var uri in EnumerateDetailUris()) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(DocumentStatuses.PendingParse, record!.Status); + Assert.NotNull(record.Metadata); + Assert.Equal(TestNoteId, record.Metadata!["certcc.noteId"]); + } + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + BsonValue summaryValue; + Assert.True(state!.Cursor.TryGetValue("summary", out summaryValue)); + var summaryDocument = Assert.IsType(summaryValue); + Assert.True(summaryDocument.TryGetValue("start", out _)); + Assert.True(summaryDocument.TryGetValue("end", out _)); + + var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue.AsBsonArray.Count + : 0; + Assert.Equal(0, pendingNotesCount); + + var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue.AsBsonArray.Count + : 0; + Assert.Equal(0, pendingSummariesCount); + + Assert.True(state.Cursor.TryGetValue("lastRun", out _)); + + Assert.True(_handler.Requests.Count >= plan.Requests.Count); + foreach (var request in _handler.Requests) + { + if (request.Headers.TryGetValue("Accept", out var accept)) + { + Assert.Contains("application/json", accept, StringComparison.OrdinalIgnoreCase); + } + } + } + + private static string BuildSummaryPayload() + { + return $$""" + { + "count": 1, + "notes": [ + "VU#{TestNoteId}" + ] + } + """; + } + + private void RegisterDetailResponses() + { + foreach (var uri in EnumerateDetailUris()) + { + var fixtureName = uri.AbsolutePath.EndsWith("/vendors/", StringComparison.OrdinalIgnoreCase) + ? "vu-294418-vendors.json" + : uri.AbsolutePath.EndsWith("/vuls/", StringComparison.OrdinalIgnoreCase) + ? "vu-294418-vuls.json" + : "vu-294418.json"; + + _handler.AddJsonResponse(uri, ReadFixture(fixtureName)); + } + } + + private static IEnumerable EnumerateDetailUris() + { + var baseUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + yield return new Uri(baseUri, $"{TestNoteId}/"); + yield return new Uri(baseUri, $"{TestNoteId}/vendors/"); + yield return new Uri(baseUri, $"{TestNoteId}/vuls/"); + } + + private async Task EnsureServiceProviderAsync(CertCcOptions template) + { + await DisposeServiceProviderAsync(); + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + options.RawDocumentRetention = TimeSpan.Zero; + options.RawDocumentRetentionTtlGrace = TimeSpan.FromMinutes(5); + options.RawDocumentRetentionSweepInterval = TimeSpan.FromHours(1); + }); + + services.AddSourceCommon(); + services.AddCertCcConnector(options => + { + options.BaseApiUri = template.BaseApiUri; + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = template.SummaryWindow.WindowSize, + Overlap = template.SummaryWindow.Overlap, + InitialBackfill = template.SummaryWindow.InitialBackfill, + MinimumWindowSize = template.SummaryWindow.MinimumWindowSize, + }; + options.MaxMonthlySummaries = template.MaxMonthlySummaries; + options.MaxNotesPerFetch = template.MaxNotesPerFetch; + options.DetailRequestDelay = template.DetailRequestDelay; + options.EnableDetailMapping = template.EnableDetailMapping; + }); + + services.Configure(CertCcOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + _serviceProvider = services.BuildServiceProvider(); + var bootstrapper = _serviceProvider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + } + + private async Task DisposeServiceProviderAsync() + { + if (_serviceProvider is null) + { + return; + } + + if (_serviceProvider is IAsyncDisposable asyncDisposable) + { + await asyncDisposable.DisposeAsync(); + } + else + { + _serviceProvider.Dispose(); + } + + _serviceProvider = null; + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var primary = Path.Combine(baseDirectory, "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + return File.ReadAllText(Path.Combine(baseDirectory, filename)); + } + + public Task InitializeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } + + public async Task DisposeAsync() + { + await DisposeServiceProviderAsync(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs index b818314f..fbd16b09 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorSnapshotTests.cs @@ -1,410 +1,410 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.CertCc; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Cursors; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; - -[Collection("mongo-fixture")] -public sealed class CertCcConnectorSnapshotTests : IAsyncLifetime -{ - private static readonly Uri SeptemberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/09/summary/"); - private static readonly Uri OctoberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); - private static readonly Uri NovemberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/11/summary/"); - private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); - private static readonly Uri VendorsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); - private static readonly Uri VulsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); - private static readonly Uri VendorStatusesDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); - - private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); - - private readonly MongoIntegrationFixture _fixture; - private ConnectorTestHarness? _harness; - - public CertCcConnectorSnapshotTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - } - - [Fact] - public async Task FetchSummaryAndDetails_ProducesDeterministicSnapshots() - { - var initialTime = new DateTimeOffset(2025, 11, 1, 8, 0, 0, TimeSpan.Zero); - var harness = await EnsureHarnessAsync(initialTime); - - RegisterSummaryResponses(harness.Handler); - RegisterDetailResponses(harness.Handler); - - var connector = harness.ServiceProvider.GetRequiredService(); - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - - var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(documentsSnapshot, "certcc-documents.snapshot.json"); - - var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(stateSnapshot, "certcc-state.snapshot.json"); - - await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); - await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); - - var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(advisoriesSnapshot, "certcc-advisories.snapshot.json"); - - harness.TimeProvider.Advance(TimeSpan.FromMinutes(30)); - RegisterSummaryNotModifiedResponses(harness.Handler); - - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - var recordedRequests = harness.Handler.Requests - .Select(request => request.Uri.ToString()) - .ToArray(); - recordedRequests.Should().Equal(new[] - { - SeptemberSummaryUri.ToString(), - OctoberSummaryUri.ToString(), - NoteDetailUri.ToString(), - VendorsDetailUri.ToString(), - VulsDetailUri.ToString(), - VendorStatusesDetailUri.ToString(), - YearlySummaryUri.ToString(), - OctoberSummaryUri.ToString(), - NovemberSummaryUri.ToString(), - YearlySummaryUri.ToString(), - }); - harness.Handler.AssertNoPendingResponses(); - - var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); - WriteOrAssertSnapshot(requestsSnapshot, "certcc-requests.snapshot.json"); - } - - private async Task EnsureHarnessAsync(DateTimeOffset initialTime) - { - if (_harness is not null) - { - return _harness; - } - - var harness = new ConnectorTestHarness(_fixture, initialTime, CertCcOptions.HttpClientName); - await harness.EnsureServiceProviderAsync(services => - { - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddCertCcConnector(options => - { - options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); - options.SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = TimeSpan.FromDays(30), - Overlap = TimeSpan.FromDays(3), - InitialBackfill = TimeSpan.FromDays(45), - MinimumWindowSize = TimeSpan.FromDays(1), - }; - options.MaxMonthlySummaries = 2; - options.MaxNotesPerFetch = 1; - options.DetailRequestDelay = TimeSpan.Zero; - options.EnableDetailMapping = true; - }); - - services.Configure(CertCcOptions.HttpClientName, options => - { - options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); - }); - }); - - _harness = harness; - return harness; - } - - private static async Task BuildDocumentsSnapshotAsync(IServiceProvider provider) - { - var documentStore = provider.GetRequiredService(); - var uris = new[] - { - SeptemberSummaryUri, - OctoberSummaryUri, - NovemberSummaryUri, - YearlySummaryUri, - NoteDetailUri, - VendorsDetailUri, - VulsDetailUri, - VendorStatusesDetailUri, - }; - - var records = new List(uris.Length); - foreach (var uri in uris) - { - var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); - if (record is null) - { - continue; - } - - var lastModified = record.Headers is not null - && record.Headers.TryGetValue("Last-Modified", out var lastModifiedHeader) - && DateTimeOffset.TryParse(lastModifiedHeader, out var parsedLastModified) - ? parsedLastModified.ToUniversalTime().ToString("O") - : record.LastModified?.ToUniversalTime().ToString("O"); - - records.Add(new - { - record.Uri, - record.Status, - record.Sha256, - record.ContentType, - LastModified = lastModified, - Metadata = record.Metadata is null - ? null - : record.Metadata - .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) - .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase), - record.Etag, - }); - } - - var ordered = records - .OrderBy(static entry => entry.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) - .ToArray(); - - return SnapshotSerializer.ToSnapshot(ordered); - } - - private static async Task BuildStateSnapshotAsync(IServiceProvider provider) - { - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - - var cursor = state!.Cursor ?? new BsonDocument(); - - BsonDocument? summaryDocument = null; - if (cursor.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDoc) - { - summaryDocument = summaryDoc; - } - - var summary = summaryDocument is null - ? null - : new - { - Start = summaryDocument.TryGetValue("start", out var startValue) ? ToIsoString(startValue) : null, - End = summaryDocument.TryGetValue("end", out var endValue) ? ToIsoString(endValue) : null, - }; - - var snapshot = new - { - Summary = summary, - PendingNotes = cursor.TryGetValue("pendingNotes", out var pendingNotesValue) - ? pendingNotesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static note => note, StringComparer.OrdinalIgnoreCase).ToArray() - : Array.Empty(), - PendingSummaries = cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) - ? pendingSummariesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static item => item, StringComparer.OrdinalIgnoreCase).ToArray() - : Array.Empty(), - LastRun = cursor.TryGetValue("lastRun", out var lastRunValue) ? ToIsoString(lastRunValue) : null, - state.LastSuccess, - state.LastFailure, - state.FailCount, - state.BackoffUntil, - }; - - return SnapshotSerializer.ToSnapshot(snapshot); - } - - private static async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) - { - var advisoryStore = provider.GetRequiredService(); - var advisories = new List(); - await foreach (var advisory in advisoryStore.StreamAsync(CancellationToken.None)) - { - advisories.Add(advisory); - } - - var ordered = advisories - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) - .ToArray(); - - return SnapshotSerializer.ToSnapshot(ordered); - } - - private static string BuildRequestsSnapshot(IReadOnlyCollection requests) - { - var ordered = requests - .OrderBy(static request => request.Timestamp) - .Select(static request => new - { - request.Method.Method, - Uri = request.Uri.ToString(), - Headers = new - { - Accept = TryGetHeader(request.Headers, "Accept"), - IfNoneMatch = TryGetHeader(request.Headers, "If-None-Match"), - IfModifiedSince = TryGetHeader(request.Headers, "If-Modified-Since"), - }, - }) - .ToArray(); - - return SnapshotSerializer.ToSnapshot(ordered); - } - - private static void RegisterSummaryResponses(CannedHttpMessageHandler handler) - { - AddJsonResponse(handler, SeptemberSummaryUri, "summary-2025-09.json", "\"certcc-summary-2025-09\"", new DateTimeOffset(2025, 9, 30, 12, 0, 0, TimeSpan.Zero)); - AddJsonResponse(handler, OctoberSummaryUri, "summary-2025-10.json", "\"certcc-summary-2025-10\"", new DateTimeOffset(2025, 10, 31, 12, 0, 0, TimeSpan.Zero)); - AddJsonResponse(handler, YearlySummaryUri, "summary-2025.json", "\"certcc-summary-2025\"", new DateTimeOffset(2025, 10, 31, 12, 1, 0, TimeSpan.Zero)); - } - - private static void RegisterSummaryNotModifiedResponses(CannedHttpMessageHandler handler) - { - AddNotModified(handler, OctoberSummaryUri, "\"certcc-summary-2025-10\""); - AddNotModified(handler, NovemberSummaryUri, "\"certcc-summary-2025-11\""); - AddNotModified(handler, YearlySummaryUri, "\"certcc-summary-2025\""); - } - - private static void RegisterDetailResponses(CannedHttpMessageHandler handler) - { - AddJsonResponse(handler, NoteDetailUri, "vu-294418.json", "\"certcc-note-294418\"", new DateTimeOffset(2025, 10, 9, 16, 52, 0, TimeSpan.Zero)); - AddJsonResponse(handler, VendorsDetailUri, "vu-294418-vendors.json", "\"certcc-vendors-294418\"", new DateTimeOffset(2025, 10, 9, 17, 5, 0, TimeSpan.Zero)); - AddJsonResponse(handler, VulsDetailUri, "vu-294418-vuls.json", "\"certcc-vuls-294418\"", new DateTimeOffset(2025, 10, 9, 17, 10, 0, TimeSpan.Zero)); - AddJsonResponse(handler, VendorStatusesDetailUri, "vendor-statuses-294418.json", "\"certcc-vendor-statuses-294418\"", new DateTimeOffset(2025, 10, 9, 17, 12, 0, TimeSpan.Zero)); - } - - private static void AddJsonResponse(CannedHttpMessageHandler handler, Uri uri, string fixtureName, string etag, DateTimeOffset lastModified) - { - var payload = ReadFixture(fixtureName); - handler.AddResponse(HttpMethod.Get, uri, _ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue(etag); - response.Headers.TryAddWithoutValidation("Last-Modified", lastModified.ToString("R")); - response.Content.Headers.LastModified = lastModified; - return response; - }); - } - - private static void AddNotModified(CannedHttpMessageHandler handler, Uri uri, string etag) - { - handler.AddResponse(HttpMethod.Get, uri, _ => - { - var response = new HttpResponseMessage(HttpStatusCode.NotModified); - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private static string ReadFixture(string filename) - { - var baseDir = AppContext.BaseDirectory; - var primary = Path.Combine(baseDir, "Fixtures", filename); - if (File.Exists(primary)) - { - return File.ReadAllText(primary); - } - - var fallback = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); - if (File.Exists(fallback)) - { - return File.ReadAllText(fallback); - } - - throw new FileNotFoundException($"Missing CERT/CC fixture '{filename}'."); - } - - private static string? TryGetHeader(IReadOnlyDictionary headers, string key) - => headers.TryGetValue(key, out var value) ? value : null; - - private static string? ToIsoString(BsonValue value) - { - return value.BsonType switch - { - BsonType.DateTime => value.ToUniversalTime().ToString("O"), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime().ToString("O"), - _ => null, - }; - } - - private static void WriteOrAssertSnapshot(string snapshot, string filename) - { - var normalizedSnapshot = Normalize(snapshot); - if (ShouldUpdateFixtures() || !FixtureExists(filename)) - { - var path = GetWritablePath(filename); - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, normalizedSnapshot); - return; - } - - var expected = ReadFixture(filename); - var normalizedExpected = Normalize(expected); - if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(Path.GetDirectoryName(GetWritablePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); - Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); - File.WriteAllText(actualPath, normalizedSnapshot); - } - - Assert.Equal(normalizedExpected, normalizedSnapshot); - } - - private static string GetWritablePath(string filename) - { - var baseDir = AppContext.BaseDirectory; - return Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); - } - - private static string Normalize(string value) - => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); - - private static bool ShouldUpdateFixtures() - { - var flag = Environment.GetEnvironmentVariable("UPDATE_CERTCC_FIXTURES"); - return string.Equals(flag, "1", StringComparison.Ordinal) || string.Equals(flag, "true", StringComparison.OrdinalIgnoreCase); - } - - private static bool FixtureExists(string filename) - { - var baseDir = AppContext.BaseDirectory; - var primary = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); - if (File.Exists(primary)) - { - return true; - } - - var fallback = Path.Combine(baseDir, "Fixtures", filename); - return File.Exists(fallback); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public async Task DisposeAsync() - { - if (_harness is not null) - { - await _harness.DisposeAsync(); - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.CertCc; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Cursors; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorSnapshotTests : IAsyncLifetime +{ + private static readonly Uri SeptemberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/09/summary/"); + private static readonly Uri OctoberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); + private static readonly Uri NovemberSummaryUri = new("https://www.kb.cert.org/vuls/api/2025/11/summary/"); + private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); + private static readonly Uri VendorsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); + private static readonly Uri VulsDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); + private static readonly Uri VendorStatusesDetailUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); + + private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); + + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public CertCcConnectorSnapshotTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchSummaryAndDetails_ProducesDeterministicSnapshots() + { + var initialTime = new DateTimeOffset(2025, 11, 1, 8, 0, 0, TimeSpan.Zero); + var harness = await EnsureHarnessAsync(initialTime); + + RegisterSummaryResponses(harness.Handler); + RegisterDetailResponses(harness.Handler); + + var connector = harness.ServiceProvider.GetRequiredService(); + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + + var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(documentsSnapshot, "certcc-documents.snapshot.json"); + + var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(stateSnapshot, "certcc-state.snapshot.json"); + + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(advisoriesSnapshot, "certcc-advisories.snapshot.json"); + + harness.TimeProvider.Advance(TimeSpan.FromMinutes(30)); + RegisterSummaryNotModifiedResponses(harness.Handler); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + var recordedRequests = harness.Handler.Requests + .Select(request => request.Uri.ToString()) + .ToArray(); + recordedRequests.Should().Equal(new[] + { + SeptemberSummaryUri.ToString(), + OctoberSummaryUri.ToString(), + NoteDetailUri.ToString(), + VendorsDetailUri.ToString(), + VulsDetailUri.ToString(), + VendorStatusesDetailUri.ToString(), + YearlySummaryUri.ToString(), + OctoberSummaryUri.ToString(), + NovemberSummaryUri.ToString(), + YearlySummaryUri.ToString(), + }); + harness.Handler.AssertNoPendingResponses(); + + var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); + WriteOrAssertSnapshot(requestsSnapshot, "certcc-requests.snapshot.json"); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return _harness; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, CertCcOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddCertCcConnector(options => + { + options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(45), + MinimumWindowSize = TimeSpan.FromDays(1), + }; + options.MaxMonthlySummaries = 2; + options.MaxNotesPerFetch = 1; + options.DetailRequestDelay = TimeSpan.Zero; + options.EnableDetailMapping = true; + }); + + services.Configure(CertCcOptions.HttpClientName, options => + { + options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); + }); + }); + + _harness = harness; + return harness; + } + + private static async Task BuildDocumentsSnapshotAsync(IServiceProvider provider) + { + var documentStore = provider.GetRequiredService(); + var uris = new[] + { + SeptemberSummaryUri, + OctoberSummaryUri, + NovemberSummaryUri, + YearlySummaryUri, + NoteDetailUri, + VendorsDetailUri, + VulsDetailUri, + VendorStatusesDetailUri, + }; + + var records = new List(uris.Length); + foreach (var uri in uris) + { + var record = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, uri.ToString(), CancellationToken.None); + if (record is null) + { + continue; + } + + var lastModified = record.Headers is not null + && record.Headers.TryGetValue("Last-Modified", out var lastModifiedHeader) + && DateTimeOffset.TryParse(lastModifiedHeader, out var parsedLastModified) + ? parsedLastModified.ToUniversalTime().ToString("O") + : record.LastModified?.ToUniversalTime().ToString("O"); + + records.Add(new + { + record.Uri, + record.Status, + record.Sha256, + record.ContentType, + LastModified = lastModified, + Metadata = record.Metadata is null + ? null + : record.Metadata + .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase), + record.Etag, + }); + } + + var ordered = records + .OrderBy(static entry => entry.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static async Task BuildStateSnapshotAsync(IServiceProvider provider) + { + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + var cursor = state!.Cursor ?? new BsonDocument(); + + BsonDocument? summaryDocument = null; + if (cursor.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDoc) + { + summaryDocument = summaryDoc; + } + + var summary = summaryDocument is null + ? null + : new + { + Start = summaryDocument.TryGetValue("start", out var startValue) ? ToIsoString(startValue) : null, + End = summaryDocument.TryGetValue("end", out var endValue) ? ToIsoString(endValue) : null, + }; + + var snapshot = new + { + Summary = summary, + PendingNotes = cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static note => note, StringComparer.OrdinalIgnoreCase).ToArray() + : Array.Empty(), + PendingSummaries = cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue.AsBsonArray.Select(static value => value.ToString()).OrderBy(static item => item, StringComparer.OrdinalIgnoreCase).ToArray() + : Array.Empty(), + LastRun = cursor.TryGetValue("lastRun", out var lastRunValue) ? ToIsoString(lastRunValue) : null, + state.LastSuccess, + state.LastFailure, + state.FailCount, + state.BackoffUntil, + }; + + return SnapshotSerializer.ToSnapshot(snapshot); + } + + private static async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) + { + var advisoryStore = provider.GetRequiredService(); + var advisories = new List(); + await foreach (var advisory in advisoryStore.StreamAsync(CancellationToken.None)) + { + advisories.Add(advisory); + } + + var ordered = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static string BuildRequestsSnapshot(IReadOnlyCollection requests) + { + var ordered = requests + .OrderBy(static request => request.Timestamp) + .Select(static request => new + { + request.Method.Method, + Uri = request.Uri.ToString(), + Headers = new + { + Accept = TryGetHeader(request.Headers, "Accept"), + IfNoneMatch = TryGetHeader(request.Headers, "If-None-Match"), + IfModifiedSince = TryGetHeader(request.Headers, "If-Modified-Since"), + }, + }) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static void RegisterSummaryResponses(CannedHttpMessageHandler handler) + { + AddJsonResponse(handler, SeptemberSummaryUri, "summary-2025-09.json", "\"certcc-summary-2025-09\"", new DateTimeOffset(2025, 9, 30, 12, 0, 0, TimeSpan.Zero)); + AddJsonResponse(handler, OctoberSummaryUri, "summary-2025-10.json", "\"certcc-summary-2025-10\"", new DateTimeOffset(2025, 10, 31, 12, 0, 0, TimeSpan.Zero)); + AddJsonResponse(handler, YearlySummaryUri, "summary-2025.json", "\"certcc-summary-2025\"", new DateTimeOffset(2025, 10, 31, 12, 1, 0, TimeSpan.Zero)); + } + + private static void RegisterSummaryNotModifiedResponses(CannedHttpMessageHandler handler) + { + AddNotModified(handler, OctoberSummaryUri, "\"certcc-summary-2025-10\""); + AddNotModified(handler, NovemberSummaryUri, "\"certcc-summary-2025-11\""); + AddNotModified(handler, YearlySummaryUri, "\"certcc-summary-2025\""); + } + + private static void RegisterDetailResponses(CannedHttpMessageHandler handler) + { + AddJsonResponse(handler, NoteDetailUri, "vu-294418.json", "\"certcc-note-294418\"", new DateTimeOffset(2025, 10, 9, 16, 52, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VendorsDetailUri, "vu-294418-vendors.json", "\"certcc-vendors-294418\"", new DateTimeOffset(2025, 10, 9, 17, 5, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VulsDetailUri, "vu-294418-vuls.json", "\"certcc-vuls-294418\"", new DateTimeOffset(2025, 10, 9, 17, 10, 0, TimeSpan.Zero)); + AddJsonResponse(handler, VendorStatusesDetailUri, "vendor-statuses-294418.json", "\"certcc-vendor-statuses-294418\"", new DateTimeOffset(2025, 10, 9, 17, 12, 0, TimeSpan.Zero)); + } + + private static void AddJsonResponse(CannedHttpMessageHandler handler, Uri uri, string fixtureName, string etag, DateTimeOffset lastModified) + { + var payload = ReadFixture(fixtureName); + handler.AddResponse(HttpMethod.Get, uri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + response.Headers.TryAddWithoutValidation("Last-Modified", lastModified.ToString("R")); + response.Content.Headers.LastModified = lastModified; + return response; + }); + } + + private static void AddNotModified(CannedHttpMessageHandler handler, Uri uri, string etag) + { + handler.AddResponse(HttpMethod.Get, uri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private static string ReadFixture(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Fixtures", filename); + if (File.Exists(primary)) + { + return File.ReadAllText(primary); + } + + var fallback = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(fallback)) + { + return File.ReadAllText(fallback); + } + + throw new FileNotFoundException($"Missing CERT/CC fixture '{filename}'."); + } + + private static string? TryGetHeader(IReadOnlyDictionary headers, string key) + => headers.TryGetValue(key, out var value) ? value : null; + + private static string? ToIsoString(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => value.ToUniversalTime().ToString("O"), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime().ToString("O"), + _ => null, + }; + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + var normalizedSnapshot = Normalize(snapshot); + if (ShouldUpdateFixtures() || !FixtureExists(filename)) + { + var path = GetWritablePath(filename); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, normalizedSnapshot); + return; + } + + var expected = ReadFixture(filename); + var normalizedExpected = Normalize(expected); + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(Path.GetDirectoryName(GetWritablePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, normalizedSnapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + } + + private static string GetWritablePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + return Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd(); + + private static bool ShouldUpdateFixtures() + { + var flag = Environment.GetEnvironmentVariable("UPDATE_CERTCC_FIXTURES"); + return string.Equals(flag, "1", StringComparison.Ordinal) || string.Equals(flag, "true", StringComparison.OrdinalIgnoreCase); + } + + private static bool FixtureExists(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(primary)) + { + return true; + } + + var fallback = Path.Combine(baseDir, "Fixtures", filename); + return File.Exists(fallback); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorTests.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorTests.cs index 920da426..5994e608 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/CertCc/CertCcConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/CertCc/CertCcConnectorTests.cs @@ -1,477 +1,477 @@ -using System; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Feedser.Source.CertCc; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Cursors; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.CertCc; - -[Collection("mongo-fixture")] -public sealed class CertCcConnectorTests : IAsyncLifetime -{ - private static readonly Uri MonthlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); - private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); - private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); - private static readonly Uri VendorsUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); - private static readonly Uri VulsUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); - private static readonly Uri VendorStatusesUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - - public CertCcConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 9, 30, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesCanonicalAdvisory() - { - await using var provider = await BuildServiceProviderAsync(); - SeedSummaryResponses(); - SeedDetailResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - advisories.Should().NotBeNull(); - advisories.Should().HaveCountGreaterThan(0); - - var advisory = advisories.FirstOrDefault(a => a.AdvisoryKey == "certcc/vu-294418"); - advisory.Should().NotBeNull(); - advisory!.Title.Should().ContainEquivalentOf("DrayOS"); - advisory.Summary.Should().NotBeNullOrWhiteSpace(); - advisory.Aliases.Should().Contain("VU#294418"); - advisory.Aliases.Should().Contain("CVE-2025-10547"); - advisory.AffectedPackages.Should().NotBeNull(); - advisory.AffectedPackages.Should().HaveCountGreaterThan(0); - advisory.AffectedPackages![0].NormalizedVersions.Should().NotBeNull(); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) - ? pendingDocsValue!.AsBsonArray.Count - : 0; - pendingDocuments.Should().Be(0); - var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) - ? pendingMappingsValue!.AsBsonArray.Count - : 0; - pendingMappings.Should().Be(0); - } - - [Fact] - public async Task Fetch_PersistsSummaryAndDetailDocuments() - { - await using var provider = await BuildServiceProviderAsync(); - SeedSummaryResponses(); - SeedDetailResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - - var documentStore = provider.GetRequiredService(); - - var summaryDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, MonthlySummaryUri.ToString(), CancellationToken.None); - summaryDocument.Should().NotBeNull(); - summaryDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - - var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); - noteDocument.Should().NotBeNull(); - noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - noteDocument.Metadata.Should().NotBeNull(); - noteDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("note"); - noteDocument.Metadata.Should().ContainKey("certcc.noteId").WhoseValue.Should().Be("294418"); - noteDocument.Metadata.Should().ContainKey("certcc.vuid").WhoseValue.Should().Be("VU#294418"); - - var vendorsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorsUri.ToString(), CancellationToken.None); - vendorsDocument.Should().NotBeNull(); - vendorsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - vendorsDocument.Metadata.Should().NotBeNull(); - vendorsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors"); - - var vulsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VulsUri.ToString(), CancellationToken.None); - vulsDocument.Should().NotBeNull(); - vulsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - vulsDocument.Metadata.Should().NotBeNull(); - vulsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vuls"); - - var vendorStatusesDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorStatusesUri.ToString(), CancellationToken.None); - vendorStatusesDocument.Should().NotBeNull(); - vendorStatusesDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - vendorStatusesDocument.Metadata.Should().NotBeNull(); - vendorStatusesDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors-vuls"); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.Should().NotBeNull(); - var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) - ? pendingNotesValue!.AsBsonArray.Count - : 0; - pendingNotesCount.Should().Be(0); - var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) - ? pendingSummariesValue!.AsBsonArray.Count - : 0; - pendingSummariesCount.Should().Be(0); - - var pendingDocumentsCount = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) - ? pendingDocumentsValue!.AsBsonArray.Count - : 0; - pendingDocumentsCount.Should().Be(4); - - var pendingMappingsCount = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) - ? pendingMappingsValue!.AsBsonArray.Count - : 0; - pendingMappingsCount.Should().Be(0); - - _handler.Requests.Should().Contain(request => request.Uri == NoteDetailUri); - } - - [Fact] - public async Task Fetch_ReusesConditionalRequestsOnSubsequentRun() - { - await using var provider = await BuildServiceProviderAsync(); - SeedSummaryResponses(summaryEtag: "\"summary-oct\"", yearlyEtag: "\"summary-year\""); - SeedDetailResponses(detailEtag: "\"note-etag\"", vendorsEtag: "\"vendors-etag\"", vulsEtag: "\"vuls-etag\"", vendorStatusesEtag: "\"vendor-statuses-etag\""); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - - _handler.Clear(); - SeedSummaryNotModifiedResponses("\"summary-oct\"", "\"summary-year\""); - SeedDetailNotModifiedResponses("\"note-etag\"", "\"vendors-etag\"", "\"vuls-etag\"", "\"vendor-statuses-etag\""); - _timeProvider.Advance(TimeSpan.FromMinutes(15)); - - await connector.FetchAsync(provider, CancellationToken.None); - - var requests = _handler.Requests.ToArray(); - requests.Should().OnlyContain(r => - r.Uri == MonthlySummaryUri - || r.Uri == YearlySummaryUri - || r.Uri == NoteDetailUri - || r.Uri == VendorsUri - || r.Uri == VulsUri - || r.Uri == VendorStatusesUri); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - var pendingNotesCount = state!.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) - ? pendingNotesValue!.AsBsonArray.Count - : 0; - pendingNotesCount.Should().Be(0); - var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) - ? pendingSummariesValue!.AsBsonArray.Count - : 0; - pendingSummaries.Should().Be(0); - - var pendingDocuments = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) - ? pendingDocumentsValue!.AsBsonArray.Count - : 0; - pendingDocuments.Should().BeGreaterThan(0); - } - - [Fact] - public async Task Fetch_DetailFailureRecordsBackoffAndKeepsPendingNote() - { - await using var provider = await BuildServiceProviderAsync(); - SeedSummaryResponses(); - SeedDetailResponses(vendorsStatus: HttpStatusCode.InternalServerError); - - var connector = provider.GetRequiredService(); - var failure = await Assert.ThrowsAnyAsync(() => connector.FetchAsync(provider, CancellationToken.None)); - Assert.True(failure is HttpRequestException || failure is InvalidOperationException); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.FailCount.Should().BeGreaterThan(0); - state.BackoffUntil.Should().NotBeNull(); - state.BackoffUntil.Should().BeAfter(_timeProvider.GetUtcNow()); - state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue).Should().BeTrue(); - pendingNotesValue!.AsBsonArray.Should().Contain(value => value.AsString == "294418"); - var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) - ? pendingSummariesValue!.AsBsonArray.Count - : 0; - pendingSummaries.Should().Be(0); - } - - [Fact] - public async Task Fetch_PartialDetailEndpointsMissing_CompletesAndMaps() - { - await using var provider = await BuildServiceProviderAsync(); - SeedSummaryResponses(); - SeedDetailResponses( - vulsStatus: HttpStatusCode.NotFound, - vendorStatusesStatus: HttpStatusCode.NotFound); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - advisories.Should().NotBeNull(); - advisories!.Should().Contain(advisory => advisory.AdvisoryKey == "certcc/vu-294418"); - - var documentStore = provider.GetRequiredService(); - var vendorsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorsUri.ToString(), CancellationToken.None); - vendorsDocument.Should().NotBeNull(); - vendorsDocument!.Status.Should().Be(DocumentStatuses.Mapped); - var vulsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VulsUri.ToString(), CancellationToken.None); - vulsDocument.Should().BeNull(); - var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); - noteDocument.Should().NotBeNull(); - noteDocument!.Status.Should().Be(DocumentStatuses.Mapped); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue).Should().BeTrue(); - pendingNotesValue!.AsBsonArray.Should().BeEmpty(); - state.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue).Should().BeTrue(); - pendingDocsValue!.AsBsonArray.Should().BeEmpty(); - state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue).Should().BeTrue(); - pendingMappingsValue!.AsBsonArray.Should().BeEmpty(); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public async Task DisposeAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - } - - [Fact] - public async Task ParseAndMap_SkipWhenDetailMappingDisabled() - { - await using var provider = await BuildServiceProviderAsync(enableDetailMapping: false); - SeedSummaryResponses(); - SeedDetailResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - advisories.Should().BeNullOrEmpty(); - - var documentStore = provider.GetRequiredService(); - var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); - noteDocument.Should().NotBeNull(); - noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) - ? pendingDocsValue!.AsBsonArray.Count - : 0; - pendingDocuments.Should().BeGreaterThan(0); - var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) - ? pendingMappingsValue!.AsBsonArray.Count - : 0; - pendingMappings.Should().Be(0); - } - - private async Task BuildServiceProviderAsync(bool enableDetailMapping = true) - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddCertCcConnector(options => - { - options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/"); - options.SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = TimeSpan.FromDays(1), - Overlap = TimeSpan.Zero, - InitialBackfill = TimeSpan.FromDays(1), - MinimumWindowSize = TimeSpan.FromHours(6), - }; - options.MaxMonthlySummaries = 1; - options.MaxNotesPerFetch = 5; - options.DetailRequestDelay = TimeSpan.Zero; - options.EnableDetailMapping = enableDetailMapping; - }); - - services.Configure(CertCcOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedSummaryResponses(string summaryEtag = "\"summary-oct\"", string yearlyEtag = "\"summary-year\"") - { - AddJsonResponse(MonthlySummaryUri, ReadFixture("summary-2025-10.json"), summaryEtag); - AddJsonResponse(YearlySummaryUri, ReadFixture("summary-2025.json"), yearlyEtag); - } - - private void SeedSummaryNotModifiedResponses(string summaryEtag, string yearlyEtag) - { - AddNotModifiedResponse(MonthlySummaryUri, summaryEtag); - AddNotModifiedResponse(YearlySummaryUri, yearlyEtag); - } - - private void SeedDetailResponses( - string detailEtag = "\"note-etag\"", - string vendorsEtag = "\"vendors-etag\"", - string vulsEtag = "\"vuls-etag\"", - string vendorStatusesEtag = "\"vendor-statuses-etag\"", - HttpStatusCode vendorsStatus = HttpStatusCode.OK, - HttpStatusCode vulsStatus = HttpStatusCode.OK, - HttpStatusCode vendorStatusesStatus = HttpStatusCode.OK) - { - AddJsonResponse(NoteDetailUri, ReadFixture("vu-294418.json"), detailEtag); - - if (vendorsStatus == HttpStatusCode.OK) - { - AddJsonResponse(VendorsUri, ReadFixture("vu-294418-vendors.json"), vendorsEtag); - } - else - { - _handler.AddResponse(VendorsUri, () => - { - var response = new HttpResponseMessage(vendorsStatus) - { - Content = new StringContent("vendors error", Encoding.UTF8, "text/plain"), - }; - response.Headers.ETag = new EntityTagHeaderValue(vendorsEtag); - return response; - }); - } - - if (vulsStatus == HttpStatusCode.OK) - { - AddJsonResponse(VulsUri, ReadFixture("vu-294418-vuls.json"), vulsEtag); - } - else - { - _handler.AddResponse(VulsUri, () => - { - var response = new HttpResponseMessage(vulsStatus) - { - Content = new StringContent("vuls error", Encoding.UTF8, "text/plain"), - }; - response.Headers.ETag = new EntityTagHeaderValue(vulsEtag); - return response; - }); - } - - if (vendorStatusesStatus == HttpStatusCode.OK) - { - AddJsonResponse(VendorStatusesUri, ReadFixture("vendor-statuses-294418.json"), vendorStatusesEtag); - } - else - { - _handler.AddResponse(VendorStatusesUri, () => - { - var response = new HttpResponseMessage(vendorStatusesStatus) - { - Content = new StringContent("vendor statuses error", Encoding.UTF8, "text/plain"), - }; - response.Headers.ETag = new EntityTagHeaderValue(vendorStatusesEtag); - return response; - }); - } - } - - private void SeedDetailNotModifiedResponses(string detailEtag, string vendorsEtag, string vulsEtag, string vendorStatusesEtag) - { - AddNotModifiedResponse(NoteDetailUri, detailEtag); - AddNotModifiedResponse(VendorsUri, vendorsEtag); - AddNotModifiedResponse(VulsUri, vulsEtag); - AddNotModifiedResponse(VendorStatusesUri, vendorStatusesEtag); - } - - private void AddJsonResponse(Uri uri, string json, string etag) - { - _handler.AddResponse(uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(json, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private void AddNotModifiedResponse(Uri uri, string etag) - { - _handler.AddResponse(uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.NotModified); - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private static string ReadFixture(string filename) - { - var baseDirectory = AppContext.BaseDirectory; - var candidate = Path.Combine(baseDirectory, "Source", "CertCc", "Fixtures", filename); - if (File.Exists(candidate)) - { - return File.ReadAllText(candidate); - } - - var fallback = Path.Combine(baseDirectory, "Fixtures", filename); - return File.ReadAllText(fallback); - } -} +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Connector.CertCc; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Cursors; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.CertCc; + +[Collection("mongo-fixture")] +public sealed class CertCcConnectorTests : IAsyncLifetime +{ + private static readonly Uri MonthlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/10/summary/"); + private static readonly Uri YearlySummaryUri = new("https://www.kb.cert.org/vuls/api/2025/summary/"); + private static readonly Uri NoteDetailUri = new("https://www.kb.cert.org/vuls/api/294418/"); + private static readonly Uri VendorsUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/"); + private static readonly Uri VulsUri = new("https://www.kb.cert.org/vuls/api/294418/vuls/"); + private static readonly Uri VendorStatusesUri = new("https://www.kb.cert.org/vuls/api/294418/vendors/vuls/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public CertCcConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 11, 9, 30, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().NotBeNull(); + advisories.Should().HaveCountGreaterThan(0); + + var advisory = advisories.FirstOrDefault(a => a.AdvisoryKey == "certcc/vu-294418"); + advisory.Should().NotBeNull(); + advisory!.Title.Should().ContainEquivalentOf("DrayOS"); + advisory.Summary.Should().NotBeNullOrWhiteSpace(); + advisory.Aliases.Should().Contain("VU#294418"); + advisory.Aliases.Should().Contain("CVE-2025-10547"); + advisory.AffectedPackages.Should().NotBeNull(); + advisory.AffectedPackages.Should().HaveCountGreaterThan(0); + advisory.AffectedPackages![0].NormalizedVersions.Should().NotBeNull(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().Be(0); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappings.Should().Be(0); + } + + [Fact] + public async Task Fetch_PersistsSummaryAndDetailDocuments() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + + var summaryDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, MonthlySummaryUri.ToString(), CancellationToken.None); + summaryDocument.Should().NotBeNull(); + summaryDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + + var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); + noteDocument.Should().NotBeNull(); + noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + noteDocument.Metadata.Should().NotBeNull(); + noteDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("note"); + noteDocument.Metadata.Should().ContainKey("certcc.noteId").WhoseValue.Should().Be("294418"); + noteDocument.Metadata.Should().ContainKey("certcc.vuid").WhoseValue.Should().Be("VU#294418"); + + var vendorsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorsUri.ToString(), CancellationToken.None); + vendorsDocument.Should().NotBeNull(); + vendorsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vendorsDocument.Metadata.Should().NotBeNull(); + vendorsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors"); + + var vulsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VulsUri.ToString(), CancellationToken.None); + vulsDocument.Should().NotBeNull(); + vulsDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vulsDocument.Metadata.Should().NotBeNull(); + vulsDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vuls"); + + var vendorStatusesDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorStatusesUri.ToString(), CancellationToken.None); + vendorStatusesDocument.Should().NotBeNull(); + vendorStatusesDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + vendorStatusesDocument.Metadata.Should().NotBeNull(); + vendorStatusesDocument.Metadata!.Should().ContainKey("certcc.endpoint").WhoseValue.Should().Be("vendors-vuls"); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + var pendingNotesCount = state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue!.AsBsonArray.Count + : 0; + pendingNotesCount.Should().Be(0); + var pendingSummariesCount = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummariesCount.Should().Be(0); + + var pendingDocumentsCount = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) + ? pendingDocumentsValue!.AsBsonArray.Count + : 0; + pendingDocumentsCount.Should().Be(4); + + var pendingMappingsCount = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappingsCount.Should().Be(0); + + _handler.Requests.Should().Contain(request => request.Uri == NoteDetailUri); + } + + [Fact] + public async Task Fetch_ReusesConditionalRequestsOnSubsequentRun() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(summaryEtag: "\"summary-oct\"", yearlyEtag: "\"summary-year\""); + SeedDetailResponses(detailEtag: "\"note-etag\"", vendorsEtag: "\"vendors-etag\"", vulsEtag: "\"vuls-etag\"", vendorStatusesEtag: "\"vendor-statuses-etag\""); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + _handler.Clear(); + SeedSummaryNotModifiedResponses("\"summary-oct\"", "\"summary-year\""); + SeedDetailNotModifiedResponses("\"note-etag\"", "\"vendors-etag\"", "\"vuls-etag\"", "\"vendor-statuses-etag\""); + _timeProvider.Advance(TimeSpan.FromMinutes(15)); + + await connector.FetchAsync(provider, CancellationToken.None); + + var requests = _handler.Requests.ToArray(); + requests.Should().OnlyContain(r => + r.Uri == MonthlySummaryUri + || r.Uri == YearlySummaryUri + || r.Uri == NoteDetailUri + || r.Uri == VendorsUri + || r.Uri == VulsUri + || r.Uri == VendorStatusesUri); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingNotesCount = state!.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue) + ? pendingNotesValue!.AsBsonArray.Count + : 0; + pendingNotesCount.Should().Be(0); + var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummaries.Should().Be(0); + + var pendingDocuments = state.Cursor.TryGetValue("pendingDocuments", out var pendingDocumentsValue) + ? pendingDocumentsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().BeGreaterThan(0); + } + + [Fact] + public async Task Fetch_DetailFailureRecordsBackoffAndKeepsPendingNote() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses(vendorsStatus: HttpStatusCode.InternalServerError); + + var connector = provider.GetRequiredService(); + var failure = await Assert.ThrowsAnyAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + Assert.True(failure is HttpRequestException || failure is InvalidOperationException); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.FailCount.Should().BeGreaterThan(0); + state.BackoffUntil.Should().NotBeNull(); + state.BackoffUntil.Should().BeAfter(_timeProvider.GetUtcNow()); + state.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue).Should().BeTrue(); + pendingNotesValue!.AsBsonArray.Should().Contain(value => value.AsString == "294418"); + var pendingSummaries = state.Cursor.TryGetValue("pendingSummaries", out var pendingSummariesValue) + ? pendingSummariesValue!.AsBsonArray.Count + : 0; + pendingSummaries.Should().Be(0); + } + + [Fact] + public async Task Fetch_PartialDetailEndpointsMissing_CompletesAndMaps() + { + await using var provider = await BuildServiceProviderAsync(); + SeedSummaryResponses(); + SeedDetailResponses( + vulsStatus: HttpStatusCode.NotFound, + vendorStatusesStatus: HttpStatusCode.NotFound); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().NotBeNull(); + advisories!.Should().Contain(advisory => advisory.AdvisoryKey == "certcc/vu-294418"); + + var documentStore = provider.GetRequiredService(); + var vendorsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VendorsUri.ToString(), CancellationToken.None); + vendorsDocument.Should().NotBeNull(); + vendorsDocument!.Status.Should().Be(DocumentStatuses.Mapped); + var vulsDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, VulsUri.ToString(), CancellationToken.None); + vulsDocument.Should().BeNull(); + var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); + noteDocument.Should().NotBeNull(); + noteDocument!.Status.Should().Be(DocumentStatuses.Mapped); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.TryGetValue("pendingNotes", out var pendingNotesValue).Should().BeTrue(); + pendingNotesValue!.AsBsonArray.Should().BeEmpty(); + state.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue).Should().BeTrue(); + pendingDocsValue!.AsBsonArray.Should().BeEmpty(); + state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue).Should().BeTrue(); + pendingMappingsValue!.AsBsonArray.Should().BeEmpty(); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } + + [Fact] + public async Task ParseAndMap_SkipWhenDetailMappingDisabled() + { + await using var provider = await BuildServiceProviderAsync(enableDetailMapping: false); + SeedSummaryResponses(); + SeedDetailResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + advisories.Should().BeNullOrEmpty(); + + var documentStore = provider.GetRequiredService(); + var noteDocument = await documentStore.FindBySourceAndUriAsync(CertCcConnectorPlugin.SourceName, NoteDetailUri.ToString(), CancellationToken.None); + noteDocument.Should().NotBeNull(); + noteDocument!.Status.Should().Be(DocumentStatuses.PendingParse); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(CertCcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + var pendingDocuments = state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocsValue) + ? pendingDocsValue!.AsBsonArray.Count + : 0; + pendingDocuments.Should().BeGreaterThan(0); + var pendingMappings = state.Cursor.TryGetValue("pendingMappings", out var pendingMappingsValue) + ? pendingMappingsValue!.AsBsonArray.Count + : 0; + pendingMappings.Should().Be(0); + } + + private async Task BuildServiceProviderAsync(bool enableDetailMapping = true) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddCertCcConnector(options => + { + options.BaseApiUri = new Uri("https://www.kb.cert.org/vuls/api/"); + options.SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(1), + Overlap = TimeSpan.Zero, + InitialBackfill = TimeSpan.FromDays(1), + MinimumWindowSize = TimeSpan.FromHours(6), + }; + options.MaxMonthlySummaries = 1; + options.MaxNotesPerFetch = 5; + options.DetailRequestDelay = TimeSpan.Zero; + options.EnableDetailMapping = enableDetailMapping; + }); + + services.Configure(CertCcOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedSummaryResponses(string summaryEtag = "\"summary-oct\"", string yearlyEtag = "\"summary-year\"") + { + AddJsonResponse(MonthlySummaryUri, ReadFixture("summary-2025-10.json"), summaryEtag); + AddJsonResponse(YearlySummaryUri, ReadFixture("summary-2025.json"), yearlyEtag); + } + + private void SeedSummaryNotModifiedResponses(string summaryEtag, string yearlyEtag) + { + AddNotModifiedResponse(MonthlySummaryUri, summaryEtag); + AddNotModifiedResponse(YearlySummaryUri, yearlyEtag); + } + + private void SeedDetailResponses( + string detailEtag = "\"note-etag\"", + string vendorsEtag = "\"vendors-etag\"", + string vulsEtag = "\"vuls-etag\"", + string vendorStatusesEtag = "\"vendor-statuses-etag\"", + HttpStatusCode vendorsStatus = HttpStatusCode.OK, + HttpStatusCode vulsStatus = HttpStatusCode.OK, + HttpStatusCode vendorStatusesStatus = HttpStatusCode.OK) + { + AddJsonResponse(NoteDetailUri, ReadFixture("vu-294418.json"), detailEtag); + + if (vendorsStatus == HttpStatusCode.OK) + { + AddJsonResponse(VendorsUri, ReadFixture("vu-294418-vendors.json"), vendorsEtag); + } + else + { + _handler.AddResponse(VendorsUri, () => + { + var response = new HttpResponseMessage(vendorsStatus) + { + Content = new StringContent("vendors error", Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(vendorsEtag); + return response; + }); + } + + if (vulsStatus == HttpStatusCode.OK) + { + AddJsonResponse(VulsUri, ReadFixture("vu-294418-vuls.json"), vulsEtag); + } + else + { + _handler.AddResponse(VulsUri, () => + { + var response = new HttpResponseMessage(vulsStatus) + { + Content = new StringContent("vuls error", Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(vulsEtag); + return response; + }); + } + + if (vendorStatusesStatus == HttpStatusCode.OK) + { + AddJsonResponse(VendorStatusesUri, ReadFixture("vendor-statuses-294418.json"), vendorStatusesEtag); + } + else + { + _handler.AddResponse(VendorStatusesUri, () => + { + var response = new HttpResponseMessage(vendorStatusesStatus) + { + Content = new StringContent("vendor statuses error", Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(vendorStatusesEtag); + return response; + }); + } + } + + private void SeedDetailNotModifiedResponses(string detailEtag, string vendorsEtag, string vulsEtag, string vendorStatusesEtag) + { + AddNotModifiedResponse(NoteDetailUri, detailEtag); + AddNotModifiedResponse(VendorsUri, vendorsEtag); + AddNotModifiedResponse(VulsUri, vulsEtag); + AddNotModifiedResponse(VendorStatusesUri, vendorStatusesEtag); + } + + private void AddJsonResponse(Uri uri, string json, string etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string etag) + { + _handler.AddResponse(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private static string ReadFixture(string filename) + { + var baseDirectory = AppContext.BaseDirectory; + var candidate = Path.Combine(baseDirectory, "Source", "CertCc", "Fixtures", filename); + if (File.Exists(candidate)) + { + return File.ReadAllText(candidate); + } + + var fallback = Path.Combine(baseDirectory, "Fixtures", filename); + return File.ReadAllText(fallback); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json similarity index 97% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json index 3608cd5f..c4eebb9a 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json @@ -1,374 +1,374 @@ -[ - { - "advisoryKey": "certcc/vu-294418", - "affectedPackages": [ - { - "type": "vendor", - "identifier": "DrayTek Corporation", - "platform": null, - "versionRanges": [ - { - "fixedVersion": null, - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": null, - "vendorExtensions": { - "certcc.vendor.name": "DrayTek Corporation", - "certcc.vendor.statement.raw": "The issue is confirmed, and here is the patch list\nV3912/V3910/V2962/V1000B 4.4.3.6/4.4.5.1\nV2927/V2865/V2866 4.5.1\nV2765/V2766/V2763/V2135 4.5.1\nV2915 4.4.6.1\nV2862/V2926 3.9.9.12\nV2952/3220 3.9.8.8\nV2860/V2925 3.9.8.6\nV2133/V2762/V2832 3.9.9.4\nV2620/LTE200 3.9.9.5", - "certcc.vendor.contactDate": "2025-09-15T19:03:33.6643450+00:00", - "certcc.vendor.statementDate": "2025-09-16T02:27:51.3463350+00:00", - "certcc.vendor.updated": "2025-10-03T11:35:31.1906610+00:00", - "certcc.vendor.statuses": "CVE-2025-10547=affected", - "certcc.vendor.patches": "3220=3.9.8.8;LTE200=3.9.9.5;V1000B=4.4.5.1;V2133=3.9.9.4;V2135=4.5.1;V2620=3.9.9.5;V2762=3.9.9.4;V2763=4.5.1;V2765=4.5.1;V2766=4.5.1;V2832=3.9.9.4;V2860=3.9.8.6;V2862=3.9.9.12;V2865=4.5.1;V2866=4.5.1;V2915=4.4.6.1;V2925=3.9.8.6;V2926=3.9.9.12;V2927=4.5.1;V2952=3.9.8.8;V2962=4.4.5.1;V3910=4.4.3.6;V3912=4.4.3.6" - } - }, - "provenance": { - "source": "cert-cc", - "kind": "vendor-range", - "value": "DrayTek Corporation", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": null, - "rangeKind": "vendor" - } - ], - "normalizedVersions": [ - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.8.6", - "notes": "DrayTek Corporation::V2860" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.8.6", - "notes": "DrayTek Corporation::V2925" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.8.8", - "notes": "DrayTek Corporation::3220" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.8.8", - "notes": "DrayTek Corporation::V2952" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.12", - "notes": "DrayTek Corporation::V2862" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.12", - "notes": "DrayTek Corporation::V2926" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.4", - "notes": "DrayTek Corporation::V2133" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.4", - "notes": "DrayTek Corporation::V2762" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.4", - "notes": "DrayTek Corporation::V2832" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.5", - "notes": "DrayTek Corporation::LTE200" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "3.9.9.5", - "notes": "DrayTek Corporation::V2620" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.4.3.6", - "notes": "DrayTek Corporation::V3910" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.4.3.6", - "notes": "DrayTek Corporation::V3912" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.4.5.1", - "notes": "DrayTek Corporation::V1000B" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.4.5.1", - "notes": "DrayTek Corporation::V2962" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.4.6.1", - "notes": "DrayTek Corporation::V2915" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2135" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2763" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2765" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2766" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2865" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2866" - }, - { - "scheme": "certcc.vendor", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "4.5.1", - "notes": "DrayTek Corporation::V2927" - } - ], - "statuses": [ - { - "provenance": { - "source": "cert-cc", - "kind": "vendor-status", - "value": "DrayTek Corporation:CVE-2025-10547", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - "status": "affected" - } - ], - "provenance": [ - { - "source": "cert-cc", - "kind": "vendor", - "value": "DrayTek Corporation", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "aliases": [ - "CVE-2025-10547", - "VU#294418" - ], - "credits": [], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-03T11:40:09.876722+00:00", - "provenance": [ - { - "source": "cert-cc", - "kind": "document", - "value": "https://www.kb.cert.org/vuls/api/294418/", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - { - "source": "cert-cc", - "kind": "map", - "value": "VU#294418", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - } - ], - "published": "2025-10-03T11:35:31.026053+00:00", - "references": [ - { - "kind": "reference", - "provenance": { - "source": "cert-cc", - "kind": "reference", - "value": "https://www.kb.cert.org/vuls/id/294418", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "certcc.public", - "summary": null, - "url": "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/" - }, - { - "kind": "reference", - "provenance": { - "source": "cert-cc", - "kind": "reference", - "value": "https://www.kb.cert.org/vuls/id/294418", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "certcc.public", - "summary": null, - "url": "https://www.draytek.com/support/resources?type=version" - }, - { - "kind": "advisory", - "provenance": { - "source": "cert-cc", - "kind": "reference", - "value": "https://www.kb.cert.org/vuls/id/294418", - "decisionReason": null, - "recordedAt": "2025-11-01T08:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "certcc.note", - "summary": null, - "url": "https://www.kb.cert.org/vuls/id/294418" - } - ], - "severity": null, - "summary": "Overview\nA remote code execution (RCE) vulnerability, tracked as CVE-2025-10547, was discovered through the EasyVPN and LAN web administration interface of Vigor routers by Draytek. A script in the LAN web administration interface uses an unitialized variable, allowing an attacker to send specially crafted HTTP requests that cause memory corruption and potentially allow arbitrary code execution.\nDescription\nVigor routers are business-grade routers, designed for small to medium-sized businesses, made by Draytek. These routers provide routing, firewall, VPN, content-filtering, bandwidth management, LAN (local area network), and multi-WAN (wide area network) features. Draytek utilizes a proprietary firmware, DrayOS, on the Vigor router line. DrayOS features the EasyVPN and LAN Web Administrator tool s to facilitate LAN and VPN setup. According to the DrayTek website, \"with EasyVPN, users no longer need to generate WireGuard keys, import OpenVPN configuration files, or upload certificates. Instead, VPN can be successfully established by simply entering the username and password or getting the OTP code by email.\"\nThe LAN Web Administrator provides a browser-based user interface for router management. When a user interacts with the LAN Web Administration interface, the user interface elements trigger actions that generate HTTP requests to interact with the local server. This process contains an uninitialized variable. Due to the uninitialized variable, an unauthenticated attacker could perform memory corruption on the router via specially crafted HTTP requests to hijack execution or inject malicious payloads. If EasyVPN is enabled, the flaw could be remotely exploited through the VPN interface.\nImpact\nA remote, unathenticated attacker can exploit this vulnerability through accessing the LAN interface—or potentially the WAN interface—if EasyVPN is enabled or remote administration over the internet is activated. If a remote, unauthenticated attacker leverages this vulnerability, they can execute arbitrary code on the router (RCE) and gain full control of the device. A successful attack could result in a attacker gaining root access to a Vigor router to then install backdoors, reconfigure network settings, or block traffic. An attacker may also pivot for lateral movement via intercepting internal communications and bypassing VPNs.\nSolution\nThe DrayTek Security team has developed a series of patches to remediate the vulnerability, and all users of Vigor routers should upgrade to the latest version ASAP. The patches can be found on the resources page of the DrayTek webpage, and the security advisory can be found within the about section of the DrayTek webpage. Consult either the CVE listing or the advisory page for a full list of affected products.\nAcknowledgements\nThanks to the reporter, Pierre-Yves MAES of ChapsVision (pymaes@chapsvision.com). This document was written by Ayushi Kriplani.", - "title": "Vigor routers running DrayOS are vulnerable to RCE via EasyVPN and LAN web administration interface" - } +[ + { + "advisoryKey": "certcc/vu-294418", + "affectedPackages": [ + { + "type": "vendor", + "identifier": "DrayTek Corporation", + "platform": null, + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certcc.vendor.name": "DrayTek Corporation", + "certcc.vendor.statement.raw": "The issue is confirmed, and here is the patch list\nV3912/V3910/V2962/V1000B 4.4.3.6/4.4.5.1\nV2927/V2865/V2866 4.5.1\nV2765/V2766/V2763/V2135 4.5.1\nV2915 4.4.6.1\nV2862/V2926 3.9.9.12\nV2952/3220 3.9.8.8\nV2860/V2925 3.9.8.6\nV2133/V2762/V2832 3.9.9.4\nV2620/LTE200 3.9.9.5", + "certcc.vendor.contactDate": "2025-09-15T19:03:33.6643450+00:00", + "certcc.vendor.statementDate": "2025-09-16T02:27:51.3463350+00:00", + "certcc.vendor.updated": "2025-10-03T11:35:31.1906610+00:00", + "certcc.vendor.statuses": "CVE-2025-10547=affected", + "certcc.vendor.patches": "3220=3.9.8.8;LTE200=3.9.9.5;V1000B=4.4.5.1;V2133=3.9.9.4;V2135=4.5.1;V2620=3.9.9.5;V2762=3.9.9.4;V2763=4.5.1;V2765=4.5.1;V2766=4.5.1;V2832=3.9.9.4;V2860=3.9.8.6;V2862=3.9.9.12;V2865=4.5.1;V2866=4.5.1;V2915=4.4.6.1;V2925=3.9.8.6;V2926=3.9.9.12;V2927=4.5.1;V2952=3.9.8.8;V2962=4.4.5.1;V3910=4.4.3.6;V3912=4.4.3.6" + } + }, + "provenance": { + "source": "cert-cc", + "kind": "vendor-range", + "value": "DrayTek Corporation", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ], + "normalizedVersions": [ + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.8.6", + "notes": "DrayTek Corporation::V2860" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.8.6", + "notes": "DrayTek Corporation::V2925" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.8.8", + "notes": "DrayTek Corporation::3220" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.8.8", + "notes": "DrayTek Corporation::V2952" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.12", + "notes": "DrayTek Corporation::V2862" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.12", + "notes": "DrayTek Corporation::V2926" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.4", + "notes": "DrayTek Corporation::V2133" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.4", + "notes": "DrayTek Corporation::V2762" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.4", + "notes": "DrayTek Corporation::V2832" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.5", + "notes": "DrayTek Corporation::LTE200" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "3.9.9.5", + "notes": "DrayTek Corporation::V2620" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.4.3.6", + "notes": "DrayTek Corporation::V3910" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.4.3.6", + "notes": "DrayTek Corporation::V3912" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.4.5.1", + "notes": "DrayTek Corporation::V1000B" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.4.5.1", + "notes": "DrayTek Corporation::V2962" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.4.6.1", + "notes": "DrayTek Corporation::V2915" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2135" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2763" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2765" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2766" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2865" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2866" + }, + { + "scheme": "certcc.vendor", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "4.5.1", + "notes": "DrayTek Corporation::V2927" + } + ], + "statuses": [ + { + "provenance": { + "source": "cert-cc", + "kind": "vendor-status", + "value": "DrayTek Corporation:CVE-2025-10547", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + "status": "affected" + } + ], + "provenance": [ + { + "source": "cert-cc", + "kind": "vendor", + "value": "DrayTek Corporation", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-10547", + "VU#294418" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-03T11:40:09.876722+00:00", + "provenance": [ + { + "source": "cert-cc", + "kind": "document", + "value": "https://www.kb.cert.org/vuls/api/294418/", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + { + "source": "cert-cc", + "kind": "map", + "value": "VU#294418", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-03T11:35:31.026053+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "source": "cert-cc", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/294418", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "certcc.public", + "summary": null, + "url": "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/" + }, + { + "kind": "reference", + "provenance": { + "source": "cert-cc", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/294418", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "certcc.public", + "summary": null, + "url": "https://www.draytek.com/support/resources?type=version" + }, + { + "kind": "advisory", + "provenance": { + "source": "cert-cc", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/294418", + "decisionReason": null, + "recordedAt": "2025-11-01T08:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "certcc.note", + "summary": null, + "url": "https://www.kb.cert.org/vuls/id/294418" + } + ], + "severity": null, + "summary": "Overview\nA remote code execution (RCE) vulnerability, tracked as CVE-2025-10547, was discovered through the EasyVPN and LAN web administration interface of Vigor routers by Draytek. A script in the LAN web administration interface uses an unitialized variable, allowing an attacker to send specially crafted HTTP requests that cause memory corruption and potentially allow arbitrary code execution.\nDescription\nVigor routers are business-grade routers, designed for small to medium-sized businesses, made by Draytek. These routers provide routing, firewall, VPN, content-filtering, bandwidth management, LAN (local area network), and multi-WAN (wide area network) features. Draytek utilizes a proprietary firmware, DrayOS, on the Vigor router line. DrayOS features the EasyVPN and LAN Web Administrator tool s to facilitate LAN and VPN setup. According to the DrayTek website, \"with EasyVPN, users no longer need to generate WireGuard keys, import OpenVPN configuration files, or upload certificates. Instead, VPN can be successfully established by simply entering the username and password or getting the OTP code by email.\"\nThe LAN Web Administrator provides a browser-based user interface for router management. When a user interacts with the LAN Web Administration interface, the user interface elements trigger actions that generate HTTP requests to interact with the local server. This process contains an uninitialized variable. Due to the uninitialized variable, an unauthenticated attacker could perform memory corruption on the router via specially crafted HTTP requests to hijack execution or inject malicious payloads. If EasyVPN is enabled, the flaw could be remotely exploited through the VPN interface.\nImpact\nA remote, unathenticated attacker can exploit this vulnerability through accessing the LAN interface—or potentially the WAN interface—if EasyVPN is enabled or remote administration over the internet is activated. If a remote, unauthenticated attacker leverages this vulnerability, they can execute arbitrary code on the router (RCE) and gain full control of the device. A successful attack could result in a attacker gaining root access to a Vigor router to then install backdoors, reconfigure network settings, or block traffic. An attacker may also pivot for lateral movement via intercepting internal communications and bypassing VPNs.\nSolution\nThe DrayTek Security team has developed a series of patches to remediate the vulnerability, and all users of Vigor routers should upgrade to the latest version ASAP. The patches can be found on the resources page of the DrayTek webpage, and the security advisory can be found within the about section of the DrayTek webpage. Consult either the CVE listing or the advisory page for a full list of affected products.\nAcknowledgements\nThanks to the reporter, Pierre-Yves MAES of ChapsVision (pymaes@chapsvision.com). This document was written by Ayushi Kriplani.", + "title": "Vigor routers running DrayOS are vulnerable to RCE via EasyVPN and LAN web administration interface" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-documents.snapshot.json similarity index 97% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-documents.snapshot.json index 8019883c..36d6251f 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-documents.snapshot.json @@ -1,106 +1,106 @@ -[ - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-summary-2025-09\"", - "lastModified": "2025-09-30T12:00:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.month": "09", - "certcc.scope": "monthly", - "certcc.year": "2025", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "0475f0766d6b96d7dc7683cf6b418055c8ecbef88a73ab5d75ce428fbd0900fc", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/2025/09/summary/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-summary-2025-10\"", - "lastModified": "2025-10-31T12:00:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.month": "10", - "certcc.scope": "monthly", - "certcc.year": "2025", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "363e3ddcd31770e5f41913328318ca0e5bf384bb059d5673ba14392f29f7296f", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-summary-2025\"", - "lastModified": "2025-10-31T12:01:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.scope": "yearly", - "certcc.year": "2025", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "363e3ddcd31770e5f41913328318ca0e5bf384bb059d5673ba14392f29f7296f", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-note-294418\"", - "lastModified": "2025-10-09T16:52:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.endpoint": "note", - "certcc.noteId": "294418", - "certcc.vuid": "VU#294418", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "5dd5c9bcd6ed6f20a2fc07a308af9f420b9a07120fe5934de2a1c26724eb36d3", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/294418/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-vendors-294418\"", - "lastModified": "2025-10-09T17:05:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.endpoint": "vendors", - "certcc.noteId": "294418", - "certcc.vuid": "VU#294418", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "b81aad835ab289c2ac68262825d0f0d5eb9212bc7b3569c84921d0fe5160734f", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-vendor-statuses-294418\"", - "lastModified": "2025-10-09T17:12:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.endpoint": "vendors-vuls", - "certcc.noteId": "294418", - "certcc.vuid": "VU#294418", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "6ad928c8a1b0410693417869d83062347747a79da6946404d94d14a2458c23ea", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/vuls/" - }, - { - "contentType": "application/json; charset=utf-8", - "etag": "\"certcc-vuls-294418\"", - "lastModified": "2025-10-09T17:10:00.0000000+00:00", - "metadata": { - "attempts": "1", - "certcc.endpoint": "vuls", - "certcc.noteId": "294418", - "certcc.vuid": "VU#294418", - "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" - }, - "sha256": "5de3b82f360e1ff06f15873f55ff10b7c4fc11ca65a5f77a3941a82018a8a7de", - "status": "pending-parse", - "uri": "https://www.kb.cert.org/vuls/api/294418/vuls/" - } +[ + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025-09\"", + "lastModified": "2025-09-30T12:00:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.month": "09", + "certcc.scope": "monthly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "0475f0766d6b96d7dc7683cf6b418055c8ecbef88a73ab5d75ce428fbd0900fc", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/09/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025-10\"", + "lastModified": "2025-10-31T12:00:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.month": "10", + "certcc.scope": "monthly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "363e3ddcd31770e5f41913328318ca0e5bf384bb059d5673ba14392f29f7296f", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-summary-2025\"", + "lastModified": "2025-10-31T12:01:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.scope": "yearly", + "certcc.year": "2025", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "363e3ddcd31770e5f41913328318ca0e5bf384bb059d5673ba14392f29f7296f", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-note-294418\"", + "lastModified": "2025-10-09T16:52:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "note", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "5dd5c9bcd6ed6f20a2fc07a308af9f420b9a07120fe5934de2a1c26724eb36d3", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-vendors-294418\"", + "lastModified": "2025-10-09T17:05:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "vendors", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "b81aad835ab289c2ac68262825d0f0d5eb9212bc7b3569c84921d0fe5160734f", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-vendor-statuses-294418\"", + "lastModified": "2025-10-09T17:12:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "vendors-vuls", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "6ad928c8a1b0410693417869d83062347747a79da6946404d94d14a2458c23ea", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/vuls/" + }, + { + "contentType": "application/json; charset=utf-8", + "etag": "\"certcc-vuls-294418\"", + "lastModified": "2025-10-09T17:10:00.0000000+00:00", + "metadata": { + "attempts": "1", + "certcc.endpoint": "vuls", + "certcc.noteId": "294418", + "certcc.vuid": "VU#294418", + "fetchedAt": "2025-11-01T08:00:00.0000000+00:00" + }, + "sha256": "5de3b82f360e1ff06f15873f55ff10b7c4fc11ca65a5f77a3941a82018a8a7de", + "status": "pending-parse", + "uri": "https://www.kb.cert.org/vuls/api/294418/vuls/" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-requests.snapshot.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-requests.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-requests.snapshot.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-requests.snapshot.json index 65782eb0..3df0ed23 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-requests.snapshot.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-requests.snapshot.json @@ -1,92 +1,92 @@ -[ - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/09/summary/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/294418/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/294418/vuls/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/vuls/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": "Fri, 31 Oct 2025 12:00:00 GMT", - "ifNoneMatch": "\"certcc-summary-2025-10\"" - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": null, - "ifNoneMatch": null - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/11/summary/" - }, - { - "headers": { - "accept": "application/json", - "ifModifiedSince": "Fri, 31 Oct 2025 12:01:00 GMT", - "ifNoneMatch": "\"certcc-summary-2025\"" - }, - "method": "GET", - "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" - } +[ + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/09/summary/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/294418/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/294418/vuls/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/294418/vendors/vuls/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": "Fri, 31 Oct 2025 12:00:00 GMT", + "ifNoneMatch": "\"certcc-summary-2025-10\"" + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/10/summary/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": null, + "ifNoneMatch": null + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/11/summary/" + }, + { + "headers": { + "accept": "application/json", + "ifModifiedSince": "Fri, 31 Oct 2025 12:01:00 GMT", + "ifNoneMatch": "\"certcc-summary-2025\"" + }, + "method": "GET", + "uri": "https://www.kb.cert.org/vuls/api/2025/summary/" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-state.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-state.snapshot.json index 2fbc20f4..ea1d2518 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-state.snapshot.json @@ -1,13 +1,13 @@ -{ - "backoffUntil": null, - "failCount": 0, - "lastFailure": null, - "lastRun": "2025-11-01T08:00:00.0000000Z", - "lastSuccess": "2025-11-01T08:00:00+00:00", - "pendingNotes": [], - "pendingSummaries": [], - "summary": { - "end": "2025-10-17T08:00:00.0000000Z", - "start": "2025-09-17T08:00:00.0000000Z" - } +{ + "backoffUntil": null, + "failCount": 0, + "lastFailure": null, + "lastRun": "2025-11-01T08:00:00.0000000Z", + "lastSuccess": "2025-11-01T08:00:00+00:00", + "pendingNotes": [], + "pendingSummaries": [], + "summary": { + "end": "2025-10-17T08:00:00.0000000Z", + "start": "2025-09-17T08:00:00.0000000Z" + } } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-09.json similarity index 90% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-09.json index af07696f..c785f81c 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-09.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-09.json @@ -1,4 +1,4 @@ -{ - "count": 0, - "notes": [] -} +{ + "count": 0, + "notes": [] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-10.json similarity index 91% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-10.json index 8f6680a4..20485ffb 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-10.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-10.json @@ -1,6 +1,6 @@ -{ - "count": 1, - "notes": [ - "VU#294418" - ] -} +{ + "count": 1, + "notes": [ + "VU#294418" + ] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-11.json similarity index 90% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-11.json index af07696f..c785f81c 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025-11.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025-11.json @@ -1,4 +1,4 @@ -{ - "count": 0, - "notes": [] -} +{ + "count": 0, + "notes": [] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025.json similarity index 91% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025.json index 8f6680a4..20485ffb 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/summary-2025.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/summary-2025.json @@ -1,6 +1,6 @@ -{ - "count": 1, - "notes": [ - "VU#294418" - ] -} +{ + "count": 1, + "notes": [ + "VU#294418" + ] +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendor-statuses-294418.json similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendor-statuses-294418.json index 4fdb02bf..7e7f8376 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendor-statuses-294418.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendor-statuses-294418.json @@ -1,11 +1,11 @@ -[ - { - "vul": "CVE-2025-10547", - "vendor": "DrayTek Corporation", - "status": "Affected", - "date_added": "2025-10-03T11:35:31.202991Z", - "dateupdated": "2025-10-03T11:40:09.944401Z", - "references": null, - "statement": null - } -] +[ + { + "vul": "CVE-2025-10547", + "vendor": "DrayTek Corporation", + "status": "Affected", + "date_added": "2025-10-03T11:35:31.202991Z", + "dateupdated": "2025-10-03T11:40:09.944401Z", + "references": null, + "statement": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendors-294418.json similarity index 98% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendors-294418.json index e26d82cd..22b5fd77 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vendors-294418.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vendors-294418.json @@ -1,12 +1,12 @@ -[ - { - "note": "294418", - "contact_date": "2025-09-15T19:03:33.664345Z", - "vendor": "DrayTek Corporation", - "references": "", - "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", - "dateupdated": "2025-10-03T11:35:31.190661Z", - "statement_date": "2025-09-16T02:27:51.346335Z", - "addendum": null - } -] +[ + { + "note": "294418", + "contact_date": "2025-09-15T19:03:33.664345Z", + "vendor": "DrayTek Corporation", + "references": "", + "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", + "dateupdated": "2025-10-03T11:35:31.190661Z", + "statement_date": "2025-09-16T02:27:51.346335Z", + "addendum": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-257161.json similarity index 98% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-257161.json index c72f66e7..b142ae16 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-257161.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-257161.json @@ -1,87 +1,87 @@ -{ - "vuid": "VU#257161", - "idnumber": "257161", - "name": "Treck IP stacks contain multiple vulnerabilities", - "keywords": null, - "overview": "### Overview\r\nTreck IP stack implementations for embedded systems are affected by multiple vulnerabilities. This set of vulnerabilities was researched and reported by JSOF, who calls them [Ripple20](https://www.jsof-tech.com/ripple20/).\r\n\r\n### Description\r\nTreck IP network stack software is designed for and used in a variety of embedded systems. The software can be licensed and integrated in various ways, including compiled from source, licensed for modification and reuse and finally as a dynamic or static linked library. Treck IP software contains multiple vulnerabilities, most of which are caused by [memory management bugs](https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152142). For more details on the vulnerabilities introduced by these bugs, see Treck's [ Vulnerability Response Information](https://treck.com/vulnerability-response-information/) and JSOF's [Ripple20 advisory](https://www.jsof-tech.com/ripple20/).\r\n\r\nHistorically-related KASAGO TCP/IP middleware from Zuken Elmic (formerly Elmic Systems) is also affected by some of these vulnerabilities. \r\n\r\nThese vulnerabilities likely affect industrial control systems and medical devices. Please see ICS-CERT Advisory [ICSA-20-168-01](https://www.us-cert.gov/ics/advisories/icsa-20-168-01) for more information.\r\n\r\n### Impact ###\r\nThe impact of these vulnerabilities will vary due to the combination of build and runtime options used while developing different embedded systems. This diversity of implementations and the lack of supply chain visibility has exasperated the problem of accurately assessing the impact of these vulnerabilities. In summary, a remote, unauthenticated attacker may be able to use specially-crafted network packets to cause a denial of service, disclose information, or execute arbitrary code.\r\n\r\n### Solution\r\n#### Apply updates\r\nUpdate to the latest stable version of Treck IP stack software (6.0.1.67 or later). Please contact Treck at . Downstream users of embedded systems that incorporate Treck IP stacks should contact their embedded system vendor.\r\n\r\n#### Block anomalous IP traffic\r\nConsider blocking network attacks via deep packet inspection. In some cases, modern switches, routers, and firewalls will drop malformed packets with no additional configuration. It is recommended that such security features are not disabled. Below is a list of possible mitigations that can be applied as appropriate to your network environment.\r\n\r\n* Normalize or reject IP fragmented packets (IP Fragments) if not supported in your environment \r\n* Disable or block IP tunneling, both IPv6-in-IPv4 or IP-in-IP tunneling if not required\r\n* Block IP source routing and any IPv6 deprecated features like routing headers (see also [VU#267289](https://www.kb.cert.org/vuls/id/267289))\r\n* Enforce TCP inspection and reject malformed TCP packets \r\n* Block unused ICMP control messages such MTU Update and Address Mask updates\r\n* Normalize DNS through a secure recursive server or application layer firewall\r\n* Ensure that you are using reliable OSI layer 2 equipment (Ethernet)\r\n* Provide DHCP/DHCPv6 security with feature like DHCP snooping\r\n* Disable or block IPv6 multicast if not used in switching infrastructure\r\n\r\nFurther recommendations are available [here](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/recommendations.md).\r\n\r\n#### Detect anomalous IP traffic\r\nSuricata IDS has built-in decoder-event rules that can be customized to detect attempts to exploit these vulnerabilities. See the rule below for an example. A larger set of selected [vu-257161.rules](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/vu-257161.rules) are available from the CERT/CC Github repository.\r\n\r\n`#IP-in-IP tunnel with fragments` \r\n`alert ip any any -> any any (msg:\"VU#257161:CVE-2020-11896, CVE-2020-11900 Fragments inside IP-in-IP tunnel https://kb.cert.org/vuls/id/257161\"; ip_proto:4; fragbits:M; sid:1367257161; rev:1;)`\r\n\r\n### Acknowledgements\r\nMoshe Kol and Shlomi Oberman of JSOF https://jsof-tech.com researched and reported these vulnerabilities. Treck worked closely with us and other stakeholders to coordinate the disclosure of these vulnerabilities.\r\n\r\nThis document was written by Vijay Sarvepalli.", - "clean_desc": null, - "impact": null, - "resolution": null, - "workarounds": null, - "sysaffected": null, - "thanks": null, - "author": null, - "public": [ - "https://www.jsof-tech.com/ripple20/", - "https://treck.com/vulnerability-response-information/", - "https://www.us-cert.gov/ics/advisories/icsa-20-168-01", - "https://jvn.jp/vu/JVNVU94736763/index.html" - ], - "cveids": [ - "CVE-2020-11902", - "CVE-2020-11913", - "CVE-2020-11898", - "CVE-2020-11907", - "CVE-2020-11901", - "CVE-2020-11903", - "CVE-2020-11904", - "CVE-2020-11906", - "CVE-2020-11910", - "CVE-2020-11911", - "CVE-2020-11912", - "CVE-2020-11914", - "CVE-2020-11899", - "CVE-2020-11896", - "CVE-2020-11897", - "CVE-2020-11905", - "CVE-2020-11908", - "CVE-2020-11900", - "CVE-2020-11909", - "CVE-2020-0597", - "CVE-2020-0595", - "CVE-2020-8674", - "CVE-2020-0594" - ], - "certadvisory": null, - "uscerttechnicalalert": null, - "datecreated": "2020-06-16T17:13:53.220714Z", - "publicdate": "2020-06-16T00:00:00Z", - "datefirstpublished": "2020-06-16T17:13:53.238540Z", - "dateupdated": "2022-09-20T01:54:35.485507Z", - "revision": 48, - "vrda_d1_directreport": null, - "vrda_d1_population": null, - "vrda_d1_impact": null, - "cam_widelyknown": null, - "cam_exploitation": null, - "cam_internetinfrastructure": null, - "cam_population": null, - "cam_impact": null, - "cam_easeofexploitation": null, - "cam_attackeraccessrequired": null, - "cam_scorecurrent": null, - "cam_scorecurrentwidelyknown": null, - "cam_scorecurrentwidelyknownexploited": null, - "ipprotocol": null, - "cvss_accessvector": null, - "cvss_accesscomplexity": null, - "cvss_authentication": null, - "cvss_confidentialityimpact": null, - "cvss_integrityimpact": null, - "cvss_availabilityimpact": null, - "cvss_exploitablity": null, - "cvss_remediationlevel": null, - "cvss_reportconfidence": null, - "cvss_collateraldamagepotential": null, - "cvss_targetdistribution": null, - "cvss_securityrequirementscr": null, - "cvss_securityrequirementsir": null, - "cvss_securityrequirementsar": null, - "cvss_basescore": null, - "cvss_basevector": null, - "cvss_temporalscore": null, - "cvss_environmentalscore": null, - "cvss_environmentalvector": null, - "metric": null, - "vulnote": 7 -} +{ + "vuid": "VU#257161", + "idnumber": "257161", + "name": "Treck IP stacks contain multiple vulnerabilities", + "keywords": null, + "overview": "### Overview\r\nTreck IP stack implementations for embedded systems are affected by multiple vulnerabilities. This set of vulnerabilities was researched and reported by JSOF, who calls them [Ripple20](https://www.jsof-tech.com/ripple20/).\r\n\r\n### Description\r\nTreck IP network stack software is designed for and used in a variety of embedded systems. The software can be licensed and integrated in various ways, including compiled from source, licensed for modification and reuse and finally as a dynamic or static linked library. Treck IP software contains multiple vulnerabilities, most of which are caused by [memory management bugs](https://wiki.sei.cmu.edu/confluence/pages/viewpage.action?pageId=87152142). For more details on the vulnerabilities introduced by these bugs, see Treck's [ Vulnerability Response Information](https://treck.com/vulnerability-response-information/) and JSOF's [Ripple20 advisory](https://www.jsof-tech.com/ripple20/).\r\n\r\nHistorically-related KASAGO TCP/IP middleware from Zuken Elmic (formerly Elmic Systems) is also affected by some of these vulnerabilities. \r\n\r\nThese vulnerabilities likely affect industrial control systems and medical devices. Please see ICS-CERT Advisory [ICSA-20-168-01](https://www.us-cert.gov/ics/advisories/icsa-20-168-01) for more information.\r\n\r\n### Impact ###\r\nThe impact of these vulnerabilities will vary due to the combination of build and runtime options used while developing different embedded systems. This diversity of implementations and the lack of supply chain visibility has exasperated the problem of accurately assessing the impact of these vulnerabilities. In summary, a remote, unauthenticated attacker may be able to use specially-crafted network packets to cause a denial of service, disclose information, or execute arbitrary code.\r\n\r\n### Solution\r\n#### Apply updates\r\nUpdate to the latest stable version of Treck IP stack software (6.0.1.67 or later). Please contact Treck at . Downstream users of embedded systems that incorporate Treck IP stacks should contact their embedded system vendor.\r\n\r\n#### Block anomalous IP traffic\r\nConsider blocking network attacks via deep packet inspection. In some cases, modern switches, routers, and firewalls will drop malformed packets with no additional configuration. It is recommended that such security features are not disabled. Below is a list of possible mitigations that can be applied as appropriate to your network environment.\r\n\r\n* Normalize or reject IP fragmented packets (IP Fragments) if not supported in your environment \r\n* Disable or block IP tunneling, both IPv6-in-IPv4 or IP-in-IP tunneling if not required\r\n* Block IP source routing and any IPv6 deprecated features like routing headers (see also [VU#267289](https://www.kb.cert.org/vuls/id/267289))\r\n* Enforce TCP inspection and reject malformed TCP packets \r\n* Block unused ICMP control messages such MTU Update and Address Mask updates\r\n* Normalize DNS through a secure recursive server or application layer firewall\r\n* Ensure that you are using reliable OSI layer 2 equipment (Ethernet)\r\n* Provide DHCP/DHCPv6 security with feature like DHCP snooping\r\n* Disable or block IPv6 multicast if not used in switching infrastructure\r\n\r\nFurther recommendations are available [here](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/recommendations.md).\r\n\r\n#### Detect anomalous IP traffic\r\nSuricata IDS has built-in decoder-event rules that can be customized to detect attempts to exploit these vulnerabilities. See the rule below for an example. A larger set of selected [vu-257161.rules](https://github.com/CERTCC/PoC-Exploits/blob/master/vu-257161/vu-257161.rules) are available from the CERT/CC Github repository.\r\n\r\n`#IP-in-IP tunnel with fragments` \r\n`alert ip any any -> any any (msg:\"VU#257161:CVE-2020-11896, CVE-2020-11900 Fragments inside IP-in-IP tunnel https://kb.cert.org/vuls/id/257161\"; ip_proto:4; fragbits:M; sid:1367257161; rev:1;)`\r\n\r\n### Acknowledgements\r\nMoshe Kol and Shlomi Oberman of JSOF https://jsof-tech.com researched and reported these vulnerabilities. Treck worked closely with us and other stakeholders to coordinate the disclosure of these vulnerabilities.\r\n\r\nThis document was written by Vijay Sarvepalli.", + "clean_desc": null, + "impact": null, + "resolution": null, + "workarounds": null, + "sysaffected": null, + "thanks": null, + "author": null, + "public": [ + "https://www.jsof-tech.com/ripple20/", + "https://treck.com/vulnerability-response-information/", + "https://www.us-cert.gov/ics/advisories/icsa-20-168-01", + "https://jvn.jp/vu/JVNVU94736763/index.html" + ], + "cveids": [ + "CVE-2020-11902", + "CVE-2020-11913", + "CVE-2020-11898", + "CVE-2020-11907", + "CVE-2020-11901", + "CVE-2020-11903", + "CVE-2020-11904", + "CVE-2020-11906", + "CVE-2020-11910", + "CVE-2020-11911", + "CVE-2020-11912", + "CVE-2020-11914", + "CVE-2020-11899", + "CVE-2020-11896", + "CVE-2020-11897", + "CVE-2020-11905", + "CVE-2020-11908", + "CVE-2020-11900", + "CVE-2020-11909", + "CVE-2020-0597", + "CVE-2020-0595", + "CVE-2020-8674", + "CVE-2020-0594" + ], + "certadvisory": null, + "uscerttechnicalalert": null, + "datecreated": "2020-06-16T17:13:53.220714Z", + "publicdate": "2020-06-16T00:00:00Z", + "datefirstpublished": "2020-06-16T17:13:53.238540Z", + "dateupdated": "2022-09-20T01:54:35.485507Z", + "revision": 48, + "vrda_d1_directreport": null, + "vrda_d1_population": null, + "vrda_d1_impact": null, + "cam_widelyknown": null, + "cam_exploitation": null, + "cam_internetinfrastructure": null, + "cam_population": null, + "cam_impact": null, + "cam_easeofexploitation": null, + "cam_attackeraccessrequired": null, + "cam_scorecurrent": null, + "cam_scorecurrentwidelyknown": null, + "cam_scorecurrentwidelyknownexploited": null, + "ipprotocol": null, + "cvss_accessvector": null, + "cvss_accesscomplexity": null, + "cvss_authentication": null, + "cvss_confidentialityimpact": null, + "cvss_integrityimpact": null, + "cvss_availabilityimpact": null, + "cvss_exploitablity": null, + "cvss_remediationlevel": null, + "cvss_reportconfidence": null, + "cvss_collateraldamagepotential": null, + "cvss_targetdistribution": null, + "cvss_securityrequirementscr": null, + "cvss_securityrequirementsir": null, + "cvss_securityrequirementsar": null, + "cvss_basescore": null, + "cvss_basevector": null, + "cvss_temporalscore": null, + "cvss_environmentalscore": null, + "cvss_environmentalvector": null, + "metric": null, + "vulnote": 7 +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vendors.json similarity index 98% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vendors.json index e26d82cd..22b5fd77 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vendors.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vendors.json @@ -1,12 +1,12 @@ -[ - { - "note": "294418", - "contact_date": "2025-09-15T19:03:33.664345Z", - "vendor": "DrayTek Corporation", - "references": "", - "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", - "dateupdated": "2025-10-03T11:35:31.190661Z", - "statement_date": "2025-09-16T02:27:51.346335Z", - "addendum": null - } -] +[ + { + "note": "294418", + "contact_date": "2025-09-15T19:03:33.664345Z", + "vendor": "DrayTek Corporation", + "references": "", + "statement": "The issue is confirmed, and here is the patch list\r\n\r\nV3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\r\nV2927/V2865/V2866\t4.5.1\r\nV2765/V2766/V2763/V2135\t4.5.1\r\nV2915\t4.4.6.1\r\nV2862/V2926\t3.9.9.12\r\nV2952/3220\t3.9.8.8\r\nV2860/V2925\t3.9.8.6\r\nV2133/V2762/V2832\t3.9.9.4\r\nV2620/LTE200\t3.9.9.5", + "dateupdated": "2025-10-03T11:35:31.190661Z", + "statement_date": "2025-09-16T02:27:51.346335Z", + "addendum": null + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vuls.json similarity index 97% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vuls.json index ec166e0b..2631351b 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418-vuls.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418-vuls.json @@ -1,11 +1,11 @@ -[ - { - "note": "294418", - "cve": "2025-10547", - "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", - "uid": "CVE-2025-10547", - "case_increment": 1, - "date_added": "2025-10-03T11:35:31.177872Z", - "dateupdated": "2025-10-03T11:40:09.915649Z" - } -] +[ + { + "note": "294418", + "cve": "2025-10547", + "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", + "uid": "CVE-2025-10547", + "case_increment": 1, + "date_added": "2025-10-03T11:35:31.177872Z", + "dateupdated": "2025-10-03T11:40:09.915649Z" + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418.json similarity index 98% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418.json index 72493215..e7fb551c 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vu-294418.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vu-294418.json @@ -1,63 +1,63 @@ -{ - "vuid": "VU#294418", - "idnumber": "294418", - "name": "Vigor routers running DrayOS are vulnerable to RCE via EasyVPN and LAN web administration interface", - "keywords": null, - "overview": "### Overview\r\nA remote code execution (RCE) vulnerability, tracked as CVE-2025-10547, was discovered through the EasyVPN and LAN web administration interface of Vigor routers by Draytek. A script in the LAN web administration interface uses an unitialized variable, allowing an attacker to send specially crafted HTTP requests that cause memory corruption and potentially allow arbitrary code execution.\r\n\t\r\n### Description\r\nVigor routers are business-grade routers, designed for small to medium-sized businesses, made by Draytek. These routers provide routing, firewall, VPN, content-filtering, bandwidth management, LAN (local area network), and multi-WAN (wide area network) features. Draytek utilizes a proprietary firmware, DrayOS, on the Vigor router line. DrayOS features the EasyVPN and LAN Web Administrator tool s to facilitate LAN and VPN setup. According to the DrayTek [website](https://www.draytek.com/support/knowledge-base/12023), \"with EasyVPN, users no longer need to generate WireGuard keys, import OpenVPN configuration files, or upload certificates. Instead, VPN can be successfully established by simply entering the username and password or getting the OTP code by email.\" \r\n\r\nThe LAN Web Administrator provides a browser-based user interface for router management. When a user interacts with the LAN Web Administration interface, the user interface elements trigger actions that generate HTTP requests to interact with the local server. This process contains an uninitialized variable. Due to the uninitialized variable, an unauthenticated attacker could perform memory corruption on the router via specially crafted HTTP requests to hijack execution or inject malicious payloads. If EasyVPN is enabled, the flaw could be remotely exploited through the VPN interface.\r\n\r\n### Impact\r\nA remote, unathenticated attacker can exploit this vulnerability through accessing the LAN interface\u2014or potentially the WAN interface\u2014if EasyVPN is enabled or remote administration over the internet is activated. If a remote, unauthenticated attacker leverages this vulnerability, they can execute arbitrary code on the router (RCE) and gain full control of the device. A successful attack could result in a attacker gaining root access to a Vigor router to then install backdoors, reconfigure network settings, or block traffic. An attacker may also pivot for lateral movement via intercepting internal communications and bypassing VPNs. \r\n\r\n### Solution\r\nThe DrayTek Security team has developed a series of patches to remediate the vulnerability, and all users of Vigor routers should upgrade to the latest version ASAP. The patches can be found on the [resources](https://www.draytek.com/support/resources?type=version) page of the DrayTek webpage, and the security advisory can be found within the [about](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) section of the DrayTek webpage. Consult either the CVE [listing](https://nvd.nist.gov/vuln/detail/CVE-2025-10547) or the [advisory page](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) for a full list of affected products. \r\n\r\n### Acknowledgements\r\nThanks to the reporter, Pierre-Yves MAES of ChapsVision (pymaes@chapsvision.com). This document was written by Ayushi Kriplani.", - "clean_desc": null, - "impact": null, - "resolution": null, - "workarounds": null, - "sysaffected": null, - "thanks": null, - "author": null, - "public": [ - "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", - "https://www.draytek.com/support/resources?type=version" - ], - "cveids": [ - "CVE-2025-10547" - ], - "certadvisory": null, - "uscerttechnicalalert": null, - "datecreated": "2025-10-03T11:35:31.224065Z", - "publicdate": "2025-10-03T11:35:31.026053Z", - "datefirstpublished": "2025-10-03T11:35:31.247121Z", - "dateupdated": "2025-10-03T11:40:09.876722Z", - "revision": 2, - "vrda_d1_directreport": null, - "vrda_d1_population": null, - "vrda_d1_impact": null, - "cam_widelyknown": null, - "cam_exploitation": null, - "cam_internetinfrastructure": null, - "cam_population": null, - "cam_impact": null, - "cam_easeofexploitation": null, - "cam_attackeraccessrequired": null, - "cam_scorecurrent": null, - "cam_scorecurrentwidelyknown": null, - "cam_scorecurrentwidelyknownexploited": null, - "ipprotocol": null, - "cvss_accessvector": null, - "cvss_accesscomplexity": null, - "cvss_authentication": null, - "cvss_confidentialityimpact": null, - "cvss_integrityimpact": null, - "cvss_availabilityimpact": null, - "cvss_exploitablity": null, - "cvss_remediationlevel": null, - "cvss_reportconfidence": null, - "cvss_collateraldamagepotential": null, - "cvss_targetdistribution": null, - "cvss_securityrequirementscr": null, - "cvss_securityrequirementsir": null, - "cvss_securityrequirementsar": null, - "cvss_basescore": null, - "cvss_basevector": null, - "cvss_temporalscore": null, - "cvss_environmentalscore": null, - "cvss_environmentalvector": null, - "metric": null, - "vulnote": 142 -} +{ + "vuid": "VU#294418", + "idnumber": "294418", + "name": "Vigor routers running DrayOS are vulnerable to RCE via EasyVPN and LAN web administration interface", + "keywords": null, + "overview": "### Overview\r\nA remote code execution (RCE) vulnerability, tracked as CVE-2025-10547, was discovered through the EasyVPN and LAN web administration interface of Vigor routers by Draytek. A script in the LAN web administration interface uses an unitialized variable, allowing an attacker to send specially crafted HTTP requests that cause memory corruption and potentially allow arbitrary code execution.\r\n\t\r\n### Description\r\nVigor routers are business-grade routers, designed for small to medium-sized businesses, made by Draytek. These routers provide routing, firewall, VPN, content-filtering, bandwidth management, LAN (local area network), and multi-WAN (wide area network) features. Draytek utilizes a proprietary firmware, DrayOS, on the Vigor router line. DrayOS features the EasyVPN and LAN Web Administrator tool s to facilitate LAN and VPN setup. According to the DrayTek [website](https://www.draytek.com/support/knowledge-base/12023), \"with EasyVPN, users no longer need to generate WireGuard keys, import OpenVPN configuration files, or upload certificates. Instead, VPN can be successfully established by simply entering the username and password or getting the OTP code by email.\" \r\n\r\nThe LAN Web Administrator provides a browser-based user interface for router management. When a user interacts with the LAN Web Administration interface, the user interface elements trigger actions that generate HTTP requests to interact with the local server. This process contains an uninitialized variable. Due to the uninitialized variable, an unauthenticated attacker could perform memory corruption on the router via specially crafted HTTP requests to hijack execution or inject malicious payloads. If EasyVPN is enabled, the flaw could be remotely exploited through the VPN interface.\r\n\r\n### Impact\r\nA remote, unathenticated attacker can exploit this vulnerability through accessing the LAN interface\u2014or potentially the WAN interface\u2014if EasyVPN is enabled or remote administration over the internet is activated. If a remote, unauthenticated attacker leverages this vulnerability, they can execute arbitrary code on the router (RCE) and gain full control of the device. A successful attack could result in a attacker gaining root access to a Vigor router to then install backdoors, reconfigure network settings, or block traffic. An attacker may also pivot for lateral movement via intercepting internal communications and bypassing VPNs. \r\n\r\n### Solution\r\nThe DrayTek Security team has developed a series of patches to remediate the vulnerability, and all users of Vigor routers should upgrade to the latest version ASAP. The patches can be found on the [resources](https://www.draytek.com/support/resources?type=version) page of the DrayTek webpage, and the security advisory can be found within the [about](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) section of the DrayTek webpage. Consult either the CVE [listing](https://nvd.nist.gov/vuln/detail/CVE-2025-10547) or the [advisory page](https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/) for a full list of affected products. \r\n\r\n### Acknowledgements\r\nThanks to the reporter, Pierre-Yves MAES of ChapsVision (pymaes@chapsvision.com). This document was written by Ayushi Kriplani.", + "clean_desc": null, + "impact": null, + "resolution": null, + "workarounds": null, + "sysaffected": null, + "thanks": null, + "author": null, + "public": [ + "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", + "https://www.draytek.com/support/resources?type=version" + ], + "cveids": [ + "CVE-2025-10547" + ], + "certadvisory": null, + "uscerttechnicalalert": null, + "datecreated": "2025-10-03T11:35:31.224065Z", + "publicdate": "2025-10-03T11:35:31.026053Z", + "datefirstpublished": "2025-10-03T11:35:31.247121Z", + "dateupdated": "2025-10-03T11:40:09.876722Z", + "revision": 2, + "vrda_d1_directreport": null, + "vrda_d1_population": null, + "vrda_d1_impact": null, + "cam_widelyknown": null, + "cam_exploitation": null, + "cam_internetinfrastructure": null, + "cam_population": null, + "cam_impact": null, + "cam_easeofexploitation": null, + "cam_attackeraccessrequired": null, + "cam_scorecurrent": null, + "cam_scorecurrentwidelyknown": null, + "cam_scorecurrentwidelyknownexploited": null, + "ipprotocol": null, + "cvss_accessvector": null, + "cvss_accesscomplexity": null, + "cvss_authentication": null, + "cvss_confidentialityimpact": null, + "cvss_integrityimpact": null, + "cvss_availabilityimpact": null, + "cvss_exploitablity": null, + "cvss_remediationlevel": null, + "cvss_reportconfidence": null, + "cvss_collateraldamagepotential": null, + "cvss_targetdistribution": null, + "cvss_securityrequirementscr": null, + "cvss_securityrequirementsir": null, + "cvss_securityrequirementsar": null, + "cvss_basescore": null, + "cvss_basevector": null, + "cvss_temporalscore": null, + "cvss_environmentalscore": null, + "cvss_environmentalvector": null, + "metric": null, + "vulnote": 142 +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vulnerabilities-294418.json similarity index 97% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vulnerabilities-294418.json index ec166e0b..2631351b 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/vulnerabilities-294418.json +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/vulnerabilities-294418.json @@ -1,11 +1,11 @@ -[ - { - "note": "294418", - "cve": "2025-10547", - "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", - "uid": "CVE-2025-10547", - "case_increment": 1, - "date_added": "2025-10-03T11:35:31.177872Z", - "dateupdated": "2025-10-03T11:40:09.915649Z" - } -] +[ + { + "note": "294418", + "cve": "2025-10547", + "description": "An uninitialized variable in the HTTP CGI request arguments processing component of Vigor Routers running DrayOS may allow an attacker the ability to perform RCE on the appliance through memory corruption.", + "uid": "CVE-2025-10547", + "case_increment": 1, + "date_added": "2025-10-03T11:35:31.177872Z", + "dateupdated": "2025-10-03T11:40:09.915649Z" + } +] diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcMapperTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcMapperTests.cs index c9304737..51ccf013 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcMapperTests.cs @@ -1,118 +1,118 @@ -using System; -using System.Globalization; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.CertCc.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; - -public sealed class CertCcMapperTests -{ - private static readonly DateTimeOffset PublishedAt = DateTimeOffset.Parse("2025-10-03T11:35:31Z", CultureInfo.InvariantCulture); - - [Fact] - public void Map_ProducesCanonicalAdvisoryWithVendorPrimitives() - { - const string vendorStatement = - "The issue is confirmed, and here is the patch list\n\n" + - "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + - "V2927/V2865/V2866\t4.5.1\n" + - "V2765/V2766/V2763/V2135\t4.5.1"; - - var vendor = new CertCcVendorDto( - "DrayTek Corporation", - ContactDate: PublishedAt.AddDays(-10), - StatementDate: PublishedAt.AddDays(-5), - Updated: PublishedAt, - Statement: vendorStatement, - Addendum: null, - References: new[] { "https://www.draytek.com/support/resources?type=version" }); - - var vendorStatus = new CertCcVendorStatusDto( - Vendor: "DrayTek Corporation", - CveId: "CVE-2025-10547", - Status: "Affected", - Statement: null, - References: Array.Empty(), - DateAdded: PublishedAt, - DateUpdated: PublishedAt); - - var vulnerability = new CertCcVulnerabilityDto( - CveId: "CVE-2025-10547", - Description: null, - DateAdded: PublishedAt, - DateUpdated: PublishedAt); - - var metadata = new CertCcNoteMetadata( - VuId: "VU#294418", - IdNumber: "294418", - Title: "Vigor routers running DrayOS RCE via EasyVPN", - Overview: "Overview", - Summary: "Summary", - Published: PublishedAt, - Updated: PublishedAt.AddMinutes(5), - Created: PublishedAt, - Revision: 2, - CveIds: new[] { "CVE-2025-10547" }, - PublicUrls: new[] - { - "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", - "https://www.draytek.com/support/resources?type=version" - }, - PrimaryUrl: "https://www.kb.cert.org/vuls/id/294418/"); - - var dto = new CertCcNoteDto( - metadata, - Vendors: new[] { vendor }, - VendorStatuses: new[] { vendorStatus }, - Vulnerabilities: new[] { vulnerability }); - - var document = new DocumentRecord( - Guid.NewGuid(), - "cert-cc", - "https://www.kb.cert.org/vuls/id/294418/", - PublishedAt, - Sha256: new string('0', 64), - Status: "pending-map", - ContentType: "application/json", - Headers: null, - Metadata: null, - Etag: null, - LastModified: PublishedAt, - GridFsId: null); - - var dtoRecord = new DtoRecord( - Id: Guid.NewGuid(), - DocumentId: document.Id, - SourceName: "cert-cc", - SchemaVersion: "certcc.vince.note.v1", - Payload: new BsonDocument(), - ValidatedAt: PublishedAt.AddMinutes(1)); - - var advisory = CertCcMapper.Map(dto, document, dtoRecord, "cert-cc"); - - Assert.Equal("certcc/vu-294418", advisory.AdvisoryKey); - Assert.Contains("VU#294418", advisory.Aliases); - Assert.Contains("CVE-2025-10547", advisory.Aliases); - Assert.Equal("en", advisory.Language); - Assert.Equal(PublishedAt, advisory.Published); - - Assert.Contains(advisory.References, reference => reference.Url.Contains("/vuls/id/294418", StringComparison.OrdinalIgnoreCase)); - - var affected = Assert.Single(advisory.AffectedPackages); - Assert.Equal("vendor", affected.Type); - Assert.Equal("DrayTek Corporation", affected.Identifier); - Assert.Contains(affected.Statuses, status => status.Status == AffectedPackageStatusCatalog.Affected); - - var range = Assert.Single(affected.VersionRanges); - Assert.NotNull(range.Primitives); - Assert.NotNull(range.Primitives!.VendorExtensions); - Assert.Contains(range.Primitives.VendorExtensions!, kvp => kvp.Key == "certcc.vendor.patches"); - - Assert.NotEmpty(affected.NormalizedVersions); - Assert.Contains(affected.NormalizedVersions, rule => rule.Scheme == "certcc.vendor" && rule.Value == "4.5.1"); - } -} +using System; +using System.Globalization; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.CertCc.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.Internal; + +public sealed class CertCcMapperTests +{ + private static readonly DateTimeOffset PublishedAt = DateTimeOffset.Parse("2025-10-03T11:35:31Z", CultureInfo.InvariantCulture); + + [Fact] + public void Map_ProducesCanonicalAdvisoryWithVendorPrimitives() + { + const string vendorStatement = + "The issue is confirmed, and here is the patch list\n\n" + + "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + + "V2927/V2865/V2866\t4.5.1\n" + + "V2765/V2766/V2763/V2135\t4.5.1"; + + var vendor = new CertCcVendorDto( + "DrayTek Corporation", + ContactDate: PublishedAt.AddDays(-10), + StatementDate: PublishedAt.AddDays(-5), + Updated: PublishedAt, + Statement: vendorStatement, + Addendum: null, + References: new[] { "https://www.draytek.com/support/resources?type=version" }); + + var vendorStatus = new CertCcVendorStatusDto( + Vendor: "DrayTek Corporation", + CveId: "CVE-2025-10547", + Status: "Affected", + Statement: null, + References: Array.Empty(), + DateAdded: PublishedAt, + DateUpdated: PublishedAt); + + var vulnerability = new CertCcVulnerabilityDto( + CveId: "CVE-2025-10547", + Description: null, + DateAdded: PublishedAt, + DateUpdated: PublishedAt); + + var metadata = new CertCcNoteMetadata( + VuId: "VU#294418", + IdNumber: "294418", + Title: "Vigor routers running DrayOS RCE via EasyVPN", + Overview: "Overview", + Summary: "Summary", + Published: PublishedAt, + Updated: PublishedAt.AddMinutes(5), + Created: PublishedAt, + Revision: 2, + CveIds: new[] { "CVE-2025-10547" }, + PublicUrls: new[] + { + "https://www.draytek.com/about/security-advisory/use-of-uninitialized-variable-vulnerabilities/", + "https://www.draytek.com/support/resources?type=version" + }, + PrimaryUrl: "https://www.kb.cert.org/vuls/id/294418/"); + + var dto = new CertCcNoteDto( + metadata, + Vendors: new[] { vendor }, + VendorStatuses: new[] { vendorStatus }, + Vulnerabilities: new[] { vulnerability }); + + var document = new DocumentRecord( + Guid.NewGuid(), + "cert-cc", + "https://www.kb.cert.org/vuls/id/294418/", + PublishedAt, + Sha256: new string('0', 64), + Status: "pending-map", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: PublishedAt, + GridFsId: null); + + var dtoRecord = new DtoRecord( + Id: Guid.NewGuid(), + DocumentId: document.Id, + SourceName: "cert-cc", + SchemaVersion: "certcc.vince.note.v1", + Payload: new BsonDocument(), + ValidatedAt: PublishedAt.AddMinutes(1)); + + var advisory = CertCcMapper.Map(dto, document, dtoRecord, "cert-cc"); + + Assert.Equal("certcc/vu-294418", advisory.AdvisoryKey); + Assert.Contains("VU#294418", advisory.Aliases); + Assert.Contains("CVE-2025-10547", advisory.Aliases); + Assert.Equal("en", advisory.Language); + Assert.Equal(PublishedAt, advisory.Published); + + Assert.Contains(advisory.References, reference => reference.Url.Contains("/vuls/id/294418", StringComparison.OrdinalIgnoreCase)); + + var affected = Assert.Single(advisory.AffectedPackages); + Assert.Equal("vendor", affected.Type); + Assert.Equal("DrayTek Corporation", affected.Identifier); + Assert.Contains(affected.Statuses, status => status.Status == AffectedPackageStatusCatalog.Affected); + + var range = Assert.Single(affected.VersionRanges); + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.VendorExtensions); + Assert.Contains(range.Primitives.VendorExtensions!, kvp => kvp.Key == "certcc.vendor.patches"); + + Assert.NotEmpty(affected.NormalizedVersions); + Assert.Contains(affected.NormalizedVersions, rule => rule.Scheme == "certcc.vendor" && rule.Value == "4.5.1"); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryParserTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryParserTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryParserTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryParserTests.cs index ddaf6213..218414ad 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryParserTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryParserTests.cs @@ -1,58 +1,58 @@ -using System.Text; -using System.Text.Json; -using StellaOps.Feedser.Source.CertCc.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; - -public sealed class CertCcSummaryParserTests -{ - [Fact] - public void ParseNotes_ReturnsTokens_FromStringArray() - { - var payload = Encoding.UTF8.GetBytes("{\"notes\":[\"VU#123456\",\"VU#654321\"]}"); - - var notes = CertCcSummaryParser.ParseNotes(payload); - - Assert.Equal(new[] { "VU#123456", "VU#654321" }, notes); - } - - [Fact] - public void ParseNotes_DeduplicatesTokens_IgnoringCaseAndWhitespace() - { - var payload = Encoding.UTF8.GetBytes("{\"notes\":[\"VU#123456\",\"vu#123456\",\" 123456 \"]}"); - - var notes = CertCcSummaryParser.ParseNotes(payload); - - Assert.Single(notes); - Assert.Equal("VU#123456", notes[0], ignoreCase: true); - } - - [Fact] - public void ParseNotes_ReadsTokens_FromObjectEntries() - { - var payload = Encoding.UTF8.GetBytes("{\"notes\":[{\"id\":\"VU#294418\"},{\"idnumber\":\"257161\"}]}"); - - var notes = CertCcSummaryParser.ParseNotes(payload); - - Assert.Equal(new[] { "VU#294418", "257161" }, notes); - } - - [Fact] - public void ParseNotes_SupportsArrayRoot() - { - var payload = Encoding.UTF8.GetBytes("[\"VU#360686\",\"VU#760160\"]"); - - var notes = CertCcSummaryParser.ParseNotes(payload); - - Assert.Equal(new[] { "VU#360686", "VU#760160" }, notes); - } - - [Fact] - public void ParseNotes_InvalidStructure_Throws() - { - var payload = Encoding.UTF8.GetBytes("\"invalid\""); - - Assert.Throws(() => CertCcSummaryParser.ParseNotes(payload)); - } -} +using System.Text; +using System.Text.Json; +using StellaOps.Concelier.Connector.CertCc.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.Internal; + +public sealed class CertCcSummaryParserTests +{ + [Fact] + public void ParseNotes_ReturnsTokens_FromStringArray() + { + var payload = Encoding.UTF8.GetBytes("{\"notes\":[\"VU#123456\",\"VU#654321\"]}"); + + var notes = CertCcSummaryParser.ParseNotes(payload); + + Assert.Equal(new[] { "VU#123456", "VU#654321" }, notes); + } + + [Fact] + public void ParseNotes_DeduplicatesTokens_IgnoringCaseAndWhitespace() + { + var payload = Encoding.UTF8.GetBytes("{\"notes\":[\"VU#123456\",\"vu#123456\",\" 123456 \"]}"); + + var notes = CertCcSummaryParser.ParseNotes(payload); + + Assert.Single(notes); + Assert.Equal("VU#123456", notes[0], ignoreCase: true); + } + + [Fact] + public void ParseNotes_ReadsTokens_FromObjectEntries() + { + var payload = Encoding.UTF8.GetBytes("{\"notes\":[{\"id\":\"VU#294418\"},{\"idnumber\":\"257161\"}]}"); + + var notes = CertCcSummaryParser.ParseNotes(payload); + + Assert.Equal(new[] { "VU#294418", "257161" }, notes); + } + + [Fact] + public void ParseNotes_SupportsArrayRoot() + { + var payload = Encoding.UTF8.GetBytes("[\"VU#360686\",\"VU#760160\"]"); + + var notes = CertCcSummaryParser.ParseNotes(payload); + + Assert.Equal(new[] { "VU#360686", "VU#760160" }, notes); + } + + [Fact] + public void ParseNotes_InvalidStructure_Throws() + { + var payload = Encoding.UTF8.GetBytes("\"invalid\""); + + Assert.Throws(() => CertCcSummaryParser.ParseNotes(payload)); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs index 0e72c3f0..626f5b90 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcSummaryPlannerTests.cs @@ -1,95 +1,95 @@ -using System; -using System.Linq; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.CertCc.Internal; -using StellaOps.Feedser.Source.Common.Cursors; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; - -public sealed class CertCcSummaryPlannerTests -{ - [Fact] - public void CreatePlan_UsesInitialBackfillWindow() - { - var options = Options.Create(new CertCcOptions - { - SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = TimeSpan.FromDays(30), - Overlap = TimeSpan.FromDays(3), - InitialBackfill = TimeSpan.FromDays(120), - MinimumWindowSize = TimeSpan.FromDays(1), - }, - }); - - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-10T12:00:00Z")); - var planner = new CertCcSummaryPlanner(options, timeProvider); - - var plan = planner.CreatePlan(state: null); - - Assert.Equal(DateTimeOffset.Parse("2025-06-12T12:00:00Z"), plan.Window.Start); - Assert.Equal(DateTimeOffset.Parse("2025-07-12T12:00:00Z"), plan.Window.End); - - Assert.Equal(3, plan.Requests.Count); - - var monthly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Monthly).ToArray(); - Assert.Collection(monthly, - request => - { - Assert.Equal(2025, request.Year); - Assert.Equal(6, request.Month); - Assert.Equal("https://www.kb.cert.org/vuls/api/2025/06/summary/", request.Uri.AbsoluteUri); - }, - request => - { - Assert.Equal(2025, request.Year); - Assert.Equal(7, request.Month); - Assert.Equal("https://www.kb.cert.org/vuls/api/2025/07/summary/", request.Uri.AbsoluteUri); - }); - - var yearly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Yearly).ToArray(); - Assert.Single(yearly); - Assert.Equal(2025, yearly[0].Year); - Assert.Null(yearly[0].Month); - Assert.Equal("https://www.kb.cert.org/vuls/api/2025/summary/", yearly[0].Uri.AbsoluteUri); - - Assert.Equal(plan.Window.End, plan.NextState.LastWindowEnd); - } - - [Fact] - public void CreatePlan_AdvancesWindowRespectingOverlap() - { - var options = Options.Create(new CertCcOptions - { - SummaryWindow = new TimeWindowCursorOptions - { - WindowSize = TimeSpan.FromDays(30), - Overlap = TimeSpan.FromDays(10), - InitialBackfill = TimeSpan.FromDays(90), - MinimumWindowSize = TimeSpan.FromDays(1), - }, - }); - - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-12-01T00:00:00Z")); - var planner = new CertCcSummaryPlanner(options, timeProvider); - - var first = planner.CreatePlan(null); - var second = planner.CreatePlan(first.NextState); - - Assert.True(second.Window.Start < second.Window.End); - Assert.Equal(first.Window.End - options.Value.SummaryWindow.Overlap, second.Window.Start); - } - - private sealed class TestTimeProvider : TimeProvider - { - private DateTimeOffset _now; - - public TestTimeProvider(DateTimeOffset now) => _now = now; - - public override DateTimeOffset GetUtcNow() => _now; - - public void Advance(TimeSpan delta) => _now = _now.Add(delta); - } -} +using System; +using System.Linq; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.CertCc.Internal; +using StellaOps.Concelier.Connector.Common.Cursors; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.Internal; + +public sealed class CertCcSummaryPlannerTests +{ + [Fact] + public void CreatePlan_UsesInitialBackfillWindow() + { + var options = Options.Create(new CertCcOptions + { + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(120), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + }); + + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-10-10T12:00:00Z")); + var planner = new CertCcSummaryPlanner(options, timeProvider); + + var plan = planner.CreatePlan(state: null); + + Assert.Equal(DateTimeOffset.Parse("2025-06-12T12:00:00Z"), plan.Window.Start); + Assert.Equal(DateTimeOffset.Parse("2025-07-12T12:00:00Z"), plan.Window.End); + + Assert.Equal(3, plan.Requests.Count); + + var monthly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Monthly).ToArray(); + Assert.Collection(monthly, + request => + { + Assert.Equal(2025, request.Year); + Assert.Equal(6, request.Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/06/summary/", request.Uri.AbsoluteUri); + }, + request => + { + Assert.Equal(2025, request.Year); + Assert.Equal(7, request.Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/07/summary/", request.Uri.AbsoluteUri); + }); + + var yearly = plan.Requests.Where(r => r.Scope == CertCcSummaryScope.Yearly).ToArray(); + Assert.Single(yearly); + Assert.Equal(2025, yearly[0].Year); + Assert.Null(yearly[0].Month); + Assert.Equal("https://www.kb.cert.org/vuls/api/2025/summary/", yearly[0].Uri.AbsoluteUri); + + Assert.Equal(plan.Window.End, plan.NextState.LastWindowEnd); + } + + [Fact] + public void CreatePlan_AdvancesWindowRespectingOverlap() + { + var options = Options.Create(new CertCcOptions + { + SummaryWindow = new TimeWindowCursorOptions + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(10), + InitialBackfill = TimeSpan.FromDays(90), + MinimumWindowSize = TimeSpan.FromDays(1), + }, + }); + + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2025-12-01T00:00:00Z")); + var planner = new CertCcSummaryPlanner(options, timeProvider); + + var first = planner.CreatePlan(null); + var second = planner.CreatePlan(first.NextState); + + Assert.True(second.Window.Start < second.Window.End); + Assert.Equal(first.Window.End - options.Value.SummaryWindow.Overlap, second.Window.Start); + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset now) => _now = now; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs similarity index 86% rename from src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs rename to src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs index 582a198a..9e14b2ad 100644 --- a/src/StellaOps.Feedser.Source.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/Internal/CertCcVendorStatementParserTests.cs @@ -1,31 +1,31 @@ -using System.Linq; -using StellaOps.Feedser.Source.CertCc.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.CertCc.Tests.Internal; - -public sealed class CertCcVendorStatementParserTests -{ - [Fact] - public void Parse_ReturnsPatchesForTabDelimitedList() - { - const string statement = - "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + - "V2927/V2865/V2866\t4.5.1\n" + - "V2765/V2766/V2763/V2135\t4.5.1"; - - var patches = CertCcVendorStatementParser.Parse(statement); - - Assert.Equal(11, patches.Count); - Assert.Contains(patches, patch => patch.Product == "V3912" && patch.Version == "4.4.3.6"); - Assert.Contains(patches, patch => patch.Product == "V2962" && patch.Version == "4.4.5.1"); - Assert.Equal(7, patches.Count(patch => patch.Version == "4.5.1")); - } - - [Fact] - public void Parse_ReturnsEmptyWhenStatementMissing() - { - var patches = CertCcVendorStatementParser.Parse(null); - Assert.Empty(patches); - } -} +using System.Linq; +using StellaOps.Concelier.Connector.CertCc.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.CertCc.Tests.Internal; + +public sealed class CertCcVendorStatementParserTests +{ + [Fact] + public void Parse_ReturnsPatchesForTabDelimitedList() + { + const string statement = + "V3912/V3910/V2962/V1000B\t4.4.3.6/4.4.5.1\n" + + "V2927/V2865/V2866\t4.5.1\n" + + "V2765/V2766/V2763/V2135\t4.5.1"; + + var patches = CertCcVendorStatementParser.Parse(statement); + + Assert.Equal(11, patches.Count); + Assert.Contains(patches, patch => patch.Product == "V3912" && patch.Version == "4.4.3.6"); + Assert.Contains(patches, patch => patch.Product == "V2962" && patch.Version == "4.4.5.1"); + Assert.Equal(7, patches.Count(patch => patch.Version == "4.5.1")); + } + + [Fact] + public void Parse_ReturnsEmptyWhenStatementMissing() + { + var patches = CertCcVendorStatementParser.Parse(null); + Assert.Empty(patches); + } +} diff --git a/src/StellaOps.Feedser.Source.Cccs.Tests/StellaOps.Feedser.Source.Cccs.Tests.csproj b/src/StellaOps.Concelier.Connector.CertCc.Tests/StellaOps.Concelier.Connector.CertCc.Tests.csproj similarity index 64% rename from src/StellaOps.Feedser.Source.Cccs.Tests/StellaOps.Feedser.Source.Cccs.Tests.csproj rename to src/StellaOps.Concelier.Connector.CertCc.Tests/StellaOps.Concelier.Connector.CertCc.Tests.csproj index 3985bbe0..949e4b0e 100644 --- a/src/StellaOps.Feedser.Source.Cccs.Tests/StellaOps.Feedser.Source.Cccs.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.CertCc.Tests/StellaOps.Concelier.Connector.CertCc.Tests.csproj @@ -1,19 +1,19 @@ - - - net10.0 - enable - enable - - - - - - - - - - - PreserveNewest - - - + + + net10.0 + enable + enable + + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.CertCc/AGENTS.md b/src/StellaOps.Concelier.Connector.CertCc/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.CertCc/AGENTS.md rename to src/StellaOps.Concelier.Connector.CertCc/AGENTS.md index 453f0c10..04a62061 100644 --- a/src/StellaOps.Feedser.Source.CertCc/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.CertCc/AGENTS.md @@ -1,38 +1,38 @@ -# AGENTS -## Role -Implement the CERT/CC (Carnegie Mellon CERT Coordination Center) advisory connector so Feedser can ingest US CERT coordination bulletins. - -## Scope -- Identify CERT/CC advisory publication format (VU#, blog, RSS, JSON) and define fetch cadence/windowing. -- Implement fetch, parse, and mapping jobs with cursor persistence and dedupe. -- Normalise advisory content (summary, impacted vendors, products, recommended mitigations, CVEs). -- Produce canonical `Advisory` objects including aliases, references, affected packages, and range primitive metadata. -- Supply fixtures and deterministic regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores and state). -- `Feedser.Models` (canonical structures). -- `Feedser.Testing` (integration tests and snapshots). - -## Interfaces & Contracts -- Job kinds: `certcc:fetch`, `certcc:parse`, `certcc:map`. -- Persist upstream caching metadata (ETag/Last-Modified) when available. -- Aliases should capture CERT/CC VU IDs and referenced CVEs. - -## In/Out of scope -In scope: -- End-to-end connector with range primitive instrumentation and telemetry. - -Out of scope: -- ICS-CERT alerts (handled by dedicated connector) or blog posts unrelated to advisories. - -## Observability & Security Expectations -- Log fetch and mapping statistics; surface failures with backoff. -- Sanitise HTML sources before persistence. -- Respect upstream throttling via retry/backoff. - -## Tests -- Add `StellaOps.Feedser.Source.CertCc.Tests` to cover fetch/parse/map with canned fixtures. -- Snapshot canonical advisories and support UPDATE flag for regeneration. -- Ensure deterministic ordering and timestamp normalisation. +# AGENTS +## Role +Implement the CERT/CC (Carnegie Mellon CERT Coordination Center) advisory connector so Concelier can ingest US CERT coordination bulletins. + +## Scope +- Identify CERT/CC advisory publication format (VU#, blog, RSS, JSON) and define fetch cadence/windowing. +- Implement fetch, parse, and mapping jobs with cursor persistence and dedupe. +- Normalise advisory content (summary, impacted vendors, products, recommended mitigations, CVEs). +- Produce canonical `Advisory` objects including aliases, references, affected packages, and range primitive metadata. +- Supply fixtures and deterministic regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores and state). +- `Concelier.Models` (canonical structures). +- `Concelier.Testing` (integration tests and snapshots). + +## Interfaces & Contracts +- Job kinds: `certcc:fetch`, `certcc:parse`, `certcc:map`. +- Persist upstream caching metadata (ETag/Last-Modified) when available. +- Aliases should capture CERT/CC VU IDs and referenced CVEs. + +## In/Out of scope +In scope: +- End-to-end connector with range primitive instrumentation and telemetry. + +Out of scope: +- ICS-CERT alerts (handled by dedicated connector) or blog posts unrelated to advisories. + +## Observability & Security Expectations +- Log fetch and mapping statistics; surface failures with backoff. +- Sanitise HTML sources before persistence. +- Respect upstream throttling via retry/backoff. + +## Tests +- Add `StellaOps.Concelier.Connector.CertCc.Tests` to cover fetch/parse/map with canned fixtures. +- Snapshot canonical advisories and support UPDATE flag for regeneration. +- Ensure deterministic ordering and timestamp normalisation. diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs b/src/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs rename to src/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs index fc4255db..bdfc56d5 100644 --- a/src/StellaOps.Feedser.Source.CertCc/CertCcConnector.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/CertCcConnector.cs @@ -1,779 +1,779 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Net; -using System.Linq; -using System.Net.Http; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.CertCc.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.CertCc; - -public sealed class CertCcConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = false, - }; - - private static readonly byte[] EmptyArrayPayload = Encoding.UTF8.GetBytes("[]"); - private static readonly string[] DetailEndpoints = { "note", "vendors", "vuls", "vendors-vuls" }; - - private readonly CertCcSummaryPlanner _summaryPlanner; - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly CertCcOptions _options; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly CertCcDiagnostics _diagnostics; - - public CertCcConnector( - CertCcSummaryPlanner summaryPlanner, - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - CertCcDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner)); - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => CertCcConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var pendingNotes = new HashSet(cursor.PendingNotes, StringComparer.OrdinalIgnoreCase); - var processedNotes = new HashSet(StringComparer.OrdinalIgnoreCase); - - var now = _timeProvider.GetUtcNow(); - var remainingBudget = _options.MaxNotesPerFetch; - - // Resume notes that previously failed before fetching new summaries. - if (pendingNotes.Count > 0 && remainingBudget > 0) - { - var replay = pendingNotes.ToArray(); - foreach (var noteId in replay) - { - if (remainingBudget <= 0) - { - break; - } - - try - { - if (!processedNotes.Add(noteId)) - { - continue; - } - - if (await HasPendingDocumentBundleAsync(noteId, pendingDocuments, cancellationToken).ConfigureAwait(false)) - { - pendingNotes.Remove(noteId); - continue; - } - - await FetchNoteBundleAsync(noteId, null, pendingDocuments, pendingNotes, cancellationToken).ConfigureAwait(false); - if (!pendingNotes.Contains(noteId)) - { - remainingBudget--; - } - } - catch (Exception ex) - { - await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - } - } - - var plan = _summaryPlanner.CreatePlan(cursor.SummaryState); - _diagnostics.PlanEvaluated(plan.Window, plan.Requests.Count); - - try - { - foreach (var request in plan.Requests) - { - cancellationToken.ThrowIfCancellationRequested(); - var shouldProcessNotes = remainingBudget > 0; - - try - { - _diagnostics.SummaryFetchAttempt(request.Scope); - var metadata = BuildSummaryMetadata(request); - var existingSummary = await _documentStore.FindBySourceAndUriAsync(SourceName, request.Uri.ToString(), cancellationToken).ConfigureAwait(false); - var fetchRequest = new SourceFetchRequest( - CertCcOptions.HttpClientName, - SourceName, - HttpMethod.Get, - request.Uri, - metadata, - existingSummary?.Etag, - existingSummary?.LastModified, - null, - new[] { "application/json" }); - - var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); - if (result.IsNotModified) - { - _diagnostics.SummaryFetchUnchanged(request.Scope); - continue; - } - - if (!result.IsSuccess || result.Document is null) - { - _diagnostics.SummaryFetchFailure(request.Scope); - continue; - } - - _diagnostics.SummaryFetchSuccess(request.Scope); - - if (!shouldProcessNotes) - { - continue; - } - - var noteTokens = await ReadSummaryNotesAsync(result.Document, cancellationToken).ConfigureAwait(false); - foreach (var token in noteTokens) - { - if (remainingBudget <= 0) - { - break; - } - - var noteId = TryNormalizeNoteToken(token, out var vuIdentifier); - if (string.IsNullOrEmpty(noteId)) - { - continue; - } - - if (!processedNotes.Add(noteId)) - { - continue; - } - - await FetchNoteBundleAsync(noteId, vuIdentifier, pendingDocuments, pendingNotes, cancellationToken).ConfigureAwait(false); - if (!pendingNotes.Contains(noteId)) - { - remainingBudget--; - } - } - } - catch - { - _diagnostics.SummaryFetchFailure(request.Scope); - throw; - } - } - } - catch (Exception ex) - { - var failureCursor = cursor - .WithPendingSummaries(Array.Empty()) - .WithPendingNotes(pendingNotes) - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastRun(now); - - await UpdateCursorAsync(failureCursor, cancellationToken).ConfigureAwait(false); - await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - var updatedCursor = cursor - .WithSummaryState(plan.NextState) - .WithPendingSummaries(Array.Empty()) - .WithPendingNotes(pendingNotes) - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastRun(now); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task HasPendingDocumentBundleAsync(string noteId, HashSet pendingDocuments, CancellationToken cancellationToken) - { - if (pendingDocuments.Count == 0) - { - return false; - } - - var required = new HashSet(DetailEndpoints, StringComparer.OrdinalIgnoreCase); - - foreach (var documentId in pendingDocuments) - { - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document?.Metadata is null) - { - continue; - } - - if (!document.Metadata.TryGetValue("certcc.noteId", out var metadataNoteId) || - !string.Equals(metadataNoteId, noteId, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var endpoint = document.Metadata.TryGetValue("certcc.endpoint", out var endpointValue) - ? endpointValue - : "note"; - - required.Remove(endpoint); - if (required.Count == 0) - { - return true; - } - } - - return false; - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - if (!_options.EnableDetailMapping) - { - return; - } - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - var groups = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingDocuments.Remove(documentId); - continue; - } - - if (!TryGetMetadata(document, "certcc.noteId", out var noteId) || string.IsNullOrWhiteSpace(noteId)) - { - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - continue; - } - - var endpoint = TryGetMetadata(document, "certcc.endpoint", out var endpointValue) - ? endpointValue - : "note"; - - var group = groups.TryGetValue(noteId, out var existing) - ? existing - : (groups[noteId] = new NoteDocumentGroup(noteId)); - - group.Add(endpoint, document); - } - - foreach (var group in groups.Values) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (group.Note is null) - { - continue; - } - - try - { - var noteBytes = await DownloadDocumentAsync(group.Note, cancellationToken).ConfigureAwait(false); - var vendorsBytes = group.Vendors is null - ? EmptyArrayPayload - : await DownloadDocumentAsync(group.Vendors, cancellationToken).ConfigureAwait(false); - var vulsBytes = group.Vuls is null - ? EmptyArrayPayload - : await DownloadDocumentAsync(group.Vuls, cancellationToken).ConfigureAwait(false); - var vendorStatusesBytes = group.VendorStatuses is null - ? EmptyArrayPayload - : await DownloadDocumentAsync(group.VendorStatuses, cancellationToken).ConfigureAwait(false); - - var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes); - var json = JsonSerializer.Serialize(dto, DtoSerializerOptions); - var payload = MongoDB.Bson.BsonDocument.Parse(json); - - _diagnostics.ParseSuccess( - dto.Vendors.Count, - dto.VendorStatuses.Count, - dto.Vulnerabilities.Count); - - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - group.Note.Id, - SourceName, - "certcc.vince.note.v1", - payload, - _timeProvider.GetUtcNow()); - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(group.Note.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - pendingMappings.Add(group.Note.Id); - pendingDocuments.Remove(group.Note.Id); - - if (group.Vendors is not null) - { - await _documentStore.UpdateStatusAsync(group.Vendors.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(group.Vendors.Id); - } - - if (group.Vuls is not null) - { - await _documentStore.UpdateStatusAsync(group.Vuls.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(group.Vuls.Id); - } - - if (group.VendorStatuses is not null) - { - await _documentStore.UpdateStatusAsync(group.VendorStatuses.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(group.VendorStatuses.Id); - } - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogError(ex, "CERT/CC parse failed for note {NoteId}", group.NoteId); - if (group.Note is not null) - { - await _documentStore.UpdateStatusAsync(group.Note.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(group.Note.Id); - } - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - if (!_options.EnableDetailMapping) - { - return; - } - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - - if (dtoRecord is null || document is null) - { - pendingMappings.Remove(documentId); - continue; - } - - try - { - var json = dtoRecord.Payload.ToJson(); - var dto = JsonSerializer.Deserialize(json, DtoSerializerOptions); - if (dto is null) - { - throw new InvalidOperationException($"CERT/CC DTO payload deserialized as null for document {documentId}."); - } - - var advisory = CertCcMapper.Map(dto, document, dtoRecord, SourceName); - var affectedCount = advisory.AffectedPackages.Length; - var normalizedRuleCount = advisory.AffectedPackages.Sum(static package => package.NormalizedVersions.Length); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - _diagnostics.MapSuccess(affectedCount, normalizedRuleCount); - } - catch (Exception ex) - { - _diagnostics.MapFailure(); - _logger.LogError(ex, "CERT/CC mapping failed for document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - } - - pendingMappings.Remove(documentId); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task FetchNoteBundleAsync( - string noteId, - string? vuIdentifier, - HashSet pendingDocuments, - HashSet pendingNotes, - CancellationToken cancellationToken) - { - var missingEndpoints = new List<(string Endpoint, HttpStatusCode? Status)>(); - - try - { - foreach (var endpoint in DetailEndpoints) - { - cancellationToken.ThrowIfCancellationRequested(); - - var uri = BuildDetailUri(noteId, endpoint); - var metadata = BuildDetailMetadata(noteId, vuIdentifier, endpoint); - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); - - var request = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri) - { - Metadata = metadata, - ETag = existing?.Etag, - LastModified = existing?.LastModified, - AcceptHeaders = new[] { "application/json" }, - }; - - SourceFetchResult result; - _diagnostics.DetailFetchAttempt(endpoint); - try - { - result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException httpEx) - { - var status = httpEx.StatusCode ?? TryParseStatusCodeFromMessage(httpEx.Message); - if (ShouldTreatAsMissing(status, endpoint)) - { - _diagnostics.DetailFetchMissing(endpoint); - missingEndpoints.Add((endpoint, status)); - continue; - } - - _diagnostics.DetailFetchFailure(endpoint); - throw; - } - Guid documentId; - if (result.IsSuccess && result.Document is not null) - { - _diagnostics.DetailFetchSuccess(endpoint); - documentId = result.Document.Id; - } - else if (result.IsNotModified) - { - _diagnostics.DetailFetchUnchanged(endpoint); - if (existing is null) - { - continue; - } - - documentId = existing.Id; - } - else - { - _diagnostics.DetailFetchFailure(endpoint); - _logger.LogWarning( - "CERT/CC detail endpoint {Endpoint} returned {StatusCode} for note {NoteId}; will retry.", - endpoint, - (int)result.StatusCode, - noteId); - - throw new HttpRequestException( - $"CERT/CC endpoint '{endpoint}' returned {(int)result.StatusCode} ({result.StatusCode}) for note {noteId}.", - null, - result.StatusCode); - } - - pendingDocuments.Add(documentId); - - if (_options.DetailRequestDelay > TimeSpan.Zero) - { - await Task.Delay(_options.DetailRequestDelay, cancellationToken).ConfigureAwait(false); - } - } - - if (missingEndpoints.Count > 0) - { - var formatted = string.Join( - ", ", - missingEndpoints.Select(item => - item.Status.HasValue - ? $"{item.Endpoint} ({(int)item.Status.Value})" - : item.Endpoint)); - - _logger.LogWarning( - "CERT/CC detail fetch completed with missing endpoints for note {NoteId}: {Endpoints}", - noteId, - formatted); - } - - pendingNotes.Remove(noteId); - } - catch (Exception ex) - { - _logger.LogError(ex, "CERT/CC detail fetch failed for note {NoteId}", noteId); - pendingNotes.Add(noteId); - throw; - } - } - - private static Dictionary BuildSummaryMetadata(CertCcSummaryRequest request) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(), - ["certcc.year"] = request.Year.ToString("D4", CultureInfo.InvariantCulture), - }; - - if (request.Month.HasValue) - { - metadata["certcc.month"] = request.Month.Value.ToString("D2", CultureInfo.InvariantCulture); - } - - return metadata; - } - - private static Dictionary BuildDetailMetadata(string noteId, string? vuIdentifier, string endpoint) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["certcc.endpoint"] = endpoint, - ["certcc.noteId"] = noteId, - }; - - if (!string.IsNullOrWhiteSpace(vuIdentifier)) - { - metadata["certcc.vuid"] = vuIdentifier; - } - - return metadata; - } - - private async Task> ReadSummaryNotesAsync(DocumentRecord document, CancellationToken cancellationToken) - { - if (!document.GridFsId.HasValue) - { - return Array.Empty(); - } - - var payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - return CertCcSummaryParser.ParseNotes(payload); - } - - private async Task DownloadDocumentAsync(DocumentRecord document, CancellationToken cancellationToken) - { - if (!document.GridFsId.HasValue) - { - throw new InvalidOperationException($"Document {document.Id} has no GridFS payload."); - } - - return await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - - private Uri BuildDetailUri(string noteId, string endpoint) - { - var suffix = endpoint switch - { - "note" => $"{noteId}/", - "vendors" => $"{noteId}/vendors/", - "vuls" => $"{noteId}/vuls/", - "vendors-vuls" => $"{noteId}/vendors/vuls/", - _ => $"{noteId}/", - }; - - return new Uri(_options.BaseApiUri, suffix); - } - - private static string? TryNormalizeNoteToken(string token, out string? vuIdentifier) - { - vuIdentifier = null; - if (string.IsNullOrWhiteSpace(token)) - { - return null; - } - - var trimmed = token.Trim(); - var digits = new string(trimmed.Where(char.IsDigit).ToArray()); - if (digits.Length == 0) - { - return null; - } - - vuIdentifier = trimmed.StartsWith("vu", StringComparison.OrdinalIgnoreCase) - ? trimmed.Replace(" ", string.Empty, StringComparison.Ordinal) - : $"VU#{digits}"; - - return digits; - } - - private static bool TryGetMetadata(DocumentRecord document, string key, out string value) - { - value = string.Empty; - if (document.Metadata is null) - { - return false; - } - - if (!document.Metadata.TryGetValue(key, out var metadataValue)) - { - return false; - } - - value = metadataValue; - return true; - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return CertCcCursor.FromBson(record?.Cursor); - } - - private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken) - { - var completedAt = _timeProvider.GetUtcNow(); - await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); - } - - private sealed class NoteDocumentGroup - { - public NoteDocumentGroup(string noteId) - { - NoteId = noteId; - } - - public string NoteId { get; } - - public DocumentRecord? Note { get; private set; } - - public DocumentRecord? Vendors { get; private set; } - - public DocumentRecord? Vuls { get; private set; } - - public DocumentRecord? VendorStatuses { get; private set; } - - public void Add(string endpoint, DocumentRecord document) - { - switch (endpoint) - { - case "note": - Note = document; - break; - case "vendors": - Vendors = document; - break; - case "vuls": - Vuls = document; - break; - case "vendors-vuls": - VendorStatuses = document; - break; - default: - Note ??= document; - break; - } - } - } - - private static bool ShouldTreatAsMissing(HttpStatusCode? statusCode, string endpoint) - { - if (statusCode is null) - { - return false; - } - - if (statusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone) - { - return !string.Equals(endpoint, "note", StringComparison.OrdinalIgnoreCase); - } - - // Treat vendors/vendors-vuls/vuls 403 as optional air-gapped responses. - if (statusCode == HttpStatusCode.Forbidden && !string.Equals(endpoint, "note", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - - return false; - } - - private static HttpStatusCode? TryParseStatusCodeFromMessage(string? message) - { - if (string.IsNullOrWhiteSpace(message)) - { - return null; - } - - const string marker = "status "; - var index = message.IndexOf(marker, StringComparison.OrdinalIgnoreCase); - if (index < 0) - { - return null; - } - - index += marker.Length; - var end = index; - while (end < message.Length && char.IsDigit(message[end])) - { - end++; - } - - if (end == index) - { - return null; - } - - if (int.TryParse(message[index..end], NumberStyles.Integer, CultureInfo.InvariantCulture, out var code) && - Enum.IsDefined(typeof(HttpStatusCode), code)) - { - return (HttpStatusCode)code; - } - - return null; - } -} +using System.Collections.Generic; +using System.Globalization; +using System.Net; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.CertCc.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.CertCc; + +public sealed class CertCcConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + private static readonly byte[] EmptyArrayPayload = Encoding.UTF8.GetBytes("[]"); + private static readonly string[] DetailEndpoints = { "note", "vendors", "vuls", "vendors-vuls" }; + + private readonly CertCcSummaryPlanner _summaryPlanner; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CertCcOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly CertCcDiagnostics _diagnostics; + + public CertCcConnector( + CertCcSummaryPlanner summaryPlanner, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + CertCcDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _summaryPlanner = summaryPlanner ?? throw new ArgumentNullException(nameof(summaryPlanner)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CertCcConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var pendingNotes = new HashSet(cursor.PendingNotes, StringComparer.OrdinalIgnoreCase); + var processedNotes = new HashSet(StringComparer.OrdinalIgnoreCase); + + var now = _timeProvider.GetUtcNow(); + var remainingBudget = _options.MaxNotesPerFetch; + + // Resume notes that previously failed before fetching new summaries. + if (pendingNotes.Count > 0 && remainingBudget > 0) + { + var replay = pendingNotes.ToArray(); + foreach (var noteId in replay) + { + if (remainingBudget <= 0) + { + break; + } + + try + { + if (!processedNotes.Add(noteId)) + { + continue; + } + + if (await HasPendingDocumentBundleAsync(noteId, pendingDocuments, cancellationToken).ConfigureAwait(false)) + { + pendingNotes.Remove(noteId); + continue; + } + + await FetchNoteBundleAsync(noteId, null, pendingDocuments, pendingNotes, cancellationToken).ConfigureAwait(false); + if (!pendingNotes.Contains(noteId)) + { + remainingBudget--; + } + } + catch (Exception ex) + { + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + } + + var plan = _summaryPlanner.CreatePlan(cursor.SummaryState); + _diagnostics.PlanEvaluated(plan.Window, plan.Requests.Count); + + try + { + foreach (var request in plan.Requests) + { + cancellationToken.ThrowIfCancellationRequested(); + var shouldProcessNotes = remainingBudget > 0; + + try + { + _diagnostics.SummaryFetchAttempt(request.Scope); + var metadata = BuildSummaryMetadata(request); + var existingSummary = await _documentStore.FindBySourceAndUriAsync(SourceName, request.Uri.ToString(), cancellationToken).ConfigureAwait(false); + var fetchRequest = new SourceFetchRequest( + CertCcOptions.HttpClientName, + SourceName, + HttpMethod.Get, + request.Uri, + metadata, + existingSummary?.Etag, + existingSummary?.LastModified, + null, + new[] { "application/json" }); + + var result = await _fetchService.FetchAsync(fetchRequest, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + _diagnostics.SummaryFetchUnchanged(request.Scope); + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.SummaryFetchFailure(request.Scope); + continue; + } + + _diagnostics.SummaryFetchSuccess(request.Scope); + + if (!shouldProcessNotes) + { + continue; + } + + var noteTokens = await ReadSummaryNotesAsync(result.Document, cancellationToken).ConfigureAwait(false); + foreach (var token in noteTokens) + { + if (remainingBudget <= 0) + { + break; + } + + var noteId = TryNormalizeNoteToken(token, out var vuIdentifier); + if (string.IsNullOrEmpty(noteId)) + { + continue; + } + + if (!processedNotes.Add(noteId)) + { + continue; + } + + await FetchNoteBundleAsync(noteId, vuIdentifier, pendingDocuments, pendingNotes, cancellationToken).ConfigureAwait(false); + if (!pendingNotes.Contains(noteId)) + { + remainingBudget--; + } + } + } + catch + { + _diagnostics.SummaryFetchFailure(request.Scope); + throw; + } + } + } + catch (Exception ex) + { + var failureCursor = cursor + .WithPendingSummaries(Array.Empty()) + .WithPendingNotes(pendingNotes) + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastRun(now); + + await UpdateCursorAsync(failureCursor, cancellationToken).ConfigureAwait(false); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var updatedCursor = cursor + .WithSummaryState(plan.NextState) + .WithPendingSummaries(Array.Empty()) + .WithPendingNotes(pendingNotes) + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastRun(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task HasPendingDocumentBundleAsync(string noteId, HashSet pendingDocuments, CancellationToken cancellationToken) + { + if (pendingDocuments.Count == 0) + { + return false; + } + + var required = new HashSet(DetailEndpoints, StringComparer.OrdinalIgnoreCase); + + foreach (var documentId in pendingDocuments) + { + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document?.Metadata is null) + { + continue; + } + + if (!document.Metadata.TryGetValue("certcc.noteId", out var metadataNoteId) || + !string.Equals(metadataNoteId, noteId, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var endpoint = document.Metadata.TryGetValue("certcc.endpoint", out var endpointValue) + ? endpointValue + : "note"; + + required.Remove(endpoint); + if (required.Count == 0) + { + return true; + } + } + + return false; + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + if (!_options.EnableDetailMapping) + { + return; + } + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + var groups = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + continue; + } + + if (!TryGetMetadata(document, "certcc.noteId", out var noteId) || string.IsNullOrWhiteSpace(noteId)) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + continue; + } + + var endpoint = TryGetMetadata(document, "certcc.endpoint", out var endpointValue) + ? endpointValue + : "note"; + + var group = groups.TryGetValue(noteId, out var existing) + ? existing + : (groups[noteId] = new NoteDocumentGroup(noteId)); + + group.Add(endpoint, document); + } + + foreach (var group in groups.Values) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (group.Note is null) + { + continue; + } + + try + { + var noteBytes = await DownloadDocumentAsync(group.Note, cancellationToken).ConfigureAwait(false); + var vendorsBytes = group.Vendors is null + ? EmptyArrayPayload + : await DownloadDocumentAsync(group.Vendors, cancellationToken).ConfigureAwait(false); + var vulsBytes = group.Vuls is null + ? EmptyArrayPayload + : await DownloadDocumentAsync(group.Vuls, cancellationToken).ConfigureAwait(false); + var vendorStatusesBytes = group.VendorStatuses is null + ? EmptyArrayPayload + : await DownloadDocumentAsync(group.VendorStatuses, cancellationToken).ConfigureAwait(false); + + var dto = CertCcNoteParser.Parse(noteBytes, vendorsBytes, vulsBytes, vendorStatusesBytes); + var json = JsonSerializer.Serialize(dto, DtoSerializerOptions); + var payload = MongoDB.Bson.BsonDocument.Parse(json); + + _diagnostics.ParseSuccess( + dto.Vendors.Count, + dto.VendorStatuses.Count, + dto.Vulnerabilities.Count); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + group.Note.Id, + SourceName, + "certcc.vince.note.v1", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(group.Note.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + pendingMappings.Add(group.Note.Id); + pendingDocuments.Remove(group.Note.Id); + + if (group.Vendors is not null) + { + await _documentStore.UpdateStatusAsync(group.Vendors.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(group.Vendors.Id); + } + + if (group.Vuls is not null) + { + await _documentStore.UpdateStatusAsync(group.Vuls.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(group.Vuls.Id); + } + + if (group.VendorStatuses is not null) + { + await _documentStore.UpdateStatusAsync(group.VendorStatuses.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(group.VendorStatuses.Id); + } + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "CERT/CC parse failed for note {NoteId}", group.NoteId); + if (group.Note is not null) + { + await _documentStore.UpdateStatusAsync(group.Note.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(group.Note.Id); + } + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + if (!_options.EnableDetailMapping) + { + return; + } + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + try + { + var json = dtoRecord.Payload.ToJson(); + var dto = JsonSerializer.Deserialize(json, DtoSerializerOptions); + if (dto is null) + { + throw new InvalidOperationException($"CERT/CC DTO payload deserialized as null for document {documentId}."); + } + + var advisory = CertCcMapper.Map(dto, document, dtoRecord, SourceName); + var affectedCount = advisory.AffectedPackages.Length; + var normalizedRuleCount = advisory.AffectedPackages.Sum(static package => package.NormalizedVersions.Length); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapSuccess(affectedCount, normalizedRuleCount); + } + catch (Exception ex) + { + _diagnostics.MapFailure(); + _logger.LogError(ex, "CERT/CC mapping failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + } + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task FetchNoteBundleAsync( + string noteId, + string? vuIdentifier, + HashSet pendingDocuments, + HashSet pendingNotes, + CancellationToken cancellationToken) + { + var missingEndpoints = new List<(string Endpoint, HttpStatusCode? Status)>(); + + try + { + foreach (var endpoint in DetailEndpoints) + { + cancellationToken.ThrowIfCancellationRequested(); + + var uri = BuildDetailUri(noteId, endpoint); + var metadata = BuildDetailMetadata(noteId, vuIdentifier, endpoint); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(CertCcOptions.HttpClientName, SourceName, uri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] { "application/json" }, + }; + + SourceFetchResult result; + _diagnostics.DetailFetchAttempt(endpoint); + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException httpEx) + { + var status = httpEx.StatusCode ?? TryParseStatusCodeFromMessage(httpEx.Message); + if (ShouldTreatAsMissing(status, endpoint)) + { + _diagnostics.DetailFetchMissing(endpoint); + missingEndpoints.Add((endpoint, status)); + continue; + } + + _diagnostics.DetailFetchFailure(endpoint); + throw; + } + Guid documentId; + if (result.IsSuccess && result.Document is not null) + { + _diagnostics.DetailFetchSuccess(endpoint); + documentId = result.Document.Id; + } + else if (result.IsNotModified) + { + _diagnostics.DetailFetchUnchanged(endpoint); + if (existing is null) + { + continue; + } + + documentId = existing.Id; + } + else + { + _diagnostics.DetailFetchFailure(endpoint); + _logger.LogWarning( + "CERT/CC detail endpoint {Endpoint} returned {StatusCode} for note {NoteId}; will retry.", + endpoint, + (int)result.StatusCode, + noteId); + + throw new HttpRequestException( + $"CERT/CC endpoint '{endpoint}' returned {(int)result.StatusCode} ({result.StatusCode}) for note {noteId}.", + null, + result.StatusCode); + } + + pendingDocuments.Add(documentId); + + if (_options.DetailRequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.DetailRequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (missingEndpoints.Count > 0) + { + var formatted = string.Join( + ", ", + missingEndpoints.Select(item => + item.Status.HasValue + ? $"{item.Endpoint} ({(int)item.Status.Value})" + : item.Endpoint)); + + _logger.LogWarning( + "CERT/CC detail fetch completed with missing endpoints for note {NoteId}: {Endpoints}", + noteId, + formatted); + } + + pendingNotes.Remove(noteId); + } + catch (Exception ex) + { + _logger.LogError(ex, "CERT/CC detail fetch failed for note {NoteId}", noteId); + pendingNotes.Add(noteId); + throw; + } + } + + private static Dictionary BuildSummaryMetadata(CertCcSummaryRequest request) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["certcc.scope"] = request.Scope.ToString().ToLowerInvariant(), + ["certcc.year"] = request.Year.ToString("D4", CultureInfo.InvariantCulture), + }; + + if (request.Month.HasValue) + { + metadata["certcc.month"] = request.Month.Value.ToString("D2", CultureInfo.InvariantCulture); + } + + return metadata; + } + + private static Dictionary BuildDetailMetadata(string noteId, string? vuIdentifier, string endpoint) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["certcc.endpoint"] = endpoint, + ["certcc.noteId"] = noteId, + }; + + if (!string.IsNullOrWhiteSpace(vuIdentifier)) + { + metadata["certcc.vuid"] = vuIdentifier; + } + + return metadata; + } + + private async Task> ReadSummaryNotesAsync(DocumentRecord document, CancellationToken cancellationToken) + { + if (!document.GridFsId.HasValue) + { + return Array.Empty(); + } + + var payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + return CertCcSummaryParser.ParseNotes(payload); + } + + private async Task DownloadDocumentAsync(DocumentRecord document, CancellationToken cancellationToken) + { + if (!document.GridFsId.HasValue) + { + throw new InvalidOperationException($"Document {document.Id} has no GridFS payload."); + } + + return await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + + private Uri BuildDetailUri(string noteId, string endpoint) + { + var suffix = endpoint switch + { + "note" => $"{noteId}/", + "vendors" => $"{noteId}/vendors/", + "vuls" => $"{noteId}/vuls/", + "vendors-vuls" => $"{noteId}/vendors/vuls/", + _ => $"{noteId}/", + }; + + return new Uri(_options.BaseApiUri, suffix); + } + + private static string? TryNormalizeNoteToken(string token, out string? vuIdentifier) + { + vuIdentifier = null; + if (string.IsNullOrWhiteSpace(token)) + { + return null; + } + + var trimmed = token.Trim(); + var digits = new string(trimmed.Where(char.IsDigit).ToArray()); + if (digits.Length == 0) + { + return null; + } + + vuIdentifier = trimmed.StartsWith("vu", StringComparison.OrdinalIgnoreCase) + ? trimmed.Replace(" ", string.Empty, StringComparison.Ordinal) + : $"VU#{digits}"; + + return digits; + } + + private static bool TryGetMetadata(DocumentRecord document, string key, out string value) + { + value = string.Empty; + if (document.Metadata is null) + { + return false; + } + + if (!document.Metadata.TryGetValue(key, out var metadataValue)) + { + return false; + } + + value = metadataValue; + return true; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var record = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return CertCcCursor.FromBson(record?.Cursor); + } + + private async Task UpdateCursorAsync(CertCcCursor cursor, CancellationToken cancellationToken) + { + var completedAt = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), completedAt, cancellationToken).ConfigureAwait(false); + } + + private sealed class NoteDocumentGroup + { + public NoteDocumentGroup(string noteId) + { + NoteId = noteId; + } + + public string NoteId { get; } + + public DocumentRecord? Note { get; private set; } + + public DocumentRecord? Vendors { get; private set; } + + public DocumentRecord? Vuls { get; private set; } + + public DocumentRecord? VendorStatuses { get; private set; } + + public void Add(string endpoint, DocumentRecord document) + { + switch (endpoint) + { + case "note": + Note = document; + break; + case "vendors": + Vendors = document; + break; + case "vuls": + Vuls = document; + break; + case "vendors-vuls": + VendorStatuses = document; + break; + default: + Note ??= document; + break; + } + } + } + + private static bool ShouldTreatAsMissing(HttpStatusCode? statusCode, string endpoint) + { + if (statusCode is null) + { + return false; + } + + if (statusCode is HttpStatusCode.NotFound or HttpStatusCode.Gone) + { + return !string.Equals(endpoint, "note", StringComparison.OrdinalIgnoreCase); + } + + // Treat vendors/vendors-vuls/vuls 403 as optional air-gapped responses. + if (statusCode == HttpStatusCode.Forbidden && !string.Equals(endpoint, "note", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + private static HttpStatusCode? TryParseStatusCodeFromMessage(string? message) + { + if (string.IsNullOrWhiteSpace(message)) + { + return null; + } + + const string marker = "status "; + var index = message.IndexOf(marker, StringComparison.OrdinalIgnoreCase); + if (index < 0) + { + return null; + } + + index += marker.Length; + var end = index; + while (end < message.Length && char.IsDigit(message[end])) + { + end++; + } + + if (end == index) + { + return null; + } + + if (int.TryParse(message[index..end], NumberStyles.Integer, CultureInfo.InvariantCulture, out var code) && + Enum.IsDefined(typeof(HttpStatusCode), code)) + { + return (HttpStatusCode)code; + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.CertCc/CertCcConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.CertCc/CertCcConnectorPlugin.cs index 468a1b20..b1119fd8 100644 --- a/src/StellaOps.Feedser.Source.CertCc/CertCcConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/CertCcConnectorPlugin.cs @@ -1,21 +1,21 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.CertCc; - -public sealed class CertCcConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "cert-cc"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) - => services.GetService() is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetRequiredService(); - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.CertCc; + +public sealed class CertCcConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cert-cc"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs similarity index 82% rename from src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs index bf09ce4a..6dda3f94 100644 --- a/src/StellaOps.Feedser.Source.CertCc/CertCcDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/CertCcDependencyInjectionRoutine.cs @@ -1,50 +1,50 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.CertCc.Configuration; - -namespace StellaOps.Feedser.Source.CertCc; - -public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:cert-cc"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddCertCcConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.CertCc.Configuration; + +namespace StellaOps.Concelier.Connector.CertCc; + +public sealed class CertCcDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:cert-cc"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCertCcConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, CertCcJobKinds.Fetch, typeof(CertCcFetchJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.CertCc/CertCcServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.CertCc/CertCcServiceCollectionExtensions.cs index 413aab63..cd5395eb 100644 --- a/src/StellaOps.Feedser.Source.CertCc/CertCcServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/CertCcServiceCollectionExtensions.cs @@ -1,37 +1,37 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.CertCc.Internal; -using StellaOps.Feedser.Source.Common.Http; - -namespace StellaOps.Feedser.Source.CertCc; - -public static class CertCcServiceCollectionExtensions -{ - public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); - - services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.BaseAddress = options.BaseApiUri; - clientOptions.UserAgent = "StellaOps.Feedser.CertCc/1.0"; - clientOptions.Timeout = TimeSpan.FromSeconds(20); - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.BaseApiUri.Host); - }); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.AddTransient(); - return services; - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.CertCc.Internal; +using StellaOps.Concelier.Connector.Common.Http; + +namespace StellaOps.Concelier.Connector.CertCc; + +public static class CertCcServiceCollectionExtensions +{ + public static IServiceCollection AddCertCcConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(CertCcOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = options.BaseApiUri; + clientOptions.UserAgent = "StellaOps.Concelier.CertCc/1.0"; + clientOptions.Timeout = TimeSpan.FromSeconds(20); + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseApiUri.Host); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs b/src/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs rename to src/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs index c6226edd..c6a5ae36 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Configuration/CertCcOptions.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Configuration/CertCcOptions.cs @@ -1,79 +1,79 @@ -using System; -using StellaOps.Feedser.Source.Common.Cursors; - -namespace StellaOps.Feedser.Source.CertCc.Configuration; - -/// -/// Connector options governing CERT/CC fetch cadence and API endpoints. -/// -public sealed class CertCcOptions -{ - public const string HttpClientName = "certcc"; - - /// - /// Root URI for the VINCE Vulnerability Notes API (must end with a slash). - /// - public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); - - /// - /// Sliding window settings controlling which summary endpoints are requested. - /// - public TimeWindowCursorOptions SummaryWindow { get; set; } = new() - { - WindowSize = TimeSpan.FromDays(30), - Overlap = TimeSpan.FromDays(3), - InitialBackfill = TimeSpan.FromDays(365), - MinimumWindowSize = TimeSpan.FromDays(1), - }; - - /// - /// Maximum number of monthly summary endpoints to request in a single plan. - /// - public int MaxMonthlySummaries { get; set; } = 6; - - /// - /// Maximum number of vulnerability notes (detail bundles) to process per fetch pass. - /// - public int MaxNotesPerFetch { get; set; } = 25; - - /// - /// Optional delay inserted between successive detail requests to respect upstream throttling. - /// - public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100); - - /// - /// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging. - /// - public bool EnableDetailMapping { get; set; } = true; - - public void Validate() - { - if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri) - { - throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI."); - } - - if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) - { - throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash."); - } - - SummaryWindow ??= new TimeWindowCursorOptions(); - SummaryWindow.EnsureValid(); - - if (MaxMonthlySummaries <= 0) - { - throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive."); - } - - if (MaxNotesPerFetch <= 0) - { - throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive."); - } - - if (DetailRequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative."); - } - } -} +using System; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Configuration; + +/// +/// Connector options governing CERT/CC fetch cadence and API endpoints. +/// +public sealed class CertCcOptions +{ + public const string HttpClientName = "certcc"; + + /// + /// Root URI for the VINCE Vulnerability Notes API (must end with a slash). + /// + public Uri BaseApiUri { get; set; } = new("https://www.kb.cert.org/vuls/api/", UriKind.Absolute); + + /// + /// Sliding window settings controlling which summary endpoints are requested. + /// + public TimeWindowCursorOptions SummaryWindow { get; set; } = new() + { + WindowSize = TimeSpan.FromDays(30), + Overlap = TimeSpan.FromDays(3), + InitialBackfill = TimeSpan.FromDays(365), + MinimumWindowSize = TimeSpan.FromDays(1), + }; + + /// + /// Maximum number of monthly summary endpoints to request in a single plan. + /// + public int MaxMonthlySummaries { get; set; } = 6; + + /// + /// Maximum number of vulnerability notes (detail bundles) to process per fetch pass. + /// + public int MaxNotesPerFetch { get; set; } = 25; + + /// + /// Optional delay inserted between successive detail requests to respect upstream throttling. + /// + public TimeSpan DetailRequestDelay { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// When disabled, parse/map stages skip detail mapping—useful for dry runs or migration staging. + /// + public bool EnableDetailMapping { get; set; } = true; + + public void Validate() + { + if (BaseApiUri is null || !BaseApiUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CertCcOptions.BaseApiUri must be an absolute URI."); + } + + if (!BaseApiUri.AbsoluteUri.EndsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("CertCcOptions.BaseApiUri must end with a trailing slash."); + } + + SummaryWindow ??= new TimeWindowCursorOptions(); + SummaryWindow.EnsureValid(); + + if (MaxMonthlySummaries <= 0) + { + throw new InvalidOperationException("CertCcOptions.MaxMonthlySummaries must be positive."); + } + + if (MaxNotesPerFetch <= 0) + { + throw new InvalidOperationException("CertCcOptions.MaxNotesPerFetch must be positive."); + } + + if (DetailRequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("CertCcOptions.DetailRequestDelay cannot be negative."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md b/src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md similarity index 73% rename from src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md rename to src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md index 16d5dd34..99374b57 100644 --- a/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md +++ b/src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-009_PLAN.md @@ -1,59 +1,59 @@ -# FEEDCONN-CERTCC-02-009 – VINCE Detail & Map Reintegration Plan - -- **Author:** BE-Conn-CERTCC (current on-call) -- **Date:** 2025-10-11 -- **Scope:** Restore VINCE detail parsing and canonical mapping in Feedser without destabilising downstream Merge/Export pipelines. - -## 1. Current State Snapshot (2025-10-11) - -- ✅ Fetch pipeline, VINCE summary planner, and detail queue are live; documents land with `DocumentStatuses.PendingParse`. -- ✅ DTO aggregate (`CertCcNoteDto`) plus mapper emit vendor-centric `normalizedVersions` (`scheme=certcc.vendor`) and provenance aligned with `src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md`. -- ✅ Regression coverage exists for fetch/parse/map flows (`CertCcConnectorSnapshotTests`), but snapshot regeneration is gated on harness refresh (FEEDCONN-CERTCC-02-007) and QA handoff (FEEDCONN-CERTCC-02-008). -- ⚠️ Parse/map jobs are not scheduled; production still operates in fetch-only mode. -- ⚠️ Downstream Merge team is finalising normalized range ingestion per `src/FASTER_MODELING_AND_NORMALIZATION.md`; we must avoid publishing canonical records until they certify compatibility. - -## 2. Required Dependencies & Coordinated Tasks - -| Dependency | Owner(s) | Blocking Condition | Handshake | -|------------|----------|--------------------|-----------| -| FEEDCONN-CERTCC-02-004 (Canonical mapping & range primitives hardening) | BE-Conn-CERTCC + Models | Ensure mapper emits deterministic `normalizedVersions` array and provenance field masks | Daily sync with Models/Merge leads; share fixture diff before each enablement phase | -| FEEDCONN-CERTCC-02-007 (Connector test harness remediation) | BE-Conn-CERTCC, QA | Restore `AddSourceCommon` harness + canned VINCE fixtures so we can shadow-run parse/map | Required before Phase 1 | -| FEEDCONN-CERTCC-02-008 (Snapshot coverage handoff) | QA | Snapshot refresh process green to surface regressions | Required before Phase 2 | -| FEEDCONN-CERTCC-02-010 (Partial-detail graceful degradation) | BE-Conn-CERTCC | Resiliency for missing VINCE endpoints to avoid job wedging after reintegration | Should land before Phase 2 cutover | - -## 3. Phased Rollout Plan - -| Phase | Window (UTC) | Actions | Success Signals | Rollback | -|-------|--------------|---------|-----------------|----------| -| **0 – Pre-flight validation** | 2025-10-11 → 2025-10-12 | • Finish FEEDCONN-CERTCC-02-007 harness fixes and regenerate fixtures.
      • Run `dotnet test src/StellaOps.Feedser.Source.CertCc.Tests` with `UPDATE_CERTCC_FIXTURES=0` to confirm deterministic baselines.
      • Generate sample advisory batch (`dotnet test … --filter SnapshotSmoke`) and deliver JSON diff to Merge for schema verification (`normalizedVersions[].scheme == certcc.vendor`, provenance masks populated). | • Harness tests green locally and in CI.
      • Merge sign-off that sample advisories conform to `FASTER_MODELING_AND_NORMALIZATION.md`. | N/A (no production enablement yet). | -| **1 – Shadow parse/map in staging** | Target start 2025-10-13 | • Register `source:cert-cc:parse` and `source:cert-cc:map` jobs, but gate them behind new config flag `feedser:sources:cert-cc:enableDetailMapping` (default `false`).
      • Deploy (restart required for options rebinding), enable flag, and point connector at staging Mongo with isolated collection (`advisories_certcc_shadow`).
      • Run connector for ≥2 cycles; compare advisory counts vs. fetch-only baseline and validate `feedser.range.primitives` metrics include `scheme=certcc.vendor`. | • No uncaught exceptions in staging logs.
      • Shadow advisories match expected vendor counts (±5%).
      • `certcc.summary.fetch.*` + new `certcc.map.duration.ms` metrics stable. | Disable flag; staging returns to fetch-only. No production impact. | -| **2 – Controlled production enablement** | Target start 2025-10-14 | • Redeploy production with flag enabled, start with job concurrency `1`, and reduce `MaxNotesPerFetch` to 5 for first 24 h.
      • Observe metrics dashboards hourly (fetch/map latency, pending queues, Mongo write throughput).
      • QA to replay latest snapshots and confirm no deterministic drift.
      • Publish advisory sample (top 10 changed docs) to Merge Slack channel for validation. | • Pending parse/mapping queues drain within expected SLA (<30 min).
      • No increase in merge dedupe anomalies.
      • Mongo writes stay within 10% of baseline. | Toggle flag off, re-run fetch-only. Clear `pendingMappings` via connector cursor reset if stuck. | -| **3 – Full production & cleanup** | Target start 2025-10-15 | • Restore `MaxNotesPerFetch` to configured default (20).
      • Remove temporary throttles and leave flag enabled by default.
      • Update `README.md` rollout notes; close FEEDCONN-CERTCC-02-009.
      • Kick off post-merge audit with Merge to ensure new advisories dedupe with other sources. | • Stable operations for ≥48 h, no degradation alerts.
      • Merge confirms conflict resolver behaviour unchanged. | If regression detected, revert to Phase 2 state or disable jobs; retain plan for reuse. | - -## 4. Monitoring & Validation Checklist - -- Dashboards: `certcc.*` meters (plan, summary fetch, detail fetch) plus `feedser.range.primitives` with tag `scheme=certcc.vendor`. -- Logs: ensure Parse/Map jobs emit `correlationId` aligned with fetch events for traceability. -- Data QA: run `tools/dump_advisory` against two VINCE notes (one multi-vendor, one single-vendor) every phase to spot-check normalized versions ordering and provenance. -- Storage: verify Mongo TTL/size for `raw_documents` and `dtos`—detail payload volume increases by ~3× when mapping resumes. - -## 5. Rollback / Contingency Playbook - -1. Disable `feedser:sources:cert-cc:enableDetailMapping` flag (and optionally set `MaxNotesPerFetch=0` for a single cycle) to halt new detail ingestion. -2. Run connector once to update cursor; verify `pendingMappings` drains. -3. If advisories already persisted, coordinate with Merge to soft-delete affected `certcc/*` advisories by advisory key hash (no schema rollback required). -4. Re-run Phase 1 shadow validation before retrying. - -## 6. Communication Cadence - -- Daily check-in with Models/Merge leads (09:30 EDT) to surface normalizedVersions/provenance diffs. -- Post-phase reports in `#feedser-certcc` Slack channel summarising metrics, advisory counts, and outstanding issues. -- Escalate blockers >12 h via Runbook SEV-3 path and annotate `TASKS.md`. - -## 7. Open Questions / Next Actions - -- [ ] Confirm whether Merge requires additional provenance field masks before Phase 2 (waiting on feedback from 2025-10-11 sample). -- [ ] Decide if CSAF endpoint ingestion (optional) should piggyback on Phase 3 or stay deferred. -- [ ] Validate that FEEDCONN-CERTCC-02-010 coverage handles mixed 200/404 VINCE endpoints during partial outages. - -Once Dependencies (Section 2) are cleared and Phase 3 completes, update `src/StellaOps.Feedser.Source.CertCc/TASKS.md` and close FEEDCONN-CERTCC-02-009. +# FEEDCONN-CERTCC-02-009 – VINCE Detail & Map Reintegration Plan + +- **Author:** BE-Conn-CERTCC (current on-call) +- **Date:** 2025-10-11 +- **Scope:** Restore VINCE detail parsing and canonical mapping in Concelier without destabilising downstream Merge/Export pipelines. + +## 1. Current State Snapshot (2025-10-11) + +- ✅ Fetch pipeline, VINCE summary planner, and detail queue are live; documents land with `DocumentStatuses.PendingParse`. +- ✅ DTO aggregate (`CertCcNoteDto`) plus mapper emit vendor-centric `normalizedVersions` (`scheme=certcc.vendor`) and provenance aligned with `src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md`. +- ✅ Regression coverage exists for fetch/parse/map flows (`CertCcConnectorSnapshotTests`), but snapshot regeneration is gated on harness refresh (FEEDCONN-CERTCC-02-007) and QA handoff (FEEDCONN-CERTCC-02-008). +- ⚠️ Parse/map jobs are not scheduled; production still operates in fetch-only mode. +- ⚠️ Downstream Merge team is finalising normalized range ingestion per `src/FASTER_MODELING_AND_NORMALIZATION.md`; we must avoid publishing canonical records until they certify compatibility. + +## 2. Required Dependencies & Coordinated Tasks + +| Dependency | Owner(s) | Blocking Condition | Handshake | +|------------|----------|--------------------|-----------| +| FEEDCONN-CERTCC-02-004 (Canonical mapping & range primitives hardening) | BE-Conn-CERTCC + Models | Ensure mapper emits deterministic `normalizedVersions` array and provenance field masks | Daily sync with Models/Merge leads; share fixture diff before each enablement phase | +| FEEDCONN-CERTCC-02-007 (Connector test harness remediation) | BE-Conn-CERTCC, QA | Restore `AddSourceCommon` harness + canned VINCE fixtures so we can shadow-run parse/map | Required before Phase 1 | +| FEEDCONN-CERTCC-02-008 (Snapshot coverage handoff) | QA | Snapshot refresh process green to surface regressions | Required before Phase 2 | +| FEEDCONN-CERTCC-02-010 (Partial-detail graceful degradation) | BE-Conn-CERTCC | Resiliency for missing VINCE endpoints to avoid job wedging after reintegration | Should land before Phase 2 cutover | + +## 3. Phased Rollout Plan + +| Phase | Window (UTC) | Actions | Success Signals | Rollback | +|-------|--------------|---------|-----------------|----------| +| **0 – Pre-flight validation** | 2025-10-11 → 2025-10-12 | • Finish FEEDCONN-CERTCC-02-007 harness fixes and regenerate fixtures.
      • Run `dotnet test src/StellaOps.Concelier.Connector.CertCc.Tests` with `UPDATE_CERTCC_FIXTURES=0` to confirm deterministic baselines.
      • Generate sample advisory batch (`dotnet test … --filter SnapshotSmoke`) and deliver JSON diff to Merge for schema verification (`normalizedVersions[].scheme == certcc.vendor`, provenance masks populated). | • Harness tests green locally and in CI.
      • Merge sign-off that sample advisories conform to `FASTER_MODELING_AND_NORMALIZATION.md`. | N/A (no production enablement yet). | +| **1 – Shadow parse/map in staging** | Target start 2025-10-13 | • Register `source:cert-cc:parse` and `source:cert-cc:map` jobs, but gate them behind new config flag `concelier:sources:cert-cc:enableDetailMapping` (default `false`).
      • Deploy (restart required for options rebinding), enable flag, and point connector at staging Mongo with isolated collection (`advisories_certcc_shadow`).
      • Run connector for ≥2 cycles; compare advisory counts vs. fetch-only baseline and validate `concelier.range.primitives` metrics include `scheme=certcc.vendor`. | • No uncaught exceptions in staging logs.
      • Shadow advisories match expected vendor counts (±5%).
      • `certcc.summary.fetch.*` + new `certcc.map.duration.ms` metrics stable. | Disable flag; staging returns to fetch-only. No production impact. | +| **2 – Controlled production enablement** | Target start 2025-10-14 | • Redeploy production with flag enabled, start with job concurrency `1`, and reduce `MaxNotesPerFetch` to 5 for first 24 h.
      • Observe metrics dashboards hourly (fetch/map latency, pending queues, Mongo write throughput).
      • QA to replay latest snapshots and confirm no deterministic drift.
      • Publish advisory sample (top 10 changed docs) to Merge Slack channel for validation. | • Pending parse/mapping queues drain within expected SLA (<30 min).
      • No increase in merge dedupe anomalies.
      • Mongo writes stay within 10% of baseline. | Toggle flag off, re-run fetch-only. Clear `pendingMappings` via connector cursor reset if stuck. | +| **3 – Full production & cleanup** | Target start 2025-10-15 | • Restore `MaxNotesPerFetch` to configured default (20).
      • Remove temporary throttles and leave flag enabled by default.
      • Update `README.md` rollout notes; close FEEDCONN-CERTCC-02-009.
      • Kick off post-merge audit with Merge to ensure new advisories dedupe with other sources. | • Stable operations for ≥48 h, no degradation alerts.
      • Merge confirms conflict resolver behaviour unchanged. | If regression detected, revert to Phase 2 state or disable jobs; retain plan for reuse. | + +## 4. Monitoring & Validation Checklist + +- Dashboards: `certcc.*` meters (plan, summary fetch, detail fetch) plus `concelier.range.primitives` with tag `scheme=certcc.vendor`. +- Logs: ensure Parse/Map jobs emit `correlationId` aligned with fetch events for traceability. +- Data QA: run `tools/dump_advisory` against two VINCE notes (one multi-vendor, one single-vendor) every phase to spot-check normalized versions ordering and provenance. +- Storage: verify Mongo TTL/size for `raw_documents` and `dtos`—detail payload volume increases by ~3× when mapping resumes. + +## 5. Rollback / Contingency Playbook + +1. Disable `concelier:sources:cert-cc:enableDetailMapping` flag (and optionally set `MaxNotesPerFetch=0` for a single cycle) to halt new detail ingestion. +2. Run connector once to update cursor; verify `pendingMappings` drains. +3. If advisories already persisted, coordinate with Merge to soft-delete affected `certcc/*` advisories by advisory key hash (no schema rollback required). +4. Re-run Phase 1 shadow validation before retrying. + +## 6. Communication Cadence + +- Daily check-in with Models/Merge leads (09:30 EDT) to surface normalizedVersions/provenance diffs. +- Post-phase reports in `#concelier-certcc` Slack channel summarising metrics, advisory counts, and outstanding issues. +- Escalate blockers >12 h via Runbook SEV-3 path and annotate `TASKS.md`. + +## 7. Open Questions / Next Actions + +- [ ] Confirm whether Merge requires additional provenance field masks before Phase 2 (waiting on feedback from 2025-10-11 sample). +- [ ] Decide if CSAF endpoint ingestion (optional) should piggyback on Phase 3 or stay deferred. +- [ ] Validate that FEEDCONN-CERTCC-02-010 coverage handles mixed 200/404 VINCE endpoints during partial outages. + +Once Dependencies (Section 2) are cleared and Phase 3 completes, update `src/StellaOps.Concelier.Connector.CertCc/TASKS.md` and close FEEDCONN-CERTCC-02-009. diff --git a/src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md b/src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md new file mode 100644 index 00000000..c710b360 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md @@ -0,0 +1,20 @@ +# FEEDCONN-CERTCC-02-012 – Schema Sync & Snapshot Regeneration + +## Summary +- Re-ran `StellaOps.Concelier.Connector.CertCc.Tests` with `UPDATE_CERTCC_FIXTURES=1`; fixtures now capture SemVer-style normalized versions (`scheme=certcc.vendor`) and `provenance.decisionReason` values emitted by the mapper. +- Recorded HTTP request ordering is persisted in `certcc-requests.snapshot.json` to keep Merge aware of the deterministic fetch plan. +- Advisories snapshot (`certcc-advisories.snapshot.json`) reflects the dual-write storage changes (normalized versions + provenance) introduced by FEEDMODELS-SCHEMA-* and FEEDSTORAGE-DATA-*. + +## Artifacts +- `src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json` +- `src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-documents.snapshot.json` +- `src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-requests.snapshot.json` +- `src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/certcc-state.snapshot.json` + +## Validation steps +```bash +dotnet test src/StellaOps.Concelier.Connector.CertCc.Tests +UPDATE_CERTCC_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.CertCc.Tests +``` + +The first command verifies deterministic behavior; the second regenerates fixtures if a future schema change occurs. Share the four snapshot files above with Merge for their backfill diff. diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs index 6feca961..219e89ff 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcCursor.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcCursor.cs @@ -1,162 +1,162 @@ -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Cursors; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal sealed record CertCcCursor( - TimeWindowCursorState SummaryState, - IReadOnlyCollection PendingSummaries, - IReadOnlyCollection PendingNotes, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - DateTimeOffset? LastRun) -{ - private static readonly Guid[] EmptyGuidArray = Array.Empty(); - private static readonly string[] EmptyStringArray = Array.Empty(); - - public static CertCcCursor Empty { get; } = new( - TimeWindowCursorState.Empty, - EmptyGuidArray, - EmptyStringArray, - EmptyGuidArray, - EmptyGuidArray, - null); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument(); - - var summary = new BsonDocument(); - SummaryState.WriteTo(summary, "start", "end"); - document["summary"] = summary; - - document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString())); - document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note)); - document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); - document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); - - if (LastRun.HasValue) - { - document["lastRun"] = LastRun.Value.UtcDateTime; - } - - return document; - } - - public static CertCcCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; - if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) - { - summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); - } - - var pendingSummaries = ReadGuidArray(document, "pendingSummaries"); - var pendingNotes = ReadStringArray(document, "pendingNotes"); - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - DateTimeOffset? lastRun = null; - if (document.TryGetValue("lastRun", out var lastRunValue)) - { - lastRun = lastRunValue.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun); - } - - public CertCcCursor WithSummaryState(TimeWindowCursorState state) - => this with { SummaryState = state ?? TimeWindowCursorState.Empty }; - - public CertCcCursor WithPendingSummaries(IEnumerable? ids) - => this with { PendingSummaries = NormalizeGuidSet(ids) }; - - public CertCcCursor WithPendingNotes(IEnumerable? notes) - => this with { PendingNotes = NormalizeStringSet(notes) }; - - public CertCcCursor WithPendingDocuments(IEnumerable? ids) - => this with { PendingDocuments = NormalizeGuidSet(ids) }; - - public CertCcCursor WithPendingMappings(IEnumerable? ids) - => this with { PendingMappings = NormalizeGuidSet(ids) }; - - public CertCcCursor WithLastRun(DateTimeOffset? timestamp) - => this with { LastRun = timestamp }; - - private static Guid[] ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) - { - return EmptyGuidArray; - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (element is BsonString bsonString && Guid.TryParse(bsonString.AsString, out var parsed)) - { - results.Add(parsed); - continue; - } - - if (element is BsonBinaryData binary && binary.GuidRepresentation != GuidRepresentation.Unspecified) - { - results.Add(binary.ToGuid()); - } - } - - return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray(); - } - - private static string[] ReadStringArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) - { - return EmptyStringArray; - } - - var results = new List(array.Count); - foreach (var element in array) - { - switch (element) - { - case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString): - results.Add(bsonString.AsString.Trim()); - break; - case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString: - results.Add(inner.AsString.Trim()); - break; - } - } - - return results.Count == 0 - ? EmptyStringArray - : results - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static Guid[] NormalizeGuidSet(IEnumerable? ids) - => ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray; - - private static string[] NormalizeStringSet(IEnumerable? values) - => values is null - ? EmptyStringArray - : values - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); -} +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal sealed record CertCcCursor( + TimeWindowCursorState SummaryState, + IReadOnlyCollection PendingSummaries, + IReadOnlyCollection PendingNotes, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + DateTimeOffset? LastRun) +{ + private static readonly Guid[] EmptyGuidArray = Array.Empty(); + private static readonly string[] EmptyStringArray = Array.Empty(); + + public static CertCcCursor Empty { get; } = new( + TimeWindowCursorState.Empty, + EmptyGuidArray, + EmptyStringArray, + EmptyGuidArray, + EmptyGuidArray, + null); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument(); + + var summary = new BsonDocument(); + SummaryState.WriteTo(summary, "start", "end"); + document["summary"] = summary; + + document["pendingSummaries"] = new BsonArray(PendingSummaries.Select(static id => id.ToString())); + document["pendingNotes"] = new BsonArray(PendingNotes.Select(static note => note)); + document["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())); + document["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())); + + if (LastRun.HasValue) + { + document["lastRun"] = LastRun.Value.UtcDateTime; + } + + return document; + } + + public static CertCcCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + TimeWindowCursorState summaryState = TimeWindowCursorState.Empty; + if (document.TryGetValue("summary", out var summaryValue) && summaryValue is BsonDocument summaryDocument) + { + summaryState = TimeWindowCursorState.FromBsonDocument(summaryDocument, "start", "end"); + } + + var pendingSummaries = ReadGuidArray(document, "pendingSummaries"); + var pendingNotes = ReadStringArray(document, "pendingNotes"); + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + DateTimeOffset? lastRun = null; + if (document.TryGetValue("lastRun", out var lastRunValue)) + { + lastRun = lastRunValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(lastRunValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(lastRunValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + return new CertCcCursor(summaryState, pendingSummaries, pendingNotes, pendingDocuments, pendingMappings, lastRun); + } + + public CertCcCursor WithSummaryState(TimeWindowCursorState state) + => this with { SummaryState = state ?? TimeWindowCursorState.Empty }; + + public CertCcCursor WithPendingSummaries(IEnumerable? ids) + => this with { PendingSummaries = NormalizeGuidSet(ids) }; + + public CertCcCursor WithPendingNotes(IEnumerable? notes) + => this with { PendingNotes = NormalizeStringSet(notes) }; + + public CertCcCursor WithPendingDocuments(IEnumerable? ids) + => this with { PendingDocuments = NormalizeGuidSet(ids) }; + + public CertCcCursor WithPendingMappings(IEnumerable? ids) + => this with { PendingMappings = NormalizeGuidSet(ids) }; + + public CertCcCursor WithLastRun(DateTimeOffset? timestamp) + => this with { LastRun = timestamp }; + + private static Guid[] ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyGuidArray; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is BsonString bsonString && Guid.TryParse(bsonString.AsString, out var parsed)) + { + results.Add(parsed); + continue; + } + + if (element is BsonBinaryData binary && binary.GuidRepresentation != GuidRepresentation.Unspecified) + { + results.Add(binary.ToGuid()); + } + } + + return results.Count == 0 ? EmptyGuidArray : results.Distinct().ToArray(); + } + + private static string[] ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array || array.Count == 0) + { + return EmptyStringArray; + } + + var results = new List(array.Count); + foreach (var element in array) + { + switch (element) + { + case BsonString bsonString when !string.IsNullOrWhiteSpace(bsonString.AsString): + results.Add(bsonString.AsString.Trim()); + break; + case BsonDocument bsonDocument when bsonDocument.TryGetValue("value", out var inner) && inner.IsString: + results.Add(inner.AsString.Trim()); + break; + } + } + + return results.Count == 0 + ? EmptyStringArray + : results + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static Guid[] NormalizeGuidSet(IEnumerable? ids) + => ids?.Where(static id => id != Guid.Empty).Distinct().ToArray() ?? EmptyGuidArray; + + private static string[] NormalizeStringSet(IEnumerable? values) + => values is null + ? EmptyStringArray + : values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcDiagnostics.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcDiagnostics.cs index 7ddaabf8..a03b2589 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcDiagnostics.cs @@ -1,214 +1,214 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; -using StellaOps.Feedser.Source.Common.Cursors; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -/// -/// Emits CERT/CC-specific telemetry for summary planning and fetch activity. -/// -public sealed class CertCcDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.CertCc"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _planWindows; - private readonly Counter _planRequests; - private readonly Histogram _planWindowDays; - private readonly Counter _summaryFetchAttempts; - private readonly Counter _summaryFetchSuccess; - private readonly Counter _summaryFetchUnchanged; - private readonly Counter _summaryFetchFailures; - private readonly Counter _detailFetchAttempts; - private readonly Counter _detailFetchSuccess; - private readonly Counter _detailFetchUnchanged; - private readonly Counter _detailFetchMissing; - private readonly Counter _detailFetchFailures; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Histogram _parseVendorCount; - private readonly Histogram _parseStatusCount; - private readonly Histogram _parseVulnerabilityCount; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Histogram _mapAffectedPackageCount; - private readonly Histogram _mapNormalizedVersionCount; - - public CertCcDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _planWindows = _meter.CreateCounter( - name: "certcc.plan.windows", - unit: "windows", - description: "Number of summary planning windows evaluated."); - _planRequests = _meter.CreateCounter( - name: "certcc.plan.requests", - unit: "requests", - description: "Total CERT/CC summary endpoints queued by the planner."); - _planWindowDays = _meter.CreateHistogram( - name: "certcc.plan.window_days", - unit: "day", - description: "Duration of each planning window in days."); - _summaryFetchAttempts = _meter.CreateCounter( - name: "certcc.summary.fetch.attempts", - unit: "operations", - description: "Number of VINCE summary fetch attempts."); - _summaryFetchSuccess = _meter.CreateCounter( - name: "certcc.summary.fetch.success", - unit: "operations", - description: "Number of VINCE summary fetches persisted to storage."); - _summaryFetchUnchanged = _meter.CreateCounter( - name: "certcc.summary.fetch.not_modified", - unit: "operations", - description: "Number of VINCE summary fetches returning HTTP 304."); - _summaryFetchFailures = _meter.CreateCounter( - name: "certcc.summary.fetch.failures", - unit: "operations", - description: "Number of VINCE summary fetches that failed after retries."); - _detailFetchAttempts = _meter.CreateCounter( - name: "certcc.detail.fetch.attempts", - unit: "operations", - description: "Number of VINCE detail fetch attempts."); - _detailFetchSuccess = _meter.CreateCounter( - name: "certcc.detail.fetch.success", - unit: "operations", - description: "Number of VINCE detail fetches that returned payloads."); - _detailFetchUnchanged = _meter.CreateCounter( - name: "certcc.detail.fetch.unchanged", - unit: "operations", - description: "Number of VINCE detail fetches returning HTTP 304."); - _detailFetchMissing = _meter.CreateCounter( - name: "certcc.detail.fetch.missing", - unit: "operations", - description: "Number of optional VINCE detail endpoints missing but tolerated."); - _detailFetchFailures = _meter.CreateCounter( - name: "certcc.detail.fetch.failures", - unit: "operations", - description: "Number of VINCE detail fetches that failed after retries."); - _parseSuccess = _meter.CreateCounter( - name: "certcc.parse.success", - unit: "documents", - description: "Number of VINCE note bundles parsed into DTOs."); - _parseFailures = _meter.CreateCounter( - name: "certcc.parse.failures", - unit: "documents", - description: "Number of VINCE note bundles that failed to parse."); - _parseVendorCount = _meter.CreateHistogram( - name: "certcc.parse.vendors.count", - unit: "vendors", - description: "Distribution of vendor statements per VINCE note."); - _parseStatusCount = _meter.CreateHistogram( - name: "certcc.parse.statuses.count", - unit: "entries", - description: "Distribution of vendor status entries per VINCE note."); - _parseVulnerabilityCount = _meter.CreateHistogram( - name: "certcc.parse.vulnerabilities.count", - unit: "entries", - description: "Distribution of vulnerability records per VINCE note."); - _mapSuccess = _meter.CreateCounter( - name: "certcc.map.success", - unit: "advisories", - description: "Number of canonical advisories emitted by the CERT/CC mapper."); - _mapFailures = _meter.CreateCounter( - name: "certcc.map.failures", - unit: "advisories", - description: "Number of CERT/CC advisory mapping attempts that failed."); - _mapAffectedPackageCount = _meter.CreateHistogram( - name: "certcc.map.affected.count", - unit: "packages", - description: "Distribution of affected packages emitted per CERT/CC advisory."); - _mapNormalizedVersionCount = _meter.CreateHistogram( - name: "certcc.map.normalized_versions.count", - unit: "rules", - description: "Distribution of normalized version rules emitted per CERT/CC advisory."); - } - - public void PlanEvaluated(TimeWindow window, int requestCount) - { - _planWindows.Add(1); - - if (requestCount > 0) - { - _planRequests.Add(requestCount); - } - - var duration = window.Duration; - if (duration > TimeSpan.Zero) - { - _planWindowDays.Record(duration.TotalDays); - } - } - - public void SummaryFetchAttempt(CertCcSummaryScope scope) - => _summaryFetchAttempts.Add(1, ScopeTag(scope)); - - public void SummaryFetchSuccess(CertCcSummaryScope scope) - => _summaryFetchSuccess.Add(1, ScopeTag(scope)); - - public void SummaryFetchUnchanged(CertCcSummaryScope scope) - => _summaryFetchUnchanged.Add(1, ScopeTag(scope)); - - public void SummaryFetchFailure(CertCcSummaryScope scope) - => _summaryFetchFailures.Add(1, ScopeTag(scope)); - - public void DetailFetchAttempt(string endpoint) - => _detailFetchAttempts.Add(1, EndpointTag(endpoint)); - - public void DetailFetchSuccess(string endpoint) - => _detailFetchSuccess.Add(1, EndpointTag(endpoint)); - - public void DetailFetchUnchanged(string endpoint) - => _detailFetchUnchanged.Add(1, EndpointTag(endpoint)); - - public void DetailFetchMissing(string endpoint) - => _detailFetchMissing.Add(1, EndpointTag(endpoint)); - - public void DetailFetchFailure(string endpoint) - => _detailFetchFailures.Add(1, EndpointTag(endpoint)); - - public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount) - { - _parseSuccess.Add(1); - if (vendorCount >= 0) - { - _parseVendorCount.Record(vendorCount); - } - if (statusCount >= 0) - { - _parseStatusCount.Record(statusCount); - } - if (vulnerabilityCount >= 0) - { - _parseVulnerabilityCount.Record(vulnerabilityCount); - } - } - - public void ParseFailure() - => _parseFailures.Add(1); - - public void MapSuccess(int affectedPackageCount, int normalizedVersionCount) - { - _mapSuccess.Add(1); - if (affectedPackageCount >= 0) - { - _mapAffectedPackageCount.Record(affectedPackageCount); - } - if (normalizedVersionCount >= 0) - { - _mapNormalizedVersionCount.Record(normalizedVersionCount); - } - } - - public void MapFailure() - => _mapFailures.Add(1); - - private static KeyValuePair ScopeTag(CertCcSummaryScope scope) - => new("scope", scope.ToString().ToLowerInvariant()); - - private static KeyValuePair EndpointTag(string endpoint) - => new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant()); - - public void Dispose() => _meter.Dispose(); -} +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +/// +/// Emits CERT/CC-specific telemetry for summary planning and fetch activity. +/// +public sealed class CertCcDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.CertCc"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _planWindows; + private readonly Counter _planRequests; + private readonly Histogram _planWindowDays; + private readonly Counter _summaryFetchAttempts; + private readonly Counter _summaryFetchSuccess; + private readonly Counter _summaryFetchUnchanged; + private readonly Counter _summaryFetchFailures; + private readonly Counter _detailFetchAttempts; + private readonly Counter _detailFetchSuccess; + private readonly Counter _detailFetchUnchanged; + private readonly Counter _detailFetchMissing; + private readonly Counter _detailFetchFailures; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Histogram _parseVendorCount; + private readonly Histogram _parseStatusCount; + private readonly Histogram _parseVulnerabilityCount; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Histogram _mapAffectedPackageCount; + private readonly Histogram _mapNormalizedVersionCount; + + public CertCcDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _planWindows = _meter.CreateCounter( + name: "certcc.plan.windows", + unit: "windows", + description: "Number of summary planning windows evaluated."); + _planRequests = _meter.CreateCounter( + name: "certcc.plan.requests", + unit: "requests", + description: "Total CERT/CC summary endpoints queued by the planner."); + _planWindowDays = _meter.CreateHistogram( + name: "certcc.plan.window_days", + unit: "day", + description: "Duration of each planning window in days."); + _summaryFetchAttempts = _meter.CreateCounter( + name: "certcc.summary.fetch.attempts", + unit: "operations", + description: "Number of VINCE summary fetch attempts."); + _summaryFetchSuccess = _meter.CreateCounter( + name: "certcc.summary.fetch.success", + unit: "operations", + description: "Number of VINCE summary fetches persisted to storage."); + _summaryFetchUnchanged = _meter.CreateCounter( + name: "certcc.summary.fetch.not_modified", + unit: "operations", + description: "Number of VINCE summary fetches returning HTTP 304."); + _summaryFetchFailures = _meter.CreateCounter( + name: "certcc.summary.fetch.failures", + unit: "operations", + description: "Number of VINCE summary fetches that failed after retries."); + _detailFetchAttempts = _meter.CreateCounter( + name: "certcc.detail.fetch.attempts", + unit: "operations", + description: "Number of VINCE detail fetch attempts."); + _detailFetchSuccess = _meter.CreateCounter( + name: "certcc.detail.fetch.success", + unit: "operations", + description: "Number of VINCE detail fetches that returned payloads."); + _detailFetchUnchanged = _meter.CreateCounter( + name: "certcc.detail.fetch.unchanged", + unit: "operations", + description: "Number of VINCE detail fetches returning HTTP 304."); + _detailFetchMissing = _meter.CreateCounter( + name: "certcc.detail.fetch.missing", + unit: "operations", + description: "Number of optional VINCE detail endpoints missing but tolerated."); + _detailFetchFailures = _meter.CreateCounter( + name: "certcc.detail.fetch.failures", + unit: "operations", + description: "Number of VINCE detail fetches that failed after retries."); + _parseSuccess = _meter.CreateCounter( + name: "certcc.parse.success", + unit: "documents", + description: "Number of VINCE note bundles parsed into DTOs."); + _parseFailures = _meter.CreateCounter( + name: "certcc.parse.failures", + unit: "documents", + description: "Number of VINCE note bundles that failed to parse."); + _parseVendorCount = _meter.CreateHistogram( + name: "certcc.parse.vendors.count", + unit: "vendors", + description: "Distribution of vendor statements per VINCE note."); + _parseStatusCount = _meter.CreateHistogram( + name: "certcc.parse.statuses.count", + unit: "entries", + description: "Distribution of vendor status entries per VINCE note."); + _parseVulnerabilityCount = _meter.CreateHistogram( + name: "certcc.parse.vulnerabilities.count", + unit: "entries", + description: "Distribution of vulnerability records per VINCE note."); + _mapSuccess = _meter.CreateCounter( + name: "certcc.map.success", + unit: "advisories", + description: "Number of canonical advisories emitted by the CERT/CC mapper."); + _mapFailures = _meter.CreateCounter( + name: "certcc.map.failures", + unit: "advisories", + description: "Number of CERT/CC advisory mapping attempts that failed."); + _mapAffectedPackageCount = _meter.CreateHistogram( + name: "certcc.map.affected.count", + unit: "packages", + description: "Distribution of affected packages emitted per CERT/CC advisory."); + _mapNormalizedVersionCount = _meter.CreateHistogram( + name: "certcc.map.normalized_versions.count", + unit: "rules", + description: "Distribution of normalized version rules emitted per CERT/CC advisory."); + } + + public void PlanEvaluated(TimeWindow window, int requestCount) + { + _planWindows.Add(1); + + if (requestCount > 0) + { + _planRequests.Add(requestCount); + } + + var duration = window.Duration; + if (duration > TimeSpan.Zero) + { + _planWindowDays.Record(duration.TotalDays); + } + } + + public void SummaryFetchAttempt(CertCcSummaryScope scope) + => _summaryFetchAttempts.Add(1, ScopeTag(scope)); + + public void SummaryFetchSuccess(CertCcSummaryScope scope) + => _summaryFetchSuccess.Add(1, ScopeTag(scope)); + + public void SummaryFetchUnchanged(CertCcSummaryScope scope) + => _summaryFetchUnchanged.Add(1, ScopeTag(scope)); + + public void SummaryFetchFailure(CertCcSummaryScope scope) + => _summaryFetchFailures.Add(1, ScopeTag(scope)); + + public void DetailFetchAttempt(string endpoint) + => _detailFetchAttempts.Add(1, EndpointTag(endpoint)); + + public void DetailFetchSuccess(string endpoint) + => _detailFetchSuccess.Add(1, EndpointTag(endpoint)); + + public void DetailFetchUnchanged(string endpoint) + => _detailFetchUnchanged.Add(1, EndpointTag(endpoint)); + + public void DetailFetchMissing(string endpoint) + => _detailFetchMissing.Add(1, EndpointTag(endpoint)); + + public void DetailFetchFailure(string endpoint) + => _detailFetchFailures.Add(1, EndpointTag(endpoint)); + + public void ParseSuccess(int vendorCount, int statusCount, int vulnerabilityCount) + { + _parseSuccess.Add(1); + if (vendorCount >= 0) + { + _parseVendorCount.Record(vendorCount); + } + if (statusCount >= 0) + { + _parseStatusCount.Record(statusCount); + } + if (vulnerabilityCount >= 0) + { + _parseVulnerabilityCount.Record(vulnerabilityCount); + } + } + + public void ParseFailure() + => _parseFailures.Add(1); + + public void MapSuccess(int affectedPackageCount, int normalizedVersionCount) + { + _mapSuccess.Add(1); + if (affectedPackageCount >= 0) + { + _mapAffectedPackageCount.Record(affectedPackageCount); + } + if (normalizedVersionCount >= 0) + { + _mapNormalizedVersionCount.Record(normalizedVersionCount); + } + } + + public void MapFailure() + => _mapFailures.Add(1); + + private static KeyValuePair ScopeTag(CertCcSummaryScope scope) + => new("scope", scope.ToString().ToLowerInvariant()); + + private static KeyValuePair EndpointTag(string endpoint) + => new("endpoint", string.IsNullOrWhiteSpace(endpoint) ? "note" : endpoint.ToLowerInvariant()); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs index d4dfc714..ab62cf07 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcMapper.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcMapper.cs @@ -1,607 +1,607 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Text.RegularExpressions; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal static class CertCcMapper -{ - private const string AdvisoryPrefix = "certcc"; - private const string VendorNormalizedVersionScheme = "certcc.vendor"; - - public static Advisory Map( - CertCcNoteDto dto, - DocumentRecord document, - DtoRecord dtoRecord, - string sourceName) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(dtoRecord); - ArgumentException.ThrowIfNullOrEmpty(sourceName); - - var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); - var fetchedAt = document.FetchedAt.ToUniversalTime(); - - var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; - - var advisoryKey = BuildAdvisoryKey(metadata); - var title = string.IsNullOrWhiteSpace(metadata.Title) ? advisoryKey : metadata.Title.Trim(); - var summary = ExtractSummary(metadata); - - var aliases = BuildAliases(dto).ToArray(); - var references = BuildReferences(dto, metadata, sourceName, recordedAt).ToArray(); - var affectedPackages = BuildAffectedPackages(dto, metadata, sourceName, recordedAt).ToArray(); - - var provenance = new[] - { - new AdvisoryProvenance(sourceName, "document", document.Uri, fetchedAt), - new AdvisoryProvenance(sourceName, "map", metadata.VuId ?? metadata.IdNumber ?? advisoryKey, recordedAt), - }; - - return new Advisory( - advisoryKey, - title, - summary, - language: "en", - metadata.Published?.ToUniversalTime(), - metadata.Updated?.ToUniversalTime(), - severity: null, - exploitKnown: false, - aliases, - references, - affectedPackages, - cvssMetrics: Array.Empty(), - provenance); - } - - private static string BuildAdvisoryKey(CertCcNoteMetadata metadata) - { - if (metadata is null) - { - return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; - } - - var vuKey = NormalizeVuId(metadata.VuId); - if (vuKey.Length > 0) - { - return $"{AdvisoryPrefix}/{vuKey}"; - } - - var id = SanitizeToken(metadata.IdNumber); - if (id.Length > 0) - { - return $"{AdvisoryPrefix}/vu-{id}"; - } - - return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; - } - - private static string NormalizeVuId(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var digits = new string(value.Where(char.IsDigit).ToArray()); - if (digits.Length > 0) - { - return $"vu-{digits}"; - } - - var sanitized = value.Trim().ToLowerInvariant(); - sanitized = sanitized.Replace("vu#", "vu-", StringComparison.OrdinalIgnoreCase); - sanitized = sanitized.Replace('#', '-'); - sanitized = sanitized.Replace(' ', '-'); - - return SanitizeToken(sanitized); - } - - private static string SanitizeToken(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var trimmed = value.Trim(); - var filtered = new string(trimmed - .Select(ch => char.IsLetterOrDigit(ch) || ch is '-' or '_' ? ch : '-') - .ToArray()); - - return filtered.Trim('-').ToLowerInvariant(); - } - - private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex WhitespaceRegex = new("[ \t\f\r]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex ParagraphRegex = new("<\\s*/?\\s*p[^>]*>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - - private static string? ExtractSummary(CertCcNoteMetadata metadata) - { - if (metadata is null) - { - return null; - } - - var summary = string.IsNullOrWhiteSpace(metadata.Summary) ? metadata.Overview : metadata.Summary; - if (string.IsNullOrWhiteSpace(summary)) - { - return null; - } - - return HtmlToPlainText(summary); - } - - private static string HtmlToPlainText(string html) - { - if (string.IsNullOrWhiteSpace(html)) - { - return string.Empty; - } - - var normalized = html - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
    • ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
      • ", "\n", StringComparison.OrdinalIgnoreCase); - - normalized = ParagraphRegex.Replace(normalized, "\n"); - - var withoutTags = HtmlTagRegex.Replace(normalized, " "); - var decoded = WebUtility.HtmlDecode(withoutTags) ?? string.Empty; - var collapsedSpaces = WhitespaceRegex.Replace(decoded, " "); - var collapsedNewlines = Regex.Replace(collapsedSpaces, "\n{2,}", "\n", RegexOptions.Compiled); - return collapsedNewlines.Trim(); - } - - private static IEnumerable BuildAliases(CertCcNoteDto dto) - { - var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); - - var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; - - if (!string.IsNullOrWhiteSpace(metadata.VuId)) - { - aliases.Add(metadata.VuId.Trim()); - } - - if (!string.IsNullOrWhiteSpace(metadata.IdNumber)) - { - aliases.Add($"VU#{metadata.IdNumber.Trim()}"); - } - - foreach (var cve in metadata.CveIds ?? Array.Empty()) - { - if (string.IsNullOrWhiteSpace(cve)) - { - continue; - } - - aliases.Add(cve.Trim()); - } - - foreach (var vulnerability in dto.Vulnerabilities ?? Array.Empty()) - { - if (string.IsNullOrWhiteSpace(vulnerability.CveId)) - { - continue; - } - - aliases.Add(vulnerability.CveId.Trim()); - } - - return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase); - } - - private static IEnumerable BuildReferences( - CertCcNoteDto dto, - CertCcNoteMetadata metadata, - string sourceName, - DateTimeOffset recordedAt) - { - var references = new List(); - var canonicalUri = !string.IsNullOrWhiteSpace(metadata.PrimaryUrl) - ? metadata.PrimaryUrl! - : (string.IsNullOrWhiteSpace(metadata.IdNumber) - ? "https://www.kb.cert.org/vuls/" - : $"https://www.kb.cert.org/vuls/id/{metadata.IdNumber.Trim()}/"); - - var provenance = new AdvisoryProvenance(sourceName, "reference", canonicalUri, recordedAt); - - TryAddReference(references, canonicalUri, "advisory", "certcc.note", null, provenance); - - foreach (var url in metadata.PublicUrls ?? Array.Empty()) - { - TryAddReference(references, url, "reference", "certcc.public", null, provenance); - } - - foreach (var vendor in dto.Vendors ?? Array.Empty()) - { - foreach (var url in vendor.References ?? Array.Empty()) - { - TryAddReference(references, url, "reference", "certcc.vendor", vendor.Vendor, provenance); - } - - var statementText = vendor.Statement ?? string.Empty; - var patches = CertCcVendorStatementParser.Parse(statementText); - foreach (var patch in patches) - { - if (!string.IsNullOrWhiteSpace(patch.RawLine) && TryFindEmbeddedUrl(patch.RawLine!, out var rawUrl)) - { - TryAddReference(references, rawUrl, "reference", "certcc.vendor.statement", vendor.Vendor, provenance); - } - } - } - - foreach (var status in dto.VendorStatuses ?? Array.Empty()) - { - foreach (var url in status.References ?? Array.Empty()) - { - TryAddReference(references, url, "reference", "certcc.vendor.status", status.Vendor, provenance); - } - - if (!string.IsNullOrWhiteSpace(status.Statement) && TryFindEmbeddedUrl(status.Statement!, out var embedded)) - { - TryAddReference(references, embedded, "reference", "certcc.vendor.status", status.Vendor, provenance); - } - } - - return references - .GroupBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .Select(static group => group - .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) - .ThenBy(static reference => reference.SourceTag ?? string.Empty, StringComparer.Ordinal) - .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .First()) - .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) - .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase); - } - - private static void TryAddReference( - ICollection references, - string? url, - string kind, - string? sourceTag, - string? summary, - AdvisoryProvenance provenance) - { - if (string.IsNullOrWhiteSpace(url)) - { - return; - } - - var candidate = url.Trim(); - if (!Uri.TryCreate(candidate, UriKind.Absolute, out var parsed)) - { - return; - } - - if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) - { - return; - } - - var normalized = parsed.ToString(); - - try - { - references.Add(new AdvisoryReference(normalized, kind, sourceTag, summary, provenance)); - } - catch (ArgumentException) - { - // ignore invalid references - } - } - - private static bool TryFindEmbeddedUrl(string text, out string? url) - { - url = null; - if (string.IsNullOrWhiteSpace(text)) - { - return false; - } - - var tokens = text.Split(new[] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); - foreach (var token in tokens) - { - var trimmed = token.Trim().TrimEnd('.', ',', ')', ';', ']', '}'); - if (trimmed.Length == 0) - { - continue; - } - - if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var parsed)) - { - continue; - } - - if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) - { - continue; - } - - url = parsed.ToString(); - return true; - } - - return false; - } - - private static IEnumerable BuildAffectedPackages( - CertCcNoteDto dto, - CertCcNoteMetadata metadata, - string sourceName, - DateTimeOffset recordedAt) - { - var vendors = dto.Vendors ?? Array.Empty(); - var statuses = dto.VendorStatuses ?? Array.Empty(); - - if (vendors.Count == 0 && statuses.Count == 0) - { - return Array.Empty(); - } - - var statusLookup = statuses - .GroupBy(static status => NormalizeVendorKey(status.Vendor)) - .ToDictionary(static group => group.Key, static group => group.ToArray(), StringComparer.OrdinalIgnoreCase); - - var packages = new List(); - - foreach (var vendor in vendors.OrderBy(static v => v.Vendor, StringComparer.OrdinalIgnoreCase)) - { - var key = NormalizeVendorKey(vendor.Vendor); - var vendorStatuses = statusLookup.TryGetValue(key, out var value) - ? value - : Array.Empty(); - - if (BuildVendorPackage(vendor, vendorStatuses, sourceName, recordedAt) is { } package) - { - packages.Add(package); - } - - statusLookup.Remove(key); - } - - foreach (var remaining in statusLookup.Values) - { - if (remaining.Length == 0) - { - continue; - } - - var vendorName = remaining[0].Vendor; - var fallbackVendor = new CertCcVendorDto( - vendorName, - ContactDate: null, - StatementDate: null, - Updated: remaining - .Select(static status => status.DateUpdated) - .Where(static update => update.HasValue) - .OrderByDescending(static update => update) - .FirstOrDefault(), - Statement: remaining - .Select(static status => status.Statement) - .FirstOrDefault(static statement => !string.IsNullOrWhiteSpace(statement)), - Addendum: null, - References: remaining - .SelectMany(static status => status.References ?? Array.Empty()) - .ToArray()); - - if (BuildVendorPackage(fallbackVendor, remaining, sourceName, recordedAt) is { } package) - { - packages.Add(package); - } - } - - return packages - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static AffectedPackage? BuildVendorPackage( - CertCcVendorDto vendor, - IReadOnlyList statuses, - string sourceName, - DateTimeOffset recordedAt) - { - var vendorName = string.IsNullOrWhiteSpace(vendor.Vendor) - ? (statuses.FirstOrDefault()?.Vendor?.Trim() ?? string.Empty) - : vendor.Vendor.Trim(); - - if (vendorName.Length == 0) - { - return null; - } - - var packageProvenance = new AdvisoryProvenance(sourceName, "vendor", vendorName, recordedAt); - var rangeProvenance = new AdvisoryProvenance(sourceName, "vendor-range", vendorName, recordedAt); - - var patches = CertCcVendorStatementParser.Parse(vendor.Statement ?? string.Empty); - var normalizedVersions = BuildNormalizedVersions(vendorName, patches); - var vendorStatuses = BuildStatuses(vendorName, statuses, sourceName, recordedAt); - var primitives = BuildRangePrimitives(vendor, vendorStatuses, patches); - - var range = new AffectedVersionRange( - rangeKind: "vendor", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: null, - provenance: rangeProvenance, - primitives: primitives); - - return new AffectedPackage( - AffectedPackageTypes.Vendor, - vendorName, - platform: null, - versionRanges: new[] { range }, - normalizedVersions: normalizedVersions, - statuses: vendorStatuses, - provenance: new[] { packageProvenance }); - } - - private static IReadOnlyList BuildNormalizedVersions( - string vendorName, - IReadOnlyList patches) - { - if (patches.Count == 0) - { - return Array.Empty(); - } - - var rules = new List(); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var patch in patches) - { - if (string.IsNullOrWhiteSpace(patch.Version)) - { - continue; - } - - var version = patch.Version.Trim(); - if (!seen.Add($"{patch.Product}|{version}")) - { - continue; - } - - var notes = string.IsNullOrWhiteSpace(patch.Product) - ? vendorName - : $"{vendorName}::{patch.Product.Trim()}"; - - rules.Add(new NormalizedVersionRule( - VendorNormalizedVersionScheme, - NormalizedVersionRuleTypes.Exact, - value: version, - notes: notes)); - } - - return rules.Count == 0 ? Array.Empty() : rules; - } - - private static IReadOnlyList BuildStatuses( - string vendorName, - IReadOnlyList statuses, - string sourceName, - DateTimeOffset recordedAt) - { - if (statuses.Count == 0) - { - return Array.Empty(); - } - - var result = new List(); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var status in statuses) - { - if (!AffectedPackageStatusCatalog.TryNormalize(status.Status, out var normalized)) - { - continue; - } - - var cve = status.CveId?.Trim() ?? string.Empty; - var key = string.IsNullOrWhiteSpace(cve) - ? normalized - : $"{normalized}|{cve}"; - - if (!seen.Add(key)) - { - continue; - } - - var provenance = new AdvisoryProvenance( - sourceName, - "vendor-status", - string.IsNullOrWhiteSpace(cve) ? vendorName : $"{vendorName}:{cve}", - recordedAt); - - result.Add(new AffectedPackageStatus(normalized, provenance)); - } - - return result - .OrderBy(static status => status.Status, StringComparer.Ordinal) - .ThenBy(static status => status.Provenance.Value ?? string.Empty, StringComparer.Ordinal) - .ToArray(); - } - - private static RangePrimitives? BuildRangePrimitives( - CertCcVendorDto vendor, - IReadOnlyList statuses, - IReadOnlyList patches) - { - var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase); - - AddVendorExtension(extensions, "certcc.vendor.name", vendor.Vendor); - AddVendorExtension(extensions, "certcc.vendor.statement.raw", HtmlToPlainText(vendor.Statement ?? string.Empty), 2048); - AddVendorExtension(extensions, "certcc.vendor.addendum", HtmlToPlainText(vendor.Addendum ?? string.Empty), 1024); - AddVendorExtension(extensions, "certcc.vendor.contactDate", FormatDate(vendor.ContactDate)); - AddVendorExtension(extensions, "certcc.vendor.statementDate", FormatDate(vendor.StatementDate)); - AddVendorExtension(extensions, "certcc.vendor.updated", FormatDate(vendor.Updated)); - - if (vendor.References is { Count: > 0 }) - { - AddVendorExtension(extensions, "certcc.vendor.references", string.Join(" ", vendor.References)); - } - - if (statuses.Count > 0) - { - var serialized = string.Join(";", statuses - .Select(static status => status.Provenance.Value is { Length: > 0 } - ? $"{status.Provenance.Value.Split(':').Last()}={status.Status}" - : status.Status)); - - AddVendorExtension(extensions, "certcc.vendor.statuses", serialized); - } - - if (patches.Count > 0) - { - var serialized = string.Join(";", patches.Select(static patch => - { - var product = string.IsNullOrWhiteSpace(patch.Product) ? "unknown" : patch.Product.Trim(); - return $"{product}={patch.Version.Trim()}"; - })); - - AddVendorExtension(extensions, "certcc.vendor.patches", serialized, 2048); - } - - return extensions.Count == 0 - ? null - : new RangePrimitives(null, null, null, extensions); - } - - private static void AddVendorExtension(IDictionary extensions, string key, string? value, int maxLength = 512) - { - if (string.IsNullOrWhiteSpace(value)) - { - return; - } - - var trimmed = value.Trim(); - if (trimmed.Length > maxLength) - { - trimmed = trimmed[..maxLength].Trim(); - } - - if (trimmed.Length == 0) - { - return; - } - - extensions[key] = trimmed; - } - - private static string? FormatDate(DateTimeOffset? value) - => value?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); - - private static string NormalizeVendorKey(string? value) - => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToLowerInvariant(); -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal static class CertCcMapper +{ + private const string AdvisoryPrefix = "certcc"; + private const string VendorNormalizedVersionScheme = "certcc.vendor"; + + public static Advisory Map( + CertCcNoteDto dto, + DocumentRecord document, + DtoRecord dtoRecord, + string sourceName) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + ArgumentException.ThrowIfNullOrEmpty(sourceName); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + var fetchedAt = document.FetchedAt.ToUniversalTime(); + + var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; + + var advisoryKey = BuildAdvisoryKey(metadata); + var title = string.IsNullOrWhiteSpace(metadata.Title) ? advisoryKey : metadata.Title.Trim(); + var summary = ExtractSummary(metadata); + + var aliases = BuildAliases(dto).ToArray(); + var references = BuildReferences(dto, metadata, sourceName, recordedAt).ToArray(); + var affectedPackages = BuildAffectedPackages(dto, metadata, sourceName, recordedAt).ToArray(); + + var provenance = new[] + { + new AdvisoryProvenance(sourceName, "document", document.Uri, fetchedAt), + new AdvisoryProvenance(sourceName, "map", metadata.VuId ?? metadata.IdNumber ?? advisoryKey, recordedAt), + }; + + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + metadata.Published?.ToUniversalTime(), + metadata.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance); + } + + private static string BuildAdvisoryKey(CertCcNoteMetadata metadata) + { + if (metadata is null) + { + return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; + } + + var vuKey = NormalizeVuId(metadata.VuId); + if (vuKey.Length > 0) + { + return $"{AdvisoryPrefix}/{vuKey}"; + } + + var id = SanitizeToken(metadata.IdNumber); + if (id.Length > 0) + { + return $"{AdvisoryPrefix}/vu-{id}"; + } + + return $"{AdvisoryPrefix}/{Guid.NewGuid():N}"; + } + + private static string NormalizeVuId(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var digits = new string(value.Where(char.IsDigit).ToArray()); + if (digits.Length > 0) + { + return $"vu-{digits}"; + } + + var sanitized = value.Trim().ToLowerInvariant(); + sanitized = sanitized.Replace("vu#", "vu-", StringComparison.OrdinalIgnoreCase); + sanitized = sanitized.Replace('#', '-'); + sanitized = sanitized.Replace(' ', '-'); + + return SanitizeToken(sanitized); + } + + private static string SanitizeToken(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var trimmed = value.Trim(); + var filtered = new string(trimmed + .Select(ch => char.IsLetterOrDigit(ch) || ch is '-' or '_' ? ch : '-') + .ToArray()); + + return filtered.Trim('-').ToLowerInvariant(); + } + + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex WhitespaceRegex = new("[ \t\f\r]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex ParagraphRegex = new("<\\s*/?\\s*p[^>]*>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + private static string? ExtractSummary(CertCcNoteMetadata metadata) + { + if (metadata is null) + { + return null; + } + + var summary = string.IsNullOrWhiteSpace(metadata.Summary) ? metadata.Overview : metadata.Summary; + if (string.IsNullOrWhiteSpace(summary)) + { + return null; + } + + return HtmlToPlainText(summary); + } + + private static string HtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var normalized = html + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
    • ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
      • ", "\n", StringComparison.OrdinalIgnoreCase); + + normalized = ParagraphRegex.Replace(normalized, "\n"); + + var withoutTags = HtmlTagRegex.Replace(normalized, " "); + var decoded = WebUtility.HtmlDecode(withoutTags) ?? string.Empty; + var collapsedSpaces = WhitespaceRegex.Replace(decoded, " "); + var collapsedNewlines = Regex.Replace(collapsedSpaces, "\n{2,}", "\n", RegexOptions.Compiled); + return collapsedNewlines.Trim(); + } + + private static IEnumerable BuildAliases(CertCcNoteDto dto) + { + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase); + + var metadata = dto.Metadata ?? CertCcNoteMetadata.Empty; + + if (!string.IsNullOrWhiteSpace(metadata.VuId)) + { + aliases.Add(metadata.VuId.Trim()); + } + + if (!string.IsNullOrWhiteSpace(metadata.IdNumber)) + { + aliases.Add($"VU#{metadata.IdNumber.Trim()}"); + } + + foreach (var cve in metadata.CveIds ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + aliases.Add(cve.Trim()); + } + + foreach (var vulnerability in dto.Vulnerabilities ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(vulnerability.CveId)) + { + continue; + } + + aliases.Add(vulnerability.CveId.Trim()); + } + + return aliases.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase); + } + + private static IEnumerable BuildReferences( + CertCcNoteDto dto, + CertCcNoteMetadata metadata, + string sourceName, + DateTimeOffset recordedAt) + { + var references = new List(); + var canonicalUri = !string.IsNullOrWhiteSpace(metadata.PrimaryUrl) + ? metadata.PrimaryUrl! + : (string.IsNullOrWhiteSpace(metadata.IdNumber) + ? "https://www.kb.cert.org/vuls/" + : $"https://www.kb.cert.org/vuls/id/{metadata.IdNumber.Trim()}/"); + + var provenance = new AdvisoryProvenance(sourceName, "reference", canonicalUri, recordedAt); + + TryAddReference(references, canonicalUri, "advisory", "certcc.note", null, provenance); + + foreach (var url in metadata.PublicUrls ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.public", null, provenance); + } + + foreach (var vendor in dto.Vendors ?? Array.Empty()) + { + foreach (var url in vendor.References ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.vendor", vendor.Vendor, provenance); + } + + var statementText = vendor.Statement ?? string.Empty; + var patches = CertCcVendorStatementParser.Parse(statementText); + foreach (var patch in patches) + { + if (!string.IsNullOrWhiteSpace(patch.RawLine) && TryFindEmbeddedUrl(patch.RawLine!, out var rawUrl)) + { + TryAddReference(references, rawUrl, "reference", "certcc.vendor.statement", vendor.Vendor, provenance); + } + } + } + + foreach (var status in dto.VendorStatuses ?? Array.Empty()) + { + foreach (var url in status.References ?? Array.Empty()) + { + TryAddReference(references, url, "reference", "certcc.vendor.status", status.Vendor, provenance); + } + + if (!string.IsNullOrWhiteSpace(status.Statement) && TryFindEmbeddedUrl(status.Statement!, out var embedded)) + { + TryAddReference(references, embedded, "reference", "certcc.vendor.status", status.Vendor, provenance); + } + } + + return references + .GroupBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .Select(static group => group + .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.SourceTag ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .First()) + .OrderBy(static reference => reference.Kind ?? string.Empty, StringComparer.Ordinal) + .ThenBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase); + } + + private static void TryAddReference( + ICollection references, + string? url, + string kind, + string? sourceTag, + string? summary, + AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + var candidate = url.Trim(); + if (!Uri.TryCreate(candidate, UriKind.Absolute, out var parsed)) + { + return; + } + + if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) + { + return; + } + + var normalized = parsed.ToString(); + + try + { + references.Add(new AdvisoryReference(normalized, kind, sourceTag, summary, provenance)); + } + catch (ArgumentException) + { + // ignore invalid references + } + } + + private static bool TryFindEmbeddedUrl(string text, out string? url) + { + url = null; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + var tokens = text.Split(new[] { ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); + foreach (var token in tokens) + { + var trimmed = token.Trim().TrimEnd('.', ',', ')', ';', ']', '}'); + if (trimmed.Length == 0) + { + continue; + } + + if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var parsed)) + { + continue; + } + + if (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps) + { + continue; + } + + url = parsed.ToString(); + return true; + } + + return false; + } + + private static IEnumerable BuildAffectedPackages( + CertCcNoteDto dto, + CertCcNoteMetadata metadata, + string sourceName, + DateTimeOffset recordedAt) + { + var vendors = dto.Vendors ?? Array.Empty(); + var statuses = dto.VendorStatuses ?? Array.Empty(); + + if (vendors.Count == 0 && statuses.Count == 0) + { + return Array.Empty(); + } + + var statusLookup = statuses + .GroupBy(static status => NormalizeVendorKey(status.Vendor)) + .ToDictionary(static group => group.Key, static group => group.ToArray(), StringComparer.OrdinalIgnoreCase); + + var packages = new List(); + + foreach (var vendor in vendors.OrderBy(static v => v.Vendor, StringComparer.OrdinalIgnoreCase)) + { + var key = NormalizeVendorKey(vendor.Vendor); + var vendorStatuses = statusLookup.TryGetValue(key, out var value) + ? value + : Array.Empty(); + + if (BuildVendorPackage(vendor, vendorStatuses, sourceName, recordedAt) is { } package) + { + packages.Add(package); + } + + statusLookup.Remove(key); + } + + foreach (var remaining in statusLookup.Values) + { + if (remaining.Length == 0) + { + continue; + } + + var vendorName = remaining[0].Vendor; + var fallbackVendor = new CertCcVendorDto( + vendorName, + ContactDate: null, + StatementDate: null, + Updated: remaining + .Select(static status => status.DateUpdated) + .Where(static update => update.HasValue) + .OrderByDescending(static update => update) + .FirstOrDefault(), + Statement: remaining + .Select(static status => status.Statement) + .FirstOrDefault(static statement => !string.IsNullOrWhiteSpace(statement)), + Addendum: null, + References: remaining + .SelectMany(static status => status.References ?? Array.Empty()) + .ToArray()); + + if (BuildVendorPackage(fallbackVendor, remaining, sourceName, recordedAt) is { } package) + { + packages.Add(package); + } + } + + return packages + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static AffectedPackage? BuildVendorPackage( + CertCcVendorDto vendor, + IReadOnlyList statuses, + string sourceName, + DateTimeOffset recordedAt) + { + var vendorName = string.IsNullOrWhiteSpace(vendor.Vendor) + ? (statuses.FirstOrDefault()?.Vendor?.Trim() ?? string.Empty) + : vendor.Vendor.Trim(); + + if (vendorName.Length == 0) + { + return null; + } + + var packageProvenance = new AdvisoryProvenance(sourceName, "vendor", vendorName, recordedAt); + var rangeProvenance = new AdvisoryProvenance(sourceName, "vendor-range", vendorName, recordedAt); + + var patches = CertCcVendorStatementParser.Parse(vendor.Statement ?? string.Empty); + var normalizedVersions = BuildNormalizedVersions(vendorName, patches); + var vendorStatuses = BuildStatuses(vendorName, statuses, sourceName, recordedAt); + var primitives = BuildRangePrimitives(vendor, vendorStatuses, patches); + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: rangeProvenance, + primitives: primitives); + + return new AffectedPackage( + AffectedPackageTypes.Vendor, + vendorName, + platform: null, + versionRanges: new[] { range }, + normalizedVersions: normalizedVersions, + statuses: vendorStatuses, + provenance: new[] { packageProvenance }); + } + + private static IReadOnlyList BuildNormalizedVersions( + string vendorName, + IReadOnlyList patches) + { + if (patches.Count == 0) + { + return Array.Empty(); + } + + var rules = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var patch in patches) + { + if (string.IsNullOrWhiteSpace(patch.Version)) + { + continue; + } + + var version = patch.Version.Trim(); + if (!seen.Add($"{patch.Product}|{version}")) + { + continue; + } + + var notes = string.IsNullOrWhiteSpace(patch.Product) + ? vendorName + : $"{vendorName}::{patch.Product.Trim()}"; + + rules.Add(new NormalizedVersionRule( + VendorNormalizedVersionScheme, + NormalizedVersionRuleTypes.Exact, + value: version, + notes: notes)); + } + + return rules.Count == 0 ? Array.Empty() : rules; + } + + private static IReadOnlyList BuildStatuses( + string vendorName, + IReadOnlyList statuses, + string sourceName, + DateTimeOffset recordedAt) + { + if (statuses.Count == 0) + { + return Array.Empty(); + } + + var result = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var status in statuses) + { + if (!AffectedPackageStatusCatalog.TryNormalize(status.Status, out var normalized)) + { + continue; + } + + var cve = status.CveId?.Trim() ?? string.Empty; + var key = string.IsNullOrWhiteSpace(cve) + ? normalized + : $"{normalized}|{cve}"; + + if (!seen.Add(key)) + { + continue; + } + + var provenance = new AdvisoryProvenance( + sourceName, + "vendor-status", + string.IsNullOrWhiteSpace(cve) ? vendorName : $"{vendorName}:{cve}", + recordedAt); + + result.Add(new AffectedPackageStatus(normalized, provenance)); + } + + return result + .OrderBy(static status => status.Status, StringComparer.Ordinal) + .ThenBy(static status => status.Provenance.Value ?? string.Empty, StringComparer.Ordinal) + .ToArray(); + } + + private static RangePrimitives? BuildRangePrimitives( + CertCcVendorDto vendor, + IReadOnlyList statuses, + IReadOnlyList patches) + { + var extensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + AddVendorExtension(extensions, "certcc.vendor.name", vendor.Vendor); + AddVendorExtension(extensions, "certcc.vendor.statement.raw", HtmlToPlainText(vendor.Statement ?? string.Empty), 2048); + AddVendorExtension(extensions, "certcc.vendor.addendum", HtmlToPlainText(vendor.Addendum ?? string.Empty), 1024); + AddVendorExtension(extensions, "certcc.vendor.contactDate", FormatDate(vendor.ContactDate)); + AddVendorExtension(extensions, "certcc.vendor.statementDate", FormatDate(vendor.StatementDate)); + AddVendorExtension(extensions, "certcc.vendor.updated", FormatDate(vendor.Updated)); + + if (vendor.References is { Count: > 0 }) + { + AddVendorExtension(extensions, "certcc.vendor.references", string.Join(" ", vendor.References)); + } + + if (statuses.Count > 0) + { + var serialized = string.Join(";", statuses + .Select(static status => status.Provenance.Value is { Length: > 0 } + ? $"{status.Provenance.Value.Split(':').Last()}={status.Status}" + : status.Status)); + + AddVendorExtension(extensions, "certcc.vendor.statuses", serialized); + } + + if (patches.Count > 0) + { + var serialized = string.Join(";", patches.Select(static patch => + { + var product = string.IsNullOrWhiteSpace(patch.Product) ? "unknown" : patch.Product.Trim(); + return $"{product}={patch.Version.Trim()}"; + })); + + AddVendorExtension(extensions, "certcc.vendor.patches", serialized, 2048); + } + + return extensions.Count == 0 + ? null + : new RangePrimitives(null, null, null, extensions); + } + + private static void AddVendorExtension(IDictionary extensions, string key, string? value, int maxLength = 512) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var trimmed = value.Trim(); + if (trimmed.Length > maxLength) + { + trimmed = trimmed[..maxLength].Trim(); + } + + if (trimmed.Length == 0) + { + return; + } + + extensions[key] = trimmed; + } + + private static string? FormatDate(DateTimeOffset? value) + => value?.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + + private static string NormalizeVendorKey(string? value) + => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToLowerInvariant(); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteDto.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteDto.cs index 1e8913cb..45bd47df 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteDto.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteDto.cs @@ -1,97 +1,97 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal sealed record CertCcNoteDto( - CertCcNoteMetadata Metadata, - IReadOnlyList Vendors, - IReadOnlyList VendorStatuses, - IReadOnlyList Vulnerabilities) -{ - public static CertCcNoteDto Empty { get; } = new( - CertCcNoteMetadata.Empty, - Array.Empty(), - Array.Empty(), - Array.Empty()); -} - -internal sealed record CertCcNoteMetadata( - string? VuId, - string IdNumber, - string Title, - string? Overview, - string? Summary, - DateTimeOffset? Published, - DateTimeOffset? Updated, - DateTimeOffset? Created, - int? Revision, - IReadOnlyList CveIds, - IReadOnlyList PublicUrls, - string? PrimaryUrl) -{ - public static CertCcNoteMetadata Empty { get; } = new( - VuId: null, - IdNumber: string.Empty, - Title: string.Empty, - Overview: null, - Summary: null, - Published: null, - Updated: null, - Created: null, - Revision: null, - CveIds: Array.Empty(), - PublicUrls: Array.Empty(), - PrimaryUrl: null); -} - -internal sealed record CertCcVendorDto( - string Vendor, - DateTimeOffset? ContactDate, - DateTimeOffset? StatementDate, - DateTimeOffset? Updated, - string? Statement, - string? Addendum, - IReadOnlyList References) -{ - public static CertCcVendorDto Empty { get; } = new( - Vendor: string.Empty, - ContactDate: null, - StatementDate: null, - Updated: null, - Statement: null, - Addendum: null, - References: Array.Empty()); -} - -internal sealed record CertCcVendorStatusDto( - string Vendor, - string CveId, - string Status, - string? Statement, - IReadOnlyList References, - DateTimeOffset? DateAdded, - DateTimeOffset? DateUpdated) -{ - public static CertCcVendorStatusDto Empty { get; } = new( - Vendor: string.Empty, - CveId: string.Empty, - Status: string.Empty, - Statement: null, - References: Array.Empty(), - DateAdded: null, - DateUpdated: null); -} - -internal sealed record CertCcVulnerabilityDto( - string CveId, - string? Description, - DateTimeOffset? DateAdded, - DateTimeOffset? DateUpdated) -{ - public static CertCcVulnerabilityDto Empty { get; } = new( - CveId: string.Empty, - Description: null, - DateAdded: null, - DateUpdated: null); -} +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal sealed record CertCcNoteDto( + CertCcNoteMetadata Metadata, + IReadOnlyList Vendors, + IReadOnlyList VendorStatuses, + IReadOnlyList Vulnerabilities) +{ + public static CertCcNoteDto Empty { get; } = new( + CertCcNoteMetadata.Empty, + Array.Empty(), + Array.Empty(), + Array.Empty()); +} + +internal sealed record CertCcNoteMetadata( + string? VuId, + string IdNumber, + string Title, + string? Overview, + string? Summary, + DateTimeOffset? Published, + DateTimeOffset? Updated, + DateTimeOffset? Created, + int? Revision, + IReadOnlyList CveIds, + IReadOnlyList PublicUrls, + string? PrimaryUrl) +{ + public static CertCcNoteMetadata Empty { get; } = new( + VuId: null, + IdNumber: string.Empty, + Title: string.Empty, + Overview: null, + Summary: null, + Published: null, + Updated: null, + Created: null, + Revision: null, + CveIds: Array.Empty(), + PublicUrls: Array.Empty(), + PrimaryUrl: null); +} + +internal sealed record CertCcVendorDto( + string Vendor, + DateTimeOffset? ContactDate, + DateTimeOffset? StatementDate, + DateTimeOffset? Updated, + string? Statement, + string? Addendum, + IReadOnlyList References) +{ + public static CertCcVendorDto Empty { get; } = new( + Vendor: string.Empty, + ContactDate: null, + StatementDate: null, + Updated: null, + Statement: null, + Addendum: null, + References: Array.Empty()); +} + +internal sealed record CertCcVendorStatusDto( + string Vendor, + string CveId, + string Status, + string? Statement, + IReadOnlyList References, + DateTimeOffset? DateAdded, + DateTimeOffset? DateUpdated) +{ + public static CertCcVendorStatusDto Empty { get; } = new( + Vendor: string.Empty, + CveId: string.Empty, + Status: string.Empty, + Statement: null, + References: Array.Empty(), + DateAdded: null, + DateUpdated: null); +} + +internal sealed record CertCcVulnerabilityDto( + string CveId, + string? Description, + DateTimeOffset? DateAdded, + DateTimeOffset? DateUpdated) +{ + public static CertCcVulnerabilityDto Empty { get; } = new( + CveId: string.Empty, + Description: null, + DateAdded: null, + DateUpdated: null); +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs index 41d5212e..5a810bff 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcNoteParser.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcNoteParser.cs @@ -1,539 +1,539 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Text; -using System.Text.Json; -using System.Text.RegularExpressions; -using Markdig; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Source.Common.Url; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal static class CertCcNoteParser -{ - private static readonly MarkdownPipeline MarkdownPipeline = new MarkdownPipelineBuilder() - .UseAdvancedExtensions() - .UseSoftlineBreakAsHardlineBreak() - .DisableHtml() - .Build(); - - private static readonly HtmlContentSanitizer HtmlSanitizer = new(); - private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); - - public static CertCcNoteDto Parse( - ReadOnlySpan noteJson, - ReadOnlySpan vendorsJson, - ReadOnlySpan vulnerabilitiesJson, - ReadOnlySpan vendorStatusesJson) - { - using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); - var (metadata, detailUri) = ParseNoteMetadata(noteDocument.RootElement); - - using var vendorsDocument = JsonDocument.Parse(vendorsJson.ToArray()); - var vendors = ParseVendors(vendorsDocument.RootElement, detailUri); - - using var vulnerabilitiesDocument = JsonDocument.Parse(vulnerabilitiesJson.ToArray()); - var vulnerabilities = ParseVulnerabilities(vulnerabilitiesDocument.RootElement); - - using var statusesDocument = JsonDocument.Parse(vendorStatusesJson.ToArray()); - var statuses = ParseVendorStatuses(statusesDocument.RootElement); - - return new CertCcNoteDto(metadata, vendors, statuses, vulnerabilities); - } - - public static CertCcNoteDto ParseNote(ReadOnlySpan noteJson) - { - using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); - var (metadata, _) = ParseNoteMetadata(noteDocument.RootElement); - return new CertCcNoteDto(metadata, Array.Empty(), Array.Empty(), Array.Empty()); - } - - private static (CertCcNoteMetadata Metadata, Uri DetailUri) ParseNoteMetadata(JsonElement root) - { - if (root.ValueKind != JsonValueKind.Object) - { - throw new JsonException("CERT/CC note payload must be a JSON object."); - } - - var vuId = GetString(root, "vuid"); - var idNumber = GetString(root, "idnumber") ?? throw new JsonException("CERT/CC note missing idnumber."); - var title = GetString(root, "name") ?? throw new JsonException("CERT/CC note missing name."); - var detailUri = BuildDetailUri(idNumber); - - var overview = NormalizeMarkdownToPlainText(root, "overview", detailUri); - var summary = NormalizeMarkdownToPlainText(root, "clean_desc", detailUri); - if (string.IsNullOrWhiteSpace(summary)) - { - summary = NormalizeMarkdownToPlainText(root, "impact", detailUri); - } - - var published = ParseDate(root, "publicdate") ?? ParseDate(root, "datefirstpublished"); - var updated = ParseDate(root, "dateupdated"); - var created = ParseDate(root, "datecreated"); - var revision = ParseInt(root, "revision"); - - var cveIds = ExtractCveIds(root, "cveids"); - var references = ExtractReferenceList(root, "public", detailUri); - - var metadata = new CertCcNoteMetadata( - VuId: string.IsNullOrWhiteSpace(vuId) ? null : vuId.Trim(), - IdNumber: idNumber.Trim(), - Title: title.Trim(), - Overview: overview, - Summary: summary, - Published: published?.ToUniversalTime(), - Updated: updated?.ToUniversalTime(), - Created: created?.ToUniversalTime(), - Revision: revision, - CveIds: cveIds, - PublicUrls: references, - PrimaryUrl: detailUri.ToString()); - - return (metadata, detailUri); - } - - private static IReadOnlyList ParseVendors(JsonElement root, Uri baseUri) - { - if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var parsed = new List(root.GetArrayLength()); - foreach (var element in root.EnumerateArray()) - { - if (element.ValueKind != JsonValueKind.Object) - { - continue; - } - - var vendor = GetString(element, "vendor"); - if (string.IsNullOrWhiteSpace(vendor)) - { - continue; - } - - var statement = NormalizeFreeformText(GetString(element, "statement")); - var addendum = NormalizeFreeformText(GetString(element, "addendum")); - var references = ExtractReferenceStringList(GetString(element, "references"), baseUri); - - parsed.Add(new CertCcVendorDto( - vendor.Trim(), - ContactDate: ParseDate(element, "contact_date"), - StatementDate: ParseDate(element, "statement_date"), - Updated: ParseDate(element, "dateupdated"), - Statement: statement, - Addendum: addendum, - References: references)); - } - - if (parsed.Count == 0) - { - return Array.Empty(); - } - - return parsed - .OrderBy(static vendor => vendor.Vendor, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList ParseVulnerabilities(JsonElement root) - { - if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var parsed = new List(root.GetArrayLength()); - foreach (var element in root.EnumerateArray()) - { - if (element.ValueKind != JsonValueKind.Object) - { - continue; - } - - var cve = GetString(element, "cve"); - if (string.IsNullOrWhiteSpace(cve)) - { - continue; - } - - parsed.Add(new CertCcVulnerabilityDto( - NormalizeCve(cve), - Description: NormalizeFreeformText(GetString(element, "description")), - DateAdded: ParseDate(element, "date_added"), - DateUpdated: ParseDate(element, "dateupdated"))); - } - - if (parsed.Count == 0) - { - return Array.Empty(); - } - - return parsed - .OrderBy(static vuln => vuln.CveId, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList ParseVendorStatuses(JsonElement root) - { - if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var parsed = new List(root.GetArrayLength()); - foreach (var element in root.EnumerateArray()) - { - if (element.ValueKind != JsonValueKind.Object) - { - continue; - } - - var vendor = GetString(element, "vendor"); - var cve = GetString(element, "vul"); - var status = GetString(element, "status"); - if (string.IsNullOrWhiteSpace(vendor) || string.IsNullOrWhiteSpace(cve) || string.IsNullOrWhiteSpace(status)) - { - continue; - } - - var references = ExtractReferenceStringList(GetString(element, "references"), baseUri: null); - parsed.Add(new CertCcVendorStatusDto( - vendor.Trim(), - NormalizeCve(cve), - status.Trim(), - NormalizeFreeformText(GetString(element, "statement")), - references, - DateAdded: ParseDate(element, "date_added"), - DateUpdated: ParseDate(element, "dateupdated"))); - } - - if (parsed.Count == 0) - { - return Array.Empty(); - } - - return parsed - .OrderBy(static entry => entry.CveId, StringComparer.OrdinalIgnoreCase) - .ThenBy(static entry => entry.Vendor, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static string? NormalizeMarkdownToPlainText(JsonElement element, string propertyName, Uri baseUri) - => NormalizeMarkdownToPlainText(GetString(element, propertyName), baseUri); - - private static string? NormalizeMarkdownToPlainText(string? markdown, Uri baseUri) - { - if (string.IsNullOrWhiteSpace(markdown)) - { - return null; - } - - var normalized = NormalizeLineEndings(markdown.Trim()); - if (normalized.Length == 0) - { - return null; - } - - var html = Markdig.Markdown.ToHtml(normalized, MarkdownPipeline); - if (string.IsNullOrWhiteSpace(html)) - { - return null; - } - - var sanitized = HtmlSanitizer.Sanitize(html, baseUri); - if (string.IsNullOrWhiteSpace(sanitized)) - { - return null; - } - - var plain = ConvertHtmlToPlainText(sanitized); - return string.IsNullOrWhiteSpace(plain) ? null : plain; - } - - private static string? NormalizeFreeformText(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - var normalized = NormalizeLineEndings(value).Trim(); - if (normalized.Length == 0) - { - return null; - } - - var lines = normalized - .Split('\n') - .Select(static line => line.TrimEnd()) - .ToArray(); - - return string.Join('\n', lines).Trim(); - } - - private static string ConvertHtmlToPlainText(string html) - { - if (string.IsNullOrWhiteSpace(html)) - { - return string.Empty; - } - - var decoded = WebUtility.HtmlDecode(html); - decoded = decoded - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) - .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase); - - decoded = Regex.Replace(decoded, "

      ", "\n\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - decoded = Regex.Replace(decoded, "
    • ", "- ", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - decoded = Regex.Replace(decoded, "
      • ", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - decoded = Regex.Replace(decoded, "", " \t", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); - - decoded = HtmlTagRegex.Replace(decoded, string.Empty); - decoded = NormalizeLineEndings(decoded); - - var lines = decoded - .Split('\n', StringSplitOptions.RemoveEmptyEntries) - .Select(static line => line.Trim()) - .ToArray(); - - return string.Join('\n', lines).Trim(); - } - - private static IReadOnlyList ExtractReferenceList(JsonElement element, string propertyName, Uri baseUri) - { - if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var references = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var candidate in raw.EnumerateArray()) - { - if (candidate.ValueKind != JsonValueKind.String) - { - continue; - } - - var text = candidate.GetString(); - if (UrlNormalizer.TryNormalize(text, baseUri, out var normalized, stripFragment: true, forceHttps: false) && normalized is not null) - { - references.Add(normalized.ToString()); - } - } - - if (references.Count == 0) - { - return Array.Empty(); - } - - return references - .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList ExtractReferenceStringList(string? value, Uri? baseUri) - { - if (string.IsNullOrWhiteSpace(value)) - { - return Array.Empty(); - } - - var buffer = ArrayPool.Shared.Rent(16); - try - { - var count = 0; - var span = value.AsSpan(); - var start = 0; - - for (var index = 0; index < span.Length; index++) - { - var ch = span[index]; - if (ch == '\r' || ch == '\n') - { - if (index > start) - { - AppendSegment(span, start, index - start, baseUri, buffer, ref count); - } - - if (ch == '\r' && index + 1 < span.Length && span[index + 1] == '\n') - { - index++; - } - - start = index + 1; - } - } - - if (start < span.Length) - { - AppendSegment(span, start, span.Length - start, baseUri, buffer, ref count); - } - - if (count == 0) - { - return Array.Empty(); - } - - return buffer.AsSpan(0, count) - .ToArray() - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - finally - { - ArrayPool.Shared.Return(buffer, clearArray: true); - } - } - - private static void AppendSegment(ReadOnlySpan span, int start, int length, Uri? baseUri, string[] buffer, ref int count) - { - var segment = span.Slice(start, length).ToString().Trim(); - if (segment.Length == 0) - { - return; - } - - if (!UrlNormalizer.TryNormalize(segment, baseUri, out var normalized, stripFragment: true, forceHttps: false) || normalized is null) - { - return; - } - - if (count >= buffer.Length) - { - return; - } - - buffer[count++] = normalized.ToString(); - } - - private static IReadOnlyList ExtractCveIds(JsonElement element, string propertyName) - { - if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var values = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var entry in raw.EnumerateArray()) - { - if (entry.ValueKind != JsonValueKind.String) - { - continue; - } - - var text = entry.GetString(); - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - values.Add(NormalizeCve(text)); - } - - if (values.Count == 0) - { - return Array.Empty(); - } - - return values - .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static string NormalizeCve(string value) - { - var trimmed = value.Trim(); - if (!trimmed.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) - { - trimmed = $"CVE-{trimmed}"; - } - - var builder = new StringBuilder(trimmed.Length); - foreach (var ch in trimmed) - { - builder.Append(char.ToUpperInvariant(ch)); - } - - return builder.ToString(); - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind switch - { - JsonValueKind.String => property.GetString(), - JsonValueKind.Number => property.ToString(), - _ => null, - }; - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - var text = GetString(element, propertyName); - if (string.IsNullOrWhiteSpace(text)) - { - return null; - } - - return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) - ? parsed.ToUniversalTime() - : null; - } - - private static int? ParseInt(JsonElement element, string propertyName) - { - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - if (property.ValueKind == JsonValueKind.Number && property.TryGetInt32(out var value)) - { - return value; - } - - var text = GetString(element, propertyName); - if (string.IsNullOrWhiteSpace(text)) - { - return null; - } - - return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : (int?)null; - } - - private static Uri BuildDetailUri(string idNumber) - { - var sanitized = idNumber.Trim(); - return new Uri($"https://www.kb.cert.org/vuls/id/{sanitized}", UriKind.Absolute); - } - - private static string NormalizeLineEndings(string value) - { - if (value.IndexOf('\r') < 0) - { - return value; - } - - return value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); - } -} +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using Markdig; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Connector.Common.Url; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal static class CertCcNoteParser +{ + private static readonly MarkdownPipeline MarkdownPipeline = new MarkdownPipelineBuilder() + .UseAdvancedExtensions() + .UseSoftlineBreakAsHardlineBreak() + .DisableHtml() + .Build(); + + private static readonly HtmlContentSanitizer HtmlSanitizer = new(); + private static readonly Regex HtmlTagRegex = new("<[^>]+>", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + public static CertCcNoteDto Parse( + ReadOnlySpan noteJson, + ReadOnlySpan vendorsJson, + ReadOnlySpan vulnerabilitiesJson, + ReadOnlySpan vendorStatusesJson) + { + using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); + var (metadata, detailUri) = ParseNoteMetadata(noteDocument.RootElement); + + using var vendorsDocument = JsonDocument.Parse(vendorsJson.ToArray()); + var vendors = ParseVendors(vendorsDocument.RootElement, detailUri); + + using var vulnerabilitiesDocument = JsonDocument.Parse(vulnerabilitiesJson.ToArray()); + var vulnerabilities = ParseVulnerabilities(vulnerabilitiesDocument.RootElement); + + using var statusesDocument = JsonDocument.Parse(vendorStatusesJson.ToArray()); + var statuses = ParseVendorStatuses(statusesDocument.RootElement); + + return new CertCcNoteDto(metadata, vendors, statuses, vulnerabilities); + } + + public static CertCcNoteDto ParseNote(ReadOnlySpan noteJson) + { + using var noteDocument = JsonDocument.Parse(noteJson.ToArray()); + var (metadata, _) = ParseNoteMetadata(noteDocument.RootElement); + return new CertCcNoteDto(metadata, Array.Empty(), Array.Empty(), Array.Empty()); + } + + private static (CertCcNoteMetadata Metadata, Uri DetailUri) ParseNoteMetadata(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Object) + { + throw new JsonException("CERT/CC note payload must be a JSON object."); + } + + var vuId = GetString(root, "vuid"); + var idNumber = GetString(root, "idnumber") ?? throw new JsonException("CERT/CC note missing idnumber."); + var title = GetString(root, "name") ?? throw new JsonException("CERT/CC note missing name."); + var detailUri = BuildDetailUri(idNumber); + + var overview = NormalizeMarkdownToPlainText(root, "overview", detailUri); + var summary = NormalizeMarkdownToPlainText(root, "clean_desc", detailUri); + if (string.IsNullOrWhiteSpace(summary)) + { + summary = NormalizeMarkdownToPlainText(root, "impact", detailUri); + } + + var published = ParseDate(root, "publicdate") ?? ParseDate(root, "datefirstpublished"); + var updated = ParseDate(root, "dateupdated"); + var created = ParseDate(root, "datecreated"); + var revision = ParseInt(root, "revision"); + + var cveIds = ExtractCveIds(root, "cveids"); + var references = ExtractReferenceList(root, "public", detailUri); + + var metadata = new CertCcNoteMetadata( + VuId: string.IsNullOrWhiteSpace(vuId) ? null : vuId.Trim(), + IdNumber: idNumber.Trim(), + Title: title.Trim(), + Overview: overview, + Summary: summary, + Published: published?.ToUniversalTime(), + Updated: updated?.ToUniversalTime(), + Created: created?.ToUniversalTime(), + Revision: revision, + CveIds: cveIds, + PublicUrls: references, + PrimaryUrl: detailUri.ToString()); + + return (metadata, detailUri); + } + + private static IReadOnlyList ParseVendors(JsonElement root, Uri baseUri) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vendor = GetString(element, "vendor"); + if (string.IsNullOrWhiteSpace(vendor)) + { + continue; + } + + var statement = NormalizeFreeformText(GetString(element, "statement")); + var addendum = NormalizeFreeformText(GetString(element, "addendum")); + var references = ExtractReferenceStringList(GetString(element, "references"), baseUri); + + parsed.Add(new CertCcVendorDto( + vendor.Trim(), + ContactDate: ParseDate(element, "contact_date"), + StatementDate: ParseDate(element, "statement_date"), + Updated: ParseDate(element, "dateupdated"), + Statement: statement, + Addendum: addendum, + References: references)); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static vendor => vendor.Vendor, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ParseVulnerabilities(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var cve = GetString(element, "cve"); + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + parsed.Add(new CertCcVulnerabilityDto( + NormalizeCve(cve), + Description: NormalizeFreeformText(GetString(element, "description")), + DateAdded: ParseDate(element, "date_added"), + DateUpdated: ParseDate(element, "dateupdated"))); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static vuln => vuln.CveId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ParseVendorStatuses(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Array || root.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var parsed = new List(root.GetArrayLength()); + foreach (var element in root.EnumerateArray()) + { + if (element.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vendor = GetString(element, "vendor"); + var cve = GetString(element, "vul"); + var status = GetString(element, "status"); + if (string.IsNullOrWhiteSpace(vendor) || string.IsNullOrWhiteSpace(cve) || string.IsNullOrWhiteSpace(status)) + { + continue; + } + + var references = ExtractReferenceStringList(GetString(element, "references"), baseUri: null); + parsed.Add(new CertCcVendorStatusDto( + vendor.Trim(), + NormalizeCve(cve), + status.Trim(), + NormalizeFreeformText(GetString(element, "statement")), + references, + DateAdded: ParseDate(element, "date_added"), + DateUpdated: ParseDate(element, "dateupdated"))); + } + + if (parsed.Count == 0) + { + return Array.Empty(); + } + + return parsed + .OrderBy(static entry => entry.CveId, StringComparer.OrdinalIgnoreCase) + .ThenBy(static entry => entry.Vendor, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string? NormalizeMarkdownToPlainText(JsonElement element, string propertyName, Uri baseUri) + => NormalizeMarkdownToPlainText(GetString(element, propertyName), baseUri); + + private static string? NormalizeMarkdownToPlainText(string? markdown, Uri baseUri) + { + if (string.IsNullOrWhiteSpace(markdown)) + { + return null; + } + + var normalized = NormalizeLineEndings(markdown.Trim()); + if (normalized.Length == 0) + { + return null; + } + + var html = Markdig.Markdown.ToHtml(normalized, MarkdownPipeline); + if (string.IsNullOrWhiteSpace(html)) + { + return null; + } + + var sanitized = HtmlSanitizer.Sanitize(html, baseUri); + if (string.IsNullOrWhiteSpace(sanitized)) + { + return null; + } + + var plain = ConvertHtmlToPlainText(sanitized); + return string.IsNullOrWhiteSpace(plain) ? null : plain; + } + + private static string? NormalizeFreeformText(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var normalized = NormalizeLineEndings(value).Trim(); + if (normalized.Length == 0) + { + return null; + } + + var lines = normalized + .Split('\n') + .Select(static line => line.TrimEnd()) + .ToArray(); + + return string.Join('\n', lines).Trim(); + } + + private static string ConvertHtmlToPlainText(string html) + { + if (string.IsNullOrWhiteSpace(html)) + { + return string.Empty; + } + + var decoded = WebUtility.HtmlDecode(html); + decoded = decoded + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase) + .Replace("
      ", "\n", StringComparison.OrdinalIgnoreCase); + + decoded = Regex.Replace(decoded, "

      ", "\n\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "
    • ", "- ", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "
      • ", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", "\n", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + decoded = Regex.Replace(decoded, "", " \t", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + decoded = HtmlTagRegex.Replace(decoded, string.Empty); + decoded = NormalizeLineEndings(decoded); + + var lines = decoded + .Split('\n', StringSplitOptions.RemoveEmptyEntries) + .Select(static line => line.Trim()) + .ToArray(); + + return string.Join('\n', lines).Trim(); + } + + private static IReadOnlyList ExtractReferenceList(JsonElement element, string propertyName, Uri baseUri) + { + if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var candidate in raw.EnumerateArray()) + { + if (candidate.ValueKind != JsonValueKind.String) + { + continue; + } + + var text = candidate.GetString(); + if (UrlNormalizer.TryNormalize(text, baseUri, out var normalized, stripFragment: true, forceHttps: false) && normalized is not null) + { + references.Add(normalized.ToString()); + } + } + + if (references.Count == 0) + { + return Array.Empty(); + } + + return references + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList ExtractReferenceStringList(string? value, Uri? baseUri) + { + if (string.IsNullOrWhiteSpace(value)) + { + return Array.Empty(); + } + + var buffer = ArrayPool.Shared.Rent(16); + try + { + var count = 0; + var span = value.AsSpan(); + var start = 0; + + for (var index = 0; index < span.Length; index++) + { + var ch = span[index]; + if (ch == '\r' || ch == '\n') + { + if (index > start) + { + AppendSegment(span, start, index - start, baseUri, buffer, ref count); + } + + if (ch == '\r' && index + 1 < span.Length && span[index + 1] == '\n') + { + index++; + } + + start = index + 1; + } + } + + if (start < span.Length) + { + AppendSegment(span, start, span.Length - start, baseUri, buffer, ref count); + } + + if (count == 0) + { + return Array.Empty(); + } + + return buffer.AsSpan(0, count) + .ToArray() + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + finally + { + ArrayPool.Shared.Return(buffer, clearArray: true); + } + } + + private static void AppendSegment(ReadOnlySpan span, int start, int length, Uri? baseUri, string[] buffer, ref int count) + { + var segment = span.Slice(start, length).ToString().Trim(); + if (segment.Length == 0) + { + return; + } + + if (!UrlNormalizer.TryNormalize(segment, baseUri, out var normalized, stripFragment: true, forceHttps: false) || normalized is null) + { + return; + } + + if (count >= buffer.Length) + { + return; + } + + buffer[count++] = normalized.ToString(); + } + + private static IReadOnlyList ExtractCveIds(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var raw) || raw.ValueKind != JsonValueKind.Array || raw.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var values = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in raw.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.String) + { + continue; + } + + var text = entry.GetString(); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + values.Add(NormalizeCve(text)); + } + + if (values.Count == 0) + { + return Array.Empty(); + } + + return values + .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static string NormalizeCve(string value) + { + var trimmed = value.Trim(); + if (!trimmed.StartsWith("CVE-", StringComparison.OrdinalIgnoreCase)) + { + trimmed = $"CVE-{trimmed}"; + } + + var builder = new StringBuilder(trimmed.Length); + foreach (var ch in trimmed) + { + builder.Append(char.ToUpperInvariant(ch)); + } + + return builder.ToString(); + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number => property.ToString(), + _ => null, + }; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var text = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static int? ParseInt(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + if (property.ValueKind == JsonValueKind.Number && property.TryGetInt32(out var value)) + { + return value; + } + + var text = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : (int?)null; + } + + private static Uri BuildDetailUri(string idNumber) + { + var sanitized = idNumber.Trim(); + return new Uri($"https://www.kb.cert.org/vuls/id/{sanitized}", UriKind.Absolute); + } + + private static string NormalizeLineEndings(string value) + { + if (value.IndexOf('\r') < 0) + { + return value; + } + + return value.Replace("\r\n", "\n", StringComparison.Ordinal).Replace('\r', '\n'); + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryParser.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryParser.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryParser.cs index 00510ac6..aa49b60f 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryParser.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryParser.cs @@ -1,108 +1,108 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text.Json; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal static class CertCcSummaryParser -{ - public static IReadOnlyList ParseNotes(byte[] payload) - { - if (payload is null || payload.Length == 0) - { - return Array.Empty(); - } - - using var document = JsonDocument.Parse(payload, new JsonDocumentOptions - { - AllowTrailingCommas = true, - CommentHandling = JsonCommentHandling.Skip, - }); - - var notesElement = document.RootElement.ValueKind switch - { - JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes, - JsonValueKind.Array => document.RootElement, - JsonValueKind.Null or JsonValueKind.Undefined => default, - _ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."), - }; - - if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0) - { - return Array.Empty(); - } - - var results = new List(notesElement.GetArrayLength()); - var seen = new HashSet(StringComparer.Ordinal); - - foreach (var element in notesElement.EnumerateArray()) - { - var token = ExtractToken(element); - if (string.IsNullOrWhiteSpace(token)) - { - continue; - } - - var normalized = token.Trim(); - var dedupKey = CreateDedupKey(normalized); - if (seen.Add(dedupKey)) - { - results.Add(normalized); - } - } - - return results.Count == 0 ? Array.Empty() : results; - } - - private static string CreateDedupKey(string token) - { - var digits = string.Concat(token.Where(char.IsDigit)); - return digits.Length > 0 - ? digits - : token.Trim().ToUpperInvariant(); - } - - private static string? ExtractToken(JsonElement element) - { - return element.ValueKind switch - { - JsonValueKind.String => element.GetString(), - JsonValueKind.Number => element.TryGetInt64(out var number) - ? number.ToString(CultureInfo.InvariantCulture) - : element.GetRawText(), - JsonValueKind.Object => ExtractFromObject(element), - _ => null, - }; - } - - private static string? ExtractFromObject(JsonElement element) - { - foreach (var propertyName in PropertyCandidates) - { - if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) - { - var value = property.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - return value; - } - } - } - - return null; - } - - private static readonly string[] PropertyCandidates = - { - "note", - "notes", - "id", - "idnumber", - "noteId", - "vu", - "vuid", - "vuId", - }; -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal static class CertCcSummaryParser +{ + public static IReadOnlyList ParseNotes(byte[] payload) + { + if (payload is null || payload.Length == 0) + { + return Array.Empty(); + } + + using var document = JsonDocument.Parse(payload, new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + }); + + var notesElement = document.RootElement.ValueKind switch + { + JsonValueKind.Object when document.RootElement.TryGetProperty("notes", out var notes) => notes, + JsonValueKind.Array => document.RootElement, + JsonValueKind.Null or JsonValueKind.Undefined => default, + _ => throw new JsonException("CERT/CC summary payload must contain a 'notes' array."), + }; + + if (notesElement.ValueKind != JsonValueKind.Array || notesElement.GetArrayLength() == 0) + { + return Array.Empty(); + } + + var results = new List(notesElement.GetArrayLength()); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var element in notesElement.EnumerateArray()) + { + var token = ExtractToken(element); + if (string.IsNullOrWhiteSpace(token)) + { + continue; + } + + var normalized = token.Trim(); + var dedupKey = CreateDedupKey(normalized); + if (seen.Add(dedupKey)) + { + results.Add(normalized); + } + } + + return results.Count == 0 ? Array.Empty() : results; + } + + private static string CreateDedupKey(string token) + { + var digits = string.Concat(token.Where(char.IsDigit)); + return digits.Length > 0 + ? digits + : token.Trim().ToUpperInvariant(); + } + + private static string? ExtractToken(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString(), + JsonValueKind.Number => element.TryGetInt64(out var number) + ? number.ToString(CultureInfo.InvariantCulture) + : element.GetRawText(), + JsonValueKind.Object => ExtractFromObject(element), + _ => null, + }; + } + + private static string? ExtractFromObject(JsonElement element) + { + foreach (var propertyName in PropertyCandidates) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + var value = property.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + } + + return null; + } + + private static readonly string[] PropertyCandidates = + { + "note", + "notes", + "id", + "idnumber", + "noteId", + "vu", + "vuid", + "vuId", + }; +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlan.cs similarity index 74% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlan.cs index affaff82..0b458c8a 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlan.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlan.cs @@ -1,22 +1,22 @@ -using System; -using System.Collections.Generic; -using StellaOps.Feedser.Source.Common.Cursors; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -public sealed record CertCcSummaryPlan( - TimeWindow Window, - IReadOnlyList Requests, - TimeWindowCursorState NextState); - -public enum CertCcSummaryScope -{ - Monthly, - Yearly, -} - -public sealed record CertCcSummaryRequest( - Uri Uri, - CertCcSummaryScope Scope, - int Year, - int? Month); +using System; +using System.Collections.Generic; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +public sealed record CertCcSummaryPlan( + TimeWindow Window, + IReadOnlyList Requests, + TimeWindowCursorState NextState); + +public enum CertCcSummaryScope +{ + Monthly, + Yearly, +} + +public sealed record CertCcSummaryRequest( + Uri Uri, + CertCcSummaryScope Scope, + int Year, + int? Month); diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlanner.cs similarity index 91% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlanner.cs index c7ba920e..9c8aa496 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcSummaryPlanner.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcSummaryPlanner.cs @@ -1,96 +1,96 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertCc.Configuration; -using StellaOps.Feedser.Source.Common.Cursors; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -/// -/// Computes which CERT/CC summary endpoints should be fetched for the next export window. -/// -public sealed class CertCcSummaryPlanner -{ - private readonly CertCcOptions _options; - private readonly TimeProvider _timeProvider; - - public CertCcSummaryPlanner( - IOptions options, - TimeProvider? timeProvider = null) - { - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state) - { - var now = _timeProvider.GetUtcNow(); - var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow); - var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window); - - var months = EnumerateYearMonths(window.Start, window.End) - .Take(_options.MaxMonthlySummaries) - .ToArray(); - - if (months.Length == 0) - { - return new CertCcSummaryPlan(window, Array.Empty(), nextState); - } - - var requests = new List(months.Length * 2); - foreach (var month in months) - { - requests.Add(new CertCcSummaryRequest( - BuildMonthlyUri(month.Year, month.Month), - CertCcSummaryScope.Monthly, - month.Year, - month.Month)); - } - - foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year)) - { - requests.Add(new CertCcSummaryRequest( - BuildYearlyUri(year), - CertCcSummaryScope.Yearly, - year, - Month: null)); - } - - return new CertCcSummaryPlan(window, requests, nextState); - } - - private Uri BuildMonthlyUri(int year, int month) - { - var path = $"{year:D4}/{month:D2}/summary/"; - return new Uri(_options.BaseApiUri, path); - } - - private Uri BuildYearlyUri(int year) - { - var path = $"{year:D4}/summary/"; - return new Uri(_options.BaseApiUri, path); - } - - private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end) - { - if (end <= start) - { - yield break; - } - - var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc); - var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc); - if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero) - { - limit = limit.AddMonths(1); - } - - while (cursor < limit) - { - yield return (cursor.Year, cursor.Month); - cursor = cursor.AddMonths(1); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.CertCc.Configuration; +using StellaOps.Concelier.Connector.Common.Cursors; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +/// +/// Computes which CERT/CC summary endpoints should be fetched for the next export window. +/// +public sealed class CertCcSummaryPlanner +{ + private readonly CertCcOptions _options; + private readonly TimeProvider _timeProvider; + + public CertCcSummaryPlanner( + IOptions options, + TimeProvider? timeProvider = null) + { + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public CertCcSummaryPlan CreatePlan(TimeWindowCursorState? state) + { + var now = _timeProvider.GetUtcNow(); + var window = TimeWindowCursorPlanner.GetNextWindow(now, state, _options.SummaryWindow); + var nextState = (state ?? TimeWindowCursorState.Empty).WithWindow(window); + + var months = EnumerateYearMonths(window.Start, window.End) + .Take(_options.MaxMonthlySummaries) + .ToArray(); + + if (months.Length == 0) + { + return new CertCcSummaryPlan(window, Array.Empty(), nextState); + } + + var requests = new List(months.Length * 2); + foreach (var month in months) + { + requests.Add(new CertCcSummaryRequest( + BuildMonthlyUri(month.Year, month.Month), + CertCcSummaryScope.Monthly, + month.Year, + month.Month)); + } + + foreach (var year in months.Select(static value => value.Year).Distinct().OrderBy(static year => year)) + { + requests.Add(new CertCcSummaryRequest( + BuildYearlyUri(year), + CertCcSummaryScope.Yearly, + year, + Month: null)); + } + + return new CertCcSummaryPlan(window, requests, nextState); + } + + private Uri BuildMonthlyUri(int year, int month) + { + var path = $"{year:D4}/{month:D2}/summary/"; + return new Uri(_options.BaseApiUri, path); + } + + private Uri BuildYearlyUri(int year) + { + var path = $"{year:D4}/summary/"; + return new Uri(_options.BaseApiUri, path); + } + + private static IEnumerable<(int Year, int Month)> EnumerateYearMonths(DateTimeOffset start, DateTimeOffset end) + { + if (end <= start) + { + yield break; + } + + var cursor = new DateTime(start.Year, start.Month, 1, 0, 0, 0, DateTimeKind.Utc); + var limit = new DateTime(end.Year, end.Month, 1, 0, 0, 0, DateTimeKind.Utc); + if (end.Day != 1 || end.TimeOfDay != TimeSpan.Zero) + { + limit = limit.AddMonths(1); + } + + while (cursor < limit) + { + yield return (cursor.Year, cursor.Month); + cursor = cursor.AddMonths(1); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs rename to src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs index 72bcdb15..510c80ad 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Internal/CertCcVendorStatementParser.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Internal/CertCcVendorStatementParser.cs @@ -1,235 +1,235 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; - -namespace StellaOps.Feedser.Source.CertCc.Internal; - -internal static class CertCcVendorStatementParser -{ - private static readonly string[] PairSeparators = - { - "\t", - " - ", - " – ", - " — ", - " : ", - ": ", - " :", - ":", - }; - - private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' }; - private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' }; - - // Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a) - private static readonly Regex VersionTokenRegex = new(@"(? Parse(string? statement) - { - if (string.IsNullOrWhiteSpace(statement)) - { - return Array.Empty(); - } - - var patches = new List(); - var lines = statement - .Replace("\r\n", "\n", StringComparison.Ordinal) - .Replace('\r', '\n') - .Split('\n', StringSplitOptions.RemoveEmptyEntries); - - foreach (var rawLine in lines) - { - var line = rawLine.Trim(); - if (line.Length == 0) - { - continue; - } - - line = TrimBulletPrefix(line); - if (line.Length == 0) - { - continue; - } - - if (!TrySplitLine(line, out var productSegment, out var versionSegment)) - { - continue; - } - - var versions = ExtractVersions(versionSegment); - if (versions.Count == 0) - { - continue; - } - - var products = ExtractProducts(productSegment); - if (products.Count == 0) - { - products.Add(string.Empty); - } - - if (versions.Count == products.Count) - { - for (var index = 0; index < versions.Count; index++) - { - patches.Add(new CertCcVendorPatch(products[index], versions[index], line)); - } - - continue; - } - - if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0) - { - var groupSize = products.Count / versions.Count; - for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++) - { - var start = versionIndex * groupSize; - var end = start + groupSize; - var version = versions[versionIndex]; - for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++) - { - patches.Add(new CertCcVendorPatch(products[productIndex], version, line)); - } - } - - continue; - } - - var primaryVersion = versions[0]; - foreach (var product in products) - { - patches.Add(new CertCcVendorPatch(product, primaryVersion, line)); - } - } - - if (patches.Count == 0) - { - return Array.Empty(); - } - - return patches - .Where(static patch => !string.IsNullOrWhiteSpace(patch.Version)) - .Distinct(CertCcVendorPatch.Comparer) - .OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase) - .ThenBy(static patch => patch.Version, StringComparer.Ordinal) - .ToArray(); - } - - private static string TrimBulletPrefix(string value) - { - var trimmed = value.TrimStart(BulletPrefixes).Trim(); - return trimmed.Length == 0 ? value.Trim() : trimmed; - } - - private static bool TrySplitLine(string line, out string productSegment, out string versionSegment) - { - foreach (var separator in PairSeparators) - { - var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries); - if (parts.Length == 2) - { - productSegment = parts[0]; - versionSegment = parts[1]; - return true; - } - } - - var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (whitespaceSplit.Length >= 2) - { - productSegment = string.Join(' ', whitespaceSplit[..^1]); - versionSegment = whitespaceSplit[^1]; - return true; - } - - productSegment = string.Empty; - versionSegment = string.Empty; - return false; - } - - private static List ExtractProducts(string segment) - { - if (string.IsNullOrWhiteSpace(segment)) - { - return new List(); - } - - var normalized = segment.Replace('\t', ' ').Trim(); - var tokens = normalized - .Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries) - .Select(static token => token.Trim()) - .Where(static token => token.Length > 0) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToList(); - - return tokens; - } - - private static List ExtractVersions(string segment) - { - if (string.IsNullOrWhiteSpace(segment)) - { - return new List(); - } - - var matches = VersionTokenRegex.Matches(segment); - if (matches.Count == 0) - { - return new List(); - } - - var versions = new List(matches.Count); - foreach (Match match in matches) - { - if (match.Groups.Count == 0) - { - continue; - } - - var value = match.Groups[1].Value.Trim(); - if (value.Length == 0) - { - continue; - } - - versions.Add(value); - } - - return versions - .Distinct(StringComparer.OrdinalIgnoreCase) - .Take(32) - .ToList(); - } -} - -internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine) -{ - public static IEqualityComparer Comparer { get; } = new CertCcVendorPatchComparer(); - - private sealed class CertCcVendorPatchComparer : IEqualityComparer - { - public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y) - { - if (ReferenceEquals(x, y)) - { - return true; - } - - if (x is null || y is null) - { - return false; - } - - return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) - && string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase); - } - - public int GetHashCode(CertCcVendorPatch obj) - { - var product = obj.Product?.ToLowerInvariant() ?? string.Empty; - var version = obj.Version?.ToLowerInvariant() ?? string.Empty; - return HashCode.Combine(product, version); - } - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Concelier.Connector.CertCc.Internal; + +internal static class CertCcVendorStatementParser +{ + private static readonly string[] PairSeparators = + { + "\t", + " - ", + " – ", + " — ", + " : ", + ": ", + " :", + ":", + }; + + private static readonly char[] BulletPrefixes = { '-', '*', '•', '+', '\t' }; + private static readonly char[] ProductDelimiters = { '/', ',', ';', '&' }; + + // Matches dotted numeric versions and simple alphanumeric suffixes (e.g., 4.4.3.6, 3.9.9.12, 10.2a) + private static readonly Regex VersionTokenRegex = new(@"(? Parse(string? statement) + { + if (string.IsNullOrWhiteSpace(statement)) + { + return Array.Empty(); + } + + var patches = new List(); + var lines = statement + .Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .Split('\n', StringSplitOptions.RemoveEmptyEntries); + + foreach (var rawLine in lines) + { + var line = rawLine.Trim(); + if (line.Length == 0) + { + continue; + } + + line = TrimBulletPrefix(line); + if (line.Length == 0) + { + continue; + } + + if (!TrySplitLine(line, out var productSegment, out var versionSegment)) + { + continue; + } + + var versions = ExtractVersions(versionSegment); + if (versions.Count == 0) + { + continue; + } + + var products = ExtractProducts(productSegment); + if (products.Count == 0) + { + products.Add(string.Empty); + } + + if (versions.Count == products.Count) + { + for (var index = 0; index < versions.Count; index++) + { + patches.Add(new CertCcVendorPatch(products[index], versions[index], line)); + } + + continue; + } + + if (versions.Count > 1 && products.Count > versions.Count && products.Count % versions.Count == 0) + { + var groupSize = products.Count / versions.Count; + for (var versionIndex = 0; versionIndex < versions.Count; versionIndex++) + { + var start = versionIndex * groupSize; + var end = start + groupSize; + var version = versions[versionIndex]; + for (var productIndex = start; productIndex < end && productIndex < products.Count; productIndex++) + { + patches.Add(new CertCcVendorPatch(products[productIndex], version, line)); + } + } + + continue; + } + + var primaryVersion = versions[0]; + foreach (var product in products) + { + patches.Add(new CertCcVendorPatch(product, primaryVersion, line)); + } + } + + if (patches.Count == 0) + { + return Array.Empty(); + } + + return patches + .Where(static patch => !string.IsNullOrWhiteSpace(patch.Version)) + .Distinct(CertCcVendorPatch.Comparer) + .OrderBy(static patch => patch.Product, StringComparer.OrdinalIgnoreCase) + .ThenBy(static patch => patch.Version, StringComparer.Ordinal) + .ToArray(); + } + + private static string TrimBulletPrefix(string value) + { + var trimmed = value.TrimStart(BulletPrefixes).Trim(); + return trimmed.Length == 0 ? value.Trim() : trimmed; + } + + private static bool TrySplitLine(string line, out string productSegment, out string versionSegment) + { + foreach (var separator in PairSeparators) + { + var parts = line.Split(separator, 2, StringSplitOptions.TrimEntries); + if (parts.Length == 2) + { + productSegment = parts[0]; + versionSegment = parts[1]; + return true; + } + } + + var whitespaceSplit = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (whitespaceSplit.Length >= 2) + { + productSegment = string.Join(' ', whitespaceSplit[..^1]); + versionSegment = whitespaceSplit[^1]; + return true; + } + + productSegment = string.Empty; + versionSegment = string.Empty; + return false; + } + + private static List ExtractProducts(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return new List(); + } + + var normalized = segment.Replace('\t', ' ').Trim(); + var tokens = normalized + .Split(ProductDelimiters, StringSplitOptions.RemoveEmptyEntries) + .Select(static token => token.Trim()) + .Where(static token => token.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + return tokens; + } + + private static List ExtractVersions(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return new List(); + } + + var matches = VersionTokenRegex.Matches(segment); + if (matches.Count == 0) + { + return new List(); + } + + var versions = new List(matches.Count); + foreach (Match match in matches) + { + if (match.Groups.Count == 0) + { + continue; + } + + var value = match.Groups[1].Value.Trim(); + if (value.Length == 0) + { + continue; + } + + versions.Add(value); + } + + return versions + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(32) + .ToList(); + } +} + +internal sealed record CertCcVendorPatch(string Product, string Version, string? RawLine) +{ + public static IEqualityComparer Comparer { get; } = new CertCcVendorPatchComparer(); + + private sealed class CertCcVendorPatchComparer : IEqualityComparer + { + public bool Equals(CertCcVendorPatch? x, CertCcVendorPatch? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Product, y.Product, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Version, y.Version, StringComparison.OrdinalIgnoreCase); + } + + public int GetHashCode(CertCcVendorPatch obj) + { + var product = obj.Product?.ToLowerInvariant() ?? string.Empty; + var version = obj.Version?.ToLowerInvariant() ?? string.Empty; + return HashCode.Combine(product, version); + } + } +} diff --git a/src/StellaOps.Feedser.Source.CertCc/Jobs.cs b/src/StellaOps.Concelier.Connector.CertCc/Jobs.cs similarity index 84% rename from src/StellaOps.Feedser.Source.CertCc/Jobs.cs rename to src/StellaOps.Concelier.Connector.CertCc/Jobs.cs index eef41998..0d14a41d 100644 --- a/src/StellaOps.Feedser.Source.CertCc/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.CertCc/Jobs.cs @@ -1,22 +1,22 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.CertCc; - -internal static class CertCcJobKinds -{ - public const string Fetch = "source:cert-cc:fetch"; -} - -internal sealed class CertCcFetchJob : IJob -{ - private readonly CertCcConnector _connector; - - public CertCcFetchJob(CertCcConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.CertCc; + +internal static class CertCcJobKinds +{ + public const string Fetch = "source:cert-cc:fetch"; +} + +internal sealed class CertCcFetchJob : IJob +{ + private readonly CertCcConnector _connector; + + public CertCcFetchJob(CertCcConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.CertCc/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.CertCc/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..19257f60 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertCc/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.CertCc.Tests")] diff --git a/src/StellaOps.Feedser.Source.CertCc/README.md b/src/StellaOps.Concelier.Connector.CertCc/README.md similarity index 89% rename from src/StellaOps.Feedser.Source.CertCc/README.md rename to src/StellaOps.Concelier.Connector.CertCc/README.md index 7a1b716d..80b15ca4 100644 --- a/src/StellaOps.Feedser.Source.CertCc/README.md +++ b/src/StellaOps.Concelier.Connector.CertCc/README.md @@ -1,63 +1,63 @@ -# CERT/CC Vulnerability Notes – Source Research - -## Canonical publication endpoints - -- **Public portal** – `https://www.kb.cert.org/vuls/` lists recently published Vulnerability Notes and exposes a “Subscribe to our feed” link for automation entry points.citeturn0search0 -- **Atom feed** – `https://www.kb.cert.org/vulfeed` returns an Atom 1.0 feed of the same notes (``, `<updated>`, `<summary>` HTML payload). Feed metadata advertises `rel="self"` at `https://kb.cert.org/vuls/atomfeed/`. Use conditional GET headers (`If-Modified-Since`, `If-None-Match`) to avoid refetching unchanged entries.citeturn0search2 - -## VINCE Vulnerability Note API - -The VINCE documentation describes an unauthenticated REST-style API for structured retrieval:citeturn1view0 - -| Endpoint | Payload | Notes | -| --- | --- | --- | -| `GET /vuls/api/{id}/` | Canonical note metadata (title, overview, markdown segments, timestamps, aliases). | Use numeric ID (e.g., `257161`). | -| `GET /vuls/api/{id}/vuls/` | Per-CVE vulnerability records tied to the note. | Includes CVE, description, timestamps. | -| `GET /vuls/api/{id}/vendors/` | Vendor statements per advisory. | Provides status text and optional references. | -| `GET /vuls/api/{id}/vendors/vuls/` | Vendor × vulnerability status matrix. | “known_affected” vs “known_not_affected” semantics. | -| `GET /vuls/api/vuls/cve/{cve}/` | Reverse lookup by CVE. | Returns combined note + vendor context. | -| `GET /vuls/api/{year}/summary/` | Annual summary listing (`count`, `notes[]`). | Year-month variants exist (`/{year}/{month}/summary/`). | -| `GET /vuls/api/{id}/csaf/` | CSAF 2.0 export generated by VINCE. | Useful for downstream CSAF tooling. | - -Operational considerations: - -- API responses are JSON (UTF-8) and publicly accessible; no authentication tokens or cookies are required.citeturn1view0 -- Monthly and annual summary endpoints enable incremental crawling without diffing the Atom feed. -- Expect high-volume notes to expose dozens of vendor records—prepare batching and pagination at the connector layer even though the API returns full arrays today. -- Apply polite backoff: the documentation does not publish explicit rate limits, but the kb.cert.org infrastructure throttles bursts; mirror existing backoff strategy (exponential with jitter) used by other connectors. -- Detail fetch tolerates missing optional endpoints (`vendors`, `vendors-vuls`, `vuls`) by logging a warning and continuing with partial data; repeated 4xx responses will not wedge the cursor. - -## Telemetry & monitoring - -The connector exposes an OpenTelemetry meter named `StellaOps.Feedser.Source.CertCc`. Key instruments include: - -- Planning: `certcc.plan.windows`, `certcc.plan.requests`, and `certcc.plan.window_days`. -- Summary fetch: `certcc.summary.fetch.attempts`, `.success`, `.not_modified`, `.failures`. -- Detail fetch: `certcc.detail.fetch.attempts`, `.success`, `.unchanged`, `.missing`, `.failures` with an `endpoint` dimension (note/vendors/vuls/vendors-vuls). -- Parsing: `certcc.parse.success`, `.failures`, plus histograms for vendor/status/vulnerability counts. -- Mapping: `certcc.map.success`, `.failures`, and histograms `certcc.map.affected.count` / `certcc.map.normalized_versions.count`. - -Structured logs surface correlation IDs across fetch, parse, and map stages. Failures emit warnings for tolerated missing endpoints and errors for retry-worthy conditions so operators can hook them into existing alert policies. - -## Historical data sets - -CERT/CC publishes a Vulnerability Data Archive (JSON exports plus tooling) for deep history or backfills. The archive is hosted on the SEI site with mirrored GitHub repositories containing normalized JSON conversions.citeturn0search3turn0search4 - -## Snapshot regression workflow - -The connector ships deterministic fixtures so QA and Merge teams can replay fetch→parse→map without live calls. Use the following flow when validating changes or refreshing snapshots: - -1. `dotnet test src/StellaOps.Feedser.Source.CertCc.Tests` – runs the connector snapshot suite against canned VINCE responses. -2. `UPDATE_CERTCC_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.CertCc.Tests` – regenerates fixtures under `src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/*.snapshot.json` and mirrors them in the test output directory (`bin/Debug/net10.0/Source/CertCc/Fixtures`). - - The harness now records every HTTP request; `certcc-requests.snapshot.json` must list summaries/months in canonical order. - - Expect `certcc-advisories.snapshot.json` to include normalized versions (`scheme=certcc.vendor`) and provenance decision reasons. -3. Review diffs and attach `certcc-*.snapshot.json` plus test logs when handing off to Merge. - -Fixtures are sorted and timestamps normalized to UTC ISO‑8601 to preserve determinism across machines. - -## Next steps for the connector - -1. Implement Atom polling for quick detection, with VINCE API lookups for structured details. `CertCcSummaryPlanner` already computes the VINCE year/month summary URIs to fetch per window; wire this into the fetch job and persist the resulting `TimeWindowCursorState`. -2. Persist `updated` timestamps and VINCE `revision` counters to drive resume logic. -3. Capture vendor statements/CSAF exports to populate range primitives once model hooks exist. -4. Evaluate using the data archive for seed fixtures covering legacy notes (pre-2010).*** +# CERT/CC Vulnerability Notes – Source Research + +## Canonical publication endpoints + +- **Public portal** – `https://www.kb.cert.org/vuls/` lists recently published Vulnerability Notes and exposes a “Subscribe to our feed” link for automation entry points.citeturn0search0 +- **Atom feed** – `https://www.kb.cert.org/vulfeed` returns an Atom 1.0 feed of the same notes (`<title>`, `<updated>`, `<summary>` HTML payload). Feed metadata advertises `rel="self"` at `https://kb.cert.org/vuls/atomfeed/`. Use conditional GET headers (`If-Modified-Since`, `If-None-Match`) to avoid refetching unchanged entries.citeturn0search2 + +## VINCE Vulnerability Note API + +The VINCE documentation describes an unauthenticated REST-style API for structured retrieval:citeturn1view0 + +| Endpoint | Payload | Notes | +| --- | --- | --- | +| `GET /vuls/api/{id}/` | Canonical note metadata (title, overview, markdown segments, timestamps, aliases). | Use numeric ID (e.g., `257161`). | +| `GET /vuls/api/{id}/vuls/` | Per-CVE vulnerability records tied to the note. | Includes CVE, description, timestamps. | +| `GET /vuls/api/{id}/vendors/` | Vendor statements per advisory. | Provides status text and optional references. | +| `GET /vuls/api/{id}/vendors/vuls/` | Vendor × vulnerability status matrix. | “known_affected” vs “known_not_affected” semantics. | +| `GET /vuls/api/vuls/cve/{cve}/` | Reverse lookup by CVE. | Returns combined note + vendor context. | +| `GET /vuls/api/{year}/summary/` | Annual summary listing (`count`, `notes[]`). | Year-month variants exist (`/{year}/{month}/summary/`). | +| `GET /vuls/api/{id}/csaf/` | CSAF 2.0 export generated by VINCE. | Useful for downstream CSAF tooling. | + +Operational considerations: + +- API responses are JSON (UTF-8) and publicly accessible; no authentication tokens or cookies are required.citeturn1view0 +- Monthly and annual summary endpoints enable incremental crawling without diffing the Atom feed. +- Expect high-volume notes to expose dozens of vendor records—prepare batching and pagination at the connector layer even though the API returns full arrays today. +- Apply polite backoff: the documentation does not publish explicit rate limits, but the kb.cert.org infrastructure throttles bursts; mirror existing backoff strategy (exponential with jitter) used by other connectors. +- Detail fetch tolerates missing optional endpoints (`vendors`, `vendors-vuls`, `vuls`) by logging a warning and continuing with partial data; repeated 4xx responses will not wedge the cursor. + +## Telemetry & monitoring + +The connector exposes an OpenTelemetry meter named `StellaOps.Concelier.Connector.CertCc`. Key instruments include: + +- Planning: `certcc.plan.windows`, `certcc.plan.requests`, and `certcc.plan.window_days`. +- Summary fetch: `certcc.summary.fetch.attempts`, `.success`, `.not_modified`, `.failures`. +- Detail fetch: `certcc.detail.fetch.attempts`, `.success`, `.unchanged`, `.missing`, `.failures` with an `endpoint` dimension (note/vendors/vuls/vendors-vuls). +- Parsing: `certcc.parse.success`, `.failures`, plus histograms for vendor/status/vulnerability counts. +- Mapping: `certcc.map.success`, `.failures`, and histograms `certcc.map.affected.count` / `certcc.map.normalized_versions.count`. + +Structured logs surface correlation IDs across fetch, parse, and map stages. Failures emit warnings for tolerated missing endpoints and errors for retry-worthy conditions so operators can hook them into existing alert policies. + +## Historical data sets + +CERT/CC publishes a Vulnerability Data Archive (JSON exports plus tooling) for deep history or backfills. The archive is hosted on the SEI site with mirrored GitHub repositories containing normalized JSON conversions.citeturn0search3turn0search4 + +## Snapshot regression workflow + +The connector ships deterministic fixtures so QA and Merge teams can replay fetch→parse→map without live calls. Use the following flow when validating changes or refreshing snapshots: + +1. `dotnet test src/StellaOps.Concelier.Connector.CertCc.Tests` – runs the connector snapshot suite against canned VINCE responses. +2. `UPDATE_CERTCC_FIXTURES=1 dotnet test src/StellaOps.Concelier.Connector.CertCc.Tests` – regenerates fixtures under `src/StellaOps.Concelier.Connector.CertCc.Tests/Fixtures/*.snapshot.json` and mirrors them in the test output directory (`bin/Debug/net10.0/Source/CertCc/Fixtures`). + - The harness now records every HTTP request; `certcc-requests.snapshot.json` must list summaries/months in canonical order. + - Expect `certcc-advisories.snapshot.json` to include normalized versions (`scheme=certcc.vendor`) and provenance decision reasons. +3. Review diffs and attach `certcc-*.snapshot.json` plus test logs when handing off to Merge. + +Fixtures are sorted and timestamps normalized to UTC ISO‑8601 to preserve determinism across machines. + +## Next steps for the connector + +1. Implement Atom polling for quick detection, with VINCE API lookups for structured details. `CertCcSummaryPlanner` already computes the VINCE year/month summary URIs to fetch per window; wire this into the fetch job and persist the resulting `TimeWindowCursorState`. +2. Persist `updated` timestamps and VINCE `revision` counters to drive resume logic. +3. Capture vendor statements/CSAF exports to populate range primitives once model hooks exist. +4. Evaluate using the data archive for seed fixtures covering legacy notes (pre-2010).*** diff --git a/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj b/src/StellaOps.Concelier.Connector.CertCc/StellaOps.Concelier.Connector.CertCc.csproj similarity index 52% rename from src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj rename to src/StellaOps.Concelier.Connector.CertCc/StellaOps.Concelier.Connector.CertCc.csproj index f5aeb10f..860f5b2a 100644 --- a/src/StellaOps.Feedser.Source.CertCc/StellaOps.Feedser.Source.CertCc.csproj +++ b/src/StellaOps.Concelier.Connector.CertCc/StellaOps.Concelier.Connector.CertCc.csproj @@ -11,8 +11,8 @@ <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> - <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> - <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> - <ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> </ItemGroup> </Project> diff --git a/src/StellaOps.Feedser.Source.CertCc/TASKS.md b/src/StellaOps.Concelier.Connector.CertCc/TASKS.md similarity index 99% rename from src/StellaOps.Feedser.Source.CertCc/TASKS.md rename to src/StellaOps.Concelier.Connector.CertCc/TASKS.md index 571922de..c1e05929 100644 --- a/src/StellaOps.Feedser.Source.CertCc/TASKS.md +++ b/src/StellaOps.Concelier.Connector.CertCc/TASKS.md @@ -1,14 +1,14 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|Document CERT/CC advisory sources|BE-Conn-CERTCC|Research|**DONE (2025-10-10)** – Catalogued Atom feed + VINCE API endpoints and archive references in `README.md`; include polling/backoff guidance.| -|Fetch pipeline & state tracking|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – Summary planner + fetch job persist monthly/yearly VINCE JSON to `DocumentStore`, hydrate the `TimeWindowCursorState`, and snapshot regression (`dotnet test` 2025-10-12) confirmed deterministic resume behaviour.| -|VINCE note detail fetcher|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – Detail bundle fetch now enqueues VU identifiers and persists note/vendors/vuls/vendors-vuls documents with ETag/Last-Modified metadata, tolerating missing optional endpoints without wedging the cursor.| -|DTO & parser implementation|BE-Conn-CERTCC|Source.Common|**DONE (2025-10-12)** – VINCE DTO aggregate materialises note/vendor/vulnerability payloads, normalises markdown to HTML-safe fragments, and surfaces vendor impact statements covered by parser unit tests.| -|Canonical mapping & range primitives|BE-Conn-CERTCC|Models|**DONE (2025-10-12)** – Mapper emits aliases (VU#, CVE), vendor range primitives, and normalizedVersions (`scheme=certcc.vendor`) with provenance masks; `certcc-advisories.snapshot.json` validates canonical output after schema sync.| -|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-11)** – Snapshot harness regenerated (`certcc-*.snapshot.json`), request ordering assertions added, and `UPDATE_CERTCC_FIXTURES` workflow verified for CI determinism.| -|Connector test harness remediation|BE-Conn-CERTCC, QA|Testing|**DONE (2025-10-11)** – Connector test harness now rebuilds `FakeTimeProvider`, wires `AddSourceCommon`, and drives canned VINCE responses across fetch→parse→map with recorded-request assertions.| -|Snapshot coverage handoff|QA|Models, Merge|**DONE (2025-10-11)** – Fixtures + request/advisory snapshots refreshed, README documents `UPDATE_CERTCC_FIXTURES` workflow, and recorded-request ordering is enforced for QA handoff.| -|FEEDCONN-CERTCC-02-010 Partial-detail graceful degradation|BE-Conn-CERTCC|Connector plan|**DONE (2025-10-12)** – Detail fetch now catches 404/410/403 responses for optional endpoints, logs missing bundles, feeds empty payloads into parsing, and ships regression coverage for mixed responses.| -|FEEDCONN-CERTCC-02-012 Schema sync & snapshot regen follow-up|QA, BE-Conn-CERTCC|Models `FEEDMODELS-SCHEMA-01-001`/`-002`/`-003`, Storage `FEEDSTORAGE-DATA-02-001`|**DONE (2025-10-12)** – Snapshot suite rerun, fixtures updated, and handoff notes (`FEEDCONN-CERTCC-02-012_HANDOFF.md`) document normalizedVersions/provenance expectations for Merge backfill.| -|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – `CertCcDiagnostics` now publishes summary/detail/parse/map metrics, README documents meter names, and structured logging guidance is captured for Ops handoff.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Document CERT/CC advisory sources|BE-Conn-CERTCC|Research|**DONE (2025-10-10)** – Catalogued Atom feed + VINCE API endpoints and archive references in `README.md`; include polling/backoff guidance.| +|Fetch pipeline & state tracking|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – Summary planner + fetch job persist monthly/yearly VINCE JSON to `DocumentStore`, hydrate the `TimeWindowCursorState`, and snapshot regression (`dotnet test` 2025-10-12) confirmed deterministic resume behaviour.| +|VINCE note detail fetcher|BE-Conn-CERTCC|Source.Common, Storage.Mongo|**DONE (2025-10-12)** – Detail bundle fetch now enqueues VU identifiers and persists note/vendors/vuls/vendors-vuls documents with ETag/Last-Modified metadata, tolerating missing optional endpoints without wedging the cursor.| +|DTO & parser implementation|BE-Conn-CERTCC|Source.Common|**DONE (2025-10-12)** – VINCE DTO aggregate materialises note/vendor/vulnerability payloads, normalises markdown to HTML-safe fragments, and surfaces vendor impact statements covered by parser unit tests.| +|Canonical mapping & range primitives|BE-Conn-CERTCC|Models|**DONE (2025-10-12)** – Mapper emits aliases (VU#, CVE), vendor range primitives, and normalizedVersions (`scheme=certcc.vendor`) with provenance masks; `certcc-advisories.snapshot.json` validates canonical output after schema sync.| +|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-11)** – Snapshot harness regenerated (`certcc-*.snapshot.json`), request ordering assertions added, and `UPDATE_CERTCC_FIXTURES` workflow verified for CI determinism.| +|Connector test harness remediation|BE-Conn-CERTCC, QA|Testing|**DONE (2025-10-11)** – Connector test harness now rebuilds `FakeTimeProvider`, wires `AddSourceCommon`, and drives canned VINCE responses across fetch→parse→map with recorded-request assertions.| +|Snapshot coverage handoff|QA|Models, Merge|**DONE (2025-10-11)** – Fixtures + request/advisory snapshots refreshed, README documents `UPDATE_CERTCC_FIXTURES` workflow, and recorded-request ordering is enforced for QA handoff.| +|FEEDCONN-CERTCC-02-010 Partial-detail graceful degradation|BE-Conn-CERTCC|Connector plan|**DONE (2025-10-12)** – Detail fetch now catches 404/410/403 responses for optional endpoints, logs missing bundles, feeds empty payloads into parsing, and ships regression coverage for mixed responses.| +|FEEDCONN-CERTCC-02-012 Schema sync & snapshot regen follow-up|QA, BE-Conn-CERTCC|Models `FEEDMODELS-SCHEMA-01-001`/`-002`/`-003`, Storage `FEEDSTORAGE-DATA-02-001`|**DONE (2025-10-12)** – Snapshot suite rerun, fixtures updated, and handoff notes (`FEEDCONN-CERTCC-02-012_HANDOFF.md`) document normalizedVersions/provenance expectations for Merge backfill.| +|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – `CertCcDiagnostics` now publishes summary/detail/parse/map metrics, README documents meter names, and structured logging guidance is captured for Ops handoff.| diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs rename to src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs index 92ccb065..a843fe7d 100644 --- a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/CertFrConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/CertFrConnectorTests.cs @@ -15,19 +15,19 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Source.CertFr; -using StellaOps.Feedser.Source.CertFr.Configuration; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Connector.CertFr; +using StellaOps.Concelier.Connector.CertFr.Configuration; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Source.CertFr.Tests; +namespace StellaOps.Concelier.Connector.CertFr.Tests; [Collection("mongo-fixture")] public sealed class CertFrConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json index d0340a94..e5e04542 100644 --- a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json +++ b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-advisories.snapshot.json @@ -1,205 +1,205 @@ -[ - { - "advisoryKey": "cert-fr/AV-2024.001", - "affectedPackages": [ - { - "identifier": "AV-2024.001", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - } - ], - "statuses": [], - "type": "vendor", - "versionRanges": [ - { - "fixedVersion": null, - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": null, - "vendorExtensions": { - "certfr.summary": "Résumé de la première alerte.", - "certfr.content": "AV-2024.001 Alerte CERT-FR AV-2024.001 L'exploitation active de la vulnérabilité est surveillée. Consultez les indications du fournisseur .", - "certfr.reference.count": "1" - } - }, - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - }, - "rangeExpression": null, - "rangeKind": "vendor" - } - ] - } - ], - "aliases": [ - "CERT-FR:AV-2024.001" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "fr", - "modified": null, - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - } - ], - "published": "2024-10-03T00:00:00+00:00", - "references": [ - { - "kind": "reference", - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - }, - "sourceTag": null, - "summary": null, - "url": "https://vendor.example.com/patch" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - }, - "sourceTag": "cert-fr", - "summary": "Résumé de la première alerte.", - "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" - } - ], - "severity": null, - "summary": "Résumé de la première alerte.", - "title": "AV-2024.001 - Première alerte" - }, - { - "advisoryKey": "cert-fr/AV-2024.002", - "affectedPackages": [ - { - "identifier": "AV-2024.002", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - } - ], - "statuses": [], - "type": "vendor", - "versionRanges": [ - { - "fixedVersion": null, - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": null, - "vendorExtensions": { - "certfr.summary": "Résumé de la deuxième alerte.", - "certfr.content": "AV-2024.002 Alerte CERT-FR AV-2024.002 Des correctifs sont disponibles pour plusieurs produits. Note de mise à jour Correctif", - "certfr.reference.count": "2" - } - }, - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - }, - "rangeExpression": null, - "rangeKind": "vendor" - } - ] - } - ], - "aliases": [ - "CERT-FR:AV-2024.002" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "fr", - "modified": null, - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - } - ], - "published": "2024-10-03T00:00:00+00:00", - "references": [ - { - "kind": "reference", - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - }, - "sourceTag": null, - "summary": null, - "url": "https://support.example.com/kb/KB-1234" - }, - { - "kind": "reference", - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - }, - "sourceTag": null, - "summary": null, - "url": "https://support.example.com/kb/KB-5678" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-10-03T00:01:00+00:00", - "source": "cert-fr", - "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - }, - "sourceTag": "cert-fr", - "summary": "Résumé de la deuxième alerte.", - "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" - } - ], - "severity": null, - "summary": "Résumé de la deuxième alerte.", - "title": "AV-2024.002 - Deuxième alerte" - } +[ + { + "advisoryKey": "cert-fr/AV-2024.001", + "affectedPackages": [ + { + "identifier": "AV-2024.001", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certfr.summary": "Résumé de la première alerte.", + "certfr.content": "AV-2024.001 Alerte CERT-FR AV-2024.001 L'exploitation active de la vulnérabilité est surveillée. Consultez les indications du fournisseur .", + "certfr.reference.count": "1" + } + }, + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CERT-FR:AV-2024.001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/patch" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la première alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.001/" + } + ], + "severity": null, + "summary": "Résumé de la première alerte.", + "title": "AV-2024.001 - Première alerte" + }, + { + "advisoryKey": "cert-fr/AV-2024.002", + "affectedPackages": [ + { + "identifier": "AV-2024.002", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certfr.summary": "Résumé de la deuxième alerte.", + "certfr.content": "AV-2024.002 Alerte CERT-FR AV-2024.002 Des correctifs sont disponibles pour plusieurs produits. Note de mise à jour Correctif", + "certfr.reference.count": "2" + } + }, + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CERT-FR:AV-2024.002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "fr", + "modified": null, + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "published": "2024-10-03T00:00:00+00:00", + "references": [ + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-1234" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": null, + "summary": null, + "url": "https://support.example.com/kb/KB-5678" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-10-03T00:01:00+00:00", + "source": "cert-fr", + "value": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + }, + "sourceTag": "cert-fr", + "summary": "Résumé de la deuxième alerte.", + "url": "https://www.cert.ssi.gouv.fr/alerte/AV-2024.002/" + } + ], + "severity": null, + "summary": "Résumé de la deuxième alerte.", + "title": "AV-2024.002 - Deuxième alerte" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html similarity index 100% rename from src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html rename to src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-001.html diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html similarity index 100% rename from src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html rename to src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-detail-AV-2024-002.html diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml b/src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml similarity index 100% rename from src/StellaOps.Feedser.Source.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml rename to src/StellaOps.Concelier.Connector.CertFr.Tests/CertFr/Fixtures/certfr-feed.xml diff --git a/src/StellaOps.Concelier.Connector.CertFr.Tests/StellaOps.Concelier.Connector.CertFr.Tests.csproj b/src/StellaOps.Concelier.Connector.CertFr.Tests/StellaOps.Concelier.Connector.CertFr.Tests.csproj new file mode 100644 index 00000000..bc08c36d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertFr.Tests/StellaOps.Concelier.Connector.CertFr.Tests.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.CertFr/StellaOps.Concelier.Connector.CertFr.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="CertFr/Fixtures/**" CopyToOutputDirectory="Always" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md b/src/StellaOps.Concelier.Connector.CertFr/AGENTS.md similarity index 81% rename from src/StellaOps.Feedser.Source.CertFr/AGENTS.md rename to src/StellaOps.Concelier.Connector.CertFr/AGENTS.md index 5e5cf3e2..15d9fbd7 100644 --- a/src/StellaOps.Feedser.Source.CertFr/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.CertFr/AGENTS.md @@ -19,9 +19,9 @@ ANSSI CERT-FR advisories connector (avis/alertes) providing national enrichment: In: advisory metadata extraction, references, severity text, watermarking. Out: OVAL or package-level authority. ## Observability & security expectations -- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=certfr`, covering fetch counts, parse failures, and map activity. +- Metrics: SourceDiagnostics emits shared `concelier.source.http.*` counters/histograms tagged `concelier.source=certfr`, covering fetch counts, parse failures, and map activity. - Logs: feed URL(s), item ids/urls, extraction durations; no PII; allowlist hostnames. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.CertFr.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.CertFr.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs b/src/StellaOps.Concelier.Connector.CertFr/CertFrConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs rename to src/StellaOps.Concelier.Connector.CertFr/CertFrConnector.cs index 264be9e2..36a6460f 100644 --- a/src/StellaOps.Feedser.Source.CertFr/CertFrConnector.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/CertFrConnector.cs @@ -5,17 +5,17 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Source.CertFr.Configuration; -using StellaOps.Feedser.Source.CertFr.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Connector.CertFr.Configuration; +using StellaOps.Concelier.Connector.CertFr.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertFr; +namespace StellaOps.Concelier.Connector.CertFr; public sealed class CertFrConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.CertFr/CertFrConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.CertFr/CertFrConnectorPlugin.cs index fb2357aa..7ed05ebe 100644 --- a/src/StellaOps.Feedser.Source.CertFr/CertFrConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/CertFrConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertFr; +namespace StellaOps.Concelier.Connector.CertFr; public sealed class CertFrConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.CertFr/CertFrDependencyInjectionRoutine.cs similarity index 84% rename from src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.CertFr/CertFrDependencyInjectionRoutine.cs index 9effd14d..8d2f0b4f 100644 --- a/src/StellaOps.Feedser.Source.CertFr/CertFrDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/CertFrDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.CertFr.Configuration; -namespace StellaOps.Feedser.Source.CertFr; +namespace StellaOps.Concelier.Connector.CertFr; public sealed class CertFrDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:cert-fr"; + private const string ConfigurationSection = "concelier:sources:cert-fr"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.CertFr/CertFrServiceCollectionExtensions.cs similarity index 78% rename from src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.CertFr/CertFrServiceCollectionExtensions.cs index 80446c97..2c3d5cb2 100644 --- a/src/StellaOps.Feedser.Source.CertFr/CertFrServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/CertFrServiceCollectionExtensions.cs @@ -2,11 +2,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertFr.Configuration; -using StellaOps.Feedser.Source.CertFr.Internal; -using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Concelier.Connector.CertFr.Configuration; +using StellaOps.Concelier.Connector.CertFr.Internal; +using StellaOps.Concelier.Connector.Common.Http; -namespace StellaOps.Feedser.Source.CertFr; +namespace StellaOps.Concelier.Connector.CertFr; public static class CertFrServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class CertFrServiceCollectionExtensions { var options = sp.GetRequiredService<IOptions<CertFrOptions>>().Value; clientOptions.BaseAddress = options.FeedUri; - clientOptions.UserAgent = "StellaOps.Feedser.CertFr/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.CertFr/1.0"; clientOptions.Timeout = TimeSpan.FromSeconds(20); clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.FeedUri.Host); diff --git a/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs b/src/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs rename to src/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs index 0593204f..0865bc1a 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Configuration/CertFrOptions.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Configuration/CertFrOptions.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Source.CertFr.Configuration; +namespace StellaOps.Concelier.Connector.CertFr.Configuration; public sealed class CertFrOptions { diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrCursor.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrCursor.cs index 434ce028..28fae7c2 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrCursor.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; internal sealed record CertFrCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDocumentMetadata.cs similarity index 93% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDocumentMetadata.cs index d8bd1d35..c29b8e88 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDocumentMetadata.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; internal sealed record CertFrDocumentMetadata( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDto.cs similarity index 89% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDto.cs index 9b25fea1..b5be2fc9 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrDto.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrDto.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; internal sealed record CertFrDto( [property: JsonPropertyName("advisoryId")] string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedClient.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedClient.cs index b160f11b..da6e09d1 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedClient.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedClient.cs @@ -8,9 +8,9 @@ using System.Threading.Tasks; using System.Xml.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertFr.Configuration; +using StellaOps.Concelier.Connector.CertFr.Configuration; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; public sealed class CertFrFeedClient { diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedItem.cs similarity index 71% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedItem.cs index 3e222528..1e723b64 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrFeedItem.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrFeedItem.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; public sealed record CertFrFeedItem( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrMapper.cs similarity index 96% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrMapper.cs index eb72ccd8..4a10b7da 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrMapper.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrMapper.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; internal static class CertFrMapper { diff --git a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs rename to src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrParser.cs index 48a87520..515f8959 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Internal/CertFrParser.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Internal/CertFrParser.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Source.CertFr.Internal; +namespace StellaOps.Concelier.Connector.CertFr.Internal; internal static class CertFrParser { diff --git a/src/StellaOps.Feedser.Source.CertFr/Jobs.cs b/src/StellaOps.Concelier.Connector.CertFr/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.CertFr/Jobs.cs rename to src/StellaOps.Concelier.Connector.CertFr/Jobs.cs index 6994fe84..5e8c1acd 100644 --- a/src/StellaOps.Feedser.Source.CertFr/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.CertFr/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.CertFr; +namespace StellaOps.Concelier.Connector.CertFr; internal static class CertFrJobKinds { diff --git a/src/StellaOps.Concelier.Connector.CertFr/StellaOps.Concelier.Connector.CertFr.csproj b/src/StellaOps.Concelier.Connector.CertFr/StellaOps.Concelier.Connector.CertFr.csproj new file mode 100644 index 00000000..cf97d924 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertFr/StellaOps.Concelier.Connector.CertFr.csproj @@ -0,0 +1,13 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.CertFr/TASKS.md b/src/StellaOps.Concelier.Connector.CertFr/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.CertFr/TASKS.md rename to src/StellaOps.Concelier.Connector.CertFr/TASKS.md diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/CertInConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs rename to src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/CertInConnectorTests.cs index 56dcabc1..5212b4ec 100644 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/CertInConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/CertInConnectorTests.cs @@ -14,21 +14,21 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.CertIn; -using StellaOps.Feedser.Source.CertIn.Configuration; -using StellaOps.Feedser.Source.CertIn.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.CertIn; +using StellaOps.Concelier.Connector.CertIn.Configuration; +using StellaOps.Concelier.Connector.CertIn.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.CertIn.Tests; +namespace StellaOps.Concelier.Connector.CertIn.Tests; [Collection("mongo-fixture")] public sealed class CertInConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/alerts-page1.json similarity index 100% rename from src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/alerts-page1.json rename to src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/alerts-page1.json diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html similarity index 100% rename from src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html rename to src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/detail-CIAD-2024-0005.html diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/expected-advisory.json similarity index 97% rename from src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json rename to src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/expected-advisory.json index e8f3dd5d..14219318 100644 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/CertIn/Fixtures/expected-advisory.json +++ b/src/StellaOps.Concelier.Connector.CertIn.Tests/CertIn/Fixtures/expected-advisory.json @@ -1,128 +1,128 @@ -{ - "advisoryKey": "CIAD-2024-0005", - "affectedPackages": [ - { - "identifier": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "affected", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" - } - ], - "statuses": [], - "type": "ics-vendor", - "versionRanges": [ - { - "fixedVersion": null, - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": null, - "vendorExtensions": { - "certin.vendor": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the " - } - }, - "provenance": { - "fieldMask": [], - "kind": "affected", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" - }, - "rangeExpression": null, - "rangeKind": "vendor" - } - ] - } - ], - "aliases": [ - "CIAD-2024-0005", - "CVE-2024-9990", - "CVE-2024-9991" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2024-04-15T10:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-04-20T00:00:00+00:00", - "source": "cert-in", - "value": "https://cert-in.example/advisory/CIAD-2024-0005" - }, - { - "fieldMask": [], - "kind": "mapping", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "CIAD-2024-0005" - } - ], - "published": "2024-04-15T10:00:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "https://cert-in.example/advisory/CIAD-2024-0005" - }, - "sourceTag": "cert-in", - "summary": null, - "url": "https://cert-in.example/advisory/CIAD-2024-0005" - }, - { - "kind": "reference", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "https://vendor.example.com/advisories/example-gateway-bulletin" - }, - "sourceTag": null, - "summary": null, - "url": "https://vendor.example.com/advisories/example-gateway-bulletin" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "https://www.cve.org/CVERecord?id=CVE-2024-9990" - }, - "sourceTag": "CVE-2024-9990", - "summary": null, - "url": "https://www.cve.org/CVERecord?id=CVE-2024-9990" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-20T00:01:00+00:00", - "source": "cert-in", - "value": "https://www.cve.org/CVERecord?id=CVE-2024-9991" - }, - "sourceTag": "CVE-2024-9991", - "summary": null, - "url": "https://www.cve.org/CVERecord?id=CVE-2024-9991" - } - ], - "severity": "high", - "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", - "title": "Multiple vulnerabilities in Example Gateway" +{ + "advisoryKey": "CIAD-2024-0005", + "affectedPackages": [ + { + "identifier": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" + } + ], + "statuses": [], + "type": "ics-vendor", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": null, + "vendorExtensions": { + "certin.vendor": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the " + } + }, + "provenance": { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "Example Gateway Technologies Pvt Ltd Organisation: Partner Systems Inc. CVE-2024-9990 and CVE-2024-9991 allow remote attackers to execute arbitrary commands. Further information is available from the" + }, + "rangeExpression": null, + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CIAD-2024-0005", + "CVE-2024-9990", + "CVE-2024-9991" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-15T10:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-20T00:00:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "CIAD-2024-0005" + } + ], + "published": "2024-04-15T10:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + "sourceTag": "cert-in", + "summary": null, + "url": "https://cert-in.example/advisory/CIAD-2024-0005" + }, + { + "kind": "reference", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + "sourceTag": null, + "summary": null, + "url": "https://vendor.example.com/advisories/example-gateway-bulletin" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + "sourceTag": "CVE-2024-9990", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9990" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-20T00:01:00+00:00", + "source": "cert-in", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + }, + "sourceTag": "CVE-2024-9991", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2024-9991" + } + ], + "severity": "high", + "summary": "Example Gateway devices vulnerable to remote code execution (CVE-2024-9990).", + "title": "Multiple vulnerabilities in Example Gateway" } \ No newline at end of file diff --git a/src/StellaOps.Concelier.Connector.CertIn.Tests/StellaOps.Concelier.Connector.CertIn.Tests.csproj b/src/StellaOps.Concelier.Connector.CertIn.Tests/StellaOps.Concelier.Connector.CertIn.Tests.csproj new file mode 100644 index 00000000..8a9c9b42 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertIn.Tests/StellaOps.Concelier.Connector.CertIn.Tests.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="CertIn/Fixtures/**" CopyToOutputDirectory="Always" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md b/src/StellaOps.Concelier.Connector.CertIn/AGENTS.md similarity index 83% rename from src/StellaOps.Feedser.Source.CertIn/AGENTS.md rename to src/StellaOps.Concelier.Connector.CertIn/AGENTS.md index e8e4dc8f..4aec2205 100644 --- a/src/StellaOps.Feedser.Source.CertIn/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.CertIn/AGENTS.md @@ -20,9 +20,9 @@ CERT-In national CERT connector; enrichment advisories for India; maps CVE lists In: enrichment, aliasing where stable, references, mitigation text. Out: package range authority; scraping behind auth walls. ## Observability & security expectations -- Metrics: shared `feedser.source.http.*` counters/histograms from SourceDiagnostics tagged `feedser.source=certin` capture fetch volume, parse failures, and map enrich counts. +- Metrics: shared `concelier.source.http.*` counters/histograms from SourceDiagnostics tagged `concelier.source=certin` capture fetch volume, parse failures, and map enrich counts. - Logs: advisory codes, CVE counts per advisory, timing; allowlist host; redact personal data if present. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.CertIn.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.CertIn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs b/src/StellaOps.Concelier.Connector.CertIn/CertInConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs rename to src/StellaOps.Concelier.Connector.CertIn/CertInConnector.cs index e6e53534..510e848e 100644 --- a/src/StellaOps.Feedser.Source.CertIn/CertInConnector.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/CertInConnector.cs @@ -7,18 +7,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.CertIn.Configuration; -using StellaOps.Feedser.Source.CertIn.Internal; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.CertIn.Configuration; +using StellaOps.Concelier.Connector.CertIn.Internal; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertIn; +namespace StellaOps.Concelier.Connector.CertIn; public sealed class CertInConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.CertIn/CertInConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.CertIn/CertInConnectorPlugin.cs index a25ae81d..4d7b08b3 100644 --- a/src/StellaOps.Feedser.Source.CertIn/CertInConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/CertInConnectorPlugin.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.CertIn; +namespace StellaOps.Concelier.Connector.CertIn; public sealed class CertInConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.CertIn/CertInDependencyInjectionRoutine.cs similarity index 84% rename from src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.CertIn/CertInDependencyInjectionRoutine.cs index 662d1411..a1c2abe3 100644 --- a/src/StellaOps.Feedser.Source.CertIn/CertInDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/CertInDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.CertIn.Configuration; -namespace StellaOps.Feedser.Source.CertIn; +namespace StellaOps.Concelier.Connector.CertIn; public sealed class CertInDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:cert-in"; + private const string ConfigurationSection = "concelier:sources:cert-in"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.CertIn/CertInServiceCollectionExtensions.cs similarity index 78% rename from src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.CertIn/CertInServiceCollectionExtensions.cs index 8b1feb47..f9861107 100644 --- a/src/StellaOps.Feedser.Source.CertIn/CertInServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/CertInServiceCollectionExtensions.cs @@ -1,11 +1,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertIn.Configuration; -using StellaOps.Feedser.Source.CertIn.Internal; -using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Concelier.Connector.CertIn.Configuration; +using StellaOps.Concelier.Connector.CertIn.Internal; +using StellaOps.Concelier.Connector.Common.Http; -namespace StellaOps.Feedser.Source.CertIn; +namespace StellaOps.Concelier.Connector.CertIn; public static class CertInServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class CertInServiceCollectionExtensions var options = sp.GetRequiredService<IOptions<CertInOptions>>().Value; clientOptions.BaseAddress = options.AlertsEndpoint; clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.CertIn/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.CertIn/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.AlertsEndpoint.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; diff --git a/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs b/src/StellaOps.Concelier.Connector.CertIn/Configuration/CertInOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs rename to src/StellaOps.Concelier.Connector.CertIn/Configuration/CertInOptions.cs index 88a69095..a2f8757f 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Configuration/CertInOptions.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Configuration/CertInOptions.cs @@ -1,7 +1,7 @@ using System; using System.Diagnostics.CodeAnalysis; -namespace StellaOps.Feedser.Source.CertIn.Configuration; +namespace StellaOps.Concelier.Connector.CertIn.Configuration; public sealed class CertInOptions { diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInAdvisoryDto.cs similarity index 83% rename from src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.CertIn/Internal/CertInAdvisoryDto.cs index 3bf3cbbf..7a8bf7a3 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInAdvisoryDto.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Immutable; -namespace StellaOps.Feedser.Source.CertIn.Internal; +namespace StellaOps.Concelier.Connector.CertIn.Internal; internal sealed record CertInAdvisoryDto( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs rename to src/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs index 6e3119f5..569fe0b3 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInClient.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInClient.cs @@ -9,9 +9,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertIn.Configuration; +using StellaOps.Concelier.Connector.CertIn.Configuration; -namespace StellaOps.Feedser.Source.CertIn.Internal; +namespace StellaOps.Concelier.Connector.CertIn.Internal; public sealed class CertInClient { diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs similarity index 94% rename from src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs rename to src/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs index 835a7194..776d7fa6 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInCursor.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.CertIn.Internal; +namespace StellaOps.Concelier.Connector.CertIn.Internal; internal sealed record CertInCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs rename to src/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs index 24da33ee..f63cec01 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInDetailParser.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInDetailParser.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Source.CertIn.Internal; +namespace StellaOps.Concelier.Connector.CertIn.Internal; internal static class CertInDetailParser { diff --git a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInListingItem.cs similarity index 71% rename from src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs rename to src/StellaOps.Concelier.Connector.CertIn/Internal/CertInListingItem.cs index 3da8a3f3..c42c311d 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Internal/CertInListingItem.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Internal/CertInListingItem.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Source.CertIn.Internal; +namespace StellaOps.Concelier.Connector.CertIn.Internal; public sealed record CertInListingItem( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.CertIn/Jobs.cs b/src/StellaOps.Concelier.Connector.CertIn/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.CertIn/Jobs.cs rename to src/StellaOps.Concelier.Connector.CertIn/Jobs.cs index 01b35089..0a9788b4 100644 --- a/src/StellaOps.Feedser.Source.CertIn/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.CertIn/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.CertIn; +namespace StellaOps.Concelier.Connector.CertIn; internal static class CertInJobKinds { diff --git a/src/StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj b/src/StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj new file mode 100644 index 00000000..44c74bcc --- /dev/null +++ b/src/StellaOps.Concelier.Connector.CertIn/StellaOps.Concelier.Connector.CertIn.csproj @@ -0,0 +1,16 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.CertIn/TASKS.md b/src/StellaOps.Concelier.Connector.CertIn/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.CertIn/TASKS.md rename to src/StellaOps.Concelier.Connector.CertIn/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/CannedHttpMessageHandlerTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/CannedHttpMessageHandlerTests.cs index ad0be9f5..01c69579 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/CannedHttpMessageHandlerTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/CannedHttpMessageHandlerTests.cs @@ -1,8 +1,8 @@ using System.Net; using System.Net.Http; -using StellaOps.Feedser.Source.Common.Testing; +using StellaOps.Concelier.Connector.Common.Testing; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class CannedHttpMessageHandlerTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/HtmlContentSanitizerTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/HtmlContentSanitizerTests.cs index 2d1eee3b..6ab1f76c 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/HtmlContentSanitizerTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/HtmlContentSanitizerTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Source.Common.Html; +using StellaOps.Concelier.Connector.Common.Html; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class HtmlContentSanitizerTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/PackageCoordinateHelperTests.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/PackageCoordinateHelperTests.cs index 3668af6e..8f4d6571 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/PackageCoordinateHelperTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/PackageCoordinateHelperTests.cs @@ -1,7 +1,7 @@ using NuGet.Versioning; -using StellaOps.Feedser.Source.Common.Packages; +using StellaOps.Concelier.Connector.Common.Packages; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class PackageCoordinateHelperTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/PdfTextExtractorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/PdfTextExtractorTests.cs index b2da19a9..6609a72e 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/PdfTextExtractorTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/PdfTextExtractorTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Source.Common.Pdf; +using StellaOps.Concelier.Connector.Common.Pdf; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class PdfTextExtractorTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceFetchServiceTests.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceFetchServiceTests.cs index ad2c3713..ff1bd09e 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceFetchServiceTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceFetchServiceTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Source.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Fetch; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class SourceFetchServiceTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceHttpClientBuilderTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceHttpClientBuilderTests.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/SourceHttpClientBuilderTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceHttpClientBuilderTests.cs index 188c1db7..0ae1d047 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/SourceHttpClientBuilderTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/SourceHttpClientBuilderTests.cs @@ -10,9 +10,9 @@ using System.Text; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Concelier.Connector.Common.Http; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class SourceHttpClientBuilderTests { @@ -62,13 +62,13 @@ public sealed class SourceHttpClientBuilderTests var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary<string, string?> { - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyAddressKey}"] = "http://proxy.local:8080", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyBypassOnLocalKey}"] = "false", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyBypassListKey}:0"] = "localhost", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyBypassListKey}:1"] = "127.0.0.1", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyUseDefaultCredentialsKey}"] = "false", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyUsernameKey}"] = "svc-feedser", - [$"feedser:httpClients:source.icscisa:{ProxySection}:{ProxyPasswordKey}"] = "s3cr3t!", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyAddressKey}"] = "http://proxy.local:8080", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyBypassOnLocalKey}"] = "false", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyBypassListKey}:0"] = "localhost", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyBypassListKey}:1"] = "127.0.0.1", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyUseDefaultCredentialsKey}"] = "false", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyUsernameKey}"] = "svc-concelier", + [$"concelier:httpClients:source.icscisa:{ProxySection}:{ProxyPasswordKey}"] = "s3cr3t!", }) .Build(); @@ -85,7 +85,7 @@ public sealed class SourceHttpClientBuilderTests var resolvedConfiguration = provider.GetRequiredService<IConfiguration>(); var proxySection = resolvedConfiguration - .GetSection("feedser") + .GetSection("concelier") .GetSection("httpClients") .GetSection("source.icscisa") .GetSection("proxy"); @@ -99,7 +99,7 @@ public sealed class SourceHttpClientBuilderTests Assert.Contains("localhost", configuredOptions.ProxyBypassList, StringComparer.OrdinalIgnoreCase); Assert.Contains("127.0.0.1", configuredOptions.ProxyBypassList); Assert.False(configuredOptions.ProxyUseDefaultCredentials); - Assert.Equal("svc-feedser", configuredOptions.ProxyUsername); + Assert.Equal("svc-concelier", configuredOptions.ProxyUsername); Assert.Equal("s3cr3t!", configuredOptions.ProxyPassword); } @@ -115,8 +115,8 @@ public sealed class SourceHttpClientBuilderTests var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary<string, string?> { - [$"feedser:httpClients:source.acsc:{AllowInvalidKey}"] = "true", - [$"feedser:httpClients:source.acsc:{TrustedRootPathsKey}:0"] = pemPath, + [$"concelier:httpClients:source.acsc:{AllowInvalidKey}"] = "true", + [$"concelier:httpClients:source.acsc:{TrustedRootPathsKey}:0"] = pemPath, }) .Build(); @@ -173,8 +173,8 @@ public sealed class SourceHttpClientBuilderTests var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary<string, string?> { - [$"feedser:{OfflineRootKey}"] = offlineRoot.FullName, - [$"feedser:httpClients:source.nkcki:{TrustedRootPathsKey}:0"] = relativePath, + [$"concelier:{OfflineRootKey}"] = offlineRoot.FullName, + [$"concelier:httpClients:source.nkcki:{TrustedRootPathsKey}:0"] = relativePath, }) .Build(); @@ -227,8 +227,8 @@ public sealed class SourceHttpClientBuilderTests var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary<string, string?> { - [$"feedser:{OfflineRootKey}"] = offlineRoot.FullName, - [$"feedser:sources:nkcki:http:{TrustedRootPathsKey}:0"] = relativePath, + [$"concelier:{OfflineRootKey}"] = offlineRoot.FullName, + [$"concelier:sources:nkcki:http:{TrustedRootPathsKey}:0"] = relativePath, }) .Build(); diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/TimeWindowCursorPlannerTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/TimeWindowCursorPlannerTests.cs index 9e6afb1e..27b43e83 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/TimeWindowCursorPlannerTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/TimeWindowCursorPlannerTests.cs @@ -1,7 +1,7 @@ using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Concelier.Connector.Common.Cursors; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class TimeWindowCursorPlannerTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Common/UrlNormalizerTests.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Common/UrlNormalizerTests.cs index 3eeeef49..d26e437b 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Common/UrlNormalizerTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Common/UrlNormalizerTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Source.Common.Url; +using StellaOps.Concelier.Connector.Common.Url; -namespace StellaOps.Feedser.Source.Common.Tests; +namespace StellaOps.Concelier.Connector.Common.Tests; public sealed class UrlNormalizerTests { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Json/JsonSchemaValidatorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Json/JsonSchemaValidatorTests.cs index f676b87c..4d916ef6 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Json/JsonSchemaValidatorTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Json/JsonSchemaValidatorTests.cs @@ -2,9 +2,9 @@ using System; using System.Text.Json; using Json.Schema; using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Source.Common.Json; +using StellaOps.Concelier.Connector.Common.Json; -namespace StellaOps.Feedser.Source.Common.Tests.Json; +namespace StellaOps.Concelier.Connector.Common.Tests.Json; public sealed class JsonSchemaValidatorTests { diff --git a/src/StellaOps.Concelier.Connector.Common.Tests/StellaOps.Concelier.Connector.Common.Tests.csproj b/src/StellaOps.Concelier.Connector.Common.Tests/StellaOps.Concelier.Connector.Common.Tests.csproj new file mode 100644 index 00000000..1fb726a4 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Common.Tests/StellaOps.Concelier.Connector.Common.Tests.csproj @@ -0,0 +1,10 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs b/src/StellaOps.Concelier.Connector.Common.Tests/Xml/XmlSchemaValidatorTests.cs similarity index 70% rename from src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs rename to src/StellaOps.Concelier.Connector.Common.Tests/Xml/XmlSchemaValidatorTests.cs index 5a2ab89f..fbaf10b9 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/Xml/XmlSchemaValidatorTests.cs +++ b/src/StellaOps.Concelier.Connector.Common.Tests/Xml/XmlSchemaValidatorTests.cs @@ -3,10 +3,10 @@ using System.Xml; using System.Xml.Linq; using System.Xml.Schema; using Microsoft.Extensions.Logging.Abstractions; -using FeedserXmlSchemaValidator = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidator; -using FeedserXmlSchemaValidationException = StellaOps.Feedser.Source.Common.Xml.XmlSchemaValidationException; +using ConcelierXmlSchemaValidator = StellaOps.Concelier.Connector.Common.Xml.XmlSchemaValidator; +using ConcelierXmlSchemaValidationException = StellaOps.Concelier.Connector.Common.Xml.XmlSchemaValidationException; -namespace StellaOps.Feedser.Source.Common.Tests.Xml; +namespace StellaOps.Concelier.Connector.Common.Tests.Xml; public sealed class XmlSchemaValidatorTests { @@ -35,7 +35,7 @@ public sealed class XmlSchemaValidatorTests { var schemaSet = CreateSchema(); var document = XDocument.Parse("<root><id>abc</id><count>3</count></root>"); - var validator = new FeedserXmlSchemaValidator(NullLogger<FeedserXmlSchemaValidator>.Instance); + var validator = new ConcelierXmlSchemaValidator(NullLogger<ConcelierXmlSchemaValidator>.Instance); var exception = Record.Exception(() => validator.Validate(document, schemaSet, "valid.xml")); @@ -47,9 +47,9 @@ public sealed class XmlSchemaValidatorTests { var schemaSet = CreateSchema(); var document = XDocument.Parse("<root><id>missing-count</id></root>"); - var validator = new FeedserXmlSchemaValidator(NullLogger<FeedserXmlSchemaValidator>.Instance); + var validator = new ConcelierXmlSchemaValidator(NullLogger<ConcelierXmlSchemaValidator>.Instance); - var ex = Assert.Throws<FeedserXmlSchemaValidationException>(() => validator.Validate(document, schemaSet, "invalid.xml")); + var ex = Assert.Throws<ConcelierXmlSchemaValidationException>(() => validator.Validate(document, schemaSet, "invalid.xml")); Assert.Equal("invalid.xml", ex.DocumentName); Assert.NotEmpty(ex.Errors); diff --git a/src/StellaOps.Feedser.Source.Common/AGENTS.md b/src/StellaOps.Concelier.Connector.Common/AGENTS.md similarity index 83% rename from src/StellaOps.Feedser.Source.Common/AGENTS.md rename to src/StellaOps.Concelier.Connector.Common/AGENTS.md index 582d2bc2..52be2af8 100644 --- a/src/StellaOps.Feedser.Source.Common/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Common/AGENTS.md @@ -22,10 +22,10 @@ Shared connector toolkit. Provides HTTP clients, retry/backoff, conditional GET In: HTTP plumbing, validators, cursor/backoff utilities, hashing. Out: connector-specific schemas/mapping rules, merge precedence. ## Observability & security expectations -- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged with `feedser.source=<connector>` plus retries/failures; connector dashboards slice on that tag instead of bespoke metric names. +- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms tagged with `concelier.source=<connector>` plus retries/failures; connector dashboards slice on that tag instead of bespoke metric names. - Logs include uri, status, retries, etag; redact tokens and auth headers. - Distributed tracing hooks and per-connector counters should be wired centrally for consistent observability. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Common.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Common.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs b/src/StellaOps.Concelier.Connector.Common/Cursors/PaginationPlanner.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs rename to src/StellaOps.Concelier.Connector.Common/Cursors/PaginationPlanner.cs index b1eec949..2b49bac5 100644 --- a/src/StellaOps.Feedser.Source.Common/Cursors/PaginationPlanner.cs +++ b/src/StellaOps.Concelier.Connector.Common/Cursors/PaginationPlanner.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Cursors; +namespace StellaOps.Concelier.Connector.Common.Cursors; /// <summary> /// Provides helpers for computing pagination start indices for sources that expose total result counts. diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs rename to src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorOptions.cs index a33c7216..a256db10 100644 --- a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorOptions.cs +++ b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorOptions.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Cursors; +namespace StellaOps.Concelier.Connector.Common.Cursors; /// <summary> /// Configuration applied when advancing sliding time-window cursors. diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorPlanner.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs rename to src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorPlanner.cs index 10b1b357..2c0941d5 100644 --- a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorPlanner.cs +++ b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorPlanner.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Cursors; +namespace StellaOps.Concelier.Connector.Common.Cursors; /// <summary> /// Utility methods for computing sliding time-window ranges used by connectors. diff --git a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs rename to src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs index 5c835eb0..cd664936 100644 --- a/src/StellaOps.Feedser.Source.Common/Cursors/TimeWindowCursorState.cs +++ b/src/StellaOps.Concelier.Connector.Common/Cursors/TimeWindowCursorState.cs @@ -1,6 +1,6 @@ using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Common.Cursors; +namespace StellaOps.Concelier.Connector.Common.Cursors; /// <summary> /// Represents the persisted state of a sliding time-window cursor. diff --git a/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs b/src/StellaOps.Concelier.Connector.Common/DocumentStatuses.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs rename to src/StellaOps.Concelier.Connector.Common/DocumentStatuses.cs index 6d265643..1207c2b3 100644 --- a/src/StellaOps.Feedser.Source.Common/DocumentStatuses.cs +++ b/src/StellaOps.Concelier.Connector.Common/DocumentStatuses.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common; +namespace StellaOps.Concelier.Connector.Common; /// <summary> /// Well-known lifecycle statuses for raw source documents as they move through fetch/parse/map stages. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/CryptoJitterSource.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/CryptoJitterSource.cs index 1c798550..1110fa8c 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/CryptoJitterSource.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/CryptoJitterSource.cs @@ -1,6 +1,6 @@ using System.Security.Cryptography; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Jitter source backed by <see cref="RandomNumberGenerator"/> for thread-safe, high-entropy delays. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/IJitterSource.cs similarity index 75% rename from src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/IJitterSource.cs index 6284e1b2..ff2874e6 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/IJitterSource.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/IJitterSource.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Produces random jitter durations used to decorrelate retries. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/RawDocumentStorage.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/RawDocumentStorage.cs index a87d4dc2..0590e201 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/RawDocumentStorage.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/RawDocumentStorage.cs @@ -2,7 +2,7 @@ using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.GridFS; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Handles persistence of raw upstream documents in GridFS buckets for later parsing. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchContentResult.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchContentResult.cs index d4106131..f1cd1dbd 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchContentResult.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchContentResult.cs @@ -1,6 +1,6 @@ using System.Net; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Result of fetching raw response content without persisting a document. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchRequest.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchRequest.cs index 1fed7047..4d4b0bac 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchRequest.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchRequest.cs @@ -1,7 +1,7 @@ using System.Collections.Generic; using System.Net.Http; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Parameters describing a fetch operation for a source connector. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchResult.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchResult.cs index a1114097..417e07ee 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchResult.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchResult.cs @@ -1,7 +1,7 @@ using System.Net; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Outcome of fetching a raw document from an upstream source. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs index c0217c54..43c0d292 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/SourceFetchService.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceFetchService.cs @@ -9,12 +9,12 @@ using System.Text; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Telemetry; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Telemetry; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Executes HTTP fetches for connectors, capturing raw responses with metadata for downstream stages. diff --git a/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs similarity index 98% rename from src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs rename to src/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs index d8105785..b6c4bd63 100644 --- a/src/StellaOps.Feedser.Source.Common/Fetch/SourceRetryPolicy.cs +++ b/src/StellaOps.Concelier.Connector.Common/Fetch/SourceRetryPolicy.cs @@ -1,7 +1,7 @@ using System.Globalization; using System.Net; -namespace StellaOps.Feedser.Source.Common.Fetch; +namespace StellaOps.Concelier.Connector.Common.Fetch; /// <summary> /// Provides retry/backoff behavior for source HTTP fetches. diff --git a/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs b/src/StellaOps.Concelier.Connector.Common/Html/HtmlContentSanitizer.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs rename to src/StellaOps.Concelier.Connector.Common/Html/HtmlContentSanitizer.cs index 15d5a8e2..bb0144ec 100644 --- a/src/StellaOps.Feedser.Source.Common/Html/HtmlContentSanitizer.cs +++ b/src/StellaOps.Concelier.Connector.Common/Html/HtmlContentSanitizer.cs @@ -1,9 +1,9 @@ using System.Linq; using AngleSharp.Dom; using AngleSharp.Html.Parser; -using StellaOps.Feedser.Source.Common.Url; +using StellaOps.Concelier.Connector.Common.Url; -namespace StellaOps.Feedser.Source.Common.Html; +namespace StellaOps.Concelier.Connector.Common.Html; /// <summary> /// Sanitizes untrusted HTML fragments produced by upstream advisories. diff --git a/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs b/src/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs rename to src/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs index 327cab59..5cb1dc10 100644 --- a/src/StellaOps.Feedser.Source.Common/Http/AllowlistedHttpMessageHandler.cs +++ b/src/StellaOps.Concelier.Connector.Common/Http/AllowlistedHttpMessageHandler.cs @@ -1,6 +1,6 @@ using System.Net.Http.Headers; -namespace StellaOps.Feedser.Source.Common.Http; +namespace StellaOps.Concelier.Connector.Common.Http; /// <summary> /// Delegating handler that enforces an allowlist of destination hosts for outbound requests. diff --git a/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Common/Http/ServiceCollectionExtensions.cs similarity index 98% rename from src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Common/Http/ServiceCollectionExtensions.cs index eea1eb7f..255b3f95 100644 --- a/src/StellaOps.Feedser.Source.Common/Http/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Common/Http/ServiceCollectionExtensions.cs @@ -4,9 +4,9 @@ using System.Net.Security; using System.Security.Cryptography.X509Certificates; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Xml; +using StellaOps.Concelier.Connector.Common.Xml; -namespace StellaOps.Feedser.Source.Common.Http; +namespace StellaOps.Concelier.Connector.Common.Http; public static class ServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientConfigurationBinder.cs b/src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientConfigurationBinder.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientConfigurationBinder.cs rename to src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientConfigurationBinder.cs index f4258832..4b2a4b9a 100644 --- a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientConfigurationBinder.cs +++ b/src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientConfigurationBinder.cs @@ -1,360 +1,360 @@ -using System.Collections.Generic; -using System.Linq; -using System.Globalization; -using System.IO; -using System.Net.Security; -using System.Security.Cryptography; -using System.Security.Cryptography.X509Certificates; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace StellaOps.Feedser.Source.Common.Http; - -internal static class SourceHttpClientConfigurationBinder -{ - private const string FeedserSection = "feedser"; - private const string HttpClientsSection = "httpClients"; - private const string SourcesSection = "sources"; - private const string HttpSection = "http"; - private const string AllowInvalidKey = "allowInvalidCertificates"; - private const string TrustedRootPathsKey = "trustedRootPaths"; - private const string ProxySection = "proxy"; - private const string ProxyAddressKey = "address"; - private const string ProxyBypassOnLocalKey = "bypassOnLocal"; - private const string ProxyBypassListKey = "bypassList"; - private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials"; - private const string ProxyUsernameKey = "username"; - private const string ProxyPasswordKey = "password"; - private const string OfflineRootKey = "offlineRoot"; - private const string OfflineRootEnvironmentVariable = "FEEDSER_OFFLINE_ROOT"; - - public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options) - { - var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration; - if (configuration is null) - { - return; - } - - var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory; - var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration"); - - var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment; - - var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase); - foreach (var section in EnumerateCandidateSections(configuration, clientName)) - { - if (section is null || !section.Exists() || !processed.Add(section.Path)) - { - continue; - } - - ApplySection(section, configuration, hostEnvironment, clientName, options, logger); - } - } - - private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName) - { - var names = BuildCandidateNames(clientName); - foreach (var name in names) - { - var httpClientSection = GetSection(configuration, FeedserSection, HttpClientsSection, name); - if (httpClientSection is not null && httpClientSection.Exists()) - { - yield return httpClientSection; - } - - var sourceHttpSection = GetSection(configuration, FeedserSection, SourcesSection, name, HttpSection); - if (sourceHttpSection is not null && sourceHttpSection.Exists()) - { - yield return sourceHttpSection; - } - } - } - - private static IEnumerable<string> BuildCandidateNames(string clientName) - { - yield return clientName; - - if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length) - { - yield return clientName["source.".Length..]; - } - - var noDots = clientName.Replace('.', '_'); - if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase)) - { - yield return noDots; - } - } - - private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments) - { - IConfiguration? current = configuration; - foreach (var segment in pathSegments) - { - if (current is null) - { - return null; - } - - current = current.GetSection(segment); - } - - return current as IConfigurationSection; - } - - private static void ApplySection( - IConfigurationSection section, - IConfiguration rootConfiguration, - IHostEnvironment? hostEnvironment, - string clientName, - SourceHttpClientOptions options, - ILogger? logger) - { - var allowInvalid = section.GetValue<bool?>(AllowInvalidKey); - if (allowInvalid == true) - { - options.AllowInvalidServerCertificates = true; - var previous = options.ServerCertificateCustomValidation; - options.ServerCertificateCustomValidation = (certificate, chain, errors) => - { - if (allowInvalid == true) - { - return true; - } - - return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None; - }; - - logger?.LogWarning( - "Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.", - clientName); - } - - var offlineRoot = section.GetValue<string?>(OfflineRootKey) - ?? rootConfiguration.GetSection(FeedserSection).GetValue<string?>(OfflineRootKey) - ?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable); - - ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger); - ApplyProxyConfiguration(section, clientName, options, logger); - } - - private static void ApplyTrustedRoots( - IConfigurationSection section, - string? offlineRoot, - IHostEnvironment? hostEnvironment, - string clientName, - SourceHttpClientOptions options, - ILogger? logger) - { - var trustedRootSection = section.GetSection(TrustedRootPathsKey); - if (!trustedRootSection.Exists()) - { - return; - } - - var paths = trustedRootSection.Get<string[]?>(); - if (paths is null || paths.Length == 0) - { - return; - } - - foreach (var rawPath in paths) - { - if (string.IsNullOrWhiteSpace(rawPath)) - { - continue; - } - - var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment); - if (!File.Exists(resolvedPath)) - { - var message = string.Format( - CultureInfo.InvariantCulture, - "Trusted root certificate '{0}' resolved to '{1}' but was not found.", - rawPath, - resolvedPath); - throw new FileNotFoundException(message, resolvedPath); - } - - foreach (var certificate in LoadCertificates(resolvedPath)) - { - try - { - AddTrustedCertificate(options, certificate); - logger?.LogInformation( - "Source HTTP client '{ClientName}' loaded trusted root certificate '{Thumbprint}' from '{Path}'.", - clientName, - certificate.Thumbprint, - resolvedPath); - } - finally - { - certificate.Dispose(); - } - } - } - } - - private static void ApplyProxyConfiguration( - IConfigurationSection section, - string clientName, - SourceHttpClientOptions options, - ILogger? logger) - { - var proxySection = section.GetSection(ProxySection); - if (!proxySection.Exists()) - { - return; - } - - var address = proxySection.GetValue<string?>(ProxyAddressKey); - if (!string.IsNullOrWhiteSpace(address)) - { - if (Uri.TryCreate(address, UriKind.Absolute, out var uri)) - { - options.ProxyAddress = uri; - } - else - { - logger?.LogWarning( - "Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.", - clientName, - address); - } - } - - var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey); - if (bypassOnLocal.HasValue) - { - options.ProxyBypassOnLocal = bypassOnLocal.Value; - } - - var bypassListSection = proxySection.GetSection(ProxyBypassListKey); - if (bypassListSection.Exists()) - { - var entries = bypassListSection.Get<string[]?>(); - options.ProxyBypassList.Clear(); - if (entries is not null) - { - foreach (var entry in entries) - { - if (!string.IsNullOrWhiteSpace(entry)) - { - options.ProxyBypassList.Add(entry.Trim()); - } - } - } - } - - var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey); - if (useDefaultCredentials.HasValue) - { - options.ProxyUseDefaultCredentials = useDefaultCredentials.Value; - } - - var username = proxySection.GetValue<string?>(ProxyUsernameKey); - if (!string.IsNullOrWhiteSpace(username)) - { - options.ProxyUsername = username.Trim(); - } - - var password = proxySection.GetValue<string?>(ProxyPasswordKey); - if (!string.IsNullOrWhiteSpace(password)) - { - options.ProxyPassword = password; - } - } - - private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment) - { - if (Path.IsPathRooted(path)) - { - return path; - } - - if (!string.IsNullOrWhiteSpace(offlineRoot)) - { - return Path.GetFullPath(Path.Combine(offlineRoot!, path)); - } - - var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory; - return Path.GetFullPath(Path.Combine(baseDirectory, path)); - } - - private static IEnumerable<X509Certificate2> LoadCertificates(string path) - { - var certificates = new List<X509Certificate2>(); - var extension = Path.GetExtension(path); - - if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase)) - { - var collection = new X509Certificate2Collection(); - try - { - collection.ImportFromPemFile(path); - } - catch (CryptographicException) - { - collection.Clear(); - } - - if (collection.Count > 0) - { - foreach (var certificate in collection) - { - certificates.Add(certificate.CopyWithPrivateKeyIfAvailable()); - } - } - else - { - certificates.Add(X509Certificate2.CreateFromPemFile(path)); - } - } - else - { - // Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.) - var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12( - File.ReadAllBytes(path), - password: null); - certificates.Add(certificate); - } - - return certificates; - } - - private static void AddTrustedCertificate(SourceHttpClientOptions options, X509Certificate2 certificate) - { - if (certificate is null) - { - return; - } - - if (options.TrustedRootCertificates.Any(existing => - string.Equals(existing.Thumbprint, certificate.Thumbprint, StringComparison.OrdinalIgnoreCase))) - { - return; - } - - options.TrustedRootCertificates.Add(certificate); - } - - // Helper extension method to copy certificate (preserves private key if present) - private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate) - { - // In .NET 9+, use X509CertificateLoader instead of obsolete constructors - if (certificate.HasPrivateKey) - { - // Export with private key and re-import using X509CertificateLoader - var exported = certificate.Export(X509ContentType.Pkcs12); - return X509CertificateLoader.LoadPkcs12(exported, password: null); - } - else - { - // For certificates without private keys, load from raw data - return X509CertificateLoader.LoadCertificate(certificate.RawData); - } - } -} +using System.Collections.Generic; +using System.Linq; +using System.Globalization; +using System.IO; +using System.Net.Security; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Concelier.Connector.Common.Http; + +internal static class SourceHttpClientConfigurationBinder +{ + private const string ConcelierSection = "concelier"; + private const string HttpClientsSection = "httpClients"; + private const string SourcesSection = "sources"; + private const string HttpSection = "http"; + private const string AllowInvalidKey = "allowInvalidCertificates"; + private const string TrustedRootPathsKey = "trustedRootPaths"; + private const string ProxySection = "proxy"; + private const string ProxyAddressKey = "address"; + private const string ProxyBypassOnLocalKey = "bypassOnLocal"; + private const string ProxyBypassListKey = "bypassList"; + private const string ProxyUseDefaultCredentialsKey = "useDefaultCredentials"; + private const string ProxyUsernameKey = "username"; + private const string ProxyPasswordKey = "password"; + private const string OfflineRootKey = "offlineRoot"; + private const string OfflineRootEnvironmentVariable = "CONCELIER_OFFLINE_ROOT"; + + public static void Apply(IServiceProvider services, string clientName, SourceHttpClientOptions options) + { + var configuration = services.GetService(typeof(IConfiguration)) as IConfiguration; + if (configuration is null) + { + return; + } + + var loggerFactory = services.GetService(typeof(ILoggerFactory)) as ILoggerFactory; + var logger = loggerFactory?.CreateLogger("SourceHttpClientConfiguration"); + + var hostEnvironment = services.GetService(typeof(IHostEnvironment)) as IHostEnvironment; + + var processed = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + foreach (var section in EnumerateCandidateSections(configuration, clientName)) + { + if (section is null || !section.Exists() || !processed.Add(section.Path)) + { + continue; + } + + ApplySection(section, configuration, hostEnvironment, clientName, options, logger); + } + } + + private static IEnumerable<IConfigurationSection> EnumerateCandidateSections(IConfiguration configuration, string clientName) + { + var names = BuildCandidateNames(clientName); + foreach (var name in names) + { + var httpClientSection = GetSection(configuration, ConcelierSection, HttpClientsSection, name); + if (httpClientSection is not null && httpClientSection.Exists()) + { + yield return httpClientSection; + } + + var sourceHttpSection = GetSection(configuration, ConcelierSection, SourcesSection, name, HttpSection); + if (sourceHttpSection is not null && sourceHttpSection.Exists()) + { + yield return sourceHttpSection; + } + } + } + + private static IEnumerable<string> BuildCandidateNames(string clientName) + { + yield return clientName; + + if (clientName.StartsWith("source.", StringComparison.OrdinalIgnoreCase) && clientName.Length > "source.".Length) + { + yield return clientName["source.".Length..]; + } + + var noDots = clientName.Replace('.', '_'); + if (!string.Equals(noDots, clientName, StringComparison.OrdinalIgnoreCase)) + { + yield return noDots; + } + } + + private static IConfigurationSection? GetSection(IConfiguration configuration, params string[] pathSegments) + { + IConfiguration? current = configuration; + foreach (var segment in pathSegments) + { + if (current is null) + { + return null; + } + + current = current.GetSection(segment); + } + + return current as IConfigurationSection; + } + + private static void ApplySection( + IConfigurationSection section, + IConfiguration rootConfiguration, + IHostEnvironment? hostEnvironment, + string clientName, + SourceHttpClientOptions options, + ILogger? logger) + { + var allowInvalid = section.GetValue<bool?>(AllowInvalidKey); + if (allowInvalid == true) + { + options.AllowInvalidServerCertificates = true; + var previous = options.ServerCertificateCustomValidation; + options.ServerCertificateCustomValidation = (certificate, chain, errors) => + { + if (allowInvalid == true) + { + return true; + } + + return previous?.Invoke(certificate, chain, errors) ?? errors == SslPolicyErrors.None; + }; + + logger?.LogWarning( + "Source HTTP client '{ClientName}' is configured to bypass TLS certificate validation.", + clientName); + } + + var offlineRoot = section.GetValue<string?>(OfflineRootKey) + ?? rootConfiguration.GetSection(ConcelierSection).GetValue<string?>(OfflineRootKey) + ?? Environment.GetEnvironmentVariable(OfflineRootEnvironmentVariable); + + ApplyTrustedRoots(section, offlineRoot, hostEnvironment, clientName, options, logger); + ApplyProxyConfiguration(section, clientName, options, logger); + } + + private static void ApplyTrustedRoots( + IConfigurationSection section, + string? offlineRoot, + IHostEnvironment? hostEnvironment, + string clientName, + SourceHttpClientOptions options, + ILogger? logger) + { + var trustedRootSection = section.GetSection(TrustedRootPathsKey); + if (!trustedRootSection.Exists()) + { + return; + } + + var paths = trustedRootSection.Get<string[]?>(); + if (paths is null || paths.Length == 0) + { + return; + } + + foreach (var rawPath in paths) + { + if (string.IsNullOrWhiteSpace(rawPath)) + { + continue; + } + + var resolvedPath = ResolvePath(rawPath, offlineRoot, hostEnvironment); + if (!File.Exists(resolvedPath)) + { + var message = string.Format( + CultureInfo.InvariantCulture, + "Trusted root certificate '{0}' resolved to '{1}' but was not found.", + rawPath, + resolvedPath); + throw new FileNotFoundException(message, resolvedPath); + } + + foreach (var certificate in LoadCertificates(resolvedPath)) + { + try + { + AddTrustedCertificate(options, certificate); + logger?.LogInformation( + "Source HTTP client '{ClientName}' loaded trusted root certificate '{Thumbprint}' from '{Path}'.", + clientName, + certificate.Thumbprint, + resolvedPath); + } + finally + { + certificate.Dispose(); + } + } + } + } + + private static void ApplyProxyConfiguration( + IConfigurationSection section, + string clientName, + SourceHttpClientOptions options, + ILogger? logger) + { + var proxySection = section.GetSection(ProxySection); + if (!proxySection.Exists()) + { + return; + } + + var address = proxySection.GetValue<string?>(ProxyAddressKey); + if (!string.IsNullOrWhiteSpace(address)) + { + if (Uri.TryCreate(address, UriKind.Absolute, out var uri)) + { + options.ProxyAddress = uri; + } + else + { + logger?.LogWarning( + "Source HTTP client '{ClientName}' has invalid proxy address '{ProxyAddress}'.", + clientName, + address); + } + } + + var bypassOnLocal = proxySection.GetValue<bool?>(ProxyBypassOnLocalKey); + if (bypassOnLocal.HasValue) + { + options.ProxyBypassOnLocal = bypassOnLocal.Value; + } + + var bypassListSection = proxySection.GetSection(ProxyBypassListKey); + if (bypassListSection.Exists()) + { + var entries = bypassListSection.Get<string[]?>(); + options.ProxyBypassList.Clear(); + if (entries is not null) + { + foreach (var entry in entries) + { + if (!string.IsNullOrWhiteSpace(entry)) + { + options.ProxyBypassList.Add(entry.Trim()); + } + } + } + } + + var useDefaultCredentials = proxySection.GetValue<bool?>(ProxyUseDefaultCredentialsKey); + if (useDefaultCredentials.HasValue) + { + options.ProxyUseDefaultCredentials = useDefaultCredentials.Value; + } + + var username = proxySection.GetValue<string?>(ProxyUsernameKey); + if (!string.IsNullOrWhiteSpace(username)) + { + options.ProxyUsername = username.Trim(); + } + + var password = proxySection.GetValue<string?>(ProxyPasswordKey); + if (!string.IsNullOrWhiteSpace(password)) + { + options.ProxyPassword = password; + } + } + + private static string ResolvePath(string path, string? offlineRoot, IHostEnvironment? hostEnvironment) + { + if (Path.IsPathRooted(path)) + { + return path; + } + + if (!string.IsNullOrWhiteSpace(offlineRoot)) + { + return Path.GetFullPath(Path.Combine(offlineRoot!, path)); + } + + var baseDirectory = hostEnvironment?.ContentRootPath ?? AppContext.BaseDirectory; + return Path.GetFullPath(Path.Combine(baseDirectory, path)); + } + + private static IEnumerable<X509Certificate2> LoadCertificates(string path) + { + var certificates = new List<X509Certificate2>(); + var extension = Path.GetExtension(path); + + if (extension.Equals(".pem", StringComparison.OrdinalIgnoreCase) || extension.Equals(".crt", StringComparison.OrdinalIgnoreCase)) + { + var collection = new X509Certificate2Collection(); + try + { + collection.ImportFromPemFile(path); + } + catch (CryptographicException) + { + collection.Clear(); + } + + if (collection.Count > 0) + { + foreach (var certificate in collection) + { + certificates.Add(certificate.CopyWithPrivateKeyIfAvailable()); + } + } + else + { + certificates.Add(X509Certificate2.CreateFromPemFile(path)); + } + } + else + { + // Use X509CertificateLoader to load certificates from PKCS#12 files (.pfx, .p12, etc.) + var certificate = System.Security.Cryptography.X509Certificates.X509CertificateLoader.LoadPkcs12( + File.ReadAllBytes(path), + password: null); + certificates.Add(certificate); + } + + return certificates; + } + + private static void AddTrustedCertificate(SourceHttpClientOptions options, X509Certificate2 certificate) + { + if (certificate is null) + { + return; + } + + if (options.TrustedRootCertificates.Any(existing => + string.Equals(existing.Thumbprint, certificate.Thumbprint, StringComparison.OrdinalIgnoreCase))) + { + return; + } + + options.TrustedRootCertificates.Add(certificate); + } + + // Helper extension method to copy certificate (preserves private key if present) + private static X509Certificate2 CopyWithPrivateKeyIfAvailable(this X509Certificate2 certificate) + { + // In .NET 9+, use X509CertificateLoader instead of obsolete constructors + if (certificate.HasPrivateKey) + { + // Export with private key and re-import using X509CertificateLoader + var exported = certificate.Export(X509ContentType.Pkcs12); + return X509CertificateLoader.LoadPkcs12(exported, password: null); + } + else + { + // For certificates without private keys, load from raw data + return X509CertificateLoader.LoadCertificate(certificate.RawData); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs b/src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientOptions.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs rename to src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientOptions.cs index b9297d04..8b5958cd 100644 --- a/src/StellaOps.Feedser.Source.Common/Http/SourceHttpClientOptions.cs +++ b/src/StellaOps.Concelier.Connector.Common/Http/SourceHttpClientOptions.cs @@ -4,7 +4,7 @@ using System.Net.Http; using System.Net.Security; using System.Security.Cryptography.X509Certificates; -namespace StellaOps.Feedser.Source.Common.Http; +namespace StellaOps.Concelier.Connector.Common.Http; /// <summary> /// Configuration applied to named HTTP clients used by connectors. @@ -27,7 +27,7 @@ public sealed class SourceHttpClientOptions /// <summary> /// Gets or sets the user-agent string applied to outgoing requests. /// </summary> - public string UserAgent { get; set; } = "StellaOps.Feedser/1.0"; + public string UserAgent { get; set; } = "StellaOps.Concelier/1.0"; /// <summary> /// Gets or sets whether redirects are allowed. Defaults to <c>true</c>. diff --git a/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs b/src/StellaOps.Concelier.Connector.Common/Json/IJsonSchemaValidator.cs similarity index 73% rename from src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs rename to src/StellaOps.Concelier.Connector.Common/Json/IJsonSchemaValidator.cs index 47317d91..c74df3a2 100644 --- a/src/StellaOps.Feedser.Source.Common/Json/IJsonSchemaValidator.cs +++ b/src/StellaOps.Concelier.Connector.Common/Json/IJsonSchemaValidator.cs @@ -1,7 +1,7 @@ using System.Text.Json; using Json.Schema; -namespace StellaOps.Feedser.Source.Common.Json; +namespace StellaOps.Concelier.Connector.Common.Json; public interface IJsonSchemaValidator { diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationError.cs similarity index 70% rename from src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs rename to src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationError.cs index f0de47f2..08da59c1 100644 --- a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationError.cs +++ b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationError.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Json; +namespace StellaOps.Concelier.Connector.Common.Json; public sealed record JsonSchemaValidationError( string InstanceLocation, diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationException.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs rename to src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationException.cs index c3ee6b2e..2451e89d 100644 --- a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidationException.cs +++ b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidationException.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Json; +namespace StellaOps.Concelier.Connector.Common.Json; public sealed class JsonSchemaValidationException : Exception { diff --git a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidator.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs rename to src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidator.cs index 98408bf5..cf2e9d9d 100644 --- a/src/StellaOps.Feedser.Source.Common/Json/JsonSchemaValidator.cs +++ b/src/StellaOps.Concelier.Connector.Common/Json/JsonSchemaValidator.cs @@ -4,7 +4,7 @@ using System.Text.Json; using Json.Schema; using Microsoft.Extensions.Logging; -namespace StellaOps.Feedser.Source.Common.Json; +namespace StellaOps.Concelier.Connector.Common.Json; public sealed class JsonSchemaValidator : IJsonSchemaValidator { private readonly ILogger<JsonSchemaValidator> _logger; diff --git a/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs b/src/StellaOps.Concelier.Connector.Common/Packages/PackageCoordinateHelper.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs rename to src/StellaOps.Concelier.Connector.Common/Packages/PackageCoordinateHelper.cs index 7505f55d..953f8e91 100644 --- a/src/StellaOps.Feedser.Source.Common/Packages/PackageCoordinateHelper.cs +++ b/src/StellaOps.Concelier.Connector.Common/Packages/PackageCoordinateHelper.cs @@ -1,9 +1,9 @@ using System.Linq; using System.Text; using NuGet.Versioning; -using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Identifiers; -namespace StellaOps.Feedser.Source.Common.Packages; +namespace StellaOps.Concelier.Connector.Common.Packages; /// <summary> /// Shared helpers for working with Package URLs and SemVer coordinates inside connectors. diff --git a/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs b/src/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs rename to src/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs index a36d98de..5a6d50ef 100644 --- a/src/StellaOps.Feedser.Source.Common/Pdf/PdfTextExtractor.cs +++ b/src/StellaOps.Concelier.Connector.Common/Pdf/PdfTextExtractor.cs @@ -6,7 +6,7 @@ using System.Text; using UglyToad.PdfPig; using UglyToad.PdfPig.Content; -namespace StellaOps.Feedser.Source.Common.Pdf; +namespace StellaOps.Concelier.Connector.Common.Pdf; /// <summary> /// Extracts text from PDF advisories using UglyToad.PdfPig without requiring native dependencies. diff --git a/src/StellaOps.Concelier.Connector.Common/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Common/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..b3710fdc --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Common/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Common.Tests")] diff --git a/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedModels.cs b/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedModels.cs new file mode 100644 index 00000000..295cebeb --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedModels.cs @@ -0,0 +1,159 @@ +using StellaOps.Concelier.Connector.Common; + +namespace StellaOps.Concelier.Connector.Common.State; + +/// <summary> +/// Describes a raw upstream document that should be persisted for a connector during seeding. +/// </summary> +public sealed record SourceStateSeedDocument +{ + /// <summary> + /// Absolute source URI. Must match the connector's upstream document identifier. + /// </summary> + public string Uri { get; init; } = string.Empty; + + /// <summary> + /// Raw document payload. Required when creating or replacing a document. + /// </summary> + public byte[] Content { get; init; } = Array.Empty<byte>(); + + /// <summary> + /// Optional explicit document identifier. When provided it overrides auto-generated IDs. + /// </summary> + public Guid? DocumentId { get; init; } + + /// <summary> + /// MIME type for the document payload. + /// </summary> + public string? ContentType { get; init; } + + /// <summary> + /// Status assigned to the document. Defaults to <see cref="DocumentStatuses.PendingParse"/>. + /// </summary> + public string Status { get; init; } = DocumentStatuses.PendingParse; + + /// <summary> + /// Optional HTTP-style headers persisted alongside the raw document. + /// </summary> + public IReadOnlyDictionary<string, string>? Headers { get; init; } + + /// <summary> + /// Source metadata (connector specific) persisted alongside the raw document. + /// </summary> + public IReadOnlyDictionary<string, string>? Metadata { get; init; } + + /// <summary> + /// Upstream ETag value, if available. + /// </summary> + public string? Etag { get; init; } + + /// <summary> + /// Upstream last-modified timestamp, if available. + /// </summary> + public DateTimeOffset? LastModified { get; init; } + + /// <summary> + /// Optional document expiration. When set a TTL will purge the raw payload after the configured retention. + /// </summary> + public DateTimeOffset? ExpiresAt { get; init; } + + /// <summary> + /// Fetch timestamp stamped onto the document. Defaults to the seed completion timestamp. + /// </summary> + public DateTimeOffset? FetchedAt { get; init; } + + /// <summary> + /// When true, the document ID will be appended to the connector cursor's <c>pendingDocuments</c> set. + /// </summary> + public bool AddToPendingDocuments { get; init; } = true; + + /// <summary> + /// When true, the document ID will be appended to the connector cursor's <c>pendingMappings</c> set. + /// </summary> + public bool AddToPendingMappings { get; init; } + + /// <summary> + /// Optional identifiers that should be recorded on the cursor to avoid duplicate ingestion. + /// </summary> + public IReadOnlyCollection<string>? KnownIdentifiers { get; init; } +} + +/// <summary> +/// Cursor updates that should accompany seeded documents. +/// </summary> +public sealed record SourceStateSeedCursor +{ + /// <summary> + /// Optional <c>pendingDocuments</c> additions expressed as document IDs. + /// </summary> + public IReadOnlyCollection<Guid>? PendingDocuments { get; init; } + + /// <summary> + /// Optional <c>pendingMappings</c> additions expressed as document IDs. + /// </summary> + public IReadOnlyCollection<Guid>? PendingMappings { get; init; } + + /// <summary> + /// Optional known advisory identifiers to merge with the cursor. + /// </summary> + public IReadOnlyCollection<string>? KnownAdvisories { get; init; } + + /// <summary> + /// Upstream window watermark tracked by connectors that rely on last-modified cursors. + /// </summary> + public DateTimeOffset? LastModifiedCursor { get; init; } + + /// <summary> + /// Optional fetch timestamp used by connectors that track the last polling instant. + /// </summary> + public DateTimeOffset? LastFetchAt { get; init; } + + /// <summary> + /// Additional cursor fields (string values) to merge. + /// </summary> + public IReadOnlyDictionary<string, string>? Additional { get; init; } +} + +/// <summary> +/// Seeding specification describing the source, documents, and cursor edits to apply. +/// </summary> +public sealed record SourceStateSeedSpecification +{ + /// <summary> + /// Source/connector name (e.g. <c>vndr.msrc</c>). + /// </summary> + public string Source { get; init; } = string.Empty; + + /// <summary> + /// Documents that should be inserted or replaced before the cursor update. + /// </summary> + public IReadOnlyList<SourceStateSeedDocument> Documents { get; init; } = Array.Empty<SourceStateSeedDocument>(); + + /// <summary> + /// Cursor adjustments applied after documents are persisted. + /// </summary> + public SourceStateSeedCursor? Cursor { get; init; } + + /// <summary> + /// Connector-level known advisory identifiers to merge into the cursor. + /// </summary> + public IReadOnlyCollection<string>? KnownAdvisories { get; init; } + + /// <summary> + /// Optional completion timestamp. Defaults to the processor's time provider. + /// </summary> + public DateTimeOffset? CompletedAt { get; init; } +} + +/// <summary> +/// Result returned after seeding completes. +/// </summary> +public sealed record SourceStateSeedResult( + int DocumentsProcessed, + int PendingDocumentsAdded, + int PendingMappingsAdded, + IReadOnlyCollection<Guid> DocumentIds, + IReadOnlyCollection<Guid> PendingDocumentIds, + IReadOnlyCollection<Guid> PendingMappingIds, + IReadOnlyCollection<string> KnownAdvisoriesAdded, + DateTimeOffset CompletedAt); diff --git a/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs b/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs new file mode 100644 index 00000000..fa747cff --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Common/State/SourceStateSeedProcessor.cs @@ -0,0 +1,329 @@ +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Common.State; + +/// <summary> +/// Persists raw documents and cursor state for connectors that require manual seeding. +/// </summary> +public sealed class SourceStateSeedProcessor +{ + private readonly IDocumentStore _documentStore; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly ISourceStateRepository _stateRepository; + private readonly TimeProvider _timeProvider; + private readonly ILogger<SourceStateSeedProcessor> _logger; + + public SourceStateSeedProcessor( + IDocumentStore documentStore, + RawDocumentStorage rawDocumentStorage, + ISourceStateRepository stateRepository, + TimeProvider? timeProvider = null, + ILogger<SourceStateSeedProcessor>? logger = null) + { + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? NullLogger<SourceStateSeedProcessor>.Instance; + } + + public async Task<SourceStateSeedResult> ProcessAsync(SourceStateSeedSpecification specification, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(specification); + ArgumentException.ThrowIfNullOrEmpty(specification.Source); + + var completedAt = specification.CompletedAt ?? _timeProvider.GetUtcNow(); + var documentIds = new List<Guid>(); + var pendingDocumentIds = new HashSet<Guid>(); + var pendingMappingIds = new HashSet<Guid>(); + var knownAdvisories = new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + AppendRange(knownAdvisories, specification.KnownAdvisories); + + if (specification.Cursor is { } cursorSeed) + { + AppendRange(pendingDocumentIds, cursorSeed.PendingDocuments); + AppendRange(pendingMappingIds, cursorSeed.PendingMappings); + AppendRange(knownAdvisories, cursorSeed.KnownAdvisories); + } + + foreach (var document in specification.Documents ?? Array.Empty<SourceStateSeedDocument>()) + { + cancellationToken.ThrowIfCancellationRequested(); + await ProcessDocumentAsync(specification.Source, document, completedAt, documentIds, pendingDocumentIds, pendingMappingIds, knownAdvisories, cancellationToken).ConfigureAwait(false); + } + + var state = await _stateRepository.TryGetAsync(specification.Source, cancellationToken).ConfigureAwait(false); + var cursor = state?.Cursor ?? new BsonDocument(); + + var newlyPendingDocuments = MergeGuidArray(cursor, "pendingDocuments", pendingDocumentIds); + var newlyPendingMappings = MergeGuidArray(cursor, "pendingMappings", pendingMappingIds); + var newlyKnownAdvisories = MergeStringArray(cursor, "knownAdvisories", knownAdvisories); + + if (specification.Cursor is { } cursorSpec) + { + if (cursorSpec.LastModifiedCursor.HasValue) + { + cursor["lastModifiedCursor"] = cursorSpec.LastModifiedCursor.Value.UtcDateTime; + } + + if (cursorSpec.LastFetchAt.HasValue) + { + cursor["lastFetchAt"] = cursorSpec.LastFetchAt.Value.UtcDateTime; + } + + if (cursorSpec.Additional is not null) + { + foreach (var kvp in cursorSpec.Additional) + { + cursor[kvp.Key] = kvp.Value; + } + } + } + + cursor["lastSeededAt"] = completedAt.UtcDateTime; + await _stateRepository.UpdateCursorAsync(specification.Source, cursor, completedAt, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Seeded {Documents} document(s) for {Source}. pendingDocuments+= {PendingDocuments}, pendingMappings+= {PendingMappings}, knownAdvisories+= {KnownAdvisories}", + documentIds.Count, + specification.Source, + newlyPendingDocuments.Count, + newlyPendingMappings.Count, + newlyKnownAdvisories.Count); + + return new SourceStateSeedResult( + DocumentsProcessed: documentIds.Count, + PendingDocumentsAdded: newlyPendingDocuments.Count, + PendingMappingsAdded: newlyPendingMappings.Count, + DocumentIds: documentIds.AsReadOnly(), + PendingDocumentIds: newlyPendingDocuments, + PendingMappingIds: newlyPendingMappings, + KnownAdvisoriesAdded: newlyKnownAdvisories, + CompletedAt: completedAt); + } + + private async Task ProcessDocumentAsync( + string source, + SourceStateSeedDocument document, + DateTimeOffset completedAt, + List<Guid> documentIds, + HashSet<Guid> pendingDocumentIds, + HashSet<Guid> pendingMappingIds, + HashSet<string> knownAdvisories, + CancellationToken cancellationToken) + { + if (document is null) + { + throw new ArgumentNullException(nameof(document)); + } + + ArgumentException.ThrowIfNullOrEmpty(document.Uri); + if (document.Content is not { Length: > 0 }) + { + throw new InvalidOperationException($"Seed entry for '{document.Uri}' is missing content bytes."); + } + + var payload = new byte[document.Content.Length]; + Buffer.BlockCopy(document.Content, 0, payload, 0, document.Content.Length); + + if (!document.Uri.Contains("://", StringComparison.Ordinal)) + { + _logger.LogWarning("Seed document URI '{Uri}' does not appear to be absolute.", document.Uri); + } + + var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + + var existing = await _documentStore.FindBySourceAndUriAsync(source, document.Uri, cancellationToken).ConfigureAwait(false); + + if (existing?.GridFsId is { } oldGridId) + { + await _rawDocumentStorage.DeleteAsync(oldGridId, cancellationToken).ConfigureAwait(false); + } + + var gridId = await _rawDocumentStorage.UploadAsync( + source, + document.Uri, + payload, + document.ContentType, + document.ExpiresAt, + cancellationToken) + .ConfigureAwait(false); + + var headers = CloneDictionary(document.Headers); + if (!string.IsNullOrWhiteSpace(document.ContentType)) + { + headers ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); + if (!headers.ContainsKey("content-type")) + { + headers["content-type"] = document.ContentType!; + } + } + + var metadata = CloneDictionary(document.Metadata); + + var record = new DocumentRecord( + document.DocumentId ?? existing?.Id ?? Guid.NewGuid(), + source, + document.Uri, + document.FetchedAt ?? completedAt, + sha256, + string.IsNullOrWhiteSpace(document.Status) ? DocumentStatuses.PendingParse : document.Status, + document.ContentType, + headers, + metadata, + document.Etag, + document.LastModified, + gridId, + document.ExpiresAt); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + + documentIds.Add(upserted.Id); + + if (document.AddToPendingDocuments) + { + pendingDocumentIds.Add(upserted.Id); + } + + if (document.AddToPendingMappings) + { + pendingMappingIds.Add(upserted.Id); + } + + AppendRange(knownAdvisories, document.KnownIdentifiers); + } + + private static Dictionary<string, string>? CloneDictionary(IReadOnlyDictionary<string, string>? values) + { + if (values is null || values.Count == 0) + { + return null; + } + + return new Dictionary<string, string>(values, StringComparer.OrdinalIgnoreCase); + } + + private static IReadOnlyCollection<Guid> MergeGuidArray(BsonDocument cursor, string field, IReadOnlyCollection<Guid> additions) + { + if (additions.Count == 0) + { + return Array.Empty<Guid>(); + } + + var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray + ? existingArray.Select(AsGuid).Where(static g => g != Guid.Empty).ToHashSet() + : new HashSet<Guid>(); + + var newlyAdded = new List<Guid>(); + foreach (var guid in additions) + { + if (guid == Guid.Empty) + { + continue; + } + + if (existing.Add(guid)) + { + newlyAdded.Add(guid); + } + } + + if (existing.Count > 0) + { + cursor[field] = new BsonArray(existing + .Select(static g => g.ToString("D")) + .OrderBy(static s => s, StringComparer.OrdinalIgnoreCase)); + } + + return newlyAdded.AsReadOnly(); + } + + private static IReadOnlyCollection<string> MergeStringArray(BsonDocument cursor, string field, IReadOnlyCollection<string> additions) + { + if (additions.Count == 0) + { + return Array.Empty<string>(); + } + + var existing = cursor.TryGetValue(field, out var value) && value is BsonArray existingArray + ? existingArray.Select(static v => v?.AsString ?? string.Empty) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .ToHashSet(StringComparer.OrdinalIgnoreCase) + : new HashSet<string>(StringComparer.OrdinalIgnoreCase); + + var newlyAdded = new List<string>(); + foreach (var entry in additions) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var normalized = entry.Trim(); + if (existing.Add(normalized)) + { + newlyAdded.Add(normalized); + } + } + + if (existing.Count > 0) + { + cursor[field] = new BsonArray(existing + .OrderBy(static s => s, StringComparer.OrdinalIgnoreCase)); + } + + return newlyAdded.AsReadOnly(); + } + + private static Guid AsGuid(BsonValue value) + { + if (value is null) + { + return Guid.Empty; + } + + return Guid.TryParse(value.ToString(), out var parsed) ? parsed : Guid.Empty; + } + + private static void AppendRange(HashSet<Guid> target, IReadOnlyCollection<Guid>? values) + { + if (values is null) + { + return; + } + + foreach (var guid in values) + { + if (guid != Guid.Empty) + { + target.Add(guid); + } + } + } + + private static void AppendRange(HashSet<string> target, IReadOnlyCollection<string>? values) + { + if (values is null) + { + return; + } + + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + target.Add(value.Trim()); + } + } + +} diff --git a/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj b/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj similarity index 64% rename from src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj rename to src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj index 5ec42c47..0096d321 100644 --- a/src/StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj +++ b/src/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj @@ -7,15 +7,14 @@ <ItemGroup> <PackageReference Include="JsonSchema.Net" Version="5.3.0" /> <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.5" /> - <PackageReference Include="MongoDB.Driver.GridFS" Version="2.22.0" /> - <PackageReference Include="MongoDB.Driver" Version="2.22.0" /> + <PackageReference Include="MongoDB.Driver" Version="3.5.0" /> <PackageReference Include="AngleSharp" Version="1.1.1" /> <PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" /> <PackageReference Include="NuGet.Versioning" Version="6.9.1" /> </ItemGroup> <ItemGroup> - <ProjectReference Include="..\StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj" /> - <ProjectReference Include="..\StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" /> <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> </ItemGroup> </Project> diff --git a/src/StellaOps.Feedser.Source.Common/TASKS.md b/src/StellaOps.Concelier.Connector.Common/TASKS.md similarity index 83% rename from src/StellaOps.Feedser.Source.Common/TASKS.md rename to src/StellaOps.Concelier.Connector.Common/TASKS.md index 5ce78e95..65ad7875 100644 --- a/src/StellaOps.Feedser.Source.Common/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Common/TASKS.md @@ -15,5 +15,5 @@ |Shared jitter source in retry policy|BE-Conn-Shared|Source.Common|**DONE** – `SourceRetryPolicy` now consumes injected `CryptoJitterSource` for thread-safe jitter.| |Allow per-request Accept header overrides|BE-Conn-Shared|Source.Common|**DONE** – `SourceFetchRequest.AcceptHeaders` honored by `SourceFetchService` plus unit tests for overrides.| |FEEDCONN-SHARED-HTTP2-001 HTTP version fallback policy|BE-Conn-Shared, Source.Common|Source.Common|**DONE (2025-10-11)** – `AddSourceHttpClient` now honours per-connector HTTP version/ policy, exposes handler customisation, and defaults to downgrade-friendly settings; unit tests cover handler configuration hook.| -|FEEDCONN-SHARED-TLS-001 Sovereign trust store support|BE-Conn-Shared, Ops|Source.Common|**DONE (2025-10-11)** – `SourceHttpClientOptions` now exposes `TrustedRootCertificates`, `ServerCertificateCustomValidation`, and `AllowInvalidServerCertificates`, and `AddSourceHttpClient` runs the shared configuration binder so connectors can pull `feedser:httpClients|sources:<name>:http` settings (incl. Offline Kit relative PEM paths via `feedser:offline:root`). Tests cover handler wiring. Ops follow-up: package RU trust roots for Offline Kit distribution.| -|FEEDCONN-SHARED-STATE-003 Source state seeding helper|Tools Guild, BE-Conn-MSRC|Tools|**TODO (2025-10-15)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands.| +|FEEDCONN-SHARED-TLS-001 Sovereign trust store support|BE-Conn-Shared, Ops|Source.Common|**DONE (2025-10-11)** – `SourceHttpClientOptions` now exposes `TrustedRootCertificates`, `ServerCertificateCustomValidation`, and `AllowInvalidServerCertificates`, and `AddSourceHttpClient` runs the shared configuration binder so connectors can pull `concelier:httpClients|sources:<name>:http` settings (incl. Offline Kit relative PEM paths via `concelier:offline:root`). Tests cover handler wiring. Ops follow-up: package RU trust roots for Offline Kit distribution.| +|FEEDCONN-SHARED-STATE-003 Source state seeding helper|Tools Guild, BE-Conn-MSRC|Tools|**DOING (2025-10-19)** – Provide a reusable CLI/utility to seed `pendingDocuments`/`pendingMappings` for connectors (MSRC backfills require scripted CVRF + detail injection). Coordinate with MSRC team for expected JSON schema and handoff once prototype lands. Prereqs confirmed none (2025-10-19).| diff --git a/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs b/src/StellaOps.Concelier.Connector.Common/Telemetry/SourceDiagnostics.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Common/Telemetry/SourceDiagnostics.cs index 1f3a520c..f3af5ee3 100644 --- a/src/StellaOps.Feedser.Source.Common/Telemetry/SourceDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Common/Telemetry/SourceDiagnostics.cs @@ -2,31 +2,31 @@ using System.Diagnostics; using System.Diagnostics.Metrics; using System.Net; -namespace StellaOps.Feedser.Source.Common.Telemetry; +namespace StellaOps.Concelier.Connector.Common.Telemetry; /// <summary> /// Central telemetry instrumentation for connector HTTP operations. /// </summary> public static class SourceDiagnostics { - public const string ActivitySourceName = "StellaOps.Feedser.Source"; - public const string MeterName = "StellaOps.Feedser.Source"; + public const string ActivitySourceName = "StellaOps.Concelier.Connector"; + public const string MeterName = "StellaOps.Concelier.Connector"; private static readonly ActivitySource ActivitySource = new(ActivitySourceName); private static readonly Meter Meter = new(MeterName); - private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("feedser.source.http.requests"); - private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("feedser.source.http.retries"); - private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("feedser.source.http.failures"); - private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("feedser.source.http.not_modified"); - private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("feedser.source.http.duration", unit: "ms"); - private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("feedser.source.http.payload_bytes", unit: "byte"); + private static readonly Counter<long> HttpRequestCounter = Meter.CreateCounter<long>("concelier.source.http.requests"); + private static readonly Counter<long> HttpRetryCounter = Meter.CreateCounter<long>("concelier.source.http.retries"); + private static readonly Counter<long> HttpFailureCounter = Meter.CreateCounter<long>("concelier.source.http.failures"); + private static readonly Counter<long> HttpNotModifiedCounter = Meter.CreateCounter<long>("concelier.source.http.not_modified"); + private static readonly Histogram<double> HttpDuration = Meter.CreateHistogram<double>("concelier.source.http.duration", unit: "ms"); + private static readonly Histogram<long> HttpPayloadBytes = Meter.CreateHistogram<long>("concelier.source.http.payload_bytes", unit: "byte"); public static Activity? StartFetch(string sourceName, Uri requestUri, string httpMethod, string? clientName) { var tags = new ActivityTagsCollection { - { "feedser.source", sourceName }, + { "concelier.source", sourceName }, { "http.method", httpMethod }, { "http.url", requestUri.ToString() }, }; @@ -70,7 +70,7 @@ public static class SourceDiagnostics { var tags = new TagList { - { "feedser.source", sourceName }, + { "concelier.source", sourceName }, { "http.retry_attempt", attempt }, { "http.retry_delay_ms", delay.TotalMilliseconds }, }; @@ -92,7 +92,7 @@ public static class SourceDiagnostics { var tags = new TagList { - { "feedser.source", sourceName }, + { "concelier.source", sourceName }, { "http.status_code", (int)statusCode }, { "http.attempts", attemptCount }, }; diff --git a/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs b/src/StellaOps.Concelier.Connector.Common/Testing/CannedHttpMessageHandler.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs rename to src/StellaOps.Concelier.Connector.Common/Testing/CannedHttpMessageHandler.cs index 76c65f7e..49e0e339 100644 --- a/src/StellaOps.Feedser.Source.Common/Testing/CannedHttpMessageHandler.cs +++ b/src/StellaOps.Concelier.Connector.Common/Testing/CannedHttpMessageHandler.cs @@ -3,7 +3,7 @@ using System.Net; using System.Net.Http; using System.Text; -namespace StellaOps.Feedser.Source.Common.Testing; +namespace StellaOps.Concelier.Connector.Common.Testing; /// <summary> /// Deterministic HTTP handler used by tests to supply canned responses keyed by request URI and method. diff --git a/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs b/src/StellaOps.Concelier.Connector.Common/Url/UrlNormalizer.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs rename to src/StellaOps.Concelier.Connector.Common/Url/UrlNormalizer.cs index a9c85cf2..bc4ef357 100644 --- a/src/StellaOps.Feedser.Source.Common/Url/UrlNormalizer.cs +++ b/src/StellaOps.Concelier.Connector.Common/Url/UrlNormalizer.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Common.Url; +namespace StellaOps.Concelier.Connector.Common.Url; /// <summary> /// Utilities for normalizing URLs from upstream feeds. diff --git a/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs b/src/StellaOps.Concelier.Connector.Common/Xml/IXmlSchemaValidator.cs similarity index 74% rename from src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs rename to src/StellaOps.Concelier.Connector.Common/Xml/IXmlSchemaValidator.cs index cb340779..62b1cfa3 100644 --- a/src/StellaOps.Feedser.Source.Common/Xml/IXmlSchemaValidator.cs +++ b/src/StellaOps.Concelier.Connector.Common/Xml/IXmlSchemaValidator.cs @@ -1,7 +1,7 @@ using System.Xml.Linq; using System.Xml.Schema; -namespace StellaOps.Feedser.Source.Common.Xml; +namespace StellaOps.Concelier.Connector.Common.Xml; public interface IXmlSchemaValidator { diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationError.cs similarity index 59% rename from src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs rename to src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationError.cs index 34ee3b2e..de35c49b 100644 --- a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationError.cs +++ b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationError.cs @@ -1,3 +1,3 @@ -namespace StellaOps.Feedser.Source.Common.Xml; +namespace StellaOps.Concelier.Connector.Common.Xml; public sealed record XmlSchemaValidationError(string Message, string? Location); diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationException.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs rename to src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationException.cs index a8b8cb3a..cfd9f101 100644 --- a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidationException.cs +++ b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidationException.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Common.Xml; +namespace StellaOps.Concelier.Connector.Common.Xml; public sealed class XmlSchemaValidationException : Exception { diff --git a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidator.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs rename to src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidator.cs index 5ea71951..fab7d904 100644 --- a/src/StellaOps.Feedser.Source.Common/Xml/XmlSchemaValidator.cs +++ b/src/StellaOps.Concelier.Connector.Common/Xml/XmlSchemaValidator.cs @@ -4,7 +4,7 @@ using System.Xml.Linq; using System.Xml.Schema; using Microsoft.Extensions.Logging; -namespace StellaOps.Feedser.Source.Common.Xml; +namespace StellaOps.Concelier.Connector.Common.Xml; public sealed class XmlSchemaValidator : IXmlSchemaValidator { diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs b/src/StellaOps.Concelier.Connector.Cve.Tests/Cve/CveConnectorTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Cve.Tests/Cve/CveConnectorTests.cs index 6736d027..40be53e3 100644 --- a/src/StellaOps.Feedser.Source.Cve.Tests/Cve/CveConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Cve.Tests/Cve/CveConnectorTests.cs @@ -1,257 +1,257 @@ -using System.Diagnostics.Metrics; -using System.IO; -using System.Net; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Cve.Configuration; -using StellaOps.Feedser.Source.Cve.Internal; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using Xunit.Abstractions; - -namespace StellaOps.Feedser.Source.Cve.Tests; - -[Collection("mongo-fixture")] -public sealed class CveConnectorTests : IAsyncLifetime -{ - private readonly MongoIntegrationFixture _fixture; - private readonly ITestOutputHelper _output; - private ConnectorTestHarness? _harness; - - public CveConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) - { - _fixture = fixture; - _output = output; - } - - [Fact] - public async Task FetchParseMap_EmitsCanonicalAdvisory() - { - var initialTime = new DateTimeOffset(2024, 10, 1, 0, 0, 0, TimeSpan.Zero); - await EnsureHarnessAsync(initialTime); - var harness = _harness!; - - var since = initialTime - TimeSpan.FromDays(30); - var listUri = new Uri($"https://cve.test/api/cve?time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&size=5"); - harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/cve-list.json")); - harness.Handler.SetFallback(request => - { - if (request.RequestUri is null) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (request.RequestUri.AbsoluteUri.Equals("https://cve.test/api/cve/CVE-2024-0001", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ReadFixture("Fixtures/cve-CVE-2024-0001.json"), Encoding.UTF8, "application/json") - }; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }); - - var metrics = new Dictionary<string, long>(StringComparer.Ordinal); - using var listener = new MeterListener - { - InstrumentPublished = (instrument, meterListener) => - { - if (instrument.Meter.Name == CveDiagnostics.MeterName) - { - meterListener.EnableMeasurementEvents(instrument); - } - } - }; - listener.SetMeasurementEventCallback<long>((instrument, value, tags, state) => - { - if (metrics.TryGetValue(instrument.Name, out var existing)) - { - metrics[instrument.Name] = existing + value; - } - else - { - metrics[instrument.Name] = value; - } - }); - listener.Start(); - - var connector = new CveConnectorPlugin().Create(harness.ServiceProvider); - - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); - await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); - - listener.Dispose(); - - var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); - var advisory = await advisoryStore.FindAsync("CVE-2024-0001", CancellationToken.None); - Assert.NotNull(advisory); - - var snapshot = SnapshotSerializer.ToSnapshot(advisory!).Replace("\r\n", "\n").TrimEnd(); - var expected = ReadFixture("Fixtures/expected-CVE-2024-0001.json").Replace("\r\n", "\n").TrimEnd(); - - if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-CVE-2024-0001.actual.json"); - Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(expected, snapshot); - harness.Handler.AssertNoPendingResponses(); - - _output.WriteLine("CVE connector smoke metrics:"); - foreach (var entry in metrics.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) - { - _output.WriteLine($" {entry.Key} = {entry.Value}"); - } - } - - [Fact] - public async Task FetchWithoutCredentials_SeedsFromDirectory() - { - var initialTime = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); - var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); - var repositoryRoot = Path.GetFullPath(Path.Combine(projectRoot, "..", "..")); - var seedDirectory = Path.Combine(repositoryRoot, "seed-data", "cve", "2025-10-15"); - Assert.True(Directory.Exists(seedDirectory), $"Seed directory '{seedDirectory}' was not found."); - - await using var harness = new ConnectorTestHarness(_fixture, initialTime, CveOptions.HttpClientName); - await harness.EnsureServiceProviderAsync(services => - { - services.AddLogging(builder => - { - builder.ClearProviders(); - builder.AddProvider(new TestOutputLoggerProvider(_output, LogLevel.Information)); - builder.SetMinimumLevel(LogLevel.Information); - }); - services.AddCveConnector(options => - { - options.BaseEndpoint = new Uri("https://cve.test/api/", UriKind.Absolute); - options.SeedDirectory = seedDirectory; - options.PageSize = 5; - options.MaxPagesPerFetch = 1; - options.InitialBackfill = TimeSpan.FromDays(30); - options.RequestDelay = TimeSpan.Zero; - }); - }); - - var connector = new CveConnectorPlugin().Create(harness.ServiceProvider); - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - - Assert.Empty(harness.Handler.Requests); - - var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - var keys = advisories.Select(advisory => advisory.AdvisoryKey).ToArray(); - - Assert.Contains("CVE-2024-0001", keys); - Assert.Contains("CVE-2024-4567", keys); - } - - private async Task EnsureHarnessAsync(DateTimeOffset initialTime) - { - if (_harness is not null) - { - return; - } - - var harness = new ConnectorTestHarness(_fixture, initialTime, CveOptions.HttpClientName); - await harness.EnsureServiceProviderAsync(services => - { - services.AddLogging(builder => - { - builder.ClearProviders(); - builder.AddProvider(new TestOutputLoggerProvider(_output, LogLevel.Information)); - builder.SetMinimumLevel(LogLevel.Information); - }); - services.AddCveConnector(options => - { - options.BaseEndpoint = new Uri("https://cve.test/api/", UriKind.Absolute); - options.ApiOrg = "test-org"; - options.ApiUser = "test-user"; - options.ApiKey = "test-key"; - options.InitialBackfill = TimeSpan.FromDays(30); - options.PageSize = 5; - options.MaxPagesPerFetch = 2; - options.RequestDelay = TimeSpan.Zero; - }); - }); - - _harness = harness; - } - - private static string ReadFixture(string relativePath) - { - var path = Path.Combine(AppContext.BaseDirectory, relativePath); - return File.ReadAllText(path); - } - - public async Task InitializeAsync() - { - await Task.CompletedTask; - } - - public async Task DisposeAsync() - { - if (_harness is not null) - { - await _harness.DisposeAsync(); - } - } - - private sealed class TestOutputLoggerProvider : ILoggerProvider - { - private readonly ITestOutputHelper _output; - private readonly LogLevel _minLevel; - - public TestOutputLoggerProvider(ITestOutputHelper output, LogLevel minLevel) - { - _output = output; - _minLevel = minLevel; - } - - public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output, _minLevel); - - public void Dispose() - { - } - - private sealed class TestOutputLogger : ILogger - { - private readonly ITestOutputHelper _output; - private readonly LogLevel _minLevel; - - public TestOutputLogger(ITestOutputHelper output, LogLevel minLevel) - { - _output = output; - _minLevel = minLevel; - } - - public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); - - public bool IsEnabled(LogLevel logLevel) => logLevel >= _minLevel; - - public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) - { - if (IsEnabled(logLevel)) - { - _output.WriteLine(formatter(state, exception)); - } - } - } - } -} +using System.Diagnostics.Metrics; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Cve.Configuration; +using StellaOps.Concelier.Connector.Cve.Internal; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using Xunit.Abstractions; + +namespace StellaOps.Concelier.Connector.Cve.Tests; + +[Collection("mongo-fixture")] +public sealed class CveConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly ITestOutputHelper _output; + private ConnectorTestHarness? _harness; + + public CveConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _output = output; + } + + [Fact] + public async Task FetchParseMap_EmitsCanonicalAdvisory() + { + var initialTime = new DateTimeOffset(2024, 10, 1, 0, 0, 0, TimeSpan.Zero); + await EnsureHarnessAsync(initialTime); + var harness = _harness!; + + var since = initialTime - TimeSpan.FromDays(30); + var listUri = new Uri($"https://cve.test/api/cve?time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&size=5"); + harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/cve-list.json")); + harness.Handler.SetFallback(request => + { + if (request.RequestUri is null) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (request.RequestUri.AbsoluteUri.Equals("https://cve.test/api/cve/CVE-2024-0001", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/cve-CVE-2024-0001.json"), Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + var metrics = new Dictionary<string, long>(StringComparer.Ordinal); + using var listener = new MeterListener + { + InstrumentPublished = (instrument, meterListener) => + { + if (instrument.Meter.Name == CveDiagnostics.MeterName) + { + meterListener.EnableMeasurementEvents(instrument); + } + } + }; + listener.SetMeasurementEventCallback<long>((instrument, value, tags, state) => + { + if (metrics.TryGetValue(instrument.Name, out var existing)) + { + metrics[instrument.Name] = existing + value; + } + else + { + metrics[instrument.Name] = value; + } + }); + listener.Start(); + + var connector = new CveConnectorPlugin().Create(harness.ServiceProvider); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + listener.Dispose(); + + var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); + var advisory = await advisoryStore.FindAsync("CVE-2024-0001", CancellationToken.None); + Assert.NotNull(advisory); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory!).Replace("\r\n", "\n").TrimEnd(); + var expected = ReadFixture("Fixtures/expected-CVE-2024-0001.json").Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-CVE-2024-0001.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + harness.Handler.AssertNoPendingResponses(); + + _output.WriteLine("CVE connector smoke metrics:"); + foreach (var entry in metrics.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + _output.WriteLine($" {entry.Key} = {entry.Value}"); + } + } + + [Fact] + public async Task FetchWithoutCredentials_SeedsFromDirectory() + { + var initialTime = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); + var repositoryRoot = Path.GetFullPath(Path.Combine(projectRoot, "..", "..")); + var seedDirectory = Path.Combine(repositoryRoot, "seed-data", "cve", "2025-10-15"); + Assert.True(Directory.Exists(seedDirectory), $"Seed directory '{seedDirectory}' was not found."); + + await using var harness = new ConnectorTestHarness(_fixture, initialTime, CveOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => + { + builder.ClearProviders(); + builder.AddProvider(new TestOutputLoggerProvider(_output, LogLevel.Information)); + builder.SetMinimumLevel(LogLevel.Information); + }); + services.AddCveConnector(options => + { + options.BaseEndpoint = new Uri("https://cve.test/api/", UriKind.Absolute); + options.SeedDirectory = seedDirectory; + options.PageSize = 5; + options.MaxPagesPerFetch = 1; + options.InitialBackfill = TimeSpan.FromDays(30); + options.RequestDelay = TimeSpan.Zero; + }); + }); + + var connector = new CveConnectorPlugin().Create(harness.ServiceProvider); + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + + Assert.Empty(harness.Handler.Requests); + + var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + var keys = advisories.Select(advisory => advisory.AdvisoryKey).ToArray(); + + Assert.Contains("CVE-2024-0001", keys); + Assert.Contains("CVE-2024-4567", keys); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, CveOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => + { + builder.ClearProviders(); + builder.AddProvider(new TestOutputLoggerProvider(_output, LogLevel.Information)); + builder.SetMinimumLevel(LogLevel.Information); + }); + services.AddCveConnector(options => + { + options.BaseEndpoint = new Uri("https://cve.test/api/", UriKind.Absolute); + options.ApiOrg = "test-org"; + options.ApiUser = "test-user"; + options.ApiKey = "test-key"; + options.InitialBackfill = TimeSpan.FromDays(30); + options.PageSize = 5; + options.MaxPagesPerFetch = 2; + options.RequestDelay = TimeSpan.Zero; + }); + }); + + _harness = harness; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath); + return File.ReadAllText(path); + } + + public async Task InitializeAsync() + { + await Task.CompletedTask; + } + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } + + private sealed class TestOutputLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _output; + private readonly LogLevel _minLevel; + + public TestOutputLoggerProvider(ITestOutputHelper output, LogLevel minLevel) + { + _output = output; + _minLevel = minLevel; + } + + public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output, _minLevel); + + public void Dispose() + { + } + + private sealed class TestOutputLogger : ILogger + { + private readonly ITestOutputHelper _output; + private readonly LogLevel _minLevel; + + public TestOutputLogger(ITestOutputHelper output, LogLevel minLevel) + { + _output = output; + _minLevel = minLevel; + } + + public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => logLevel >= _minLevel; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + if (IsEnabled(logLevel)) + { + _output.WriteLine(formatter(state, exception)); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-CVE-2024-0001.json similarity index 96% rename from src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json rename to src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-CVE-2024-0001.json index b9b89bfc..88158371 100644 --- a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-CVE-2024-0001.json +++ b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-CVE-2024-0001.json @@ -1,72 +1,72 @@ -{ - "dataType": "CVE_RECORD", - "dataVersion": "5.0", - "cveMetadata": { - "cveId": "CVE-2024-0001", - "assignerShortName": "ExampleOrg", - "state": "PUBLISHED", - "dateReserved": "2024-01-01T00:00:00Z", - "datePublished": "2024-09-10T12:00:00Z", - "dateUpdated": "2024-09-15T12:00:00Z" - }, - "containers": { - "cna": { - "title": "Example Product Remote Code Execution", - "descriptions": [ - { - "lang": "en", - "value": "An example vulnerability allowing remote attackers to execute arbitrary code." - } - ], - "affected": [ - { - "vendor": "ExampleVendor", - "product": "ExampleProduct", - "platform": "linux", - "defaultStatus": "affected", - "versions": [ - { - "status": "affected", - "version": "1.0.0", - "lessThan": "1.2.0", - "versionType": "semver" - }, - { - "status": "unaffected", - "version": "1.2.0", - "versionType": "semver" - } - ] - } - ], - "references": [ - { - "url": "https://example.com/security/advisory", - "name": "Vendor Advisory", - "tags": [ - "vendor-advisory" - ] - }, - { - "url": "https://cve.example.com/CVE-2024-0001", - "tags": [ - "third-party-advisory" - ] - } - ], - "metrics": [ - { - "cvssV3_1": { - "version": "3.1", - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "baseScore": 9.8, - "baseSeverity": "CRITICAL" - } - } - ], - "aliases": [ - "GHSA-xxxx-yyyy-zzzz" - ] - } - } -} +{ + "dataType": "CVE_RECORD", + "dataVersion": "5.0", + "cveMetadata": { + "cveId": "CVE-2024-0001", + "assignerShortName": "ExampleOrg", + "state": "PUBLISHED", + "dateReserved": "2024-01-01T00:00:00Z", + "datePublished": "2024-09-10T12:00:00Z", + "dateUpdated": "2024-09-15T12:00:00Z" + }, + "containers": { + "cna": { + "title": "Example Product Remote Code Execution", + "descriptions": [ + { + "lang": "en", + "value": "An example vulnerability allowing remote attackers to execute arbitrary code." + } + ], + "affected": [ + { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux", + "defaultStatus": "affected", + "versions": [ + { + "status": "affected", + "version": "1.0.0", + "lessThan": "1.2.0", + "versionType": "semver" + }, + { + "status": "unaffected", + "version": "1.2.0", + "versionType": "semver" + } + ] + } + ], + "references": [ + { + "url": "https://example.com/security/advisory", + "name": "Vendor Advisory", + "tags": [ + "vendor-advisory" + ] + }, + { + "url": "https://cve.example.com/CVE-2024-0001", + "tags": [ + "third-party-advisory" + ] + } + ], + "metrics": [ + { + "cvssV3_1": { + "version": "3.1", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ], + "aliases": [ + "GHSA-xxxx-yyyy-zzzz" + ] + } + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-list.json similarity index 94% rename from src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json rename to src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-list.json index c2c15927..98a78ed6 100644 --- a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/cve-list.json +++ b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/cve-list.json @@ -1,18 +1,18 @@ -{ - "dataType": "CVE_RECORD_LIST", - "dataVersion": "5.0", - "data": [ - { - "cveMetadata": { - "cveId": "CVE-2024-0001", - "state": "PUBLISHED", - "dateUpdated": "2024-09-15T12:00:00Z" - } - } - ], - "pagination": { - "page": 1, - "totalCount": 1, - "itemsPerPage": 5 - } -} +{ + "dataType": "CVE_RECORD_LIST", + "dataVersion": "5.0", + "data": [ + { + "cveMetadata": { + "cveId": "CVE-2024-0001", + "state": "PUBLISHED", + "dateUpdated": "2024-09-15T12:00:00Z" + } + } + ], + "pagination": { + "page": 1, + "totalCount": 1, + "itemsPerPage": 5 + } +} diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/expected-CVE-2024-0001.json similarity index 96% rename from src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json rename to src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/expected-CVE-2024-0001.json index 4e6d0b4b..0abdfa98 100644 --- a/src/StellaOps.Feedser.Source.Cve.Tests/Fixtures/expected-CVE-2024-0001.json +++ b/src/StellaOps.Concelier.Connector.Cve.Tests/Fixtures/expected-CVE-2024-0001.json @@ -1,221 +1,221 @@ -{ - "advisoryKey": "CVE-2024-0001", - "affectedPackages": [ - { - "type": "vendor", - "identifier": "examplevendor:exampleproduct", - "platform": "linux", - "versionRanges": [ - { - "fixedVersion": "1.2.0", - "introducedVersion": "1.0.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": "version=1.0.0, < 1.2.0", - "exactValue": null, - "fixed": "1.2.0", - "fixedInclusive": false, - "introduced": "1.0.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": { - "vendor": "ExampleVendor", - "product": "ExampleProduct", - "platform": "linux", - "version": "1.0.0", - "lessThan": "1.2.0", - "versionType": "semver" - } - }, - "provenance": { - "source": "cve", - "kind": "affected-range", - "value": "examplevendor:exampleproduct", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": "version=1.0.0, < 1.2.0", - "rangeKind": "semver" - }, - { - "fixedVersion": "1.2.0", - "introducedVersion": "1.2.0", - "lastAffectedVersion": "1.2.0", - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": "version=1.2.0", - "exactValue": null, - "fixed": "1.2.0", - "fixedInclusive": false, - "introduced": "1.2.0", - "introducedInclusive": true, - "lastAffected": "1.2.0", - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": { - "vendor": "ExampleVendor", - "product": "ExampleProduct", - "platform": "linux", - "version": "1.2.0", - "versionType": "semver" - } - }, - "provenance": { - "source": "cve", - "kind": "affected-range", - "value": "examplevendor:exampleproduct", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": "version=1.2.0", - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "1.2.0", - "notes": "cve:cve-2024-0001:examplevendor:exampleproduct" - }, - { - "scheme": "semver", - "type": "range", - "min": "1.0.0", - "minInclusive": true, - "max": "1.2.0", - "maxInclusive": false, - "value": null, - "notes": "cve:cve-2024-0001:examplevendor:exampleproduct" - } - ], - "statuses": [ - { - "provenance": { - "source": "cve", - "kind": "affected-status", - "value": "examplevendor:exampleproduct", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "status": "affected" - }, - { - "provenance": { - "source": "cve", - "kind": "affected-status", - "value": "examplevendor:exampleproduct", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "status": "not_affected" - } - ], - "provenance": [ - { - "source": "cve", - "kind": "affected", - "value": "examplevendor:exampleproduct", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "aliases": [ - "CVE-2024-0001", - "GHSA-xxxx-yyyy-zzzz" - ], - "credits": [], - "cvssMetrics": [ - { - "baseScore": 9.8, - "baseSeverity": "critical", - "provenance": { - "source": "cve", - "kind": "cvss", - "value": "cve/CVE-2024-0001", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2024-09-15T12:00:00+00:00", - "provenance": [ - { - "source": "cve", - "kind": "document", - "value": "cve/CVE-2024-0001", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - { - "source": "cve", - "kind": "mapping", - "value": "CVE-2024-0001", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - } - ], - "published": "2024-09-10T12:00:00+00:00", - "references": [ - { - "kind": "third-party-advisory", - "provenance": { - "source": "cve", - "kind": "reference", - "value": "https://cve.example.com/CVE-2024-0001", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": null, - "summary": null, - "url": "https://cve.example.com/CVE-2024-0001" - }, - { - "kind": "vendor-advisory", - "provenance": { - "source": "cve", - "kind": "reference", - "value": "https://example.com/security/advisory", - "decisionReason": null, - "recordedAt": "2024-10-01T00:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "Vendor Advisory", - "summary": null, - "url": "https://example.com/security/advisory" - } - ], - "severity": "critical", - "summary": "An example vulnerability allowing remote attackers to execute arbitrary code.", - "title": "Example Product Remote Code Execution" +{ + "advisoryKey": "CVE-2024-0001", + "affectedPackages": [ + { + "type": "vendor", + "identifier": "examplevendor:exampleproduct", + "platform": "linux", + "versionRanges": [ + { + "fixedVersion": "1.2.0", + "introducedVersion": "1.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": "version=1.0.0, < 1.2.0", + "exactValue": null, + "fixed": "1.2.0", + "fixedInclusive": false, + "introduced": "1.0.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux", + "version": "1.0.0", + "lessThan": "1.2.0", + "versionType": "semver" + } + }, + "provenance": { + "source": "cve", + "kind": "affected-range", + "value": "examplevendor:exampleproduct", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": "version=1.0.0, < 1.2.0", + "rangeKind": "semver" + }, + { + "fixedVersion": "1.2.0", + "introducedVersion": "1.2.0", + "lastAffectedVersion": "1.2.0", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": "version=1.2.0", + "exactValue": null, + "fixed": "1.2.0", + "fixedInclusive": false, + "introduced": "1.2.0", + "introducedInclusive": true, + "lastAffected": "1.2.0", + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": { + "vendor": "ExampleVendor", + "product": "ExampleProduct", + "platform": "linux", + "version": "1.2.0", + "versionType": "semver" + } + }, + "provenance": { + "source": "cve", + "kind": "affected-range", + "value": "examplevendor:exampleproduct", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": "version=1.2.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.2.0", + "notes": "cve:cve-2024-0001:examplevendor:exampleproduct" + }, + { + "scheme": "semver", + "type": "range", + "min": "1.0.0", + "minInclusive": true, + "max": "1.2.0", + "maxInclusive": false, + "value": null, + "notes": "cve:cve-2024-0001:examplevendor:exampleproduct" + } + ], + "statuses": [ + { + "provenance": { + "source": "cve", + "kind": "affected-status", + "value": "examplevendor:exampleproduct", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "status": "affected" + }, + { + "provenance": { + "source": "cve", + "kind": "affected-status", + "value": "examplevendor:exampleproduct", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "status": "not_affected" + } + ], + "provenance": [ + { + "source": "cve", + "kind": "affected", + "value": "examplevendor:exampleproduct", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-0001", + "GHSA-xxxx-yyyy-zzzz" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "cve", + "kind": "cvss", + "value": "cve/CVE-2024-0001", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-09-15T12:00:00+00:00", + "provenance": [ + { + "source": "cve", + "kind": "document", + "value": "cve/CVE-2024-0001", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + { + "source": "cve", + "kind": "mapping", + "value": "CVE-2024-0001", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-09-10T12:00:00+00:00", + "references": [ + { + "kind": "third-party-advisory", + "provenance": { + "source": "cve", + "kind": "reference", + "value": "https://cve.example.com/CVE-2024-0001", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": null, + "url": "https://cve.example.com/CVE-2024-0001" + }, + { + "kind": "vendor-advisory", + "provenance": { + "source": "cve", + "kind": "reference", + "value": "https://example.com/security/advisory", + "decisionReason": null, + "recordedAt": "2024-10-01T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "Vendor Advisory", + "summary": null, + "url": "https://example.com/security/advisory" + } + ], + "severity": "critical", + "summary": "An example vulnerability allowing remote attackers to execute arbitrary code.", + "title": "Example Product Remote Code Execution" } \ No newline at end of file diff --git a/src/StellaOps.Concelier.Connector.Cve.Tests/StellaOps.Concelier.Connector.Cve.Tests.csproj b/src/StellaOps.Concelier.Connector.Cve.Tests/StellaOps.Concelier.Connector.Cve.Tests.csproj new file mode 100644 index 00000000..069ddfc0 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Cve.Tests/StellaOps.Concelier.Connector.Cve.Tests.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Cve/StellaOps.Concelier.Connector.Cve.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="Fixtures/*.json" CopyToOutputDirectory="Always" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Cve/AGENTS.md b/src/StellaOps.Concelier.Connector.Cve/AGENTS.md similarity index 86% rename from src/StellaOps.Feedser.Source.Cve/AGENTS.md rename to src/StellaOps.Concelier.Connector.Cve/AGENTS.md index 2d7c464a..a1ff634d 100644 --- a/src/StellaOps.Feedser.Source.Cve/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Cve/AGENTS.md @@ -1,38 +1,38 @@ -# AGENTS -## Role -Create a dedicated CVE connector when we need raw CVE stream ingestion outside of NVD/OSV/National feeds (e.g., CVE JSON 5 API or CNA disclosures). - -## Scope -- Determine whether this connector should consume the official CVE JSON 5 API, CNA disclosures, or another stream. -- Implement fetch/windowing aligned with CVE publication cadence; manage cursors for incremental backfills. -- Parse CVE payloads into DTOs capturing descriptions, affected vendors/products, references, and metrics. -- Map CVEs into canonical `Advisory` records (aliases, references, affected packages, range primitives). -- Deliver deterministic fixtures/tests for fetch/parse/map lifecycle. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores & source state). -- `Feedser.Models` (canonical data model). -- `Feedser.Testing` (integration fixtures, snapshot helpers). - -## Interfaces & Contracts -- Job kinds: `cve:fetch`, `cve:parse`, `cve:map`. -- Persist upstream metadata (e.g., `If-Modified-Since`, `cveMetadataDate`) for incremental fetching. -- Aliases must include primary CVE ID along with CNA-specific identifiers when available. - -## In/Out of scope -In scope: -- Core pipeline for CVE ingestion with provenance/range primitives. - -Out of scope: -- Downstream impact scoring or enrichment (handled by other teams). - -## Observability & Security Expectations -- Log fetch batch sizes, update timestamps, and mapping counts. -- Handle rate limits politely with exponential backoff. -- Sanitize and validate payloads before persistence. - -## Tests -- Add `StellaOps.Feedser.Source.Cve.Tests` with canned CVE JSON fixtures covering fetch/parse/map. -- Snapshot canonical advisories; include env flag for fixture regeneration. -- Ensure deterministic ordering and timestamp handling. +# AGENTS +## Role +Create a dedicated CVE connector when we need raw CVE stream ingestion outside of NVD/OSV/National feeds (e.g., CVE JSON 5 API or CNA disclosures). + +## Scope +- Determine whether this connector should consume the official CVE JSON 5 API, CNA disclosures, or another stream. +- Implement fetch/windowing aligned with CVE publication cadence; manage cursors for incremental backfills. +- Parse CVE payloads into DTOs capturing descriptions, affected vendors/products, references, and metrics. +- Map CVEs into canonical `Advisory` records (aliases, references, affected packages, range primitives). +- Deliver deterministic fixtures/tests for fetch/parse/map lifecycle. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores & source state). +- `Concelier.Models` (canonical data model). +- `Concelier.Testing` (integration fixtures, snapshot helpers). + +## Interfaces & Contracts +- Job kinds: `cve:fetch`, `cve:parse`, `cve:map`. +- Persist upstream metadata (e.g., `If-Modified-Since`, `cveMetadataDate`) for incremental fetching. +- Aliases must include primary CVE ID along with CNA-specific identifiers when available. + +## In/Out of scope +In scope: +- Core pipeline for CVE ingestion with provenance/range primitives. + +Out of scope: +- Downstream impact scoring or enrichment (handled by other teams). + +## Observability & Security Expectations +- Log fetch batch sizes, update timestamps, and mapping counts. +- Handle rate limits politely with exponential backoff. +- Sanitize and validate payloads before persistence. + +## Tests +- Add `StellaOps.Concelier.Connector.Cve.Tests` with canned CVE JSON fixtures covering fetch/parse/map. +- Snapshot canonical advisories; include env flag for fixture regeneration. +- Ensure deterministic ordering and timestamp handling. diff --git a/src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs b/src/StellaOps.Concelier.Connector.Cve/Configuration/CveOptions.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs rename to src/StellaOps.Concelier.Connector.Cve/Configuration/CveOptions.cs index 3a48f133..aeb89342 100644 --- a/src/StellaOps.Feedser.Source.Cve/Configuration/CveOptions.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Configuration/CveOptions.cs @@ -1,126 +1,126 @@ -using System; -using System.Diagnostics.CodeAnalysis; -using System.IO; - -namespace StellaOps.Feedser.Source.Cve.Configuration; - -public sealed class CveOptions -{ - public static string HttpClientName => "source.cve"; - - public Uri BaseEndpoint { get; set; } = new("https://cveawg.mitre.org/api/", UriKind.Absolute); - - /// <summary> - /// CVE Services requires an organisation identifier for authenticated requests. - /// </summary> - public string ApiOrg { get; set; } = string.Empty; - - /// <summary> - /// CVE Services user identifier. Typically the username registered with the CVE Program. - /// </summary> - public string ApiUser { get; set; } = string.Empty; - - /// <summary> - /// API key issued by the CVE Program for the configured organisation/user pair. - /// </summary> - public string ApiKey { get; set; } = string.Empty; - - /// <summary> - /// Optional path containing seed CVE JSON documents used when live credentials are unavailable. - /// </summary> - public string? SeedDirectory { get; set; } - - /// <summary> - /// Results fetched per page when querying CVE Services. Valid range 1-500. - /// </summary> - public int PageSize { get; set; } = 200; - - /// <summary> - /// Maximum number of pages to fetch in a single run. Guards against runaway backfills. - /// </summary> - public int MaxPagesPerFetch { get; set; } = 5; - - /// <summary> - /// Sliding look-back window when no previous cursor is available. - /// </summary> - public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); - - /// <summary> - /// Delay between paginated requests to respect API throttling guidance. - /// </summary> - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - /// <summary> - /// Backoff applied when the connector encounters an unrecoverable failure. - /// </summary> - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(10); - - [MemberNotNull(nameof(BaseEndpoint))] - public void Validate() - { - if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); - } - - var hasCredentials = !string.IsNullOrWhiteSpace(ApiOrg) - && !string.IsNullOrWhiteSpace(ApiUser) - && !string.IsNullOrWhiteSpace(ApiKey); - var hasSeedDirectory = !string.IsNullOrWhiteSpace(SeedDirectory); - - if (!hasCredentials && !hasSeedDirectory) - { - throw new InvalidOperationException("Api credentials must be provided unless a SeedDirectory is configured."); - } - - if (hasCredentials && string.IsNullOrWhiteSpace(ApiOrg)) - { - throw new InvalidOperationException("ApiOrg must be provided."); - } - - if (hasCredentials && string.IsNullOrWhiteSpace(ApiUser)) - { - throw new InvalidOperationException("ApiUser must be provided."); - } - - if (hasCredentials && string.IsNullOrWhiteSpace(ApiKey)) - { - throw new InvalidOperationException("ApiKey must be provided."); - } - - if (hasSeedDirectory && !Directory.Exists(SeedDirectory!)) - { - throw new InvalidOperationException($"SeedDirectory '{SeedDirectory}' does not exist."); - } - - if (PageSize is < 1 or > 500) - { - throw new InvalidOperationException("PageSize must be between 1 and 500."); - } - - if (MaxPagesPerFetch <= 0) - { - throw new InvalidOperationException("MaxPagesPerFetch must be a positive integer."); - } - - if (InitialBackfill < TimeSpan.Zero) - { - throw new InvalidOperationException("InitialBackfill cannot be negative."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("RequestDelay cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("FailureBackoff must be greater than zero."); - } - } - - public bool HasCredentials() - => !string.IsNullOrWhiteSpace(ApiOrg) - && !string.IsNullOrWhiteSpace(ApiUser) - && !string.IsNullOrWhiteSpace(ApiKey); -} +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; + +namespace StellaOps.Concelier.Connector.Cve.Configuration; + +public sealed class CveOptions +{ + public static string HttpClientName => "source.cve"; + + public Uri BaseEndpoint { get; set; } = new("https://cveawg.mitre.org/api/", UriKind.Absolute); + + /// <summary> + /// CVE Services requires an organisation identifier for authenticated requests. + /// </summary> + public string ApiOrg { get; set; } = string.Empty; + + /// <summary> + /// CVE Services user identifier. Typically the username registered with the CVE Program. + /// </summary> + public string ApiUser { get; set; } = string.Empty; + + /// <summary> + /// API key issued by the CVE Program for the configured organisation/user pair. + /// </summary> + public string ApiKey { get; set; } = string.Empty; + + /// <summary> + /// Optional path containing seed CVE JSON documents used when live credentials are unavailable. + /// </summary> + public string? SeedDirectory { get; set; } + + /// <summary> + /// Results fetched per page when querying CVE Services. Valid range 1-500. + /// </summary> + public int PageSize { get; set; } = 200; + + /// <summary> + /// Maximum number of pages to fetch in a single run. Guards against runaway backfills. + /// </summary> + public int MaxPagesPerFetch { get; set; } = 5; + + /// <summary> + /// Sliding look-back window when no previous cursor is available. + /// </summary> + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + /// <summary> + /// Delay between paginated requests to respect API throttling guidance. + /// </summary> + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// <summary> + /// Backoff applied when the connector encounters an unrecoverable failure. + /// </summary> + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(10); + + [MemberNotNull(nameof(BaseEndpoint))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); + } + + var hasCredentials = !string.IsNullOrWhiteSpace(ApiOrg) + && !string.IsNullOrWhiteSpace(ApiUser) + && !string.IsNullOrWhiteSpace(ApiKey); + var hasSeedDirectory = !string.IsNullOrWhiteSpace(SeedDirectory); + + if (!hasCredentials && !hasSeedDirectory) + { + throw new InvalidOperationException("Api credentials must be provided unless a SeedDirectory is configured."); + } + + if (hasCredentials && string.IsNullOrWhiteSpace(ApiOrg)) + { + throw new InvalidOperationException("ApiOrg must be provided."); + } + + if (hasCredentials && string.IsNullOrWhiteSpace(ApiUser)) + { + throw new InvalidOperationException("ApiUser must be provided."); + } + + if (hasCredentials && string.IsNullOrWhiteSpace(ApiKey)) + { + throw new InvalidOperationException("ApiKey must be provided."); + } + + if (hasSeedDirectory && !Directory.Exists(SeedDirectory!)) + { + throw new InvalidOperationException($"SeedDirectory '{SeedDirectory}' does not exist."); + } + + if (PageSize is < 1 or > 500) + { + throw new InvalidOperationException("PageSize must be between 1 and 500."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be a positive integer."); + } + + if (InitialBackfill < TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill cannot be negative."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be greater than zero."); + } + } + + public bool HasCredentials() + => !string.IsNullOrWhiteSpace(ApiOrg) + && !string.IsNullOrWhiteSpace(ApiUser) + && !string.IsNullOrWhiteSpace(ApiKey); +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveConnector.cs b/src/StellaOps.Concelier.Connector.Cve/CveConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Cve/CveConnector.cs rename to src/StellaOps.Concelier.Connector.Cve/CveConnector.cs index f9785a44..8948a379 100644 --- a/src/StellaOps.Feedser.Source.Cve/CveConnector.cs +++ b/src/StellaOps.Concelier.Connector.Cve/CveConnector.cs @@ -1,609 +1,609 @@ -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Security.Cryptography; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Text; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Cve.Configuration; -using StellaOps.Feedser.Source.Cve.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Cve; - -public sealed class CveConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - WriteIndented = false, - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly CveOptions _options; - private readonly CveDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger<CveConnector> _logger; - - public CveConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions<CveOptions> options, - CveDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger<CveConnector> logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => CveConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var now = _timeProvider.GetUtcNow(); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - - if (!_options.HasCredentials()) - { - if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) - { - return; - } - - _logger.LogWarning("CVEs fetch skipped: no credentials configured and no seed data found at {SeedDirectory}.", _options.SeedDirectory ?? "(seed directory not configured)"); - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var initialPendingDocuments = pendingDocuments.Count; - var initialPendingMappings = pendingMappings.Count; - var documentsFetched = 0; - var detailFailures = 0; - var detailUnchanged = 0; - var listSuccessCount = 0; - var listUnchangedCount = 0; - - var since = cursor.CurrentWindowStart ?? cursor.LastModifiedExclusive ?? now - _options.InitialBackfill; - if (since > now) - { - since = now; - } - - var windowEnd = cursor.CurrentWindowEnd ?? now; - if (windowEnd <= since) - { - windowEnd = since + TimeSpan.FromMinutes(1); - } - - var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; - var pagesFetched = 0; - var hasMorePages = true; - DateTimeOffset? maxModified = cursor.LastModifiedExclusive; - - while (hasMorePages && pagesFetched < _options.MaxPagesPerFetch) - { - cancellationToken.ThrowIfCancellationRequested(); - - var requestUri = BuildListRequestUri(since, windowEnd, page, _options.PageSize); - var metadata = new Dictionary<string, string>(StringComparer.Ordinal) - { - ["since"] = since.ToString("O"), - ["until"] = windowEnd.ToString("O"), - ["page"] = page.ToString(CultureInfo.InvariantCulture), - ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), - }; - - SourceFetchContentResult listResult; - try - { - _diagnostics.FetchAttempt(); - listResult = await _fetchService.FetchContentAsync( - new SourceFetchRequest( - CveOptions.HttpClientName, - SourceName, - requestUri) - { - Metadata = metadata, - AcceptHeaders = new[] { "application/json" }, - }, - cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException ex) when (IsAuthenticationFailure(ex)) - { - _logger.LogWarning("CVEs fetch requires API credentials ({StatusCode}); falling back to seed data if available.", ex.StatusCode); - if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) - { - return; - } - - _logger.LogWarning("CVEs fetch aborted: no seed data available (SeedDirectory={SeedDirectory}).", _options.SeedDirectory ?? "(seed directory not configured)"); - return; - } - catch (HttpRequestException ex) - { - _diagnostics.FetchFailure(); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (listResult.IsNotModified) - { - _diagnostics.FetchUnchanged(); - listUnchangedCount++; - break; - } - - if (!listResult.IsSuccess || listResult.Content is null) - { - _diagnostics.FetchFailure(); - break; - } - - _diagnostics.FetchSuccess(); - listSuccessCount++; - - var pageModel = CveListParser.Parse(listResult.Content, page, _options.PageSize); - - if (pageModel.Items.Count == 0) - { - hasMorePages = false; - } - - foreach (var item in pageModel.Items) - { - cancellationToken.ThrowIfCancellationRequested(); - - var detailUri = BuildDetailRequestUri(item.CveId); - var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) - { - ["cveId"] = item.CveId, - ["page"] = page.ToString(CultureInfo.InvariantCulture), - ["since"] = since.ToString("O"), - ["until"] = windowEnd.ToString("O"), - }; - - SourceFetchResult detailResult; - try - { - detailResult = await _fetchService.FetchAsync( - new SourceFetchRequest( - CveOptions.HttpClientName, - SourceName, - detailUri) - { - Metadata = detailMetadata, - AcceptHeaders = new[] { "application/json" }, - }, - cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException ex) when (IsAuthenticationFailure(ex)) - { - _diagnostics.FetchFailure(); - _logger.LogWarning(ex, "Failed fetching CVE record {CveId} due to authentication. Seeding if possible.", item.CveId); - if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) - { - return; - } - - _logger.LogWarning("CVE record {CveId} skipped; missing credentials and no seed data available.", item.CveId); - continue; - } - catch (HttpRequestException ex) - { - _diagnostics.FetchFailure(); - _logger.LogWarning(ex, "Failed fetching CVE record {CveId}", item.CveId); - continue; - } - - if (detailResult.IsNotModified) - { - _diagnostics.FetchUnchanged(); - detailUnchanged++; - continue; - } - - if (!detailResult.IsSuccess || detailResult.Document is null) - { - _diagnostics.FetchFailure(); - detailFailures++; - continue; - } - - _diagnostics.FetchDocument(); - if (pendingDocuments.Add(detailResult.Document.Id)) - { - documentsFetched++; - } - pendingMappings.Add(detailResult.Document.Id); - } - - if (pageModel.MaxModified.HasValue) - { - if (!maxModified.HasValue || pageModel.MaxModified > maxModified) - { - maxModified = pageModel.MaxModified; - } - } - - hasMorePages = pageModel.HasMorePages; - page = pageModel.NextPageCandidate; - pagesFetched++; - - if (hasMorePages && _options.RequestDelay > TimeSpan.Zero) - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - if (hasMorePages) - { - updatedCursor = updatedCursor - .WithCurrentWindowStart(since) - .WithCurrentWindowEnd(windowEnd) - .WithNextPage(page); - } - else - { - var nextSince = maxModified ?? windowEnd; - updatedCursor = updatedCursor - .WithLastModifiedExclusive(nextSince) - .WithCurrentWindowStart(null) - .WithCurrentWindowEnd(null) - .WithNextPage(1); - } - - var nextWindowStart = hasMorePages ? since : maxModified ?? windowEnd; - DateTimeOffset? nextWindowEnd = hasMorePages ? windowEnd : null; - var nextPage = hasMorePages ? page : 1; - var windowStartString = since.ToString("O"); - var windowEndString = windowEnd.ToString("O"); - var nextWindowStartString = nextWindowStart.ToString("O"); - var nextWindowEndString = nextWindowEnd?.ToString("O") ?? "(none)"; - - _logger.LogInformation( - "CVEs fetch window {WindowStart}->{WindowEnd} pages={PagesFetched} listSuccess={ListSuccess} detailDocuments={DocumentsFetched} detailFailures={DetailFailures} detailUnchanged={DetailUnchanged} pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter} hasMorePages={HasMorePages} nextWindowStart={NextWindowStart} nextWindowEnd={NextWindowEnd} nextPage={NextPage}", - windowStartString, - windowEndString, - pagesFetched, - listSuccessCount, - documentsFetched, - detailFailures, - detailUnchanged, - initialPendingDocuments, - pendingDocuments.Count, - initialPendingMappings, - pendingMappings.Count, - hasMorePages, - nextWindowStartString, - nextWindowEndString, - nextPage); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToList(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - remainingDocuments.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(); - _logger.LogWarning("CVEs document {DocumentId} missing GridFS content", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - continue; - } - - byte[] rawBytes; - try - { - rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogError(ex, "Unable to download CVE raw document {DocumentId}", documentId); - throw; - } - - CveRecordDto dto; - try - { - dto = CveRecordParser.Parse(rawBytes); - } - catch (JsonException ex) - { - _diagnostics.ParseQuarantine(); - _logger.LogError(ex, "Malformed CVE JSON for {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - continue; - } - - var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - document.Id, - SourceName, - "cve/5.0", - payload, - _timeProvider.GetUtcNow()); - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - _diagnostics.ParseSuccess(); - } - - var updatedCursor = cursor - .WithPendingDocuments(remainingDocuments); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - - if (dtoRecord is null || document is null) - { - _logger.LogWarning("Skipping CVE mapping for {DocumentId}: DTO or document missing", documentId); - pendingMappings.Remove(documentId); - continue; - } - - CveRecordDto dto; - try - { - dto = JsonSerializer.Deserialize<CveRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) - ?? throw new InvalidOperationException("Deserialized DTO was null."); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to deserialize CVE DTO for {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - var recordedAt = dtoRecord.ValidatedAt; - var advisory = CveMapper.Map(dto, document, recordedAt); - - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - _diagnostics.MapSuccess(1); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task<bool> TrySeedFromDirectoryAsync(CveCursor cursor, DateTimeOffset now, CancellationToken cancellationToken) - { - var seedDirectory = _options.SeedDirectory; - if (string.IsNullOrWhiteSpace(seedDirectory) || !Directory.Exists(seedDirectory)) - { - return false; - } - - var detailFiles = Directory.EnumerateFiles(seedDirectory, "CVE-*.json", SearchOption.AllDirectories) - .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - if (detailFiles.Length == 0) - { - return false; - } - - var seeded = 0; - DateTimeOffset? maxModified = cursor.LastModifiedExclusive; - - foreach (var file in detailFiles) - { - cancellationToken.ThrowIfCancellationRequested(); - - byte[] payload; - try - { - payload = await File.ReadAllBytesAsync(file, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Unable to read CVE seed file {File}", file); - continue; - } - - CveRecordDto dto; - try - { - dto = CveRecordParser.Parse(payload); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Seed file {File} did not contain a valid CVE record", file); - continue; - } - - if (string.IsNullOrWhiteSpace(dto.CveId)) - { - _logger.LogWarning("Seed file {File} missing CVE identifier", file); - continue; - } - - var uri = $"seed://{dto.CveId}"; - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri, cancellationToken).ConfigureAwait(false); - var documentId = existing?.Id ?? Guid.NewGuid(); - - var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); - var lastModified = dto.Modified ?? dto.Published ?? now; - ObjectId gridId = ObjectId.Empty; - - try - { - if (existing?.GridFsId is ObjectId existingGrid && existingGrid != ObjectId.Empty) - { - gridId = existingGrid; - } - else - { - gridId = await _rawDocumentStorage.UploadAsync(SourceName, uri, payload, "application/json", cancellationToken).ConfigureAwait(false); - } - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Unable to store CVE seed payload for {CveId}", dto.CveId); - continue; - } - - var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) - { - ["seed.file"] = Path.GetFileName(file), - ["seed.directory"] = seedDirectory, - }; - - var document = new DocumentRecord( - documentId, - SourceName, - uri, - now, - sha256, - DocumentStatuses.Mapped, - "application/json", - Headers: null, - Metadata: metadata, - Etag: null, - LastModified: lastModified, - GridFsId: gridId); - - await _documentStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false); - - var advisory = CveMapper.Map(dto, document, now); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - - if (!maxModified.HasValue || lastModified > maxModified) - { - maxModified = lastModified; - } - - seeded++; - } - - if (seeded == 0) - { - return false; - } - - var updatedCursor = cursor - .WithPendingDocuments(Array.Empty<Guid>()) - .WithPendingMappings(Array.Empty<Guid>()) - .WithLastModifiedExclusive(maxModified ?? now) - .WithCurrentWindowStart(null) - .WithCurrentWindowEnd(null) - .WithNextPage(1); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - - _logger.LogWarning("Seeded {SeededCount} CVE advisories from {SeedDirectory}; live fetch will resume when credentials are configured.", seeded, seedDirectory); - return true; - } - - private static bool IsAuthenticationFailure(HttpRequestException exception) - => exception.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden; - - private async Task<CveCursor> GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? CveCursor.Empty : CveCursor.FromBson(state.Cursor); - } - - private async Task UpdateCursorAsync(CveCursor cursor, CancellationToken cancellationToken) - { - await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); - } - - private static Uri BuildListRequestUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) - { - var query = $"time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(until.ToString("O"))}&page={page}&size={pageSize}"; - return new Uri($"cve?{query}", UriKind.Relative); - } - - private static Uri BuildDetailRequestUri(string cveId) - { - var encoded = Uri.EscapeDataString(cveId); - return new Uri($"cve/{encoded}", UriKind.Relative); - } -} +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Text; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Cve.Configuration; +using StellaOps.Concelier.Connector.Cve.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Cve; + +public sealed class CveConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly CveOptions _options; + private readonly CveDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger<CveConnector> _logger; + + public CveConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<CveOptions> options, + CveDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger<CveConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => CveConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + if (!_options.HasCredentials()) + { + if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) + { + return; + } + + _logger.LogWarning("CVEs fetch skipped: no credentials configured and no seed data found at {SeedDirectory}.", _options.SeedDirectory ?? "(seed directory not configured)"); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var initialPendingDocuments = pendingDocuments.Count; + var initialPendingMappings = pendingMappings.Count; + var documentsFetched = 0; + var detailFailures = 0; + var detailUnchanged = 0; + var listSuccessCount = 0; + var listUnchangedCount = 0; + + var since = cursor.CurrentWindowStart ?? cursor.LastModifiedExclusive ?? now - _options.InitialBackfill; + if (since > now) + { + since = now; + } + + var windowEnd = cursor.CurrentWindowEnd ?? now; + if (windowEnd <= since) + { + windowEnd = since + TimeSpan.FromMinutes(1); + } + + var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; + var pagesFetched = 0; + var hasMorePages = true; + DateTimeOffset? maxModified = cursor.LastModifiedExclusive; + + while (hasMorePages && pagesFetched < _options.MaxPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var requestUri = BuildListRequestUri(since, windowEnd, page, _options.PageSize); + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["since"] = since.ToString("O"), + ["until"] = windowEnd.ToString("O"), + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + }; + + SourceFetchContentResult listResult; + try + { + _diagnostics.FetchAttempt(); + listResult = await _fetchService.FetchContentAsync( + new SourceFetchRequest( + CveOptions.HttpClientName, + SourceName, + requestUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) when (IsAuthenticationFailure(ex)) + { + _logger.LogWarning("CVEs fetch requires API credentials ({StatusCode}); falling back to seed data if available.", ex.StatusCode); + if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) + { + return; + } + + _logger.LogWarning("CVEs fetch aborted: no seed data available (SeedDirectory={SeedDirectory}).", _options.SeedDirectory ?? "(seed directory not configured)"); + return; + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (listResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + listUnchangedCount++; + break; + } + + if (!listResult.IsSuccess || listResult.Content is null) + { + _diagnostics.FetchFailure(); + break; + } + + _diagnostics.FetchSuccess(); + listSuccessCount++; + + var pageModel = CveListParser.Parse(listResult.Content, page, _options.PageSize); + + if (pageModel.Items.Count == 0) + { + hasMorePages = false; + } + + foreach (var item in pageModel.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = BuildDetailRequestUri(item.CveId); + var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["cveId"] = item.CveId, + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["since"] = since.ToString("O"), + ["until"] = windowEnd.ToString("O"), + }; + + SourceFetchResult detailResult; + try + { + detailResult = await _fetchService.FetchAsync( + new SourceFetchRequest( + CveOptions.HttpClientName, + SourceName, + detailUri) + { + Metadata = detailMetadata, + AcceptHeaders = new[] { "application/json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) when (IsAuthenticationFailure(ex)) + { + _diagnostics.FetchFailure(); + _logger.LogWarning(ex, "Failed fetching CVE record {CveId} due to authentication. Seeding if possible.", item.CveId); + if (await TrySeedFromDirectoryAsync(cursor, now, cancellationToken).ConfigureAwait(false)) + { + return; + } + + _logger.LogWarning("CVE record {CveId} skipped; missing credentials and no seed data available.", item.CveId); + continue; + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + _logger.LogWarning(ex, "Failed fetching CVE record {CveId}", item.CveId); + continue; + } + + if (detailResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + detailUnchanged++; + continue; + } + + if (!detailResult.IsSuccess || detailResult.Document is null) + { + _diagnostics.FetchFailure(); + detailFailures++; + continue; + } + + _diagnostics.FetchDocument(); + if (pendingDocuments.Add(detailResult.Document.Id)) + { + documentsFetched++; + } + pendingMappings.Add(detailResult.Document.Id); + } + + if (pageModel.MaxModified.HasValue) + { + if (!maxModified.HasValue || pageModel.MaxModified > maxModified) + { + maxModified = pageModel.MaxModified; + } + } + + hasMorePages = pageModel.HasMorePages; + page = pageModel.NextPageCandidate; + pagesFetched++; + + if (hasMorePages && _options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + if (hasMorePages) + { + updatedCursor = updatedCursor + .WithCurrentWindowStart(since) + .WithCurrentWindowEnd(windowEnd) + .WithNextPage(page); + } + else + { + var nextSince = maxModified ?? windowEnd; + updatedCursor = updatedCursor + .WithLastModifiedExclusive(nextSince) + .WithCurrentWindowStart(null) + .WithCurrentWindowEnd(null) + .WithNextPage(1); + } + + var nextWindowStart = hasMorePages ? since : maxModified ?? windowEnd; + DateTimeOffset? nextWindowEnd = hasMorePages ? windowEnd : null; + var nextPage = hasMorePages ? page : 1; + var windowStartString = since.ToString("O"); + var windowEndString = windowEnd.ToString("O"); + var nextWindowStartString = nextWindowStart.ToString("O"); + var nextWindowEndString = nextWindowEnd?.ToString("O") ?? "(none)"; + + _logger.LogInformation( + "CVEs fetch window {WindowStart}->{WindowEnd} pages={PagesFetched} listSuccess={ListSuccess} detailDocuments={DocumentsFetched} detailFailures={DetailFailures} detailUnchanged={DetailUnchanged} pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter} hasMorePages={HasMorePages} nextWindowStart={NextWindowStart} nextWindowEnd={NextWindowEnd} nextPage={NextPage}", + windowStartString, + windowEndString, + pagesFetched, + listSuccessCount, + documentsFetched, + detailFailures, + detailUnchanged, + initialPendingDocuments, + pendingDocuments.Count, + initialPendingMappings, + pendingMappings.Count, + hasMorePages, + nextWindowStartString, + nextWindowEndString, + nextPage); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("CVEs document {DocumentId} missing GridFS content", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Unable to download CVE raw document {DocumentId}", documentId); + throw; + } + + CveRecordDto dto; + try + { + dto = CveRecordParser.Parse(rawBytes); + } + catch (JsonException ex) + { + _diagnostics.ParseQuarantine(); + _logger.LogError(ex, "Malformed CVE JSON for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "cve/5.0", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + _diagnostics.ParseSuccess(); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + _logger.LogWarning("Skipping CVE mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + CveRecordDto dto; + try + { + dto = JsonSerializer.Deserialize<CveRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize CVE DTO for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var recordedAt = dtoRecord.ValidatedAt; + var advisory = CveMapper.Map(dto, document, recordedAt); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(1); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task<bool> TrySeedFromDirectoryAsync(CveCursor cursor, DateTimeOffset now, CancellationToken cancellationToken) + { + var seedDirectory = _options.SeedDirectory; + if (string.IsNullOrWhiteSpace(seedDirectory) || !Directory.Exists(seedDirectory)) + { + return false; + } + + var detailFiles = Directory.EnumerateFiles(seedDirectory, "CVE-*.json", SearchOption.AllDirectories) + .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (detailFiles.Length == 0) + { + return false; + } + + var seeded = 0; + DateTimeOffset? maxModified = cursor.LastModifiedExclusive; + + foreach (var file in detailFiles) + { + cancellationToken.ThrowIfCancellationRequested(); + + byte[] payload; + try + { + payload = await File.ReadAllBytesAsync(file, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to read CVE seed file {File}", file); + continue; + } + + CveRecordDto dto; + try + { + dto = CveRecordParser.Parse(payload); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Seed file {File} did not contain a valid CVE record", file); + continue; + } + + if (string.IsNullOrWhiteSpace(dto.CveId)) + { + _logger.LogWarning("Seed file {File} missing CVE identifier", file); + continue; + } + + var uri = $"seed://{dto.CveId}"; + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, uri, cancellationToken).ConfigureAwait(false); + var documentId = existing?.Id ?? Guid.NewGuid(); + + var sha256 = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var lastModified = dto.Modified ?? dto.Published ?? now; + ObjectId gridId = ObjectId.Empty; + + try + { + if (existing?.GridFsId is ObjectId existingGrid && existingGrid != ObjectId.Empty) + { + gridId = existingGrid; + } + else + { + gridId = await _rawDocumentStorage.UploadAsync(SourceName, uri, payload, "application/json", cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to store CVE seed payload for {CveId}", dto.CveId); + continue; + } + + var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["seed.file"] = Path.GetFileName(file), + ["seed.directory"] = seedDirectory, + }; + + var document = new DocumentRecord( + documentId, + SourceName, + uri, + now, + sha256, + DocumentStatuses.Mapped, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: lastModified, + GridFsId: gridId); + + await _documentStore.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + + var advisory = CveMapper.Map(dto, document, now); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + + if (!maxModified.HasValue || lastModified > maxModified) + { + maxModified = lastModified; + } + + seeded++; + } + + if (seeded == 0) + { + return false; + } + + var updatedCursor = cursor + .WithPendingDocuments(Array.Empty<Guid>()) + .WithPendingMappings(Array.Empty<Guid>()) + .WithLastModifiedExclusive(maxModified ?? now) + .WithCurrentWindowStart(null) + .WithCurrentWindowEnd(null) + .WithNextPage(1); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + + _logger.LogWarning("Seeded {SeededCount} CVE advisories from {SeedDirectory}; live fetch will resume when credentials are configured.", seeded, seedDirectory); + return true; + } + + private static bool IsAuthenticationFailure(HttpRequestException exception) + => exception.StatusCode is HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden; + + private async Task<CveCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? CveCursor.Empty : CveCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(CveCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private static Uri BuildListRequestUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) + { + var query = $"time_modified.gte={Uri.EscapeDataString(since.ToString("O"))}&time_modified.lte={Uri.EscapeDataString(until.ToString("O"))}&page={page}&size={pageSize}"; + return new Uri($"cve?{query}", UriKind.Relative); + } + + private static Uri BuildDetailRequestUri(string cveId) + { + var encoded = Uri.EscapeDataString(cveId); + return new Uri($"cve/{encoded}", UriKind.Relative); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Cve/CveConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Cve/CveConnectorPlugin.cs index 1e534726..af05f5a7 100644 --- a/src/StellaOps.Feedser.Source.Cve/CveConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Cve/CveConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Cve; - -public sealed class CveConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "cve"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance<CveConnector>(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Cve; + +public sealed class CveConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "cve"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<CveConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Cve/CveDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Cve/CveDependencyInjectionRoutine.cs index 519bf529..c1af9ceb 100644 --- a/src/StellaOps.Feedser.Source.Cve/CveDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Cve/CveDependencyInjectionRoutine.cs @@ -1,54 +1,54 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Cve.Configuration; - -namespace StellaOps.Feedser.Source.Cve; - -public sealed class CveDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:cve"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddCveConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient<CveFetchJob>(); - services.AddTransient<CveParseJob>(); - services.AddTransient<CveMapJob>(); - - services.PostConfigure<JobSchedulerOptions>(options => - { - EnsureJob(options, CveJobKinds.Fetch, typeof(CveFetchJob)); - EnsureJob(options, CveJobKinds.Parse, typeof(CveParseJob)); - EnsureJob(options, CveJobKinds.Map, typeof(CveMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Cve.Configuration; + +namespace StellaOps.Concelier.Connector.Cve; + +public sealed class CveDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:cve"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddCveConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient<CveFetchJob>(); + services.AddTransient<CveParseJob>(); + services.AddTransient<CveMapJob>(); + + services.PostConfigure<JobSchedulerOptions>(options => + { + EnsureJob(options, CveJobKinds.Fetch, typeof(CveFetchJob)); + EnsureJob(options, CveJobKinds.Parse, typeof(CveParseJob)); + EnsureJob(options, CveJobKinds.Map, typeof(CveMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Cve/CveServiceCollectionExtensions.cs similarity index 82% rename from src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Cve/CveServiceCollectionExtensions.cs index 55105ebb..62f8f3c6 100644 --- a/src/StellaOps.Feedser.Source.Cve/CveServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Cve/CveServiceCollectionExtensions.cs @@ -1,41 +1,41 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Cve.Configuration; -using StellaOps.Feedser.Source.Cve.Internal; - -namespace StellaOps.Feedser.Source.Cve; - -public static class CveServiceCollectionExtensions -{ - public static IServiceCollection AddCveConnector(this IServiceCollection services, Action<CveOptions> configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions<CveOptions>() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()); - - services.AddSourceHttpClient(CveOptions.HttpClientName, (sp, clientOptions) => - { - var options = sp.GetRequiredService<IOptions<CveOptions>>().Value; - clientOptions.BaseAddress = options.BaseEndpoint; - clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Cve/1.0"; - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); - clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; - if (options.HasCredentials()) - { - clientOptions.DefaultRequestHeaders["CVE-API-ORG"] = options.ApiOrg; - clientOptions.DefaultRequestHeaders["CVE-API-USER"] = options.ApiUser; - clientOptions.DefaultRequestHeaders["CVE-API-KEY"] = options.ApiKey; - } - }); - - services.AddSingleton<CveDiagnostics>(); - services.AddTransient<CveConnector>(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Cve.Configuration; +using StellaOps.Concelier.Connector.Cve.Internal; + +namespace StellaOps.Concelier.Connector.Cve; + +public static class CveServiceCollectionExtensions +{ + public static IServiceCollection AddCveConnector(this IServiceCollection services, Action<CveOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<CveOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(CveOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService<IOptions<CveOptions>>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Concelier.Cve/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + if (options.HasCredentials()) + { + clientOptions.DefaultRequestHeaders["CVE-API-ORG"] = options.ApiOrg; + clientOptions.DefaultRequestHeaders["CVE-API-USER"] = options.ApiUser; + clientOptions.DefaultRequestHeaders["CVE-API-KEY"] = options.ApiKey; + } + }); + + services.AddSingleton<CveDiagnostics>(); + services.AddTransient<CveConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs index b1a3136d..8f247e86 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveCursor.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveCursor.cs @@ -1,135 +1,135 @@ -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -internal sealed record CveCursor( - DateTimeOffset? LastModifiedExclusive, - DateTimeOffset? CurrentWindowStart, - DateTimeOffset? CurrentWindowEnd, - int NextPage, - IReadOnlyCollection<Guid> PendingDocuments, - IReadOnlyCollection<Guid> PendingMappings) -{ - private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); - - public static CveCursor Empty { get; } = new( - LastModifiedExclusive: null, - CurrentWindowStart: null, - CurrentWindowEnd: null, - NextPage: 1, - PendingDocuments: EmptyGuidList, - PendingMappings: EmptyGuidList); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["nextPage"] = NextPage, - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastModifiedExclusive.HasValue) - { - document["lastModifiedExclusive"] = LastModifiedExclusive.Value.UtcDateTime; - } - - if (CurrentWindowStart.HasValue) - { - document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; - } - - if (CurrentWindowEnd.HasValue) - { - document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; - } - - return document; - } - - public static CveCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var lastModifiedExclusive = document.TryGetValue("lastModifiedExclusive", out var lastModifiedValue) - ? ParseDate(lastModifiedValue) - : null; - var currentWindowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) - ? ParseDate(windowStartValue) - : null; - var currentWindowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) - ? ParseDate(windowEndValue) - : null; - var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 - ? Math.Max(1, nextPageValue.AsInt32) - : 1; - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new CveCursor( - LastModifiedExclusive: lastModifiedExclusive, - CurrentWindowStart: currentWindowStart, - CurrentWindowEnd: currentWindowEnd, - NextPage: nextPage, - PendingDocuments: pendingDocuments, - PendingMappings: pendingMappings); - } - - public CveCursor WithPendingDocuments(IEnumerable<Guid> ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; - - public CveCursor WithPendingMappings(IEnumerable<Guid> ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; - - public CveCursor WithLastModifiedExclusive(DateTimeOffset? timestamp) - => this with { LastModifiedExclusive = timestamp }; - - public CveCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) - => this with { CurrentWindowEnd = timestamp }; - - public CveCursor WithCurrentWindowStart(DateTimeOffset? timestamp) - => this with { CurrentWindowStart = timestamp }; - - public CveCursor WithNextPage(int page) - => this with { NextPage = page < 1 ? 1 : page }; - - private static DateTimeOffset? ParseDate(BsonValue value) - { - return value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuidList; - } - - var results = new List<Guid>(array.Count); - foreach (var element in array) - { - if (element is null) - { - continue; - } - - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } -} +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +internal sealed record CveCursor( + DateTimeOffset? LastModifiedExclusive, + DateTimeOffset? CurrentWindowStart, + DateTimeOffset? CurrentWindowEnd, + int NextPage, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings) +{ + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + + public static CveCursor Empty { get; } = new( + LastModifiedExclusive: null, + CurrentWindowStart: null, + CurrentWindowEnd: null, + NextPage: 1, + PendingDocuments: EmptyGuidList, + PendingMappings: EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["nextPage"] = NextPage, + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastModifiedExclusive.HasValue) + { + document["lastModifiedExclusive"] = LastModifiedExclusive.Value.UtcDateTime; + } + + if (CurrentWindowStart.HasValue) + { + document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; + } + + if (CurrentWindowEnd.HasValue) + { + document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; + } + + return document; + } + + public static CveCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastModifiedExclusive = document.TryGetValue("lastModifiedExclusive", out var lastModifiedValue) + ? ParseDate(lastModifiedValue) + : null; + var currentWindowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) + ? ParseDate(windowStartValue) + : null; + var currentWindowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) + ? ParseDate(windowEndValue) + : null; + var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 + ? Math.Max(1, nextPageValue.AsInt32) + : 1; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new CveCursor( + LastModifiedExclusive: lastModifiedExclusive, + CurrentWindowStart: currentWindowStart, + CurrentWindowEnd: currentWindowEnd, + NextPage: nextPage, + PendingDocuments: pendingDocuments, + PendingMappings: pendingMappings); + } + + public CveCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public CveCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public CveCursor WithLastModifiedExclusive(DateTimeOffset? timestamp) + => this with { LastModifiedExclusive = timestamp }; + + public CveCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) + => this with { CurrentWindowEnd = timestamp }; + + public CveCursor WithCurrentWindowStart(DateTimeOffset? timestamp) + => this with { CurrentWindowStart = timestamp }; + + public CveCursor WithNextPage(int page) + => this with { NextPage = page < 1 ? 1 : page }; + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List<Guid>(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveDiagnostics.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveDiagnostics.cs index 570af127..cc2b1878 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveDiagnostics.cs @@ -1,81 +1,81 @@ -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -public sealed class CveDiagnostics : IDisposable -{ - public const string MeterName = "StellaOps.Feedser.Source.Cve"; - public const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter<long> _fetchAttempts; - private readonly Counter<long> _fetchDocuments; - private readonly Counter<long> _fetchSuccess; - private readonly Counter<long> _fetchFailures; - private readonly Counter<long> _fetchUnchanged; - private readonly Counter<long> _parseSuccess; - private readonly Counter<long> _parseFailures; - private readonly Counter<long> _parseQuarantine; - private readonly Counter<long> _mapSuccess; - - public CveDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchAttempts = _meter.CreateCounter<long>( - name: "cve.fetch.attempts", - unit: "operations", - description: "Number of CVE fetch operations attempted."); - _fetchSuccess = _meter.CreateCounter<long>( - name: "cve.fetch.success", - unit: "operations", - description: "Number of CVE fetch operations that completed successfully."); - _fetchDocuments = _meter.CreateCounter<long>( - name: "cve.fetch.documents", - unit: "documents", - description: "Count of CVE documents fetched and persisted."); - _fetchFailures = _meter.CreateCounter<long>( - name: "cve.fetch.failures", - unit: "operations", - description: "Count of CVE fetch attempts that resulted in an error."); - _fetchUnchanged = _meter.CreateCounter<long>( - name: "cve.fetch.unchanged", - unit: "operations", - description: "Count of CVE fetch attempts returning 304 Not Modified."); - _parseSuccess = _meter.CreateCounter<long>( - name: "cve.parse.success", - unit: "documents", - description: "Count of CVE documents successfully parsed into DTOs."); - _parseFailures = _meter.CreateCounter<long>( - name: "cve.parse.failures", - unit: "documents", - description: "Count of CVE documents that could not be parsed."); - _parseQuarantine = _meter.CreateCounter<long>( - name: "cve.parse.quarantine", - unit: "documents", - description: "Count of CVE documents quarantined after schema validation errors."); - _mapSuccess = _meter.CreateCounter<long>( - name: "cve.map.success", - unit: "advisories", - description: "Count of canonical advisories emitted by the CVE mapper."); - } - - public void FetchAttempt() => _fetchAttempts.Add(1); - - public void FetchDocument() => _fetchDocuments.Add(1); - - public void FetchSuccess() => _fetchSuccess.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void ParseSuccess() => _parseSuccess.Add(1); - - public void ParseFailure() => _parseFailures.Add(1); - - public void ParseQuarantine() => _parseQuarantine.Add(1); - - public void MapSuccess(long count) => _mapSuccess.Add(count); - - public void Dispose() => _meter.Dispose(); -} +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +public sealed class CveDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Concelier.Connector.Cve"; + public const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter<long> _fetchAttempts; + private readonly Counter<long> _fetchDocuments; + private readonly Counter<long> _fetchSuccess; + private readonly Counter<long> _fetchFailures; + private readonly Counter<long> _fetchUnchanged; + private readonly Counter<long> _parseSuccess; + private readonly Counter<long> _parseFailures; + private readonly Counter<long> _parseQuarantine; + private readonly Counter<long> _mapSuccess; + + public CveDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter<long>( + name: "cve.fetch.attempts", + unit: "operations", + description: "Number of CVE fetch operations attempted."); + _fetchSuccess = _meter.CreateCounter<long>( + name: "cve.fetch.success", + unit: "operations", + description: "Number of CVE fetch operations that completed successfully."); + _fetchDocuments = _meter.CreateCounter<long>( + name: "cve.fetch.documents", + unit: "documents", + description: "Count of CVE documents fetched and persisted."); + _fetchFailures = _meter.CreateCounter<long>( + name: "cve.fetch.failures", + unit: "operations", + description: "Count of CVE fetch attempts that resulted in an error."); + _fetchUnchanged = _meter.CreateCounter<long>( + name: "cve.fetch.unchanged", + unit: "operations", + description: "Count of CVE fetch attempts returning 304 Not Modified."); + _parseSuccess = _meter.CreateCounter<long>( + name: "cve.parse.success", + unit: "documents", + description: "Count of CVE documents successfully parsed into DTOs."); + _parseFailures = _meter.CreateCounter<long>( + name: "cve.parse.failures", + unit: "documents", + description: "Count of CVE documents that could not be parsed."); + _parseQuarantine = _meter.CreateCounter<long>( + name: "cve.parse.quarantine", + unit: "documents", + description: "Count of CVE documents quarantined after schema validation errors."); + _mapSuccess = _meter.CreateCounter<long>( + name: "cve.map.success", + unit: "advisories", + description: "Count of canonical advisories emitted by the CVE mapper."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchSuccess() => _fetchSuccess.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count) => _mapSuccess.Add(count); + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveListParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveListParser.cs index 93c3141e..6758b6b1 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveListParser.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveListParser.cs @@ -1,264 +1,264 @@ -using System.Globalization; -using System.Text.Json; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -internal static class CveListParser -{ - public static CveListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) - { - using var document = JsonDocument.Parse(content.ToArray()); - var root = document.RootElement; - - var items = new List<CveListItem>(); - DateTimeOffset? maxModified = null; - - foreach (var element in EnumerateItemElements(root)) - { - var cveId = ExtractCveId(element); - if (string.IsNullOrWhiteSpace(cveId)) - { - continue; - } - - var modified = ExtractModified(element); - if (modified.HasValue && (!maxModified.HasValue || modified > maxModified)) - { - maxModified = modified; - } - - items.Add(new CveListItem(cveId, modified)); - } - - var hasMore = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); - - return new CveListPage(items, maxModified, hasMore, nextPage ?? currentPage + 1); - } - - private static IEnumerable<JsonElement> EnumerateItemElements(JsonElement root) - { - if (root.TryGetProperty("data", out var dataElement) && dataElement.ValueKind == JsonValueKind.Array) - { - foreach (var item in dataElement.EnumerateArray()) - { - yield return item; - } - yield break; - } - - if (root.TryGetProperty("vulnerabilities", out var vulnerabilities) && vulnerabilities.ValueKind == JsonValueKind.Array) - { - foreach (var item in vulnerabilities.EnumerateArray()) - { - yield return item; - } - yield break; - } - - if (root.ValueKind == JsonValueKind.Array) - { - foreach (var item in root.EnumerateArray()) - { - yield return item; - } - } - } - - private static string? ExtractCveId(JsonElement element) - { - if (element.TryGetProperty("cveId", out var cveId) && cveId.ValueKind == JsonValueKind.String) - { - return cveId.GetString(); - } - - if (element.TryGetProperty("cveMetadata", out var metadata)) - { - if (metadata.TryGetProperty("cveId", out var metadataId) && metadataId.ValueKind == JsonValueKind.String) - { - return metadataId.GetString(); - } - } - - if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) - { - if (cve.TryGetProperty("cveMetadata", out var nestedMeta) && nestedMeta.ValueKind == JsonValueKind.Object) - { - if (nestedMeta.TryGetProperty("cveId", out var nestedId) && nestedId.ValueKind == JsonValueKind.String) - { - return nestedId.GetString(); - } - } - - if (cve.TryGetProperty("id", out var cveIdElement) && cveIdElement.ValueKind == JsonValueKind.String) - { - return cveIdElement.GetString(); - } - } - - return null; - } - - private static DateTimeOffset? ExtractModified(JsonElement element) - { - static DateTimeOffset? Parse(JsonElement candidate) - { - return candidate.ValueKind switch - { - JsonValueKind.String when DateTimeOffset.TryParse(candidate.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) - => parsed.ToUniversalTime(), - _ => null, - }; - } - - if (element.TryGetProperty("dateUpdated", out var dateUpdated)) - { - var parsed = Parse(dateUpdated); - if (parsed.HasValue) - { - return parsed; - } - } - - if (element.TryGetProperty("cveMetadata", out var metadata)) - { - if (metadata.TryGetProperty("dateUpdated", out var metadataUpdated)) - { - var parsed = Parse(metadataUpdated); - if (parsed.HasValue) - { - return parsed; - } - } - } - - if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) - { - if (cve.TryGetProperty("cveMetadata", out var nestedMeta)) - { - if (nestedMeta.TryGetProperty("dateUpdated", out var nestedUpdated)) - { - var parsed = Parse(nestedUpdated); - if (parsed.HasValue) - { - return parsed; - } - } - } - - if (cve.TryGetProperty("lastModified", out var lastModified)) - { - var parsed = Parse(lastModified); - if (parsed.HasValue) - { - return parsed; - } - } - } - - return null; - } - - private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) - { - nextPage = null; - - if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) - { - var totalPages = TryGetInt(pagination, "totalPages") - ?? TryGetInt(pagination, "pageCount") - ?? TryGetInt(pagination, "totalPagesCount"); - if (totalPages.HasValue) - { - if (currentPage < totalPages.Value) - { - nextPage = currentPage + 1; - return true; - } - - return false; - } - - var totalCount = TryGetInt(pagination, "totalCount") - ?? TryGetInt(pagination, "totalResults"); - var limit = TryGetInt(pagination, "limit") - ?? TryGetInt(pagination, "itemsPerPage") - ?? TryGetInt(pagination, "pageSize") - ?? pageSize; - - if (totalCount.HasValue) - { - var processed = (currentPage - 1) * limit + itemCount; - if (processed < totalCount.Value) - { - nextPage = currentPage + 1; - return true; - } - - return false; - } - - if (pagination.TryGetProperty("nextPage", out var nextPageElement)) - { - switch (nextPageElement.ValueKind) - { - case JsonValueKind.Number when nextPageElement.TryGetInt32(out var value): - nextPage = value; - return true; - case JsonValueKind.String when int.TryParse(nextPageElement.GetString(), out var parsed): - nextPage = parsed; - return true; - case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageElement.GetString()): - nextPage = currentPage + 1; - return true; - } - } - } - - if (root.TryGetProperty("nextPage", out var nextPageValue)) - { - switch (nextPageValue.ValueKind) - { - case JsonValueKind.Number when nextPageValue.TryGetInt32(out var value): - nextPage = value; - return true; - case JsonValueKind.String when int.TryParse(nextPageValue.GetString(), out var parsed): - nextPage = parsed; - return true; - case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageValue.GetString()): - nextPage = currentPage + 1; - return true; - } - } - - if (itemCount >= pageSize) - { - nextPage = currentPage + 1; - return true; - } - - return false; - } - - private static int? TryGetInt(JsonElement element, string propertyName) - { - if (!element.TryGetProperty(propertyName, out var value)) - { - return null; - } - - return value.ValueKind switch - { - JsonValueKind.Number when value.TryGetInt32(out var number) => number, - JsonValueKind.String when int.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed, - _ => null, - }; - } -} - -internal sealed record CveListPage( - IReadOnlyList<CveListItem> Items, - DateTimeOffset? MaxModified, - bool HasMorePages, - int NextPageCandidate); - -internal sealed record CveListItem(string CveId, DateTimeOffset? DateUpdated); +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +internal static class CveListParser +{ + public static CveListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var items = new List<CveListItem>(); + DateTimeOffset? maxModified = null; + + foreach (var element in EnumerateItemElements(root)) + { + var cveId = ExtractCveId(element); + if (string.IsNullOrWhiteSpace(cveId)) + { + continue; + } + + var modified = ExtractModified(element); + if (modified.HasValue && (!maxModified.HasValue || modified > maxModified)) + { + maxModified = modified; + } + + items.Add(new CveListItem(cveId, modified)); + } + + var hasMore = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); + + return new CveListPage(items, maxModified, hasMore, nextPage ?? currentPage + 1); + } + + private static IEnumerable<JsonElement> EnumerateItemElements(JsonElement root) + { + if (root.TryGetProperty("data", out var dataElement) && dataElement.ValueKind == JsonValueKind.Array) + { + foreach (var item in dataElement.EnumerateArray()) + { + yield return item; + } + yield break; + } + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilities) && vulnerabilities.ValueKind == JsonValueKind.Array) + { + foreach (var item in vulnerabilities.EnumerateArray()) + { + yield return item; + } + yield break; + } + + if (root.ValueKind == JsonValueKind.Array) + { + foreach (var item in root.EnumerateArray()) + { + yield return item; + } + } + } + + private static string? ExtractCveId(JsonElement element) + { + if (element.TryGetProperty("cveId", out var cveId) && cveId.ValueKind == JsonValueKind.String) + { + return cveId.GetString(); + } + + if (element.TryGetProperty("cveMetadata", out var metadata)) + { + if (metadata.TryGetProperty("cveId", out var metadataId) && metadataId.ValueKind == JsonValueKind.String) + { + return metadataId.GetString(); + } + } + + if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) + { + if (cve.TryGetProperty("cveMetadata", out var nestedMeta) && nestedMeta.ValueKind == JsonValueKind.Object) + { + if (nestedMeta.TryGetProperty("cveId", out var nestedId) && nestedId.ValueKind == JsonValueKind.String) + { + return nestedId.GetString(); + } + } + + if (cve.TryGetProperty("id", out var cveIdElement) && cveIdElement.ValueKind == JsonValueKind.String) + { + return cveIdElement.GetString(); + } + } + + return null; + } + + private static DateTimeOffset? ExtractModified(JsonElement element) + { + static DateTimeOffset? Parse(JsonElement candidate) + { + return candidate.ValueKind switch + { + JsonValueKind.String when DateTimeOffset.TryParse(candidate.GetString(), CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + => parsed.ToUniversalTime(), + _ => null, + }; + } + + if (element.TryGetProperty("dateUpdated", out var dateUpdated)) + { + var parsed = Parse(dateUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + + if (element.TryGetProperty("cveMetadata", out var metadata)) + { + if (metadata.TryGetProperty("dateUpdated", out var metadataUpdated)) + { + var parsed = Parse(metadataUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + } + + if (element.TryGetProperty("cve", out var cve) && cve.ValueKind == JsonValueKind.Object) + { + if (cve.TryGetProperty("cveMetadata", out var nestedMeta)) + { + if (nestedMeta.TryGetProperty("dateUpdated", out var nestedUpdated)) + { + var parsed = Parse(nestedUpdated); + if (parsed.HasValue) + { + return parsed; + } + } + } + + if (cve.TryGetProperty("lastModified", out var lastModified)) + { + var parsed = Parse(lastModified); + if (parsed.HasValue) + { + return parsed; + } + } + } + + return null; + } + + private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) + { + nextPage = null; + + if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) + { + var totalPages = TryGetInt(pagination, "totalPages") + ?? TryGetInt(pagination, "pageCount") + ?? TryGetInt(pagination, "totalPagesCount"); + if (totalPages.HasValue) + { + if (currentPage < totalPages.Value) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + var totalCount = TryGetInt(pagination, "totalCount") + ?? TryGetInt(pagination, "totalResults"); + var limit = TryGetInt(pagination, "limit") + ?? TryGetInt(pagination, "itemsPerPage") + ?? TryGetInt(pagination, "pageSize") + ?? pageSize; + + if (totalCount.HasValue) + { + var processed = (currentPage - 1) * limit + itemCount; + if (processed < totalCount.Value) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + if (pagination.TryGetProperty("nextPage", out var nextPageElement)) + { + switch (nextPageElement.ValueKind) + { + case JsonValueKind.Number when nextPageElement.TryGetInt32(out var value): + nextPage = value; + return true; + case JsonValueKind.String when int.TryParse(nextPageElement.GetString(), out var parsed): + nextPage = parsed; + return true; + case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageElement.GetString()): + nextPage = currentPage + 1; + return true; + } + } + } + + if (root.TryGetProperty("nextPage", out var nextPageValue)) + { + switch (nextPageValue.ValueKind) + { + case JsonValueKind.Number when nextPageValue.TryGetInt32(out var value): + nextPage = value; + return true; + case JsonValueKind.String when int.TryParse(nextPageValue.GetString(), out var parsed): + nextPage = parsed; + return true; + case JsonValueKind.String when !string.IsNullOrWhiteSpace(nextPageValue.GetString()): + nextPage = currentPage + 1; + return true; + } + } + + if (itemCount >= pageSize) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + private static int? TryGetInt(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind switch + { + JsonValueKind.Number when value.TryGetInt32(out var number) => number, + JsonValueKind.String when int.TryParse(value.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) => parsed, + _ => null, + }; + } +} + +internal sealed record CveListPage( + IReadOnlyList<CveListItem> Items, + DateTimeOffset? MaxModified, + bool HasMorePages, + int NextPageCandidate); + +internal sealed record CveListItem(string CveId, DateTimeOffset? DateUpdated); diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveMapper.cs index 83fbcc1a..86890bfc 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveMapper.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveMapper.cs @@ -1,451 +1,451 @@ -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Storage.Mongo.Documents; -using NuGet.Versioning; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -internal static class CveMapper -{ - private static readonly string[] SeverityOrder = - { - "critical", - "high", - "medium", - "low", - "informational", - "none", - "unknown", - }; - - public static Advisory Map(CveRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var fetchProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); - var mapProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "mapping", dto.CveId, recordedAt); - - var aliases = dto.Aliases - .Append(dto.CveId) - .Where(static alias => !string.IsNullOrWhiteSpace(alias)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - var references = dto.References - .Select(reference => CreateReference(reference, recordedAt)) - .Where(static reference => reference is not null) - .Cast<AdvisoryReference>() - .ToList(); - - var affected = CreateAffectedPackages(dto, recordedAt); - var cvssMetrics = CreateCvssMetrics(dto, recordedAt, document.Uri); - var severity = DetermineSeverity(cvssMetrics); - - var provenance = new[] - { - fetchProvenance, - mapProvenance, - }; - - var title = string.IsNullOrWhiteSpace(dto.Title) ? dto.CveId : dto.Title!; - - return new Advisory( - advisoryKey: dto.CveId, - title: title, - summary: dto.Summary, - language: dto.Language, - published: dto.Published, - modified: dto.Modified ?? dto.Published, - severity: severity, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affected, - cvssMetrics: cvssMetrics, - provenance: provenance); - } - - private static AdvisoryReference? CreateReference(CveReferenceDto dto, DateTimeOffset recordedAt) - { - if (string.IsNullOrWhiteSpace(dto.Url) || !Validation.LooksLikeHttpUrl(dto.Url)) - { - return null; - } - - var kind = dto.Tags.FirstOrDefault(); - return new AdvisoryReference( - dto.Url, - kind, - dto.Source, - summary: null, - provenance: new AdvisoryProvenance(CveConnectorPlugin.SourceName, "reference", dto.Url, recordedAt)); - } - - private static IReadOnlyList<AffectedPackage> CreateAffectedPackages(CveRecordDto dto, DateTimeOffset recordedAt) - { - if (dto.Affected.Count == 0) - { - return Array.Empty<AffectedPackage>(); - } - - var packages = new List<AffectedPackage>(dto.Affected.Count); - foreach (var affected in dto.Affected) - { - var vendor = string.IsNullOrWhiteSpace(affected.Vendor) ? "unknown-vendor" : affected.Vendor!.Trim(); - var product = string.IsNullOrWhiteSpace(affected.Product) ? "unknown-product" : affected.Product!.Trim(); - var identifier = string.Equals(product, vendor, StringComparison.OrdinalIgnoreCase) - ? vendor.ToLowerInvariant() - : $"{vendor}:{product}".ToLowerInvariant(); - - var provenance = new[] - { - new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected", identifier, recordedAt), - }; - - var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) - { - ["vendor"] = vendor, - ["product"] = product, - }; - if (!string.IsNullOrWhiteSpace(affected.Platform)) - { - vendorExtensions["platform"] = affected.Platform!; - } - - var note = BuildNormalizedNote(dto.CveId, identifier); - var (ranges, normalizedVersions) = CreateVersionArtifacts(affected, recordedAt, identifier, vendorExtensions, note); - var statuses = CreateStatuses(affected, recordedAt, identifier); - - if (ranges.Count == 0) - { - var fallbackPrimitives = vendorExtensions.Count == 0 - ? null - : new RangePrimitives(null, null, null, vendorExtensions); - - ranges.Add(new AffectedVersionRange( - rangeKind: "vendor", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: null, - provenance: provenance[0], - primitives: fallbackPrimitives)); - } - - packages.Add(new AffectedPackage( - type: AffectedPackageTypes.Vendor, - identifier: identifier, - platform: affected.Platform, - versionRanges: ranges, - statuses: statuses, - provenance: provenance, - normalizedVersions: normalizedVersions.Count == 0 - ? Array.Empty<NormalizedVersionRule>() - : normalizedVersions.ToArray())); - } - - return packages; - } - - private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> Normalized) CreateVersionArtifacts( - CveAffectedDto affected, - DateTimeOffset recordedAt, - string identifier, - IReadOnlyDictionary<string, string> vendorExtensions, - string normalizedNote) - { - var ranges = new List<AffectedVersionRange>(); - var normalized = new List<NormalizedVersionRule>(); - - foreach (var version in affected.Versions) - { - var range = BuildVersionRange(version, recordedAt, identifier, vendorExtensions); - if (range is null) - { - continue; - } - - ranges.Add(range); - - var rule = range.ToNormalizedVersionRule(normalizedNote); - if (rule is not null) - { - normalized.Add(rule); - } - } - - return (ranges, normalized); - } - - private static AffectedVersionRange? BuildVersionRange( - CveVersionDto version, - DateTimeOffset recordedAt, - string identifier, - IReadOnlyDictionary<string, string> baseVendorExtensions) - { - var vendor = new Dictionary<string, string>(baseVendorExtensions, StringComparer.OrdinalIgnoreCase); - - void AddExtension(string key, string? value) - { - if (!string.IsNullOrWhiteSpace(value)) - { - vendor[key] = value.Trim(); - } - } - - AddExtension("version", version.Version); - AddExtension("lessThan", version.LessThan); - AddExtension("lessThanOrEqual", version.LessThanOrEqual); - AddExtension("versionType", version.VersionType); - AddExtension("versionRange", version.Range); - - var introduced = Normalize(version.Version); - var fixedExclusive = Normalize(version.LessThan); - var lastInclusive = Normalize(version.LessThanOrEqual); - - var rangeExpression = Normalize(string.IsNullOrWhiteSpace(version.Range) - ? BuildRangeExpression(version) - : version.Range); - - var provenance = new AdvisoryProvenance( - CveConnectorPlugin.SourceName, - "affected-range", - identifier, - recordedAt); - - var semVerPrimitive = TryBuildSemVerPrimitive( - version.VersionType, - introduced, - fixedExclusive, - lastInclusive, - rangeExpression, - out var primitive); - - if (semVerPrimitive) - { - introduced = primitive!.Introduced ?? introduced; - fixedExclusive = primitive.Fixed ?? fixedExclusive; - lastInclusive = primitive.LastAffected ?? lastInclusive; - } - - if (introduced is null && fixedExclusive is null && lastInclusive is null && rangeExpression is null && primitive is null && vendor.Count == baseVendorExtensions.Count) - { - return null; - } - - var rangeKind = primitive is not null - ? NormalizedVersionSchemes.SemVer - : (string.IsNullOrWhiteSpace(version.VersionType) - ? "vendor" - : version.VersionType!.Trim().ToLowerInvariant()); - - var rangePrimitives = primitive is null && vendor.Count == 0 - ? null - : new RangePrimitives( - primitive, - Nevra: null, - Evr: null, - VendorExtensions: vendor.Count == 0 ? null : vendor); - - return new AffectedVersionRange( - rangeKind: rangeKind, - introducedVersion: introduced, - fixedVersion: fixedExclusive, - lastAffectedVersion: lastInclusive, - rangeExpression: rangeExpression, - provenance: provenance, - primitives: rangePrimitives); - } - - private static List<AffectedPackageStatus> CreateStatuses(CveAffectedDto affected, DateTimeOffset recordedAt, string identifier) - { - var statuses = new List<AffectedPackageStatus>(); - - void AddStatus(string? status) - { - if (string.IsNullOrWhiteSpace(status)) - { - return; - } - - statuses.Add(new AffectedPackageStatus( - status, - new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected-status", identifier, recordedAt))); - } - - AddStatus(affected.DefaultStatus); - - foreach (var version in affected.Versions) - { - AddStatus(version.Status); - } - - return statuses; - } - - private static string? Normalize(string? value) - => string.IsNullOrWhiteSpace(value) || value is "*" or "-" ? null : value.Trim(); - - private static string? BuildRangeExpression(CveVersionDto version) - { - var builder = new List<string>(); - if (!string.IsNullOrWhiteSpace(version.Version)) - { - builder.Add($"version={version.Version}"); - } - - if (!string.IsNullOrWhiteSpace(version.LessThan)) - { - builder.Add($"< {version.LessThan}"); - } - - if (!string.IsNullOrWhiteSpace(version.LessThanOrEqual)) - { - builder.Add($"<= {version.LessThanOrEqual}"); - } - - if (builder.Count == 0) - { - return null; - } - - return string.Join(", ", builder); - } - - private static string BuildNormalizedNote(string cveId, string identifier) - { - var baseId = string.IsNullOrWhiteSpace(cveId) - ? "unknown" - : cveId.Trim().ToLowerInvariant(); - return $"cve:{baseId}:{identifier}"; - } - - private static bool TryBuildSemVerPrimitive( - string? versionType, - string? introduced, - string? fixedExclusive, - string? lastInclusive, - string? constraintExpression, - out SemVerPrimitive? primitive) - { - primitive = null; - - if (!string.Equals(versionType, "semver", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - if (!TryNormalizeSemVer(introduced, out var normalizedIntroduced) - || !TryNormalizeSemVer(fixedExclusive, out var normalizedFixed) - || !TryNormalizeSemVer(lastInclusive, out var normalizedLast)) - { - normalizedIntroduced = introduced; - normalizedFixed = fixedExclusive; - normalizedLast = lastInclusive; - return false; - } - - if (normalizedIntroduced is null && normalizedFixed is null && normalizedLast is null) - { - return false; - } - - var introducedInclusive = true; - var fixedInclusive = false; - var lastInclusiveFlag = true; - - if (normalizedFixed is null && normalizedLast is null && normalizedIntroduced is not null) - { - // Exact version. Treat as introduced/fixed equality. - normalizedFixed = normalizedIntroduced; - normalizedLast = normalizedIntroduced; - } - - primitive = new SemVerPrimitive( - Introduced: normalizedIntroduced, - IntroducedInclusive: introducedInclusive, - Fixed: normalizedFixed, - FixedInclusive: fixedInclusive, - LastAffected: normalizedLast, - LastAffectedInclusive: lastInclusiveFlag, - ConstraintExpression: constraintExpression, - ExactValue: normalizedFixed is not null && normalizedIntroduced is not null && normalizedFixed == normalizedIntroduced - ? normalizedIntroduced - : null); - - return true; - } - - private static bool TryNormalizeSemVer(string? value, out string? normalized) - { - normalized = null; - if (string.IsNullOrWhiteSpace(value)) - { - return true; - } - - var trimmed = value.Trim(); - if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 1) - { - trimmed = trimmed[1..]; - } - - if (!NuGetVersion.TryParse(trimmed, out var parsed)) - { - return false; - } - - normalized = parsed.ToNormalizedString(); - return true; - } - - private static IReadOnlyList<CvssMetric> CreateCvssMetrics(CveRecordDto dto, DateTimeOffset recordedAt, string sourceUri) - { - if (dto.Metrics.Count == 0) - { - return Array.Empty<CvssMetric>(); - } - - var provenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "cvss", sourceUri, recordedAt); - var metrics = new List<CvssMetric>(dto.Metrics.Count); - foreach (var metric in dto.Metrics) - { - if (!CvssMetricNormalizer.TryNormalize(metric.Version, metric.Vector, metric.BaseScore, metric.BaseSeverity, out var normalized)) - { - continue; - } - - metrics.Add(new CvssMetric( - normalized.Version, - normalized.Vector, - normalized.BaseScore, - normalized.BaseSeverity, - provenance)); - } - - return metrics; - } - - private static string? DetermineSeverity(IReadOnlyList<CvssMetric> metrics) - { - if (metrics.Count == 0) - { - return null; - } - - foreach (var level in SeverityOrder) - { - if (metrics.Any(metric => string.Equals(metric.BaseSeverity, level, StringComparison.OrdinalIgnoreCase))) - { - return level; - } - } - - return metrics - .Select(metric => metric.BaseSeverity) - .FirstOrDefault(); - } -} +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Storage.Mongo.Documents; +using NuGet.Versioning; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +internal static class CveMapper +{ + private static readonly string[] SeverityOrder = + { + "critical", + "high", + "medium", + "low", + "informational", + "none", + "unknown", + }; + + public static Advisory Map(CveRecordDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var fetchProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "document", document.Uri, document.FetchedAt); + var mapProvenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "mapping", dto.CveId, recordedAt); + + var aliases = dto.Aliases + .Append(dto.CveId) + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var references = dto.References + .Select(reference => CreateReference(reference, recordedAt)) + .Where(static reference => reference is not null) + .Cast<AdvisoryReference>() + .ToList(); + + var affected = CreateAffectedPackages(dto, recordedAt); + var cvssMetrics = CreateCvssMetrics(dto, recordedAt, document.Uri); + var severity = DetermineSeverity(cvssMetrics); + + var provenance = new[] + { + fetchProvenance, + mapProvenance, + }; + + var title = string.IsNullOrWhiteSpace(dto.Title) ? dto.CveId : dto.Title!; + + return new Advisory( + advisoryKey: dto.CveId, + title: title, + summary: dto.Summary, + language: dto.Language, + published: dto.Published, + modified: dto.Modified ?? dto.Published, + severity: severity, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affected, + cvssMetrics: cvssMetrics, + provenance: provenance); + } + + private static AdvisoryReference? CreateReference(CveReferenceDto dto, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(dto.Url) || !Validation.LooksLikeHttpUrl(dto.Url)) + { + return null; + } + + var kind = dto.Tags.FirstOrDefault(); + return new AdvisoryReference( + dto.Url, + kind, + dto.Source, + summary: null, + provenance: new AdvisoryProvenance(CveConnectorPlugin.SourceName, "reference", dto.Url, recordedAt)); + } + + private static IReadOnlyList<AffectedPackage> CreateAffectedPackages(CveRecordDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty<AffectedPackage>(); + } + + var packages = new List<AffectedPackage>(dto.Affected.Count); + foreach (var affected in dto.Affected) + { + var vendor = string.IsNullOrWhiteSpace(affected.Vendor) ? "unknown-vendor" : affected.Vendor!.Trim(); + var product = string.IsNullOrWhiteSpace(affected.Product) ? "unknown-product" : affected.Product!.Trim(); + var identifier = string.Equals(product, vendor, StringComparison.OrdinalIgnoreCase) + ? vendor.ToLowerInvariant() + : $"{vendor}:{product}".ToLowerInvariant(); + + var provenance = new[] + { + new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected", identifier, recordedAt), + }; + + var vendorExtensions = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["vendor"] = vendor, + ["product"] = product, + }; + if (!string.IsNullOrWhiteSpace(affected.Platform)) + { + vendorExtensions["platform"] = affected.Platform!; + } + + var note = BuildNormalizedNote(dto.CveId, identifier); + var (ranges, normalizedVersions) = CreateVersionArtifacts(affected, recordedAt, identifier, vendorExtensions, note); + var statuses = CreateStatuses(affected, recordedAt, identifier); + + if (ranges.Count == 0) + { + var fallbackPrimitives = vendorExtensions.Count == 0 + ? null + : new RangePrimitives(null, null, null, vendorExtensions); + + ranges.Add(new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance[0], + primitives: fallbackPrimitives)); + } + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: identifier, + platform: affected.Platform, + versionRanges: ranges, + statuses: statuses, + provenance: provenance, + normalizedVersions: normalizedVersions.Count == 0 + ? Array.Empty<NormalizedVersionRule>() + : normalizedVersions.ToArray())); + } + + return packages; + } + + private static (List<AffectedVersionRange> Ranges, List<NormalizedVersionRule> Normalized) CreateVersionArtifacts( + CveAffectedDto affected, + DateTimeOffset recordedAt, + string identifier, + IReadOnlyDictionary<string, string> vendorExtensions, + string normalizedNote) + { + var ranges = new List<AffectedVersionRange>(); + var normalized = new List<NormalizedVersionRule>(); + + foreach (var version in affected.Versions) + { + var range = BuildVersionRange(version, recordedAt, identifier, vendorExtensions); + if (range is null) + { + continue; + } + + ranges.Add(range); + + var rule = range.ToNormalizedVersionRule(normalizedNote); + if (rule is not null) + { + normalized.Add(rule); + } + } + + return (ranges, normalized); + } + + private static AffectedVersionRange? BuildVersionRange( + CveVersionDto version, + DateTimeOffset recordedAt, + string identifier, + IReadOnlyDictionary<string, string> baseVendorExtensions) + { + var vendor = new Dictionary<string, string>(baseVendorExtensions, StringComparer.OrdinalIgnoreCase); + + void AddExtension(string key, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + vendor[key] = value.Trim(); + } + } + + AddExtension("version", version.Version); + AddExtension("lessThan", version.LessThan); + AddExtension("lessThanOrEqual", version.LessThanOrEqual); + AddExtension("versionType", version.VersionType); + AddExtension("versionRange", version.Range); + + var introduced = Normalize(version.Version); + var fixedExclusive = Normalize(version.LessThan); + var lastInclusive = Normalize(version.LessThanOrEqual); + + var rangeExpression = Normalize(string.IsNullOrWhiteSpace(version.Range) + ? BuildRangeExpression(version) + : version.Range); + + var provenance = new AdvisoryProvenance( + CveConnectorPlugin.SourceName, + "affected-range", + identifier, + recordedAt); + + var semVerPrimitive = TryBuildSemVerPrimitive( + version.VersionType, + introduced, + fixedExclusive, + lastInclusive, + rangeExpression, + out var primitive); + + if (semVerPrimitive) + { + introduced = primitive!.Introduced ?? introduced; + fixedExclusive = primitive.Fixed ?? fixedExclusive; + lastInclusive = primitive.LastAffected ?? lastInclusive; + } + + if (introduced is null && fixedExclusive is null && lastInclusive is null && rangeExpression is null && primitive is null && vendor.Count == baseVendorExtensions.Count) + { + return null; + } + + var rangeKind = primitive is not null + ? NormalizedVersionSchemes.SemVer + : (string.IsNullOrWhiteSpace(version.VersionType) + ? "vendor" + : version.VersionType!.Trim().ToLowerInvariant()); + + var rangePrimitives = primitive is null && vendor.Count == 0 + ? null + : new RangePrimitives( + primitive, + Nevra: null, + Evr: null, + VendorExtensions: vendor.Count == 0 ? null : vendor); + + return new AffectedVersionRange( + rangeKind: rangeKind, + introducedVersion: introduced, + fixedVersion: fixedExclusive, + lastAffectedVersion: lastInclusive, + rangeExpression: rangeExpression, + provenance: provenance, + primitives: rangePrimitives); + } + + private static List<AffectedPackageStatus> CreateStatuses(CveAffectedDto affected, DateTimeOffset recordedAt, string identifier) + { + var statuses = new List<AffectedPackageStatus>(); + + void AddStatus(string? status) + { + if (string.IsNullOrWhiteSpace(status)) + { + return; + } + + statuses.Add(new AffectedPackageStatus( + status, + new AdvisoryProvenance(CveConnectorPlugin.SourceName, "affected-status", identifier, recordedAt))); + } + + AddStatus(affected.DefaultStatus); + + foreach (var version in affected.Versions) + { + AddStatus(version.Status); + } + + return statuses; + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) || value is "*" or "-" ? null : value.Trim(); + + private static string? BuildRangeExpression(CveVersionDto version) + { + var builder = new List<string>(); + if (!string.IsNullOrWhiteSpace(version.Version)) + { + builder.Add($"version={version.Version}"); + } + + if (!string.IsNullOrWhiteSpace(version.LessThan)) + { + builder.Add($"< {version.LessThan}"); + } + + if (!string.IsNullOrWhiteSpace(version.LessThanOrEqual)) + { + builder.Add($"<= {version.LessThanOrEqual}"); + } + + if (builder.Count == 0) + { + return null; + } + + return string.Join(", ", builder); + } + + private static string BuildNormalizedNote(string cveId, string identifier) + { + var baseId = string.IsNullOrWhiteSpace(cveId) + ? "unknown" + : cveId.Trim().ToLowerInvariant(); + return $"cve:{baseId}:{identifier}"; + } + + private static bool TryBuildSemVerPrimitive( + string? versionType, + string? introduced, + string? fixedExclusive, + string? lastInclusive, + string? constraintExpression, + out SemVerPrimitive? primitive) + { + primitive = null; + + if (!string.Equals(versionType, "semver", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (!TryNormalizeSemVer(introduced, out var normalizedIntroduced) + || !TryNormalizeSemVer(fixedExclusive, out var normalizedFixed) + || !TryNormalizeSemVer(lastInclusive, out var normalizedLast)) + { + normalizedIntroduced = introduced; + normalizedFixed = fixedExclusive; + normalizedLast = lastInclusive; + return false; + } + + if (normalizedIntroduced is null && normalizedFixed is null && normalizedLast is null) + { + return false; + } + + var introducedInclusive = true; + var fixedInclusive = false; + var lastInclusiveFlag = true; + + if (normalizedFixed is null && normalizedLast is null && normalizedIntroduced is not null) + { + // Exact version. Treat as introduced/fixed equality. + normalizedFixed = normalizedIntroduced; + normalizedLast = normalizedIntroduced; + } + + primitive = new SemVerPrimitive( + Introduced: normalizedIntroduced, + IntroducedInclusive: introducedInclusive, + Fixed: normalizedFixed, + FixedInclusive: fixedInclusive, + LastAffected: normalizedLast, + LastAffectedInclusive: lastInclusiveFlag, + ConstraintExpression: constraintExpression, + ExactValue: normalizedFixed is not null && normalizedIntroduced is not null && normalizedFixed == normalizedIntroduced + ? normalizedIntroduced + : null); + + return true; + } + + private static bool TryNormalizeSemVer(string? value, out string? normalized) + { + normalized = null; + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + var trimmed = value.Trim(); + if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase) && trimmed.Length > 1) + { + trimmed = trimmed[1..]; + } + + if (!NuGetVersion.TryParse(trimmed, out var parsed)) + { + return false; + } + + normalized = parsed.ToNormalizedString(); + return true; + } + + private static IReadOnlyList<CvssMetric> CreateCvssMetrics(CveRecordDto dto, DateTimeOffset recordedAt, string sourceUri) + { + if (dto.Metrics.Count == 0) + { + return Array.Empty<CvssMetric>(); + } + + var provenance = new AdvisoryProvenance(CveConnectorPlugin.SourceName, "cvss", sourceUri, recordedAt); + var metrics = new List<CvssMetric>(dto.Metrics.Count); + foreach (var metric in dto.Metrics) + { + if (!CvssMetricNormalizer.TryNormalize(metric.Version, metric.Vector, metric.BaseScore, metric.BaseSeverity, out var normalized)) + { + continue; + } + + metrics.Add(new CvssMetric( + normalized.Version, + normalized.Vector, + normalized.BaseScore, + normalized.BaseSeverity, + provenance)); + } + + return metrics; + } + + private static string? DetermineSeverity(IReadOnlyList<CvssMetric> metrics) + { + if (metrics.Count == 0) + { + return null; + } + + foreach (var level in SeverityOrder) + { + if (metrics.Any(metric => string.Equals(metric.BaseSeverity, level, StringComparison.OrdinalIgnoreCase))) + { + return level; + } + } + + return metrics + .Select(metric => metric.BaseSeverity) + .FirstOrDefault(); + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordDto.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordDto.cs index 8dbe9d54..95f1a57f 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordDto.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordDto.cs @@ -1,105 +1,105 @@ -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -internal sealed record CveRecordDto -{ - [JsonPropertyName("cveId")] - public string CveId { get; init; } = string.Empty; - - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("language")] - public string? Language { get; init; } - - [JsonPropertyName("state")] - public string State { get; init; } = "PUBLISHED"; - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("modified")] - public DateTimeOffset? Modified { get; init; } - - [JsonPropertyName("aliases")] - public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); - - [JsonPropertyName("references")] - public IReadOnlyList<CveReferenceDto> References { get; init; } = Array.Empty<CveReferenceDto>(); - - [JsonPropertyName("affected")] - public IReadOnlyList<CveAffectedDto> Affected { get; init; } = Array.Empty<CveAffectedDto>(); - - [JsonPropertyName("metrics")] - public IReadOnlyList<CveCvssMetricDto> Metrics { get; init; } = Array.Empty<CveCvssMetricDto>(); -} - -internal sealed record CveReferenceDto -{ - [JsonPropertyName("url")] - public string Url { get; init; } = string.Empty; - - [JsonPropertyName("source")] - public string? Source { get; init; } - - [JsonPropertyName("tags")] - public IReadOnlyList<string> Tags { get; init; } = Array.Empty<string>(); -} - -internal sealed record CveAffectedDto -{ - [JsonPropertyName("vendor")] - public string? Vendor { get; init; } - - [JsonPropertyName("product")] - public string? Product { get; init; } - - [JsonPropertyName("platform")] - public string? Platform { get; init; } - - [JsonPropertyName("defaultStatus")] - public string? DefaultStatus { get; init; } - - [JsonPropertyName("versions")] - public IReadOnlyList<CveVersionDto> Versions { get; init; } = Array.Empty<CveVersionDto>(); -} - -internal sealed record CveVersionDto -{ - [JsonPropertyName("status")] - public string? Status { get; init; } - - [JsonPropertyName("version")] - public string? Version { get; init; } - - [JsonPropertyName("lessThan")] - public string? LessThan { get; init; } - - [JsonPropertyName("lessThanOrEqual")] - public string? LessThanOrEqual { get; init; } - - [JsonPropertyName("versionType")] - public string? VersionType { get; init; } - - [JsonPropertyName("versionRange")] - public string? Range { get; init; } -} - -internal sealed record CveCvssMetricDto -{ - [JsonPropertyName("version")] - public string? Version { get; init; } - - [JsonPropertyName("vector")] - public string? Vector { get; init; } - - [JsonPropertyName("baseScore")] - public double? BaseScore { get; init; } - - [JsonPropertyName("baseSeverity")] - public string? BaseSeverity { get; init; } -} +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +internal sealed record CveRecordDto +{ + [JsonPropertyName("cveId")] + public string CveId { get; init; } = string.Empty; + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("language")] + public string? Language { get; init; } + + [JsonPropertyName("state")] + public string State { get; init; } = "PUBLISHED"; + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("modified")] + public DateTimeOffset? Modified { get; init; } + + [JsonPropertyName("aliases")] + public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); + + [JsonPropertyName("references")] + public IReadOnlyList<CveReferenceDto> References { get; init; } = Array.Empty<CveReferenceDto>(); + + [JsonPropertyName("affected")] + public IReadOnlyList<CveAffectedDto> Affected { get; init; } = Array.Empty<CveAffectedDto>(); + + [JsonPropertyName("metrics")] + public IReadOnlyList<CveCvssMetricDto> Metrics { get; init; } = Array.Empty<CveCvssMetricDto>(); +} + +internal sealed record CveReferenceDto +{ + [JsonPropertyName("url")] + public string Url { get; init; } = string.Empty; + + [JsonPropertyName("source")] + public string? Source { get; init; } + + [JsonPropertyName("tags")] + public IReadOnlyList<string> Tags { get; init; } = Array.Empty<string>(); +} + +internal sealed record CveAffectedDto +{ + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("product")] + public string? Product { get; init; } + + [JsonPropertyName("platform")] + public string? Platform { get; init; } + + [JsonPropertyName("defaultStatus")] + public string? DefaultStatus { get; init; } + + [JsonPropertyName("versions")] + public IReadOnlyList<CveVersionDto> Versions { get; init; } = Array.Empty<CveVersionDto>(); +} + +internal sealed record CveVersionDto +{ + [JsonPropertyName("status")] + public string? Status { get; init; } + + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("lessThan")] + public string? LessThan { get; init; } + + [JsonPropertyName("lessThanOrEqual")] + public string? LessThanOrEqual { get; init; } + + [JsonPropertyName("versionType")] + public string? VersionType { get; init; } + + [JsonPropertyName("versionRange")] + public string? Range { get; init; } +} + +internal sealed record CveCvssMetricDto +{ + [JsonPropertyName("version")] + public string? Version { get; init; } + + [JsonPropertyName("vector")] + public string? Vector { get; init; } + + [JsonPropertyName("baseScore")] + public double? BaseScore { get; init; } + + [JsonPropertyName("baseSeverity")] + public string? BaseSeverity { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs b/src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs rename to src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordParser.cs index d9fc89d7..7fd3bdd9 100644 --- a/src/StellaOps.Feedser.Source.Cve/Internal/CveRecordParser.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Internal/CveRecordParser.cs @@ -1,346 +1,346 @@ -using System.Globalization; -using System.Linq; -using System.Text.Json; -using StellaOps.Feedser.Normalization.Text; - -namespace StellaOps.Feedser.Source.Cve.Internal; - -internal static class CveRecordParser -{ - public static CveRecordDto Parse(ReadOnlySpan<byte> content) - { - using var document = JsonDocument.Parse(content.ToArray()); - var root = document.RootElement; - - var metadata = TryGetProperty(root, "cveMetadata"); - if (metadata.ValueKind != JsonValueKind.Object) - { - throw new JsonException("cveMetadata section missing."); - } - - var containers = TryGetProperty(root, "containers"); - var cna = TryGetProperty(containers, "cna"); - - var cveId = GetString(metadata, "cveId") ?? throw new JsonException("cveMetadata.cveId missing."); - var state = GetString(metadata, "state") ?? "PUBLISHED"; - var published = GetDate(metadata, "datePublished"); - var modified = GetDate(metadata, "dateUpdated") ?? GetDate(metadata, "dateReserved"); - - var description = ParseDescription(cna); - - var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) - { - cveId, - }; - foreach (var alias in ParseAliases(cna)) - { - aliases.Add(alias); - } - - var references = ParseReferences(cna); - var affected = ParseAffected(cna); - var metrics = ParseMetrics(cna); - - return new CveRecordDto - { - CveId = cveId, - Title = GetString(cna, "title") ?? cveId, - Summary = description.Text, - Language = description.Language, - State = state, - Published = published, - Modified = modified, - Aliases = aliases.ToArray(), - References = references, - Affected = affected, - Metrics = metrics, - }; - } - - private static NormalizedDescription ParseDescription(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>()); - } - - if (!element.TryGetProperty("descriptions", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array) - { - return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>()); - } - - var items = new List<LocalizedText>(descriptions.GetArrayLength()); - foreach (var entry in descriptions.EnumerateArray()) - { - if (entry.ValueKind != JsonValueKind.Object) - { - continue; - } - - var text = GetString(entry, "value"); - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - var lang = GetString(entry, "lang"); - items.Add(new LocalizedText(text, lang)); - } - - return DescriptionNormalizer.Normalize(items); - } - - private static IEnumerable<string> ParseAliases(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - yield break; - } - - if (!element.TryGetProperty("aliases", out var aliases) || aliases.ValueKind != JsonValueKind.Array) - { - yield break; - } - - foreach (var alias in aliases.EnumerateArray()) - { - if (alias.ValueKind == JsonValueKind.String) - { - var value = alias.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - yield return value; - } - } - } - } - - private static IReadOnlyList<CveReferenceDto> ParseReferences(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - return Array.Empty<CveReferenceDto>(); - } - - if (!element.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) - { - return Array.Empty<CveReferenceDto>(); - } - - var list = new List<CveReferenceDto>(references.GetArrayLength()); - foreach (var reference in references.EnumerateArray()) - { - if (reference.ValueKind != JsonValueKind.Object) - { - continue; - } - - var url = GetString(reference, "url"); - if (string.IsNullOrWhiteSpace(url)) - { - continue; - } - - var tags = Array.Empty<string>(); - if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) - { - tags = tagsElement - .EnumerateArray() - .Where(static t => t.ValueKind == JsonValueKind.String) - .Select(static t => t.GetString()!) - .Where(static v => !string.IsNullOrWhiteSpace(v)) - .ToArray(); - } - - var source = GetString(reference, "name") ?? GetString(reference, "source"); - list.Add(new CveReferenceDto - { - Url = url, - Source = source, - Tags = tags, - }); - } - - return list; - } - - private static IReadOnlyList<CveAffectedDto> ParseAffected(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - return Array.Empty<CveAffectedDto>(); - } - - if (!element.TryGetProperty("affected", out var affected) || affected.ValueKind != JsonValueKind.Array) - { - return Array.Empty<CveAffectedDto>(); - } - - var list = new List<CveAffectedDto>(affected.GetArrayLength()); - foreach (var item in affected.EnumerateArray()) - { - if (item.ValueKind != JsonValueKind.Object) - { - continue; - } - - var versions = new List<CveVersionDto>(); - if (item.TryGetProperty("versions", out var versionsElement) && versionsElement.ValueKind == JsonValueKind.Array) - { - foreach (var versionEntry in versionsElement.EnumerateArray()) - { - if (versionEntry.ValueKind != JsonValueKind.Object) - { - continue; - } - - versions.Add(new CveVersionDto - { - Status = GetString(versionEntry, "status"), - Version = GetString(versionEntry, "version"), - LessThan = GetString(versionEntry, "lessThan"), - LessThanOrEqual = GetString(versionEntry, "lessThanOrEqual"), - VersionType = GetString(versionEntry, "versionType"), - Range = GetString(versionEntry, "versionRange"), - }); - } - } - - list.Add(new CveAffectedDto - { - Vendor = GetString(item, "vendor") ?? GetString(item, "vendorName"), - Product = GetString(item, "product") ?? GetString(item, "productName"), - Platform = GetString(item, "platform"), - DefaultStatus = GetString(item, "defaultStatus"), - Versions = versions, - }); - } - - return list; - } - - private static IReadOnlyList<CveCvssMetricDto> ParseMetrics(JsonElement element) - { - if (element.ValueKind != JsonValueKind.Object) - { - return Array.Empty<CveCvssMetricDto>(); - } - - if (!element.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Array) - { - return Array.Empty<CveCvssMetricDto>(); - } - - var list = new List<CveCvssMetricDto>(metrics.GetArrayLength()); - foreach (var metric in metrics.EnumerateArray()) - { - if (metric.ValueKind != JsonValueKind.Object) - { - continue; - } - - if (metric.TryGetProperty("cvssV4_0", out var cvss40) && cvss40.ValueKind == JsonValueKind.Object) - { - list.Add(ParseCvss(cvss40, "4.0")); - } - else if (metric.TryGetProperty("cvssV3_1", out var cvss31) && cvss31.ValueKind == JsonValueKind.Object) - { - list.Add(ParseCvss(cvss31, "3.1")); - } - else if (metric.TryGetProperty("cvssV3", out var cvss3) && cvss3.ValueKind == JsonValueKind.Object) - { - list.Add(ParseCvss(cvss3, "3.0")); - } - else if (metric.TryGetProperty("cvssV2", out var cvss2) && cvss2.ValueKind == JsonValueKind.Object) - { - list.Add(ParseCvss(cvss2, "2.0")); - } - } - - return list; - } - - private static CveCvssMetricDto ParseCvss(JsonElement element, string fallbackVersion) - { - var version = GetString(element, "version") ?? fallbackVersion; - var vector = GetString(element, "vectorString") ?? GetString(element, "vector"); - var baseScore = GetDouble(element, "baseScore"); - var severity = GetString(element, "baseSeverity") ?? GetString(element, "severity"); - - return new CveCvssMetricDto - { - Version = version, - Vector = vector, - BaseScore = baseScore, - BaseSeverity = severity, - }; - } - - private static JsonElement TryGetProperty(JsonElement element, string propertyName) - { - if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var property)) - { - return property; - } - - return default; - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind switch - { - JsonValueKind.String => property.GetString(), - JsonValueKind.Number when property.TryGetDouble(out var number) => number.ToString(CultureInfo.InvariantCulture), - _ => null, - }; - } - - private static DateTimeOffset? GetDate(JsonElement element, string propertyName) - { - var value = GetString(element, propertyName); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) - ? parsed.ToUniversalTime() - : null; - } - - private static double? GetDouble(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - if (property.ValueKind == JsonValueKind.Number && property.TryGetDouble(out var number)) - { - return number; - } - - if (property.ValueKind == JsonValueKind.String && double.TryParse(property.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) - { - return parsed; - } - - return null; - } -} +using System.Globalization; +using System.Linq; +using System.Text.Json; +using StellaOps.Concelier.Normalization.Text; + +namespace StellaOps.Concelier.Connector.Cve.Internal; + +internal static class CveRecordParser +{ + public static CveRecordDto Parse(ReadOnlySpan<byte> content) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var metadata = TryGetProperty(root, "cveMetadata"); + if (metadata.ValueKind != JsonValueKind.Object) + { + throw new JsonException("cveMetadata section missing."); + } + + var containers = TryGetProperty(root, "containers"); + var cna = TryGetProperty(containers, "cna"); + + var cveId = GetString(metadata, "cveId") ?? throw new JsonException("cveMetadata.cveId missing."); + var state = GetString(metadata, "state") ?? "PUBLISHED"; + var published = GetDate(metadata, "datePublished"); + var modified = GetDate(metadata, "dateUpdated") ?? GetDate(metadata, "dateReserved"); + + var description = ParseDescription(cna); + + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + cveId, + }; + foreach (var alias in ParseAliases(cna)) + { + aliases.Add(alias); + } + + var references = ParseReferences(cna); + var affected = ParseAffected(cna); + var metrics = ParseMetrics(cna); + + return new CveRecordDto + { + CveId = cveId, + Title = GetString(cna, "title") ?? cveId, + Summary = description.Text, + Language = description.Language, + State = state, + Published = published, + Modified = modified, + Aliases = aliases.ToArray(), + References = references, + Affected = affected, + Metrics = metrics, + }; + } + + private static NormalizedDescription ParseDescription(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>()); + } + + if (!element.TryGetProperty("descriptions", out var descriptions) || descriptions.ValueKind != JsonValueKind.Array) + { + return DescriptionNormalizer.Normalize(Array.Empty<LocalizedText>()); + } + + var items = new List<LocalizedText>(descriptions.GetArrayLength()); + foreach (var entry in descriptions.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var text = GetString(entry, "value"); + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + var lang = GetString(entry, "lang"); + items.Add(new LocalizedText(text, lang)); + } + + return DescriptionNormalizer.Normalize(items); + } + + private static IEnumerable<string> ParseAliases(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + yield break; + } + + if (!element.TryGetProperty("aliases", out var aliases) || aliases.ValueKind != JsonValueKind.Array) + { + yield break; + } + + foreach (var alias in aliases.EnumerateArray()) + { + if (alias.ValueKind == JsonValueKind.String) + { + var value = alias.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + yield return value; + } + } + } + } + + private static IReadOnlyList<CveReferenceDto> ParseReferences(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty<CveReferenceDto>(); + } + + if (!element.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) + { + return Array.Empty<CveReferenceDto>(); + } + + var list = new List<CveReferenceDto>(references.GetArrayLength()); + foreach (var reference in references.EnumerateArray()) + { + if (reference.ValueKind != JsonValueKind.Object) + { + continue; + } + + var url = GetString(reference, "url"); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + var tags = Array.Empty<string>(); + if (reference.TryGetProperty("tags", out var tagsElement) && tagsElement.ValueKind == JsonValueKind.Array) + { + tags = tagsElement + .EnumerateArray() + .Where(static t => t.ValueKind == JsonValueKind.String) + .Select(static t => t.GetString()!) + .Where(static v => !string.IsNullOrWhiteSpace(v)) + .ToArray(); + } + + var source = GetString(reference, "name") ?? GetString(reference, "source"); + list.Add(new CveReferenceDto + { + Url = url, + Source = source, + Tags = tags, + }); + } + + return list; + } + + private static IReadOnlyList<CveAffectedDto> ParseAffected(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty<CveAffectedDto>(); + } + + if (!element.TryGetProperty("affected", out var affected) || affected.ValueKind != JsonValueKind.Array) + { + return Array.Empty<CveAffectedDto>(); + } + + var list = new List<CveAffectedDto>(affected.GetArrayLength()); + foreach (var item in affected.EnumerateArray()) + { + if (item.ValueKind != JsonValueKind.Object) + { + continue; + } + + var versions = new List<CveVersionDto>(); + if (item.TryGetProperty("versions", out var versionsElement) && versionsElement.ValueKind == JsonValueKind.Array) + { + foreach (var versionEntry in versionsElement.EnumerateArray()) + { + if (versionEntry.ValueKind != JsonValueKind.Object) + { + continue; + } + + versions.Add(new CveVersionDto + { + Status = GetString(versionEntry, "status"), + Version = GetString(versionEntry, "version"), + LessThan = GetString(versionEntry, "lessThan"), + LessThanOrEqual = GetString(versionEntry, "lessThanOrEqual"), + VersionType = GetString(versionEntry, "versionType"), + Range = GetString(versionEntry, "versionRange"), + }); + } + } + + list.Add(new CveAffectedDto + { + Vendor = GetString(item, "vendor") ?? GetString(item, "vendorName"), + Product = GetString(item, "product") ?? GetString(item, "productName"), + Platform = GetString(item, "platform"), + DefaultStatus = GetString(item, "defaultStatus"), + Versions = versions, + }); + } + + return list; + } + + private static IReadOnlyList<CveCvssMetricDto> ParseMetrics(JsonElement element) + { + if (element.ValueKind != JsonValueKind.Object) + { + return Array.Empty<CveCvssMetricDto>(); + } + + if (!element.TryGetProperty("metrics", out var metrics) || metrics.ValueKind != JsonValueKind.Array) + { + return Array.Empty<CveCvssMetricDto>(); + } + + var list = new List<CveCvssMetricDto>(metrics.GetArrayLength()); + foreach (var metric in metrics.EnumerateArray()) + { + if (metric.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (metric.TryGetProperty("cvssV4_0", out var cvss40) && cvss40.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss40, "4.0")); + } + else if (metric.TryGetProperty("cvssV3_1", out var cvss31) && cvss31.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss31, "3.1")); + } + else if (metric.TryGetProperty("cvssV3", out var cvss3) && cvss3.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss3, "3.0")); + } + else if (metric.TryGetProperty("cvssV2", out var cvss2) && cvss2.ValueKind == JsonValueKind.Object) + { + list.Add(ParseCvss(cvss2, "2.0")); + } + } + + return list; + } + + private static CveCvssMetricDto ParseCvss(JsonElement element, string fallbackVersion) + { + var version = GetString(element, "version") ?? fallbackVersion; + var vector = GetString(element, "vectorString") ?? GetString(element, "vector"); + var baseScore = GetDouble(element, "baseScore"); + var severity = GetString(element, "baseSeverity") ?? GetString(element, "severity"); + + return new CveCvssMetricDto + { + Version = version, + Vector = vector, + BaseScore = baseScore, + BaseSeverity = severity, + }; + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + { + if (element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var property)) + { + return property; + } + + return default; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + JsonValueKind.Number when property.TryGetDouble(out var number) => number.ToString(CultureInfo.InvariantCulture), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } + + private static double? GetDouble(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + if (property.ValueKind == JsonValueKind.Number && property.TryGetDouble(out var number)) + { + return number; + } + + if (property.ValueKind == JsonValueKind.String && double.TryParse(property.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Source.Cve/Jobs.cs b/src/StellaOps.Concelier.Connector.Cve/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Cve/Jobs.cs rename to src/StellaOps.Concelier.Connector.Cve/Jobs.cs index b413a27f..5e25f8de 100644 --- a/src/StellaOps.Feedser.Source.Cve/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Cve/Jobs.cs @@ -1,43 +1,43 @@ -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Cve; - -internal static class CveJobKinds -{ - public const string Fetch = "source:cve:fetch"; - public const string Parse = "source:cve:parse"; - public const string Map = "source:cve:map"; -} - -internal sealed class CveFetchJob : IJob -{ - private readonly CveConnector _connector; - - public CveFetchJob(CveConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class CveParseJob : IJob -{ - private readonly CveConnector _connector; - - public CveParseJob(CveConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class CveMapJob : IJob -{ - private readonly CveConnector _connector; - - public CveMapJob(CveConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Cve; + +internal static class CveJobKinds +{ + public const string Fetch = "source:cve:fetch"; + public const string Parse = "source:cve:parse"; + public const string Map = "source:cve:map"; +} + +internal sealed class CveFetchJob : IJob +{ + private readonly CveConnector _connector; + + public CveFetchJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class CveParseJob : IJob +{ + private readonly CveConnector _connector; + + public CveParseJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class CveMapJob : IJob +{ + private readonly CveConnector _connector; + + public CveMapJob(CveConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj b/src/StellaOps.Concelier.Connector.Cve/StellaOps.Concelier.Connector.Cve.csproj similarity index 57% rename from src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj rename to src/StellaOps.Concelier.Connector.Cve/StellaOps.Concelier.Connector.Cve.csproj index f7f2c154..d97e92d3 100644 --- a/src/StellaOps.Feedser.Source.Cve/StellaOps.Feedser.Source.Cve.csproj +++ b/src/StellaOps.Concelier.Connector.Cve/StellaOps.Concelier.Connector.Cve.csproj @@ -9,8 +9,8 @@ <ItemGroup> <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> - <ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" /> - <ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> </ItemGroup> </Project> diff --git a/src/StellaOps.Feedser.Source.Cve/TASKS.md b/src/StellaOps.Concelier.Connector.Cve/TASKS.md similarity index 80% rename from src/StellaOps.Feedser.Source.Cve/TASKS.md rename to src/StellaOps.Concelier.Connector.Cve/TASKS.md index 2a4751ee..94f5041f 100644 --- a/src/StellaOps.Feedser.Source.Cve/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Cve/TASKS.md @@ -5,8 +5,8 @@ |Fetch/cursor implementation|BE-Conn-CVE|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Time-window + page-aware cursor with SourceFetchService fetching list/detail pairs, resumable state persisted via `CveCursor`.| |DTOs & parser|BE-Conn-CVE|Source.Common|**DONE (2025-10-10)** – `CveRecordParser` and DTOs capture aliases, references, metrics, vendor ranges; sanitises text and timestamps.| |Canonical mapping & range primitives|BE-Conn-CVE|Models|**DONE (2025-10-10)** – `CveMapper` emits canonical advisories, vendor range primitives, SemVer/range statuses, references, CVSS normalization.<br>2025-10-11 research trail: confirm subsequent MR adds `NormalizedVersions` shaped like `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"nvd:CVE-2025-XXXX"}]` so storage provenance joins continue to work.| -|Deterministic tests & fixtures|QA|Testing|**DONE (2025-10-10)** – Added `StellaOps.Feedser.Source.Cve.Tests` harness with canned fixtures + snapshot regression covering fetch/parse/map.| +|Deterministic tests & fixtures|QA|Testing|**DONE (2025-10-10)** – Added `StellaOps.Concelier.Connector.Cve.Tests` harness with canned fixtures + snapshot regression covering fetch/parse/map.| |Observability & docs|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`cve.fetch.*`, etc.) wired; options/usage documented via `CveServiceCollectionExtensions`.| -|Operator rollout playbook|BE-Conn-CVE, Ops|Docs|**DONE (2025-10-12)** – Refreshed `docs/ops/feedser-cve-kev-operations.md` with credential checklist, smoke book, PromQL guardrails, and linked Grafana pack (`docs/ops/feedser-cve-kev-grafana-dashboard.json`).| -|Live smoke & monitoring|QA, BE-Conn-CVE|WebService, Observability|**DONE (2025-10-15)** – Executed connector harness smoke using CVE Services sample window (CVE-2024-0001), confirmed fetch/parse/map telemetry (`cve.fetch.*`, `cve.map.success`) all incremented once, and archived the summary log + Grafana import guidance in `docs/ops/feedser-cve-kev-operations.md` (“Staging smoke 2025-10-15”).| -|FEEDCONN-CVE-02-003 Normalized versions rollout|BE-Conn-CVE|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Confirmed SemVer primitives map to normalized rules with `cve:{cveId}:{identifier}` notes and refreshed snapshots; `dotnet test src/StellaOps.Feedser.Source.Cve.Tests` passes on net10 preview.| +|Operator rollout playbook|BE-Conn-CVE, Ops|Docs|**DONE (2025-10-12)** – Refreshed `docs/ops/concelier-cve-kev-operations.md` with credential checklist, smoke book, PromQL guardrails, and linked Grafana pack (`docs/ops/concelier-cve-kev-grafana-dashboard.json`).| +|Live smoke & monitoring|QA, BE-Conn-CVE|WebService, Observability|**DONE (2025-10-15)** – Executed connector harness smoke using CVE Services sample window (CVE-2024-0001), confirmed fetch/parse/map telemetry (`cve.fetch.*`, `cve.map.success`) all incremented once, and archived the summary log + Grafana import guidance in `docs/ops/concelier-cve-kev-operations.md` (“Staging smoke 2025-10-15”).| +|FEEDCONN-CVE-02-003 Normalized versions rollout|BE-Conn-CVE|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Confirmed SemVer primitives map to normalized rules with `cve:{cveId}:{identifier}` notes and refreshed snapshots; `dotnet test src/StellaOps.Concelier.Connector.Cve.Tests` passes on net10 preview.| diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianConnectorTests.cs index bbec302a..04937dba 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianConnectorTests.cs @@ -1,281 +1,281 @@ -using System.Collections.Generic; -using System; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Distro.Debian.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; -using Xunit.Abstractions; - -namespace StellaOps.Feedser.Source.Distro.Debian.Tests; - -[Collection("mongo-fixture")] -public sealed class DebianConnectorTests : IAsyncLifetime -{ - private static readonly Uri ListUri = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); - private static readonly Uri DetailResolved = new("https://security-tracker.debian.org/tracker/DSA-2024-123"); - private static readonly Uri DetailOpen = new("https://security-tracker.debian.org/tracker/DSA-2024-124"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - private readonly Dictionary<Uri, Func<HttpRequestMessage, HttpResponseMessage>> _fallbackFactories = new(); - private readonly ITestOutputHelper _output; - - public DebianConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) - { - _fixture = fixture; - _handler = new CannedHttpMessageHandler(); - _handler.SetFallback(request => - { - if (request.RequestUri is null) - { - throw new InvalidOperationException("Request URI required for fallback response."); - } - - if (_fallbackFactories.TryGetValue(request.RequestUri, out var factory)) - { - return factory(request); - } - - throw new InvalidOperationException($"No canned or fallback response registered for {request.Method} {request.RequestUri}."); - }); - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 12, 0, 0, 0, TimeSpan.Zero)); - _output = output; - } - - [Fact] - public async Task FetchParseMap_PopulatesRangePrimitivesAndResumesWithNotModified() - { - await using var provider = await BuildServiceProviderAsync(); - - SeedInitialResponses(); - - var connector = provider.GetRequiredService<DebianConnector>(); - await connector.FetchAsync(provider, CancellationToken.None); - _timeProvider.Advance(TimeSpan.FromMinutes(1)); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(2, advisories.Count); - - var resolved = advisories.Single(a => a.AdvisoryKey == "DSA-2024-123"); - _output.WriteLine("Resolved aliases: " + string.Join(",", resolved.Aliases)); - var resolvedBookworm = Assert.Single(resolved.AffectedPackages, p => p.Platform == "bookworm"); - var resolvedRange = Assert.Single(resolvedBookworm.VersionRanges); - Assert.Equal("evr", resolvedRange.RangeKind); - Assert.Equal("1:1.1.1n-0+deb11u2", resolvedRange.IntroducedVersion); - Assert.Equal("1:1.1.1n-0+deb11u5", resolvedRange.FixedVersion); - Assert.NotNull(resolvedRange.Primitives); - Assert.NotNull(resolvedRange.Primitives!.Evr); - Assert.Equal(1, resolvedRange.Primitives.Evr!.Introduced!.Epoch); - Assert.Equal("1.1.1n", resolvedRange.Primitives.Evr.Introduced.UpstreamVersion); - - var open = advisories.Single(a => a.AdvisoryKey == "DSA-2024-124"); - var openBookworm = Assert.Single(open.AffectedPackages, p => p.Platform == "bookworm"); - var openRange = Assert.Single(openBookworm.VersionRanges); - Assert.Equal("evr", openRange.RangeKind); - Assert.Equal("1:1.3.1-1", openRange.IntroducedVersion); - Assert.Null(openRange.FixedVersion); - Assert.NotNull(openRange.Primitives); - Assert.NotNull(openRange.Primitives!.Evr); - - // Ensure data persisted through Mongo round-trip. - var found = await advisoryStore.FindAsync("DSA-2024-123", CancellationToken.None); - Assert.NotNull(found); - var persistedRange = Assert.Single(found!.AffectedPackages, pkg => pkg.Platform == "bookworm").VersionRanges.Single(); - Assert.NotNull(persistedRange.Primitives); - Assert.NotNull(persistedRange.Primitives!.Evr); - - // Second run should issue conditional requests and no additional parsing/mapping. - SeedNotModifiedResponses(); - await connector.FetchAsync(provider, CancellationToken.None); - _timeProvider.Advance(TimeSpan.FromMinutes(1)); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var documents = provider.GetRequiredService<IDocumentStore>(); - var listDoc = await documents.FindBySourceAndUriAsync(DebianConnectorPlugin.SourceName, DetailResolved.ToString(), CancellationToken.None); - Assert.NotNull(listDoc); - - var refreshed = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(2, refreshed.Count); - } - - private async Task<ServiceProvider> BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName, CancellationToken.None); - _handler.Clear(); - _fallbackFactories.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(new TestOutputLoggerProvider(_output))); - services.AddSingleton<TimeProvider>(_timeProvider); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddDebianConnector(options => - { - options.ListEndpoint = ListUri; - options.DetailBaseUri = new Uri("https://security-tracker.debian.org/tracker/"); - options.MaxAdvisoriesPerFetch = 10; - options.RequestDelay = TimeSpan.Zero; - }); - - services.Configure<HttpClientFactoryOptions>(DebianOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedInitialResponses() - { - AddListResponse("debian-list.txt", "\"list-v1\""); - AddDetailResponse(DetailResolved, "debian-detail-dsa-2024-123.html", "\"detail-123\""); - AddDetailResponse(DetailOpen, "debian-detail-dsa-2024-124.html", "\"detail-124\""); - } - - private void SeedNotModifiedResponses() - { - AddNotModifiedResponse(ListUri, "\"list-v1\""); - AddNotModifiedResponse(DetailResolved, "\"detail-123\""); - AddNotModifiedResponse(DetailOpen, "\"detail-124\""); - } - - private void AddListResponse(string fixture, string etag) - { - RegisterResponseFactory(ListUri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/plain"), - }; - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private void AddDetailResponse(Uri uri, string fixture, string etag) - { - RegisterResponseFactory(uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), - }; - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private void AddNotModifiedResponse(Uri uri, string etag) - { - RegisterResponseFactory(uri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.NotModified); - response.Headers.ETag = new EntityTagHeaderValue(etag); - return response; - }); - } - - private void RegisterResponseFactory(Uri uri, Func<HttpResponseMessage> factory) - { - _handler.AddResponse(uri, () => factory()); - _fallbackFactories[uri] = _ => factory(); - } - - private static string ReadFixture(string filename) - { - var candidates = new[] - { - Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Debian", "Fixtures", filename), - Path.Combine(AppContext.BaseDirectory, "Distro", "Debian", "Fixtures", filename), - Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Source", "Distro", "Debian", "Fixtures", filename), - }; - - foreach (var candidate in candidates) - { - var fullPath = Path.GetFullPath(candidate); - if (File.Exists(fullPath)) - { - return File.ReadAllText(fullPath); - } - } - - throw new FileNotFoundException($"Fixture '{filename}' not found", filename); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; - - private sealed class TestOutputLoggerProvider : ILoggerProvider - { - private readonly ITestOutputHelper _output; - - public TestOutputLoggerProvider(ITestOutputHelper output) => _output = output; - - public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output); - - public void Dispose() - { - } - - private sealed class TestOutputLogger : ILogger - { - private readonly ITestOutputHelper _output; - - public TestOutputLogger(ITestOutputHelper output) => _output = output; - - public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); - - public bool IsEnabled(LogLevel logLevel) => false; - - public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) - { - if (IsEnabled(logLevel)) - { - _output.WriteLine(formatter(state, exception)); - } - } - } - } -} +using System.Collections.Generic; +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Distro.Debian.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Concelier.Connector.Distro.Debian.Tests; + +[Collection("mongo-fixture")] +public sealed class DebianConnectorTests : IAsyncLifetime +{ + private static readonly Uri ListUri = new("https://salsa.debian.org/security-tracker-team/security-tracker/-/raw/master/data/DSA/list"); + private static readonly Uri DetailResolved = new("https://security-tracker.debian.org/tracker/DSA-2024-123"); + private static readonly Uri DetailOpen = new("https://security-tracker.debian.org/tracker/DSA-2024-124"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + private readonly Dictionary<Uri, Func<HttpRequestMessage, HttpResponseMessage>> _fallbackFactories = new(); + private readonly ITestOutputHelper _output; + + public DebianConnectorTests(MongoIntegrationFixture fixture, ITestOutputHelper output) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + _handler.SetFallback(request => + { + if (request.RequestUri is null) + { + throw new InvalidOperationException("Request URI required for fallback response."); + } + + if (_fallbackFactories.TryGetValue(request.RequestUri, out var factory)) + { + return factory(request); + } + + throw new InvalidOperationException($"No canned or fallback response registered for {request.Method} {request.RequestUri}."); + }); + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2024, 9, 12, 0, 0, 0, TimeSpan.Zero)); + _output = output; + } + + [Fact] + public async Task FetchParseMap_PopulatesRangePrimitivesAndResumesWithNotModified() + { + await using var provider = await BuildServiceProviderAsync(); + + SeedInitialResponses(); + + var connector = provider.GetRequiredService<DebianConnector>(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService<IAdvisoryStore>(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var resolved = advisories.Single(a => a.AdvisoryKey == "DSA-2024-123"); + _output.WriteLine("Resolved aliases: " + string.Join(",", resolved.Aliases)); + var resolvedBookworm = Assert.Single(resolved.AffectedPackages, p => p.Platform == "bookworm"); + var resolvedRange = Assert.Single(resolvedBookworm.VersionRanges); + Assert.Equal("evr", resolvedRange.RangeKind); + Assert.Equal("1:1.1.1n-0+deb11u2", resolvedRange.IntroducedVersion); + Assert.Equal("1:1.1.1n-0+deb11u5", resolvedRange.FixedVersion); + Assert.NotNull(resolvedRange.Primitives); + Assert.NotNull(resolvedRange.Primitives!.Evr); + Assert.Equal(1, resolvedRange.Primitives.Evr!.Introduced!.Epoch); + Assert.Equal("1.1.1n", resolvedRange.Primitives.Evr.Introduced.UpstreamVersion); + + var open = advisories.Single(a => a.AdvisoryKey == "DSA-2024-124"); + var openBookworm = Assert.Single(open.AffectedPackages, p => p.Platform == "bookworm"); + var openRange = Assert.Single(openBookworm.VersionRanges); + Assert.Equal("evr", openRange.RangeKind); + Assert.Equal("1:1.3.1-1", openRange.IntroducedVersion); + Assert.Null(openRange.FixedVersion); + Assert.NotNull(openRange.Primitives); + Assert.NotNull(openRange.Primitives!.Evr); + + // Ensure data persisted through Mongo round-trip. + var found = await advisoryStore.FindAsync("DSA-2024-123", CancellationToken.None); + Assert.NotNull(found); + var persistedRange = Assert.Single(found!.AffectedPackages, pkg => pkg.Platform == "bookworm").VersionRanges.Single(); + Assert.NotNull(persistedRange.Primitives); + Assert.NotNull(persistedRange.Primitives!.Evr); + + // Second run should issue conditional requests and no additional parsing/mapping. + SeedNotModifiedResponses(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var documents = provider.GetRequiredService<IDocumentStore>(); + var listDoc = await documents.FindBySourceAndUriAsync(DebianConnectorPlugin.SourceName, DetailResolved.ToString(), CancellationToken.None); + Assert.NotNull(listDoc); + + var refreshed = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, refreshed.Count); + } + + private async Task<ServiceProvider> BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName, CancellationToken.None); + _handler.Clear(); + _fallbackFactories.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(new TestOutputLoggerProvider(_output))); + services.AddSingleton<TimeProvider>(_timeProvider); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddDebianConnector(options => + { + options.ListEndpoint = ListUri; + options.DetailBaseUri = new Uri("https://security-tracker.debian.org/tracker/"); + options.MaxAdvisoriesPerFetch = 10; + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure<HttpClientFactoryOptions>(DebianOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService<MongoBootstrapper>(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedInitialResponses() + { + AddListResponse("debian-list.txt", "\"list-v1\""); + AddDetailResponse(DetailResolved, "debian-detail-dsa-2024-123.html", "\"detail-123\""); + AddDetailResponse(DetailOpen, "debian-detail-dsa-2024-124.html", "\"detail-124\""); + } + + private void SeedNotModifiedResponses() + { + AddNotModifiedResponse(ListUri, "\"list-v1\""); + AddNotModifiedResponse(DetailResolved, "\"detail-123\""); + AddNotModifiedResponse(DetailOpen, "\"detail-124\""); + } + + private void AddListResponse(string fixture, string etag) + { + RegisterResponseFactory(ListUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/plain"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddDetailResponse(Uri uri, string fixture, string etag) + { + RegisterResponseFactory(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture(fixture), Encoding.UTF8, "text/html"), + }; + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void AddNotModifiedResponse(Uri uri, string etag) + { + RegisterResponseFactory(uri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(etag); + return response; + }); + } + + private void RegisterResponseFactory(Uri uri, Func<HttpResponseMessage> factory) + { + _handler.AddResponse(uri, () => factory()); + _fallbackFactories[uri] = _ => factory(); + } + + private static string ReadFixture(string filename) + { + var candidates = new[] + { + Path.Combine(AppContext.BaseDirectory, "Source", "Distro", "Debian", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "Distro", "Debian", "Fixtures", filename), + Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Source", "Distro", "Debian", "Fixtures", filename), + }; + + foreach (var candidate in candidates) + { + var fullPath = Path.GetFullPath(candidate); + if (File.Exists(fullPath)) + { + return File.ReadAllText(fullPath); + } + } + + throw new FileNotFoundException($"Fixture '{filename}' not found", filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private sealed class TestOutputLoggerProvider : ILoggerProvider + { + private readonly ITestOutputHelper _output; + + public TestOutputLoggerProvider(ITestOutputHelper output) => _output = output; + + public ILogger CreateLogger(string categoryName) => new TestOutputLogger(_output); + + public void Dispose() + { + } + + private sealed class TestOutputLogger : ILogger + { + private readonly ITestOutputHelper _output; + + public TestOutputLogger(ITestOutputHelper output) => _output = output; + + public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullLogger.Instance.BeginScope(state); + + public bool IsEnabled(LogLevel logLevel) => false; + + public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter) + { + if (IsEnabled(logLevel)) + { + _output.WriteLine(formatter(state, exception)); + } + } + } + } +} diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianMapperTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianMapperTests.cs index 03f8d5fb..8b75a5b0 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/DebianMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/DebianMapperTests.cs @@ -1,11 +1,11 @@ using System; using Xunit; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Distro.Debian; -using StellaOps.Feedser.Source.Distro.Debian.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Distro.Debian; +using StellaOps.Concelier.Connector.Distro.Debian.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Distro.Debian.Tests; +namespace StellaOps.Concelier.Connector.Distro.Debian.Tests; public sealed class DebianMapperTests { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html rename to src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-123.html diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html rename to src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-detail-dsa-2024-124.html diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt rename to src/StellaOps.Concelier.Connector.Distro.Debian.Tests/Source/Distro/Debian/Fixtures/debian-list.txt diff --git a/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/StellaOps.Concelier.Connector.Distro.Debian.Tests.csproj b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/StellaOps.Concelier.Connector.Distro.Debian.Tests.csproj new file mode 100644 index 00000000..373f2b25 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Debian.Tests/StellaOps.Concelier.Connector.Distro.Debian.Tests.csproj @@ -0,0 +1,13 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Concelier.Connector.Distro.Debian/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/AssemblyInfo.cs new file mode 100644 index 00000000..ed109896 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Distro.Debian.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Configuration/DebianOptions.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Configuration/DebianOptions.cs index c5c82e4a..f7573f23 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Configuration/DebianOptions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Configuration/DebianOptions.cs @@ -1,10 +1,10 @@ using System; -namespace StellaOps.Feedser.Source.Distro.Debian.Configuration; +namespace StellaOps.Concelier.Connector.Distro.Debian.Configuration; public sealed class DebianOptions { - public const string HttpClientName = "feedser.debian"; + public const string HttpClientName = "concelier.debian"; /// <summary> /// Raw advisory list published by the Debian security tracker team. @@ -45,7 +45,7 @@ public sealed class DebianOptions /// <summary> /// Custom user-agent for Debian tracker courtesy. /// </summary> - public string UserAgent { get; set; } = "StellaOps.Feedser.Debian/0.1 (+https://stella-ops.org)"; + public string UserAgent { get; set; } = "StellaOps.Concelier.Debian/0.1 (+https://stella-ops.org)"; public void Validate() { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnector.cs index ddd1c1de..3666fd92 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnector.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnector.cs @@ -9,18 +9,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Distro.Debian.Configuration; -using StellaOps.Feedser.Source.Distro.Debian.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Distro.Debian.Configuration; +using StellaOps.Concelier.Connector.Distro.Debian.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Debian; +namespace StellaOps.Concelier.Connector.Distro.Debian; public sealed class DebianConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnectorPlugin.cs index 5f4aced7..18df76e0 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/DebianConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianConnectorPlugin.cs @@ -4,7 +4,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Debian; +namespace StellaOps.Concelier.Connector.Distro.Debian; public sealed class DebianConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/DebianDependencyInjectionRoutine.cs index 562ded3d..f4dc07dd 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/DebianDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Distro.Debian.Configuration; -namespace StellaOps.Feedser.Source.Distro.Debian; +namespace StellaOps.Concelier.Connector.Distro.Debian; public sealed class DebianDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:debian"; + private const string ConfigurationSection = "concelier:sources:debian"; private const string FetchSchedule = "*/30 * * * *"; private const string ParseSchedule = "7,37 * * * *"; private const string MapSchedule = "12,42 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianServiceCollectionExtensions.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/DebianServiceCollectionExtensions.cs index 5df031df..f1d03a6a 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/DebianServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/DebianServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Distro.Debian.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Distro.Debian.Configuration; -namespace StellaOps.Feedser.Source.Distro.Debian; +namespace StellaOps.Concelier.Connector.Distro.Debian; public static class DebianServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianAdvisoryDto.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianAdvisoryDto.cs index b1a88845..729acc60 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianAdvisoryDto.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal sealed record DebianAdvisoryDto( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianCursor.cs index 1b0d6bff..a8c77bb5 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianCursor.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal sealed record DebianCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianDetailMetadata.cs similarity index 76% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianDetailMetadata.cs index 56e95339..2d08e5e4 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianDetailMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianDetailMetadata.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal sealed record DebianDetailMetadata( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianFetchCacheEntry.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianFetchCacheEntry.cs index 3be35b6b..c06e49ff 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianFetchCacheEntry.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianFetchCacheEntry.cs @@ -1,13 +1,13 @@ using System; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) { public static DebianFetchCacheEntry Empty { get; } = new(null, null); - public static DebianFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public static DebianFetchCacheEntry FromDocument(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) => new(document.Etag, document.LastModified); public static DebianFetchCacheEntry FromBson(BsonDocument document) @@ -54,7 +54,7 @@ internal sealed record DebianFetchCacheEntry(string? ETag, DateTimeOffset? LastM return document; } - public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public bool Matches(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) { if (document is null) { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianHtmlParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianHtmlParser.cs index 34b22a25..3438995f 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianHtmlParser.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianHtmlParser.cs @@ -5,7 +5,7 @@ using System.Linq; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal static class DebianHtmlParser { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListEntry.cs similarity index 74% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListEntry.cs index 81e708fa..658126f5 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListEntry.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListEntry.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal sealed record DebianListEntry( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListParser.cs index 3e22e4b0..7a6d5e17 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianListParser.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianListParser.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal static class DebianListParser { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianMapper.cs index 0c6d89e1..9570c2b2 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Internal/DebianMapper.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Internal/DebianMapper.cs @@ -1,12 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Distro; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Distro; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Distro.Debian.Internal; +namespace StellaOps.Concelier.Connector.Distro.Debian.Internal; internal static class DebianMapper { diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs b/src/StellaOps.Concelier.Connector.Distro.Debian/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs rename to src/StellaOps.Concelier.Connector.Distro.Debian/Jobs.cs index 0c770787..f0526fe3 100644 --- a/src/StellaOps.Feedser.Source.Distro.Debian/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Distro.Debian; +namespace StellaOps.Concelier.Connector.Distro.Debian; internal static class DebianJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj b/src/StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj new file mode 100644 index 00000000..23396f92 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Debian/StellaOps.Concelier.Connector.Distro.Debian.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0001.json diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0002.json diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/csaf-rhsa-2025-0003.json diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json similarity index 77% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json index da219872..b08d01d7 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json @@ -1,149 +1,163 @@ -{ - "advisoryKey": "RHSA-2025:0001", - "affectedPackages": [ - { - "identifier": "cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", - "platform": "Red Hat Enterprise Linux 8", - "provenance": [ - { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "8Base-RHEL-8" - } - ], - "statuses": [ - { - "provenance": { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "8Base-RHEL-8" - }, - "status": "known_affected" - } - ], - "type": "cpe", - "versionRanges": [] - }, - { - "identifier": "kernel-0:4.18.0-513.5.1.el8.x86_64", - "platform": "Red Hat Enterprise Linux 8", - "provenance": [ - { - "fieldMask": [], - "kind": "package.nevra", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" - } - ], - "statuses": [], - "type": "rpm", - "versionRanges": [ - { - "fixedVersion": "kernel-0:4.18.0-513.5.1.el8.x86_64", - "introducedVersion": null, - "lastAffectedVersion": "kernel-0:4.18.0-500.1.0.el8.x86_64", - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": { - "fixed": { - "architecture": "x86_64", - "epoch": 0, - "name": "kernel", - "release": "513.5.1.el8", - "version": "4.18.0" - }, - "introduced": null, - "lastAffected": { - "architecture": "x86_64", - "epoch": 0, - "name": "kernel", - "release": "500.1.0.el8", - "version": "4.18.0" - } - }, - "semVer": null, - "vendorExtensions": null - }, - "provenance": { - "fieldMask": [], - "kind": "package.nevra", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "kernel-0:4.18.0-513.5.1.el8.x86_64" - }, - "rangeExpression": null, - "rangeKind": "nevra" - } - ] - } - ], - "aliases": [ - "CVE-2025-0001", - "RHSA-2025:0001" - ], - "cvssMetrics": [ - { - "baseScore": 9.8, - "baseSeverity": "critical", - "provenance": { - "fieldMask": [], - "kind": "cvss", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "CVE-2025-0001" - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-03T00:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "advisory", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "RHSA-2025:0001" - } - ], - "published": "2025-10-02T00:00:00+00:00", - "references": [ - { - "kind": "self", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "https://access.redhat.com/errata/RHSA-2025:0001" - }, - "sourceTag": null, - "summary": "RHSA advisory", - "url": "https://access.redhat.com/errata/RHSA-2025:0001" - }, - { - "kind": "external", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-05T00:00:00+00:00", - "source": "redhat", - "value": "https://www.cve.org/CVERecord?id=CVE-2025-0001" - }, - "sourceTag": null, - "summary": "CVE record", - "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" - } - ], - "severity": "high", - "summary": "An update fixes a critical kernel issue.", - "title": "Red Hat Security Advisory: Example kernel update" +{ + "advisoryKey": "RHSA-2025:0001", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:2.3:o:redhat:enterprise_linux:8:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 8", + "versionRanges": [], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "oval", + "value": "8Base-RHEL-8", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + }, + "status": "known_affected" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "oval", + "value": "8Base-RHEL-8", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "rpm", + "identifier": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "platform": "Red Hat Enterprise Linux 8", + "versionRanges": [ + { + "fixedVersion": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "introducedVersion": null, + "lastAffectedVersion": "kernel-0:4.18.0-500.1.0.el8.x86_64", + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": { + "fixed": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "513.5.1.el8", + "version": "4.18.0" + }, + "introduced": null, + "lastAffected": { + "architecture": "x86_64", + "epoch": 0, + "name": "kernel", + "release": "500.1.0.el8", + "version": "4.18.0" + } + }, + "semVer": null, + "vendorExtensions": null + }, + "provenance": { + "source": "redhat", + "kind": "package.nevra", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "nevra" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "redhat", + "kind": "package.nevra", + "value": "kernel-0:4.18.0-513.5.1.el8.x86_64", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-0001", + "RHSA-2025:0001" + ], + "canonicalMetricId": null, + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "redhat", + "kind": "cvss", + "value": "CVE-2025-0001", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [], + "description": null, + "exploitKnown": false, + "language": "en", + "modified": "2025-10-03T00:00:00+00:00", + "provenance": [ + { + "source": "redhat", + "kind": "advisory", + "value": "RHSA-2025:0001", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-02T00:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://access.redhat.com/errata/RHSA-2025:0001", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0001" + }, + { + "kind": "external", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0001", + "decisionReason": null, + "recordedAt": "2025-10-05T00:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0001" + } + ], + "severity": "high", + "summary": "An update fixes a critical kernel issue.", + "title": "Red Hat Security Advisory: Example kernel update" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json similarity index 70% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json index acdd5763..b03905ba 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json @@ -1,118 +1,132 @@ -{ - "advisoryKey": "RHSA-2025:0002", - "affectedPackages": [ - { - "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", - "platform": "Red Hat Enterprise Linux 9", - "provenance": [ - { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "9Base-RHEL-9" - } - ], - "statuses": [ - { - "provenance": { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "9Base-RHEL-9" - }, - "status": "known_not_affected" - }, - { - "provenance": { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "9Base-RHEL-9" - }, - "status": "under_investigation" - } - ], - "type": "cpe", - "versionRanges": [] - }, - { - "identifier": "kernel-0:5.14.0-400.el9.x86_64", - "platform": "Red Hat Enterprise Linux 9", - "provenance": [ - { - "fieldMask": [], - "kind": "package.nevra", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "kernel-0:5.14.0-400.el9.x86_64" - } - ], - "statuses": [ - { - "provenance": { - "fieldMask": [], - "kind": "package.nevra", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "kernel-0:5.14.0-400.el9.x86_64" - }, - "status": "known_not_affected" - } - ], - "type": "rpm", - "versionRanges": [] - } - ], - "aliases": [ - "CVE-2025-0002", - "RHSA-2025:0002" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-05T12:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "advisory", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "RHSA-2025:0002" - } - ], - "published": "2025-10-05T12:00:00+00:00", - "references": [ - { - "kind": "self", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "https://access.redhat.com/errata/RHSA-2025:0002" - }, - "sourceTag": null, - "summary": "RHSA advisory", - "url": "https://access.redhat.com/errata/RHSA-2025:0002" - }, - { - "kind": "external", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-05T12:00:00+00:00", - "source": "redhat", - "value": "https://www.cve.org/CVERecord?id=CVE-2025-0002" - }, - "sourceTag": null, - "summary": "CVE record", - "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" - } - ], - "severity": "medium", - "summary": "Second advisory covering unaffected packages.", - "title": "Red Hat Security Advisory: Follow-up kernel status" +{ + "advisoryKey": "RHSA-2025:0002", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "versionRanges": [], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "oval", + "value": "9Base-RHEL-9", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + }, + "status": "known_not_affected" + }, + { + "provenance": { + "source": "redhat", + "kind": "oval", + "value": "9Base-RHEL-9", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + }, + "status": "under_investigation" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "oval", + "value": "9Base-RHEL-9", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + } + ] + }, + { + "type": "rpm", + "identifier": "kernel-0:5.14.0-400.el9.x86_64", + "platform": "Red Hat Enterprise Linux 9", + "versionRanges": [], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "package.nevra", + "value": "kernel-0:5.14.0-400.el9.x86_64", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + }, + "status": "known_not_affected" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "package.nevra", + "value": "kernel-0:5.14.0-400.el9.x86_64", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-0002", + "RHSA-2025:0002" + ], + "canonicalMetricId": null, + "credits": [], + "cvssMetrics": [], + "cwes": [], + "description": null, + "exploitKnown": false, + "language": "en", + "modified": "2025-10-05T12:00:00+00:00", + "provenance": [ + { + "source": "redhat", + "kind": "advisory", + "value": "RHSA-2025:0002", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-05T12:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://access.redhat.com/errata/RHSA-2025:0002", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "RHSA advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0002" + }, + { + "kind": "external", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0002", + "decisionReason": null, + "recordedAt": "2025-10-05T12:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0002" + } + ], + "severity": "medium", + "summary": "Second advisory covering unaffected packages.", + "title": "Red Hat Security Advisory: Follow-up kernel status" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json similarity index 74% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json index 9649bae7..c0c30ad8 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json @@ -1,121 +1,134 @@ -{ - "advisoryKey": "RHSA-2025:0003", - "affectedPackages": [ - { - "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", - "platform": "Red Hat Enterprise Linux 9", - "provenance": [ - { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "9Base-RHEL-9" - } - ], - "statuses": [ - { - "provenance": { - "fieldMask": [], - "kind": "oval", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "9Base-RHEL-9" - }, - "status": "known_affected" - } - ], - "type": "cpe", - "versionRanges": [] - } - ], - "aliases": [ - "CVE-2025-0003", - "RHSA-2025:0003" - ], - "cvssMetrics": [ - { - "baseScore": 7.5, - "baseSeverity": "high", - "provenance": { - "fieldMask": [], - "kind": "cvss", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "CVE-2025-0003" - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-06T09:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "advisory", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "RHSA-2025:0003" - } - ], - "published": "2025-10-06T09:00:00+00:00", - "references": [ - { - "kind": "self", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "https://access.redhat.com/errata/RHSA-2025:0003" - }, - "sourceTag": null, - "summary": "Primary advisory", - "url": "https://access.redhat.com/errata/RHSA-2025:0003" - }, - { - "kind": "mitigation", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "https://access.redhat.com/solutions/999999" - }, - "sourceTag": null, - "summary": "Knowledge base guidance", - "url": "https://access.redhat.com/solutions/999999" - }, - { - "kind": "exploit", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" - }, - "sourceTag": null, - "summary": "Exploit tracking", - "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" - }, - { - "kind": "external", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2025-10-06T09:00:00+00:00", - "source": "redhat", - "value": "https://www.cve.org/CVERecord?id=CVE-2025-0003" - }, - "sourceTag": null, - "summary": "CVE record", - "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" - } - ], - "severity": "high", - "summary": "Advisory with mixed reference sources to verify dedupe ordering.", - "title": "Red Hat Security Advisory: Reference dedupe validation" +{ + "advisoryKey": "RHSA-2025:0003", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:2.3:o:redhat:enterprise_linux:9:*:*:*:*:*:*:*", + "platform": "Red Hat Enterprise Linux 9", + "versionRanges": [], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "oval", + "value": "9Base-RHEL-9", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "status": "known_affected" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "oval", + "value": "9Base-RHEL-9", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2025-0003", + "RHSA-2025:0003" + ], + "canonicalMetricId": null, + "credits": [], + "cvssMetrics": [ + { + "baseScore": 7.5, + "baseSeverity": "high", + "provenance": { + "source": "redhat", + "kind": "cvss", + "value": "CVE-2025-0003", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "version": "3.1" + } + ], + "cwes": [], + "description": null, + "exploitKnown": false, + "language": "en", + "modified": "2025-10-06T09:00:00+00:00", + "provenance": [ + { + "source": "redhat", + "kind": "advisory", + "value": "RHSA-2025:0003", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2025-10-06T09:00:00+00:00", + "references": [ + { + "kind": "self", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://access.redhat.com/errata/RHSA-2025:0003", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "Primary advisory", + "url": "https://access.redhat.com/errata/RHSA-2025:0003" + }, + { + "kind": "mitigation", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://access.redhat.com/solutions/999999", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "Knowledge base guidance", + "url": "https://access.redhat.com/solutions/999999" + }, + { + "kind": "exploit", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "Exploit tracking", + "url": "https://bugzilla.redhat.com/show_bug.cgi?id=2222222" + }, + { + "kind": "external", + "provenance": { + "source": "redhat", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2025-0003", + "decisionReason": null, + "recordedAt": "2025-10-06T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": null, + "summary": "CVE record", + "url": "https://www.cve.org/CVERecord?id=CVE-2025-0003" + } + ], + "severity": "high", + "summary": "Advisory with mixed reference sources to verify dedupe ordering.", + "title": "Red Hat Security Advisory: Reference dedupe validation" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json index 9b713b60..daab87ad 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1-repeat.json @@ -1,14 +1,14 @@ -[ - { - "RHSA": "RHSA-2025:0001", - "severity": "important", - "released_on": "2025-10-03T00:00:00Z", - "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" - }, - { - "RHSA": "RHSA-2025:0002", - "severity": "moderate", - "released_on": "2025-10-05T12:00:00Z", - "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" - } -] +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T00:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + }, + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json index 7f158304..025fa550 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page1.json @@ -1,14 +1,14 @@ -[ - { - "RHSA": "RHSA-2025:0001", - "severity": "important", - "released_on": "2025-10-03T12:00:00Z", - "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" - }, - { - "RHSA": "RHSA-2025:0002", - "severity": "moderate", - "released_on": "2025-10-05T12:00:00Z", - "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" - } -] +[ + { + "RHSA": "RHSA-2025:0001", + "severity": "important", + "released_on": "2025-10-03T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0001.json" + }, + { + "RHSA": "RHSA-2025:0002", + "severity": "moderate", + "released_on": "2025-10-05T12:00:00Z", + "resource_url": "https://access.redhat.com/hydra/rest/securitydata/csaf/RHSA-2025:0002.json" + } +] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page2.json diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/summary-page3.json diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs index 463f513c..74686796 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorHarnessTests.cs @@ -3,15 +3,15 @@ using System.IO; using System.Linq; using Microsoft.Extensions.DependencyInjection; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Distro.RedHat; -using StellaOps.Feedser.Source.Distro.RedHat.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Distro.RedHat; +using StellaOps.Concelier.Connector.Distro.RedHat.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Tests; [Collection("mongo-fixture")] public sealed class RedHatConnectorHarnessTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs index 873980c5..ce2a4534 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/RedHatConnectorTests.cs @@ -14,25 +14,25 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Distro.RedHat; -using StellaOps.Feedser.Source.Distro.RedHat.Configuration; -using StellaOps.Feedser.Source.Distro.RedHat.Internal; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Distro.RedHat; +using StellaOps.Concelier.Connector.Distro.RedHat.Configuration; +using StellaOps.Concelier.Connector.Distro.RedHat.Internal; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; using StellaOps.Plugin; using Xunit; using Xunit.Abstractions; -namespace StellaOps.Feedser.Source.Distro.RedHat.Tests; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Tests; [Collection("mongo-fixture")] public sealed class RedHatConnectorTests : IAsyncLifetime @@ -567,9 +567,9 @@ public sealed class RedHatConnectorTests : IAsyncLifetime services.Configure<JobSchedulerOptions>(schedulerOptions => { - var fetchType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatFetchJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; - var parseType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatParseJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; - var mapType = Type.GetType("StellaOps.Feedser.Source.Distro.RedHat.RedHatMapJob, StellaOps.Feedser.Source.Distro.RedHat", throwOnError: true)!; + var fetchType = Type.GetType("StellaOps.Concelier.Connector.Distro.RedHat.RedHatFetchJob, StellaOps.Concelier.Connector.Distro.RedHat", throwOnError: true)!; + var parseType = Type.GetType("StellaOps.Concelier.Connector.Distro.RedHat.RedHatParseJob, StellaOps.Concelier.Connector.Distro.RedHat", throwOnError: true)!; + var mapType = Type.GetType("StellaOps.Concelier.Connector.Distro.RedHat.RedHatMapJob, StellaOps.Concelier.Connector.Distro.RedHat", throwOnError: true)!; schedulerOptions.Definitions["source:redhat:fetch"] = new JobDefinition("source:redhat:fetch", fetchType, TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *", true); schedulerOptions.Definitions["source:redhat:parse"] = new JobDefinition("source:redhat:parse", parseType, TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *", true); diff --git a/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/StellaOps.Concelier.Connector.Distro.RedHat.Tests.csproj b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/StellaOps.Concelier.Connector.Distro.RedHat.Tests.csproj new file mode 100644 index 00000000..591ae8de --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/StellaOps.Concelier.Connector.Distro.RedHat.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Distro.RedHat/StellaOps.Concelier.Connector.Distro.RedHat.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="RedHat/Fixtures/*.json" + CopyToOutputDirectory="Always" + TargetPath="Source/Distro/RedHat/Fixtures/%(Filename)%(Extension)" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md b/src/StellaOps.Concelier.Connector.Distro.RedHat/AGENTS.md similarity index 81% rename from src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md rename to src/StellaOps.Concelier.Connector.Distro.RedHat/AGENTS.md index 89956ea5..0394fed9 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/AGENTS.md @@ -19,9 +19,9 @@ Red Hat distro connector (Security Data API and OVAL) providing authoritative OS In: authoritative rpm ranges, RHSA mapping, OVAL interpretation, watermarking. Out: building RPM artifacts; cross-distro reconciliation beyond Red Hat. ## Observability & security expectations -- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=redhat`, capturing fetch volumes, parse/OVAL failures, and map affected counts without bespoke metric names. +- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms tagged `concelier.source=redhat`, capturing fetch volumes, parse/OVAL failures, and map affected counts without bespoke metric names. - Logs: cursor bounds, advisory ids, NEVRA counts; allowlist Red Hat endpoints. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Distro.RedHat.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Distro.RedHat.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md b/src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md similarity index 82% rename from src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md rename to src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md index 55298baf..9407b28e 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/CONFLICT_RESOLVER_NOTES.md @@ -1,25 +1,25 @@ -# RHSA Fixture Diffs for Conflict Resolver (Sprint 1) - -_Status date: 2025-10-11_ - -The Red Hat connector fixtures were re-baselined after the model helper rollout so that the conflict resolver receives the canonical payload shape expected for range reconciliation. - -## Key schema deltas - -- `affectedPackages[]` now emits the `type` field ahead of the identifier and always carries a `normalizedVersions` array (empty for NEVRA/CPE today) alongside existing `versionRanges`. -- All nested `provenance` objects (package ranges, statuses, advisory-level metadata, references) now serialize in canonical order – `source`, `kind`, `value`, `decisionReason`, `recordedAt`, `fieldMask` – to align with `AdvisoryProvenance` equality used by the conflict resolver. -- `decisionReason` is now present (null) on provenance payloads so future precedence decisions can annotate overrides without another fixture bump. - -## Impact on conflict resolver - -- Range merge logic must accept an optional `normalizedVersions` array even when it is empty; RPM reconciliation continues to rely on NEVRA primitives (`rangeKind: "nevra"`). -- Provenance comparisons should treat the new property ordering and `decisionReason` field as canonical; older snapshots that lacked these fields are obsolete. -- Advisory/reference provenance now matches the structure that merge emits, so deterministic hashing of resolver inputs will remain stable across connectors. - -## Updated goldens - -- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json` -- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json` -- `src/StellaOps.Feedser.Source.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json` - -Keep these notes in sync with any future provenance or normalized-rule updates so the conflict resolver team can reason about fixture-driven regressions. +# RHSA Fixture Diffs for Conflict Resolver (Sprint 1) + +_Status date: 2025-10-11_ + +The Red Hat connector fixtures were re-baselined after the model helper rollout so that the conflict resolver receives the canonical payload shape expected for range reconciliation. + +## Key schema deltas + +- `affectedPackages[]` now emits the `type` field ahead of the identifier and always carries a `normalizedVersions` array (empty for NEVRA/CPE today) alongside existing `versionRanges`. +- All nested `provenance` objects (package ranges, statuses, advisory-level metadata, references) now serialize in canonical order – `source`, `kind`, `value`, `decisionReason`, `recordedAt`, `fieldMask` – to align with `AdvisoryProvenance` equality used by the conflict resolver. +- `decisionReason` is now present (null) on provenance payloads so future precedence decisions can annotate overrides without another fixture bump. + +## Impact on conflict resolver + +- Range merge logic must accept an optional `normalizedVersions` array even when it is empty; RPM reconciliation continues to rely on NEVRA primitives (`rangeKind: "nevra"`). +- Provenance comparisons should treat the new property ordering and `decisionReason` field as canonical; older snapshots that lacked these fields are obsolete. +- Advisory/reference provenance now matches the structure that merge emits, so deterministic hashing of resolver inputs will remain stable across connectors. + +## Updated goldens + +- `src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0001.snapshot.json` +- `src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0002.snapshot.json` +- `src/StellaOps.Concelier.Connector.Distro.RedHat.Tests/RedHat/Fixtures/rhsa-2025-0003.snapshot.json` + +Keep these notes in sync with any future provenance or normalized-rule updates so the conflict resolver team can reason about fixture-driven regressions. diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Configuration/RedHatOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Configuration/RedHatOptions.cs index 81739d7a..f9d90b28 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Configuration/RedHatOptions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Configuration/RedHatOptions.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Distro.RedHat.Configuration; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Configuration; public sealed class RedHatOptions { @@ -50,7 +50,7 @@ public sealed class RedHatOptions /// <summary> /// Custom user-agent presented to Red Hat endpoints (kept short to satisfy Jetty header limits). /// </summary> - public string UserAgent { get; set; } = "StellaOps.Feedser.RedHat/1.0"; + public string UserAgent { get; set; } = "StellaOps.Concelier.RedHat/1.0"; public void Validate() { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/Models/RedHatCsafModels.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/Models/RedHatCsafModels.cs index 3fc6375e..d97167e3 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/Models/RedHatCsafModels.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/Models/RedHatCsafModels.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal.Models; internal sealed class RedHatCsafEnvelope { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatCursor.cs index cbb807d1..e8e107ff 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatCursor.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal; internal sealed record RedHatCursor( DateTimeOffset? LastReleasedOn, diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatMapper.cs index e9d8a7ad..448c88bf 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatMapper.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatMapper.cs @@ -3,16 +3,16 @@ using System.Collections.Generic; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Distro.RedHat.Internal.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.Distro; -using StellaOps.Feedser.Normalization.Identifiers; -using StellaOps.Feedser.Normalization.Text; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Distro.RedHat.Internal.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.Distro; +using StellaOps.Concelier.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Text; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; -namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal; internal static class RedHatMapper { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatSummaryItem.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatSummaryItem.cs index 0e63349d..81afa8d7 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Internal/RedHatSummaryItem.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Internal/RedHatSummaryItem.cs @@ -1,7 +1,7 @@ using System; using System.Text.Json; -namespace StellaOps.Feedser.Source.Distro.RedHat.Internal; +namespace StellaOps.Concelier.Connector.Distro.RedHat.Internal; internal readonly record struct RedHatSummaryItem(string AdvisoryId, DateTimeOffset ReleasedOn, Uri ResourceUri) { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/Jobs.cs index 86d93841..1ae03368 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Distro.RedHat; +namespace StellaOps.Concelier.Connector.Distro.RedHat; internal static class RedHatJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Distro.RedHat/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..ad7d4030 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Distro.RedHat.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnector.cs index f59381bd..5efab6be 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnector.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnector.cs @@ -7,18 +7,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Distro.RedHat.Configuration; -using StellaOps.Feedser.Source.Distro.RedHat.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Distro.RedHat.Configuration; +using StellaOps.Concelier.Connector.Distro.RedHat.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.RedHat; +namespace StellaOps.Concelier.Connector.Distro.RedHat; public sealed class RedHatConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnectorPlugin.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnectorPlugin.cs index 6b2a03ae..4b5f39b3 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatConnectorPlugin.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.RedHat; +namespace StellaOps.Concelier.Connector.Distro.RedHat; public sealed class RedHatConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatDependencyInjectionRoutine.cs index 39574db0..2cdacf7b 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Distro.RedHat.Configuration; -namespace StellaOps.Feedser.Source.Distro.RedHat; +namespace StellaOps.Concelier.Connector.Distro.RedHat; public sealed class RedHatDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:redhat"; + private const string ConfigurationSection = "concelier:sources:redhat"; private const string FetchCron = "0,15,30,45 * * * *"; private const string ParseCron = "5,20,35,50 * * * *"; private const string MapCron = "10,25,40,55 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatServiceCollectionExtensions.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatServiceCollectionExtensions.cs index 5cf81f46..d872d84e 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/RedHatServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/RedHatServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Distro.RedHat.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Distro.RedHat.Configuration; -namespace StellaOps.Feedser.Source.Distro.RedHat; +namespace StellaOps.Concelier.Connector.Distro.RedHat; public static class RedHatServiceCollectionExtensions { diff --git a/src/StellaOps.Concelier.Connector.Distro.RedHat/StellaOps.Concelier.Connector.Distro.RedHat.csproj b/src/StellaOps.Concelier.Connector.Distro.RedHat/StellaOps.Concelier.Connector.Distro.RedHat.csproj new file mode 100644 index 00000000..f04c7094 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/StellaOps.Concelier.Connector.Distro.RedHat.csproj @@ -0,0 +1,15 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + <ProjectReference Include="../StellaOps.DependencyInjection/StellaOps.DependencyInjection.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" /> + <ProjectReference Include="..\StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md b/src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md similarity index 89% rename from src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md rename to src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md index 2c431a34..96e535cc 100644 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Distro.RedHat/TASKS.md @@ -13,4 +13,4 @@ |Express unaffected/investigation statuses without overloading range fields|BE-Conn-RH|Models|**DONE** – Introduced AffectedPackageStatus collection and updated mapper/tests.| |Reference dedupe & ordering in mapper|BE-Conn-RH|Models|DONE – mapper consolidates by URL, merges metadata, deterministic ordering validated in tests.| |Hydra summary fetch through SourceFetchService|BE-Conn-RH|Source.Common|DONE – summary pages now fetched via SourceFetchService with cache + conditional headers.| -|Fixture validation sweep|QA|Testing|**DOING (2025-10-10)** – Regenerate RHSA fixtures once mapper fixes land, review snapshot diffs, and update docs; blocked by outstanding range provenance patches.| +|Fixture validation sweep|QA|None|**DOING (2025-10-19)** – Prereqs confirmed none; continuing RHSA fixture regeneration and diff review alongside mapper provenance updates.| diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-changes.csv diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0001-1.json diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/Source/Distro/Suse/Fixtures/suse-su-2025_0002-1.json diff --git a/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/StellaOps.Concelier.Connector.Distro.Suse.Tests.csproj b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/StellaOps.Concelier.Connector.Distro.Suse.Tests.csproj new file mode 100644 index 00000000..34253211 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/StellaOps.Concelier.Connector.Distro.Suse.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Connector.Distro.Suse/StellaOps.Concelier.Connector.Distro.Suse.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Source\Distro\Suse\Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseConnectorTests.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseConnectorTests.cs index a1a2c403..3a5f961c 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseConnectorTests.cs @@ -12,21 +12,21 @@ using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Distro.Suse; -using StellaOps.Feedser.Source.Distro.Suse.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Distro.Suse; +using StellaOps.Concelier.Connector.Distro.Suse.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; using Xunit; using Xunit.Abstractions; -namespace StellaOps.Feedser.Source.Distro.Suse.Tests; +namespace StellaOps.Concelier.Connector.Distro.Suse.Tests; [Collection("mongo-fixture")] public sealed class SuseConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseCsafParserTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseCsafParserTests.cs index fdde44fc..edfdea30 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseCsafParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseCsafParserTests.cs @@ -2,10 +2,10 @@ using System; using System.IO; using System.Linq; using System.Text.Json; -using StellaOps.Feedser.Source.Distro.Suse.Internal; +using StellaOps.Concelier.Connector.Distro.Suse.Internal; using Xunit; -namespace StellaOps.Feedser.Source.Distro.Suse.Tests; +namespace StellaOps.Concelier.Connector.Distro.Suse.Tests; public sealed class SuseCsafParserTests { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseMapperTests.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseMapperTests.cs index f5faf065..243485a2 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/SuseMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse.Tests/SuseMapperTests.cs @@ -2,14 +2,14 @@ using System; using System.Collections.Generic; using System.IO; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Distro.Suse; -using StellaOps.Feedser.Source.Distro.Suse.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Distro.Suse; +using StellaOps.Concelier.Connector.Distro.Suse.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; using Xunit; -namespace StellaOps.Feedser.Source.Distro.Suse.Tests; +namespace StellaOps.Concelier.Connector.Distro.Suse.Tests; public sealed class SuseMapperTests { diff --git a/src/StellaOps.Concelier.Connector.Distro.Suse/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/AssemblyInfo.cs new file mode 100644 index 00000000..8a1cf0e4 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Distro.Suse.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Configuration/SuseOptions.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Configuration/SuseOptions.cs index b1849d96..02bd9508 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Configuration/SuseOptions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Configuration/SuseOptions.cs @@ -1,10 +1,10 @@ using System; -namespace StellaOps.Feedser.Source.Distro.Suse.Configuration; +namespace StellaOps.Concelier.Connector.Distro.Suse.Configuration; public sealed class SuseOptions { - public const string HttpClientName = "feedser.suse"; + public const string HttpClientName = "concelier.suse"; /// <summary> /// CSV index enumerating CSAF advisories with their last modification timestamps. @@ -39,7 +39,7 @@ public sealed class SuseOptions /// <summary> /// Custom user agent presented to SUSE endpoints. /// </summary> - public string UserAgent { get; set; } = "StellaOps.Feedser.Suse/0.1 (+https://stella-ops.org)"; + public string UserAgent { get; set; } = "StellaOps.Concelier.Suse/0.1 (+https://stella-ops.org)"; /// <summary> /// Timeout override applied to HTTP requests (defaults to 60 seconds when unset). diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseAdvisoryDto.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseAdvisoryDto.cs index a79b35b8..14d445ab 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseAdvisoryDto.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal sealed record SuseAdvisoryDto( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangeRecord.cs similarity index 60% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangeRecord.cs index 99c8833f..54870611 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangeRecord.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangeRecord.cs @@ -1,5 +1,5 @@ using System; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal sealed record SuseChangeRecord(string FileName, DateTimeOffset ModifiedAt); diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangesParser.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangesParser.cs index 4c61b48b..94893386 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseChangesParser.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseChangesParser.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal static class SuseChangesParser { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCsafParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCsafParser.cs index 3aebaec6..2ae5359c 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCsafParser.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCsafParser.cs @@ -3,9 +3,9 @@ using System.Buffers.Text; using System.Collections.Generic; using System.Globalization; using System.Text.Json; -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Normalization.Distro; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal static class SuseCsafParser { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCursor.cs index 8ac220ad..b2e4acc8 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseCursor.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal sealed record SuseCursor( DateTimeOffset? LastModified, diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseFetchCacheEntry.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseFetchCacheEntry.cs index 2d70dcb5..88cfbdf7 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseFetchCacheEntry.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseFetchCacheEntry.cs @@ -1,13 +1,13 @@ using System; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal sealed record SuseFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) { public static SuseFetchCacheEntry Empty { get; } = new(null, null); - public static SuseFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public static SuseFetchCacheEntry FromDocument(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) => new(document.Etag, document.LastModified); public static SuseFetchCacheEntry FromBson(BsonDocument document) @@ -54,7 +54,7 @@ internal sealed record SuseFetchCacheEntry(string? ETag, DateTimeOffset? LastMod return document; } - public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public bool Matches(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) { if (document is null) { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseMapper.cs index 6135c08a..3a052e08 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Internal/SuseMapper.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Internal/SuseMapper.cs @@ -2,11 +2,11 @@ using System; using System.Collections.Generic; using System.Globalization; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Distro; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Distro; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Distro.Suse.Internal; +namespace StellaOps.Concelier.Connector.Distro.Suse.Internal; internal static class SuseMapper { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/Jobs.cs index d2fcd2fa..cd88105e 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Distro.Suse; +namespace StellaOps.Concelier.Connector.Distro.Suse; internal static class SuseJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Distro.Suse/StellaOps.Concelier.Connector.Distro.Suse.csproj b/src/StellaOps.Concelier.Connector.Distro.Suse/StellaOps.Concelier.Connector.Distro.Suse.csproj new file mode 100644 index 00000000..23396f92 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/StellaOps.Concelier.Connector.Distro.Suse.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnector.cs index c21b19d8..96f947d4 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnector.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnector.cs @@ -11,18 +11,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Distro.Suse.Configuration; -using StellaOps.Feedser.Source.Distro.Suse.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Distro.Suse.Configuration; +using StellaOps.Concelier.Connector.Distro.Suse.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Suse; +namespace StellaOps.Concelier.Connector.Distro.Suse; public sealed class SuseConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnectorPlugin.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnectorPlugin.cs index 00be404e..b6423b57 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/SuseConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Suse; +namespace StellaOps.Concelier.Connector.Distro.Suse; public sealed class SuseConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/SuseDependencyInjectionRoutine.cs index 60da18e7..d4fa8498 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/SuseDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Distro.Suse.Configuration; -namespace StellaOps.Feedser.Source.Distro.Suse; +namespace StellaOps.Concelier.Connector.Distro.Suse; public sealed class SuseDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:suse"; + private const string ConfigurationSection = "concelier:sources:suse"; private const string FetchCron = "*/30 * * * *"; private const string ParseCron = "5,35 * * * *"; private const string MapCron = "10,40 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseServiceCollectionExtensions.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Distro.Suse/SuseServiceCollectionExtensions.cs index 51aff135..12973cb5 100644 --- a/src/StellaOps.Feedser.Source.Distro.Suse/SuseServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Suse/SuseServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Distro.Suse.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Distro.Suse.Configuration; -namespace StellaOps.Feedser.Source.Distro.Suse; +namespace StellaOps.Concelier.Connector.Distro.Suse; public static class SuseServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page0.json diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/Fixtures/ubuntu-notices-page1.json diff --git a/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests.csproj b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests.csproj new file mode 100644 index 00000000..db8d3994 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests.csproj @@ -0,0 +1,18 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> + <ItemGroup> + <None Update="Fixtures\**\*"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs index 4db488bd..5b0e3c4b 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu.Tests/UbuntuConnectorTests.cs @@ -12,18 +12,18 @@ using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Distro.Ubuntu; -using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Distro.Ubuntu; +using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Testing; using Xunit; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Tests; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Tests; [Collection("mongo-fixture")] public sealed class UbuntuConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Configuration/UbuntuOptions.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Configuration/UbuntuOptions.cs index 97596c2f..dd8c89f8 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Configuration/UbuntuOptions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Configuration/UbuntuOptions.cs @@ -1,10 +1,10 @@ using System; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration; public sealed class UbuntuOptions { - public const string HttpClientName = "feedser.ubuntu"; + public const string HttpClientName = "concelier.ubuntu"; public const int MaxPageSize = 20; /// <summary> @@ -27,7 +27,7 @@ public sealed class UbuntuOptions public int IndexPageSize { get; set; } = 20; - public string UserAgent { get; set; } = "StellaOps.Feedser.Ubuntu/0.1 (+https://stella-ops.org)"; + public string UserAgent { get; set; } = "StellaOps.Concelier.Ubuntu/0.1 (+https://stella-ops.org)"; public void Validate() { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuCursor.cs index 56d3bb63..a1584df4 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuCursor.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; internal sealed record UbuntuCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs index 29d739af..157181ac 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuFetchCacheEntry.cs @@ -1,13 +1,13 @@ using System; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; internal sealed record UbuntuFetchCacheEntry(string? ETag, DateTimeOffset? LastModified) { public static UbuntuFetchCacheEntry Empty { get; } = new(null, null); - public static UbuntuFetchCacheEntry FromDocument(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public static UbuntuFetchCacheEntry FromDocument(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) => new(document.Etag, document.LastModified); public static UbuntuFetchCacheEntry FromBson(BsonDocument document) @@ -54,7 +54,7 @@ internal sealed record UbuntuFetchCacheEntry(string? ETag, DateTimeOffset? LastM return doc; } - public bool Matches(StellaOps.Feedser.Storage.Mongo.Documents.DocumentRecord document) + public bool Matches(StellaOps.Concelier.Storage.Mongo.Documents.DocumentRecord document) { if (document is null) { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuMapper.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuMapper.cs index 8176235b..36f8b549 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuMapper.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuMapper.cs @@ -1,11 +1,11 @@ using System; using System.Collections.Generic; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Distro; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Distro; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; internal static class UbuntuMapper { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs index 18b8afa7..184e9e64 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeDto.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; internal sealed record UbuntuNoticeDto( string NoticeId, diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs index 3a175e44..2c10ca38 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Internal/UbuntuNoticeParser.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.Text.Json; -namespace StellaOps.Feedser.Source.Distro.Ubuntu.Internal; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; internal static class UbuntuNoticeParser { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/Jobs.cs index 5fd98027..86c23540 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Distro.Ubuntu; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu; internal static class UbuntuJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj new file mode 100644 index 00000000..23396f92 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/StellaOps.Concelier.Connector.Distro.Ubuntu.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/TASKS.md rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnector.cs index 150e4458..3a6c0229 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnector.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnector.cs @@ -7,18 +7,18 @@ using System.Security.Cryptography; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; -using StellaOps.Feedser.Source.Distro.Ubuntu.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration; +using StellaOps.Concelier.Connector.Distro.Ubuntu.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Ubuntu; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu; public sealed class UbuntuConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnectorPlugin.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnectorPlugin.cs index 12a80e6c..bb8f1e9a 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Distro.Ubuntu; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu; public sealed class UbuntuConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs index 07a4b704..baff7c2d 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration; -namespace StellaOps.Feedser.Source.Distro.Ubuntu; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu; public sealed class UbuntuDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:ubuntu"; + private const string ConfigurationSection = "concelier:sources:ubuntu"; private const string FetchCron = "*/20 * * * *"; private const string ParseCron = "7,27,47 * * * *"; private const string MapCron = "10,30,50 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs index ff362f3d..547beb2e 100644 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Distro.Ubuntu/UbuntuServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Distro.Ubuntu.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Distro.Ubuntu.Configuration; -namespace StellaOps.Feedser.Source.Distro.Ubuntu; +namespace StellaOps.Concelier.Connector.Distro.Ubuntu; public static class UbuntuServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json index dcafd7fa..81aa82ec 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/conflict-ghsa.canonical.json @@ -1,195 +1,195 @@ -{ - "advisoryKey": "GHSA-qqqq-wwww-eeee", - "affectedPackages": [ - { - "type": "semver", - "identifier": "npm:conflict/package", - "platform": null, - "versionRanges": [ - { - "fixedVersion": "1.4.0", - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": "< 1.4.0", - "exactValue": null, - "fixed": "1.4.0", - "fixedInclusive": false, - "introduced": null, - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": false, - "style": "lessThan" - }, - "vendorExtensions": { - "ecosystem": "npm", - "package": "conflict/package" - } - }, - "provenance": { - "source": "ghsa", - "kind": "affected-range", - "value": "npm:conflict/package", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": "< 1.4.0", - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "lt", - "min": null, - "minInclusive": null, - "max": "1.4.0", - "maxInclusive": false, - "value": null, - "notes": "ghsa:npm:conflict/package" - } - ], - "statuses": [ - { - "provenance": { - "source": "ghsa", - "kind": "affected-status", - "value": "npm:conflict/package", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "affectedpackages[].statuses[]" - ] - }, - "status": "affected" - } - ], - "provenance": [ - { - "source": "ghsa", - "kind": "affected", - "value": "npm:conflict/package", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2025-4242", - "GHSA-qqqq-wwww-eeee" - ], - "canonicalMetricId": "ghsa:severity/high", - "credits": [ - { - "displayName": "maintainer-team", - "role": "remediation_developer", - "contacts": [ - "https://github.com/conflict/package" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "maintainer-team", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "security-researcher", - "role": "reporter", - "contacts": [ - "https://github.com/sec-researcher" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "security-researcher", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "cwes": [], - "description": "Container escape vulnerability allowing privilege escalation in conflict-package.", - "exploitKnown": false, - "language": "en", - "modified": "2025-03-02T12:00:00+00:00", - "provenance": [ - { - "source": "ghsa", - "kind": "document", - "value": "https://github.com/advisories/GHSA-qqqq-wwww-eeee", - "decisionReason": null, - "recordedAt": "2025-03-03T18:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "ghsa", - "kind": "mapping", - "value": "GHSA-qqqq-wwww-eeee", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-02-25T00:00:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-qqqq-wwww-eeee", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-qqqq-wwww-eeee" - }, - { - "kind": "fix", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://github.com/conflict/package/releases/tag/v1.4.0", - "decisionReason": null, - "recordedAt": "2025-03-04T08:30:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/conflict/package/releases/tag/v1.4.0" - } - ], - "severity": "high", - "summary": "Container escape in conflict-package", - "title": "Container escape in conflict-package" -} +{ + "advisoryKey": "GHSA-qqqq-wwww-eeee", + "affectedPackages": [ + { + "type": "semver", + "identifier": "npm:conflict/package", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.4.0", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": "< 1.4.0", + "exactValue": null, + "fixed": "1.4.0", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false, + "style": "lessThan" + }, + "vendorExtensions": { + "ecosystem": "npm", + "package": "conflict/package" + } + }, + "provenance": { + "source": "ghsa", + "kind": "affected-range", + "value": "npm:conflict/package", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 1.4.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "lt", + "min": null, + "minInclusive": null, + "max": "1.4.0", + "maxInclusive": false, + "value": null, + "notes": "ghsa:npm:conflict/package" + } + ], + "statuses": [ + { + "provenance": { + "source": "ghsa", + "kind": "affected-status", + "value": "npm:conflict/package", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "affectedpackages[].statuses[]" + ] + }, + "status": "affected" + } + ], + "provenance": [ + { + "source": "ghsa", + "kind": "affected", + "value": "npm:conflict/package", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-4242", + "GHSA-qqqq-wwww-eeee" + ], + "canonicalMetricId": "ghsa:severity/high", + "credits": [ + { + "displayName": "maintainer-team", + "role": "remediation_developer", + "contacts": [ + "https://github.com/conflict/package" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "maintainer-team", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "security-researcher", + "role": "reporter", + "contacts": [ + "https://github.com/sec-researcher" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "security-researcher", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "cwes": [], + "description": "Container escape vulnerability allowing privilege escalation in conflict-package.", + "exploitKnown": false, + "language": "en", + "modified": "2025-03-02T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "document", + "value": "https://github.com/advisories/GHSA-qqqq-wwww-eeee", + "decisionReason": null, + "recordedAt": "2025-03-03T18:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "ghsa", + "kind": "mapping", + "value": "GHSA-qqqq-wwww-eeee", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-02-25T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-qqqq-wwww-eeee", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-qqqq-wwww-eeee" + }, + { + "kind": "fix", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/conflict/package/releases/tag/v1.4.0", + "decisionReason": null, + "recordedAt": "2025-03-04T08:30:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/conflict/package/releases/tag/v1.4.0" + } + ], + "severity": "high", + "summary": "Container escape in conflict-package", + "title": "Container escape in conflict-package" +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.ghsa.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.ghsa.json similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.ghsa.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.ghsa.json index 5cd5ddf8..fc6096fc 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.ghsa.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.ghsa.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "GHSA-credit-parity", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "ghsa:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "ghsa:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "ghsa", - "kind": "document", - "value": "security/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "ghsa", - "kind": "mapping", - "value": "GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "patch", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://example.com/ghsa/patch", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://example.com/ghsa/patch" - }, - { - "kind": "advisory", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-credit-parity" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "GHSA-credit-parity", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "ghsa:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "ghsa:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "document", + "value": "security/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "ghsa", + "kind": "mapping", + "value": "GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://example.com/ghsa/patch", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://example.com/ghsa/patch" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-credit-parity" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.nvd.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.nvd.json index 4c75a6cc..da751c42 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.nvd.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "CVE-2025-5555", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "nvd", - "kind": "credit", - "value": "nvd:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "nvd", - "kind": "credit", - "value": "nvd:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "nvd", - "kind": "document", - "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "nvd", - "kind": "mapping", - "value": "CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "report", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://example.com/nvd/reference", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://example.com/nvd/reference" - }, - { - "kind": "advisory", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "CVE-2025-5555", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "nvd", + "kind": "credit", + "value": "nvd:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "nvd", + "kind": "credit", + "value": "nvd:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "nvd", + "kind": "document", + "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "nvd", + "kind": "mapping", + "value": "CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "report", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://example.com/nvd/reference", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://example.com/nvd/reference" + }, + { + "kind": "advisory", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.osv.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.osv.json similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.osv.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.osv.json index 878c01b6..f7d1d602 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.osv.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/credit-parity.osv.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "GHSA-credit-parity", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "osv", - "kind": "credit", - "value": "osv:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "osv", - "kind": "credit", - "value": "osv:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-credit-parity" - }, - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://osv.dev/vulnerability/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://osv.dev/vulnerability/GHSA-credit-parity" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "GHSA-credit-parity", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "osv", + "kind": "credit", + "value": "osv:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "osv", + "kind": "credit", + "value": "osv:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-credit-parity" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://osv.dev/vulnerability/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://osv.dev/vulnerability/GHSA-credit-parity" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json index e6e7bb65..69fc5320 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json @@ -1,229 +1,229 @@ -{ - "advisoryKey": "GHSA-xxxx-yyyy-zzzz", - "affectedPackages": [ - { - "type": "semver", - "identifier": "npm:example/package", - "platform": null, - "versionRanges": [ - { - "fixedVersion": "1.5.0", - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": "< 1.5.0", - "exactValue": null, - "fixed": "1.5.0", - "fixedInclusive": false, - "introduced": null, - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": false, - "style": "lessThan" - }, - "vendorExtensions": { - "ecosystem": "npm", - "package": "example/package" - } - }, - "provenance": { - "source": "ghsa", - "kind": "affected-range", - "value": "npm:example/package", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": "< 1.5.0", - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "lt", - "min": null, - "minInclusive": null, - "max": "1.5.0", - "maxInclusive": false, - "value": null, - "notes": "ghsa:npm:example/package" - } - ], - "statuses": [ - { - "provenance": { - "source": "ghsa", - "kind": "affected-status", - "value": "npm:example/package", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "affectedpackages[].statuses[]" - ] - }, - "status": "affected" - } - ], - "provenance": [ - { - "source": "ghsa", - "kind": "affected", - "value": "npm:example/package", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2024-1111", - "GHSA-xxxx-yyyy-zzzz" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "credits": [ - { - "displayName": "maintainer-team", - "role": "remediation_developer", - "contacts": [ - "https://github.com/maintainer-team" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "maintainer-team", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "security-reporter", - "role": "reporter", - "contacts": [ - "https://github.com/security-reporter" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "security-reporter", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [ - { - "baseScore": 9.8, - "baseSeverity": "critical", - "provenance": { - "source": "ghsa", - "kind": "cvss", - "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "cvssmetrics[]" - ] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-79", - "name": "Cross-site Scripting", - "uri": "https://cwe.mitre.org/data/definitions/79.html", - "provenance": [ - { - "source": "unknown", - "kind": "unspecified", - "value": null, - "decisionReason": null, - "recordedAt": "1970-01-01T00:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "description": "An example advisory describing a supply chain risk.", - "exploitKnown": false, - "language": "en", - "modified": "2024-09-20T12:00:00+00:00", - "provenance": [ - { - "source": "ghsa", - "kind": "document", - "value": "security/advisories/GHSA-xxxx-yyyy-zzzz", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "ghsa", - "kind": "mapping", - "value": "GHSA-xxxx-yyyy-zzzz", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2024-09-10T13:00:00+00:00", - "references": [ - { - "kind": "fix", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://example.com/patch", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "Vendor Fix", - "summary": null, - "url": "https://example.com/patch" - }, - { - "kind": "advisory", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz", - "decisionReason": null, - "recordedAt": "2024-10-02T00:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz" - } - ], - "severity": "critical", - "summary": "Example GHSA vulnerability", - "title": "Example GHSA vulnerability" +{ + "advisoryKey": "GHSA-xxxx-yyyy-zzzz", + "affectedPackages": [ + { + "type": "semver", + "identifier": "npm:example/package", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.5.0", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": "< 1.5.0", + "exactValue": null, + "fixed": "1.5.0", + "fixedInclusive": false, + "introduced": null, + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false, + "style": "lessThan" + }, + "vendorExtensions": { + "ecosystem": "npm", + "package": "example/package" + } + }, + "provenance": { + "source": "ghsa", + "kind": "affected-range", + "value": "npm:example/package", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": "< 1.5.0", + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "lt", + "min": null, + "minInclusive": null, + "max": "1.5.0", + "maxInclusive": false, + "value": null, + "notes": "ghsa:npm:example/package" + } + ], + "statuses": [ + { + "provenance": { + "source": "ghsa", + "kind": "affected-status", + "value": "npm:example/package", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "affectedpackages[].statuses[]" + ] + }, + "status": "affected" + } + ], + "provenance": [ + { + "source": "ghsa", + "kind": "affected", + "value": "npm:example/package", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2024-1111", + "GHSA-xxxx-yyyy-zzzz" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "credits": [ + { + "displayName": "maintainer-team", + "role": "remediation_developer", + "contacts": [ + "https://github.com/maintainer-team" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "maintainer-team", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "security-reporter", + "role": "reporter", + "contacts": [ + "https://github.com/security-reporter" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "security-reporter", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "ghsa", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-79", + "name": "Cross-site Scripting", + "uri": "https://cwe.mitre.org/data/definitions/79.html", + "provenance": [ + { + "source": "unknown", + "kind": "unspecified", + "value": null, + "decisionReason": null, + "recordedAt": "1970-01-01T00:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "description": "An example advisory describing a supply chain risk.", + "exploitKnown": false, + "language": "en", + "modified": "2024-09-20T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "document", + "value": "security/advisories/GHSA-xxxx-yyyy-zzzz", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "ghsa", + "kind": "mapping", + "value": "GHSA-xxxx-yyyy-zzzz", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2024-09-10T13:00:00+00:00", + "references": [ + { + "kind": "fix", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://example.com/patch", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "Vendor Fix", + "summary": null, + "url": "https://example.com/patch" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz", + "decisionReason": null, + "recordedAt": "2024-10-02T00:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz" + } + ], + "severity": "critical", + "summary": "Example GHSA vulnerability", + "title": "Example GHSA vulnerability" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json index 422b0b72..86e80540 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json @@ -1,61 +1,61 @@ -{ - "ghsa_id": "GHSA-xxxx-yyyy-zzzz", - "summary": "Example GHSA vulnerability", - "description": "An example advisory describing a supply chain risk.", - "severity": "CRITICAL", - "published_at": "2024-09-10T13:00:00Z", - "updated_at": "2024-09-20T12:00:00Z", - "cve_ids": [ - "CVE-2024-1111" - ], - "references": [ - { - "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz", - "type": "ADVISORY" - }, - { - "url": "https://example.com/patch", - "type": "FIX", - "name": "Vendor Fix" - } - ], - "credits": [ - { - "type": "reporter", - "user": { - "login": "security-reporter", - "html_url": "https://github.com/security-reporter" - } - }, - { - "type": "remediation_developer", - "user": { - "login": "maintainer-team", - "html_url": "https://github.com/maintainer-team" - } - } - ], - "cvss": { - "score": 9.8, - "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "severity": "CRITICAL" - }, - "cwes": [ - { - "cwe_id": "CWE-79", - "name": "Cross-site Scripting" - } - ], - "vulnerabilities": [ - { - "package": { - "name": "example/package", - "ecosystem": "npm" - }, - "vulnerable_version_range": "< 1.5.0", - "first_patched_version": { - "identifier": "1.5.0" - } - } - ] -} +{ + "ghsa_id": "GHSA-xxxx-yyyy-zzzz", + "summary": "Example GHSA vulnerability", + "description": "An example advisory describing a supply chain risk.", + "severity": "CRITICAL", + "published_at": "2024-09-10T13:00:00Z", + "updated_at": "2024-09-20T12:00:00Z", + "cve_ids": [ + "CVE-2024-1111" + ], + "references": [ + { + "url": "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz", + "type": "ADVISORY" + }, + { + "url": "https://example.com/patch", + "type": "FIX", + "name": "Vendor Fix" + } + ], + "credits": [ + { + "type": "reporter", + "user": { + "login": "security-reporter", + "html_url": "https://github.com/security-reporter" + } + }, + { + "type": "remediation_developer", + "user": { + "login": "maintainer-team", + "html_url": "https://github.com/maintainer-team" + } + } + ], + "cvss": { + "score": 9.8, + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "severity": "CRITICAL" + }, + "cwes": [ + { + "cwe_id": "CWE-79", + "name": "Cross-site Scripting" + } + ], + "vulnerabilities": [ + { + "package": { + "name": "example/package", + "ecosystem": "npm" + }, + "vulnerable_version_range": "< 1.5.0", + "first_patched_version": { + "identifier": "1.5.0" + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-list.json similarity index 93% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-list.json index b73b82ae..c28d9fa8 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/ghsa-list.json +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Fixtures/ghsa-list.json @@ -1,12 +1,12 @@ -{ - "advisories": [ - { - "ghsa_id": "GHSA-xxxx-yyyy-zzzz", - "updated_at": "2024-09-20T12:00:00Z" - } - ], - "pagination": { - "page": 1, - "has_next_page": false - } -} +{ + "advisories": [ + { + "ghsa_id": "GHSA-xxxx-yyyy-zzzz", + "updated_at": "2024-09-20T12:00:00Z" + } + ], + "pagination": { + "page": 1, + "has_next_page": false + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs index b954bfbf..e3b8010b 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConflictFixtureTests.cs @@ -1,94 +1,94 @@ -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Ghsa.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Ghsa.Tests; - -public sealed class GhsaConflictFixtureTests -{ - [Fact] - public void ConflictFixture_MatchesSnapshot() - { - var recordedAt = new DateTimeOffset(2025, 3, 4, 8, 30, 0, TimeSpan.Zero); - var document = new DocumentRecord( - Id: Guid.Parse("2f5c4d67-fcac-4ec9-a8d4-8a9c5a6d0fc9"), - SourceName: GhsaConnectorPlugin.SourceName, - Uri: "https://github.com/advisories/GHSA-qqqq-wwww-eeee", - FetchedAt: new DateTimeOffset(2025, 3, 3, 18, 0, 0, TimeSpan.Zero), - Sha256: "sha256-ghsa-conflict-fixture", - Status: "completed", - ContentType: "application/json", - Headers: null, - Metadata: null, - Etag: "\"etag-ghsa-conflict\"", - LastModified: new DateTimeOffset(2025, 3, 3, 18, 0, 0, TimeSpan.Zero), - GridFsId: null); - - var dto = new GhsaRecordDto - { - GhsaId = "GHSA-qqqq-wwww-eeee", - Summary = "Container escape in conflict-package", - Description = "Container escape vulnerability allowing privilege escalation in conflict-package.", - Severity = "HIGH", - PublishedAt = new DateTimeOffset(2025, 2, 25, 0, 0, 0, TimeSpan.Zero), - UpdatedAt = new DateTimeOffset(2025, 3, 2, 12, 0, 0, TimeSpan.Zero), - Aliases = new[] { "GHSA-qqqq-wwww-eeee", "CVE-2025-4242" }, - References = new[] - { - new GhsaReferenceDto - { - Url = "https://github.com/advisories/GHSA-qqqq-wwww-eeee", - Type = "ADVISORY" - }, - new GhsaReferenceDto - { - Url = "https://github.com/conflict/package/releases/tag/v1.4.0", - Type = "FIX" - } - }, - Affected = new[] - { - new GhsaAffectedDto - { - PackageName = "conflict/package", - Ecosystem = "npm", - VulnerableRange = "< 1.4.0", - PatchedVersion = "1.4.0" - } - }, - Credits = new[] - { - new GhsaCreditDto - { - Type = "reporter", - Name = "security-researcher", - Login = "sec-researcher", - ProfileUrl = "https://github.com/sec-researcher" - }, - new GhsaCreditDto - { - Type = "remediation_developer", - Name = "maintainer-team", - Login = "conflict-maintainer", - ProfileUrl = "https://github.com/conflict/package" - } - } - }; - - var advisory = GhsaMapper.Map(dto, document, recordedAt); - Assert.Equal("ghsa:severity/high", advisory.CanonicalMetricId); - Assert.True(advisory.CvssMetrics.IsEmpty); - var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); - - var expectedPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-ghsa.canonical.json"); - var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); - - if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-ghsa.canonical.actual.json"); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(expected, snapshot); - } -} +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests; + +public sealed class GhsaConflictFixtureTests +{ + [Fact] + public void ConflictFixture_MatchesSnapshot() + { + var recordedAt = new DateTimeOffset(2025, 3, 4, 8, 30, 0, TimeSpan.Zero); + var document = new DocumentRecord( + Id: Guid.Parse("2f5c4d67-fcac-4ec9-a8d4-8a9c5a6d0fc9"), + SourceName: GhsaConnectorPlugin.SourceName, + Uri: "https://github.com/advisories/GHSA-qqqq-wwww-eeee", + FetchedAt: new DateTimeOffset(2025, 3, 3, 18, 0, 0, TimeSpan.Zero), + Sha256: "sha256-ghsa-conflict-fixture", + Status: "completed", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: "\"etag-ghsa-conflict\"", + LastModified: new DateTimeOffset(2025, 3, 3, 18, 0, 0, TimeSpan.Zero), + GridFsId: null); + + var dto = new GhsaRecordDto + { + GhsaId = "GHSA-qqqq-wwww-eeee", + Summary = "Container escape in conflict-package", + Description = "Container escape vulnerability allowing privilege escalation in conflict-package.", + Severity = "HIGH", + PublishedAt = new DateTimeOffset(2025, 2, 25, 0, 0, 0, TimeSpan.Zero), + UpdatedAt = new DateTimeOffset(2025, 3, 2, 12, 0, 0, TimeSpan.Zero), + Aliases = new[] { "GHSA-qqqq-wwww-eeee", "CVE-2025-4242" }, + References = new[] + { + new GhsaReferenceDto + { + Url = "https://github.com/advisories/GHSA-qqqq-wwww-eeee", + Type = "ADVISORY" + }, + new GhsaReferenceDto + { + Url = "https://github.com/conflict/package/releases/tag/v1.4.0", + Type = "FIX" + } + }, + Affected = new[] + { + new GhsaAffectedDto + { + PackageName = "conflict/package", + Ecosystem = "npm", + VulnerableRange = "< 1.4.0", + PatchedVersion = "1.4.0" + } + }, + Credits = new[] + { + new GhsaCreditDto + { + Type = "reporter", + Name = "security-researcher", + Login = "sec-researcher", + ProfileUrl = "https://github.com/sec-researcher" + }, + new GhsaCreditDto + { + Type = "remediation_developer", + Name = "maintainer-team", + Login = "conflict-maintainer", + ProfileUrl = "https://github.com/conflict/package" + } + } + }; + + var advisory = GhsaMapper.Map(dto, document, recordedAt); + Assert.Equal("ghsa:severity/high", advisory.CanonicalMetricId); + Assert.True(advisory.CvssMetrics.IsEmpty); + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); + + var expectedPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-ghsa.canonical.json"); + var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-ghsa.canonical.actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs index 1afcaf6d..511094db 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaConnectorTests.cs @@ -1,204 +1,204 @@ -using System.Net; -using System.Net.Http; -using System.Text; -using MongoDB.Bson; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Ghsa.Configuration; -using StellaOps.Feedser.Source.Ghsa.Internal; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; - -namespace StellaOps.Feedser.Source.Ghsa.Tests; - -[Collection("mongo-fixture")] -public sealed class GhsaConnectorTests : IAsyncLifetime -{ - private readonly MongoIntegrationFixture _fixture; - private ConnectorTestHarness? _harness; - - public GhsaConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - } - - [Fact] - public async Task FetchParseMap_EmitsCanonicalAdvisory() - { - var initialTime = new DateTimeOffset(2024, 10, 2, 0, 0, 0, TimeSpan.Zero); - await EnsureHarnessAsync(initialTime); - var harness = _harness!; - - var since = initialTime - TimeSpan.FromDays(30); - var listUri = new Uri($"https://ghsa.test/security/advisories?updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&per_page=5"); - harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/ghsa-list.json")); - harness.Handler.SetFallback(request => - { - if (request.RequestUri is null) - { - return new HttpResponseMessage(HttpStatusCode.NotFound); - } - - if (request.RequestUri.AbsoluteUri.Equals("https://ghsa.test/security/advisories/GHSA-xxxx-yyyy-zzzz", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ReadFixture("Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json"), Encoding.UTF8, "application/json") - }; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound); - }); - - var connector = new GhsaConnectorPlugin().Create(harness.ServiceProvider); - - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); - await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); - - var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); - var advisory = await advisoryStore.FindAsync("GHSA-xxxx-yyyy-zzzz", CancellationToken.None); - Assert.NotNull(advisory); - - Assert.Collection(advisory!.Credits, - credit => - { - Assert.Equal("remediation_developer", credit.Role); - Assert.Equal("maintainer-team", credit.DisplayName); - Assert.Contains("https://github.com/maintainer-team", credit.Contacts); - }, - credit => - { - Assert.Equal("reporter", credit.Role); - Assert.Equal("security-reporter", credit.DisplayName); - Assert.Contains("https://github.com/security-reporter", credit.Contacts); - }); - - var weakness = Assert.Single(advisory.Cwes); - Assert.Equal("CWE-79", weakness.Identifier); - Assert.Equal("https://cwe.mitre.org/data/definitions/79.html", weakness.Uri); - - var metric = Assert.Single(advisory.CvssMetrics); - Assert.Equal("3.1", metric.Version); - Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", metric.Vector); - Assert.Equal("critical", metric.BaseSeverity); - Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", advisory.CanonicalMetricId); - - var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); - var expected = ReadFixture("Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json").Replace("\r\n", "\n").TrimEnd(); - - if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-GHSA-xxxx-yyyy-zzzz.actual.json"); - Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(expected, snapshot); - harness.Handler.AssertNoPendingResponses(); - } - - [Fact] - public async Task FetchAsync_RateLimitDefersWindowAndRecordsSnapshot() - { - var initialTime = new DateTimeOffset(2024, 10, 5, 0, 0, 0, TimeSpan.Zero); - await EnsureHarnessAsync(initialTime); - var harness = _harness!; - - var since = initialTime - TimeSpan.FromDays(30); - var until = initialTime; - var listUri = new Uri($"https://ghsa.test/security/advisories?updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page=1&per_page=5"); - - harness.Handler.AddResponse(HttpMethod.Get, listUri, _ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ReadFixture("Fixtures/ghsa-list.json"), Encoding.UTF8, "application/json") - }; - response.Headers.TryAddWithoutValidation("X-RateLimit-Resource", "core"); - response.Headers.TryAddWithoutValidation("X-RateLimit-Limit", "5000"); - response.Headers.TryAddWithoutValidation("X-RateLimit-Remaining", "0"); - return response; - }); - - harness.Handler.SetFallback(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); - - var connector = new GhsaConnectorPlugin().Create(harness.ServiceProvider); - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - - Assert.Single(harness.Handler.Requests); - - var diagnostics = harness.ServiceProvider.GetRequiredService<GhsaDiagnostics>(); - var snapshot = diagnostics.GetLastRateLimitSnapshot(); - Assert.True(snapshot.HasValue); - Assert.Equal("list", snapshot!.Value.Phase); - Assert.Equal("core", snapshot.Value.Resource); - Assert.Equal(0, snapshot.Value.Remaining); - - var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>(); - var state = await stateRepository.TryGetAsync(GhsaConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - - Assert.True(state!.Cursor.TryGetValue("currentWindowStart", out var startValue)); - Assert.True(state.Cursor.TryGetValue("currentWindowEnd", out var endValue)); - Assert.True(state.Cursor.TryGetValue("nextPage", out var nextPageValue)); - - Assert.Equal(since.UtcDateTime, startValue.ToUniversalTime()); - Assert.Equal(until.UtcDateTime, endValue.ToUniversalTime()); - Assert.Equal(1, nextPageValue.AsInt32); - - Assert.True(state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); - Assert.Empty(pendingDocs.AsBsonArray); - Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); - Assert.Empty(pendingMappings.AsBsonArray); - } - - private async Task EnsureHarnessAsync(DateTimeOffset initialTime) - { - if (_harness is not null) - { - return; - } - - var harness = new ConnectorTestHarness(_fixture, initialTime, GhsaOptions.HttpClientName); - await harness.EnsureServiceProviderAsync(services => - { - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddGhsaConnector(options => - { - options.BaseEndpoint = new Uri("https://ghsa.test/", UriKind.Absolute); - options.ApiToken = "test-token"; - options.PageSize = 5; - options.MaxPagesPerFetch = 2; - options.RequestDelay = TimeSpan.Zero; - options.InitialBackfill = TimeSpan.FromDays(30); - options.SecondaryRateLimitBackoff = TimeSpan.FromMilliseconds(10); - }); - }); - - _harness = harness; - } - - private static string ReadFixture(string relativePath) - { - var path = Path.Combine(AppContext.BaseDirectory, relativePath); - return File.ReadAllText(path); - } - - public async Task InitializeAsync() - { - await Task.CompletedTask; - } - - public async Task DisposeAsync() - { - if (_harness is not null) - { - await _harness.DisposeAsync(); - } - } -} +using System.Net; +using System.Net.Http; +using System.Text; +using MongoDB.Bson; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests; + +[Collection("mongo-fixture")] +public sealed class GhsaConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public GhsaConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchParseMap_EmitsCanonicalAdvisory() + { + var initialTime = new DateTimeOffset(2024, 10, 2, 0, 0, 0, TimeSpan.Zero); + await EnsureHarnessAsync(initialTime); + var harness = _harness!; + + var since = initialTime - TimeSpan.FromDays(30); + var listUri = new Uri($"https://ghsa.test/security/advisories?updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(initialTime.ToString("O"))}&page=1&per_page=5"); + harness.Handler.AddJsonResponse(listUri, ReadFixture("Fixtures/ghsa-list.json")); + harness.Handler.SetFallback(request => + { + if (request.RequestUri is null) + { + return new HttpResponseMessage(HttpStatusCode.NotFound); + } + + if (request.RequestUri.AbsoluteUri.Equals("https://ghsa.test/security/advisories/GHSA-xxxx-yyyy-zzzz", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ghsa-GHSA-xxxx-yyyy-zzzz.json"), Encoding.UTF8, "application/json") + }; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + var connector = new GhsaConnectorPlugin().Create(harness.ServiceProvider); + + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var advisoryStore = harness.ServiceProvider.GetRequiredService<IAdvisoryStore>(); + var advisory = await advisoryStore.FindAsync("GHSA-xxxx-yyyy-zzzz", CancellationToken.None); + Assert.NotNull(advisory); + + Assert.Collection(advisory!.Credits, + credit => + { + Assert.Equal("remediation_developer", credit.Role); + Assert.Equal("maintainer-team", credit.DisplayName); + Assert.Contains("https://github.com/maintainer-team", credit.Contacts); + }, + credit => + { + Assert.Equal("reporter", credit.Role); + Assert.Equal("security-reporter", credit.DisplayName); + Assert.Contains("https://github.com/security-reporter", credit.Contacts); + }); + + var weakness = Assert.Single(advisory.Cwes); + Assert.Equal("CWE-79", weakness.Identifier); + Assert.Equal("https://cwe.mitre.org/data/definitions/79.html", weakness.Uri); + + var metric = Assert.Single(advisory.CvssMetrics); + Assert.Equal("3.1", metric.Version); + Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", metric.Vector); + Assert.Equal("critical", metric.BaseSeverity); + Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", advisory.CanonicalMetricId); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); + var expected = ReadFixture("Fixtures/expected-GHSA-xxxx-yyyy-zzzz.json").Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "expected-GHSA-xxxx-yyyy-zzzz.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + harness.Handler.AssertNoPendingResponses(); + } + + [Fact] + public async Task FetchAsync_RateLimitDefersWindowAndRecordsSnapshot() + { + var initialTime = new DateTimeOffset(2024, 10, 5, 0, 0, 0, TimeSpan.Zero); + await EnsureHarnessAsync(initialTime); + var harness = _harness!; + + var since = initialTime - TimeSpan.FromDays(30); + var until = initialTime; + var listUri = new Uri($"https://ghsa.test/security/advisories?updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page=1&per_page=5"); + + harness.Handler.AddResponse(HttpMethod.Get, listUri, _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ReadFixture("Fixtures/ghsa-list.json"), Encoding.UTF8, "application/json") + }; + response.Headers.TryAddWithoutValidation("X-RateLimit-Resource", "core"); + response.Headers.TryAddWithoutValidation("X-RateLimit-Limit", "5000"); + response.Headers.TryAddWithoutValidation("X-RateLimit-Remaining", "0"); + return response; + }); + + harness.Handler.SetFallback(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + + var connector = new GhsaConnectorPlugin().Create(harness.ServiceProvider); + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + + Assert.Single(harness.Handler.Requests); + + var diagnostics = harness.ServiceProvider.GetRequiredService<GhsaDiagnostics>(); + var snapshot = diagnostics.GetLastRateLimitSnapshot(); + Assert.True(snapshot.HasValue); + Assert.Equal("list", snapshot!.Value.Phase); + Assert.Equal("core", snapshot.Value.Resource); + Assert.Equal(0, snapshot.Value.Remaining); + + var stateRepository = harness.ServiceProvider.GetRequiredService<ISourceStateRepository>(); + var state = await stateRepository.TryGetAsync(GhsaConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + Assert.True(state!.Cursor.TryGetValue("currentWindowStart", out var startValue)); + Assert.True(state.Cursor.TryGetValue("currentWindowEnd", out var endValue)); + Assert.True(state.Cursor.TryGetValue("nextPage", out var nextPageValue)); + + Assert.Equal(since.UtcDateTime, startValue.ToUniversalTime()); + Assert.Equal(until.UtcDateTime, endValue.ToUniversalTime()); + Assert.Equal(1, nextPageValue.AsInt32); + + Assert.True(state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs)); + Assert.Empty(pendingDocs.AsBsonArray); + Assert.True(state.Cursor.TryGetValue("pendingMappings", out var pendingMappings)); + Assert.Empty(pendingMappings.AsBsonArray); + } + + private async Task EnsureHarnessAsync(DateTimeOffset initialTime) + { + if (_harness is not null) + { + return; + } + + var harness = new ConnectorTestHarness(_fixture, initialTime, GhsaOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddGhsaConnector(options => + { + options.BaseEndpoint = new Uri("https://ghsa.test/", UriKind.Absolute); + options.ApiToken = "test-token"; + options.PageSize = 5; + options.MaxPagesPerFetch = 2; + options.RequestDelay = TimeSpan.Zero; + options.InitialBackfill = TimeSpan.FromDays(30); + options.SecondaryRateLimitBackoff = TimeSpan.FromMilliseconds(10); + }); + }); + + _harness = harness; + } + + private static string ReadFixture(string relativePath) + { + var path = Path.Combine(AppContext.BaseDirectory, relativePath); + return File.ReadAllText(path); + } + + public async Task InitializeAsync() + { + await Task.CompletedTask; + } + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs index 6fc87bcf..241ab59f 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaCreditParityRegressionTests.cs @@ -1,51 +1,51 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Source.Ghsa.Tests.Ghsa; - -public sealed class GhsaCreditParityRegressionTests -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); - - [Fact] - public void CreditParity_FixturesRemainInSyncAcrossSources() - { - var ghsa = LoadFixture("credit-parity.ghsa.json"); - var osv = LoadFixture("credit-parity.osv.json"); - var nvd = LoadFixture("credit-parity.nvd.json"); - - var ghsaCredits = NormalizeCredits(ghsa); - var osvCredits = NormalizeCredits(osv); - var nvdCredits = NormalizeCredits(nvd); - - Assert.NotEmpty(ghsaCredits); - Assert.Equal(ghsaCredits, osvCredits); - Assert.Equal(ghsaCredits, nvdCredits); - } - - private static Advisory LoadFixture(string fileName) - { - var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName); - return JsonSerializer.Deserialize<Advisory>(File.ReadAllText(path), SerializerOptions) - ?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'."); - } - - private static HashSet<string> NormalizeCredits(Advisory advisory) - { - var set = new HashSet<string>(StringComparer.Ordinal); - foreach (var credit in advisory.Credits) - { - var contactList = credit.Contacts.IsDefaultOrEmpty - ? Array.Empty<string>() - : credit.Contacts.ToArray(); - var contacts = string.Join("|", contactList.OrderBy(static contact => contact, StringComparer.Ordinal)); - var key = string.Join("||", credit.Role ?? string.Empty, credit.DisplayName ?? string.Empty, contacts); - set.Add(key); - } - - return set; - } -} +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests.Ghsa; + +public sealed class GhsaCreditParityRegressionTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public void CreditParity_FixturesRemainInSyncAcrossSources() + { + var ghsa = LoadFixture("credit-parity.ghsa.json"); + var osv = LoadFixture("credit-parity.osv.json"); + var nvd = LoadFixture("credit-parity.nvd.json"); + + var ghsaCredits = NormalizeCredits(ghsa); + var osvCredits = NormalizeCredits(osv); + var nvdCredits = NormalizeCredits(nvd); + + Assert.NotEmpty(ghsaCredits); + Assert.Equal(ghsaCredits, osvCredits); + Assert.Equal(ghsaCredits, nvdCredits); + } + + private static Advisory LoadFixture(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName); + return JsonSerializer.Deserialize<Advisory>(File.ReadAllText(path), SerializerOptions) + ?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'."); + } + + private static HashSet<string> NormalizeCredits(Advisory advisory) + { + var set = new HashSet<string>(StringComparer.Ordinal); + foreach (var credit in advisory.Credits) + { + var contactList = credit.Contacts.IsDefaultOrEmpty + ? Array.Empty<string>() + : credit.Contacts.ToArray(); + var contacts = string.Join("|", contactList.OrderBy(static contact => contact, StringComparer.Ordinal)); + var key = string.Join("||", credit.Role ?? string.Empty, credit.DisplayName ?? string.Empty, contacts); + set.Add(key); + } + + return set; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs similarity index 82% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs index b60c86de..287ad3ea 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDependencyInjectionRoutineTests.cs @@ -1,71 +1,71 @@ -using System; -using System.Collections.Generic; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ghsa; -using StellaOps.Feedser.Source.Ghsa.Configuration; -using Xunit; - -namespace StellaOps.Feedser.Source.Ghsa.Tests.Ghsa; - -public sealed class GhsaDependencyInjectionRoutineTests -{ - [Fact] - public void Register_ConfiguresConnectorAndScheduler() - { - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddOptions(); - services.AddSourceCommon(); - - var configuration = new ConfigurationBuilder() - .AddInMemoryCollection(new Dictionary<string, string?> - { - ["feedser:sources:ghsa:apiToken"] = "test-token", - ["feedser:sources:ghsa:pageSize"] = "25", - ["feedser:sources:ghsa:maxPagesPerFetch"] = "3", - ["feedser:sources:ghsa:initialBackfill"] = "1.00:00:00", - }) - .Build(); - - var routine = new GhsaDependencyInjectionRoutine(); - routine.Register(services, configuration); - - services.Configure<JobSchedulerOptions>(_ => { }); - - var provider = services.BuildServiceProvider(validateScopes: true); - - var ghsaOptions = provider.GetRequiredService<IOptions<GhsaOptions>>().Value; - Assert.Equal("test-token", ghsaOptions.ApiToken); - Assert.Equal(25, ghsaOptions.PageSize); - Assert.Equal(TimeSpan.FromDays(1), ghsaOptions.InitialBackfill); - - var schedulerOptions = provider.GetRequiredService<IOptions<JobSchedulerOptions>>().Value; - Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Fetch, out var fetchDefinition)); - Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Parse, out var parseDefinition)); - Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Map, out var mapDefinition)); - - Assert.Equal(typeof(GhsaFetchJob), fetchDefinition.JobType); - Assert.Equal(TimeSpan.FromMinutes(6), fetchDefinition.Timeout); - Assert.Equal(TimeSpan.FromMinutes(4), fetchDefinition.LeaseDuration); - Assert.Equal("1,11,21,31,41,51 * * * *", fetchDefinition.CronExpression); - Assert.True(fetchDefinition.Enabled); - - Assert.Equal(typeof(GhsaParseJob), parseDefinition.JobType); - Assert.Equal(TimeSpan.FromMinutes(5), parseDefinition.Timeout); - Assert.Equal(TimeSpan.FromMinutes(4), parseDefinition.LeaseDuration); - Assert.Equal("3,13,23,33,43,53 * * * *", parseDefinition.CronExpression); - Assert.True(parseDefinition.Enabled); - - Assert.Equal(typeof(GhsaMapJob), mapDefinition.JobType); - Assert.Equal(TimeSpan.FromMinutes(5), mapDefinition.Timeout); - Assert.Equal(TimeSpan.FromMinutes(4), mapDefinition.LeaseDuration); - Assert.Equal("5,15,25,35,45,55 * * * *", mapDefinition.CronExpression); - Assert.True(mapDefinition.Enabled); - } -} +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ghsa; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests.Ghsa; + +public sealed class GhsaDependencyInjectionRoutineTests +{ + [Fact] + public void Register_ConfiguresConnectorAndScheduler() + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddOptions(); + services.AddSourceCommon(); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary<string, string?> + { + ["concelier:sources:ghsa:apiToken"] = "test-token", + ["concelier:sources:ghsa:pageSize"] = "25", + ["concelier:sources:ghsa:maxPagesPerFetch"] = "3", + ["concelier:sources:ghsa:initialBackfill"] = "1.00:00:00", + }) + .Build(); + + var routine = new GhsaDependencyInjectionRoutine(); + routine.Register(services, configuration); + + services.Configure<JobSchedulerOptions>(_ => { }); + + var provider = services.BuildServiceProvider(validateScopes: true); + + var ghsaOptions = provider.GetRequiredService<IOptions<GhsaOptions>>().Value; + Assert.Equal("test-token", ghsaOptions.ApiToken); + Assert.Equal(25, ghsaOptions.PageSize); + Assert.Equal(TimeSpan.FromDays(1), ghsaOptions.InitialBackfill); + + var schedulerOptions = provider.GetRequiredService<IOptions<JobSchedulerOptions>>().Value; + Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Fetch, out var fetchDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Parse, out var parseDefinition)); + Assert.True(schedulerOptions.Definitions.TryGetValue(GhsaJobKinds.Map, out var mapDefinition)); + + Assert.Equal(typeof(GhsaFetchJob), fetchDefinition.JobType); + Assert.Equal(TimeSpan.FromMinutes(6), fetchDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(4), fetchDefinition.LeaseDuration); + Assert.Equal("1,11,21,31,41,51 * * * *", fetchDefinition.CronExpression); + Assert.True(fetchDefinition.Enabled); + + Assert.Equal(typeof(GhsaParseJob), parseDefinition.JobType); + Assert.Equal(TimeSpan.FromMinutes(5), parseDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(4), parseDefinition.LeaseDuration); + Assert.Equal("3,13,23,33,43,53 * * * *", parseDefinition.CronExpression); + Assert.True(parseDefinition.Enabled); + + Assert.Equal(typeof(GhsaMapJob), mapDefinition.JobType); + Assert.Equal(TimeSpan.FromMinutes(5), mapDefinition.Timeout); + Assert.Equal(TimeSpan.FromMinutes(4), mapDefinition.LeaseDuration); + Assert.Equal("5,15,25,35,45,55 * * * *", mapDefinition.CronExpression); + Assert.True(mapDefinition.Enabled); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs index 5f7e7584..3af10165 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaDiagnosticsTests.cs @@ -1,35 +1,35 @@ -using System; -using StellaOps.Feedser.Source.Ghsa.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ghsa.Tests.Ghsa; - -public class GhsaDiagnosticsTests : IDisposable -{ - private readonly GhsaDiagnostics diagnostics = new(); - - [Fact] - public void RecordRateLimit_PersistsSnapshot() - { - var snapshot = new GhsaRateLimitSnapshot( - Phase: "list", - Resource: "core", - Limit: 5000, - Remaining: 100, - Used: 4900, - ResetAt: DateTimeOffset.UtcNow.AddMinutes(1), - ResetAfter: TimeSpan.FromMinutes(1), - RetryAfter: TimeSpan.FromSeconds(10)); - - diagnostics.RecordRateLimit(snapshot); - - var stored = diagnostics.GetLastRateLimitSnapshot(); - Assert.NotNull(stored); - Assert.Equal(snapshot, stored); - } - - public void Dispose() - { - diagnostics.Dispose(); - } -} +using System; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests.Ghsa; + +public class GhsaDiagnosticsTests : IDisposable +{ + private readonly GhsaDiagnostics diagnostics = new(); + + [Fact] + public void RecordRateLimit_PersistsSnapshot() + { + var snapshot = new GhsaRateLimitSnapshot( + Phase: "list", + Resource: "core", + Limit: 5000, + Remaining: 100, + Used: 4900, + ResetAt: DateTimeOffset.UtcNow.AddMinutes(1), + ResetAfter: TimeSpan.FromMinutes(1), + RetryAfter: TimeSpan.FromSeconds(10)); + + diagnostics.RecordRateLimit(snapshot); + + var stored = diagnostics.GetLastRateLimitSnapshot(); + Assert.NotNull(stored); + Assert.Equal(snapshot, stored); + } + + public void Dispose() + { + diagnostics.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaMapperTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaMapperTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaMapperTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaMapperTests.cs index f93923fa..20802584 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaMapperTests.cs @@ -1,7 +1,7 @@ -using StellaOps.Feedser.Source.Ghsa.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Ghsa.Tests; +namespace StellaOps.Concelier.Connector.Ghsa.Tests; public sealed class GhsaMapperTests { diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs rename to src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs index a54b737e..bde55541 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/Ghsa/GhsaRateLimitParserTests.cs @@ -1,60 +1,60 @@ -using System.Collections.Generic; -using System.Globalization; -using StellaOps.Feedser.Source.Ghsa.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ghsa.Tests.Ghsa; - -public class GhsaRateLimitParserTests -{ - [Fact] - public void TryParse_ReturnsSnapshot_WhenHeadersPresent() - { - var now = DateTimeOffset.UtcNow; - var reset = now.AddMinutes(5); - - var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) - { - ["X-RateLimit-Limit"] = "5000", - ["X-RateLimit-Remaining"] = "42", - ["X-RateLimit-Used"] = "4958", - ["X-RateLimit-Reset"] = reset.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture), - ["X-RateLimit-Resource"] = "core" - }; - - var snapshot = GhsaRateLimitParser.TryParse(headers, now, "list"); - - Assert.True(snapshot.HasValue); - Assert.Equal("list", snapshot.Value.Phase); - Assert.Equal("core", snapshot.Value.Resource); - Assert.Equal(5000, snapshot.Value.Limit); - Assert.Equal(42, snapshot.Value.Remaining); - Assert.Equal(4958, snapshot.Value.Used); - Assert.NotNull(snapshot.Value.ResetAfter); - Assert.True(snapshot.Value.ResetAfter!.Value.TotalMinutes <= 5.1 && snapshot.Value.ResetAfter.Value.TotalMinutes >= 4.9); - } - - [Fact] - public void TryParse_ReturnsNull_WhenHeadersMissing() - { - var snapshot = GhsaRateLimitParser.TryParse(null, DateTimeOffset.UtcNow, "list"); - Assert.Null(snapshot); - } - - [Fact] - public void TryParse_HandlesRetryAfter() - { - var now = DateTimeOffset.UtcNow; - var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) - { - ["Retry-After"] = "60" - }; - - var snapshot = GhsaRateLimitParser.TryParse(headers, now, "detail"); - - Assert.True(snapshot.HasValue); - Assert.Equal("detail", snapshot.Value.Phase); - Assert.NotNull(snapshot.Value.RetryAfter); - Assert.Equal(60, Math.Round(snapshot.Value.RetryAfter!.Value.TotalSeconds)); - } -} +using System.Collections.Generic; +using System.Globalization; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ghsa.Tests.Ghsa; + +public class GhsaRateLimitParserTests +{ + [Fact] + public void TryParse_ReturnsSnapshot_WhenHeadersPresent() + { + var now = DateTimeOffset.UtcNow; + var reset = now.AddMinutes(5); + + var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["X-RateLimit-Limit"] = "5000", + ["X-RateLimit-Remaining"] = "42", + ["X-RateLimit-Used"] = "4958", + ["X-RateLimit-Reset"] = reset.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture), + ["X-RateLimit-Resource"] = "core" + }; + + var snapshot = GhsaRateLimitParser.TryParse(headers, now, "list"); + + Assert.True(snapshot.HasValue); + Assert.Equal("list", snapshot.Value.Phase); + Assert.Equal("core", snapshot.Value.Resource); + Assert.Equal(5000, snapshot.Value.Limit); + Assert.Equal(42, snapshot.Value.Remaining); + Assert.Equal(4958, snapshot.Value.Used); + Assert.NotNull(snapshot.Value.ResetAfter); + Assert.True(snapshot.Value.ResetAfter!.Value.TotalMinutes <= 5.1 && snapshot.Value.ResetAfter.Value.TotalMinutes >= 4.9); + } + + [Fact] + public void TryParse_ReturnsNull_WhenHeadersMissing() + { + var snapshot = GhsaRateLimitParser.TryParse(null, DateTimeOffset.UtcNow, "list"); + Assert.Null(snapshot); + } + + [Fact] + public void TryParse_HandlesRetryAfter() + { + var now = DateTimeOffset.UtcNow; + var headers = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase) + { + ["Retry-After"] = "60" + }; + + var snapshot = GhsaRateLimitParser.TryParse(headers, now, "detail"); + + Assert.True(snapshot.HasValue); + Assert.Equal("detail", snapshot.Value.Phase); + Assert.NotNull(snapshot.Value.RetryAfter); + Assert.Equal(60, Math.Round(snapshot.Value.RetryAfter!.Value.TotalSeconds)); + } +} diff --git a/src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj b/src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj new file mode 100644 index 00000000..dfb9dd10 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + <ItemGroup> + <ProjectReference Include="../StellaOps.Concelier.Connector.Ghsa/StellaOps.Concelier.Connector.Ghsa.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj" /> + </ItemGroup> + <ItemGroup> + <None Include="Fixtures/*.json" CopyToOutputDirectory="Always" /> + </ItemGroup> +</Project> diff --git a/src/StellaOps.Feedser.Source.Ghsa/AGENTS.md b/src/StellaOps.Concelier.Connector.Ghsa/AGENTS.md similarity index 87% rename from src/StellaOps.Feedser.Source.Ghsa/AGENTS.md rename to src/StellaOps.Concelier.Connector.Ghsa/AGENTS.md index e80bd4c5..be116480 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Ghsa/AGENTS.md @@ -1,39 +1,39 @@ -# AGENTS -## Role -Implement a connector for GitHub Security Advisories (GHSA) when we need to ingest GHSA content directly (instead of crosswalking via OSV/NVD). - -## Scope -- Determine the optimal GHSA data source (GraphQL API, REST, or ecosystem export) and required authentication. -- Implement fetch logic with pagination, updated-since filtering, and cursor persistence. -- Parse GHSA records (identifiers, summaries, affected packages, versions, references, severity). -- Map advisories into canonical `Advisory` objects with aliases, references, affected packages, and range primitives. -- Provide deterministic fixtures and regression tests for the full pipeline. - -## Participants -- `Source.Common` (HTTP clients, fetch service, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores and source state). -- `Feedser.Models` (canonical advisory types). -- `Feedser.Testing` (integration harness, snapshot helpers). - -## Interfaces & Contracts -- Job kinds: `ghsa:fetch`, `ghsa:parse`, `ghsa:map`. -- Support GitHub API authentication & rate limiting (token, retry/backoff). -- Alias set must include GHSA IDs and linked CVE IDs. - -## In/Out of scope -In scope: -- Full GHSA connector implementation with range primitives and provenance instrumentation. - -Out of scope: -- Repo-specific advisory ingest (handled via GitHub repo exports). -- Downstream ecosystem-specific enrichments. - -## Observability & Security Expectations -- Log fetch pagination, throttling, and mapping stats. -- Handle GitHub API rate limits with exponential backoff and `Retry-After`. -- Sanitize/validate payloads before persistence. - -## Tests -- Add `StellaOps.Feedser.Source.Ghsa.Tests` with canned GraphQL/REST fixtures. -- Snapshot canonical advisories; enable fixture regeneration with env flag. -- Confirm deterministic ordering/time normalisation. +# AGENTS +## Role +Implement a connector for GitHub Security Advisories (GHSA) when we need to ingest GHSA content directly (instead of crosswalking via OSV/NVD). + +## Scope +- Determine the optimal GHSA data source (GraphQL API, REST, or ecosystem export) and required authentication. +- Implement fetch logic with pagination, updated-since filtering, and cursor persistence. +- Parse GHSA records (identifiers, summaries, affected packages, versions, references, severity). +- Map advisories into canonical `Advisory` objects with aliases, references, affected packages, and range primitives. +- Provide deterministic fixtures and regression tests for the full pipeline. + +## Participants +- `Source.Common` (HTTP clients, fetch service, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores and source state). +- `Concelier.Models` (canonical advisory types). +- `Concelier.Testing` (integration harness, snapshot helpers). + +## Interfaces & Contracts +- Job kinds: `ghsa:fetch`, `ghsa:parse`, `ghsa:map`. +- Support GitHub API authentication & rate limiting (token, retry/backoff). +- Alias set must include GHSA IDs and linked CVE IDs. + +## In/Out of scope +In scope: +- Full GHSA connector implementation with range primitives and provenance instrumentation. + +Out of scope: +- Repo-specific advisory ingest (handled via GitHub repo exports). +- Downstream ecosystem-specific enrichments. + +## Observability & Security Expectations +- Log fetch pagination, throttling, and mapping stats. +- Handle GitHub API rate limits with exponential backoff and `Retry-After`. +- Sanitize/validate payloads before persistence. + +## Tests +- Add `StellaOps.Concelier.Connector.Ghsa.Tests` with canned GraphQL/REST fixtures. +- Snapshot canonical advisories; enable fixture regeneration with env flag. +- Confirm deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs b/src/StellaOps.Concelier.Connector.Ghsa/Configuration/GhsaOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Configuration/GhsaOptions.cs index 3020caf0..5f6ac65a 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Configuration/GhsaOptions.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Configuration/GhsaOptions.cs @@ -1,75 +1,75 @@ -using System.Diagnostics.CodeAnalysis; - -namespace StellaOps.Feedser.Source.Ghsa.Configuration; - -public sealed class GhsaOptions -{ - public static string HttpClientName => "source.ghsa"; - - public Uri BaseEndpoint { get; set; } = new("https://api.github.com/", UriKind.Absolute); - - public string ApiToken { get; set; } = string.Empty; - - public int PageSize { get; set; } = 50; - - public int MaxPagesPerFetch { get; set; } = 5; - - public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); - - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(200); - - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - public int RateLimitWarningThreshold { get; set; } = 500; - - public TimeSpan SecondaryRateLimitBackoff { get; set; } = TimeSpan.FromMinutes(2); - - [MemberNotNull(nameof(BaseEndpoint), nameof(ApiToken))] - public void Validate() - { - if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); - } - - if (string.IsNullOrWhiteSpace(ApiToken)) - { - throw new InvalidOperationException("ApiToken must be provided."); - } - - if (PageSize is < 1 or > 100) - { - throw new InvalidOperationException("PageSize must be between 1 and 100."); - } - - if (MaxPagesPerFetch <= 0) - { - throw new InvalidOperationException("MaxPagesPerFetch must be positive."); - } - - if (InitialBackfill < TimeSpan.Zero) - { - throw new InvalidOperationException("InitialBackfill cannot be negative."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("RequestDelay cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("FailureBackoff must be greater than zero."); - } - - if (RateLimitWarningThreshold < 0) - { - throw new InvalidOperationException("RateLimitWarningThreshold cannot be negative."); - } - - if (SecondaryRateLimitBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("SecondaryRateLimitBackoff must be greater than zero."); - } - } -} +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Concelier.Connector.Ghsa.Configuration; + +public sealed class GhsaOptions +{ + public static string HttpClientName => "source.ghsa"; + + public Uri BaseEndpoint { get; set; } = new("https://api.github.com/", UriKind.Absolute); + + public string ApiToken { get; set; } = string.Empty; + + public int PageSize { get; set; } = 50; + + public int MaxPagesPerFetch { get; set; } = 5; + + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(30); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(200); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + public int RateLimitWarningThreshold { get; set; } = 500; + + public TimeSpan SecondaryRateLimitBackoff { get; set; } = TimeSpan.FromMinutes(2); + + [MemberNotNull(nameof(BaseEndpoint), nameof(ApiToken))] + public void Validate() + { + if (BaseEndpoint is null || !BaseEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("BaseEndpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ApiToken)) + { + throw new InvalidOperationException("ApiToken must be provided."); + } + + if (PageSize is < 1 or > 100) + { + throw new InvalidOperationException("PageSize must be between 1 and 100."); + } + + if (MaxPagesPerFetch <= 0) + { + throw new InvalidOperationException("MaxPagesPerFetch must be positive."); + } + + if (InitialBackfill < TimeSpan.Zero) + { + throw new InvalidOperationException("InitialBackfill cannot be negative."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be greater than zero."); + } + + if (RateLimitWarningThreshold < 0) + { + throw new InvalidOperationException("RateLimitWarningThreshold cannot be negative."); + } + + if (SecondaryRateLimitBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("SecondaryRateLimitBackoff must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs b/src/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs rename to src/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs index c5eaaf0e..b5e43f05 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnector.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/GhsaConnector.cs @@ -1,547 +1,547 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net.Http; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Ghsa.Configuration; -using StellaOps.Feedser.Source.Ghsa.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ghsa; - -public sealed class GhsaConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - WriteIndented = false, - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly GhsaOptions _options; - private readonly GhsaDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger<GhsaConnector> _logger; - private readonly object _rateLimitWarningLock = new(); - private readonly Dictionary<(string Phase, string Resource), bool> _rateLimitWarnings = new(); - - public GhsaConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions<GhsaOptions> options, - GhsaDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger<GhsaConnector> logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => GhsaConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var now = _timeProvider.GetUtcNow(); - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - _options.InitialBackfill; - if (since > now) - { - since = now; - } - - var until = cursor.CurrentWindowEnd ?? now; - if (until <= since) - { - until = since + TimeSpan.FromMinutes(1); - } - - var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; - var pagesFetched = 0; - var hasMore = true; - var rateLimitHit = false; - DateTimeOffset? maxUpdated = cursor.LastUpdatedExclusive; - - while (hasMore && pagesFetched < _options.MaxPagesPerFetch) - { - cancellationToken.ThrowIfCancellationRequested(); - - var listUri = BuildListUri(since, until, page, _options.PageSize); - var metadata = new Dictionary<string, string>(StringComparer.Ordinal) - { - ["since"] = since.ToString("O"), - ["until"] = until.ToString("O"), - ["page"] = page.ToString(CultureInfo.InvariantCulture), - ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), - }; - - SourceFetchContentResult listResult; - try - { - _diagnostics.FetchAttempt(); - listResult = await _fetchService.FetchContentAsync( - new SourceFetchRequest( - GhsaOptions.HttpClientName, - SourceName, - listUri) - { - Metadata = metadata, - AcceptHeaders = new[] { "application/vnd.github+json" }, - }, - cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException ex) - { - _diagnostics.FetchFailure(); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (listResult.IsNotModified) - { - _diagnostics.FetchUnchanged(); - break; - } - - if (!listResult.IsSuccess || listResult.Content is null) - { - _diagnostics.FetchFailure(); - break; - } - - var deferList = await ApplyRateLimitAsync(listResult.Headers, "list", cancellationToken).ConfigureAwait(false); - if (deferList) - { - rateLimitHit = true; - break; - } - - var pageModel = GhsaListParser.Parse(listResult.Content, page, _options.PageSize); - - if (pageModel.Items.Count == 0) - { - hasMore = false; - } - - foreach (var item in pageModel.Items) - { - cancellationToken.ThrowIfCancellationRequested(); - - var detailUri = BuildDetailUri(item.GhsaId); - var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) - { - ["ghsaId"] = item.GhsaId, - ["page"] = page.ToString(CultureInfo.InvariantCulture), - ["since"] = since.ToString("O"), - ["until"] = until.ToString("O"), - }; - - SourceFetchResult detailResult; - try - { - detailResult = await _fetchService.FetchAsync( - new SourceFetchRequest( - GhsaOptions.HttpClientName, - SourceName, - detailUri) - { - Metadata = detailMetadata, - AcceptHeaders = new[] { "application/vnd.github+json" }, - }, - cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException ex) - { - _diagnostics.FetchFailure(); - _logger.LogWarning(ex, "Failed fetching GHSA advisory {GhsaId}", item.GhsaId); - continue; - } - - if (detailResult.IsNotModified) - { - _diagnostics.FetchUnchanged(); - continue; - } - - if (!detailResult.IsSuccess || detailResult.Document is null) - { - _diagnostics.FetchFailure(); - continue; - } - - _diagnostics.FetchDocument(); - pendingDocuments.Add(detailResult.Document.Id); - pendingMappings.Add(detailResult.Document.Id); - - var deferDetail = await ApplyRateLimitAsync(detailResult.Document.Headers, "detail", cancellationToken).ConfigureAwait(false); - if (deferDetail) - { - rateLimitHit = true; - break; - } - } - - if (rateLimitHit) - { - break; - } - - if (pageModel.MaxUpdated.HasValue) - { - if (!maxUpdated.HasValue || pageModel.MaxUpdated > maxUpdated) - { - maxUpdated = pageModel.MaxUpdated; - } - } - - hasMore = pageModel.HasMorePages; - page = pageModel.NextPageCandidate; - pagesFetched++; - - if (!rateLimitHit && hasMore && _options.RequestDelay > TimeSpan.Zero) - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - if (hasMore || rateLimitHit) - { - updatedCursor = updatedCursor - .WithCurrentWindowStart(since) - .WithCurrentWindowEnd(until) - .WithNextPage(page); - } - else - { - var nextSince = maxUpdated ?? until; - updatedCursor = updatedCursor - .WithLastUpdatedExclusive(nextSince) - .WithCurrentWindowStart(null) - .WithCurrentWindowEnd(null) - .WithNextPage(1); - } - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToList(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - remainingDocuments.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(); - _logger.LogWarning("GHSA document {DocumentId} missing GridFS content", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - continue; - } - - byte[] rawBytes; - try - { - rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogError(ex, "Unable to download GHSA raw document {DocumentId}", documentId); - throw; - } - - GhsaRecordDto dto; - try - { - dto = GhsaRecordParser.Parse(rawBytes); - } - catch (JsonException ex) - { - _diagnostics.ParseQuarantine(); - _logger.LogError(ex, "Malformed GHSA JSON for {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - continue; - } - - var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - document.Id, - SourceName, - "ghsa/1.0", - payload, - _timeProvider.GetUtcNow()); - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - _diagnostics.ParseSuccess(); - } - - var updatedCursor = cursor.WithPendingDocuments(remainingDocuments); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - - if (dtoRecord is null || document is null) - { - _logger.LogWarning("Skipping GHSA mapping for {DocumentId}: DTO or document missing", documentId); - pendingMappings.Remove(documentId); - continue; - } - - GhsaRecordDto dto; - try - { - dto = JsonSerializer.Deserialize<GhsaRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) - ?? throw new InvalidOperationException("Deserialized DTO was null."); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to deserialize GHSA DTO for {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - var advisory = GhsaMapper.Map(dto, document, dtoRecord.ValidatedAt); - - if (advisory.CvssMetrics.IsEmpty && !string.IsNullOrWhiteSpace(advisory.CanonicalMetricId)) - { - var fallbackSeverity = string.IsNullOrWhiteSpace(advisory.Severity) - ? "unknown" - : advisory.Severity!; - _diagnostics.CanonicalMetricFallback(advisory.CanonicalMetricId!, fallbackSeverity); - if (_logger.IsEnabled(LogLevel.Debug)) - { - _logger.LogDebug( - "GHSA {GhsaId} emitted canonical metric fallback {CanonicalMetricId} (severity {Severity})", - advisory.AdvisoryKey, - advisory.CanonicalMetricId, - fallbackSeverity); - } - } - - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - _diagnostics.MapSuccess(1); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private static Uri BuildListUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) - { - var query = $"updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page={page}&per_page={pageSize}"; - return new Uri($"security/advisories?{query}", UriKind.Relative); - } - - private static Uri BuildDetailUri(string ghsaId) - { - var encoded = Uri.EscapeDataString(ghsaId); - return new Uri($"security/advisories/{encoded}", UriKind.Relative); - } - - private async Task<GhsaCursor> GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? GhsaCursor.Empty : GhsaCursor.FromBson(state.Cursor); - } - - private async Task UpdateCursorAsync(GhsaCursor cursor, CancellationToken cancellationToken) - { - await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); - } - - private bool ShouldLogRateLimitWarning(in GhsaRateLimitSnapshot snapshot, out bool recovered) - { - recovered = false; - - if (!snapshot.Remaining.HasValue) - { - return false; - } - - var key = (snapshot.Phase, snapshot.Resource ?? "global"); - var warn = snapshot.Remaining.Value <= _options.RateLimitWarningThreshold; - - lock (_rateLimitWarningLock) - { - var previouslyWarned = _rateLimitWarnings.TryGetValue(key, out var flagged) && flagged; - - if (warn) - { - if (previouslyWarned) - { - return false; - } - - _rateLimitWarnings[key] = true; - return true; - } - - if (previouslyWarned) - { - _rateLimitWarnings.Remove(key); - recovered = true; - } - - return false; - } - } - - private static double? CalculateHeadroomPercentage(in GhsaRateLimitSnapshot snapshot) - { - if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) - { - return null; - } - - var limit = snapshot.Limit.Value; - if (limit <= 0) - { - return null; - } - - return (double)snapshot.Remaining.Value / limit * 100d; - } - - private static string FormatHeadroom(double? headroomPct) - => headroomPct.HasValue ? $" (headroom {headroomPct.Value:F1}%)" : string.Empty; - - private async Task<bool> ApplyRateLimitAsync(IReadOnlyDictionary<string, string>? headers, string phase, CancellationToken cancellationToken) - { - var snapshot = GhsaRateLimitParser.TryParse(headers, _timeProvider.GetUtcNow(), phase); - if (snapshot is null || !snapshot.Value.HasData) - { - return false; - } - - _diagnostics.RecordRateLimit(snapshot.Value); - - var headroomPct = CalculateHeadroomPercentage(snapshot.Value); - if (ShouldLogRateLimitWarning(snapshot.Value, out var recovered)) - { - var resetMessage = snapshot.Value.ResetAfter.HasValue - ? $" (resets in {snapshot.Value.ResetAfter.Value:c})" - : snapshot.Value.ResetAt.HasValue ? $" (resets at {snapshot.Value.ResetAt.Value:O})" : string.Empty; - - _logger.LogWarning( - "GHSA rate limit warning: remaining {Remaining} of {Limit} for {Phase} {Resource}{ResetMessage}{Headroom}", - snapshot.Value.Remaining, - snapshot.Value.Limit, - phase, - snapshot.Value.Resource ?? "global", - resetMessage, - FormatHeadroom(headroomPct)); - } - else if (recovered) - { - _logger.LogInformation( - "GHSA rate limit recovered for {Phase} {Resource}: remaining {Remaining} of {Limit}{Headroom}", - phase, - snapshot.Value.Resource ?? "global", - snapshot.Value.Remaining, - snapshot.Value.Limit, - FormatHeadroom(headroomPct)); - } - - if (snapshot.Value.Remaining.HasValue && snapshot.Value.Remaining.Value <= 0) - { - _diagnostics.RateLimitExhausted(phase); - var delay = snapshot.Value.RetryAfter ?? snapshot.Value.ResetAfter ?? _options.SecondaryRateLimitBackoff; - - if (delay > TimeSpan.Zero) - { - _logger.LogWarning( - "GHSA rate limit exhausted for {Phase} {Resource}; delaying {Delay}{Headroom}", - phase, - snapshot.Value.Resource ?? "global", - delay, - FormatHeadroom(headroomPct)); - await Task.Delay(delay, cancellationToken).ConfigureAwait(false); - } - - return true; - } - - return false; - } -} +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ghsa; + +public sealed class GhsaConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly GhsaOptions _options; + private readonly GhsaDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger<GhsaConnector> _logger; + private readonly object _rateLimitWarningLock = new(); + private readonly Dictionary<(string Phase, string Resource), bool> _rateLimitWarnings = new(); + + public GhsaConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions<GhsaOptions> options, + GhsaDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger<GhsaConnector> logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => GhsaConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + var since = cursor.CurrentWindowStart ?? cursor.LastUpdatedExclusive ?? now - _options.InitialBackfill; + if (since > now) + { + since = now; + } + + var until = cursor.CurrentWindowEnd ?? now; + if (until <= since) + { + until = since + TimeSpan.FromMinutes(1); + } + + var page = cursor.NextPage <= 0 ? 1 : cursor.NextPage; + var pagesFetched = 0; + var hasMore = true; + var rateLimitHit = false; + DateTimeOffset? maxUpdated = cursor.LastUpdatedExclusive; + + while (hasMore && pagesFetched < _options.MaxPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var listUri = BuildListUri(since, until, page, _options.PageSize); + var metadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["since"] = since.ToString("O"), + ["until"] = until.ToString("O"), + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["pageSize"] = _options.PageSize.ToString(CultureInfo.InvariantCulture), + }; + + SourceFetchContentResult listResult; + try + { + _diagnostics.FetchAttempt(); + listResult = await _fetchService.FetchContentAsync( + new SourceFetchRequest( + GhsaOptions.HttpClientName, + SourceName, + listUri) + { + Metadata = metadata, + AcceptHeaders = new[] { "application/vnd.github+json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (listResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + break; + } + + if (!listResult.IsSuccess || listResult.Content is null) + { + _diagnostics.FetchFailure(); + break; + } + + var deferList = await ApplyRateLimitAsync(listResult.Headers, "list", cancellationToken).ConfigureAwait(false); + if (deferList) + { + rateLimitHit = true; + break; + } + + var pageModel = GhsaListParser.Parse(listResult.Content, page, _options.PageSize); + + if (pageModel.Items.Count == 0) + { + hasMore = false; + } + + foreach (var item in pageModel.Items) + { + cancellationToken.ThrowIfCancellationRequested(); + + var detailUri = BuildDetailUri(item.GhsaId); + var detailMetadata = new Dictionary<string, string>(StringComparer.Ordinal) + { + ["ghsaId"] = item.GhsaId, + ["page"] = page.ToString(CultureInfo.InvariantCulture), + ["since"] = since.ToString("O"), + ["until"] = until.ToString("O"), + }; + + SourceFetchResult detailResult; + try + { + detailResult = await _fetchService.FetchAsync( + new SourceFetchRequest( + GhsaOptions.HttpClientName, + SourceName, + detailUri) + { + Metadata = detailMetadata, + AcceptHeaders = new[] { "application/vnd.github+json" }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) + { + _diagnostics.FetchFailure(); + _logger.LogWarning(ex, "Failed fetching GHSA advisory {GhsaId}", item.GhsaId); + continue; + } + + if (detailResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + continue; + } + + if (!detailResult.IsSuccess || detailResult.Document is null) + { + _diagnostics.FetchFailure(); + continue; + } + + _diagnostics.FetchDocument(); + pendingDocuments.Add(detailResult.Document.Id); + pendingMappings.Add(detailResult.Document.Id); + + var deferDetail = await ApplyRateLimitAsync(detailResult.Document.Headers, "detail", cancellationToken).ConfigureAwait(false); + if (deferDetail) + { + rateLimitHit = true; + break; + } + } + + if (rateLimitHit) + { + break; + } + + if (pageModel.MaxUpdated.HasValue) + { + if (!maxUpdated.HasValue || pageModel.MaxUpdated > maxUpdated) + { + maxUpdated = pageModel.MaxUpdated; + } + } + + hasMore = pageModel.HasMorePages; + page = pageModel.NextPageCandidate; + pagesFetched++; + + if (!rateLimitHit && hasMore && _options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + if (hasMore || rateLimitHit) + { + updatedCursor = updatedCursor + .WithCurrentWindowStart(since) + .WithCurrentWindowEnd(until) + .WithNextPage(page); + } + else + { + var nextSince = maxUpdated ?? until; + updatedCursor = updatedCursor + .WithLastUpdatedExclusive(nextSince) + .WithCurrentWindowStart(null) + .WithCurrentWindowEnd(null) + .WithNextPage(1); + } + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("GHSA document {DocumentId} missing GridFS content", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Unable to download GHSA raw document {DocumentId}", documentId); + throw; + } + + GhsaRecordDto dto; + try + { + dto = GhsaRecordParser.Parse(rawBytes); + } + catch (JsonException ex) + { + _diagnostics.ParseQuarantine(); + _logger.LogError(ex, "Malformed GHSA JSON for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + continue; + } + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + "ghsa/1.0", + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + _diagnostics.ParseSuccess(); + } + + var updatedCursor = cursor.WithPendingDocuments(remainingDocuments); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + _logger.LogWarning("Skipping GHSA mapping for {DocumentId}: DTO or document missing", documentId); + pendingMappings.Remove(documentId); + continue; + } + + GhsaRecordDto dto; + try + { + dto = JsonSerializer.Deserialize<GhsaRecordDto>(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Deserialized DTO was null."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize GHSA DTO for {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var advisory = GhsaMapper.Map(dto, document, dtoRecord.ValidatedAt); + + if (advisory.CvssMetrics.IsEmpty && !string.IsNullOrWhiteSpace(advisory.CanonicalMetricId)) + { + var fallbackSeverity = string.IsNullOrWhiteSpace(advisory.Severity) + ? "unknown" + : advisory.Severity!; + _diagnostics.CanonicalMetricFallback(advisory.CanonicalMetricId!, fallbackSeverity); + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug( + "GHSA {GhsaId} emitted canonical metric fallback {CanonicalMetricId} (severity {Severity})", + advisory.AdvisoryKey, + advisory.CanonicalMetricId, + fallbackSeverity); + } + } + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(1); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private static Uri BuildListUri(DateTimeOffset since, DateTimeOffset until, int page, int pageSize) + { + var query = $"updated_since={Uri.EscapeDataString(since.ToString("O"))}&updated_until={Uri.EscapeDataString(until.ToString("O"))}&page={page}&per_page={pageSize}"; + return new Uri($"security/advisories?{query}", UriKind.Relative); + } + + private static Uri BuildDetailUri(string ghsaId) + { + var encoded = Uri.EscapeDataString(ghsaId); + return new Uri($"security/advisories/{encoded}", UriKind.Relative); + } + + private async Task<GhsaCursor> GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? GhsaCursor.Empty : GhsaCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(GhsaCursor cursor, CancellationToken cancellationToken) + { + await _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } + + private bool ShouldLogRateLimitWarning(in GhsaRateLimitSnapshot snapshot, out bool recovered) + { + recovered = false; + + if (!snapshot.Remaining.HasValue) + { + return false; + } + + var key = (snapshot.Phase, snapshot.Resource ?? "global"); + var warn = snapshot.Remaining.Value <= _options.RateLimitWarningThreshold; + + lock (_rateLimitWarningLock) + { + var previouslyWarned = _rateLimitWarnings.TryGetValue(key, out var flagged) && flagged; + + if (warn) + { + if (previouslyWarned) + { + return false; + } + + _rateLimitWarnings[key] = true; + return true; + } + + if (previouslyWarned) + { + _rateLimitWarnings.Remove(key); + recovered = true; + } + + return false; + } + } + + private static double? CalculateHeadroomPercentage(in GhsaRateLimitSnapshot snapshot) + { + if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) + { + return null; + } + + var limit = snapshot.Limit.Value; + if (limit <= 0) + { + return null; + } + + return (double)snapshot.Remaining.Value / limit * 100d; + } + + private static string FormatHeadroom(double? headroomPct) + => headroomPct.HasValue ? $" (headroom {headroomPct.Value:F1}%)" : string.Empty; + + private async Task<bool> ApplyRateLimitAsync(IReadOnlyDictionary<string, string>? headers, string phase, CancellationToken cancellationToken) + { + var snapshot = GhsaRateLimitParser.TryParse(headers, _timeProvider.GetUtcNow(), phase); + if (snapshot is null || !snapshot.Value.HasData) + { + return false; + } + + _diagnostics.RecordRateLimit(snapshot.Value); + + var headroomPct = CalculateHeadroomPercentage(snapshot.Value); + if (ShouldLogRateLimitWarning(snapshot.Value, out var recovered)) + { + var resetMessage = snapshot.Value.ResetAfter.HasValue + ? $" (resets in {snapshot.Value.ResetAfter.Value:c})" + : snapshot.Value.ResetAt.HasValue ? $" (resets at {snapshot.Value.ResetAt.Value:O})" : string.Empty; + + _logger.LogWarning( + "GHSA rate limit warning: remaining {Remaining} of {Limit} for {Phase} {Resource}{ResetMessage}{Headroom}", + snapshot.Value.Remaining, + snapshot.Value.Limit, + phase, + snapshot.Value.Resource ?? "global", + resetMessage, + FormatHeadroom(headroomPct)); + } + else if (recovered) + { + _logger.LogInformation( + "GHSA rate limit recovered for {Phase} {Resource}: remaining {Remaining} of {Limit}{Headroom}", + phase, + snapshot.Value.Resource ?? "global", + snapshot.Value.Remaining, + snapshot.Value.Limit, + FormatHeadroom(headroomPct)); + } + + if (snapshot.Value.Remaining.HasValue && snapshot.Value.Remaining.Value <= 0) + { + _diagnostics.RateLimitExhausted(phase); + var delay = snapshot.Value.RetryAfter ?? snapshot.Value.ResetAfter ?? _options.SecondaryRateLimitBackoff; + + if (delay > TimeSpan.Zero) + { + _logger.LogWarning( + "GHSA rate limit exhausted for {Phase} {Resource}; delaying {Delay}{Headroom}", + phase, + snapshot.Value.Resource ?? "global", + delay, + FormatHeadroom(headroomPct)); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Ghsa/GhsaConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Ghsa/GhsaConnectorPlugin.cs index 7f95dd9f..82e7c84e 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/GhsaConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/GhsaConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ghsa; - -public sealed class GhsaConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "ghsa"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance<GhsaConnector>(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ghsa; + +public sealed class GhsaConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ghsa"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance<GhsaConnector>(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs index d727460d..a8ad2c84 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/GhsaDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/GhsaDependencyInjectionRoutine.cs @@ -1,53 +1,53 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Ghsa.Configuration; - -namespace StellaOps.Feedser.Source.Ghsa; - -public sealed class GhsaDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:ghsa"; - private const string FetchCron = "1,11,21,31,41,51 * * * *"; - private const string ParseCron = "3,13,23,33,43,53 * * * *"; - private const string MapCron = "5,15,25,35,45,55 * * * *"; - - private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); - private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5); - private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(5); - private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(4); - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddGhsaConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - var scheduler = new JobSchedulerBuilder(services); - scheduler - .AddJob<GhsaFetchJob>( - GhsaJobKinds.Fetch, - cronExpression: FetchCron, - timeout: FetchTimeout, - leaseDuration: LeaseDuration) - .AddJob<GhsaParseJob>( - GhsaJobKinds.Parse, - cronExpression: ParseCron, - timeout: ParseTimeout, - leaseDuration: LeaseDuration) - .AddJob<GhsaMapJob>( - GhsaJobKinds.Map, - cronExpression: MapCron, - timeout: MapTimeout, - leaseDuration: LeaseDuration); - - return services; - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Ghsa.Configuration; + +namespace StellaOps.Concelier.Connector.Ghsa; + +public sealed class GhsaDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:ghsa"; + private const string FetchCron = "1,11,21,31,41,51 * * * *"; + private const string ParseCron = "3,13,23,33,43,53 * * * *"; + private const string MapCron = "5,15,25,35,45,55 * * * *"; + + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(6); + private static readonly TimeSpan ParseTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan MapTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(4); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddGhsaConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + var scheduler = new JobSchedulerBuilder(services); + scheduler + .AddJob<GhsaFetchJob>( + GhsaJobKinds.Fetch, + cronExpression: FetchCron, + timeout: FetchTimeout, + leaseDuration: LeaseDuration) + .AddJob<GhsaParseJob>( + GhsaJobKinds.Parse, + cronExpression: ParseCron, + timeout: ParseTimeout, + leaseDuration: LeaseDuration) + .AddJob<GhsaMapJob>( + GhsaJobKinds.Map, + cronExpression: MapCron, + timeout: MapTimeout, + leaseDuration: LeaseDuration); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs similarity index 81% rename from src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs index 7777b355..8c0e3bc8 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/GhsaServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/GhsaServiceCollectionExtensions.cs @@ -1,37 +1,37 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ghsa.Configuration; -using StellaOps.Feedser.Source.Ghsa.Internal; - -namespace StellaOps.Feedser.Source.Ghsa; - -public static class GhsaServiceCollectionExtensions -{ - public static IServiceCollection AddGhsaConnector(this IServiceCollection services, Action<GhsaOptions> configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions<GhsaOptions>() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()); - - services.AddSourceHttpClient(GhsaOptions.HttpClientName, (sp, clientOptions) => - { - var options = sp.GetRequiredService<IOptions<GhsaOptions>>().Value; - clientOptions.BaseAddress = options.BaseEndpoint; - clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Ghsa/1.0"; - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); - clientOptions.DefaultRequestHeaders["Accept"] = "application/vnd.github+json"; - clientOptions.DefaultRequestHeaders["Authorization"] = $"Bearer {options.ApiToken}"; - clientOptions.DefaultRequestHeaders["X-GitHub-Api-Version"] = "2022-11-28"; - }); - - services.AddSingleton<GhsaDiagnostics>(); - services.AddTransient<GhsaConnector>(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ghsa.Configuration; +using StellaOps.Concelier.Connector.Ghsa.Internal; + +namespace StellaOps.Concelier.Connector.Ghsa; + +public static class GhsaServiceCollectionExtensions +{ + public static IServiceCollection AddGhsaConnector(this IServiceCollection services, Action<GhsaOptions> configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions<GhsaOptions>() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(GhsaOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService<IOptions<GhsaOptions>>().Value; + clientOptions.BaseAddress = options.BaseEndpoint; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Concelier.Ghsa/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/vnd.github+json"; + clientOptions.DefaultRequestHeaders["Authorization"] = $"Bearer {options.ApiToken}"; + clientOptions.DefaultRequestHeaders["X-GitHub-Api-Version"] = "2022-11-28"; + }); + + services.AddSingleton<GhsaDiagnostics>(); + services.AddTransient<GhsaConnector>(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaCursor.cs index 09f3069a..0fc6cb49 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaCursor.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaCursor.cs @@ -1,135 +1,135 @@ -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal sealed record GhsaCursor( - DateTimeOffset? LastUpdatedExclusive, - DateTimeOffset? CurrentWindowStart, - DateTimeOffset? CurrentWindowEnd, - int NextPage, - IReadOnlyCollection<Guid> PendingDocuments, - IReadOnlyCollection<Guid> PendingMappings) -{ - private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); - - public static GhsaCursor Empty { get; } = new( - null, - null, - null, - 1, - EmptyGuidList, - EmptyGuidList); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["nextPage"] = NextPage, - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastUpdatedExclusive.HasValue) - { - document["lastUpdatedExclusive"] = LastUpdatedExclusive.Value.UtcDateTime; - } - - if (CurrentWindowStart.HasValue) - { - document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; - } - - if (CurrentWindowEnd.HasValue) - { - document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; - } - - return document; - } - - public static GhsaCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var lastUpdatedExclusive = document.TryGetValue("lastUpdatedExclusive", out var lastUpdated) - ? ParseDate(lastUpdated) - : null; - var windowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) - ? ParseDate(windowStartValue) - : null; - var windowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) - ? ParseDate(windowEndValue) - : null; - var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 - ? Math.Max(1, nextPageValue.AsInt32) - : 1; - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new GhsaCursor( - lastUpdatedExclusive, - windowStart, - windowEnd, - nextPage, - pendingDocuments, - pendingMappings); - } - - public GhsaCursor WithPendingDocuments(IEnumerable<Guid> ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; - - public GhsaCursor WithPendingMappings(IEnumerable<Guid> ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; - - public GhsaCursor WithLastUpdatedExclusive(DateTimeOffset? timestamp) - => this with { LastUpdatedExclusive = timestamp }; - - public GhsaCursor WithCurrentWindowStart(DateTimeOffset? timestamp) - => this with { CurrentWindowStart = timestamp }; - - public GhsaCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) - => this with { CurrentWindowEnd = timestamp }; - - public GhsaCursor WithNextPage(int page) - => this with { NextPage = page < 1 ? 1 : page }; - - private static DateTimeOffset? ParseDate(BsonValue value) - { - return value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuidList; - } - - var results = new List<Guid>(array.Count); - foreach (var element in array) - { - if (element is null) - { - continue; - } - - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } -} +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal sealed record GhsaCursor( + DateTimeOffset? LastUpdatedExclusive, + DateTimeOffset? CurrentWindowStart, + DateTimeOffset? CurrentWindowEnd, + int NextPage, + IReadOnlyCollection<Guid> PendingDocuments, + IReadOnlyCollection<Guid> PendingMappings) +{ + private static readonly IReadOnlyCollection<Guid> EmptyGuidList = Array.Empty<Guid>(); + + public static GhsaCursor Empty { get; } = new( + null, + null, + null, + 1, + EmptyGuidList, + EmptyGuidList); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["nextPage"] = NextPage, + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastUpdatedExclusive.HasValue) + { + document["lastUpdatedExclusive"] = LastUpdatedExclusive.Value.UtcDateTime; + } + + if (CurrentWindowStart.HasValue) + { + document["currentWindowStart"] = CurrentWindowStart.Value.UtcDateTime; + } + + if (CurrentWindowEnd.HasValue) + { + document["currentWindowEnd"] = CurrentWindowEnd.Value.UtcDateTime; + } + + return document; + } + + public static GhsaCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastUpdatedExclusive = document.TryGetValue("lastUpdatedExclusive", out var lastUpdated) + ? ParseDate(lastUpdated) + : null; + var windowStart = document.TryGetValue("currentWindowStart", out var windowStartValue) + ? ParseDate(windowStartValue) + : null; + var windowEnd = document.TryGetValue("currentWindowEnd", out var windowEndValue) + ? ParseDate(windowEndValue) + : null; + var nextPage = document.TryGetValue("nextPage", out var nextPageValue) && nextPageValue.IsInt32 + ? Math.Max(1, nextPageValue.AsInt32) + : 1; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new GhsaCursor( + lastUpdatedExclusive, + windowStart, + windowEnd, + nextPage, + pendingDocuments, + pendingMappings); + } + + public GhsaCursor WithPendingDocuments(IEnumerable<Guid> ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public GhsaCursor WithPendingMappings(IEnumerable<Guid> ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidList }; + + public GhsaCursor WithLastUpdatedExclusive(DateTimeOffset? timestamp) + => this with { LastUpdatedExclusive = timestamp }; + + public GhsaCursor WithCurrentWindowStart(DateTimeOffset? timestamp) + => this with { CurrentWindowStart = timestamp }; + + public GhsaCursor WithCurrentWindowEnd(DateTimeOffset? timestamp) + => this with { CurrentWindowEnd = timestamp }; + + public GhsaCursor WithNextPage(int page) + => this with { NextPage = page < 1 ? 1 : page }; + + private static DateTimeOffset? ParseDate(BsonValue value) + { + return value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuidList; + } + + var results = new List<Guid>(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaDiagnostics.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaDiagnostics.cs index 7ded3f38..4ed93004 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaDiagnostics.cs @@ -1,164 +1,164 @@ -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -public sealed class GhsaDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.Ghsa"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter<long> _fetchAttempts; - private readonly Counter<long> _fetchDocuments; - private readonly Counter<long> _fetchFailures; - private readonly Counter<long> _fetchUnchanged; - private readonly Counter<long> _parseSuccess; - private readonly Counter<long> _parseFailures; - private readonly Counter<long> _parseQuarantine; - private readonly Counter<long> _mapSuccess; - private readonly Histogram<long> _rateLimitRemaining; - private readonly Histogram<long> _rateLimitLimit; - private readonly Histogram<double> _rateLimitResetSeconds; - private readonly Histogram<double> _rateLimitHeadroomPct; - private readonly ObservableGauge<double> _rateLimitHeadroomGauge; - private readonly Counter<long> _rateLimitExhausted; - private readonly Counter<long> _canonicalMetricFallbacks; - private readonly object _rateLimitLock = new(); - private GhsaRateLimitSnapshot? _lastRateLimitSnapshot; - private readonly Dictionary<(string Phase, string? Resource), GhsaRateLimitSnapshot> _rateLimitSnapshots = new(); - - public GhsaDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchAttempts = _meter.CreateCounter<long>("ghsa.fetch.attempts", unit: "operations"); - _fetchDocuments = _meter.CreateCounter<long>("ghsa.fetch.documents", unit: "documents"); - _fetchFailures = _meter.CreateCounter<long>("ghsa.fetch.failures", unit: "operations"); - _fetchUnchanged = _meter.CreateCounter<long>("ghsa.fetch.unchanged", unit: "operations"); - _parseSuccess = _meter.CreateCounter<long>("ghsa.parse.success", unit: "documents"); - _parseFailures = _meter.CreateCounter<long>("ghsa.parse.failures", unit: "documents"); - _parseQuarantine = _meter.CreateCounter<long>("ghsa.parse.quarantine", unit: "documents"); - _mapSuccess = _meter.CreateCounter<long>("ghsa.map.success", unit: "advisories"); - _rateLimitRemaining = _meter.CreateHistogram<long>("ghsa.ratelimit.remaining", unit: "requests"); - _rateLimitLimit = _meter.CreateHistogram<long>("ghsa.ratelimit.limit", unit: "requests"); - _rateLimitResetSeconds = _meter.CreateHistogram<double>("ghsa.ratelimit.reset_seconds", unit: "s"); - _rateLimitHeadroomPct = _meter.CreateHistogram<double>("ghsa.ratelimit.headroom_pct", unit: "percent"); - _rateLimitHeadroomGauge = _meter.CreateObservableGauge("ghsa.ratelimit.headroom_pct_current", ObserveHeadroom, unit: "percent"); - _rateLimitExhausted = _meter.CreateCounter<long>("ghsa.ratelimit.exhausted", unit: "events"); - _canonicalMetricFallbacks = _meter.CreateCounter<long>("ghsa.map.canonical_metric_fallbacks", unit: "advisories"); - } - - public void FetchAttempt() => _fetchAttempts.Add(1); - - public void FetchDocument() => _fetchDocuments.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void ParseSuccess() => _parseSuccess.Add(1); - - public void ParseFailure() => _parseFailures.Add(1); - - public void ParseQuarantine() => _parseQuarantine.Add(1); - - public void MapSuccess(long count) => _mapSuccess.Add(count); - - internal void RecordRateLimit(GhsaRateLimitSnapshot snapshot) - { - var tags = new KeyValuePair<string, object?>[] - { - new("phase", snapshot.Phase), - new("resource", snapshot.Resource ?? "unknown") - }; - - if (snapshot.Limit.HasValue) - { - _rateLimitLimit.Record(snapshot.Limit.Value, tags); - } - - if (snapshot.Remaining.HasValue) - { - _rateLimitRemaining.Record(snapshot.Remaining.Value, tags); - } - - if (snapshot.ResetAfter.HasValue) - { - _rateLimitResetSeconds.Record(snapshot.ResetAfter.Value.TotalSeconds, tags); - } - - if (TryCalculateHeadroom(snapshot, out var headroom)) - { - _rateLimitHeadroomPct.Record(headroom, tags); - } - - lock (_rateLimitLock) - { - _lastRateLimitSnapshot = snapshot; - _rateLimitSnapshots[(snapshot.Phase, snapshot.Resource)] = snapshot; - } - } - - internal void RateLimitExhausted(string phase) - => _rateLimitExhausted.Add(1, new KeyValuePair<string, object?>("phase", phase)); - - public void CanonicalMetricFallback(string canonicalMetricId, string severity) - => _canonicalMetricFallbacks.Add( - 1, - new KeyValuePair<string, object?>("canonical_metric_id", canonicalMetricId), - new KeyValuePair<string, object?>("severity", severity), - new KeyValuePair<string, object?>("reason", "no_cvss")); - - internal GhsaRateLimitSnapshot? GetLastRateLimitSnapshot() - { - lock (_rateLimitLock) - { - return _lastRateLimitSnapshot; - } - } - - private IEnumerable<Measurement<double>> ObserveHeadroom() - { - lock (_rateLimitLock) - { - if (_rateLimitSnapshots.Count == 0) - { - yield break; - } - - foreach (var snapshot in _rateLimitSnapshots.Values) - { - if (TryCalculateHeadroom(snapshot, out var headroom)) - { - yield return new Measurement<double>( - headroom, - new KeyValuePair<string, object?>("phase", snapshot.Phase), - new KeyValuePair<string, object?>("resource", snapshot.Resource ?? "unknown")); - } - } - } - } - - private static bool TryCalculateHeadroom(in GhsaRateLimitSnapshot snapshot, out double headroomPct) - { - headroomPct = 0; - if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) - { - return false; - } - - var limit = snapshot.Limit.Value; - if (limit <= 0) - { - return false; - } - - headroomPct = (double)snapshot.Remaining.Value / limit * 100d; - return true; - } - - public void Dispose() - { - _meter.Dispose(); - } -} +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +public sealed class GhsaDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.Ghsa"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter<long> _fetchAttempts; + private readonly Counter<long> _fetchDocuments; + private readonly Counter<long> _fetchFailures; + private readonly Counter<long> _fetchUnchanged; + private readonly Counter<long> _parseSuccess; + private readonly Counter<long> _parseFailures; + private readonly Counter<long> _parseQuarantine; + private readonly Counter<long> _mapSuccess; + private readonly Histogram<long> _rateLimitRemaining; + private readonly Histogram<long> _rateLimitLimit; + private readonly Histogram<double> _rateLimitResetSeconds; + private readonly Histogram<double> _rateLimitHeadroomPct; + private readonly ObservableGauge<double> _rateLimitHeadroomGauge; + private readonly Counter<long> _rateLimitExhausted; + private readonly Counter<long> _canonicalMetricFallbacks; + private readonly object _rateLimitLock = new(); + private GhsaRateLimitSnapshot? _lastRateLimitSnapshot; + private readonly Dictionary<(string Phase, string? Resource), GhsaRateLimitSnapshot> _rateLimitSnapshots = new(); + + public GhsaDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter<long>("ghsa.fetch.attempts", unit: "operations"); + _fetchDocuments = _meter.CreateCounter<long>("ghsa.fetch.documents", unit: "documents"); + _fetchFailures = _meter.CreateCounter<long>("ghsa.fetch.failures", unit: "operations"); + _fetchUnchanged = _meter.CreateCounter<long>("ghsa.fetch.unchanged", unit: "operations"); + _parseSuccess = _meter.CreateCounter<long>("ghsa.parse.success", unit: "documents"); + _parseFailures = _meter.CreateCounter<long>("ghsa.parse.failures", unit: "documents"); + _parseQuarantine = _meter.CreateCounter<long>("ghsa.parse.quarantine", unit: "documents"); + _mapSuccess = _meter.CreateCounter<long>("ghsa.map.success", unit: "advisories"); + _rateLimitRemaining = _meter.CreateHistogram<long>("ghsa.ratelimit.remaining", unit: "requests"); + _rateLimitLimit = _meter.CreateHistogram<long>("ghsa.ratelimit.limit", unit: "requests"); + _rateLimitResetSeconds = _meter.CreateHistogram<double>("ghsa.ratelimit.reset_seconds", unit: "s"); + _rateLimitHeadroomPct = _meter.CreateHistogram<double>("ghsa.ratelimit.headroom_pct", unit: "percent"); + _rateLimitHeadroomGauge = _meter.CreateObservableGauge("ghsa.ratelimit.headroom_pct_current", ObserveHeadroom, unit: "percent"); + _rateLimitExhausted = _meter.CreateCounter<long>("ghsa.ratelimit.exhausted", unit: "events"); + _canonicalMetricFallbacks = _meter.CreateCounter<long>("ghsa.map.canonical_metric_fallbacks", unit: "advisories"); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchDocument() => _fetchDocuments.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseSuccess() => _parseSuccess.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void ParseQuarantine() => _parseQuarantine.Add(1); + + public void MapSuccess(long count) => _mapSuccess.Add(count); + + internal void RecordRateLimit(GhsaRateLimitSnapshot snapshot) + { + var tags = new KeyValuePair<string, object?>[] + { + new("phase", snapshot.Phase), + new("resource", snapshot.Resource ?? "unknown") + }; + + if (snapshot.Limit.HasValue) + { + _rateLimitLimit.Record(snapshot.Limit.Value, tags); + } + + if (snapshot.Remaining.HasValue) + { + _rateLimitRemaining.Record(snapshot.Remaining.Value, tags); + } + + if (snapshot.ResetAfter.HasValue) + { + _rateLimitResetSeconds.Record(snapshot.ResetAfter.Value.TotalSeconds, tags); + } + + if (TryCalculateHeadroom(snapshot, out var headroom)) + { + _rateLimitHeadroomPct.Record(headroom, tags); + } + + lock (_rateLimitLock) + { + _lastRateLimitSnapshot = snapshot; + _rateLimitSnapshots[(snapshot.Phase, snapshot.Resource)] = snapshot; + } + } + + internal void RateLimitExhausted(string phase) + => _rateLimitExhausted.Add(1, new KeyValuePair<string, object?>("phase", phase)); + + public void CanonicalMetricFallback(string canonicalMetricId, string severity) + => _canonicalMetricFallbacks.Add( + 1, + new KeyValuePair<string, object?>("canonical_metric_id", canonicalMetricId), + new KeyValuePair<string, object?>("severity", severity), + new KeyValuePair<string, object?>("reason", "no_cvss")); + + internal GhsaRateLimitSnapshot? GetLastRateLimitSnapshot() + { + lock (_rateLimitLock) + { + return _lastRateLimitSnapshot; + } + } + + private IEnumerable<Measurement<double>> ObserveHeadroom() + { + lock (_rateLimitLock) + { + if (_rateLimitSnapshots.Count == 0) + { + yield break; + } + + foreach (var snapshot in _rateLimitSnapshots.Values) + { + if (TryCalculateHeadroom(snapshot, out var headroom)) + { + yield return new Measurement<double>( + headroom, + new KeyValuePair<string, object?>("phase", snapshot.Phase), + new KeyValuePair<string, object?>("resource", snapshot.Resource ?? "unknown")); + } + } + } + } + + private static bool TryCalculateHeadroom(in GhsaRateLimitSnapshot snapshot, out double headroomPct) + { + headroomPct = 0; + if (!snapshot.Limit.HasValue || !snapshot.Remaining.HasValue) + { + return false; + } + + var limit = snapshot.Limit.Value; + if (limit <= 0) + { + return false; + } + + headroomPct = (double)snapshot.Remaining.Value / limit * 100d; + return true; + } + + public void Dispose() + { + _meter.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaListParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaListParser.cs index 034e9b5e..ee7b5676 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaListParser.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaListParser.cs @@ -1,115 +1,115 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Text.Json; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal static class GhsaListParser -{ - public static GhsaListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) - { - using var document = JsonDocument.Parse(content.ToArray()); - var root = document.RootElement; - - var items = new List<GhsaListItem>(); - DateTimeOffset? maxUpdated = null; - - if (root.TryGetProperty("advisories", out var advisories) && advisories.ValueKind == JsonValueKind.Array) - { - foreach (var advisory in advisories.EnumerateArray()) - { - if (advisory.ValueKind != JsonValueKind.Object) - { - continue; - } - - var id = GetString(advisory, "ghsa_id"); - if (string.IsNullOrWhiteSpace(id)) - { - continue; - } - - var updated = GetDate(advisory, "updated_at"); - if (updated.HasValue && (!maxUpdated.HasValue || updated > maxUpdated)) - { - maxUpdated = updated; - } - - items.Add(new GhsaListItem(id, updated)); - } - } - - var hasMorePages = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); - - return new GhsaListPage(items, maxUpdated, hasMorePages, nextPage ?? currentPage + 1); - } - - private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) - { - nextPage = null; - - if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) - { - var hasNextPage = pagination.TryGetProperty("has_next_page", out var hasNext) && hasNext.ValueKind == JsonValueKind.True; - if (hasNextPage) - { - nextPage = currentPage + 1; - return true; - } - - if (pagination.TryGetProperty("total_pages", out var totalPagesElement) && totalPagesElement.ValueKind == JsonValueKind.Number && totalPagesElement.TryGetInt32(out var totalPages)) - { - if (currentPage < totalPages) - { - nextPage = currentPage + 1; - return true; - } - } - - return false; - } - - if (itemCount >= pageSize) - { - nextPage = currentPage + 1; - return true; - } - - return false; - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind switch - { - JsonValueKind.String => property.GetString(), - _ => null, - }; - } - - private static DateTimeOffset? GetDate(JsonElement element, string propertyName) - { - var value = GetString(element, propertyName); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) - ? parsed.ToUniversalTime() - : null; - } -} - -internal sealed record GhsaListPage( - IReadOnlyList<GhsaListItem> Items, - DateTimeOffset? MaxUpdated, - bool HasMorePages, - int NextPageCandidate); - -internal sealed record GhsaListItem(string GhsaId, DateTimeOffset? UpdatedAt); +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal static class GhsaListParser +{ + public static GhsaListPage Parse(ReadOnlySpan<byte> content, int currentPage, int pageSize) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var items = new List<GhsaListItem>(); + DateTimeOffset? maxUpdated = null; + + if (root.TryGetProperty("advisories", out var advisories) && advisories.ValueKind == JsonValueKind.Array) + { + foreach (var advisory in advisories.EnumerateArray()) + { + if (advisory.ValueKind != JsonValueKind.Object) + { + continue; + } + + var id = GetString(advisory, "ghsa_id"); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + var updated = GetDate(advisory, "updated_at"); + if (updated.HasValue && (!maxUpdated.HasValue || updated > maxUpdated)) + { + maxUpdated = updated; + } + + items.Add(new GhsaListItem(id, updated)); + } + } + + var hasMorePages = TryDetermineHasMore(root, currentPage, pageSize, items.Count, out var nextPage); + + return new GhsaListPage(items, maxUpdated, hasMorePages, nextPage ?? currentPage + 1); + } + + private static bool TryDetermineHasMore(JsonElement root, int currentPage, int pageSize, int itemCount, out int? nextPage) + { + nextPage = null; + + if (root.TryGetProperty("pagination", out var pagination) && pagination.ValueKind == JsonValueKind.Object) + { + var hasNextPage = pagination.TryGetProperty("has_next_page", out var hasNext) && hasNext.ValueKind == JsonValueKind.True; + if (hasNextPage) + { + nextPage = currentPage + 1; + return true; + } + + if (pagination.TryGetProperty("total_pages", out var totalPagesElement) && totalPagesElement.ValueKind == JsonValueKind.Number && totalPagesElement.TryGetInt32(out var totalPages)) + { + if (currentPage < totalPages) + { + nextPage = currentPage + 1; + return true; + } + } + + return false; + } + + if (itemCount >= pageSize) + { + nextPage = currentPage + 1; + return true; + } + + return false; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} + +internal sealed record GhsaListPage( + IReadOnlyList<GhsaListItem> Items, + DateTimeOffset? MaxUpdated, + bool HasMorePages, + int NextPageCandidate); + +internal sealed record GhsaListItem(string GhsaId, DateTimeOffset? UpdatedAt); diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaMapper.cs similarity index 98% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaMapper.cs index 330b048a..c5e04bdc 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaMapper.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaMapper.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; using System.Text; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.SemVer; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.SemVer; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Ghsa.Internal; +namespace StellaOps.Concelier.Connector.Ghsa.Internal; internal static class GhsaMapper { diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitParser.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitParser.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitParser.cs index 18c825e9..7b3932a9 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitParser.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitParser.cs @@ -1,111 +1,111 @@ -using System; -using System.Collections.Generic; -using System.Globalization; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal static class GhsaRateLimitParser -{ - public static GhsaRateLimitSnapshot? TryParse(IReadOnlyDictionary<string, string>? headers, DateTimeOffset now, string phase) - { - if (headers is null || headers.Count == 0) - { - return null; - } - - string? resource = null; - long? limit = null; - long? remaining = null; - long? used = null; - DateTimeOffset? resetAt = null; - TimeSpan? resetAfter = null; - TimeSpan? retryAfter = null; - var hasData = false; - - if (TryGet(headers, "X-RateLimit-Resource", out var resourceValue) && !string.IsNullOrWhiteSpace(resourceValue)) - { - resource = resourceValue; - hasData = true; - } - - if (TryParseLong(headers, "X-RateLimit-Limit", out var limitValue)) - { - limit = limitValue; - hasData = true; - } - - if (TryParseLong(headers, "X-RateLimit-Remaining", out var remainingValue)) - { - remaining = remainingValue; - hasData = true; - } - - if (TryParseLong(headers, "X-RateLimit-Used", out var usedValue)) - { - used = usedValue; - hasData = true; - } - - if (TryParseLong(headers, "X-RateLimit-Reset", out var resetValue)) - { - resetAt = DateTimeOffset.FromUnixTimeSeconds(resetValue); - var delta = resetAt.Value - now; - if (delta > TimeSpan.Zero) - { - resetAfter = delta; - } - hasData = true; - } - - if (TryGet(headers, "Retry-After", out var retryAfterValue) && !string.IsNullOrWhiteSpace(retryAfterValue)) - { - if (double.TryParse(retryAfterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds > 0) - { - retryAfter = TimeSpan.FromSeconds(seconds); - } - else if (DateTimeOffset.TryParse(retryAfterValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var retryAfterDate)) - { - var delta = retryAfterDate - now; - if (delta > TimeSpan.Zero) - { - retryAfter = delta; - } - } - hasData = true; - } - - if (!hasData) - { - return null; - } - - return new GhsaRateLimitSnapshot(phase, resource, limit, remaining, used, resetAt, resetAfter, retryAfter); - } - - private static bool TryGet(IReadOnlyDictionary<string, string> headers, string key, out string value) - { - foreach (var pair in headers) - { - if (pair.Key.Equals(key, StringComparison.OrdinalIgnoreCase)) - { - value = pair.Value; - return true; - } - } - - value = string.Empty; - return false; - } - - private static bool TryParseLong(IReadOnlyDictionary<string, string> headers, string key, out long result) - { - result = 0; - if (TryGet(headers, key, out var value) && long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) - { - result = parsed; - return true; - } - - return false; - } -} +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal static class GhsaRateLimitParser +{ + public static GhsaRateLimitSnapshot? TryParse(IReadOnlyDictionary<string, string>? headers, DateTimeOffset now, string phase) + { + if (headers is null || headers.Count == 0) + { + return null; + } + + string? resource = null; + long? limit = null; + long? remaining = null; + long? used = null; + DateTimeOffset? resetAt = null; + TimeSpan? resetAfter = null; + TimeSpan? retryAfter = null; + var hasData = false; + + if (TryGet(headers, "X-RateLimit-Resource", out var resourceValue) && !string.IsNullOrWhiteSpace(resourceValue)) + { + resource = resourceValue; + hasData = true; + } + + if (TryParseLong(headers, "X-RateLimit-Limit", out var limitValue)) + { + limit = limitValue; + hasData = true; + } + + if (TryParseLong(headers, "X-RateLimit-Remaining", out var remainingValue)) + { + remaining = remainingValue; + hasData = true; + } + + if (TryParseLong(headers, "X-RateLimit-Used", out var usedValue)) + { + used = usedValue; + hasData = true; + } + + if (TryParseLong(headers, "X-RateLimit-Reset", out var resetValue)) + { + resetAt = DateTimeOffset.FromUnixTimeSeconds(resetValue); + var delta = resetAt.Value - now; + if (delta > TimeSpan.Zero) + { + resetAfter = delta; + } + hasData = true; + } + + if (TryGet(headers, "Retry-After", out var retryAfterValue) && !string.IsNullOrWhiteSpace(retryAfterValue)) + { + if (double.TryParse(retryAfterValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var seconds) && seconds > 0) + { + retryAfter = TimeSpan.FromSeconds(seconds); + } + else if (DateTimeOffset.TryParse(retryAfterValue, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var retryAfterDate)) + { + var delta = retryAfterDate - now; + if (delta > TimeSpan.Zero) + { + retryAfter = delta; + } + } + hasData = true; + } + + if (!hasData) + { + return null; + } + + return new GhsaRateLimitSnapshot(phase, resource, limit, remaining, used, resetAt, resetAfter, retryAfter); + } + + private static bool TryGet(IReadOnlyDictionary<string, string> headers, string key, out string value) + { + foreach (var pair in headers) + { + if (pair.Key.Equals(key, StringComparison.OrdinalIgnoreCase)) + { + value = pair.Value; + return true; + } + } + + value = string.Empty; + return false; + } + + private static bool TryParseLong(IReadOnlyDictionary<string, string> headers, string key, out long result) + { + result = 0; + if (TryGet(headers, key, out var value) && long.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + result = parsed; + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitSnapshot.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitSnapshot.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitSnapshot.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitSnapshot.cs index 42db39bc..16073b2a 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRateLimitSnapshot.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRateLimitSnapshot.cs @@ -1,23 +1,23 @@ -using System; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal readonly record struct GhsaRateLimitSnapshot( - string Phase, - string? Resource, - long? Limit, - long? Remaining, - long? Used, - DateTimeOffset? ResetAt, - TimeSpan? ResetAfter, - TimeSpan? RetryAfter) -{ - public bool HasData => - Limit.HasValue || - Remaining.HasValue || - Used.HasValue || - ResetAt.HasValue || - ResetAfter.HasValue || - RetryAfter.HasValue || - !string.IsNullOrEmpty(Resource); -} +using System; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal readonly record struct GhsaRateLimitSnapshot( + string Phase, + string? Resource, + long? Limit, + long? Remaining, + long? Used, + DateTimeOffset? ResetAt, + TimeSpan? ResetAfter, + TimeSpan? RetryAfter) +{ + public bool HasData => + Limit.HasValue || + Remaining.HasValue || + Used.HasValue || + ResetAt.HasValue || + ResetAfter.HasValue || + RetryAfter.HasValue || + !string.IsNullOrEmpty(Resource); +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordDto.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordDto.cs index b3b3e3c9..918dcc6e 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordDto.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordDto.cs @@ -1,75 +1,75 @@ -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal sealed record GhsaRecordDto -{ - public string GhsaId { get; init; } = string.Empty; - - public string? Summary { get; init; } - - public string? Description { get; init; } - - public string? Severity { get; init; } - - public DateTimeOffset? PublishedAt { get; init; } - - public DateTimeOffset? UpdatedAt { get; init; } - - public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); - - public IReadOnlyList<GhsaReferenceDto> References { get; init; } = Array.Empty<GhsaReferenceDto>(); - - public IReadOnlyList<GhsaAffectedDto> Affected { get; init; } = Array.Empty<GhsaAffectedDto>(); - - public IReadOnlyList<GhsaCreditDto> Credits { get; init; } = Array.Empty<GhsaCreditDto>(); - - public IReadOnlyList<GhsaWeaknessDto> Cwes { get; init; } = Array.Empty<GhsaWeaknessDto>(); - - public GhsaCvssDto? Cvss { get; init; } -} - -internal sealed record GhsaReferenceDto -{ - public string Url { get; init; } = string.Empty; - - public string? Type { get; init; } - - public string? Name { get; init; } -} - -internal sealed record GhsaAffectedDto -{ - public string PackageName { get; init; } = string.Empty; - - public string Ecosystem { get; init; } = string.Empty; - - public string? VulnerableRange { get; init; } - - public string? PatchedVersion { get; init; } -} - -internal sealed record GhsaCreditDto -{ - public string? Type { get; init; } - - public string? Name { get; init; } - - public string? Login { get; init; } - - public string? ProfileUrl { get; init; } -} - -internal sealed record GhsaWeaknessDto -{ - public string? CweId { get; init; } - - public string? Name { get; init; } -} - -internal sealed record GhsaCvssDto -{ - public double? Score { get; init; } - - public string? VectorString { get; init; } - - public string? Severity { get; init; } -} +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal sealed record GhsaRecordDto +{ + public string GhsaId { get; init; } = string.Empty; + + public string? Summary { get; init; } + + public string? Description { get; init; } + + public string? Severity { get; init; } + + public DateTimeOffset? PublishedAt { get; init; } + + public DateTimeOffset? UpdatedAt { get; init; } + + public IReadOnlyList<string> Aliases { get; init; } = Array.Empty<string>(); + + public IReadOnlyList<GhsaReferenceDto> References { get; init; } = Array.Empty<GhsaReferenceDto>(); + + public IReadOnlyList<GhsaAffectedDto> Affected { get; init; } = Array.Empty<GhsaAffectedDto>(); + + public IReadOnlyList<GhsaCreditDto> Credits { get; init; } = Array.Empty<GhsaCreditDto>(); + + public IReadOnlyList<GhsaWeaknessDto> Cwes { get; init; } = Array.Empty<GhsaWeaknessDto>(); + + public GhsaCvssDto? Cvss { get; init; } +} + +internal sealed record GhsaReferenceDto +{ + public string Url { get; init; } = string.Empty; + + public string? Type { get; init; } + + public string? Name { get; init; } +} + +internal sealed record GhsaAffectedDto +{ + public string PackageName { get; init; } = string.Empty; + + public string Ecosystem { get; init; } = string.Empty; + + public string? VulnerableRange { get; init; } + + public string? PatchedVersion { get; init; } +} + +internal sealed record GhsaCreditDto +{ + public string? Type { get; init; } + + public string? Name { get; init; } + + public string? Login { get; init; } + + public string? ProfileUrl { get; init; } +} + +internal sealed record GhsaWeaknessDto +{ + public string? CweId { get; init; } + + public string? Name { get; init; } +} + +internal sealed record GhsaCvssDto +{ + public double? Score { get; init; } + + public string? VectorString { get; init; } + + public string? Severity { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordParser.cs index 99937985..3e832610 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Internal/GhsaRecordParser.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Internal/GhsaRecordParser.cs @@ -1,269 +1,269 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Text.Json; - -namespace StellaOps.Feedser.Source.Ghsa.Internal; - -internal static class GhsaRecordParser -{ - public static GhsaRecordDto Parse(ReadOnlySpan<byte> content) - { - using var document = JsonDocument.Parse(content.ToArray()); - var root = document.RootElement; - - var ghsaId = GetString(root, "ghsa_id") ?? throw new JsonException("ghsa_id missing"); - var summary = GetString(root, "summary"); - var description = GetString(root, "description"); - var severity = GetString(root, "severity"); - var publishedAt = GetDate(root, "published_at"); - var updatedAt = GetDate(root, "updated_at") ?? publishedAt; - - var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) - { - ghsaId, - }; - - if (root.TryGetProperty("cve_ids", out var cveIds) && cveIds.ValueKind == JsonValueKind.Array) - { - foreach (var cve in cveIds.EnumerateArray()) - { - if (cve.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cve.GetString())) - { - aliases.Add(cve.GetString()!); - } - } - } - - var references = ParseReferences(root); - var affected = ParseAffected(root); - var credits = ParseCredits(root); - var cwes = ParseCwes(root); - var cvss = ParseCvss(root); - - return new GhsaRecordDto - { - GhsaId = ghsaId, - Summary = summary, - Description = description, - Severity = severity, - PublishedAt = publishedAt, - UpdatedAt = updatedAt, - Aliases = aliases.ToArray(), - References = references, - Affected = affected, - Credits = credits, - Cwes = cwes, - Cvss = cvss, - }; - } - - private static IReadOnlyList<GhsaReferenceDto> ParseReferences(JsonElement root) - { - if (!root.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) - { - return Array.Empty<GhsaReferenceDto>(); - } - - var list = new List<GhsaReferenceDto>(references.GetArrayLength()); - foreach (var reference in references.EnumerateArray()) - { - if (reference.ValueKind != JsonValueKind.Object) - { - continue; - } - - var url = GetString(reference, "url"); - if (string.IsNullOrWhiteSpace(url)) - { - continue; - } - - list.Add(new GhsaReferenceDto - { - Url = url, - Type = GetString(reference, "type"), - Name = GetString(reference, "name"), - }); - } - - return list; - } - - private static IReadOnlyList<GhsaAffectedDto> ParseAffected(JsonElement root) - { - if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) - { - return Array.Empty<GhsaAffectedDto>(); - } - - var list = new List<GhsaAffectedDto>(vulnerabilities.GetArrayLength()); - foreach (var entry in vulnerabilities.EnumerateArray()) - { - if (entry.ValueKind != JsonValueKind.Object) - { - continue; - } - - var package = entry.TryGetProperty("package", out var packageElement) && packageElement.ValueKind == JsonValueKind.Object - ? packageElement - : default; - - var packageName = GetString(package, "name") ?? "unknown-package"; - var ecosystem = GetString(package, "ecosystem") ?? "unknown"; - var vulnerableRange = GetString(entry, "vulnerable_version_range"); - - string? patchedVersion = null; - if (entry.TryGetProperty("first_patched_version", out var patchedElement) && patchedElement.ValueKind == JsonValueKind.Object) - { - patchedVersion = GetString(patchedElement, "identifier"); - } - - list.Add(new GhsaAffectedDto - { - PackageName = packageName, - Ecosystem = ecosystem, - VulnerableRange = vulnerableRange, - PatchedVersion = patchedVersion, - }); - } - - return list; - } - - private static IReadOnlyList<GhsaCreditDto> ParseCredits(JsonElement root) - { - if (!root.TryGetProperty("credits", out var credits) || credits.ValueKind != JsonValueKind.Array) - { - return Array.Empty<GhsaCreditDto>(); - } - - var list = new List<GhsaCreditDto>(credits.GetArrayLength()); - foreach (var credit in credits.EnumerateArray()) - { - if (credit.ValueKind != JsonValueKind.Object) - { - continue; - } - - var type = GetString(credit, "type"); - var name = GetString(credit, "name"); - string? login = null; - string? profile = null; - - if (credit.TryGetProperty("user", out var user) && user.ValueKind == JsonValueKind.Object) - { - login = GetString(user, "login"); - profile = GetString(user, "html_url") ?? GetString(user, "url"); - name ??= GetString(user, "name"); - } - - name ??= login; - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - - list.Add(new GhsaCreditDto - { - Type = type, - Name = name, - Login = login, - ProfileUrl = profile, - }); - } - - return list; - } - - private static IReadOnlyList<GhsaWeaknessDto> ParseCwes(JsonElement root) - { - if (!root.TryGetProperty("cwes", out var cwes) || cwes.ValueKind != JsonValueKind.Array) - { - return Array.Empty<GhsaWeaknessDto>(); - } - - var list = new List<GhsaWeaknessDto>(cwes.GetArrayLength()); - foreach (var entry in cwes.EnumerateArray()) - { - if (entry.ValueKind != JsonValueKind.Object) - { - continue; - } - - var cweId = GetString(entry, "cwe_id"); - if (string.IsNullOrWhiteSpace(cweId)) - { - continue; - } - - list.Add(new GhsaWeaknessDto - { - CweId = cweId, - Name = GetString(entry, "name"), - }); - } - - return list.Count == 0 ? Array.Empty<GhsaWeaknessDto>() : list; - } - - private static GhsaCvssDto? ParseCvss(JsonElement root) - { - if (!root.TryGetProperty("cvss", out var cvss) || cvss.ValueKind != JsonValueKind.Object) - { - return null; - } - - double? score = null; - if (cvss.TryGetProperty("score", out var scoreElement) && scoreElement.ValueKind == JsonValueKind.Number) - { - score = scoreElement.GetDouble(); - } - - var vector = GetString(cvss, "vector_string") ?? GetString(cvss, "vectorString"); - var severity = GetString(cvss, "severity"); - - if (score is null && string.IsNullOrWhiteSpace(vector) && string.IsNullOrWhiteSpace(severity)) - { - return null; - } - - return new GhsaCvssDto - { - Score = score, - VectorString = vector, - Severity = severity, - }; - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind switch - { - JsonValueKind.String => property.GetString(), - _ => null, - }; - } - - private static DateTimeOffset? GetDate(JsonElement element, string propertyName) - { - var value = GetString(element, propertyName); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) - ? parsed.ToUniversalTime() - : null; - } -} +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; + +namespace StellaOps.Concelier.Connector.Ghsa.Internal; + +internal static class GhsaRecordParser +{ + public static GhsaRecordDto Parse(ReadOnlySpan<byte> content) + { + using var document = JsonDocument.Parse(content.ToArray()); + var root = document.RootElement; + + var ghsaId = GetString(root, "ghsa_id") ?? throw new JsonException("ghsa_id missing"); + var summary = GetString(root, "summary"); + var description = GetString(root, "description"); + var severity = GetString(root, "severity"); + var publishedAt = GetDate(root, "published_at"); + var updatedAt = GetDate(root, "updated_at") ?? publishedAt; + + var aliases = new HashSet<string>(StringComparer.OrdinalIgnoreCase) + { + ghsaId, + }; + + if (root.TryGetProperty("cve_ids", out var cveIds) && cveIds.ValueKind == JsonValueKind.Array) + { + foreach (var cve in cveIds.EnumerateArray()) + { + if (cve.ValueKind == JsonValueKind.String && !string.IsNullOrWhiteSpace(cve.GetString())) + { + aliases.Add(cve.GetString()!); + } + } + } + + var references = ParseReferences(root); + var affected = ParseAffected(root); + var credits = ParseCredits(root); + var cwes = ParseCwes(root); + var cvss = ParseCvss(root); + + return new GhsaRecordDto + { + GhsaId = ghsaId, + Summary = summary, + Description = description, + Severity = severity, + PublishedAt = publishedAt, + UpdatedAt = updatedAt, + Aliases = aliases.ToArray(), + References = references, + Affected = affected, + Credits = credits, + Cwes = cwes, + Cvss = cvss, + }; + } + + private static IReadOnlyList<GhsaReferenceDto> ParseReferences(JsonElement root) + { + if (!root.TryGetProperty("references", out var references) || references.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaReferenceDto>(); + } + + var list = new List<GhsaReferenceDto>(references.GetArrayLength()); + foreach (var reference in references.EnumerateArray()) + { + if (reference.ValueKind != JsonValueKind.Object) + { + continue; + } + + var url = GetString(reference, "url"); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + list.Add(new GhsaReferenceDto + { + Url = url, + Type = GetString(reference, "type"), + Name = GetString(reference, "name"), + }); + } + + return list; + } + + private static IReadOnlyList<GhsaAffectedDto> ParseAffected(JsonElement root) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) || vulnerabilities.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaAffectedDto>(); + } + + var list = new List<GhsaAffectedDto>(vulnerabilities.GetArrayLength()); + foreach (var entry in vulnerabilities.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var package = entry.TryGetProperty("package", out var packageElement) && packageElement.ValueKind == JsonValueKind.Object + ? packageElement + : default; + + var packageName = GetString(package, "name") ?? "unknown-package"; + var ecosystem = GetString(package, "ecosystem") ?? "unknown"; + var vulnerableRange = GetString(entry, "vulnerable_version_range"); + + string? patchedVersion = null; + if (entry.TryGetProperty("first_patched_version", out var patchedElement) && patchedElement.ValueKind == JsonValueKind.Object) + { + patchedVersion = GetString(patchedElement, "identifier"); + } + + list.Add(new GhsaAffectedDto + { + PackageName = packageName, + Ecosystem = ecosystem, + VulnerableRange = vulnerableRange, + PatchedVersion = patchedVersion, + }); + } + + return list; + } + + private static IReadOnlyList<GhsaCreditDto> ParseCredits(JsonElement root) + { + if (!root.TryGetProperty("credits", out var credits) || credits.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaCreditDto>(); + } + + var list = new List<GhsaCreditDto>(credits.GetArrayLength()); + foreach (var credit in credits.EnumerateArray()) + { + if (credit.ValueKind != JsonValueKind.Object) + { + continue; + } + + var type = GetString(credit, "type"); + var name = GetString(credit, "name"); + string? login = null; + string? profile = null; + + if (credit.TryGetProperty("user", out var user) && user.ValueKind == JsonValueKind.Object) + { + login = GetString(user, "login"); + profile = GetString(user, "html_url") ?? GetString(user, "url"); + name ??= GetString(user, "name"); + } + + name ??= login; + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + list.Add(new GhsaCreditDto + { + Type = type, + Name = name, + Login = login, + ProfileUrl = profile, + }); + } + + return list; + } + + private static IReadOnlyList<GhsaWeaknessDto> ParseCwes(JsonElement root) + { + if (!root.TryGetProperty("cwes", out var cwes) || cwes.ValueKind != JsonValueKind.Array) + { + return Array.Empty<GhsaWeaknessDto>(); + } + + var list = new List<GhsaWeaknessDto>(cwes.GetArrayLength()); + foreach (var entry in cwes.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var cweId = GetString(entry, "cwe_id"); + if (string.IsNullOrWhiteSpace(cweId)) + { + continue; + } + + list.Add(new GhsaWeaknessDto + { + CweId = cweId, + Name = GetString(entry, "name"), + }); + } + + return list.Count == 0 ? Array.Empty<GhsaWeaknessDto>() : list; + } + + private static GhsaCvssDto? ParseCvss(JsonElement root) + { + if (!root.TryGetProperty("cvss", out var cvss) || cvss.ValueKind != JsonValueKind.Object) + { + return null; + } + + double? score = null; + if (cvss.TryGetProperty("score", out var scoreElement) && scoreElement.ValueKind == JsonValueKind.Number) + { + score = scoreElement.GetDouble(); + } + + var vector = GetString(cvss, "vector_string") ?? GetString(cvss, "vectorString"); + var severity = GetString(cvss, "severity"); + + if (score is null && string.IsNullOrWhiteSpace(vector) && string.IsNullOrWhiteSpace(severity)) + { + return null; + } + + return new GhsaCvssDto + { + Score = score, + VectorString = vector, + Severity = severity, + }; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind switch + { + JsonValueKind.String => property.GetString(), + _ => null, + }; + } + + private static DateTimeOffset? GetDate(JsonElement element, string propertyName) + { + var value = GetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var parsed) + ? parsed.ToUniversalTime() + : null; + } +} diff --git a/src/StellaOps.Feedser.Source.Ghsa/Jobs.cs b/src/StellaOps.Concelier.Connector.Ghsa/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ghsa/Jobs.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Jobs.cs index 67759407..b7ad6848 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Jobs.cs @@ -1,43 +1,43 @@ -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Ghsa; - -internal static class GhsaJobKinds -{ - public const string Fetch = "source:ghsa:fetch"; - public const string Parse = "source:ghsa:parse"; - public const string Map = "source:ghsa:map"; -} - -internal sealed class GhsaFetchJob : IJob -{ - private readonly GhsaConnector _connector; - - public GhsaFetchJob(GhsaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class GhsaParseJob : IJob -{ - private readonly GhsaConnector _connector; - - public GhsaParseJob(GhsaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class GhsaMapJob : IJob -{ - private readonly GhsaConnector _connector; - - public GhsaMapJob(GhsaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Ghsa; + +internal static class GhsaJobKinds +{ + public const string Fetch = "source:ghsa:fetch"; + public const string Parse = "source:ghsa:parse"; + public const string Map = "source:ghsa:map"; +} + +internal sealed class GhsaFetchJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaFetchJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class GhsaParseJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaParseJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class GhsaMapJob : IJob +{ + private readonly GhsaConnector _connector; + + public GhsaMapJob(GhsaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Nvd/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Ghsa/Properties/AssemblyInfo.cs similarity index 52% rename from src/StellaOps.Feedser.Source.Nvd/Properties/AssemblyInfo.cs rename to src/StellaOps.Concelier.Connector.Ghsa/Properties/AssemblyInfo.cs index 9a1bdc8f..3db3a93f 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Properties/AssemblyInfo.cs +++ b/src/StellaOps.Concelier.Connector.Ghsa/Properties/AssemblyInfo.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Nvd.Tests")] -[assembly: InternalsVisibleTo("FixtureUpdater")] +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("FixtureUpdater")] +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ghsa.Tests")] diff --git a/src/StellaOps.Concelier.Connector.Ghsa/StellaOps.Concelier.Connector.Ghsa.csproj b/src/StellaOps.Concelier.Connector.Ghsa/StellaOps.Concelier.Connector.Ghsa.csproj new file mode 100644 index 00000000..71801984 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ghsa/StellaOps.Concelier.Connector.Ghsa.csproj @@ -0,0 +1,17 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <TargetFramework>net10.0</TargetFramework> + <ImplicitUsings>enable</ImplicitUsings> + <Nullable>enable</Nullable> + </PropertyGroup> + + <ItemGroup> + <ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" /> + + <ProjectReference Include="../StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" /> + <ProjectReference Include="../StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj" /> + </ItemGroup> +</Project> + diff --git a/src/StellaOps.Feedser.Source.Ghsa/TASKS.md b/src/StellaOps.Concelier.Connector.Ghsa/TASKS.md similarity index 90% rename from src/StellaOps.Feedser.Source.Ghsa/TASKS.md rename to src/StellaOps.Concelier.Connector.Ghsa/TASKS.md index da091701..03723827 100644 --- a/src/StellaOps.Feedser.Source.Ghsa/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Ghsa/TASKS.md @@ -5,15 +5,15 @@ |Fetch pipeline & state management|BE-Conn-GHSA|Source.Common, Storage.Mongo|**DONE (2025-10-10)** – Implemented list/detail fetch using `GhsaCursor` (time window + page), resumable SourceState and backoff controls.| |DTO & parser implementation|BE-Conn-GHSA|Source.Common|**DONE (2025-10-10)** – Added `GhsaRecordParser`/DTOs extracting aliases, references, severity, vulnerable ranges, patched versions.| |Canonical mapping & range primitives|BE-Conn-GHSA|Models|**DONE (2025-10-10)** – `GhsaMapper` emits GHSA advisories with SemVer packages, vendor extensions (ecosystem/package) and deterministic references.<br>2025-10-11 research trail: upcoming normalized array should follow `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"ghsa:GHSA-xxxx"}]`; include patched-only advisories as `lt`/`lte` when no explicit floor.| -|Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-10)** – New `StellaOps.Feedser.Source.Ghsa.Tests` regression covers fetch/parse/map via canned GHSA fixtures and snapshot assertions.| +|Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-10)** – New `StellaOps.Concelier.Connector.Ghsa.Tests` regression covers fetch/parse/map via canned GHSA fixtures and snapshot assertions.| |Telemetry & documentation|DevEx|Docs|**DONE (2025-10-10)** – Diagnostics meter (`ghsa.fetch.*`) wired; DI extension documents token/headers and job registrations.| |GitHub quota monitoring & retries|BE-Conn-GHSA, Observability|Source.Common|**DONE (2025-10-12)** – Rate-limit metrics/logs added, retry/backoff handles 403 secondary limits, and ops runbook documents dashboards + mitigation steps.| -|Production credential & scheduler rollout|Ops, BE-Conn-GHSA|Docs, WebService|**DONE (2025-10-12)** – Scheduler defaults registered via `JobSchedulerBuilder`, credential provisioning documented (Compose/Helm samples), and staged backfill guidance captured in `docs/ops/feedser-ghsa-operations.md`.| -|FEEDCONN-GHSA-04-002 Conflict regression fixtures|BE-Conn-GHSA, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Added `conflict-ghsa.canonical.json` + `GhsaConflictFixtureTests`; SemVer ranges and credits align with merge precedence triple and shareable with QA. Validation: `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests`.| +|Production credential & scheduler rollout|Ops, BE-Conn-GHSA|Docs, WebService|**DONE (2025-10-12)** – Scheduler defaults registered via `JobSchedulerBuilder`, credential provisioning documented (Compose/Helm samples), and staged backfill guidance captured in `docs/ops/concelier-ghsa-operations.md`.| +|FEEDCONN-GHSA-04-002 Conflict regression fixtures|BE-Conn-GHSA, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Added `conflict-ghsa.canonical.json` + `GhsaConflictFixtureTests`; SemVer ranges and credits align with merge precedence triple and shareable with QA. Validation: `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj --filter GhsaConflictFixtureTests`.| |FEEDCONN-GHSA-02-004 GHSA credits & ecosystem severity mapping|BE-Conn-GHSA|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – Mapper emits advisory credits with provenance masks, fixtures assert role/contact ordering, and severity normalization remains unchanged.| |FEEDCONN-GHSA-02-007 Credit parity regression fixtures|BE-Conn-GHSA, QA|Source.Nvd, Source.Osv|**DONE (2025-10-12)** – Parity fixtures regenerated via `tools/FixtureUpdater`, normalized SemVer notes verified against GHSA/NVD/OSV snapshots, and the fixtures guide now documents the headroom checks.| |FEEDCONN-GHSA-02-001 Normalized versions rollout|BE-Conn-GHSA|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** – GHSA mapper now emits SemVer primitives + normalized ranges, fixtures refreshed, connector tests passing; report logged via FEEDMERGE-COORD-02-900.| |FEEDCONN-GHSA-02-005 Quota monitoring hardening|BE-Conn-GHSA, Observability|Source.Common metrics|**DONE (2025-10-12)** – Diagnostics expose headroom histograms/gauges, warning logs dedupe below the configured threshold, and the ops runbook gained alerting and mitigation guidance.| |FEEDCONN-GHSA-02-006 Scheduler rollout integration|BE-Conn-GHSA, Ops|Job scheduler|**DONE (2025-10-12)** – Dependency routine tests assert cron/timeouts, and the runbook highlights cron overrides plus backoff toggles for staged rollouts.| |FEEDCONN-GHSA-04-003 Description/CWE/metric parity rollout|BE-Conn-GHSA|Models, Core|**DONE (2025-10-15)** – Mapper emits advisory description, CWE weaknesses, and canonical CVSS metric id with updated fixtures (`osv-ghsa.osv.json` parity suite) and connector regression covers the new fields. Reported completion to Merge coordination.| -|FEEDCONN-GHSA-04-004 Canonical metric fallback coverage|BE-Conn-GHSA|Models, Merge|**DONE (2025-10-16)** – Ensure canonical metric ids remain populated when GitHub omits CVSS vectors/scores; add fixtures capturing severity-only advisories, document precedence with Merge, and emit analytics to track fallback usage.<br>2025-10-16: Mapper now emits `ghsa:severity/<level>` canonical ids when vectors are missing, diagnostics expose `ghsa.map.canonical_metric_fallbacks`, conflict/mapper fixtures updated, and runbook documents Merge precedence. Tests: `dotnet test src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj`.| +|FEEDCONN-GHSA-04-004 Canonical metric fallback coverage|BE-Conn-GHSA|Models, Merge|**DONE (2025-10-16)** – Ensure canonical metric ids remain populated when GitHub omits CVSS vectors/scores; add fixtures capturing severity-only advisories, document precedence with Merge, and emit analytics to track fallback usage.<br>2025-10-16: Mapper now emits `ghsa:severity/<level>` canonical ids when vectors are missing, diagnostics expose `ghsa.map.canonical_metric_fallbacks`, conflict/mapper fixtures updated, and runbook documents Merge precedence. Tests: `dotnet test src/StellaOps.Concelier.Connector.Ghsa.Tests/StellaOps.Concelier.Connector.Ghsa.Tests.csproj`.| diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html similarity index 98% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html index 1a1921fd..8caff893 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsa-25-123-01.html @@ -1,13 +1,13 @@ -<article> - <h1>ICSA-25-123-01: Example ICS Advisory</h1> - <p>The Cybersecurity and Infrastructure Security Agency (CISA) is aware of vulnerabilities affecting ControlSuite 4.2.</p> - <p><strong>Vendor:</strong> Example Corp</p> - <p><strong>Products:</strong> ControlSuite 4.2</p> - <p><a href="https://files.cisa.gov/docs/icsa-25-123-01.pdf">Download PDF advisory</a></p> - <p>For additional information see the <a href="https://example.com/security/icsa-25-123-01">vendor bulletin</a>.</p> - <h2>Mitigations</h2> - <ul> - <li>Apply ControlSuite firmware version 4.2.1 or later.</li> - <li>Restrict network access to the engineering workstation and monitor remote connections.</li> - </ul> -</article> +<article> + <h1>ICSA-25-123-01: Example ICS Advisory</h1> + <p>The Cybersecurity and Infrastructure Security Agency (CISA) is aware of vulnerabilities affecting ControlSuite 4.2.</p> + <p><strong>Vendor:</strong> Example Corp</p> + <p><strong>Products:</strong> ControlSuite 4.2</p> + <p><a href="https://files.cisa.gov/docs/icsa-25-123-01.pdf">Download PDF advisory</a></p> + <p>For additional information see the <a href="https://example.com/security/icsa-25-123-01">vendor bulletin</a>.</p> + <h2>Mitigations</h2> + <ul> + <li>Apply ControlSuite firmware version 4.2.1 or later.</li> + <li>Restrict network access to the engineering workstation and monitor remote connections.</li> + </ul> +</article> diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html similarity index 98% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html index 1c1f45ab..9eab3d35 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/icsma-25-045-01.html @@ -1,9 +1,9 @@ -<article> - <h1>ICSMA-25-045-01: Example Medical Advisory</h1> - <p>HealthTech InfusionManager 2.1 devices contain multiple vulnerabilities.</p> - <p><strong>Vendor:</strong> HealthTech</p> - <p><strong>Products:</strong> InfusionManager 2.1</p> - <p><a href="https://www.cisa.gov/sites/default/files/2025-10/ICSMA-25-045-01_Supplement.pdf">Supplemental guidance</a></p> - <h2>Mitigations</h2> - <p>Contact HealthTech support to obtain firmware 2.1.5 and enable multi-factor authentication for remote sessions.</p> -</article> +<article> + <h1>ICSMA-25-045-01: Example Medical Advisory</h1> + <p>HealthTech InfusionManager 2.1 devices contain multiple vulnerabilities.</p> + <p><strong>Vendor:</strong> HealthTech</p> + <p><strong>Products:</strong> InfusionManager 2.1</p> + <p><a href="https://www.cisa.gov/sites/default/files/2025-10/ICSMA-25-045-01_Supplement.pdf">Supplemental guidance</a></p> + <h2>Mitigations</h2> + <p>Contact HealthTech support to obtain firmware 2.1.5 and enable multi-factor authentication for remote sessions.</p> +</article> diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml similarity index 97% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml index a227a6e4..75f07863 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/Fixtures/sample-feed.xml @@ -1,27 +1,27 @@ -<?xml version="1.0" encoding="UTF-8"?> -<rss version="2.0"> - <channel> - <title>CISA ICS Advisories - - ICSA-25-123-01: Example ICS Advisory - https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01 - Mon, 13 Oct 2025 12:00:00 GMT - Vendor: Example Corp

      -

      Products: ControlSuite 4.2

      -

      Download vendor PDF

      -

      CVE-2024-12345 allows remote code execution.

      - ]]>       -       - - ICSMA-25-045-01: Example Medical Advisory - https://www.cisa.gov/news-events/ics-medical-advisories/icsma-25-045-01 - Tue, 14 Oct 2025 09:30:00 GMT - Vendor: HealthTech

      -

      Products: InfusionManager 2.1

      -

      Multiple vulnerabilities including CVE-2025-11111 and CVE-2025-22222.

      - ]]>       -       - - + + + + CISA ICS Advisories + + ICSA-25-123-01: Example ICS Advisory + https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01 + Mon, 13 Oct 2025 12:00:00 GMT + Vendor: Example Corp

      +

      Products: ControlSuite 4.2

      +

      Download vendor PDF

      +

      CVE-2024-12345 allows remote code execution.

      + ]]>       +       + + ICSMA-25-045-01: Example Medical Advisory + https://www.cisa.gov/news-events/ics-medical-advisories/icsma-25-045-01 + Tue, 14 Oct 2025 09:30:00 GMT + Vendor: HealthTech

      +

      Products: InfusionManager 2.1

      +

      Multiple vulnerabilities including CVE-2025-11111 and CVE-2025-22222.

      + ]]>       +       +       +       diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs index 5300fe64..6b03c996 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaConnectorMappingTests.cs @@ -1,101 +1,101 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Ics.Cisa; -using StellaOps.Feedser.Source.Ics.Cisa.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Tests.IcsCisa; - -public class IcsCisaConnectorMappingTests -{ - private static readonly DateTimeOffset RecordedAt = new(2025, 10, 14, 12, 0, 0, TimeSpan.Zero); - - [Fact] - public void BuildReferences_MergesFeedAndDetailAttachments() - { - var dto = new IcsCisaAdvisoryDto - { - AdvisoryId = "ICSA-25-123-01", - Title = "Sample Advisory", - Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01", - Summary = "Summary", - DescriptionHtml = "

      Summary

      ", - Published = RecordedAt, - Updated = RecordedAt, - IsMedical = false, - References = new[] - { - "https://example.org/advisory", - "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01" - }, - Attachments = new List - { - new() { Title = "PDF Attachment", Url = "https://files.cisa.gov/docs/icsa-25-123-01.pdf" }, - } - }; - - var references = IcsCisaConnector.BuildReferences(dto, RecordedAt); - - Assert.Equal(3, references.Count); - Assert.Contains(references, reference => reference.Kind == "attachment" && reference.Url == "https://files.cisa.gov/docs/icsa-25-123-01.pdf"); - Assert.Contains(references, reference => reference.Url == "https://example.org/advisory"); - Assert.Contains(references, reference => reference.Url == "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01"); - } - - - [Fact] - public void BuildMitigationReferences_ProducesReferences() - { - var dto = new IcsCisaAdvisoryDto - { - AdvisoryId = "ICSA-25-999-01", - Title = "Mitigation Test", - Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-999-01", - Mitigations = new[] { "Apply firmware 9.9.1", "Limit network access" }, - Published = RecordedAt, - Updated = RecordedAt, - IsMedical = false, - }; - - var references = IcsCisaConnector.BuildMitigationReferences(dto, RecordedAt); - - Assert.Equal(2, references.Count); - var first = references.First(); - Assert.Equal("mitigation", first.Kind); - Assert.Equal("icscisa-mitigation", first.SourceTag); - Assert.EndsWith("#mitigation-1", first.Url, StringComparison.Ordinal); - Assert.Contains("Apply firmware", first.Summary); - } - - [Fact] - public void BuildAffectedPackages_EmitsProductRangesWithSemVer() - { - var dto = new IcsCisaAdvisoryDto - { - AdvisoryId = "ICSA-25-456-02", - Title = "Vendor Advisory", - Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-456-02", - DescriptionHtml = "", - Summary = null, - Published = RecordedAt, - Vendors = new[] { "Example Corp" }, - Products = new[] { "ControlSuite 4.2" } - }; - - var packages = IcsCisaConnector.BuildAffectedPackages(dto, RecordedAt); - - var productPackage = Assert.Single(packages); - Assert.Equal(AffectedPackageTypes.IcsVendor, productPackage.Type); - Assert.Equal("ControlSuite", productPackage.Identifier); - var range = Assert.Single(productPackage.VersionRanges); - Assert.Equal("product", range.RangeKind); - Assert.Equal("4.2", range.RangeExpression); - Assert.NotNull(range.Primitives); - Assert.Equal("Example Corp", range.Primitives!.VendorExtensions!["ics.vendors"]); - Assert.Equal("ControlSuite", range.Primitives.VendorExtensions!["ics.product"]); - Assert.NotNull(range.Primitives.SemVer); - Assert.Equal("4.2.0", range.Primitives.SemVer!.ExactValue); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Ics.Cisa; +using StellaOps.Concelier.Connector.Ics.Cisa.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Tests.IcsCisa; + +public class IcsCisaConnectorMappingTests +{ + private static readonly DateTimeOffset RecordedAt = new(2025, 10, 14, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public void BuildReferences_MergesFeedAndDetailAttachments() + { + var dto = new IcsCisaAdvisoryDto + { + AdvisoryId = "ICSA-25-123-01", + Title = "Sample Advisory", + Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01", + Summary = "Summary", + DescriptionHtml = "

      Summary

      ", + Published = RecordedAt, + Updated = RecordedAt, + IsMedical = false, + References = new[] + { + "https://example.org/advisory", + "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01" + }, + Attachments = new List + { + new() { Title = "PDF Attachment", Url = "https://files.cisa.gov/docs/icsa-25-123-01.pdf" }, + } + }; + + var references = IcsCisaConnector.BuildReferences(dto, RecordedAt); + + Assert.Equal(3, references.Count); + Assert.Contains(references, reference => reference.Kind == "attachment" && reference.Url == "https://files.cisa.gov/docs/icsa-25-123-01.pdf"); + Assert.Contains(references, reference => reference.Url == "https://example.org/advisory"); + Assert.Contains(references, reference => reference.Url == "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01"); + } + + + [Fact] + public void BuildMitigationReferences_ProducesReferences() + { + var dto = new IcsCisaAdvisoryDto + { + AdvisoryId = "ICSA-25-999-01", + Title = "Mitigation Test", + Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-999-01", + Mitigations = new[] { "Apply firmware 9.9.1", "Limit network access" }, + Published = RecordedAt, + Updated = RecordedAt, + IsMedical = false, + }; + + var references = IcsCisaConnector.BuildMitigationReferences(dto, RecordedAt); + + Assert.Equal(2, references.Count); + var first = references.First(); + Assert.Equal("mitigation", first.Kind); + Assert.Equal("icscisa-mitigation", first.SourceTag); + Assert.EndsWith("#mitigation-1", first.Url, StringComparison.Ordinal); + Assert.Contains("Apply firmware", first.Summary); + } + + [Fact] + public void BuildAffectedPackages_EmitsProductRangesWithSemVer() + { + var dto = new IcsCisaAdvisoryDto + { + AdvisoryId = "ICSA-25-456-02", + Title = "Vendor Advisory", + Link = "https://www.cisa.gov/news-events/ics-advisories/icsa-25-456-02", + DescriptionHtml = "", + Summary = null, + Published = RecordedAt, + Vendors = new[] { "Example Corp" }, + Products = new[] { "ControlSuite 4.2" } + }; + + var packages = IcsCisaConnector.BuildAffectedPackages(dto, RecordedAt); + + var productPackage = Assert.Single(packages); + Assert.Equal(AffectedPackageTypes.IcsVendor, productPackage.Type); + Assert.Equal("ControlSuite", productPackage.Identifier); + var range = Assert.Single(productPackage.VersionRanges); + Assert.Equal("product", range.RangeKind); + Assert.Equal("4.2", range.RangeExpression); + Assert.NotNull(range.Primitives); + Assert.Equal("Example Corp", range.Primitives!.VendorExtensions!["ics.vendors"]); + Assert.Equal("ControlSuite", range.Primitives.VendorExtensions!["ics.product"]); + Assert.NotNull(range.Primitives.SemVer); + Assert.Equal("4.2.0", range.Primitives.SemVer!.ExactValue); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs index c280017f..0c43eea9 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisa/IcsCisaFeedParserTests.cs @@ -1,38 +1,38 @@ -using System; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using StellaOps.Feedser.Source.Ics.Cisa.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Tests.IcsCisa; - -public class IcsCisaFeedParserTests -{ - [Fact] - public void Parse_ReturnsAdvisories() - { - var parser = new IcsCisaFeedParser(); - using var stream = File.OpenRead(Path.Combine("IcsCisa", "Fixtures", "sample-feed.xml")); - - var advisories = parser.Parse(stream, isMedicalTopic: false, topicUri: new Uri("https://content.govdelivery.com/accounts/USDHSCISA/topics.rss")); - - Assert.Equal(2, advisories.Count); - - var first = advisories.First(); - Console.WriteLine("Description:" + first.DescriptionHtml); - Console.WriteLine("Attachments:" + string.Join(",", first.Attachments.Select(a => a.Url))); - Console.WriteLine("References:" + string.Join(",", first.References)); - Assert.Equal("ICSA-25-123-01", first.AdvisoryId); - Assert.Contains("CVE-2024-12345", first.CveIds); - Assert.Contains("Example Corp", first.Vendors); - Assert.Contains("ControlSuite 4.2", first.Products); - Assert.Contains(first.Attachments, attachment => attachment.Url == "https://example.com/security/icsa-25-123-01.pdf"); - Assert.Contains(first.References, reference => reference == "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01"); - - var second = advisories.Last(); - Assert.True(second.IsMedical); - Assert.Contains("CVE-2025-11111", second.CveIds); - Assert.Contains("HealthTech", second.Vendors); - } -} +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using StellaOps.Concelier.Connector.Ics.Cisa.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Tests.IcsCisa; + +public class IcsCisaFeedParserTests +{ + [Fact] + public void Parse_ReturnsAdvisories() + { + var parser = new IcsCisaFeedParser(); + using var stream = File.OpenRead(Path.Combine("IcsCisa", "Fixtures", "sample-feed.xml")); + + var advisories = parser.Parse(stream, isMedicalTopic: false, topicUri: new Uri("https://content.govdelivery.com/accounts/USDHSCISA/topics.rss")); + + Assert.Equal(2, advisories.Count); + + var first = advisories.First(); + Console.WriteLine("Description:" + first.DescriptionHtml); + Console.WriteLine("Attachments:" + string.Join(",", first.Attachments.Select(a => a.Url))); + Console.WriteLine("References:" + string.Join(",", first.References)); + Assert.Equal("ICSA-25-123-01", first.AdvisoryId); + Assert.Contains("CVE-2024-12345", first.CveIds); + Assert.Contains("Example Corp", first.Vendors); + Assert.Contains("ControlSuite 4.2", first.Products); + Assert.Contains(first.Attachments, attachment => attachment.Url == "https://example.com/security/icsa-25-123-01.pdf"); + Assert.Contains(first.References, reference => reference == "https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01"); + + var second = advisories.Last(); + Assert.True(second.IsMedical); + Assert.Contains("CVE-2025-11111", second.CveIds); + Assert.Contains("HealthTech", second.Vendors); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisaConnectorTests.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisaConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisaConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisaConnectorTests.cs index 1d928560..b3d50857 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/IcsCisaConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/IcsCisaConnectorTests.cs @@ -1,156 +1,156 @@ -using System; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ics.Cisa; -using StellaOps.Feedser.Source.Ics.Cisa.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Tests; - -[Collection("mongo-fixture")] -public sealed class IcsCisaConnectorTests : IAsyncLifetime -{ - private readonly MongoIntegrationFixture _fixture; - private readonly CannedHttpMessageHandler _handler = new(); - - public IcsCisaConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); - } - - [Fact] - public async Task FetchParseMap_EndToEnd_ProducesCanonicalAdvisories() - { - await using var provider = await BuildServiceProviderAsync(); - RegisterResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - _handler.AssertNoPendingResponses(); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - - Assert.Equal(2, advisories.Count); - - var icsa = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSA-25-123-01"); - Console.WriteLine("ProductsRaw:" + string.Join("|", icsa.AffectedPackages.SelectMany(p => p.Provenance).Select(p => p.Value ?? ""))); - Assert.Contains("CVE-2024-12345", icsa.Aliases); - Assert.Contains(icsa.References, reference => reference.Url == "https://example.com/security/icsa-25-123-01"); - Assert.Contains(icsa.References, reference => reference.Url == "https://files.cisa.gov/docs/icsa-25-123-01.pdf" && reference.Kind == "attachment"); - var icsaMitigations = icsa.References.Where(reference => reference.Kind == "mitigation").ToList(); - Assert.Equal(2, icsaMitigations.Count); - Assert.Contains("Apply ControlSuite firmware version 4.2.1 or later.", icsaMitigations[0].Summary, StringComparison.Ordinal); - Assert.EndsWith("#mitigation-1", icsaMitigations[0].Url, StringComparison.Ordinal); - Assert.Contains("Restrict network access", icsaMitigations[1].Summary, StringComparison.Ordinal); - - var controlSuitePackage = Assert.Single(icsa.AffectedPackages, package => string.Equals(package.Identifier, "ControlSuite", StringComparison.OrdinalIgnoreCase)); - var controlSuiteRange = Assert.Single(controlSuitePackage.VersionRanges); - Assert.Equal("product", controlSuiteRange.RangeKind); - Assert.Equal("4.2", controlSuiteRange.RangeExpression); - Assert.NotNull(controlSuiteRange.Primitives); - Assert.NotNull(controlSuiteRange.Primitives!.SemVer); - Assert.Equal("4.2.0", controlSuiteRange.Primitives.SemVer!.ExactValue); - Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.product", out var controlSuiteProduct) && controlSuiteProduct == "ControlSuite"); - Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.version", out var controlSuiteVersion) && controlSuiteVersion == "4.2"); - Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.vendors", out var controlSuiteVendors) && controlSuiteVendors == "Example Corp"); - - var icsma = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSMA-25-045-01"); - Assert.Contains("CVE-2025-11111", icsma.Aliases); - var icsmaMitigation = Assert.Single(icsma.References.Where(reference => reference.Kind == "mitigation")); - Assert.Contains("Contact HealthTech support", icsmaMitigation.Summary, StringComparison.Ordinal); - Assert.Contains(icsma.References, reference => reference.Url == "https://www.cisa.gov/sites/default/files/2025-10/ICSMA-25-045-01_Supplement.pdf"); - var infusionPackage = Assert.Single(icsma.AffectedPackages, package => string.Equals(package.Identifier, "InfusionManager", StringComparison.OrdinalIgnoreCase)); - var infusionRange = Assert.Single(infusionPackage.VersionRanges); - Assert.Equal("2.1", infusionRange.RangeExpression); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddIcsCisaConnector(options => - { - options.GovDeliveryCode = "TESTCODE"; - options.TopicsEndpoint = new Uri("https://feed.test/topics.rss", UriKind.Absolute); - options.TopicIds.Clear(); - options.TopicIds.Add("USDHSCISA_TEST"); - options.RequestDelay = TimeSpan.Zero; - options.DetailBaseUri = new Uri("https://www.cisa.gov/", UriKind.Absolute); - options.AdditionalHosts.Add("files.cisa.gov"); - }); - - services.Configure(IcsCisaOptions.HttpClientName, builder => - { - builder.HttpMessageHandlerBuilderActions.Add(handlerBuilder => - { - handlerBuilder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void RegisterResponses() - { - var feedUri = new Uri("https://feed.test/topics.rss?code=TESTCODE&format=xml&topic_id=USDHSCISA_TEST", UriKind.Absolute); - _handler.AddResponse(feedUri, () => CreateTextResponse("IcsCisa/Fixtures/sample-feed.xml", "application/rss+xml")); - - var icsaDetail = new Uri("https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01", UriKind.Absolute); - _handler.AddResponse(icsaDetail, () => CreateTextResponse("IcsCisa/Fixtures/icsa-25-123-01.html", "text/html")); - - var icsmaDetail = new Uri("https://www.cisa.gov/news-events/ics-medical-advisories/icsma-25-045-01", UriKind.Absolute); - _handler.AddResponse(icsmaDetail, () => CreateTextResponse("IcsCisa/Fixtures/icsma-25-045-01.html", "text/html")); - } - - private static HttpResponseMessage CreateTextResponse(string relativePath, string contentType) - { - var fullPath = Path.Combine(AppContext.BaseDirectory, relativePath); - var content = File.ReadAllText(fullPath); - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(content, Encoding.UTF8, contentType), - }; - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - _handler.Clear(); - return Task.CompletedTask; - } -} +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ics.Cisa; +using StellaOps.Concelier.Connector.Ics.Cisa.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Tests; + +[Collection("mongo-fixture")] +public sealed class IcsCisaConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler = new(); + + public IcsCisaConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture ?? throw new ArgumentNullException(nameof(fixture)); + } + + [Fact] + public async Task FetchParseMap_EndToEnd_ProducesCanonicalAdvisories() + { + await using var provider = await BuildServiceProviderAsync(); + RegisterResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + _handler.AssertNoPendingResponses(); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + + Assert.Equal(2, advisories.Count); + + var icsa = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSA-25-123-01"); + Console.WriteLine("ProductsRaw:" + string.Join("|", icsa.AffectedPackages.SelectMany(p => p.Provenance).Select(p => p.Value ?? ""))); + Assert.Contains("CVE-2024-12345", icsa.Aliases); + Assert.Contains(icsa.References, reference => reference.Url == "https://example.com/security/icsa-25-123-01"); + Assert.Contains(icsa.References, reference => reference.Url == "https://files.cisa.gov/docs/icsa-25-123-01.pdf" && reference.Kind == "attachment"); + var icsaMitigations = icsa.References.Where(reference => reference.Kind == "mitigation").ToList(); + Assert.Equal(2, icsaMitigations.Count); + Assert.Contains("Apply ControlSuite firmware version 4.2.1 or later.", icsaMitigations[0].Summary, StringComparison.Ordinal); + Assert.EndsWith("#mitigation-1", icsaMitigations[0].Url, StringComparison.Ordinal); + Assert.Contains("Restrict network access", icsaMitigations[1].Summary, StringComparison.Ordinal); + + var controlSuitePackage = Assert.Single(icsa.AffectedPackages, package => string.Equals(package.Identifier, "ControlSuite", StringComparison.OrdinalIgnoreCase)); + var controlSuiteRange = Assert.Single(controlSuitePackage.VersionRanges); + Assert.Equal("product", controlSuiteRange.RangeKind); + Assert.Equal("4.2", controlSuiteRange.RangeExpression); + Assert.NotNull(controlSuiteRange.Primitives); + Assert.NotNull(controlSuiteRange.Primitives!.SemVer); + Assert.Equal("4.2.0", controlSuiteRange.Primitives.SemVer!.ExactValue); + Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.product", out var controlSuiteProduct) && controlSuiteProduct == "ControlSuite"); + Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.version", out var controlSuiteVersion) && controlSuiteVersion == "4.2"); + Assert.True(controlSuiteRange.Primitives.VendorExtensions!.TryGetValue("ics.vendors", out var controlSuiteVendors) && controlSuiteVendors == "Example Corp"); + + var icsma = Assert.Single(advisories, advisory => advisory.AdvisoryKey == "ICSMA-25-045-01"); + Assert.Contains("CVE-2025-11111", icsma.Aliases); + var icsmaMitigation = Assert.Single(icsma.References.Where(reference => reference.Kind == "mitigation")); + Assert.Contains("Contact HealthTech support", icsmaMitigation.Summary, StringComparison.Ordinal); + Assert.Contains(icsma.References, reference => reference.Url == "https://www.cisa.gov/sites/default/files/2025-10/ICSMA-25-045-01_Supplement.pdf"); + var infusionPackage = Assert.Single(icsma.AffectedPackages, package => string.Equals(package.Identifier, "InfusionManager", StringComparison.OrdinalIgnoreCase)); + var infusionRange = Assert.Single(infusionPackage.VersionRanges); + Assert.Equal("2.1", infusionRange.RangeExpression); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddIcsCisaConnector(options => + { + options.GovDeliveryCode = "TESTCODE"; + options.TopicsEndpoint = new Uri("https://feed.test/topics.rss", UriKind.Absolute); + options.TopicIds.Clear(); + options.TopicIds.Add("USDHSCISA_TEST"); + options.RequestDelay = TimeSpan.Zero; + options.DetailBaseUri = new Uri("https://www.cisa.gov/", UriKind.Absolute); + options.AdditionalHosts.Add("files.cisa.gov"); + }); + + services.Configure(IcsCisaOptions.HttpClientName, builder => + { + builder.HttpMessageHandlerBuilderActions.Add(handlerBuilder => + { + handlerBuilder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void RegisterResponses() + { + var feedUri = new Uri("https://feed.test/topics.rss?code=TESTCODE&format=xml&topic_id=USDHSCISA_TEST", UriKind.Absolute); + _handler.AddResponse(feedUri, () => CreateTextResponse("IcsCisa/Fixtures/sample-feed.xml", "application/rss+xml")); + + var icsaDetail = new Uri("https://www.cisa.gov/news-events/ics-advisories/icsa-25-123-01", UriKind.Absolute); + _handler.AddResponse(icsaDetail, () => CreateTextResponse("IcsCisa/Fixtures/icsa-25-123-01.html", "text/html")); + + var icsmaDetail = new Uri("https://www.cisa.gov/news-events/ics-medical-advisories/icsma-25-045-01", UriKind.Absolute); + _handler.AddResponse(icsmaDetail, () => CreateTextResponse("IcsCisa/Fixtures/icsma-25-045-01.html", "text/html")); + } + + private static HttpResponseMessage CreateTextResponse(string relativePath, string contentType) + { + var fullPath = Path.Combine(AppContext.BaseDirectory, relativePath); + var content = File.ReadAllText(fullPath); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/StellaOps.Concelier.Connector.Ics.Cisa.Tests.csproj b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/StellaOps.Concelier.Connector.Ics.Cisa.Tests.csproj new file mode 100644 index 00000000..96934e35 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/StellaOps.Concelier.Connector.Ics.Cisa.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md b/src/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md similarity index 85% rename from src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md rename to src/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md index 01ca3f43..b9fa7005 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/AGENTS.md @@ -1,39 +1,39 @@ -# AGENTS -## Role -Implement the CISA ICS advisory connector to ingest US CISA Industrial Control Systems advisories (distinct from the general CERT feed). - -## Scope -- Locate the official CISA ICS advisory feed/API (currently HTML/RSS) and define fetch cadence/windowing. -- Build fetch/cursor pipeline with retry/backoff and raw document storage. -- Parse advisory content for summary, impacted vendors/products, mitigation, CVEs. -- Map advisories into canonical `Advisory` records with aliases, references, affected ICS packages, and range primitives. -- Provide deterministic fixtures and automated regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). -- `Feedser.Models` (canonical advisory structures). -- `Feedser.Testing` (integration fixtures and snapshots). - -## Interfaces & Contracts -- Job kinds: `ics-cisa:fetch`, `ics-cisa:parse`, `ics-cisa:map`. -- Persist upstream caching metadata (ETag/Last-Modified) when available. -- Alias set should include CISA ICS advisory IDs and referenced CVE IDs. - -## In/Out of scope -In scope: -- ICS-specific advisories from CISA. -- Range primitives capturing vendor/equipment metadata. - -Out of scope: -- General CISA alerts (covered elsewhere). - -## Observability & Security Expectations -- Log fetch attempts, advisory counts, and mapping results. -- Sanitize HTML, removing scripts/styles before persistence. -- Honour upstream rate limits with exponential backoff. - -## Tests -- Add `StellaOps.Feedser.Source.Ics.Cisa.Tests` to cover fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Implement the CISA ICS advisory connector to ingest US CISA Industrial Control Systems advisories (distinct from the general CERT feed). + +## Scope +- Locate the official CISA ICS advisory feed/API (currently HTML/RSS) and define fetch cadence/windowing. +- Build fetch/cursor pipeline with retry/backoff and raw document storage. +- Parse advisory content for summary, impacted vendors/products, mitigation, CVEs. +- Map advisories into canonical `Advisory` records with aliases, references, affected ICS packages, and range primitives. +- Provide deterministic fixtures and automated regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Concelier.Models` (canonical advisory structures). +- `Concelier.Testing` (integration fixtures and snapshots). + +## Interfaces & Contracts +- Job kinds: `ics-cisa:fetch`, `ics-cisa:parse`, `ics-cisa:map`. +- Persist upstream caching metadata (ETag/Last-Modified) when available. +- Alias set should include CISA ICS advisory IDs and referenced CVE IDs. + +## In/Out of scope +In scope: +- ICS-specific advisories from CISA. +- Range primitives capturing vendor/equipment metadata. + +Out of scope: +- General CISA alerts (covered elsewhere). + +## Observability & Security Expectations +- Log fetch attempts, advisory counts, and mapping results. +- Sanitize HTML, removing scripts/styles before persistence. +- Honour upstream rate limits with exponential backoff. + +## Tests +- Add `StellaOps.Concelier.Connector.Ics.Cisa.Tests` to cover fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Configuration/IcsCisaOptions.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Configuration/IcsCisaOptions.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Configuration/IcsCisaOptions.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Configuration/IcsCisaOptions.cs index eee231d0..607600af 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Configuration/IcsCisaOptions.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Configuration/IcsCisaOptions.cs @@ -1,182 +1,182 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Net; -using System.Net.Http; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Configuration; - -public sealed class IcsCisaOptions -{ - public static string HttpClientName => "source.ics.cisa"; - - /// - /// GovDelivery topics RSS endpoint. Feed URIs are constructed from this base. - /// - public Uri TopicsEndpoint { get; set; } = new("https://content.govdelivery.com/accounts/USDHSCISA/topics.rss", UriKind.Absolute); - - /// - /// GovDelivery personalised subscription code (code=...). - /// - public string GovDeliveryCode { get; set; } = string.Empty; - - /// - /// Topic identifiers to pull (e.g. USDHSCISA_16 for general ICS advisories). - /// - public IList TopicIds { get; } = new List - { - "USDHSCISA_16", // ICS advisories (ICSA) - "USDHSCISA_19", // ICS medical advisories (ICSMA) - "USDHSCISA_17", // ICS alerts - }; - - /// - /// Optional delay between sequential topic fetches to appease GovDelivery throttling. - /// - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - public TimeSpan DocumentExpiry { get; set; } = TimeSpan.FromDays(30); - - /// - /// Optional proxy endpoint used when Akamai blocks direct pulls. - /// - public Uri? ProxyUri { get; set; } - - /// - /// HTTP version requested when contacting GovDelivery. - /// - public Version RequestVersion { get; set; } = HttpVersion.Version11; - - /// - /// Negotiation policy applied to HTTP requests. - /// - public HttpVersionPolicy RequestVersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower; - - /// - /// Maximum number of retry attempts for RSS fetches. - /// - public int MaxAttempts { get; set; } = 4; - - /// - /// Base delay used for exponential backoff between attempts. - /// - public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(3); - - /// - /// Base URI used when fetching HTML detail pages. - /// - public Uri DetailBaseUri { get; set; } = new("https://www.cisa.gov/", UriKind.Absolute); - - /// - /// Optional timeout override applied to detail page fetches. - /// - public TimeSpan DetailRequestTimeout { get; set; } = TimeSpan.FromSeconds(25); - - /// - /// Additional hosts allowed by the connector (detail pages, attachments). - /// - public IList AdditionalHosts { get; } = new List - { - "www.cisa.gov", - "cisa.gov" - }; - - public bool EnableDetailScrape { get; set; } = true; - - public bool CaptureAttachments { get; set; } = true; - - [MemberNotNull(nameof(TopicsEndpoint), nameof(GovDeliveryCode))] - public void Validate() - { - if (TopicsEndpoint is null || !TopicsEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("TopicsEndpoint must be an absolute URI."); - } - - if (string.IsNullOrWhiteSpace(GovDeliveryCode)) - { - throw new InvalidOperationException("GovDeliveryCode must be provided."); - } - - if (TopicIds.Count == 0) - { - throw new InvalidOperationException("At least one GovDelivery topic identifier is required."); - } - - foreach (var topic in TopicIds) - { - if (string.IsNullOrWhiteSpace(topic)) - { - throw new InvalidOperationException("Topic identifiers cannot be blank."); - } - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("RequestDelay cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("FailureBackoff must be greater than zero."); - } - - if (DocumentExpiry <= TimeSpan.Zero) - { - throw new InvalidOperationException("DocumentExpiry must be greater than zero."); - } - - if (MaxAttempts <= 0) - { - throw new InvalidOperationException("MaxAttempts must be positive."); - } - - if (BaseDelay <= TimeSpan.Zero) - { - throw new InvalidOperationException("BaseDelay must be greater than zero."); - } - - if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri) - { - throw new InvalidOperationException("DetailBaseUri must be an absolute URI."); - } - - if (DetailRequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("DetailRequestTimeout must be greater than zero."); - } - - if (ProxyUri is not null && !ProxyUri.IsAbsoluteUri) - { - throw new InvalidOperationException("ProxyUri must be an absolute URI when specified."); - } - - foreach (var host in AdditionalHosts) - { - if (string.IsNullOrWhiteSpace(host)) - { - throw new InvalidOperationException("Additional host entries cannot be blank."); - } - } - } - - public Uri BuildTopicUri(string topicId) - { - ArgumentException.ThrowIfNullOrEmpty(topicId); - Validate(); - - var builder = new UriBuilder(TopicsEndpoint); - var query = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["code"] = GovDeliveryCode, - ["format"] = "xml", - ["topic_id"] = topicId.Trim(), - }; - - builder.Query = string.Join("&", query.Select(pair => $"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}")); - return builder.Uri; - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Net; +using System.Net.Http; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Configuration; + +public sealed class IcsCisaOptions +{ + public static string HttpClientName => "source.ics.cisa"; + + /// + /// GovDelivery topics RSS endpoint. Feed URIs are constructed from this base. + /// + public Uri TopicsEndpoint { get; set; } = new("https://content.govdelivery.com/accounts/USDHSCISA/topics.rss", UriKind.Absolute); + + /// + /// GovDelivery personalised subscription code (code=...). + /// + public string GovDeliveryCode { get; set; } = string.Empty; + + /// + /// Topic identifiers to pull (e.g. USDHSCISA_16 for general ICS advisories). + /// + public IList TopicIds { get; } = new List + { + "USDHSCISA_16", // ICS advisories (ICSA) + "USDHSCISA_19", // ICS medical advisories (ICSMA) + "USDHSCISA_17", // ICS alerts + }; + + /// + /// Optional delay between sequential topic fetches to appease GovDelivery throttling. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + public TimeSpan DocumentExpiry { get; set; } = TimeSpan.FromDays(30); + + /// + /// Optional proxy endpoint used when Akamai blocks direct pulls. + /// + public Uri? ProxyUri { get; set; } + + /// + /// HTTP version requested when contacting GovDelivery. + /// + public Version RequestVersion { get; set; } = HttpVersion.Version11; + + /// + /// Negotiation policy applied to HTTP requests. + /// + public HttpVersionPolicy RequestVersionPolicy { get; set; } = HttpVersionPolicy.RequestVersionOrLower; + + /// + /// Maximum number of retry attempts for RSS fetches. + /// + public int MaxAttempts { get; set; } = 4; + + /// + /// Base delay used for exponential backoff between attempts. + /// + public TimeSpan BaseDelay { get; set; } = TimeSpan.FromSeconds(3); + + /// + /// Base URI used when fetching HTML detail pages. + /// + public Uri DetailBaseUri { get; set; } = new("https://www.cisa.gov/", UriKind.Absolute); + + /// + /// Optional timeout override applied to detail page fetches. + /// + public TimeSpan DetailRequestTimeout { get; set; } = TimeSpan.FromSeconds(25); + + /// + /// Additional hosts allowed by the connector (detail pages, attachments). + /// + public IList AdditionalHosts { get; } = new List + { + "www.cisa.gov", + "cisa.gov" + }; + + public bool EnableDetailScrape { get; set; } = true; + + public bool CaptureAttachments { get; set; } = true; + + [MemberNotNull(nameof(TopicsEndpoint), nameof(GovDeliveryCode))] + public void Validate() + { + if (TopicsEndpoint is null || !TopicsEndpoint.IsAbsoluteUri) + { + throw new InvalidOperationException("TopicsEndpoint must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(GovDeliveryCode)) + { + throw new InvalidOperationException("GovDeliveryCode must be provided."); + } + + if (TopicIds.Count == 0) + { + throw new InvalidOperationException("At least one GovDelivery topic identifier is required."); + } + + foreach (var topic in TopicIds) + { + if (string.IsNullOrWhiteSpace(topic)) + { + throw new InvalidOperationException("Topic identifiers cannot be blank."); + } + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be greater than zero."); + } + + if (DocumentExpiry <= TimeSpan.Zero) + { + throw new InvalidOperationException("DocumentExpiry must be greater than zero."); + } + + if (MaxAttempts <= 0) + { + throw new InvalidOperationException("MaxAttempts must be positive."); + } + + if (BaseDelay <= TimeSpan.Zero) + { + throw new InvalidOperationException("BaseDelay must be greater than zero."); + } + + if (DetailBaseUri is null || !DetailBaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("DetailBaseUri must be an absolute URI."); + } + + if (DetailRequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("DetailRequestTimeout must be greater than zero."); + } + + if (ProxyUri is not null && !ProxyUri.IsAbsoluteUri) + { + throw new InvalidOperationException("ProxyUri must be an absolute URI when specified."); + } + + foreach (var host in AdditionalHosts) + { + if (string.IsNullOrWhiteSpace(host)) + { + throw new InvalidOperationException("Additional host entries cannot be blank."); + } + } + } + + public Uri BuildTopicUri(string topicId) + { + ArgumentException.ThrowIfNullOrEmpty(topicId); + Validate(); + + var builder = new UriBuilder(TopicsEndpoint); + var query = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["code"] = GovDeliveryCode, + ["format"] = "xml", + ["topic_id"] = topicId.Trim(), + }; + + builder.Query = string.Join("&", query.Select(pair => $"{Uri.EscapeDataString(pair.Key)}={Uri.EscapeDataString(pair.Value)}")); + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/HANDOVER.md b/src/StellaOps.Concelier.Connector.Ics.Cisa/HANDOVER.md similarity index 71% rename from src/StellaOps.Feedser.Source.Ics.Cisa/HANDOVER.md rename to src/StellaOps.Concelier.Connector.Ics.Cisa/HANDOVER.md index 6f5ed806..8b1c3eab 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/HANDOVER.md +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/HANDOVER.md @@ -1,21 +1,21 @@ -# ICS CISA Connector – Status (2025-10-16) - -## Context -- Proxy plumbing for GovDelivery (`SourceHttpClientOptions.Proxy*`) is implemented and covered by `SourceHttpClientBuilderTests.AddSourceHttpClient_LoadsProxyConfiguration`. -- Detail enrichment now extracts mitigation paragraphs/bullets, merges them with feed data, and emits `mitigation` references plus combined alias sets. -- `BuildAffectedPackages` parses product/version pairs and now persists SemVer exact values for canonical ranges via the advisory store. - -## Current Outcomes -- Feed parser fixtures were refreshed so vendor PDFs stay surfaced as attachments; DTO references continue including canonical links. -- SemVer primitive deserialisation now restores `exactValue` (e.g., `"4.2"` → `"4.2.0"`), keeping connector snapshots deterministic. -- Console debugging noise was removed from connector/parser code. -- Ops runbook documents attachment + SemVer validation steps for dry runs. -- `dotnet test src/StellaOps.Feedser.Source.Ics.Cisa.Tests/StellaOps.Feedser.Source.Ics.Cisa.Tests.csproj` passes (2025-10-16). - -## Outstanding Items -- None. Continue monitoring Akamai access decisions and proxy requirements via Ops feedback. - -## Verification Checklist -- ✅ `dotnet test src/StellaOps.Feedser.Source.Ics.Cisa.Tests/StellaOps.Feedser.Source.Ics.Cisa.Tests.csproj` -- ☐ `dotnet test src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj` (proxy support) — rerun when Source.Common changes land. -- Keep this summary aligned with `TASKS.md` as further work emerges. +# ICS CISA Connector – Status (2025-10-16) + +## Context +- Proxy plumbing for GovDelivery (`SourceHttpClientOptions.Proxy*`) is implemented and covered by `SourceHttpClientBuilderTests.AddSourceHttpClient_LoadsProxyConfiguration`. +- Detail enrichment now extracts mitigation paragraphs/bullets, merges them with feed data, and emits `mitigation` references plus combined alias sets. +- `BuildAffectedPackages` parses product/version pairs and now persists SemVer exact values for canonical ranges via the advisory store. + +## Current Outcomes +- Feed parser fixtures were refreshed so vendor PDFs stay surfaced as attachments; DTO references continue including canonical links. +- SemVer primitive deserialisation now restores `exactValue` (e.g., `"4.2"` → `"4.2.0"`), keeping connector snapshots deterministic. +- Console debugging noise was removed from connector/parser code. +- Ops runbook documents attachment + SemVer validation steps for dry runs. +- `dotnet test src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/StellaOps.Concelier.Connector.Ics.Cisa.Tests.csproj` passes (2025-10-16). + +## Outstanding Items +- None. Continue monitoring Akamai access decisions and proxy requirements via Ops feedback. + +## Verification Checklist +- ✅ `dotnet test src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/StellaOps.Concelier.Connector.Ics.Cisa.Tests.csproj` +- ☐ `dotnet test src/StellaOps.Concelier.Connector.Common.Tests/StellaOps.Concelier.Connector.Common.Tests.csproj` (proxy support) — rerun when Source.Common changes land. +- Keep this summary aligned with `TASKS.md` as further work emerges. diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnector.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnector.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnector.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnector.cs index e00825cd..0c133d01 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnector.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnector.cs @@ -1,1248 +1,1248 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using AngleSharp.Html.Dom; -using AngleSharp.Html.Parser; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Source.Ics.Cisa.Configuration; -using StellaOps.Feedser.Source.Ics.Cisa.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ics.Cisa; - -public sealed class IcsCisaConnector : IFeedConnector -{ - private const string SchemaVersion = "ics.cisa.feed.v1"; - - private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" }; - private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" }; - private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly IcsCisaOptions _options; - private readonly IcsCisaFeedParser _parser; - private readonly IcsCisaDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly HtmlContentSanitizer _htmlSanitizer = new(); - private readonly HtmlParser _htmlParser = new(); - - public IcsCisaConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - IcsCisaFeedParser parser, - IcsCisaDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _parser = parser ?? throw new ArgumentNullException(nameof(parser)); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => IcsCisaConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var now = _timeProvider.GetUtcNow(); - var touched = false; - - foreach (var topic in _options.TopicIds) - { - cancellationToken.ThrowIfCancellationRequested(); - - _diagnostics.FetchAttempt(topic); - var topicUri = _options.BuildTopicUri(topic); - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, topicUri.ToString(), cancellationToken).ConfigureAwait(false); - - var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, topicUri) - { - AcceptHeaders = RssAcceptHeaders, - Metadata = new Dictionary(StringComparer.Ordinal) - { - ["icscisa.topicId"] = topic, - }, - }; - - if (existing is not null) - { - request = request with - { - ETag = existing.Etag, - LastModified = existing.LastModified, - }; - } - - SourceFetchResult? result = null; - var documentsAdded = 0; - var usedFallback = false; - - try - { - result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); - } - catch (HttpRequestException ex) when (ShouldRetryWithFallback(ex)) - { - _logger.LogWarning(ex, "Retrying CISA ICS topic {TopicId} via Akamai fallback", topic); - _diagnostics.FetchFallback(topic); - usedFallback = true; - var fallbackRequest = request with - { - AcceptHeaders = RssFallbackAcceptHeaders, - Metadata = AppendMetadata(request.Metadata, "icscisa.retry", "akamai"), - }; - - try - { - result = await _fetchService.FetchAsync(fallbackRequest, cancellationToken).ConfigureAwait(false); - } - catch (Exception fallbackEx) when (fallbackEx is HttpRequestException or TaskCanceledException) - { - _diagnostics.FetchFailure(topic); - _logger.LogError(fallbackEx, "Fallback fetch failed for CISA ICS topic {TopicId}", topic); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, fallbackEx.Message, cancellationToken).ConfigureAwait(false); - throw; - } - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - _diagnostics.FetchFailure(topic); - _logger.LogError(ex, "Failed to fetch CISA ICS topic {TopicId}", topic); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (result is null) - { - _diagnostics.FetchFailure(topic); - continue; - } - - if (result.IsNotModified) - { - _diagnostics.FetchNotModified(topic); - _logger.LogDebug("CISA ICS topic {TopicId} not modified", topic); - } - else if (result.IsSuccess && result.Document is not null) - { - pendingDocuments.Add(result.Document.Id); - pendingMappings.Remove(result.Document.Id); - touched = true; - documentsAdded++; - _diagnostics.FetchSuccess(topic, 1); - _logger.LogInformation("Fetched CISA ICS topic {TopicId} document {DocumentId}", topic, result.Document.Id); - } - else if (result.IsSuccess) - { - _diagnostics.FetchSuccess(topic, 0); - _logger.LogDebug("CISA ICS topic {TopicId} fetch succeeded without new document (fallback={Fallback})", topic, usedFallback); - } - else - { - _diagnostics.FetchFailure(topic); - _logger.LogWarning("CISA ICS topic {TopicId} returned status {StatusCode}", topic, result.StatusCode); - } - - if (documentsAdded > 0) - { - _logger.LogInformation("CISA ICS topic {TopicId} added {DocumentsAdded} document(s) (fallbackUsed={Fallback})", topic, documentsAdded, usedFallback); - } - - if (_options.RequestDelay > TimeSpan.Zero) - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - } - - if (!touched) - { - await UpdateCursorAsync(cursor.WithPendingDocuments(pendingDocuments).WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false); - return; - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - DateTimeOffset? latestPublished = cursor.LastPublished; - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var topicId = "unknown"; - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(topicId); - continue; - } - - if (document.Metadata is not null && document.Metadata.TryGetValue("icscisa.topicId", out var topicValue)) - { - topicId = topicValue; - } - - if (!document.GridFsId.HasValue) - { - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(topicId); - continue; - } - - byte[] bytes; - try - { - bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to download CISA ICS payload {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(topicId); - continue; - } - - IReadOnlyCollection advisories; - try - { - using var stream = new MemoryStream(bytes, writable: false); - var topicUri = Uri.TryCreate(document.Uri, UriKind.Absolute, out var parsed) ? parsed : null; - advisories = _parser.Parse(stream, string.Equals(topicId, "USDHSCISA_19", StringComparison.OrdinalIgnoreCase), parsed); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to parse CISA ICS feed {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(topicId); - continue; - } - - var advisoryList = advisories.ToList(); - var detailAttempts = 0; - if (_options.EnableDetailScrape) - { - var enriched = new List(advisoryList.Count); - foreach (var advisory in advisoryList) - { - if (NeedsDetailFetch(advisory)) - { - detailAttempts++; - } - var enrichedAdvisory = await EnrichAdvisoryAsync(advisory, cancellationToken).ConfigureAwait(false); - enriched.Add(enrichedAdvisory); - } - - advisoryList = enriched; - } - - var attachmentTotal = advisoryList.Sum(static advisory => advisory.Attachments is null ? 0 : advisory.Attachments.Count); - - var feedDto = new IcsCisaFeedDto - { - TopicId = topicId, - FeedUri = document.Uri, - Advisories = advisoryList, - }; - - try - { - var json = JsonSerializer.Serialize(feedDto, new JsonSerializerOptions(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = false, - }); - var bson = BsonDocument.Parse(json); - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - document.Id, - SourceName, - SchemaVersion, - bson, - _timeProvider.GetUtcNow()); - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - pendingMappings.Add(document.Id); - - var docPublished = advisoryList.Count > 0 ? advisoryList.Max(a => a.Published) : (DateTimeOffset?)null; - if (docPublished.HasValue && docPublished > latestPublished) - { - latestPublished = docPublished; - } - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to persist CISA ICS DTO {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - _diagnostics.ParseFailure(topicId); - continue; - } - - _diagnostics.ParseSuccess(topicId, advisoryList.Count, attachmentTotal, detailAttempts); - _logger.LogInformation( - "CISA ICS parse produced advisories={Advisories} attachments={Attachments} detailAttempts={DetailAttempts} topic={TopicId}", - advisoryList.Count, - attachmentTotal, - detailAttempts, - topicId); - } - - var updatedCursor = cursor - .WithPendingDocuments(remainingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastPublished(latestPublished); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - pendingMappings.Remove(documentId); - _diagnostics.MapFailure("unknown"); - continue; - } - - IcsCisaFeedDto? feedDto; - try - { - var json = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson }); - feedDto = JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web)); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to deserialize CISA ICS DTO {DtoId}", dtoRecord.Id); - pendingMappings.Remove(documentId); - await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - _diagnostics.MapFailure("unknown"); - continue; - } - - if (feedDto is null) - { - pendingMappings.Remove(documentId); - await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - _diagnostics.MapFailure("unknown"); - continue; - } - - var allMapped = true; - var mappedCount = 0; - foreach (var advisoryDto in feedDto.Advisories) - { - try - { - var advisory = MapAdvisory(dtoRecord, feedDto, advisoryDto); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - _diagnostics.MapSuccess( - advisoryDto.AdvisoryId, - advisory.References.Length, - advisory.AffectedPackages.Length, - advisory.Aliases.Length); - mappedCount++; - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to map CISA ICS advisory {AdvisoryId}", advisoryDto.AdvisoryId); - _diagnostics.MapFailure(advisoryDto.AdvisoryId); - allMapped = false; - } - } - - pendingMappings.Remove(documentId); - - if (!allMapped) - { - _logger.LogWarning( - "CISA ICS mapping failed for document {DocumentId} (mapped={MappedCount} of {Total})", - documentId, - mappedCount, - feedDto.Advisories.Count); - await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - continue; - } - - await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - _logger.LogInformation("CISA ICS mapped {MappedCount} advisories from document {DocumentId}", mappedCount, documentId); - } - - await UpdateCursorAsync(cursor.WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false); - } - - private Advisory MapAdvisory(DtoRecord dtoRecord, IcsCisaFeedDto feedDto, IcsCisaAdvisoryDto advisoryDto) - { - var recordedAt = dtoRecord.ValidatedAt; - var fetchProvenance = new AdvisoryProvenance(SourceName, "feed", feedDto.FeedUri, recordedAt); - var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryId, _timeProvider.GetUtcNow()); - - var aliases = CombineAliases(advisoryDto); - var references = BuildReferences(advisoryDto, recordedAt).ToList(); - var mitigationReferences = BuildMitigationReferences(advisoryDto, recordedAt); - if (mitigationReferences.Count > 0) - { - references.AddRange(mitigationReferences); - } - - var affectedPackages = BuildAffectedPackages(advisoryDto, recordedAt); - - return new Advisory( - advisoryDto.AdvisoryId, - advisoryDto.Title, - advisoryDto.Summary, - language: "en", - published: advisoryDto.Published, - modified: advisoryDto.Updated ?? advisoryDto.Published, - severity: null, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affectedPackages, - cvssMetrics: Array.Empty(), - provenance: new[] { fetchProvenance, mappingProvenance }); - } - - internal static IReadOnlyCollection CombineAliases(IcsCisaAdvisoryDto advisoryDto) - { - var set = new HashSet(StringComparer.OrdinalIgnoreCase); - - if (advisoryDto.Aliases is not null) - { - foreach (var alias in advisoryDto.Aliases) - { - if (string.IsNullOrWhiteSpace(alias)) - { - continue; - } - - set.Add(alias.Trim()); - } - } - - if (advisoryDto.CveIds is not null) - { - foreach (var cve in advisoryDto.CveIds) - { - if (string.IsNullOrWhiteSpace(cve)) - { - continue; - } - - set.Add(cve.Trim()); - } - } - - return set.Count == 0 - ? Array.Empty() - : set.OrderBy(static value => value, StringComparer.Ordinal).ToArray(); - } - - internal static IReadOnlyCollection BuildMitigationReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) - { - if (advisoryDto.Mitigations is null || advisoryDto.Mitigations.Count == 0) - { - return Array.Empty(); - } - - var references = new List(); - var baseUrl = Validation.LooksLikeHttpUrl(advisoryDto.Link) ? advisoryDto.Link : null; - var sourceTag = advisoryDto.IsMedical ? "icscisa-medical-mitigation" : "icscisa-mitigation"; - - var index = 0; - foreach (var mitigation in advisoryDto.Mitigations) - { - index++; - if (string.IsNullOrWhiteSpace(mitigation)) - { - continue; - } - - var summary = mitigation.Trim(); - var url = baseUrl is not null - ? $"{baseUrl}#mitigation-{index}" - : $"icscisa:mitigation:{advisoryDto.AdvisoryId}:{index}"; - - references.Add(new AdvisoryReference( - url, - kind: "mitigation", - sourceTag: sourceTag, - summary: summary, - provenance: new AdvisoryProvenance("ics-cisa", "mitigation", url, recordedAt))); - } - - return references.Count == 0 ? Array.Empty() : references; - } - - internal static IReadOnlyCollection BuildReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) - { - var references = new List(); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - if (advisoryDto.Attachments is { Count: > 0 }) - { - foreach (var attachment in advisoryDto.Attachments) - { - if (attachment is null || !Validation.LooksLikeHttpUrl(attachment.Url)) - { - continue; - } - - var url = attachment.Url; - if (!seen.Add(url)) - { - continue; - } - - try - { - references.Add(new AdvisoryReference( - url, - kind: "attachment", - sourceTag: advisoryDto.IsMedical ? "icscisa-medical-attachment" : "icscisa-attachment", - summary: attachment.Title, - provenance: new AdvisoryProvenance("ics-cisa", "attachment", url, recordedAt))); - } - catch (ArgumentException) - { - // ignore invalid URIs - } - } - } - - foreach (var reference in advisoryDto.References ?? Array.Empty()) - { - if (!Validation.LooksLikeHttpUrl(reference)) - { - continue; - } - - if (!seen.Add(reference)) - { - continue; - } - - try - { - references.Add(new AdvisoryReference( - reference, - kind: "advisory", - sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa", - summary: null, - provenance: new AdvisoryProvenance("ics-cisa", "reference", reference, recordedAt))); - } - catch (ArgumentException) - { - // ignore invalid URIs - } - } - - if (references.Count == 0 && Validation.LooksLikeHttpUrl(advisoryDto.Link) && seen.Add(advisoryDto.Link)) - { - references.Add(new AdvisoryReference( - advisoryDto.Link, - kind: "advisory", - sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa", - summary: null, - provenance: new AdvisoryProvenance("ics-cisa", "reference", advisoryDto.Link, recordedAt))); - } - - return references; - } - - internal static IReadOnlyCollection BuildAffectedPackages(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) - { - var packages = new List(); - var vendors = advisoryDto.Vendors ?? Array.Empty(); - var normalizedVendors = vendors - .Where(static vendor => !string.IsNullOrWhiteSpace(vendor)) - .Select(static vendor => vendor.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - var parsedProducts = (advisoryDto.Products ?? Array.Empty()) - .Where(static product => !string.IsNullOrWhiteSpace(product)) - .Select(ParseProductInfo) - .Where(static product => !string.IsNullOrWhiteSpace(product.Name)) - .ToArray(); - - if (parsedProducts.Length > 0) - { - foreach (var product in parsedProducts) - { - } - - foreach (var product in parsedProducts) - { - var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt); - var vendorExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["ics.product"] = product.Name! - }; - - if (!string.IsNullOrWhiteSpace(product.VersionExpression)) - { - vendorExtensions["ics.version"] = product.VersionExpression!; - } - - if (normalizedVendors.Length > 0) - { - vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors); - } - - var semVer = TryCreateSemVerPrimitive(product.VersionExpression); - var range = new AffectedVersionRange( - rangeKind: "product", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: product.VersionExpression, - provenance: provenance, - primitives: new RangePrimitives(semVer, null, null, vendorExtensions)); - - packages.Add(new AffectedPackage( - AffectedPackageTypes.IcsVendor, - product.Name!, - platform: null, - versionRanges: new[] { range }, - statuses: Array.Empty(), - provenance: new[] { provenance })); - } - - return packages; - } - - if (normalizedVendors.Length == 0) - { - return packages; - } - - foreach (var vendor in normalizedVendors) - { - var provenance = new AdvisoryProvenance("ics-cisa", "affected", vendor, recordedAt); - var vendorExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["ics.vendor"] = vendor - }; - - var range = new AffectedVersionRange( - rangeKind: "vendor", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: null, - provenance: provenance, - primitives: new RangePrimitives(null, null, null, vendorExtensions)); - - packages.Add(new AffectedPackage( - AffectedPackageTypes.IcsVendor, - vendor, - platform: null, - versionRanges: new[] { range }, - statuses: Array.Empty(), - provenance: new[] { provenance })); - } - - return packages; - } - - - private static ProductInfo ParseProductInfo(string raw) - { - var trimmed = raw?.Trim(); - if (string.IsNullOrWhiteSpace(trimmed)) - { - return new ProductInfo(null, null); - } - - if (trimmed.Contains(':', StringComparison.Ordinal)) - { - var parts = trimmed.Split(':', 2); - var name = parts[0].Trim(); - var versionSegment = parts[1].Trim(); - return new ProductInfo( - string.IsNullOrWhiteSpace(name) ? trimmed : name, - string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment); - } - - var lastSpace = trimmed.LastIndexOf(' '); - if (lastSpace > 0) - { - var candidateVersion = trimmed[(lastSpace + 1)..].Trim(); - if (Regex.IsMatch(candidateVersion, "^[vV]?[0-9].*")) - { - var name = trimmed[..lastSpace].Trim(); - return new ProductInfo( - string.IsNullOrWhiteSpace(name) ? trimmed : name, - candidateVersion); - } - } - - return new ProductInfo(trimmed, null); - } - - private static SemVerPrimitive? TryCreateSemVerPrimitive(string? versionExpression) - { - if (string.IsNullOrWhiteSpace(versionExpression)) - { - return null; - } - - var normalized = NormalizeSemVer(versionExpression); - if (normalized is null) - { - var trimmed = versionExpression.Trim(); - if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) - { - trimmed = trimmed[1..]; - } - - if (Version.TryParse(trimmed, out var parsed)) - { - normalized = string.Join('.', new[] - { - parsed.Major.ToString(CultureInfo.InvariantCulture), - parsed.Minor >= 0 ? parsed.Minor.ToString(CultureInfo.InvariantCulture) : "0", - parsed.Build >= 0 ? parsed.Build.ToString(CultureInfo.InvariantCulture) : "0", - }); - } - } - - if (normalized is null) - { - return null; - } - - return new SemVerPrimitive( - null, - true, - null, - true, - null, - true, - null, - normalized); - } - - private static string? NormalizeSemVer(string rawVersion) - { - var trimmed = rawVersion.Trim(); - if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) - { - trimmed = trimmed[1..]; - } - - if (!Regex.IsMatch(trimmed, @"^[0-9]+(\.[0-9]+){0,2}$")) - { - return null; - } - - var parts = trimmed.Split('.', StringSplitOptions.RemoveEmptyEntries); - var components = parts.Take(3).ToList(); - while (components.Count < 3) - { - components.Add("0"); - } - - return string.Join('.', components); - } - - private sealed record ProductInfo(string? Name, string? VersionExpression); - - private async Task EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken) - { - if (!NeedsDetailFetch(advisory)) - { - return advisory; - } - - if (!Uri.TryCreate(advisory.Link, UriKind.Absolute, out var detailUri)) - { - return advisory; - } - - var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, detailUri) - { - AcceptHeaders = DetailAcceptHeaders, - Metadata = AppendMetadata(null, "icscisa.detail", advisory.AdvisoryId), - TimeoutOverride = _options.DetailRequestTimeout, - }; - - try - { - var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - if (!result.IsSuccess || result.Content is null) - { - _diagnostics.DetailFetchFailure(advisory.AdvisoryId); - return advisory; - } - - var html = Encoding.UTF8.GetString(result.Content); - var sanitized = _htmlSanitizer.Sanitize(html, detailUri); - if (string.IsNullOrWhiteSpace(sanitized)) - { - _diagnostics.DetailFetchSuccess(advisory.AdvisoryId); - return advisory with { DetailHtml = sanitized }; - } - - var detailAttachments = _options.CaptureAttachments - ? ParseAttachmentsFromHtml(sanitized, detailUri) - : Array.Empty(); - var mergedAttachments = _options.CaptureAttachments - ? MergeAttachments(advisory.Attachments, detailAttachments) - : advisory.Attachments; - - var detailMitigations = ParseMitigationsFromHtml(sanitized); - var mergedMitigations = MergeMitigations(advisory.Mitigations, detailMitigations); - - var detailReferences = ParseReferencesFromHtml(sanitized, detailUri); - var mergedReferences = MergeReferences(advisory.References, detailReferences); - - var summary = string.IsNullOrWhiteSpace(advisory.Summary) - ? ExtractFirstSentence(sanitized) - : advisory.Summary; - - var descriptionHtml = string.IsNullOrWhiteSpace(advisory.DescriptionHtml) - ? sanitized - : advisory.DescriptionHtml; - - return advisory with - { - DetailHtml = sanitized, - DescriptionHtml = descriptionHtml, - Summary = summary, - References = mergedReferences, - Attachments = mergedAttachments, - Mitigations = mergedMitigations, - }; - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - _logger.LogWarning(ex, "Failed to fetch detail page for {AdvisoryId}", advisory.AdvisoryId); - _diagnostics.DetailFetchFailure(advisory.AdvisoryId); - return advisory; - } - } - - private bool NeedsDetailFetch(IcsCisaAdvisoryDto advisory) - { - if (!_options.EnableDetailScrape) - { - return false; - } - - if (string.IsNullOrWhiteSpace(advisory.DescriptionHtml)) - { - return true; - } - - if (string.IsNullOrWhiteSpace(advisory.Summary)) - { - return true; - } - - if (advisory.Mitigations is null || advisory.Mitigations.Count == 0) - { - return true; - } - - if (_options.CaptureAttachments && (advisory.Attachments is null || advisory.Attachments.Count == 0)) - { - return true; - } - - return false; - } - - private IReadOnlyCollection ParseMitigationsFromHtml(string sanitizedHtml) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return Array.Empty(); - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var mitigations = new List(); - - foreach (var heading in document.QuerySelectorAll("h1, h2, h3, h4, h5, h6")) - { - var headingText = heading.TextContent?.Trim(); - if (!IsMitigationHeading(headingText)) - { - continue; - } - - var node = heading.NextElementSibling; - while (node is not null && node is not IHtmlHeadingElement) - { - if (node is IHtmlParagraphElement or IHtmlDivElement) - { - var content = Validation.CollapseWhitespace(node.TextContent); - if (!string.IsNullOrWhiteSpace(content)) - { - mitigations.Add(content); - } - } - else if (node is IHtmlElement element && (string.Equals(element.TagName, "UL", StringComparison.OrdinalIgnoreCase) || string.Equals(element.TagName, "OL", StringComparison.OrdinalIgnoreCase))) - { - foreach (var item in element.Children) - { - var content = Validation.CollapseWhitespace(item.TextContent); - if (!string.IsNullOrWhiteSpace(content)) - { - mitigations.Add(content); - } - } - } - - node = node.NextElementSibling; - } - } - - return mitigations.Count == 0 ? Array.Empty() : mitigations; - } - catch - { - return Array.Empty(); - } - } - - private static bool IsMitigationHeading(string? headingText) - { - if (string.IsNullOrWhiteSpace(headingText)) - { - return false; - } - - return headingText.Contains("mitigation", StringComparison.OrdinalIgnoreCase); - } - - private IReadOnlyCollection ParseAttachmentsFromHtml(string sanitizedHtml, Uri baseUri) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return Array.Empty(); - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var attachments = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var anchor in document.QuerySelectorAll("a")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!Uri.TryCreate(baseUri, href, out var resolved)) - { - continue; - } - - var url = resolved.ToString(); - if (!url.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase) && - !url.Contains("/pdf", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - attachments[url] = new IcsCisaAttachmentDto - { - Title = anchor.TextContent?.Trim(), - Url = url, - }; - } - - return attachments.Count == 0 - ? Array.Empty() - : attachments.Values.ToArray(); - } - catch - { - return Array.Empty(); - } - } - - private IReadOnlyCollection ParseReferencesFromHtml(string sanitizedHtml, Uri baseUri) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return Array.Empty(); - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var links = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var anchor in document.QuerySelectorAll("a")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (Uri.TryCreate(baseUri, href, out var resolved) && Validation.LooksLikeHttpUrl(resolved.ToString())) - { - links.Add(resolved.ToString()); - } - } - - return links.Count == 0 ? Array.Empty() : links.ToArray(); - } - catch - { - return Array.Empty(); - } - } - - internal static IReadOnlyCollection MergeMitigations(IReadOnlyCollection? existing, IReadOnlyCollection incoming) - { - if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0)) - { - return Array.Empty(); - } - - var set = new HashSet(StringComparer.OrdinalIgnoreCase); - var merged = new List(); - - if (existing is not null) - { - foreach (var mitigation in existing) - { - var value = mitigation?.Trim(); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - if (set.Add(value)) - { - merged.Add(value); - } - } - } - - if (incoming is not null) - { - foreach (var mitigation in incoming) - { - var value = mitigation?.Trim(); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - if (set.Add(value)) - { - merged.Add(value); - } - } - } - - return merged.Count == 0 ? Array.Empty() : merged; - } - - internal static IReadOnlyCollection MergeAttachments(IReadOnlyCollection? existing, IReadOnlyCollection incoming) - { - if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0)) - { - return Array.Empty(); - } - - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - - if (existing is not null) - { - foreach (var attachment in existing) - { - if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url)) - { - continue; - } - - map[attachment.Url] = attachment; - } - } - - if (incoming is not null) - { - foreach (var attachment in incoming) - { - if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url)) - { - continue; - } - - if (!map.ContainsKey(attachment.Url) || string.IsNullOrWhiteSpace(map[attachment.Url].Title)) - { - map[attachment.Url] = attachment; - } - } - } - - return map.Count == 0 ? Array.Empty() : map.Values.ToArray(); - } - - internal static IReadOnlyCollection MergeReferences(IReadOnlyCollection? existing, IReadOnlyCollection incoming) - { - var links = new HashSet(existing ?? Array.Empty(), StringComparer.OrdinalIgnoreCase); - foreach (var link in incoming) - { - if (Validation.LooksLikeHttpUrl(link)) - { - links.Add(link); - } - } - - return links.Count == 0 ? Array.Empty() : links.ToArray(); - } - - internal static string? ExtractFirstSentence(string sanitizedHtml) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return null; - } - - var text = Validation.CollapseWhitespace(sanitizedHtml); - if (text.Length <= 280) - { - return text; - } - - var terminator = text.IndexOf('.', StringComparison.Ordinal); - if (terminator <= 0 || terminator > 280) - { - return text[..Math.Min(280, text.Length)].Trim(); - } - - return text[..(terminator + 1)].Trim(); - } - - internal static IReadOnlyDictionary AppendMetadata(IReadOnlyDictionary? metadata, string key, string value) - { - var dictionary = new Dictionary(StringComparer.Ordinal); - - if (metadata is not null) - { - foreach (var pair in metadata) - { - dictionary[pair.Key] = pair.Value; - } - } - - dictionary[key] = value; - return dictionary; - } - - internal static bool ShouldRetryWithFallback(HttpRequestException exception) - { - var message = exception.Message ?? string.Empty; - return message.Contains(" 403", StringComparison.OrdinalIgnoreCase) - || message.Contains("403", StringComparison.OrdinalIgnoreCase) - || message.Contains("406", StringComparison.OrdinalIgnoreCase); - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? IcsCisaCursor.Empty : IcsCisaCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(IcsCisaCursor cursor, CancellationToken cancellationToken) - => _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Bson.IO; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Connector.Ics.Cisa.Configuration; +using StellaOps.Concelier.Connector.Ics.Cisa.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ics.Cisa; + +public sealed class IcsCisaConnector : IFeedConnector +{ + private const string SchemaVersion = "ics.cisa.feed.v1"; + + private static readonly string[] RssAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml" }; + private static readonly string[] RssFallbackAcceptHeaders = { "application/rss+xml", "application/xml", "text/xml", "*/*" }; + private static readonly string[] DetailAcceptHeaders = { "text/html", "application/xhtml+xml", "*/*" }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly IcsCisaOptions _options; + private readonly IcsCisaFeedParser _parser; + private readonly IcsCisaDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly HtmlContentSanitizer _htmlSanitizer = new(); + private readonly HtmlParser _htmlParser = new(); + + public IcsCisaConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + IcsCisaFeedParser parser, + IcsCisaDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _parser = parser ?? throw new ArgumentNullException(nameof(parser)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => IcsCisaConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var now = _timeProvider.GetUtcNow(); + var touched = false; + + foreach (var topic in _options.TopicIds) + { + cancellationToken.ThrowIfCancellationRequested(); + + _diagnostics.FetchAttempt(topic); + var topicUri = _options.BuildTopicUri(topic); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, topicUri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, topicUri) + { + AcceptHeaders = RssAcceptHeaders, + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["icscisa.topicId"] = topic, + }, + }; + + if (existing is not null) + { + request = request with + { + ETag = existing.Etag, + LastModified = existing.LastModified, + }; + } + + SourceFetchResult? result = null; + var documentsAdded = 0; + var usedFallback = false; + + try + { + result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException ex) when (ShouldRetryWithFallback(ex)) + { + _logger.LogWarning(ex, "Retrying CISA ICS topic {TopicId} via Akamai fallback", topic); + _diagnostics.FetchFallback(topic); + usedFallback = true; + var fallbackRequest = request with + { + AcceptHeaders = RssFallbackAcceptHeaders, + Metadata = AppendMetadata(request.Metadata, "icscisa.retry", "akamai"), + }; + + try + { + result = await _fetchService.FetchAsync(fallbackRequest, cancellationToken).ConfigureAwait(false); + } + catch (Exception fallbackEx) when (fallbackEx is HttpRequestException or TaskCanceledException) + { + _diagnostics.FetchFailure(topic); + _logger.LogError(fallbackEx, "Fallback fetch failed for CISA ICS topic {TopicId}", topic); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, fallbackEx.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _diagnostics.FetchFailure(topic); + _logger.LogError(ex, "Failed to fetch CISA ICS topic {TopicId}", topic); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result is null) + { + _diagnostics.FetchFailure(topic); + continue; + } + + if (result.IsNotModified) + { + _diagnostics.FetchNotModified(topic); + _logger.LogDebug("CISA ICS topic {TopicId} not modified", topic); + } + else if (result.IsSuccess && result.Document is not null) + { + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + touched = true; + documentsAdded++; + _diagnostics.FetchSuccess(topic, 1); + _logger.LogInformation("Fetched CISA ICS topic {TopicId} document {DocumentId}", topic, result.Document.Id); + } + else if (result.IsSuccess) + { + _diagnostics.FetchSuccess(topic, 0); + _logger.LogDebug("CISA ICS topic {TopicId} fetch succeeded without new document (fallback={Fallback})", topic, usedFallback); + } + else + { + _diagnostics.FetchFailure(topic); + _logger.LogWarning("CISA ICS topic {TopicId} returned status {StatusCode}", topic, result.StatusCode); + } + + if (documentsAdded > 0) + { + _logger.LogInformation("CISA ICS topic {TopicId} added {DocumentsAdded} document(s) (fallbackUsed={Fallback})", topic, documentsAdded, usedFallback); + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (!touched) + { + await UpdateCursorAsync(cursor.WithPendingDocuments(pendingDocuments).WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false); + return; + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + DateTimeOffset? latestPublished = cursor.LastPublished; + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var topicId = "unknown"; + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(topicId); + continue; + } + + if (document.Metadata is not null && document.Metadata.TryGetValue("icscisa.topicId", out var topicValue)) + { + topicId = topicValue; + } + + if (!document.GridFsId.HasValue) + { + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(topicId); + continue; + } + + byte[] bytes; + try + { + bytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to download CISA ICS payload {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(topicId); + continue; + } + + IReadOnlyCollection advisories; + try + { + using var stream = new MemoryStream(bytes, writable: false); + var topicUri = Uri.TryCreate(document.Uri, UriKind.Absolute, out var parsed) ? parsed : null; + advisories = _parser.Parse(stream, string.Equals(topicId, "USDHSCISA_19", StringComparison.OrdinalIgnoreCase), parsed); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to parse CISA ICS feed {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(topicId); + continue; + } + + var advisoryList = advisories.ToList(); + var detailAttempts = 0; + if (_options.EnableDetailScrape) + { + var enriched = new List(advisoryList.Count); + foreach (var advisory in advisoryList) + { + if (NeedsDetailFetch(advisory)) + { + detailAttempts++; + } + var enrichedAdvisory = await EnrichAdvisoryAsync(advisory, cancellationToken).ConfigureAwait(false); + enriched.Add(enrichedAdvisory); + } + + advisoryList = enriched; + } + + var attachmentTotal = advisoryList.Sum(static advisory => advisory.Attachments is null ? 0 : advisory.Attachments.Count); + + var feedDto = new IcsCisaFeedDto + { + TopicId = topicId, + FeedUri = document.Uri, + Advisories = advisoryList, + }; + + try + { + var json = JsonSerializer.Serialize(feedDto, new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }); + var bson = BsonDocument.Parse(json); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + SchemaVersion, + bson, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + + var docPublished = advisoryList.Count > 0 ? advisoryList.Max(a => a.Published) : (DateTimeOffset?)null; + if (docPublished.HasValue && docPublished > latestPublished) + { + latestPublished = docPublished; + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to persist CISA ICS DTO {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + _diagnostics.ParseFailure(topicId); + continue; + } + + _diagnostics.ParseSuccess(topicId, advisoryList.Count, attachmentTotal, detailAttempts); + _logger.LogInformation( + "CISA ICS parse produced advisories={Advisories} attachments={Attachments} detailAttempts={DetailAttempts} topic={TopicId}", + advisoryList.Count, + attachmentTotal, + detailAttempts, + topicId); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPublished(latestPublished); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + pendingMappings.Remove(documentId); + _diagnostics.MapFailure("unknown"); + continue; + } + + IcsCisaFeedDto? feedDto; + try + { + var json = dtoRecord.Payload.ToJson(new JsonWriterSettings { OutputMode = JsonOutputMode.RelaxedExtendedJson }); + feedDto = JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to deserialize CISA ICS DTO {DtoId}", dtoRecord.Id); + pendingMappings.Remove(documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + _diagnostics.MapFailure("unknown"); + continue; + } + + if (feedDto is null) + { + pendingMappings.Remove(documentId); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + _diagnostics.MapFailure("unknown"); + continue; + } + + var allMapped = true; + var mappedCount = 0; + foreach (var advisoryDto in feedDto.Advisories) + { + try + { + var advisory = MapAdvisory(dtoRecord, feedDto, advisoryDto); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + _diagnostics.MapSuccess( + advisoryDto.AdvisoryId, + advisory.References.Length, + advisory.AffectedPackages.Length, + advisory.Aliases.Length); + mappedCount++; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to map CISA ICS advisory {AdvisoryId}", advisoryDto.AdvisoryId); + _diagnostics.MapFailure(advisoryDto.AdvisoryId); + allMapped = false; + } + } + + pendingMappings.Remove(documentId); + + if (!allMapped) + { + _logger.LogWarning( + "CISA ICS mapping failed for document {DocumentId} (mapped={MappedCount} of {Total})", + documentId, + mappedCount, + feedDto.Advisories.Count); + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + continue; + } + + await _documentStore.UpdateStatusAsync(documentId, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("CISA ICS mapped {MappedCount} advisories from document {DocumentId}", mappedCount, documentId); + } + + await UpdateCursorAsync(cursor.WithPendingMappings(pendingMappings), cancellationToken).ConfigureAwait(false); + } + + private Advisory MapAdvisory(DtoRecord dtoRecord, IcsCisaFeedDto feedDto, IcsCisaAdvisoryDto advisoryDto) + { + var recordedAt = dtoRecord.ValidatedAt; + var fetchProvenance = new AdvisoryProvenance(SourceName, "feed", feedDto.FeedUri, recordedAt); + var mappingProvenance = new AdvisoryProvenance(SourceName, "mapping", advisoryDto.AdvisoryId, _timeProvider.GetUtcNow()); + + var aliases = CombineAliases(advisoryDto); + var references = BuildReferences(advisoryDto, recordedAt).ToList(); + var mitigationReferences = BuildMitigationReferences(advisoryDto, recordedAt); + if (mitigationReferences.Count > 0) + { + references.AddRange(mitigationReferences); + } + + var affectedPackages = BuildAffectedPackages(advisoryDto, recordedAt); + + return new Advisory( + advisoryDto.AdvisoryId, + advisoryDto.Title, + advisoryDto.Summary, + language: "en", + published: advisoryDto.Published, + modified: advisoryDto.Updated ?? advisoryDto.Published, + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affectedPackages, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mappingProvenance }); + } + + internal static IReadOnlyCollection CombineAliases(IcsCisaAdvisoryDto advisoryDto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (advisoryDto.Aliases is not null) + { + foreach (var alias in advisoryDto.Aliases) + { + if (string.IsNullOrWhiteSpace(alias)) + { + continue; + } + + set.Add(alias.Trim()); + } + } + + if (advisoryDto.CveIds is not null) + { + foreach (var cve in advisoryDto.CveIds) + { + if (string.IsNullOrWhiteSpace(cve)) + { + continue; + } + + set.Add(cve.Trim()); + } + } + + return set.Count == 0 + ? Array.Empty() + : set.OrderBy(static value => value, StringComparer.Ordinal).ToArray(); + } + + internal static IReadOnlyCollection BuildMitigationReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) + { + if (advisoryDto.Mitigations is null || advisoryDto.Mitigations.Count == 0) + { + return Array.Empty(); + } + + var references = new List(); + var baseUrl = Validation.LooksLikeHttpUrl(advisoryDto.Link) ? advisoryDto.Link : null; + var sourceTag = advisoryDto.IsMedical ? "icscisa-medical-mitigation" : "icscisa-mitigation"; + + var index = 0; + foreach (var mitigation in advisoryDto.Mitigations) + { + index++; + if (string.IsNullOrWhiteSpace(mitigation)) + { + continue; + } + + var summary = mitigation.Trim(); + var url = baseUrl is not null + ? $"{baseUrl}#mitigation-{index}" + : $"icscisa:mitigation:{advisoryDto.AdvisoryId}:{index}"; + + references.Add(new AdvisoryReference( + url, + kind: "mitigation", + sourceTag: sourceTag, + summary: summary, + provenance: new AdvisoryProvenance("ics-cisa", "mitigation", url, recordedAt))); + } + + return references.Count == 0 ? Array.Empty() : references; + } + + internal static IReadOnlyCollection BuildReferences(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) + { + var references = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (advisoryDto.Attachments is { Count: > 0 }) + { + foreach (var attachment in advisoryDto.Attachments) + { + if (attachment is null || !Validation.LooksLikeHttpUrl(attachment.Url)) + { + continue; + } + + var url = attachment.Url; + if (!seen.Add(url)) + { + continue; + } + + try + { + references.Add(new AdvisoryReference( + url, + kind: "attachment", + sourceTag: advisoryDto.IsMedical ? "icscisa-medical-attachment" : "icscisa-attachment", + summary: attachment.Title, + provenance: new AdvisoryProvenance("ics-cisa", "attachment", url, recordedAt))); + } + catch (ArgumentException) + { + // ignore invalid URIs + } + } + } + + foreach (var reference in advisoryDto.References ?? Array.Empty()) + { + if (!Validation.LooksLikeHttpUrl(reference)) + { + continue; + } + + if (!seen.Add(reference)) + { + continue; + } + + try + { + references.Add(new AdvisoryReference( + reference, + kind: "advisory", + sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa", + summary: null, + provenance: new AdvisoryProvenance("ics-cisa", "reference", reference, recordedAt))); + } + catch (ArgumentException) + { + // ignore invalid URIs + } + } + + if (references.Count == 0 && Validation.LooksLikeHttpUrl(advisoryDto.Link) && seen.Add(advisoryDto.Link)) + { + references.Add(new AdvisoryReference( + advisoryDto.Link, + kind: "advisory", + sourceTag: advisoryDto.IsMedical ? "icscisa-medical" : "icscisa", + summary: null, + provenance: new AdvisoryProvenance("ics-cisa", "reference", advisoryDto.Link, recordedAt))); + } + + return references; + } + + internal static IReadOnlyCollection BuildAffectedPackages(IcsCisaAdvisoryDto advisoryDto, DateTimeOffset recordedAt) + { + var packages = new List(); + var vendors = advisoryDto.Vendors ?? Array.Empty(); + var normalizedVendors = vendors + .Where(static vendor => !string.IsNullOrWhiteSpace(vendor)) + .Select(static vendor => vendor.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var parsedProducts = (advisoryDto.Products ?? Array.Empty()) + .Where(static product => !string.IsNullOrWhiteSpace(product)) + .Select(ParseProductInfo) + .Where(static product => !string.IsNullOrWhiteSpace(product.Name)) + .ToArray(); + + if (parsedProducts.Length > 0) + { + foreach (var product in parsedProducts) + { + } + + foreach (var product in parsedProducts) + { + var provenance = new AdvisoryProvenance("ics-cisa", "affected", product.Name!, recordedAt); + var vendorExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ics.product"] = product.Name! + }; + + if (!string.IsNullOrWhiteSpace(product.VersionExpression)) + { + vendorExtensions["ics.version"] = product.VersionExpression!; + } + + if (normalizedVendors.Length > 0) + { + vendorExtensions["ics.vendors"] = string.Join(",", normalizedVendors); + } + + var semVer = TryCreateSemVerPrimitive(product.VersionExpression); + var range = new AffectedVersionRange( + rangeKind: "product", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: product.VersionExpression, + provenance: provenance, + primitives: new RangePrimitives(semVer, null, null, vendorExtensions)); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.IcsVendor, + product.Name!, + platform: null, + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance })); + } + + return packages; + } + + if (normalizedVendors.Length == 0) + { + return packages; + } + + foreach (var vendor in normalizedVendors) + { + var provenance = new AdvisoryProvenance("ics-cisa", "affected", vendor, recordedAt); + var vendorExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ics.vendor"] = vendor + }; + + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: provenance, + primitives: new RangePrimitives(null, null, null, vendorExtensions)); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.IcsVendor, + vendor, + platform: null, + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { provenance })); + } + + return packages; + } + + + private static ProductInfo ParseProductInfo(string raw) + { + var trimmed = raw?.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + return new ProductInfo(null, null); + } + + if (trimmed.Contains(':', StringComparison.Ordinal)) + { + var parts = trimmed.Split(':', 2); + var name = parts[0].Trim(); + var versionSegment = parts[1].Trim(); + return new ProductInfo( + string.IsNullOrWhiteSpace(name) ? trimmed : name, + string.IsNullOrWhiteSpace(versionSegment) ? null : versionSegment); + } + + var lastSpace = trimmed.LastIndexOf(' '); + if (lastSpace > 0) + { + var candidateVersion = trimmed[(lastSpace + 1)..].Trim(); + if (Regex.IsMatch(candidateVersion, "^[vV]?[0-9].*")) + { + var name = trimmed[..lastSpace].Trim(); + return new ProductInfo( + string.IsNullOrWhiteSpace(name) ? trimmed : name, + candidateVersion); + } + } + + return new ProductInfo(trimmed, null); + } + + private static SemVerPrimitive? TryCreateSemVerPrimitive(string? versionExpression) + { + if (string.IsNullOrWhiteSpace(versionExpression)) + { + return null; + } + + var normalized = NormalizeSemVer(versionExpression); + if (normalized is null) + { + var trimmed = versionExpression.Trim(); + if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[1..]; + } + + if (Version.TryParse(trimmed, out var parsed)) + { + normalized = string.Join('.', new[] + { + parsed.Major.ToString(CultureInfo.InvariantCulture), + parsed.Minor >= 0 ? parsed.Minor.ToString(CultureInfo.InvariantCulture) : "0", + parsed.Build >= 0 ? parsed.Build.ToString(CultureInfo.InvariantCulture) : "0", + }); + } + } + + if (normalized is null) + { + return null; + } + + return new SemVerPrimitive( + null, + true, + null, + true, + null, + true, + null, + normalized); + } + + private static string? NormalizeSemVer(string rawVersion) + { + var trimmed = rawVersion.Trim(); + if (trimmed.StartsWith("v", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[1..]; + } + + if (!Regex.IsMatch(trimmed, @"^[0-9]+(\.[0-9]+){0,2}$")) + { + return null; + } + + var parts = trimmed.Split('.', StringSplitOptions.RemoveEmptyEntries); + var components = parts.Take(3).ToList(); + while (components.Count < 3) + { + components.Add("0"); + } + + return string.Join('.', components); + } + + private sealed record ProductInfo(string? Name, string? VersionExpression); + + private async Task EnrichAdvisoryAsync(IcsCisaAdvisoryDto advisory, CancellationToken cancellationToken) + { + if (!NeedsDetailFetch(advisory)) + { + return advisory; + } + + if (!Uri.TryCreate(advisory.Link, UriKind.Absolute, out var detailUri)) + { + return advisory; + } + + var request = new SourceFetchRequest(IcsCisaOptions.HttpClientName, SourceName, detailUri) + { + AcceptHeaders = DetailAcceptHeaders, + Metadata = AppendMetadata(null, "icscisa.detail", advisory.AdvisoryId), + TimeoutOverride = _options.DetailRequestTimeout, + }; + + try + { + var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + if (!result.IsSuccess || result.Content is null) + { + _diagnostics.DetailFetchFailure(advisory.AdvisoryId); + return advisory; + } + + var html = Encoding.UTF8.GetString(result.Content); + var sanitized = _htmlSanitizer.Sanitize(html, detailUri); + if (string.IsNullOrWhiteSpace(sanitized)) + { + _diagnostics.DetailFetchSuccess(advisory.AdvisoryId); + return advisory with { DetailHtml = sanitized }; + } + + var detailAttachments = _options.CaptureAttachments + ? ParseAttachmentsFromHtml(sanitized, detailUri) + : Array.Empty(); + var mergedAttachments = _options.CaptureAttachments + ? MergeAttachments(advisory.Attachments, detailAttachments) + : advisory.Attachments; + + var detailMitigations = ParseMitigationsFromHtml(sanitized); + var mergedMitigations = MergeMitigations(advisory.Mitigations, detailMitigations); + + var detailReferences = ParseReferencesFromHtml(sanitized, detailUri); + var mergedReferences = MergeReferences(advisory.References, detailReferences); + + var summary = string.IsNullOrWhiteSpace(advisory.Summary) + ? ExtractFirstSentence(sanitized) + : advisory.Summary; + + var descriptionHtml = string.IsNullOrWhiteSpace(advisory.DescriptionHtml) + ? sanitized + : advisory.DescriptionHtml; + + return advisory with + { + DetailHtml = sanitized, + DescriptionHtml = descriptionHtml, + Summary = summary, + References = mergedReferences, + Attachments = mergedAttachments, + Mitigations = mergedMitigations, + }; + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch detail page for {AdvisoryId}", advisory.AdvisoryId); + _diagnostics.DetailFetchFailure(advisory.AdvisoryId); + return advisory; + } + } + + private bool NeedsDetailFetch(IcsCisaAdvisoryDto advisory) + { + if (!_options.EnableDetailScrape) + { + return false; + } + + if (string.IsNullOrWhiteSpace(advisory.DescriptionHtml)) + { + return true; + } + + if (string.IsNullOrWhiteSpace(advisory.Summary)) + { + return true; + } + + if (advisory.Mitigations is null || advisory.Mitigations.Count == 0) + { + return true; + } + + if (_options.CaptureAttachments && (advisory.Attachments is null || advisory.Attachments.Count == 0)) + { + return true; + } + + return false; + } + + private IReadOnlyCollection ParseMitigationsFromHtml(string sanitizedHtml) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return Array.Empty(); + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var mitigations = new List(); + + foreach (var heading in document.QuerySelectorAll("h1, h2, h3, h4, h5, h6")) + { + var headingText = heading.TextContent?.Trim(); + if (!IsMitigationHeading(headingText)) + { + continue; + } + + var node = heading.NextElementSibling; + while (node is not null && node is not IHtmlHeadingElement) + { + if (node is IHtmlParagraphElement or IHtmlDivElement) + { + var content = Validation.CollapseWhitespace(node.TextContent); + if (!string.IsNullOrWhiteSpace(content)) + { + mitigations.Add(content); + } + } + else if (node is IHtmlElement element && (string.Equals(element.TagName, "UL", StringComparison.OrdinalIgnoreCase) || string.Equals(element.TagName, "OL", StringComparison.OrdinalIgnoreCase))) + { + foreach (var item in element.Children) + { + var content = Validation.CollapseWhitespace(item.TextContent); + if (!string.IsNullOrWhiteSpace(content)) + { + mitigations.Add(content); + } + } + } + + node = node.NextElementSibling; + } + } + + return mitigations.Count == 0 ? Array.Empty() : mitigations; + } + catch + { + return Array.Empty(); + } + } + + private static bool IsMitigationHeading(string? headingText) + { + if (string.IsNullOrWhiteSpace(headingText)) + { + return false; + } + + return headingText.Contains("mitigation", StringComparison.OrdinalIgnoreCase); + } + + private IReadOnlyCollection ParseAttachmentsFromHtml(string sanitizedHtml, Uri baseUri) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return Array.Empty(); + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var attachments = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var anchor in document.QuerySelectorAll("a")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(baseUri, href, out var resolved)) + { + continue; + } + + var url = resolved.ToString(); + if (!url.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase) && + !url.Contains("/pdf", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + attachments[url] = new IcsCisaAttachmentDto + { + Title = anchor.TextContent?.Trim(), + Url = url, + }; + } + + return attachments.Count == 0 + ? Array.Empty() + : attachments.Values.ToArray(); + } + catch + { + return Array.Empty(); + } + } + + private IReadOnlyCollection ParseReferencesFromHtml(string sanitizedHtml, Uri baseUri) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return Array.Empty(); + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var links = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var anchor in document.QuerySelectorAll("a")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (Uri.TryCreate(baseUri, href, out var resolved) && Validation.LooksLikeHttpUrl(resolved.ToString())) + { + links.Add(resolved.ToString()); + } + } + + return links.Count == 0 ? Array.Empty() : links.ToArray(); + } + catch + { + return Array.Empty(); + } + } + + internal static IReadOnlyCollection MergeMitigations(IReadOnlyCollection? existing, IReadOnlyCollection incoming) + { + if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0)) + { + return Array.Empty(); + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + var merged = new List(); + + if (existing is not null) + { + foreach (var mitigation in existing) + { + var value = mitigation?.Trim(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (set.Add(value)) + { + merged.Add(value); + } + } + } + + if (incoming is not null) + { + foreach (var mitigation in incoming) + { + var value = mitigation?.Trim(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + if (set.Add(value)) + { + merged.Add(value); + } + } + } + + return merged.Count == 0 ? Array.Empty() : merged; + } + + internal static IReadOnlyCollection MergeAttachments(IReadOnlyCollection? existing, IReadOnlyCollection incoming) + { + if ((existing is null || existing.Count == 0) && (incoming is null || incoming.Count == 0)) + { + return Array.Empty(); + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (existing is not null) + { + foreach (var attachment in existing) + { + if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url)) + { + continue; + } + + map[attachment.Url] = attachment; + } + } + + if (incoming is not null) + { + foreach (var attachment in incoming) + { + if (attachment is null || string.IsNullOrWhiteSpace(attachment.Url)) + { + continue; + } + + if (!map.ContainsKey(attachment.Url) || string.IsNullOrWhiteSpace(map[attachment.Url].Title)) + { + map[attachment.Url] = attachment; + } + } + } + + return map.Count == 0 ? Array.Empty() : map.Values.ToArray(); + } + + internal static IReadOnlyCollection MergeReferences(IReadOnlyCollection? existing, IReadOnlyCollection incoming) + { + var links = new HashSet(existing ?? Array.Empty(), StringComparer.OrdinalIgnoreCase); + foreach (var link in incoming) + { + if (Validation.LooksLikeHttpUrl(link)) + { + links.Add(link); + } + } + + return links.Count == 0 ? Array.Empty() : links.ToArray(); + } + + internal static string? ExtractFirstSentence(string sanitizedHtml) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return null; + } + + var text = Validation.CollapseWhitespace(sanitizedHtml); + if (text.Length <= 280) + { + return text; + } + + var terminator = text.IndexOf('.', StringComparison.Ordinal); + if (terminator <= 0 || terminator > 280) + { + return text[..Math.Min(280, text.Length)].Trim(); + } + + return text[..(terminator + 1)].Trim(); + } + + internal static IReadOnlyDictionary AppendMetadata(IReadOnlyDictionary? metadata, string key, string value) + { + var dictionary = new Dictionary(StringComparer.Ordinal); + + if (metadata is not null) + { + foreach (var pair in metadata) + { + dictionary[pair.Key] = pair.Value; + } + } + + dictionary[key] = value; + return dictionary; + } + + internal static bool ShouldRetryWithFallback(HttpRequestException exception) + { + var message = exception.Message ?? string.Empty; + return message.Contains(" 403", StringComparison.OrdinalIgnoreCase) + || message.Contains("403", StringComparison.OrdinalIgnoreCase) + || message.Contains("406", StringComparison.OrdinalIgnoreCase); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? IcsCisaCursor.Empty : IcsCisaCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(IcsCisaCursor cursor, CancellationToken cancellationToken) + => _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnectorPlugin.cs index bf13cb9d..d25736b2 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ics.Cisa; - -public sealed class IcsCisaConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "ics-cisa"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ics.Cisa; + +public sealed class IcsCisaConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ics-cisa"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs index 3b84d7b7..4cf880be 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaDependencyInjectionRoutine.cs @@ -1,56 +1,56 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Ics.Cisa.Configuration; - -namespace StellaOps.Feedser.Source.Ics.Cisa; - -public sealed class IcsCisaDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:ics-cisa"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddIcsCisaConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, IcsCisaJobKinds.Fetch, typeof(IcsCisaFetchJob)); - EnsureJob(options, IcsCisaJobKinds.Parse, typeof(IcsCisaParseJob)); - EnsureJob(options, IcsCisaJobKinds.Map, typeof(IcsCisaMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - ArgumentNullException.ThrowIfNull(options); - - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Ics.Cisa.Configuration; + +namespace StellaOps.Concelier.Connector.Ics.Cisa; + +public sealed class IcsCisaDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:ics-cisa"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddIcsCisaConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, IcsCisaJobKinds.Fetch, typeof(IcsCisaFetchJob)); + EnsureJob(options, IcsCisaJobKinds.Parse, typeof(IcsCisaParseJob)); + EnsureJob(options, IcsCisaJobKinds.Map, typeof(IcsCisaMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs index c7b87a93..9c0675ba 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/IcsCisaServiceCollectionExtensions.cs @@ -1,60 +1,60 @@ -using System; -using System.Net; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ics.Cisa.Configuration; -using StellaOps.Feedser.Source.Ics.Cisa.Internal; - -namespace StellaOps.Feedser.Source.Ics.Cisa; - -public static class IcsCisaServiceCollectionExtensions -{ - public static IServiceCollection AddIcsCisaConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()); - - services.AddSourceHttpClient(IcsCisaOptions.HttpClientName, (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.BaseAddress = new Uri(options.TopicsEndpoint.GetLeftPart(UriPartial.Authority)); - clientOptions.Timeout = TimeSpan.FromSeconds(45); - clientOptions.UserAgent = "StellaOps.Feedser.IcsCisa/1.0"; - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.TopicsEndpoint.Host); - clientOptions.AllowedHosts.Add(options.DetailBaseUri.Host); - foreach (var host in options.AdditionalHosts) - { - clientOptions.AllowedHosts.Add(host); - } - clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml"; - clientOptions.RequestVersion = options.RequestVersion; - clientOptions.VersionPolicy = options.RequestVersionPolicy; - clientOptions.MaxAttempts = options.MaxAttempts; - clientOptions.BaseDelay = options.BaseDelay; - clientOptions.EnableMultipleHttp2Connections = false; - - clientOptions.ConfigureHandler = handler => - { - handler.AutomaticDecompression = DecompressionMethods.All; - }; - - if (options.ProxyUri is not null) - { - clientOptions.ProxyAddress = options.ProxyUri; - clientOptions.ProxyBypassOnLocal = false; - } - }); - - services.AddSingleton(); - services.AddSingleton(); - services.AddTransient(); - - return services; - } -} +using System; +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ics.Cisa.Configuration; +using StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +namespace StellaOps.Concelier.Connector.Ics.Cisa; + +public static class IcsCisaServiceCollectionExtensions +{ + public static IServiceCollection AddIcsCisaConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()); + + services.AddSourceHttpClient(IcsCisaOptions.HttpClientName, (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.BaseAddress = new Uri(options.TopicsEndpoint.GetLeftPart(UriPartial.Authority)); + clientOptions.Timeout = TimeSpan.FromSeconds(45); + clientOptions.UserAgent = "StellaOps.Concelier.IcsCisa/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.TopicsEndpoint.Host); + clientOptions.AllowedHosts.Add(options.DetailBaseUri.Host); + foreach (var host in options.AdditionalHosts) + { + clientOptions.AllowedHosts.Add(host); + } + clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml"; + clientOptions.RequestVersion = options.RequestVersion; + clientOptions.VersionPolicy = options.RequestVersionPolicy; + clientOptions.MaxAttempts = options.MaxAttempts; + clientOptions.BaseDelay = options.BaseDelay; + clientOptions.EnableMultipleHttp2Connections = false; + + clientOptions.ConfigureHandler = handler => + { + handler.AutomaticDecompression = DecompressionMethods.All; + }; + + if (options.ProxyUri is not null) + { + clientOptions.ProxyAddress = options.ProxyUri; + clientOptions.ProxyBypassOnLocal = false; + } + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs index fd3d86c5..3b125dda 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAdvisoryDto.cs @@ -1,56 +1,56 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -public sealed record IcsCisaAdvisoryDto -{ - [JsonPropertyName("advisoryId")] - public required string AdvisoryId { get; init; } - - [JsonPropertyName("title")] - public required string Title { get; init; } - - [JsonPropertyName("link")] - public required string Link { get; init; } - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("descriptionHtml")] - public string DescriptionHtml { get; init; } = string.Empty; - - [JsonPropertyName("published")] - public DateTimeOffset Published { get; init; } - - [JsonPropertyName("updated")] - public DateTimeOffset? Updated { get; init; } - - [JsonPropertyName("medical")] - public bool IsMedical { get; init; } - - [JsonPropertyName("aliases")] - public IReadOnlyCollection Aliases { get; init; } = Array.Empty(); - - [JsonPropertyName("cveIds")] - public IReadOnlyCollection CveIds { get; init; } = Array.Empty(); - - [JsonPropertyName("vendors")] - public IReadOnlyCollection Vendors { get; init; } = Array.Empty(); - - [JsonPropertyName("products")] - public IReadOnlyCollection Products { get; init; } = Array.Empty(); - - [JsonPropertyName("references")] - public IReadOnlyCollection References { get; init; } = Array.Empty(); - - [JsonPropertyName("attachments")] - public IReadOnlyCollection Attachments { get; init; } = Array.Empty(); - - [JsonPropertyName("mitigations")] - public IReadOnlyCollection Mitigations { get; init; } = Array.Empty(); - - [JsonPropertyName("detailHtml")] - public string? DetailHtml { get; init; } -} +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +public sealed record IcsCisaAdvisoryDto +{ + [JsonPropertyName("advisoryId")] + public required string AdvisoryId { get; init; } + + [JsonPropertyName("title")] + public required string Title { get; init; } + + [JsonPropertyName("link")] + public required string Link { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("descriptionHtml")] + public string DescriptionHtml { get; init; } = string.Empty; + + [JsonPropertyName("published")] + public DateTimeOffset Published { get; init; } + + [JsonPropertyName("updated")] + public DateTimeOffset? Updated { get; init; } + + [JsonPropertyName("medical")] + public bool IsMedical { get; init; } + + [JsonPropertyName("aliases")] + public IReadOnlyCollection Aliases { get; init; } = Array.Empty(); + + [JsonPropertyName("cveIds")] + public IReadOnlyCollection CveIds { get; init; } = Array.Empty(); + + [JsonPropertyName("vendors")] + public IReadOnlyCollection Vendors { get; init; } = Array.Empty(); + + [JsonPropertyName("products")] + public IReadOnlyCollection Products { get; init; } = Array.Empty(); + + [JsonPropertyName("references")] + public IReadOnlyCollection References { get; init; } = Array.Empty(); + + [JsonPropertyName("attachments")] + public IReadOnlyCollection Attachments { get; init; } = Array.Empty(); + + [JsonPropertyName("mitigations")] + public IReadOnlyCollection Mitigations { get; init; } = Array.Empty(); + + [JsonPropertyName("detailHtml")] + public string? DetailHtml { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs similarity index 76% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs index 0518d058..f943935c 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaAttachmentDto.cs @@ -1,12 +1,12 @@ -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -public sealed record IcsCisaAttachmentDto -{ - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("url")] - public required string Url { get; init; } -} +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +public sealed record IcsCisaAttachmentDto +{ + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("url")] + public required string Url { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaCursor.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaCursor.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaCursor.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaCursor.cs index 88bd1ce4..c6e6901e 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaCursor.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaCursor.cs @@ -1,88 +1,88 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -internal sealed record IcsCisaCursor( - DateTimeOffset? LastPublished, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - public static IcsCisaCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), - }; - - if (LastPublished.HasValue) - { - document["lastPublished"] = LastPublished.Value.UtcDateTime; - } - - return document; - } - - public static IcsCisaCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) - ? ParseDate(publishedValue) - : null; - - return new IcsCisaCursor( - lastPublished, - ReadGuidArray(document, "pendingDocuments"), - ReadGuidArray(document, "pendingMappings")); - } - - public IcsCisaCursor WithPendingDocuments(IEnumerable ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; - - public IcsCisaCursor WithPendingMappings(IEnumerable ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; - - public IcsCisaCursor WithLastPublished(DateTimeOffset? published) - => this with { LastPublished = published?.ToUniversalTime() }; - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return Array.Empty(); - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (element is null) - { - continue; - } - - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +internal sealed record IcsCisaCursor( + DateTimeOffset? LastPublished, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static IcsCisaCursor Empty { get; } = new(null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + return document; + } + + public static IcsCisaCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) + ? ParseDate(publishedValue) + : null; + + return new IcsCisaCursor( + lastPublished, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public IcsCisaCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public IcsCisaCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public IcsCisaCursor WithLastPublished(DateTimeOffset? published) + => this with { LastPublished = published?.ToUniversalTime() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaDiagnostics.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaDiagnostics.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaDiagnostics.cs index c81cde4a..091c1e52 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaDiagnostics.cs @@ -1,171 +1,171 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -public sealed class IcsCisaDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.Ics.Cisa"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - - private readonly Counter _fetchAttempts; - private readonly Counter _fetchSuccess; - private readonly Counter _fetchFailures; - private readonly Counter _fetchNotModified; - private readonly Counter _fetchFallbacks; - private readonly Histogram _fetchDocuments; - - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Histogram _parseAdvisoryCount; - private readonly Histogram _parseAttachmentCount; - private readonly Histogram _parseDetailCount; - - private readonly Counter _detailSuccess; - private readonly Counter _detailFailures; - - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Histogram _mapReferenceCount; - private readonly Histogram _mapPackageCount; - private readonly Histogram _mapAliasCount; - - public IcsCisaDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - - _fetchAttempts = _meter.CreateCounter("icscisa.fetch.attempts", unit: "operations"); - _fetchSuccess = _meter.CreateCounter("icscisa.fetch.success", unit: "operations"); - _fetchFailures = _meter.CreateCounter("icscisa.fetch.failures", unit: "operations"); - _fetchNotModified = _meter.CreateCounter("icscisa.fetch.not_modified", unit: "operations"); - _fetchFallbacks = _meter.CreateCounter("icscisa.fetch.fallbacks", unit: "operations"); - _fetchDocuments = _meter.CreateHistogram("icscisa.fetch.documents", unit: "documents"); - - _parseSuccess = _meter.CreateCounter("icscisa.parse.success", unit: "documents"); - _parseFailures = _meter.CreateCounter("icscisa.parse.failures", unit: "documents"); - _parseAdvisoryCount = _meter.CreateHistogram("icscisa.parse.advisories", unit: "advisories"); - _parseAttachmentCount = _meter.CreateHistogram("icscisa.parse.attachments", unit: "attachments"); - _parseDetailCount = _meter.CreateHistogram("icscisa.parse.detail_fetches", unit: "fetches"); - - _detailSuccess = _meter.CreateCounter("icscisa.detail.success", unit: "operations"); - _detailFailures = _meter.CreateCounter("icscisa.detail.failures", unit: "operations"); - - _mapSuccess = _meter.CreateCounter("icscisa.map.success", unit: "advisories"); - _mapFailures = _meter.CreateCounter("icscisa.map.failures", unit: "advisories"); - _mapReferenceCount = _meter.CreateHistogram("icscisa.map.references", unit: "references"); - _mapPackageCount = _meter.CreateHistogram("icscisa.map.packages", unit: "packages"); - _mapAliasCount = _meter.CreateHistogram("icscisa.map.aliases", unit: "aliases"); - } - - public void FetchAttempt(string topicId) - { - _fetchAttempts.Add(1, BuildTopicTags(topicId)); - } - - public void FetchSuccess(string topicId, int documentsAdded) - { - var tags = BuildTopicTags(topicId); - _fetchSuccess.Add(1, tags); - if (documentsAdded > 0) - { - _fetchDocuments.Record(documentsAdded, tags); - } - } - - public void FetchNotModified(string topicId) - { - _fetchNotModified.Add(1, BuildTopicTags(topicId)); - } - - public void FetchFallback(string topicId) - { - _fetchFallbacks.Add(1, BuildTopicTags(topicId)); - } - - public void FetchFailure(string topicId) - { - _fetchFailures.Add(1, BuildTopicTags(topicId)); - } - - public void ParseSuccess(string topicId, int advisoryCount, int attachmentCount, int detailFetchCount) - { - var tags = BuildTopicTags(topicId); - _parseSuccess.Add(1, tags); - if (advisoryCount >= 0) - { - _parseAdvisoryCount.Record(advisoryCount, tags); - } - - if (attachmentCount >= 0) - { - _parseAttachmentCount.Record(attachmentCount, tags); - } - - if (detailFetchCount >= 0) - { - _parseDetailCount.Record(detailFetchCount, tags); - } - } - - public void ParseFailure(string topicId) - { - _parseFailures.Add(1, BuildTopicTags(topicId)); - } - - public void DetailFetchSuccess(string advisoryId) - { - _detailSuccess.Add(1, BuildAdvisoryTags(advisoryId)); - } - - public void DetailFetchFailure(string advisoryId) - { - _detailFailures.Add(1, BuildAdvisoryTags(advisoryId)); - } - - public void MapSuccess(string advisoryId, int referenceCount, int packageCount, int aliasCount) - { - var tags = BuildAdvisoryTags(advisoryId); - _mapSuccess.Add(1, tags); - if (referenceCount >= 0) - { - _mapReferenceCount.Record(referenceCount, tags); - } - - if (packageCount >= 0) - { - _mapPackageCount.Record(packageCount, tags); - } - - if (aliasCount >= 0) - { - _mapAliasCount.Record(aliasCount, tags); - } - } - - public void MapFailure(string advisoryId) - { - _mapFailures.Add(1, BuildAdvisoryTags(advisoryId)); - } - - private static KeyValuePair[] BuildTopicTags(string? topicId) - => new[] - { - new KeyValuePair("feedser.source", IcsCisaConnectorPlugin.SourceName), - new KeyValuePair("icscisa.topic", string.IsNullOrWhiteSpace(topicId) ? "unknown" : topicId) - }; - - private static KeyValuePair[] BuildAdvisoryTags(string? advisoryId) - => new[] - { - new KeyValuePair("feedser.source", IcsCisaConnectorPlugin.SourceName), - new KeyValuePair("icscisa.advisory", string.IsNullOrWhiteSpace(advisoryId) ? "unknown" : advisoryId) - }; - - public void Dispose() - { - _meter.Dispose(); - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +public sealed class IcsCisaDiagnostics : IDisposable +{ + private const string MeterName = "StellaOps.Concelier.Connector.Ics.Cisa"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + + private readonly Counter _fetchAttempts; + private readonly Counter _fetchSuccess; + private readonly Counter _fetchFailures; + private readonly Counter _fetchNotModified; + private readonly Counter _fetchFallbacks; + private readonly Histogram _fetchDocuments; + + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Histogram _parseAdvisoryCount; + private readonly Histogram _parseAttachmentCount; + private readonly Histogram _parseDetailCount; + + private readonly Counter _detailSuccess; + private readonly Counter _detailFailures; + + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Histogram _mapReferenceCount; + private readonly Histogram _mapPackageCount; + private readonly Histogram _mapAliasCount; + + public IcsCisaDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + + _fetchAttempts = _meter.CreateCounter("icscisa.fetch.attempts", unit: "operations"); + _fetchSuccess = _meter.CreateCounter("icscisa.fetch.success", unit: "operations"); + _fetchFailures = _meter.CreateCounter("icscisa.fetch.failures", unit: "operations"); + _fetchNotModified = _meter.CreateCounter("icscisa.fetch.not_modified", unit: "operations"); + _fetchFallbacks = _meter.CreateCounter("icscisa.fetch.fallbacks", unit: "operations"); + _fetchDocuments = _meter.CreateHistogram("icscisa.fetch.documents", unit: "documents"); + + _parseSuccess = _meter.CreateCounter("icscisa.parse.success", unit: "documents"); + _parseFailures = _meter.CreateCounter("icscisa.parse.failures", unit: "documents"); + _parseAdvisoryCount = _meter.CreateHistogram("icscisa.parse.advisories", unit: "advisories"); + _parseAttachmentCount = _meter.CreateHistogram("icscisa.parse.attachments", unit: "attachments"); + _parseDetailCount = _meter.CreateHistogram("icscisa.parse.detail_fetches", unit: "fetches"); + + _detailSuccess = _meter.CreateCounter("icscisa.detail.success", unit: "operations"); + _detailFailures = _meter.CreateCounter("icscisa.detail.failures", unit: "operations"); + + _mapSuccess = _meter.CreateCounter("icscisa.map.success", unit: "advisories"); + _mapFailures = _meter.CreateCounter("icscisa.map.failures", unit: "advisories"); + _mapReferenceCount = _meter.CreateHistogram("icscisa.map.references", unit: "references"); + _mapPackageCount = _meter.CreateHistogram("icscisa.map.packages", unit: "packages"); + _mapAliasCount = _meter.CreateHistogram("icscisa.map.aliases", unit: "aliases"); + } + + public void FetchAttempt(string topicId) + { + _fetchAttempts.Add(1, BuildTopicTags(topicId)); + } + + public void FetchSuccess(string topicId, int documentsAdded) + { + var tags = BuildTopicTags(topicId); + _fetchSuccess.Add(1, tags); + if (documentsAdded > 0) + { + _fetchDocuments.Record(documentsAdded, tags); + } + } + + public void FetchNotModified(string topicId) + { + _fetchNotModified.Add(1, BuildTopicTags(topicId)); + } + + public void FetchFallback(string topicId) + { + _fetchFallbacks.Add(1, BuildTopicTags(topicId)); + } + + public void FetchFailure(string topicId) + { + _fetchFailures.Add(1, BuildTopicTags(topicId)); + } + + public void ParseSuccess(string topicId, int advisoryCount, int attachmentCount, int detailFetchCount) + { + var tags = BuildTopicTags(topicId); + _parseSuccess.Add(1, tags); + if (advisoryCount >= 0) + { + _parseAdvisoryCount.Record(advisoryCount, tags); + } + + if (attachmentCount >= 0) + { + _parseAttachmentCount.Record(attachmentCount, tags); + } + + if (detailFetchCount >= 0) + { + _parseDetailCount.Record(detailFetchCount, tags); + } + } + + public void ParseFailure(string topicId) + { + _parseFailures.Add(1, BuildTopicTags(topicId)); + } + + public void DetailFetchSuccess(string advisoryId) + { + _detailSuccess.Add(1, BuildAdvisoryTags(advisoryId)); + } + + public void DetailFetchFailure(string advisoryId) + { + _detailFailures.Add(1, BuildAdvisoryTags(advisoryId)); + } + + public void MapSuccess(string advisoryId, int referenceCount, int packageCount, int aliasCount) + { + var tags = BuildAdvisoryTags(advisoryId); + _mapSuccess.Add(1, tags); + if (referenceCount >= 0) + { + _mapReferenceCount.Record(referenceCount, tags); + } + + if (packageCount >= 0) + { + _mapPackageCount.Record(packageCount, tags); + } + + if (aliasCount >= 0) + { + _mapAliasCount.Record(aliasCount, tags); + } + } + + public void MapFailure(string advisoryId) + { + _mapFailures.Add(1, BuildAdvisoryTags(advisoryId)); + } + + private static KeyValuePair[] BuildTopicTags(string? topicId) + => new[] + { + new KeyValuePair("concelier.source", IcsCisaConnectorPlugin.SourceName), + new KeyValuePair("icscisa.topic", string.IsNullOrWhiteSpace(topicId) ? "unknown" : topicId) + }; + + private static KeyValuePair[] BuildAdvisoryTags(string? advisoryId) + => new[] + { + new KeyValuePair("concelier.source", IcsCisaConnectorPlugin.SourceName), + new KeyValuePair("icscisa.advisory", string.IsNullOrWhiteSpace(advisoryId) ? "unknown" : advisoryId) + }; + + public void Dispose() + { + _meter.Dispose(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedDto.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedDto.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedDto.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedDto.cs index 6c2c82af..a54d20fd 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedDto.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedDto.cs @@ -1,16 +1,16 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -public sealed record IcsCisaFeedDto -{ - [JsonPropertyName("topicId")] - public required string TopicId { get; init; } - - [JsonPropertyName("feedUri")] - public required string FeedUri { get; init; } - - [JsonPropertyName("advisories")] - public IReadOnlyCollection Advisories { get; init; } = new List(); -} +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +public sealed record IcsCisaFeedDto +{ + [JsonPropertyName("topicId")] + public required string TopicId { get; init; } + + [JsonPropertyName("feedUri")] + public required string FeedUri { get; init; } + + [JsonPropertyName("advisories")] + public IReadOnlyCollection Advisories { get; init; } = new List(); +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedParser.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedParser.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedParser.cs index 2bef37d6..2e4c2326 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Internal/IcsCisaFeedParser.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Internal/IcsCisaFeedParser.cs @@ -1,402 +1,402 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.ServiceModel.Syndication; -using System.Text; -using System.Text.RegularExpressions; -using System.Xml; -using AngleSharp.Html.Parser; -using AngleSharp.Html.Dom; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.Ics.Cisa.Internal; - -public sealed class IcsCisaFeedParser -{ - private static readonly Regex AdvisoryIdRegex = new(@"^(?ICS[AM]?A?-?\d{2}-\d{3}[A-Z]?(-\d{2})?)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); - - private readonly HtmlContentSanitizer _sanitizer = new(); - private readonly HtmlParser _htmlParser = new(); - - public IReadOnlyCollection Parse(Stream rssStream, bool isMedicalTopic, Uri? topicUri) - { - if (rssStream is null) - { - return Array.Empty(); - } - - using var reader = XmlReader.Create(rssStream, new XmlReaderSettings - { - DtdProcessing = DtdProcessing.Ignore, - IgnoreComments = true, - IgnoreProcessingInstructions = true, - }); - - var feed = SyndicationFeed.Load(reader); - if (feed is null || feed.Items is null) - { - return Array.Empty(); - } - - var advisories = new List(); - foreach (var item in feed.Items) - { - var dto = ConvertItem(item, isMedicalTopic, topicUri); - if (dto is not null) - { - advisories.Add(dto); - } - } - - return advisories; - } - - private IcsCisaAdvisoryDto? ConvertItem(SyndicationItem item, bool isMedicalTopic, Uri? topicUri) - { - if (item is null) - { - return null; - } - - var title = item.Title?.Text?.Trim(); - if (string.IsNullOrWhiteSpace(title)) - { - return null; - } - - var advisoryId = ExtractAdvisoryId(title); - if (string.IsNullOrWhiteSpace(advisoryId)) - { - return null; - } - - var linkUri = item.Links.FirstOrDefault()?.Uri; - if (linkUri is null && !string.IsNullOrWhiteSpace(item.Id) && Uri.TryCreate(item.Id, UriKind.Absolute, out var fallback)) - { - linkUri = fallback; - } - - if (linkUri is null) - { - return null; - } - - var contentHtml = ExtractContentHtml(item); - var sanitizedHtml = _sanitizer.Sanitize(contentHtml, linkUri); - var textContent = ExtractTextContent(sanitizedHtml); - - var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { advisoryId }; - var cveIds = ExtractCveIds(textContent, aliases); - var vendors = ExtractList(sanitizedHtml, textContent, "Vendor"); - var products = ExtractList(sanitizedHtml, textContent, "Products"); - if (products.Count == 0) - { - products = ExtractList(sanitizedHtml, textContent, "Product"); - } - var attachments = ExtractAttachments(sanitizedHtml, linkUri); - var references = ExtractReferences(sanitizedHtml, linkUri); - - var published = item.PublishDate != DateTimeOffset.MinValue - ? item.PublishDate.ToUniversalTime() - : item.LastUpdatedTime.ToUniversalTime(); - - var updated = item.LastUpdatedTime != DateTimeOffset.MinValue - ? item.LastUpdatedTime.ToUniversalTime() - : (DateTimeOffset?)null; - - return new IcsCisaAdvisoryDto - { - AdvisoryId = advisoryId, - Title = title, - Link = linkUri.ToString(), - Summary = item.Summary?.Text?.Trim(), - DescriptionHtml = sanitizedHtml, - Published = published, - Updated = updated, - IsMedical = isMedicalTopic || advisoryId.StartsWith("ICSMA", StringComparison.OrdinalIgnoreCase), - Aliases = aliases.ToArray(), - CveIds = cveIds, - Vendors = vendors, - Products = products, - References = references, - Attachments = attachments, - }; - } - - private static string ExtractAdvisoryId(string title) - { - if (string.IsNullOrWhiteSpace(title)) - { - return string.Empty; - } - - var colonIndex = title.IndexOf(':'); - var candidate = colonIndex > 0 ? title[..colonIndex] : title; - var match = AdvisoryIdRegex.Match(candidate); - if (match.Success) - { - var id = match.Groups["id"].Value.Trim(); - return id.ToUpperInvariant(); - } - - return candidate.Trim(); - } - - private static string ExtractContentHtml(SyndicationItem item) - { - if (item.Content is TextSyndicationContent textContent) - { - return textContent.Text ?? string.Empty; - } - - if (item.Summary is not null) - { - return item.Summary.Text ?? string.Empty; - } - - if (item.ElementExtensions is not null) - { - foreach (var extension in item.ElementExtensions) - { - try - { - var value = extension.GetObject(); - if (!string.IsNullOrWhiteSpace(value)) - { - return value; - } - } - catch - { - // ignore malformed extensions - } - } - } - - return string.Empty; - } - - private static IReadOnlyCollection ExtractCveIds(string text, HashSet aliases) - { - if (string.IsNullOrWhiteSpace(text)) - { - return Array.Empty(); - } - - var matches = CveRegex.Matches(text); - if (matches.Count == 0) - { - return Array.Empty(); - } - - var ids = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (Match match in matches) - { - if (!match.Success) - { - continue; - } - - var value = match.Value.ToUpperInvariant(); - if (ids.Add(value)) - { - aliases.Add(value); - } - } - - return ids.ToArray(); - } - - private IReadOnlyCollection ExtractList(string sanitizedHtml, string textContent, string key) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return Array.Empty(); - } - - var items = new HashSet(StringComparer.OrdinalIgnoreCase); - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - foreach (var element in document.All) - { - if (element is IHtmlParagraphElement or IHtmlDivElement or IHtmlSpanElement or IHtmlListItemElement) - { - var content = element.TextContent?.Trim(); - if (string.IsNullOrWhiteSpace(content)) - { - continue; - } - - if (content.StartsWith($"{key}:", StringComparison.OrdinalIgnoreCase)) - { - var line = content[(key.Length + 1)..].Trim(); - foreach (var part in line.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) - { - var value = part.Trim(); - if (!string.IsNullOrWhiteSpace(value)) - { - items.Add(value); - } - } - } - } - } - } - catch - { - // ignore HTML parsing failures; fallback to text processing below - } - - if (items.Count == 0 && !string.IsNullOrWhiteSpace(textContent)) - { - using var reader = new StringReader(textContent); - string? line; - while ((line = reader.ReadLine()) is not null) - { - if (line.StartsWith($"{key}:", StringComparison.OrdinalIgnoreCase)) - { - var raw = line[(key.Length + 1)..].Trim(); - foreach (var part in raw.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) - { - var value = part.Trim(); - if (!string.IsNullOrWhiteSpace(value)) - { - items.Add(value); - } - } - } - } - } - - return items.ToArray(); - } - - private IReadOnlyCollection ExtractAttachments(string sanitizedHtml, Uri linkUri) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return Array.Empty(); - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var attachments = new List(); - - foreach (var anchor in document.QuerySelectorAll("a")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!Uri.TryCreate(linkUri, href, out var resolved)) - { - continue; - } - - var url = resolved.ToString(); - if (!url.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase) && - !url.Contains("/pdf", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - attachments.Add(new IcsCisaAttachmentDto - { - Title = anchor.TextContent?.Trim(), - Url = url, - }); - } - - return attachments.Count == 0 ? Array.Empty() : attachments; - } - catch - { - return Array.Empty(); - } - } - - private IReadOnlyCollection ExtractReferences(string sanitizedHtml, Uri linkUri) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return new[] { linkUri.ToString() }; - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var links = new HashSet(StringComparer.OrdinalIgnoreCase) - { - linkUri.ToString() - }; - - foreach (var anchor in document.QuerySelectorAll("a")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (Uri.TryCreate(linkUri, href, out var resolved)) - { - links.Add(resolved.ToString()); - } - } - - return links.ToArray(); - } - catch - { - return new[] { linkUri.ToString() }; - } - } - - private string ExtractTextContent(string sanitizedHtml) - { - if (string.IsNullOrWhiteSpace(sanitizedHtml)) - { - return string.Empty; - } - - try - { - var document = _htmlParser.ParseDocument(sanitizedHtml); - var builder = new StringBuilder(); - var body = document.Body ?? document.DocumentElement; - if (body is null) - { - return string.Empty; - } - - foreach (var node in body.ChildNodes) - { - var text = node.TextContent; - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - if (builder.Length > 0) - { - builder.AppendLine(); - } - - builder.Append(text.Trim()); - } - - return builder.ToString(); - } - catch - { - return sanitizedHtml; - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.ServiceModel.Syndication; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; +using AngleSharp.Html.Parser; +using AngleSharp.Html.Dom; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.Ics.Cisa.Internal; + +public sealed class IcsCisaFeedParser +{ + private static readonly Regex AdvisoryIdRegex = new(@"^(?ICS[AM]?A?-?\d{2}-\d{3}[A-Z]?(-\d{2})?)", RegexOptions.IgnoreCase | RegexOptions.Compiled); + private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private readonly HtmlContentSanitizer _sanitizer = new(); + private readonly HtmlParser _htmlParser = new(); + + public IReadOnlyCollection Parse(Stream rssStream, bool isMedicalTopic, Uri? topicUri) + { + if (rssStream is null) + { + return Array.Empty(); + } + + using var reader = XmlReader.Create(rssStream, new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Ignore, + IgnoreComments = true, + IgnoreProcessingInstructions = true, + }); + + var feed = SyndicationFeed.Load(reader); + if (feed is null || feed.Items is null) + { + return Array.Empty(); + } + + var advisories = new List(); + foreach (var item in feed.Items) + { + var dto = ConvertItem(item, isMedicalTopic, topicUri); + if (dto is not null) + { + advisories.Add(dto); + } + } + + return advisories; + } + + private IcsCisaAdvisoryDto? ConvertItem(SyndicationItem item, bool isMedicalTopic, Uri? topicUri) + { + if (item is null) + { + return null; + } + + var title = item.Title?.Text?.Trim(); + if (string.IsNullOrWhiteSpace(title)) + { + return null; + } + + var advisoryId = ExtractAdvisoryId(title); + if (string.IsNullOrWhiteSpace(advisoryId)) + { + return null; + } + + var linkUri = item.Links.FirstOrDefault()?.Uri; + if (linkUri is null && !string.IsNullOrWhiteSpace(item.Id) && Uri.TryCreate(item.Id, UriKind.Absolute, out var fallback)) + { + linkUri = fallback; + } + + if (linkUri is null) + { + return null; + } + + var contentHtml = ExtractContentHtml(item); + var sanitizedHtml = _sanitizer.Sanitize(contentHtml, linkUri); + var textContent = ExtractTextContent(sanitizedHtml); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { advisoryId }; + var cveIds = ExtractCveIds(textContent, aliases); + var vendors = ExtractList(sanitizedHtml, textContent, "Vendor"); + var products = ExtractList(sanitizedHtml, textContent, "Products"); + if (products.Count == 0) + { + products = ExtractList(sanitizedHtml, textContent, "Product"); + } + var attachments = ExtractAttachments(sanitizedHtml, linkUri); + var references = ExtractReferences(sanitizedHtml, linkUri); + + var published = item.PublishDate != DateTimeOffset.MinValue + ? item.PublishDate.ToUniversalTime() + : item.LastUpdatedTime.ToUniversalTime(); + + var updated = item.LastUpdatedTime != DateTimeOffset.MinValue + ? item.LastUpdatedTime.ToUniversalTime() + : (DateTimeOffset?)null; + + return new IcsCisaAdvisoryDto + { + AdvisoryId = advisoryId, + Title = title, + Link = linkUri.ToString(), + Summary = item.Summary?.Text?.Trim(), + DescriptionHtml = sanitizedHtml, + Published = published, + Updated = updated, + IsMedical = isMedicalTopic || advisoryId.StartsWith("ICSMA", StringComparison.OrdinalIgnoreCase), + Aliases = aliases.ToArray(), + CveIds = cveIds, + Vendors = vendors, + Products = products, + References = references, + Attachments = attachments, + }; + } + + private static string ExtractAdvisoryId(string title) + { + if (string.IsNullOrWhiteSpace(title)) + { + return string.Empty; + } + + var colonIndex = title.IndexOf(':'); + var candidate = colonIndex > 0 ? title[..colonIndex] : title; + var match = AdvisoryIdRegex.Match(candidate); + if (match.Success) + { + var id = match.Groups["id"].Value.Trim(); + return id.ToUpperInvariant(); + } + + return candidate.Trim(); + } + + private static string ExtractContentHtml(SyndicationItem item) + { + if (item.Content is TextSyndicationContent textContent) + { + return textContent.Text ?? string.Empty; + } + + if (item.Summary is not null) + { + return item.Summary.Text ?? string.Empty; + } + + if (item.ElementExtensions is not null) + { + foreach (var extension in item.ElementExtensions) + { + try + { + var value = extension.GetObject(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + catch + { + // ignore malformed extensions + } + } + } + + return string.Empty; + } + + private static IReadOnlyCollection ExtractCveIds(string text, HashSet aliases) + { + if (string.IsNullOrWhiteSpace(text)) + { + return Array.Empty(); + } + + var matches = CveRegex.Matches(text); + if (matches.Count == 0) + { + return Array.Empty(); + } + + var ids = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (Match match in matches) + { + if (!match.Success) + { + continue; + } + + var value = match.Value.ToUpperInvariant(); + if (ids.Add(value)) + { + aliases.Add(value); + } + } + + return ids.ToArray(); + } + + private IReadOnlyCollection ExtractList(string sanitizedHtml, string textContent, string key) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return Array.Empty(); + } + + var items = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + foreach (var element in document.All) + { + if (element is IHtmlParagraphElement or IHtmlDivElement or IHtmlSpanElement or IHtmlListItemElement) + { + var content = element.TextContent?.Trim(); + if (string.IsNullOrWhiteSpace(content)) + { + continue; + } + + if (content.StartsWith($"{key}:", StringComparison.OrdinalIgnoreCase)) + { + var line = content[(key.Length + 1)..].Trim(); + foreach (var part in line.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + var value = part.Trim(); + if (!string.IsNullOrWhiteSpace(value)) + { + items.Add(value); + } + } + } + } + } + } + catch + { + // ignore HTML parsing failures; fallback to text processing below + } + + if (items.Count == 0 && !string.IsNullOrWhiteSpace(textContent)) + { + using var reader = new StringReader(textContent); + string? line; + while ((line = reader.ReadLine()) is not null) + { + if (line.StartsWith($"{key}:", StringComparison.OrdinalIgnoreCase)) + { + var raw = line[(key.Length + 1)..].Trim(); + foreach (var part in raw.Split(new[] { ',', ';' }, StringSplitOptions.RemoveEmptyEntries)) + { + var value = part.Trim(); + if (!string.IsNullOrWhiteSpace(value)) + { + items.Add(value); + } + } + } + } + } + + return items.ToArray(); + } + + private IReadOnlyCollection ExtractAttachments(string sanitizedHtml, Uri linkUri) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return Array.Empty(); + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var attachments = new List(); + + foreach (var anchor in document.QuerySelectorAll("a")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(linkUri, href, out var resolved)) + { + continue; + } + + var url = resolved.ToString(); + if (!url.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase) && + !url.Contains("/pdf", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + attachments.Add(new IcsCisaAttachmentDto + { + Title = anchor.TextContent?.Trim(), + Url = url, + }); + } + + return attachments.Count == 0 ? Array.Empty() : attachments; + } + catch + { + return Array.Empty(); + } + } + + private IReadOnlyCollection ExtractReferences(string sanitizedHtml, Uri linkUri) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return new[] { linkUri.ToString() }; + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var links = new HashSet(StringComparer.OrdinalIgnoreCase) + { + linkUri.ToString() + }; + + foreach (var anchor in document.QuerySelectorAll("a")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (Uri.TryCreate(linkUri, href, out var resolved)) + { + links.Add(resolved.ToString()); + } + } + + return links.ToArray(); + } + catch + { + return new[] { linkUri.ToString() }; + } + } + + private string ExtractTextContent(string sanitizedHtml) + { + if (string.IsNullOrWhiteSpace(sanitizedHtml)) + { + return string.Empty; + } + + try + { + var document = _htmlParser.ParseDocument(sanitizedHtml); + var builder = new StringBuilder(); + var body = document.Body ?? document.DocumentElement; + if (body is null) + { + return string.Empty; + } + + foreach (var node in body.ChildNodes) + { + var text = node.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + if (builder.Length > 0) + { + builder.AppendLine(); + } + + builder.Append(text.Trim()); + } + + return builder.ToString(); + } + catch + { + return sanitizedHtml; + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/Jobs.cs b/src/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ics.Cisa/Jobs.cs rename to src/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs index 44a1f005..0c27afa2 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/Jobs.cs @@ -1,46 +1,46 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Ics.Cisa; - -internal static class IcsCisaJobKinds -{ - public const string Fetch = "source:ics-cisa:fetch"; - public const string Parse = "source:ics-cisa:parse"; - public const string Map = "source:ics-cisa:map"; -} - -internal sealed class IcsCisaFetchJob : IJob -{ - private readonly IcsCisaConnector _connector; - - public IcsCisaFetchJob(IcsCisaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class IcsCisaParseJob : IJob -{ - private readonly IcsCisaConnector _connector; - - public IcsCisaParseJob(IcsCisaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class IcsCisaMapJob : IJob -{ - private readonly IcsCisaConnector _connector; - - public IcsCisaMapJob(IcsCisaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Ics.Cisa; + +internal static class IcsCisaJobKinds +{ + public const string Fetch = "source:ics-cisa:fetch"; + public const string Parse = "source:ics-cisa:parse"; + public const string Map = "source:ics-cisa:map"; +} + +internal sealed class IcsCisaFetchJob : IJob +{ + private readonly IcsCisaConnector _connector; + + public IcsCisaFetchJob(IcsCisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class IcsCisaParseJob : IJob +{ + private readonly IcsCisaConnector _connector; + + public IcsCisaParseJob(IcsCisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class IcsCisaMapJob : IJob +{ + private readonly IcsCisaConnector _connector; + + public IcsCisaMapJob(IcsCisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj b/src/StellaOps.Concelier.Connector.Ics.Cisa/StellaOps.Concelier.Connector.Ics.Cisa.csproj similarity index 51% rename from src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj rename to src/StellaOps.Concelier.Connector.Ics.Cisa/StellaOps.Concelier.Connector.Ics.Cisa.csproj index 8da045aa..fd85683b 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/StellaOps.Feedser.Source.Ics.Cisa.csproj +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/StellaOps.Concelier.Connector.Ics.Cisa.csproj @@ -1,28 +1,28 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - - - - - - - <_Parameter1>StellaOps.Feedser.Source.Ics.Cisa.Tests - - - - + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + + + <_Parameter1>StellaOps.Concelier.Connector.Ics.Cisa.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md b/src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md similarity index 75% rename from src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md rename to src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md index 747a6169..11baa8c7 100644 --- a/src/StellaOps.Feedser.Source.Ics.Cisa/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Ics.Cisa/TASKS.md @@ -1,14 +1,15 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-ICSCISA-02-001 Document CISA ICS feed contract|BE-Conn-ICS-CISA|Research|**DONE (2025-10-11)** – `https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml` and legacy `/sites/default/files/feeds/...` return Akamai 403 even with browser UA; HTML landing page blocked as well. Logged full headers (x-reference-error, AkamaiGHost) in `docs/feedser-connector-research-20251011.md` and initiated GovDelivery access request.| -|FEEDCONN-ICSCISA-02-002 Fetch pipeline & cursor storage|BE-Conn-ICS-CISA|Source.Common, Storage.Mongo|**DONE (2025-10-16)** – Confirmed proxy knobs + cursor state behave with the refreshed fixtures; ops runbook now captures proxy usage/validation so the fetch stage is production-ready.| -|FEEDCONN-ICSCISA-02-003 DTO/parser implementation|BE-Conn-ICS-CISA|Source.Common|**DONE (2025-10-16)** – Feed parser fixtures updated to retain vendor PDFs as attachments while maintaining reference coverage; console diagnostics removed.| -|FEEDCONN-ICSCISA-02-004 Canonical mapping & range primitives|BE-Conn-ICS-CISA|Models|**DONE (2025-10-16)** – `TryCreateSemVerPrimitive` flow + Mongo deserialiser now persist `exactValue` (`4.2` → `4.2.0`), unblocking canonical snapshots.| -|FEEDCONN-ICSCISA-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-16)** – `dotnet test src/StellaOps.Feedser.Source.Ics.Cisa.Tests/...` passes; fixtures assert attachment handling + SemVer semantics.| -|FEEDCONN-ICSCISA-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-16)** – Ops guide documents attachment checks, SemVer exact values, and proxy guidance; diagnostics remain unchanged.| -|FEEDCONN-ICSCISA-02-007 Detail document inventory|BE-Conn-ICS-CISA|Research|**DONE (2025-10-16)** – Validated canned detail pages vs feed output so attachment inventories stay aligned; archived expectations noted in `HANDOVER.md`.| -|FEEDCONN-ICSCISA-02-008 Distribution fallback strategy|BE-Conn-ICS-CISA|Research|**DONE (2025-10-11)** – Outlined GovDelivery token request, HTML scrape + email digest fallback, and dependency on Ops for credential workflow; awaiting decision before fetch implementation.| -|FEEDCONN-ICSCISA-02-009 GovDelivery credential onboarding|Ops, BE-Conn-ICS-CISA|Ops|**DONE (2025-10-14)** – GovDelivery onboarding runbook captured in `docs/ops/feedser-icscisa-operations.md`; secret vault path and Offline Kit handling documented.| -|FEEDCONN-ICSCISA-02-010 Mitigation & SemVer polish|BE-Conn-ICS-CISA|02-003, 02-004|**DONE (2025-10-16)** – Attachment + mitigation references now land as expected and SemVer primitives carry exact values; end-to-end suite green (see `HANDOVER.md`).| -|FEEDCONN-ICSCISA-02-011 Docs & telemetry refresh|DevEx|02-006|**DONE (2025-10-16)** – Ops documentation refreshed (attachments, SemVer validation, proxy knobs) and telemetry notes verified.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-ICSCISA-02-001 Document CISA ICS feed contract|BE-Conn-ICS-CISA|Research|**DONE (2025-10-11)** – `https://www.cisa.gov/cybersecurity-advisories/ics-advisories.xml` and legacy `/sites/default/files/feeds/...` return Akamai 403 even with browser UA; HTML landing page blocked as well. Logged full headers (x-reference-error, AkamaiGHost) in `docs/concelier-connector-research-20251011.md` and initiated GovDelivery access request.| +|FEEDCONN-ICSCISA-02-002 Fetch pipeline & cursor storage|BE-Conn-ICS-CISA|Source.Common, Storage.Mongo|**DONE (2025-10-16)** – Confirmed proxy knobs + cursor state behave with the refreshed fixtures; ops runbook now captures proxy usage/validation so the fetch stage is production-ready.| +|FEEDCONN-ICSCISA-02-003 DTO/parser implementation|BE-Conn-ICS-CISA|Source.Common|**DONE (2025-10-16)** – Feed parser fixtures updated to retain vendor PDFs as attachments while maintaining reference coverage; console diagnostics removed.| +|FEEDCONN-ICSCISA-02-004 Canonical mapping & range primitives|BE-Conn-ICS-CISA|Models|**DONE (2025-10-16)** – `TryCreateSemVerPrimitive` flow + Mongo deserialiser now persist `exactValue` (`4.2` → `4.2.0`), unblocking canonical snapshots.| +|FEEDCONN-ICSCISA-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-16)** – `dotnet test src/StellaOps.Concelier.Connector.Ics.Cisa.Tests/...` passes; fixtures assert attachment handling + SemVer semantics.| +|FEEDCONN-ICSCISA-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-16)** – Ops guide documents attachment checks, SemVer exact values, and proxy guidance; diagnostics remain unchanged.| +|FEEDCONN-ICSCISA-02-007 Detail document inventory|BE-Conn-ICS-CISA|Research|**DONE (2025-10-16)** – Validated canned detail pages vs feed output so attachment inventories stay aligned; archived expectations noted in `HANDOVER.md`.| +|FEEDCONN-ICSCISA-02-008 Distribution fallback strategy|BE-Conn-ICS-CISA|Research|**DONE (2025-10-11)** – Outlined GovDelivery token request, HTML scrape + email digest fallback, and dependency on Ops for credential workflow; awaiting decision before fetch implementation.| +|FEEDCONN-ICSCISA-02-009 GovDelivery credential onboarding|Ops, BE-Conn-ICS-CISA|Ops|**DONE (2025-10-14)** – GovDelivery onboarding runbook captured in `docs/ops/concelier-icscisa-operations.md`; secret vault path and Offline Kit handling documented.| +|FEEDCONN-ICSCISA-02-010 Mitigation & SemVer polish|BE-Conn-ICS-CISA|02-003, 02-004|**DONE (2025-10-16)** – Attachment + mitigation references now land as expected and SemVer primitives carry exact values; end-to-end suite green (see `HANDOVER.md`).| +|FEEDCONN-ICSCISA-02-011 Docs & telemetry refresh|DevEx|02-006|**DONE (2025-10-16)** – Ops documentation refreshed (attachments, SemVer validation, proxy knobs) and telemetry notes verified.| +|FEEDCONN-ICSCISA-02-012 Normalized version decision|BE-Conn-ICS-CISA|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-23)** – Promote existing `SemVerPrimitive` exact values into `NormalizedVersions` via `.ToNormalizedVersionRule("ics-cisa:{advisoryId}:{product}")`, add regression coverage, and open Models ticket if non-SemVer firmware requires a new scheme.| diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html similarity index 100% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/detail-acme-controller-2024.html diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/expected-advisory.json diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml similarity index 100% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/Fixtures/feed-page1.xml diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs index 9b72d06a..3c846ff2 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/Kaspersky/KasperskyConnectorTests.cs @@ -12,20 +12,20 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Ics.Kaspersky; -using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Ics.Kaspersky; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Tests; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Tests; [Collection("mongo-fixture")] public sealed class KasperskyConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests.csproj b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests.csproj new file mode 100644 index 00000000..fcb1a72c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests/StellaOps.Concelier.Connector.Ics.Kaspersky.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md index f76285ff..ed709759 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/AGENTS.md @@ -20,9 +20,9 @@ Kaspersky ICS-CERT connector; authoritative for OT/ICS vendor advisories covered In: ICS advisory mapping, affected vendor products, mitigation references. Out: firmware downloads; reverse-engineering artifacts. ## Observability & security expectations -- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms with `feedser.source=ics-kaspersky` to track fetch totals, parse failures, and mapped affected counts. +- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms with `concelier.source=ics-kaspersky` to track fetch totals, parse failures, and mapped affected counts. - Logs: slugs, vendor/product counts, timing; allowlist host. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Ics.Kaspersky.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Ics.Kaspersky.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Configuration/KasperskyOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Configuration/KasperskyOptions.cs index 06fccede..0caae5b4 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Configuration/KasperskyOptions.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Configuration/KasperskyOptions.cs @@ -1,7 +1,7 @@ using System; using System.Diagnostics.CodeAnalysis; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; public sealed class KasperskyOptions { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs index db2f6ae4..6adf3294 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryDto.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Immutable; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; internal sealed record KasperskyAdvisoryDto( string AdvisoryKey, diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs index 0a3d5419..41003023 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyAdvisoryParser.cs @@ -6,7 +6,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; internal static class KasperskyAdvisoryParser { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyCursor.cs index 74ff6fad..ebbffe59 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyCursor.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; internal sealed record KasperskyCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedClient.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedClient.cs index efe11249..703ce358 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedClient.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedClient.cs @@ -10,9 +10,9 @@ using System.Threading.Tasks; using System.Xml.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; public sealed class KasperskyFeedClient { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedItem.cs similarity index 65% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedItem.cs index eaa552d2..724afa42 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Internal/KasperskyFeedItem.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Internal/KasperskyFeedItem.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; public sealed record KasperskyFeedItem( string Title, diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs index 2d752305..07e9b7f8 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Ics.Kaspersky; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky; internal static class KasperskyJobKinds { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnector.cs index 727e6821..8c47b7bc 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnector.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnector.cs @@ -7,18 +7,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; -using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Ics.Kaspersky; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky; public sealed class KasperskyConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnectorPlugin.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnectorPlugin.cs index 45a05b99..4c0a74ab 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyConnectorPlugin.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Ics.Kaspersky; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky; public sealed class KasperskyConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs index a3559421..d6983adc 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; -namespace StellaOps.Feedser.Source.Ics.Kaspersky; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky; public sealed class KasperskyDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:ics-kaspersky"; + private const string ConfigurationSection = "concelier:sources:ics-kaspersky"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs similarity index 77% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs index 216fbfaf..0b22c576 100644 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/KasperskyServiceCollectionExtensions.cs @@ -1,11 +1,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ics.Kaspersky.Configuration; -using StellaOps.Feedser.Source.Ics.Kaspersky.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Configuration; +using StellaOps.Concelier.Connector.Ics.Kaspersky.Internal; -namespace StellaOps.Feedser.Source.Ics.Kaspersky; +namespace StellaOps.Concelier.Connector.Ics.Kaspersky; public static class KasperskyServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class KasperskyServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = options.FeedUri; clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.IcsKaspersky/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.IcsKaspersky/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.FeedUri.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/rss+xml"; diff --git a/src/StellaOps.Concelier.Connector.Ics.Kaspersky/StellaOps.Concelier.Connector.Ics.Kaspersky.csproj b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/StellaOps.Concelier.Connector.Ics.Kaspersky.csproj new file mode 100644 index 00000000..44c74bcc --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/StellaOps.Concelier.Connector.Ics.Kaspersky.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md b/src/StellaOps.Concelier.Connector.Ics.Kaspersky/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Ics.Kaspersky/TASKS.md rename to src/StellaOps.Concelier.Connector.Ics.Kaspersky/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/expected-advisory.json similarity index 96% rename from src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json rename to src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/expected-advisory.json index 4a691565..f706a590 100644 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/expected-advisory.json +++ b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/expected-advisory.json @@ -1,87 +1,87 @@ -{ - "advisoryKey": "JVNDB-2024-123456", - "affectedPackages": [], - "aliases": [ - "CVE-2024-5555", - "JVNDB-2024-123456" - ], - "cvssMetrics": [ - { - "baseScore": 8.8, - "baseSeverity": "high", - "provenance": { - "fieldMask": [], - "kind": "cvss", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "Base" - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2024-03-10T02:30:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-03-10T00:00:00+00:00", - "source": "jvn", - "value": "https://jvndb.jvn.jp/myjvn?method=getVulnDetailInfo&feed=hnd&lang=en&vulnId=JVNDB-2024-123456" - }, - { - "fieldMask": [], - "kind": "mapping", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "JVNDB-2024-123456" - } - ], - "published": "2024-03-09T02:00:00+00:00", - "references": [ - { - "kind": "weakness", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "https://cwe.mitre.org/data/definitions/287.html" - }, - "sourceTag": "CWE-287", - "summary": "JVNDB", - "url": "https://cwe.mitre.org/data/definitions/287.html" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "https://vendor.example.com/advisories/EX-2024-01" - }, - "sourceTag": "EX-2024-01", - "summary": "Example ICS Vendor Advisory", - "url": "https://vendor.example.com/advisories/EX-2024-01" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-03-10T00:01:00+00:00", - "source": "jvn", - "value": "https://www.cve.org/CVERecord?id=CVE-2024-5555" - }, - "sourceTag": "CVE-2024-5555", - "summary": "Common Vulnerabilities and Exposures (CVE)", - "url": "https://www.cve.org/CVERecord?id=CVE-2024-5555" - } - ], - "severity": "high", - "summary": "Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability.", - "title": "Example vulnerability in Imaginary ICS Controller" +{ + "advisoryKey": "JVNDB-2024-123456", + "affectedPackages": [], + "aliases": [ + "CVE-2024-5555", + "JVNDB-2024-123456" + ], + "cvssMetrics": [ + { + "baseScore": 8.8, + "baseSeverity": "high", + "provenance": { + "fieldMask": [], + "kind": "cvss", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "Base" + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-03-10T02:30:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-03-10T00:00:00+00:00", + "source": "jvn", + "value": "https://jvndb.jvn.jp/myjvn?method=getVulnDetailInfo&feed=hnd&lang=en&vulnId=JVNDB-2024-123456" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "JVNDB-2024-123456" + } + ], + "published": "2024-03-09T02:00:00+00:00", + "references": [ + { + "kind": "weakness", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://cwe.mitre.org/data/definitions/287.html" + }, + "sourceTag": "CWE-287", + "summary": "JVNDB", + "url": "https://cwe.mitre.org/data/definitions/287.html" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://vendor.example.com/advisories/EX-2024-01" + }, + "sourceTag": "EX-2024-01", + "summary": "Example ICS Vendor Advisory", + "url": "https://vendor.example.com/advisories/EX-2024-01" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-03-10T00:01:00+00:00", + "source": "jvn", + "value": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + }, + "sourceTag": "CVE-2024-5555", + "summary": "Common Vulnerabilities and Exposures (CVE)", + "url": "https://www.cve.org/CVERecord?id=CVE-2024-5555" + } + ], + "severity": "high", + "summary": "Imaginary ICS Controller provided by Example Industrial Corporation contains an authentication bypass vulnerability.", + "title": "Example vulnerability in Imaginary ICS Controller" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml rename to src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/jvnrss-window1.xml diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml rename to src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/Fixtures/vuldef-JVNDB-2024-123456.xml diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/JvnConnectorTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/JvnConnectorTests.cs index eafb6cc2..682675dc 100644 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/Jvn/JvnConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Jvn.Tests/Jvn/JvnConnectorTests.cs @@ -12,22 +12,22 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Jvn; -using StellaOps.Feedser.Source.Jvn.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Jvn; +using StellaOps.Concelier.Connector.Jvn.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.JpFlags; using Xunit.Abstractions; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Jvn.Tests; +namespace StellaOps.Concelier.Connector.Jvn.Tests; [Collection("mongo-fixture")] public sealed class JvnConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Concelier.Connector.Jvn.Tests/StellaOps.Concelier.Connector.Jvn.Tests.csproj b/src/StellaOps.Concelier.Connector.Jvn.Tests/StellaOps.Concelier.Connector.Jvn.Tests.csproj new file mode 100644 index 00000000..69a9f212 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Jvn.Tests/StellaOps.Concelier.Connector.Jvn.Tests.csproj @@ -0,0 +1,16 @@ + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md b/src/StellaOps.Concelier.Connector.Jvn/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.Jvn/AGENTS.md rename to src/StellaOps.Concelier.Connector.Jvn/AGENTS.md index 8c2234d1..38e3f736 100644 --- a/src/StellaOps.Feedser.Source.Jvn/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Jvn/AGENTS.md @@ -21,9 +21,9 @@ Japan JVN/MyJVN connector; national CERT enrichment with strong identifiers (JVN In: JVN/MyJVN ingestion, aliases, jp_flags, enrichment mapping, watermarking. Out: overriding distro or PSIRT ranges without concrete evidence; scraping unofficial mirrors. ## Observability & security expectations -- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=jvn`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. +- Metrics: SourceDiagnostics emits `concelier.source.http.*` counters/histograms tagged `concelier.source=jvn`, enabling dashboards to track fetch requests, item counts, parse failures, and enrichment/map activity (including jp_flags) via tag filters. - Logs: window bounds, jvndb ids processed, vendor_status distribution; redact API keys. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Jvn.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Jvn.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs b/src/StellaOps.Concelier.Connector.Jvn/Configuration/JvnOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs rename to src/StellaOps.Concelier.Connector.Jvn/Configuration/JvnOptions.cs index 56a3fa81..a2c5bb7c 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Configuration/JvnOptions.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Configuration/JvnOptions.cs @@ -1,6 +1,6 @@ using System.Diagnostics.CodeAnalysis; -namespace StellaOps.Feedser.Source.Jvn.Configuration; +namespace StellaOps.Concelier.Connector.Jvn.Configuration; /// /// Options controlling the JVN connector fetch cadence and HTTP client configuration. diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnAdvisoryMapper.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnAdvisoryMapper.cs index b6544e86..535d42c6 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnAdvisoryMapper.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnAdvisoryMapper.cs @@ -1,16 +1,16 @@ using System; using System.Collections.Generic; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.Identifiers; -using StellaOps.Feedser.Normalization.Text; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Text; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.JpFlags; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal static class JvnAdvisoryMapper { diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnConstants.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnConstants.cs index a3f3fb43..edc3849d 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnConstants.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnConstants.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal static class JvnConstants { diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnCursor.cs index 25be9887..585799b8 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnCursor.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnCursor.cs @@ -1,7 +1,7 @@ using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal sealed record JvnCursor( DateTimeOffset? WindowStart, diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailDto.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailDto.cs index 9e3449bb..b8c2e05c 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailDto.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailDto.cs @@ -1,6 +1,6 @@ using System.Collections.Immutable; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal sealed record JvnDetailDto( string VulnerabilityId, diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailParser.cs index 24faacb9..4ad414e4 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnDetailParser.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnDetailParser.cs @@ -8,7 +8,7 @@ using System.Xml; using System.Xml.Linq; using System.Xml.Schema; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal static class JvnDetailParser { diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewItem.cs similarity index 74% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewItem.cs index cb421e0c..a80468cc 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewItem.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewItem.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal sealed record JvnOverviewItem( string VulnerabilityId, diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewPage.cs similarity index 71% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewPage.cs index f63779ba..7a73846b 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnOverviewPage.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnOverviewPage.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal sealed record JvnOverviewPage( IReadOnlyList Items, diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaProvider.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaProvider.cs index 075ce424..7d85f559 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaProvider.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaProvider.cs @@ -7,7 +7,7 @@ using System.Threading; using System.Xml; using System.Xml.Schema; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal static class JvnSchemaProvider { @@ -47,7 +47,7 @@ internal static class JvnSchemaProvider private static Dictionary CreateResourceMap() { - var baseNamespace = typeof(JvnSchemaProvider).Namespace ?? "StellaOps.Feedser.Source.Jvn.Internal"; + var baseNamespace = typeof(JvnSchemaProvider).Namespace ?? "StellaOps.Concelier.Connector.Jvn.Internal"; var prefix = baseNamespace.Replace(".Internal", string.Empty, StringComparison.Ordinal); return new Dictionary(StringComparer.OrdinalIgnoreCase) diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaValidationException.cs similarity index 81% rename from src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaValidationException.cs index 0015fa5f..0bebf0bd 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/JvnSchemaValidationException.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/JvnSchemaValidationException.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; internal sealed class JvnSchemaValidationException : Exception { diff --git a/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs b/src/StellaOps.Concelier.Connector.Jvn/Internal/MyJvnClient.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs rename to src/StellaOps.Concelier.Connector.Jvn/Internal/MyJvnClient.cs index 319358cc..09edf103 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Internal/MyJvnClient.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Internal/MyJvnClient.cs @@ -9,9 +9,9 @@ using System.Xml; using System.Xml.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Concelier.Connector.Jvn.Configuration; -namespace StellaOps.Feedser.Source.Jvn.Internal; +namespace StellaOps.Concelier.Connector.Jvn.Internal; public sealed class MyJvnClient { diff --git a/src/StellaOps.Feedser.Source.Jvn/Jobs.cs b/src/StellaOps.Concelier.Connector.Jvn/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Jvn/Jobs.cs rename to src/StellaOps.Concelier.Connector.Jvn/Jobs.cs index e56571d1..b1d6123f 100644 --- a/src/StellaOps.Feedser.Source.Jvn/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Jvn; +namespace StellaOps.Concelier.Connector.Jvn; internal static class JvnJobKinds { diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs b/src/StellaOps.Concelier.Connector.Jvn/JvnConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs rename to src/StellaOps.Concelier.Connector.Jvn/JvnConnector.cs index 0be9aadf..14493215 100644 --- a/src/StellaOps.Feedser.Source.Jvn/JvnConnector.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/JvnConnector.cs @@ -4,19 +4,19 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Jvn.Configuration; -using StellaOps.Feedser.Source.Jvn.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.JpFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Jvn.Configuration; +using StellaOps.Concelier.Connector.Jvn.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.JpFlags; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Jvn; +namespace StellaOps.Concelier.Connector.Jvn; public sealed class JvnConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs index 406c4cb2..8c253f02 100644 --- a/src/StellaOps.Feedser.Source.Jvn/JvnConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/JvnConnectorPlugin.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Jvn; +namespace StellaOps.Concelier.Connector.Jvn; public sealed class JvnConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs index 1b7accd1..d98393f8 100644 --- a/src/StellaOps.Feedser.Source.Jvn/JvnDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/JvnDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Jvn.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Jvn.Configuration; -namespace StellaOps.Feedser.Source.Jvn; +namespace StellaOps.Concelier.Connector.Jvn; public sealed class JvnDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:jvn"; + private const string ConfigurationSection = "concelier:sources:jvn"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Jvn/JvnServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Jvn/JvnServiceCollectionExtensions.cs index 2ce8d662..d7ddb5d9 100644 --- a/src/StellaOps.Feedser.Source.Jvn/JvnServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Jvn/JvnServiceCollectionExtensions.cs @@ -1,11 +1,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Jvn.Configuration; -using StellaOps.Feedser.Source.Jvn.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Jvn.Configuration; +using StellaOps.Concelier.Connector.Jvn.Internal; -namespace StellaOps.Feedser.Source.Jvn; +namespace StellaOps.Concelier.Connector.Jvn; public static class JvnServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class JvnServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = options.BaseEndpoint; clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Jvn/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.Jvn/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/xml"; diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/data_marking.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/data_marking.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/data_marking.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/jvnrss_3.2.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/jvnrss_3.2.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/jvnrss_3.2.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/mod_sec_3.0.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/mod_sec_3.0.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/mod_sec_3.0.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/status_3.3.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/status_3.3.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/status_3.3.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/tlp_marking.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/tlp_marking.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/tlp_marking.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/vuldef_3.2.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/vuldef_3.2.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/vuldef_3.2.xsd diff --git a/src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd b/src/StellaOps.Concelier.Connector.Jvn/Schemas/xml.xsd similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/Schemas/xml.xsd rename to src/StellaOps.Concelier.Connector.Jvn/Schemas/xml.xsd diff --git a/src/StellaOps.Concelier.Connector.Jvn/StellaOps.Concelier.Connector.Jvn.csproj b/src/StellaOps.Concelier.Connector.Jvn/StellaOps.Concelier.Connector.Jvn.csproj new file mode 100644 index 00000000..6662142d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Jvn/StellaOps.Concelier.Connector.Jvn.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Jvn/TASKS.md b/src/StellaOps.Concelier.Connector.Jvn/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Jvn/TASKS.md rename to src/StellaOps.Concelier.Connector.Jvn/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Kev.Tests/Kev/Fixtures/kev-advisories.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/Fixtures/kev-catalog.json similarity index 97% rename from src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json rename to src/StellaOps.Concelier.Connector.Kev.Tests/Kev/Fixtures/kev-catalog.json index f9733e7d..07d5a912 100644 --- a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/Fixtures/kev-catalog.json +++ b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/Fixtures/kev-catalog.json @@ -1,38 +1,38 @@ -{ - "title": "CISA Catalog of Known Exploited Vulnerabilities", - "catalogVersion": "2025.10.09", - "dateReleased": "2025-10-09T16:52:28.6547Z", - "count": 2, - "vulnerabilities": [ - { - "cveID": "CVE-2021-43798", - "vendorProject": "Grafana Labs", - "product": "Grafana", - "vulnerabilityName": "Grafana Path Traversal Vulnerability", - "dateAdded": "2025-10-09", - "shortDescription": "Grafana contains a path traversal vulnerability that could allow access to local files.", - "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.", - "dueDate": "2025-10-30", - "knownRansomwareCampaignUse": "Unknown", - "notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", - "cwes": [ - "CWE-22" - ] - }, - { - "cveID": "CVE-2024-12345", - "vendorProject": "Acme Corp", - "product": "Acme Widget", - "vulnerabilityName": "Acme Widget Buffer Overflow", - "dateAdded": "2025-08-01", - "shortDescription": "Acme Widget contains a buffer overflow that may allow remote code execution.", - "requiredAction": "Apply vendor patch KB-1234.", - "knownRansomwareCampaignUse": "Confirmed", - "notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored", - "cwes": [ - "CWE-120", - "CWE-787" - ] - } - ] -} +{ + "title": "CISA Catalog of Known Exploited Vulnerabilities", + "catalogVersion": "2025.10.09", + "dateReleased": "2025-10-09T16:52:28.6547Z", + "count": 2, + "vulnerabilities": [ + { + "cveID": "CVE-2021-43798", + "vendorProject": "Grafana Labs", + "product": "Grafana", + "vulnerabilityName": "Grafana Path Traversal Vulnerability", + "dateAdded": "2025-10-09", + "shortDescription": "Grafana contains a path traversal vulnerability that could allow access to local files.", + "requiredAction": "Apply mitigations per vendor instructions, follow applicable BOD 22-01 guidance for cloud services, or discontinue use of the product if mitigations are unavailable.", + "dueDate": "2025-10-30", + "knownRansomwareCampaignUse": "Unknown", + "notes": "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", + "cwes": [ + "CWE-22" + ] + }, + { + "cveID": "CVE-2024-12345", + "vendorProject": "Acme Corp", + "product": "Acme Widget", + "vulnerabilityName": "Acme Widget Buffer Overflow", + "dateAdded": "2025-08-01", + "shortDescription": "Acme Widget contains a buffer overflow that may allow remote code execution.", + "requiredAction": "Apply vendor patch KB-1234.", + "knownRansomwareCampaignUse": "Confirmed", + "notes": "https://acme.example/advisories/KB-1234 https://nvd.nist.gov/vuln/detail/CVE-2024-12345 additional context ignored", + "cwes": [ + "CWE-120", + "CWE-787" + ] + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs index d681011c..a466dff2 100644 --- a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevConnectorTests.cs @@ -1,218 +1,218 @@ -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Kev; -using StellaOps.Feedser.Source.Kev.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Kev.Tests; - -[Collection("mongo-fixture")] -public sealed class KevConnectorTests : IAsyncLifetime -{ - private static readonly Uri FeedUri = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); - private const string CatalogEtag = "\"kev-2025-10-09\""; - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - - public KevConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesDeterministicSnapshot() - { - await using var provider = await BuildServiceProviderAsync(); - SeedCatalogResponse(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - _timeProvider.Advance(TimeSpan.FromMinutes(1)); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.NotEmpty(advisories); - - var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); - var snapshot = SnapshotSerializer.ToSnapshot(ordered); - WriteOrAssertSnapshot(snapshot, "kev-advisories.snapshot.json"); - - var documentStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(KevConnectorPlugin.SourceName, FeedUri.ToString(), CancellationToken.None); - Assert.NotNull(document); - Assert.Equal(DocumentStatuses.Mapped, document!.Status); - - SeedNotModifiedResponse(); - await connector.FetchAsync(provider, CancellationToken.None); - _handler.AssertNoPendingResponses(); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(KevConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.Equal("2025.10.09", state!.Cursor.TryGetValue("catalogVersion", out var versionValue) ? versionValue.AsString : null); - Assert.True(state.Cursor.TryGetValue("catalogReleased", out var releasedValue) && releasedValue.BsonType is BsonType.DateTime); - Assert.True(IsEmptyArray(state.Cursor, "pendingDocuments")); - Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddKevConnector(options => - { - options.FeedUri = FeedUri; - options.RequestTimeout = TimeSpan.FromSeconds(10); - }); - - services.Configure(KevOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedCatalogResponse() - { - var payload = ReadFixture("kev-catalog.json"); - _handler.AddResponse(FeedUri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); - response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero); - return response; - }); - } - - private void SeedNotModifiedResponse() - { - _handler.AddResponse(FeedUri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.NotModified); - response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); - return response; - }); - } - - private static bool IsEmptyArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return false; - } - - return array.Count == 0; - } - - private static string ReadFixture(string filename) - { - var path = GetExistingFixturePath(filename); - return File.ReadAllText(path); - } - - private static void WriteOrAssertSnapshot(string snapshot, string filename) - { - if (ShouldUpdateFixtures()) - { - var target = GetWritableFixturePath(filename); - File.WriteAllText(target, snapshot); - return; - } - - var expected = ReadFixture(filename); - var normalizedExpected = Normalize(expected); - var normalizedSnapshot = Normalize(snapshot); - - if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(normalizedExpected, normalizedSnapshot); - } - - private static bool ShouldUpdateFixtures() - { - var value = Environment.GetEnvironmentVariable("UPDATE_KEV_FIXTURES"); - return string.Equals(value, "1", StringComparison.Ordinal) || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); - } - - private static string Normalize(string value) - => value.Replace("\r\n", "\n", StringComparison.Ordinal); - - private static string GetExistingFixturePath(string filename) - { - var baseDir = AppContext.BaseDirectory; - var primary = Path.Combine(baseDir, "Source", "Kev", "Fixtures", filename); - if (File.Exists(primary)) - { - return primary; - } - - var fallback = Path.Combine(baseDir, "Kev", "Fixtures", filename); - if (File.Exists(fallback)) - { - return fallback; - } - - throw new FileNotFoundException($"Unable to locate KEV fixture '{filename}'."); - } - - private static string GetWritableFixturePath(string filename) - { - var baseDir = AppContext.BaseDirectory; - var primaryDir = Path.Combine(baseDir, "Source", "Kev", "Fixtures"); - Directory.CreateDirectory(primaryDir); - return Path.Combine(primaryDir, filename); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public async Task DisposeAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - } -} +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Kev; +using StellaOps.Concelier.Connector.Kev.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Kev.Tests; + +[Collection("mongo-fixture")] +public sealed class KevConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); + private const string CatalogEtag = "\"kev-2025-10-09\""; + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public KevConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedCatalogResponse(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.NotEmpty(advisories); + + var ordered = advisories.OrderBy(static a => a.AdvisoryKey, StringComparer.Ordinal).ToArray(); + var snapshot = SnapshotSerializer.ToSnapshot(ordered); + WriteOrAssertSnapshot(snapshot, "kev-advisories.snapshot.json"); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(KevConnectorPlugin.SourceName, FeedUri.ToString(), CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + SeedNotModifiedResponse(); + await connector.FetchAsync(provider, CancellationToken.None); + _handler.AssertNoPendingResponses(); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KevConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.Equal("2025.10.09", state!.Cursor.TryGetValue("catalogVersion", out var versionValue) ? versionValue.AsString : null); + Assert.True(state.Cursor.TryGetValue("catalogReleased", out var releasedValue) && releasedValue.BsonType is BsonType.DateTime); + Assert.True(IsEmptyArray(state.Cursor, "pendingDocuments")); + Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddKevConnector(options => + { + options.FeedUri = FeedUri; + options.RequestTimeout = TimeSpan.FromSeconds(10); + }); + + services.Configure(KevOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedCatalogResponse() + { + var payload = ReadFixture("kev-catalog.json"); + _handler.AddResponse(FeedUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero); + return response; + }); + } + + private void SeedNotModifiedResponse() + { + _handler.AddResponse(FeedUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.NotModified); + response.Headers.ETag = new EntityTagHeaderValue(CatalogEtag); + return response; + }); + } + + private static bool IsEmptyArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return false; + } + + return array.Count == 0; + } + + private static string ReadFixture(string filename) + { + var path = GetExistingFixturePath(filename); + return File.ReadAllText(path); + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + if (ShouldUpdateFixtures()) + { + var target = GetWritableFixturePath(filename); + File.WriteAllText(target, snapshot); + return; + } + + var expected = ReadFixture(filename); + var normalizedExpected = Normalize(expected); + var normalizedSnapshot = Normalize(snapshot); + + if (!string.Equals(normalizedExpected, normalizedSnapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(Path.GetDirectoryName(GetWritableFixturePath(filename))!, Path.GetFileNameWithoutExtension(filename) + ".actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(normalizedExpected, normalizedSnapshot); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable("UPDATE_KEV_FIXTURES"); + return string.Equals(value, "1", StringComparison.Ordinal) || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string Normalize(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string GetExistingFixturePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primary = Path.Combine(baseDir, "Source", "Kev", "Fixtures", filename); + if (File.Exists(primary)) + { + return primary; + } + + var fallback = Path.Combine(baseDir, "Kev", "Fixtures", filename); + if (File.Exists(fallback)) + { + return fallback; + } + + throw new FileNotFoundException($"Unable to locate KEV fixture '{filename}'."); + } + + private static string GetWritableFixturePath(string filename) + { + var baseDir = AppContext.BaseDirectory; + var primaryDir = Path.Combine(baseDir, "Source", "Kev", "Fixtures"); + Directory.CreateDirectory(primaryDir); + return Path.Combine(primaryDir, filename); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevMapperTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs rename to src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevMapperTests.cs index 50032c5c..9e2395be 100644 --- a/src/StellaOps.Feedser.Source.Kev.Tests/Kev/KevMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Kev.Tests/Kev/KevMapperTests.cs @@ -1,93 +1,93 @@ -using System; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Kev; -using StellaOps.Feedser.Source.Kev.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Kev.Tests; - -public sealed class KevMapperTests -{ - [Fact] - public void Map_BuildsVendorRangePrimitivesWithDueDate() - { - var catalog = new KevCatalogDto - { - CatalogVersion = "2025.10.09", - DateReleased = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero), - Vulnerabilities = new[] - { - new KevVulnerabilityDto - { - CveId = "CVE-2021-43798", - VendorProject = "Grafana Labs", - Product = "Grafana", - VulnerabilityName = "Grafana Path Traversal Vulnerability", - DateAdded = "2025-10-09", - ShortDescription = "Grafana contains a path traversal vulnerability that could allow access to local files.", - RequiredAction = "Apply mitigations per vendor instructions or discontinue use.", - DueDate = "2025-10-30", - KnownRansomwareCampaignUse = "Unknown", - Notes = "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", - Cwes = new[] { "CWE-22" } - } - } - }; - - var feedUri = new Uri("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); - var fetchedAt = new DateTimeOffset(2025, 10, 9, 17, 0, 0, TimeSpan.Zero); - var validatedAt = fetchedAt.AddMinutes(1); - - var advisories = KevMapper.Map(catalog, KevConnectorPlugin.SourceName, feedUri, fetchedAt, validatedAt); - - var advisory = Assert.Single(advisories); - Assert.True(advisory.ExploitKnown); - Assert.Contains("cve-2021-43798", advisory.Aliases, StringComparer.OrdinalIgnoreCase); - - var affected = Assert.Single(advisory.AffectedPackages); - Assert.Equal(AffectedPackageTypes.Vendor, affected.Type); - Assert.Equal("Grafana Labs::Grafana", affected.Identifier); - - Assert.Collection( - affected.NormalizedVersions, - rule => - { - Assert.Equal("kev.catalog", rule.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); - Assert.Equal("2025.10.09", rule.Value); - Assert.Equal("Grafana Labs::Grafana", rule.Notes); - }, - rule => - { - Assert.Equal("kev.date-added", rule.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); - Assert.Equal("2025-10-09", rule.Value); - }, - rule => - { - Assert.Equal("kev.due-date", rule.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, rule.Type); - Assert.Equal("2025-10-30", rule.Max); - Assert.True(rule.MaxInclusive); - }); - - var range = Assert.Single(affected.VersionRanges); - Assert.Equal(AffectedPackageTypes.Vendor, range.RangeKind); - var primitives = range.Primitives; - Assert.NotNull(primitives); - - Assert.True(primitives!.HasVendorExtensions); - var extensions = primitives!.VendorExtensions!; - Assert.Equal("Grafana Labs", extensions["kev.vendorProject"]); - Assert.Equal("Grafana", extensions["kev.product"]); - Assert.Equal("2025-10-30", extensions["kev.dueDate"]); - Assert.Equal("Unknown", extensions["kev.knownRansomwareCampaignUse"]); - Assert.Equal("CWE-22", extensions["kev.cwe"]); - - var references = advisory.References.Select(reference => reference.Url).ToArray(); - Assert.Contains("https://grafana.com/security/advisory", references); - Assert.Contains("https://nvd.nist.gov/vuln/detail/CVE-2021-43798", references); - Assert.Contains("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798", references); - } -} +using System; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Kev; +using StellaOps.Concelier.Connector.Kev.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Kev.Tests; + +public sealed class KevMapperTests +{ + [Fact] + public void Map_BuildsVendorRangePrimitivesWithDueDate() + { + var catalog = new KevCatalogDto + { + CatalogVersion = "2025.10.09", + DateReleased = new DateTimeOffset(2025, 10, 9, 16, 52, 28, TimeSpan.Zero), + Vulnerabilities = new[] + { + new KevVulnerabilityDto + { + CveId = "CVE-2021-43798", + VendorProject = "Grafana Labs", + Product = "Grafana", + VulnerabilityName = "Grafana Path Traversal Vulnerability", + DateAdded = "2025-10-09", + ShortDescription = "Grafana contains a path traversal vulnerability that could allow access to local files.", + RequiredAction = "Apply mitigations per vendor instructions or discontinue use.", + DueDate = "2025-10-30", + KnownRansomwareCampaignUse = "Unknown", + Notes = "https://grafana.com/security/advisory; https://nvd.nist.gov/vuln/detail/CVE-2021-43798", + Cwes = new[] { "CWE-22" } + } + } + }; + + var feedUri = new Uri("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"); + var fetchedAt = new DateTimeOffset(2025, 10, 9, 17, 0, 0, TimeSpan.Zero); + var validatedAt = fetchedAt.AddMinutes(1); + + var advisories = KevMapper.Map(catalog, KevConnectorPlugin.SourceName, feedUri, fetchedAt, validatedAt); + + var advisory = Assert.Single(advisories); + Assert.True(advisory.ExploitKnown); + Assert.Contains("cve-2021-43798", advisory.Aliases, StringComparer.OrdinalIgnoreCase); + + var affected = Assert.Single(advisory.AffectedPackages); + Assert.Equal(AffectedPackageTypes.Vendor, affected.Type); + Assert.Equal("Grafana Labs::Grafana", affected.Identifier); + + Assert.Collection( + affected.NormalizedVersions, + rule => + { + Assert.Equal("kev.catalog", rule.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); + Assert.Equal("2025.10.09", rule.Value); + Assert.Equal("Grafana Labs::Grafana", rule.Notes); + }, + rule => + { + Assert.Equal("kev.date-added", rule.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); + Assert.Equal("2025-10-09", rule.Value); + }, + rule => + { + Assert.Equal("kev.due-date", rule.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, rule.Type); + Assert.Equal("2025-10-30", rule.Max); + Assert.True(rule.MaxInclusive); + }); + + var range = Assert.Single(affected.VersionRanges); + Assert.Equal(AffectedPackageTypes.Vendor, range.RangeKind); + var primitives = range.Primitives; + Assert.NotNull(primitives); + + Assert.True(primitives!.HasVendorExtensions); + var extensions = primitives!.VendorExtensions!; + Assert.Equal("Grafana Labs", extensions["kev.vendorProject"]); + Assert.Equal("Grafana", extensions["kev.product"]); + Assert.Equal("2025-10-30", extensions["kev.dueDate"]); + Assert.Equal("Unknown", extensions["kev.knownRansomwareCampaignUse"]); + Assert.Equal("CWE-22", extensions["kev.cwe"]); + + var references = advisory.References.Select(reference => reference.Url).ToArray(); + Assert.Contains("https://grafana.com/security/advisory", references); + Assert.Contains("https://nvd.nist.gov/vuln/detail/CVE-2021-43798", references); + Assert.Contains("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search=CVE-2021-43798", references); + } +} diff --git a/src/StellaOps.Concelier.Connector.Kev.Tests/StellaOps.Concelier.Connector.Kev.Tests.csproj b/src/StellaOps.Concelier.Connector.Kev.Tests/StellaOps.Concelier.Connector.Kev.Tests.csproj new file mode 100644 index 00000000..1fbd9bbc --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Kev.Tests/StellaOps.Concelier.Connector.Kev.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Kev/AGENTS.md b/src/StellaOps.Concelier.Connector.Kev/AGENTS.md similarity index 88% rename from src/StellaOps.Feedser.Source.Kev/AGENTS.md rename to src/StellaOps.Concelier.Connector.Kev/AGENTS.md index 9665f59e..4b238198 100644 --- a/src/StellaOps.Feedser.Source.Kev/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Kev/AGENTS.md @@ -1,44 +1,44 @@ -# AGENTS -## Role -Implement the CISA Known Exploited Vulnerabilities (KEV) catalogue connector to ingest KEV entries for enrichment and policy checks. - -## Scope -- Integrate with the official KEV JSON feed; understand schema, update cadence, and pagination (if any). -- Implement fetch job with incremental updates, checksum validation, and cursor persistence. -- Parse KEV entries (CVE ID, vendor/product, required actions, due dates). -- Map entries into canonical `Advisory` (or augmentation) records with aliases, references, affected packages, and range primitives capturing enforcement metadata. -- Deliver deterministic fixtures and regression tests. - -## Participants -- `Source.Common` (HTTP client, fetch service, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). -- `Feedser.Models` (advisory + range primitive types). -- `Feedser.Testing` (integration fixtures & snapshots). - -## Interfaces & Contracts -- Job kinds: `kev:fetch`, `kev:parse`, `kev:map`. -- Persist upstream `catalogLastUpdated` / ETag to detect changes. -- Alias list must include CVE ID; references should point to CISA KEV listing and vendor advisories. - -## In/Out of scope -In scope: -- KEV feed ingestion and canonical mapping. -- Range primitives capturing remediation due dates or vendor requirements. - -Out of scope: -- Compliance policy enforcement (handled elsewhere). - -## Observability & Security Expectations -- Log fetch timestamps, updated entry counts, and mapping stats. -- Handle data anomalies and record failures with backoff. -- Validate JSON payloads before persistence. -- Structured informational logs should surface the catalog version, release timestamp, and advisory counts for each successful parse/map cycle. - -## Operational Notes -- HTTP allowlist is limited to `www.cisa.gov`; operators should mirror / proxy that hostname for air-gapped deployments. -- CISA publishes KEV updates daily (catalogVersion follows `yyyy.MM.dd`). Expect releases near 16:30–17:00 UTC and retain overlap when scheduling fetches. - -## Tests -- Add `StellaOps.Feedser.Source.Kev.Tests` covering fetch/parse/map with KEV JSON fixtures. -- Snapshot canonical output; allow fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Implement the CISA Known Exploited Vulnerabilities (KEV) catalogue connector to ingest KEV entries for enrichment and policy checks. + +## Scope +- Integrate with the official KEV JSON feed; understand schema, update cadence, and pagination (if any). +- Implement fetch job with incremental updates, checksum validation, and cursor persistence. +- Parse KEV entries (CVE ID, vendor/product, required actions, due dates). +- Map entries into canonical `Advisory` (or augmentation) records with aliases, references, affected packages, and range primitives capturing enforcement metadata. +- Deliver deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP client, fetch service, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Concelier.Models` (advisory + range primitive types). +- `Concelier.Testing` (integration fixtures & snapshots). + +## Interfaces & Contracts +- Job kinds: `kev:fetch`, `kev:parse`, `kev:map`. +- Persist upstream `catalogLastUpdated` / ETag to detect changes. +- Alias list must include CVE ID; references should point to CISA KEV listing and vendor advisories. + +## In/Out of scope +In scope: +- KEV feed ingestion and canonical mapping. +- Range primitives capturing remediation due dates or vendor requirements. + +Out of scope: +- Compliance policy enforcement (handled elsewhere). + +## Observability & Security Expectations +- Log fetch timestamps, updated entry counts, and mapping stats. +- Handle data anomalies and record failures with backoff. +- Validate JSON payloads before persistence. +- Structured informational logs should surface the catalog version, release timestamp, and advisory counts for each successful parse/map cycle. + +## Operational Notes +- HTTP allowlist is limited to `www.cisa.gov`; operators should mirror / proxy that hostname for air-gapped deployments. +- CISA publishes KEV updates daily (catalogVersion follows `yyyy.MM.dd`). Expect releases near 16:30–17:00 UTC and retain overlap when scheduling fetches. + +## Tests +- Add `StellaOps.Concelier.Connector.Kev.Tests` covering fetch/parse/map with KEV JSON fixtures. +- Snapshot canonical output; allow fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs b/src/StellaOps.Concelier.Connector.Kev/Configuration/KevOptions.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs rename to src/StellaOps.Concelier.Connector.Kev/Configuration/KevOptions.cs index c8de4041..3bb3b20a 100644 --- a/src/StellaOps.Feedser.Source.Kev/Configuration/KevOptions.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Configuration/KevOptions.cs @@ -1,33 +1,33 @@ -using System; -using System.Diagnostics.CodeAnalysis; - -namespace StellaOps.Feedser.Source.Kev.Configuration; - -public sealed class KevOptions -{ - public static string HttpClientName => "source.kev"; - - /// - /// Official CISA Known Exploited Vulnerabilities JSON feed. - /// - public Uri FeedUri { get; set; } = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", UriKind.Absolute); - - /// - /// Timeout applied to KEV feed requests. - /// - public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); - - [MemberNotNull(nameof(FeedUri))] - public void Validate() - { - if (FeedUri is null || !FeedUri.IsAbsoluteUri) - { - throw new InvalidOperationException("FeedUri must be an absolute URI."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("RequestTimeout must be greater than zero."); - } - } -} +using System; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Concelier.Connector.Kev.Configuration; + +public sealed class KevOptions +{ + public static string HttpClientName => "source.kev"; + + /// + /// Official CISA Known Exploited Vulnerabilities JSON feed. + /// + public Uri FeedUri { get; set; } = new("https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", UriKind.Absolute); + + /// + /// Timeout applied to KEV feed requests. + /// + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + [MemberNotNull(nameof(FeedUri))] + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("FeedUri must be an absolute URI."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RequestTimeout must be greater than zero."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs b/src/StellaOps.Concelier.Connector.Kev/Internal/KevCatalogDto.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs rename to src/StellaOps.Concelier.Connector.Kev/Internal/KevCatalogDto.cs index f786cc82..74372540 100644 --- a/src/StellaOps.Feedser.Source.Kev/Internal/KevCatalogDto.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Internal/KevCatalogDto.cs @@ -1,59 +1,59 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Kev.Internal; - -internal sealed record KevCatalogDto -{ - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("catalogVersion")] - public string? CatalogVersion { get; init; } - - [JsonPropertyName("dateReleased")] - public DateTimeOffset? DateReleased { get; init; } - - [JsonPropertyName("count")] - public int Count { get; init; } - - [JsonPropertyName("vulnerabilities")] - public IReadOnlyList Vulnerabilities { get; init; } = Array.Empty(); -} - -internal sealed record KevVulnerabilityDto -{ - [JsonPropertyName("cveID")] - public string? CveId { get; init; } - - [JsonPropertyName("vendorProject")] - public string? VendorProject { get; init; } - - [JsonPropertyName("product")] - public string? Product { get; init; } - - [JsonPropertyName("vulnerabilityName")] - public string? VulnerabilityName { get; init; } - - [JsonPropertyName("dateAdded")] - public string? DateAdded { get; init; } - - [JsonPropertyName("shortDescription")] - public string? ShortDescription { get; init; } - - [JsonPropertyName("requiredAction")] - public string? RequiredAction { get; init; } - - [JsonPropertyName("dueDate")] - public string? DueDate { get; init; } - - [JsonPropertyName("knownRansomwareCampaignUse")] - public string? KnownRansomwareCampaignUse { get; init; } - - [JsonPropertyName("notes")] - public string? Notes { get; init; } - - [JsonPropertyName("cwes")] - public IReadOnlyList Cwes { get; init; } = Array.Empty(); -} +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Kev.Internal; + +internal sealed record KevCatalogDto +{ + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("catalogVersion")] + public string? CatalogVersion { get; init; } + + [JsonPropertyName("dateReleased")] + public DateTimeOffset? DateReleased { get; init; } + + [JsonPropertyName("count")] + public int Count { get; init; } + + [JsonPropertyName("vulnerabilities")] + public IReadOnlyList Vulnerabilities { get; init; } = Array.Empty(); +} + +internal sealed record KevVulnerabilityDto +{ + [JsonPropertyName("cveID")] + public string? CveId { get; init; } + + [JsonPropertyName("vendorProject")] + public string? VendorProject { get; init; } + + [JsonPropertyName("product")] + public string? Product { get; init; } + + [JsonPropertyName("vulnerabilityName")] + public string? VulnerabilityName { get; init; } + + [JsonPropertyName("dateAdded")] + public string? DateAdded { get; init; } + + [JsonPropertyName("shortDescription")] + public string? ShortDescription { get; init; } + + [JsonPropertyName("requiredAction")] + public string? RequiredAction { get; init; } + + [JsonPropertyName("dueDate")] + public string? DueDate { get; init; } + + [JsonPropertyName("knownRansomwareCampaignUse")] + public string? KnownRansomwareCampaignUse { get; init; } + + [JsonPropertyName("notes")] + public string? Notes { get; init; } + + [JsonPropertyName("cwes")] + public IReadOnlyList Cwes { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs b/src/StellaOps.Concelier.Connector.Kev/Internal/KevCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs rename to src/StellaOps.Concelier.Connector.Kev/Internal/KevCursor.cs index 7a1028dd..2662ebae 100644 --- a/src/StellaOps.Feedser.Source.Kev/Internal/KevCursor.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Internal/KevCursor.cs @@ -1,103 +1,103 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Kev.Internal; - -internal sealed record KevCursor( - string? CatalogVersion, - DateTimeOffset? CatalogReleased, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - public static KevCursor Empty { get; } = new(null, null, Array.Empty(), Array.Empty()); - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), - }; - - if (!string.IsNullOrEmpty(CatalogVersion)) - { - document["catalogVersion"] = CatalogVersion; - } - - if (CatalogReleased.HasValue) - { - document["catalogReleased"] = CatalogReleased.Value.UtcDateTime; - } - - return document; - } - - public static KevCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var version = document.TryGetValue("catalogVersion", out var versionValue) - ? versionValue.AsString - : null; - - var released = document.TryGetValue("catalogReleased", out var releasedValue) - ? ParseDate(releasedValue) - : null; - - return new KevCursor( - version, - released, - ReadGuidArray(document, "pendingDocuments"), - ReadGuidArray(document, "pendingMappings")); - } - - public KevCursor WithCatalogMetadata(string? version, DateTimeOffset? released) - => this with - { - CatalogVersion = string.IsNullOrWhiteSpace(version) ? null : version.Trim(), - CatalogReleased = released?.ToUniversalTime(), - }; - - public KevCursor WithPendingDocuments(IEnumerable ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; - - public KevCursor WithPendingMappings(IEnumerable ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return Array.Empty(); - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (element is null) - { - continue; - } - - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Kev.Internal; + +internal sealed record KevCursor( + string? CatalogVersion, + DateTimeOffset? CatalogReleased, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + public static KevCursor Empty { get; } = new(null, null, Array.Empty(), Array.Empty()); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(static id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(static id => id.ToString())), + }; + + if (!string.IsNullOrEmpty(CatalogVersion)) + { + document["catalogVersion"] = CatalogVersion; + } + + if (CatalogReleased.HasValue) + { + document["catalogReleased"] = CatalogReleased.Value.UtcDateTime; + } + + return document; + } + + public static KevCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var version = document.TryGetValue("catalogVersion", out var versionValue) + ? versionValue.AsString + : null; + + var released = document.TryGetValue("catalogReleased", out var releasedValue) + ? ParseDate(releasedValue) + : null; + + return new KevCursor( + version, + released, + ReadGuidArray(document, "pendingDocuments"), + ReadGuidArray(document, "pendingMappings")); + } + + public KevCursor WithCatalogMetadata(string? version, DateTimeOffset? released) + => this with + { + CatalogVersion = string.IsNullOrWhiteSpace(version) ? null : version.Trim(), + CatalogReleased = released?.ToUniversalTime(), + }; + + public KevCursor WithPendingDocuments(IEnumerable ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? Array.Empty() }; + + public KevCursor WithPendingMappings(IEnumerable ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? Array.Empty() }; + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return Array.Empty(); + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs b/src/StellaOps.Concelier.Connector.Kev/Internal/KevDiagnostics.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Kev/Internal/KevDiagnostics.cs index 8602dd7d..5aad43e7 100644 --- a/src/StellaOps.Feedser.Source.Kev/Internal/KevDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Internal/KevDiagnostics.cs @@ -1,113 +1,113 @@ -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Kev.Internal; - -public sealed class KevDiagnostics : IDisposable -{ - public const string MeterName = "StellaOps.Feedser.Source.Kev"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _fetchAttempts; - private readonly Counter _fetchSuccess; - private readonly Counter _fetchFailures; - private readonly Counter _fetchUnchanged; - private readonly Counter _parsedEntries; - private readonly Counter _parseFailures; - private readonly Counter _parseAnomalies; - private readonly Counter _mappedAdvisories; - - public KevDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchAttempts = _meter.CreateCounter( - name: "kev.fetch.attempts", - unit: "operations", - description: "Number of KEV fetch attempts performed."); - _fetchSuccess = _meter.CreateCounter( - name: "kev.fetch.success", - unit: "operations", - description: "Number of KEV fetch attempts that produced new catalog content."); - _fetchFailures = _meter.CreateCounter( - name: "kev.fetch.failures", - unit: "operations", - description: "Number of KEV fetch attempts that failed."); - _fetchUnchanged = _meter.CreateCounter( - name: "kev.fetch.unchanged", - unit: "operations", - description: "Number of KEV fetch attempts returning HTTP 304 / unchanged catalog."); - _parsedEntries = _meter.CreateCounter( - name: "kev.parse.entries", - unit: "entries", - description: "Number of KEV vulnerabilities parsed from the catalog."); - _parseFailures = _meter.CreateCounter( - name: "kev.parse.failures", - unit: "documents", - description: "Number of KEV catalog parse operations that failed or were quarantined."); - _parseAnomalies = _meter.CreateCounter( - name: "kev.parse.anomalies", - unit: "entries", - description: "Number of KEV entries skipped or flagged during parsing due to anomalies."); - _mappedAdvisories = _meter.CreateCounter( - name: "kev.map.advisories", - unit: "advisories", - description: "Number of KEV advisories emitted during mapping."); - } - - public void FetchAttempt() => _fetchAttempts.Add(1); - - public void FetchSuccess() => _fetchSuccess.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void CatalogParsed(string? catalogVersion, int entryCount) - { - if (entryCount <= 0) - { - return; - } - - _parsedEntries.Add(entryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); - } - - public void ParseFailure(string reason, string? catalogVersion = null) - { - var tags = string.IsNullOrWhiteSpace(catalogVersion) - ? new[] { new KeyValuePair("reason", reason) } - : new[] - { - new KeyValuePair("reason", reason), - new KeyValuePair("catalogVersion", catalogVersion) - }; - - _parseFailures.Add(1, tags); - } - - public void RecordAnomaly(string reason, string? catalogVersion = null) - { - var tags = string.IsNullOrWhiteSpace(catalogVersion) - ? new[] { new KeyValuePair("reason", reason) } - : new[] - { - new KeyValuePair("reason", reason), - new KeyValuePair("catalogVersion", catalogVersion) - }; - - _parseAnomalies.Add(1, tags); - } - - public void AdvisoriesMapped(string? catalogVersion, int advisoryCount) - { - if (advisoryCount <= 0) - { - return; - } - - _mappedAdvisories.Add(advisoryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); - } - - public void Dispose() => _meter.Dispose(); -} +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Kev.Internal; + +public sealed class KevDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Concelier.Connector.Kev"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchAttempts; + private readonly Counter _fetchSuccess; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parsedEntries; + private readonly Counter _parseFailures; + private readonly Counter _parseAnomalies; + private readonly Counter _mappedAdvisories; + + public KevDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchAttempts = _meter.CreateCounter( + name: "kev.fetch.attempts", + unit: "operations", + description: "Number of KEV fetch attempts performed."); + _fetchSuccess = _meter.CreateCounter( + name: "kev.fetch.success", + unit: "operations", + description: "Number of KEV fetch attempts that produced new catalog content."); + _fetchFailures = _meter.CreateCounter( + name: "kev.fetch.failures", + unit: "operations", + description: "Number of KEV fetch attempts that failed."); + _fetchUnchanged = _meter.CreateCounter( + name: "kev.fetch.unchanged", + unit: "operations", + description: "Number of KEV fetch attempts returning HTTP 304 / unchanged catalog."); + _parsedEntries = _meter.CreateCounter( + name: "kev.parse.entries", + unit: "entries", + description: "Number of KEV vulnerabilities parsed from the catalog."); + _parseFailures = _meter.CreateCounter( + name: "kev.parse.failures", + unit: "documents", + description: "Number of KEV catalog parse operations that failed or were quarantined."); + _parseAnomalies = _meter.CreateCounter( + name: "kev.parse.anomalies", + unit: "entries", + description: "Number of KEV entries skipped or flagged during parsing due to anomalies."); + _mappedAdvisories = _meter.CreateCounter( + name: "kev.map.advisories", + unit: "advisories", + description: "Number of KEV advisories emitted during mapping."); + } + + public void FetchAttempt() => _fetchAttempts.Add(1); + + public void FetchSuccess() => _fetchSuccess.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void CatalogParsed(string? catalogVersion, int entryCount) + { + if (entryCount <= 0) + { + return; + } + + _parsedEntries.Add(entryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); + } + + public void ParseFailure(string reason, string? catalogVersion = null) + { + var tags = string.IsNullOrWhiteSpace(catalogVersion) + ? new[] { new KeyValuePair("reason", reason) } + : new[] + { + new KeyValuePair("reason", reason), + new KeyValuePair("catalogVersion", catalogVersion) + }; + + _parseFailures.Add(1, tags); + } + + public void RecordAnomaly(string reason, string? catalogVersion = null) + { + var tags = string.IsNullOrWhiteSpace(catalogVersion) + ? new[] { new KeyValuePair("reason", reason) } + : new[] + { + new KeyValuePair("reason", reason), + new KeyValuePair("catalogVersion", catalogVersion) + }; + + _parseAnomalies.Add(1, tags); + } + + public void AdvisoriesMapped(string? catalogVersion, int advisoryCount) + { + if (advisoryCount <= 0) + { + return; + } + + _mappedAdvisories.Add(advisoryCount, new KeyValuePair("catalogVersion", catalogVersion ?? string.Empty)); + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs b/src/StellaOps.Concelier.Connector.Kev/Internal/KevMapper.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs rename to src/StellaOps.Concelier.Connector.Kev/Internal/KevMapper.cs index 635144d6..5b80b819 100644 --- a/src/StellaOps.Feedser.Source.Kev/Internal/KevMapper.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Internal/KevMapper.cs @@ -1,373 +1,373 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Source.Kev.Internal; - -internal static class KevMapper -{ - public static IReadOnlyList Map( - KevCatalogDto catalog, - string sourceName, - Uri feedUri, - DateTimeOffset fetchedAt, - DateTimeOffset validatedAt) - { - ArgumentNullException.ThrowIfNull(catalog); - ArgumentNullException.ThrowIfNull(sourceName); - ArgumentNullException.ThrowIfNull(feedUri); - - var advisories = new List(); - var fetchProvenance = new AdvisoryProvenance(sourceName, "document", feedUri.ToString(), fetchedAt); - var mappingProvenance = new AdvisoryProvenance( - sourceName, - "mapping", - catalog.CatalogVersion ?? feedUri.ToString(), - validatedAt); - - if (catalog.Vulnerabilities is null || catalog.Vulnerabilities.Count == 0) - { - return advisories; - } - - foreach (var entry in catalog.Vulnerabilities) - { - if (entry is null) - { - continue; - } - - var cveId = Normalize(entry.CveId); - if (string.IsNullOrEmpty(cveId)) - { - continue; - } - - var advisoryKey = $"kev/{cveId.ToLowerInvariant()}"; - var title = Normalize(entry.VulnerabilityName) ?? cveId; - var summary = Normalize(entry.ShortDescription); - var published = ParseDate(entry.DateAdded); - var dueDate = ParseDate(entry.DueDate); - - var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { cveId }; - - var references = BuildReferences(entry, sourceName, mappingProvenance, feedUri, cveId).ToArray(); - - var affectedPackages = BuildAffectedPackages( - entry, - catalog, - sourceName, - mappingProvenance, - published, - dueDate).ToArray(); - - var provenance = new[] - { - fetchProvenance, - mappingProvenance - }; - - advisories.Add(new Advisory( - advisoryKey, - title, - summary, - language: "en", - published, - modified: catalog.DateReleased?.ToUniversalTime(), - severity: null, - exploitKnown: true, - aliases, - references, - affectedPackages, - cvssMetrics: Array.Empty(), - provenance)); - } - - return advisories - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) - .ToArray(); - } - - private static IEnumerable BuildReferences( - KevVulnerabilityDto entry, - string sourceName, - AdvisoryProvenance mappingProvenance, - Uri feedUri, - string cveId) - { - var references = new List(); - var provenance = new AdvisoryProvenance(sourceName, "reference", cveId, mappingProvenance.RecordedAt); - - var catalogUrl = BuildCatalogSearchUrl(cveId); - if (catalogUrl is not null) - { - TryAddReference(references, catalogUrl, "advisory", "cisa-kev", provenance); - } - - TryAddReference(references, feedUri.ToString(), "reference", "cisa-kev-feed", provenance); - - foreach (var url in ExtractUrls(entry.Notes)) - { - TryAddReference(references, url, "reference", "kev.notes", provenance); - } - - return references - .GroupBy(static r => r.Url, StringComparer.OrdinalIgnoreCase) - .Select(static group => group - .OrderBy(static r => r.Kind, StringComparer.Ordinal) - .ThenBy(static r => r.SourceTag, StringComparer.Ordinal) - .First()) - .OrderBy(static r => r.Kind, StringComparer.Ordinal) - .ThenBy(static r => r.Url, StringComparer.Ordinal) - .ToArray(); - } - - private static void TryAddReference( - ICollection references, - string? url, - string kind, - string? sourceTag, - AdvisoryProvenance provenance) - { - if (string.IsNullOrWhiteSpace(url)) - { - return; - } - - if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed) - || (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps)) - { - return; - } - - try - { - references.Add(new AdvisoryReference(parsed.ToString(), kind, sourceTag, null, provenance)); - } - catch (ArgumentException) - { - // Ignore invalid references while leaving traceability via diagnostics elsewhere. - } - } - - private static string? BuildCatalogSearchUrl(string cveId) - { - if (string.IsNullOrWhiteSpace(cveId)) - { - return null; - } - - var builder = new StringBuilder("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search="); - builder.Append(Uri.EscapeDataString(cveId)); - return builder.ToString(); - } - - private static IEnumerable BuildAffectedPackages( - KevVulnerabilityDto entry, - KevCatalogDto catalog, - string sourceName, - AdvisoryProvenance mappingProvenance, - DateTimeOffset? published, - DateTimeOffset? dueDate) - { - var identifier = BuildIdentifier(entry) ?? entry.CveId ?? "kev"; - var rangeExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase); - - void TryAddExtension(string key, string? value, int maxLength = 512) - { - if (string.IsNullOrWhiteSpace(value)) - { - return; - } - - var trimmed = value.Trim(); - if (trimmed.Length > maxLength) - { - trimmed = trimmed[..maxLength].Trim(); - } - - if (trimmed.Length > 0) - { - rangeExtensions[key] = trimmed; - } - } - - TryAddExtension("kev.vendorProject", entry.VendorProject, 256); - TryAddExtension("kev.product", entry.Product, 256); - TryAddExtension("kev.requiredAction", entry.RequiredAction); - TryAddExtension("kev.knownRansomwareCampaignUse", entry.KnownRansomwareCampaignUse, 64); - TryAddExtension("kev.notes", entry.Notes); - TryAddExtension("kev.catalogVersion", catalog.CatalogVersion, 64); - - if (catalog.DateReleased.HasValue) - { - TryAddExtension("kev.catalogReleased", catalog.DateReleased.Value.ToString("O", CultureInfo.InvariantCulture)); - } - - if (published.HasValue) - { - TryAddExtension("kev.dateAdded", published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); - } - - if (dueDate.HasValue) - { - TryAddExtension("kev.dueDate", dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); - } - - if (entry.Cwes is { Count: > 0 }) - { - TryAddExtension("kev.cwe", string.Join(",", entry.Cwes.Where(static cwe => !string.IsNullOrWhiteSpace(cwe)).OrderBy(static cwe => cwe, StringComparer.Ordinal))); - } - - if (rangeExtensions.Count == 0) - { - return Array.Empty(); - } - - var rangeProvenance = new AdvisoryProvenance(sourceName, "kev-range", identifier, mappingProvenance.RecordedAt); - var range = new AffectedVersionRange( - rangeKind: AffectedPackageTypes.Vendor, - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: null, - provenance: rangeProvenance, - primitives: new RangePrimitives(null, null, null, rangeExtensions)); - - var normalizedVersions = BuildNormalizedVersions(identifier, catalog, published, dueDate); - - var affectedPackage = new AffectedPackage( - AffectedPackageTypes.Vendor, - identifier, - platform: null, - versionRanges: new[] { range }, - statuses: Array.Empty(), - provenance: new[] { mappingProvenance }, - normalizedVersions: normalizedVersions); - - return new[] { affectedPackage }; - } - - private static string? BuildIdentifier(KevVulnerabilityDto entry) - { - var vendor = Normalize(entry.VendorProject); - var product = Normalize(entry.Product); - - if (!string.IsNullOrEmpty(vendor) && !string.IsNullOrEmpty(product)) - { - return $"{vendor}::{product}"; - } - - return vendor ?? product; - } - - private static IEnumerable ExtractUrls(string? notes) - { - if (string.IsNullOrWhiteSpace(notes)) - { - return Array.Empty(); - } - - var tokens = notes.Split(new[] { ';', ',', ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); - var results = new List(); - - foreach (var token in tokens) - { - var trimmed = token.Trim().TrimEnd('.', ')', ';', ','); - if (trimmed.Length == 0) - { - continue; - } - - if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) - && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) - { - results.Add(uri.ToString()); - } - } - - return results.Count == 0 - ? Array.Empty() - : results.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(static value => value, StringComparer.Ordinal).ToArray(); - } - - private static string? Normalize(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - var trimmed = value.Trim(); - return trimmed.Length == 0 ? null : trimmed; - } - - private static DateTimeOffset? ParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) - { - return parsed.ToUniversalTime(); - } - - if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var date)) - { - return new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Utc)); - } - - return null; - } - - private static IReadOnlyList BuildNormalizedVersions( - string identifier, - KevCatalogDto catalog, - DateTimeOffset? published, - DateTimeOffset? dueDate) - { - var rules = new List(); - var notes = Validation.TrimToNull(identifier); - - if (!string.IsNullOrWhiteSpace(catalog.CatalogVersion)) - { - rules.Add(new NormalizedVersionRule( - scheme: "kev.catalog", - type: NormalizedVersionRuleTypes.Exact, - value: catalog.CatalogVersion.Trim(), - notes: notes)); - } - - if (published.HasValue) - { - rules.Add(new NormalizedVersionRule( - scheme: "kev.date-added", - type: NormalizedVersionRuleTypes.Exact, - value: published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), - notes: notes)); - } - - if (dueDate.HasValue) - { - rules.Add(new NormalizedVersionRule( - scheme: "kev.due-date", - type: NormalizedVersionRuleTypes.LessThanOrEqual, - max: dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), - maxInclusive: true, - notes: notes)); - } - - return rules.Count == 0 - ? Array.Empty() - : rules - .OrderBy(static rule => rule.Scheme, StringComparer.Ordinal) - .ThenBy(static rule => rule.Type, StringComparer.Ordinal) - .ThenBy(static rule => rule.Value ?? rule.Max ?? string.Empty, StringComparer.Ordinal) - .ToArray(); - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Connector.Kev.Internal; + +internal static class KevMapper +{ + public static IReadOnlyList Map( + KevCatalogDto catalog, + string sourceName, + Uri feedUri, + DateTimeOffset fetchedAt, + DateTimeOffset validatedAt) + { + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(sourceName); + ArgumentNullException.ThrowIfNull(feedUri); + + var advisories = new List(); + var fetchProvenance = new AdvisoryProvenance(sourceName, "document", feedUri.ToString(), fetchedAt); + var mappingProvenance = new AdvisoryProvenance( + sourceName, + "mapping", + catalog.CatalogVersion ?? feedUri.ToString(), + validatedAt); + + if (catalog.Vulnerabilities is null || catalog.Vulnerabilities.Count == 0) + { + return advisories; + } + + foreach (var entry in catalog.Vulnerabilities) + { + if (entry is null) + { + continue; + } + + var cveId = Normalize(entry.CveId); + if (string.IsNullOrEmpty(cveId)) + { + continue; + } + + var advisoryKey = $"kev/{cveId.ToLowerInvariant()}"; + var title = Normalize(entry.VulnerabilityName) ?? cveId; + var summary = Normalize(entry.ShortDescription); + var published = ParseDate(entry.DateAdded); + var dueDate = ParseDate(entry.DueDate); + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { cveId }; + + var references = BuildReferences(entry, sourceName, mappingProvenance, feedUri, cveId).ToArray(); + + var affectedPackages = BuildAffectedPackages( + entry, + catalog, + sourceName, + mappingProvenance, + published, + dueDate).ToArray(); + + var provenance = new[] + { + fetchProvenance, + mappingProvenance + }; + + advisories.Add(new Advisory( + advisoryKey, + title, + summary, + language: "en", + published, + modified: catalog.DateReleased?.ToUniversalTime(), + severity: null, + exploitKnown: true, + aliases, + references, + affectedPackages, + cvssMetrics: Array.Empty(), + provenance)); + } + + return advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + } + + private static IEnumerable BuildReferences( + KevVulnerabilityDto entry, + string sourceName, + AdvisoryProvenance mappingProvenance, + Uri feedUri, + string cveId) + { + var references = new List(); + var provenance = new AdvisoryProvenance(sourceName, "reference", cveId, mappingProvenance.RecordedAt); + + var catalogUrl = BuildCatalogSearchUrl(cveId); + if (catalogUrl is not null) + { + TryAddReference(references, catalogUrl, "advisory", "cisa-kev", provenance); + } + + TryAddReference(references, feedUri.ToString(), "reference", "cisa-kev-feed", provenance); + + foreach (var url in ExtractUrls(entry.Notes)) + { + TryAddReference(references, url, "reference", "kev.notes", provenance); + } + + return references + .GroupBy(static r => r.Url, StringComparer.OrdinalIgnoreCase) + .Select(static group => group + .OrderBy(static r => r.Kind, StringComparer.Ordinal) + .ThenBy(static r => r.SourceTag, StringComparer.Ordinal) + .First()) + .OrderBy(static r => r.Kind, StringComparer.Ordinal) + .ThenBy(static r => r.Url, StringComparer.Ordinal) + .ToArray(); + } + + private static void TryAddReference( + ICollection references, + string? url, + string kind, + string? sourceTag, + AdvisoryProvenance provenance) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + if (!Uri.TryCreate(url, UriKind.Absolute, out var parsed) + || (parsed.Scheme != Uri.UriSchemeHttp && parsed.Scheme != Uri.UriSchemeHttps)) + { + return; + } + + try + { + references.Add(new AdvisoryReference(parsed.ToString(), kind, sourceTag, null, provenance)); + } + catch (ArgumentException) + { + // Ignore invalid references while leaving traceability via diagnostics elsewhere. + } + } + + private static string? BuildCatalogSearchUrl(string cveId) + { + if (string.IsNullOrWhiteSpace(cveId)) + { + return null; + } + + var builder = new StringBuilder("https://www.cisa.gov/known-exploited-vulnerabilities-catalog?search="); + builder.Append(Uri.EscapeDataString(cveId)); + return builder.ToString(); + } + + private static IEnumerable BuildAffectedPackages( + KevVulnerabilityDto entry, + KevCatalogDto catalog, + string sourceName, + AdvisoryProvenance mappingProvenance, + DateTimeOffset? published, + DateTimeOffset? dueDate) + { + var identifier = BuildIdentifier(entry) ?? entry.CveId ?? "kev"; + var rangeExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + + void TryAddExtension(string key, string? value, int maxLength = 512) + { + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + var trimmed = value.Trim(); + if (trimmed.Length > maxLength) + { + trimmed = trimmed[..maxLength].Trim(); + } + + if (trimmed.Length > 0) + { + rangeExtensions[key] = trimmed; + } + } + + TryAddExtension("kev.vendorProject", entry.VendorProject, 256); + TryAddExtension("kev.product", entry.Product, 256); + TryAddExtension("kev.requiredAction", entry.RequiredAction); + TryAddExtension("kev.knownRansomwareCampaignUse", entry.KnownRansomwareCampaignUse, 64); + TryAddExtension("kev.notes", entry.Notes); + TryAddExtension("kev.catalogVersion", catalog.CatalogVersion, 64); + + if (catalog.DateReleased.HasValue) + { + TryAddExtension("kev.catalogReleased", catalog.DateReleased.Value.ToString("O", CultureInfo.InvariantCulture)); + } + + if (published.HasValue) + { + TryAddExtension("kev.dateAdded", published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + } + + if (dueDate.HasValue) + { + TryAddExtension("kev.dueDate", dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); + } + + if (entry.Cwes is { Count: > 0 }) + { + TryAddExtension("kev.cwe", string.Join(",", entry.Cwes.Where(static cwe => !string.IsNullOrWhiteSpace(cwe)).OrderBy(static cwe => cwe, StringComparer.Ordinal))); + } + + if (rangeExtensions.Count == 0) + { + return Array.Empty(); + } + + var rangeProvenance = new AdvisoryProvenance(sourceName, "kev-range", identifier, mappingProvenance.RecordedAt); + var range = new AffectedVersionRange( + rangeKind: AffectedPackageTypes.Vendor, + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: rangeProvenance, + primitives: new RangePrimitives(null, null, null, rangeExtensions)); + + var normalizedVersions = BuildNormalizedVersions(identifier, catalog, published, dueDate); + + var affectedPackage = new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: new[] { range }, + statuses: Array.Empty(), + provenance: new[] { mappingProvenance }, + normalizedVersions: normalizedVersions); + + return new[] { affectedPackage }; + } + + private static string? BuildIdentifier(KevVulnerabilityDto entry) + { + var vendor = Normalize(entry.VendorProject); + var product = Normalize(entry.Product); + + if (!string.IsNullOrEmpty(vendor) && !string.IsNullOrEmpty(product)) + { + return $"{vendor}::{product}"; + } + + return vendor ?? product; + } + + private static IEnumerable ExtractUrls(string? notes) + { + if (string.IsNullOrWhiteSpace(notes)) + { + return Array.Empty(); + } + + var tokens = notes.Split(new[] { ';', ',', ' ', '\r', '\n', '\t' }, StringSplitOptions.RemoveEmptyEntries); + var results = new List(); + + foreach (var token in tokens) + { + var trimmed = token.Trim().TrimEnd('.', ')', ';', ','); + if (trimmed.Length == 0) + { + continue; + } + + if (Uri.TryCreate(trimmed, UriKind.Absolute, out var uri) + && (uri.Scheme == Uri.UriSchemeHttp || uri.Scheme == Uri.UriSchemeHttps)) + { + results.Add(uri.ToString()); + } + } + + return results.Count == 0 + ? Array.Empty() + : results.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(static value => value, StringComparer.Ordinal).ToArray(); + } + + private static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + return trimmed.Length == 0 ? null : trimmed; + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed.ToUniversalTime(); + } + + if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var date)) + { + return new DateTimeOffset(DateTime.SpecifyKind(date, DateTimeKind.Utc)); + } + + return null; + } + + private static IReadOnlyList BuildNormalizedVersions( + string identifier, + KevCatalogDto catalog, + DateTimeOffset? published, + DateTimeOffset? dueDate) + { + var rules = new List(); + var notes = Validation.TrimToNull(identifier); + + if (!string.IsNullOrWhiteSpace(catalog.CatalogVersion)) + { + rules.Add(new NormalizedVersionRule( + scheme: "kev.catalog", + type: NormalizedVersionRuleTypes.Exact, + value: catalog.CatalogVersion.Trim(), + notes: notes)); + } + + if (published.HasValue) + { + rules.Add(new NormalizedVersionRule( + scheme: "kev.date-added", + type: NormalizedVersionRuleTypes.Exact, + value: published.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + notes: notes)); + } + + if (dueDate.HasValue) + { + rules.Add(new NormalizedVersionRule( + scheme: "kev.due-date", + type: NormalizedVersionRuleTypes.LessThanOrEqual, + max: dueDate.Value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture), + maxInclusive: true, + notes: notes)); + } + + return rules.Count == 0 + ? Array.Empty() + : rules + .OrderBy(static rule => rule.Scheme, StringComparer.Ordinal) + .ThenBy(static rule => rule.Type, StringComparer.Ordinal) + .ThenBy(static rule => rule.Value ?? rule.Max ?? string.Empty, StringComparer.Ordinal) + .ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs b/src/StellaOps.Concelier.Connector.Kev/Internal/KevSchemaProvider.cs similarity index 80% rename from src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs rename to src/StellaOps.Concelier.Connector.Kev/Internal/KevSchemaProvider.cs index 53f1287b..9ed4c511 100644 --- a/src/StellaOps.Feedser.Source.Kev/Internal/KevSchemaProvider.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Internal/KevSchemaProvider.cs @@ -1,25 +1,25 @@ -using System.IO; -using System.Reflection; -using System.Threading; -using Json.Schema; - -namespace StellaOps.Feedser.Source.Kev.Internal; - -internal static class KevSchemaProvider -{ - private const string ResourceName = "StellaOps.Feedser.Source.Kev.Schemas.kev-catalog.schema.json"; - - private static readonly Lazy CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); - - public static JsonSchema Schema => CachedSchema.Value; - - private static JsonSchema LoadSchema() - { - var assembly = typeof(KevSchemaProvider).GetTypeInfo().Assembly; - using var stream = assembly.GetManifestResourceStream(ResourceName) - ?? throw new InvalidOperationException($"Embedded schema '{ResourceName}' was not found."); - using var reader = new StreamReader(stream); - var schemaJson = reader.ReadToEnd(); - return JsonSchema.FromText(schemaJson); - } -} +using System.IO; +using System.Reflection; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Concelier.Connector.Kev.Internal; + +internal static class KevSchemaProvider +{ + private const string ResourceName = "StellaOps.Concelier.Connector.Kev.Schemas.kev-catalog.schema.json"; + + private static readonly Lazy CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => CachedSchema.Value; + + private static JsonSchema LoadSchema() + { + var assembly = typeof(KevSchemaProvider).GetTypeInfo().Assembly; + using var stream = assembly.GetManifestResourceStream(ResourceName) + ?? throw new InvalidOperationException($"Embedded schema '{ResourceName}' was not found."); + using var reader = new StreamReader(stream); + var schemaJson = reader.ReadToEnd(); + return JsonSchema.FromText(schemaJson); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Jobs.cs b/src/StellaOps.Concelier.Connector.Kev/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Kev/Jobs.cs rename to src/StellaOps.Concelier.Connector.Kev/Jobs.cs index f9323c9d..e77b9010 100644 --- a/src/StellaOps.Feedser.Source.Kev/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Kev/Jobs.cs @@ -1,46 +1,46 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Kev; - -internal static class KevJobKinds -{ - public const string Fetch = "source:kev:fetch"; - public const string Parse = "source:kev:parse"; - public const string Map = "source:kev:map"; -} - -internal sealed class KevFetchJob : IJob -{ - private readonly KevConnector _connector; - - public KevFetchJob(KevConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class KevParseJob : IJob -{ - private readonly KevConnector _connector; - - public KevParseJob(KevConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class KevMapJob : IJob -{ - private readonly KevConnector _connector; - - public KevMapJob(KevConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Kev; + +internal static class KevJobKinds +{ + public const string Fetch = "source:kev:fetch"; + public const string Parse = "source:kev:parse"; + public const string Map = "source:kev:map"; +} + +internal sealed class KevFetchJob : IJob +{ + private readonly KevConnector _connector; + + public KevFetchJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class KevParseJob : IJob +{ + private readonly KevConnector _connector; + + public KevParseJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class KevMapJob : IJob +{ + private readonly KevConnector _connector; + + public KevMapJob(KevConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevConnector.cs b/src/StellaOps.Concelier.Connector.Kev/KevConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kev/KevConnector.cs rename to src/StellaOps.Concelier.Connector.Kev/KevConnector.cs index a970d6d3..5b68656e 100644 --- a/src/StellaOps.Feedser.Source.Kev/KevConnector.cs +++ b/src/StellaOps.Concelier.Connector.Kev/KevConnector.cs @@ -1,441 +1,441 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Json; -using StellaOps.Feedser.Source.Kev.Configuration; -using StellaOps.Feedser.Source.Kev.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Kev; - -public sealed class KevConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private const string SchemaVersion = "kev.catalog.v1"; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly KevOptions _options; - private readonly IJsonSchemaValidator _schemaValidator; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly KevDiagnostics _diagnostics; - - public KevConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - IJsonSchemaValidator schemaValidator, - KevDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => KevConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var now = _timeProvider.GetUtcNow(); - - try - { - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, _options.FeedUri.ToString(), cancellationToken).ConfigureAwait(false); - - var request = new SourceFetchRequest( - KevOptions.HttpClientName, - SourceName, - _options.FeedUri) - { - Metadata = new Dictionary(StringComparer.Ordinal) - { - ["kev.cursor.catalogVersion"] = cursor.CatalogVersion ?? string.Empty, - ["kev.cursor.catalogReleased"] = cursor.CatalogReleased?.ToString("O") ?? string.Empty, - }, - ETag = existing?.Etag, - LastModified = existing?.LastModified, - TimeoutOverride = _options.RequestTimeout, - AcceptHeaders = new[] { "application/json", "text/json" }, - }; - - _diagnostics.FetchAttempt(); - var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); - if (result.IsNotModified) - { - _diagnostics.FetchUnchanged(); - _logger.LogInformation( - "KEV catalog not modified (catalogVersion={CatalogVersion}, etag={Etag})", - cursor.CatalogVersion ?? "(unknown)", - existing?.Etag ?? "(none)"); - await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); - return; - } - - if (!result.IsSuccess || result.Document is null) - { - _diagnostics.FetchFailure(); - await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), "KEV feed returned no content.", cancellationToken).ConfigureAwait(false); - return; - } - - _diagnostics.FetchSuccess(); - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var pendingDocumentsBefore = pendingDocuments.Count; - var pendingMappingsBefore = pendingMappings.Count; - - pendingDocuments.Add(result.Document.Id); - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - var document = result.Document; - var lastModified = document.LastModified?.ToUniversalTime().ToString("O") ?? "(unknown)"; - _logger.LogInformation( - "Fetched KEV catalog document {DocumentId} (etag={Etag}, lastModified={LastModified}) pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter}", - document.Id, - document.Etag ?? "(none)", - lastModified, - pendingDocumentsBefore, - pendingDocuments.Count, - pendingMappingsBefore, - pendingMappings.Count); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "KEV fetch failed for {Uri}", _options.FeedUri); - await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var latestCatalogVersion = cursor.CatalogVersion; - var latestCatalogReleased = cursor.CatalogReleased; - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure("missingPayload", cursor.CatalogVersion); - _logger.LogWarning("KEV document {DocumentId} missing GridFS payload", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - byte[] rawBytes; - try - { - rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure("download", cursor.CatalogVersion); - _logger.LogError(ex, "KEV parse failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - KevCatalogDto? catalog = null; - string? catalogVersion = null; - try - { - using var jsonDocument = JsonDocument.Parse(rawBytes); - catalogVersion = TryGetCatalogVersion(jsonDocument.RootElement); - _schemaValidator.Validate(jsonDocument, KevSchemaProvider.Schema, document.Uri); - catalog = jsonDocument.RootElement.Deserialize(SerializerOptions); - } - catch (JsonSchemaValidationException ex) - { - _diagnostics.ParseFailure("schema", catalogVersion); - _logger.LogWarning(ex, "KEV schema validation failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - catch (JsonException ex) - { - _diagnostics.ParseFailure("invalidJson", catalogVersion); - _logger.LogError(ex, "KEV JSON parsing failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - catch (Exception ex) - { - _diagnostics.ParseFailure("deserialize", catalogVersion); - _logger.LogError(ex, "KEV catalog deserialization failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (catalog is null) - { - _diagnostics.ParseFailure("emptyCatalog", catalogVersion); - _logger.LogWarning("KEV catalog payload was empty for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var entryCount = catalog.Vulnerabilities?.Count ?? 0; - var released = catalog.DateReleased?.ToUniversalTime(); - RecordCatalogAnomalies(catalog); - - try - { - var payloadJson = JsonSerializer.Serialize(catalog, SerializerOptions); - var payload = BsonDocument.Parse(payloadJson); - - _logger.LogInformation( - "Parsed KEV catalog document {DocumentId} (version={CatalogVersion}, released={Released}, entries={EntryCount})", - document.Id, - catalog.CatalogVersion ?? "(unknown)", - released, - entryCount); - _diagnostics.CatalogParsed(catalog.CatalogVersion, entryCount); - - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - document.Id, - SourceName, - SchemaVersion, - payload, - _timeProvider.GetUtcNow()); - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - pendingMappings.Add(document.Id); - - latestCatalogVersion = catalog.CatalogVersion ?? latestCatalogVersion; - latestCatalogReleased = catalog.DateReleased ?? latestCatalogReleased; - } - catch (Exception ex) - { - _logger.LogError(ex, "KEV DTO persistence failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(remainingDocuments) - .WithPendingMappings(pendingMappings) - .WithCatalogMetadata(latestCatalogVersion, latestCatalogReleased); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - - if (dtoRecord is null || document is null) - { - pendingMappings.Remove(documentId); - continue; - } - - KevCatalogDto? catalog; - try - { - var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings - { - OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, - }); - - catalog = JsonSerializer.Deserialize(dtoJson, SerializerOptions); - } - catch (Exception ex) - { - _logger.LogError(ex, "KEV mapping: failed to deserialize DTO for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - if (catalog is null) - { - _logger.LogWarning("KEV mapping: DTO payload was empty for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - var feedUri = TryParseUri(document.Uri) ?? _options.FeedUri; - var advisories = KevMapper.Map(catalog, SourceName, feedUri, document.FetchedAt, dtoRecord.ValidatedAt); - var entryCount = catalog.Vulnerabilities?.Count ?? 0; - var mappedCount = advisories.Count; - var skippedCount = Math.Max(0, entryCount - mappedCount); - _logger.LogInformation( - "Mapped {MappedCount}/{EntryCount} KEV advisories from catalog version {CatalogVersion} (skipped={SkippedCount})", - mappedCount, - entryCount, - catalog.CatalogVersion ?? "(unknown)", - skippedCount); - _diagnostics.AdvisoriesMapped(catalog.CatalogVersion, mappedCount); - - foreach (var advisory in advisories) - { - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - } - - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? KevCursor.Empty : KevCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(KevCursor cursor, CancellationToken cancellationToken) - { - return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); - } - - private void RecordCatalogAnomalies(KevCatalogDto catalog) - { - ArgumentNullException.ThrowIfNull(catalog); - - var version = catalog.CatalogVersion; - var vulnerabilities = catalog.Vulnerabilities ?? Array.Empty(); - - if (catalog.Count != vulnerabilities.Count) - { - _diagnostics.RecordAnomaly("countMismatch", version); - } - - foreach (var entry in vulnerabilities) - { - if (entry is null) - { - _diagnostics.RecordAnomaly("nullEntry", version); - continue; - } - - if (string.IsNullOrWhiteSpace(entry.CveId)) - { - _diagnostics.RecordAnomaly("missingCveId", version); - } - } - } - - private static string? TryGetCatalogVersion(JsonElement root) - { - if (root.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (root.TryGetProperty("catalogVersion", out var versionElement) && versionElement.ValueKind == JsonValueKind.String) - { - return versionElement.GetString(); - } - - return null; - } - - private static Uri? TryParseUri(string? value) - => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Json; +using StellaOps.Concelier.Connector.Kev.Configuration; +using StellaOps.Concelier.Connector.Kev.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Kev; + +public sealed class KevConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private const string SchemaVersion = "kev.catalog.v1"; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly KevOptions _options; + private readonly IJsonSchemaValidator _schemaValidator; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly KevDiagnostics _diagnostics; + + public KevConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + IJsonSchemaValidator schemaValidator, + KevDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _schemaValidator = schemaValidator ?? throw new ArgumentNullException(nameof(schemaValidator)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => KevConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + + try + { + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, _options.FeedUri.ToString(), cancellationToken).ConfigureAwait(false); + + var request = new SourceFetchRequest( + KevOptions.HttpClientName, + SourceName, + _options.FeedUri) + { + Metadata = new Dictionary(StringComparer.Ordinal) + { + ["kev.cursor.catalogVersion"] = cursor.CatalogVersion ?? string.Empty, + ["kev.cursor.catalogReleased"] = cursor.CatalogReleased?.ToString("O") ?? string.Empty, + }, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + TimeoutOverride = _options.RequestTimeout, + AcceptHeaders = new[] { "application/json", "text/json" }, + }; + + _diagnostics.FetchAttempt(); + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + _diagnostics.FetchUnchanged(); + _logger.LogInformation( + "KEV catalog not modified (catalogVersion={CatalogVersion}, etag={Etag})", + cursor.CatalogVersion ?? "(unknown)", + existing?.Etag ?? "(none)"); + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.FetchFailure(); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), "KEV feed returned no content.", cancellationToken).ConfigureAwait(false); + return; + } + + _diagnostics.FetchSuccess(); + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var pendingDocumentsBefore = pendingDocuments.Count; + var pendingMappingsBefore = pendingMappings.Count; + + pendingDocuments.Add(result.Document.Id); + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + var document = result.Document; + var lastModified = document.LastModified?.ToUniversalTime().ToString("O") ?? "(unknown)"; + _logger.LogInformation( + "Fetched KEV catalog document {DocumentId} (etag={Etag}, lastModified={LastModified}) pendingDocuments={PendingDocumentsBefore}->{PendingDocumentsAfter} pendingMappings={PendingMappingsBefore}->{PendingMappingsAfter}", + document.Id, + document.Etag ?? "(none)", + lastModified, + pendingDocumentsBefore, + pendingDocuments.Count, + pendingMappingsBefore, + pendingMappings.Count); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "KEV fetch failed for {Uri}", _options.FeedUri); + await _stateRepository.MarkFailureAsync(SourceName, now, TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var latestCatalogVersion = cursor.CatalogVersion; + var latestCatalogReleased = cursor.CatalogReleased; + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure("missingPayload", cursor.CatalogVersion); + _logger.LogWarning("KEV document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + byte[] rawBytes; + try + { + rawBytes = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure("download", cursor.CatalogVersion); + _logger.LogError(ex, "KEV parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + KevCatalogDto? catalog = null; + string? catalogVersion = null; + try + { + using var jsonDocument = JsonDocument.Parse(rawBytes); + catalogVersion = TryGetCatalogVersion(jsonDocument.RootElement); + _schemaValidator.Validate(jsonDocument, KevSchemaProvider.Schema, document.Uri); + catalog = jsonDocument.RootElement.Deserialize(SerializerOptions); + } + catch (JsonSchemaValidationException ex) + { + _diagnostics.ParseFailure("schema", catalogVersion); + _logger.LogWarning(ex, "KEV schema validation failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + catch (JsonException ex) + { + _diagnostics.ParseFailure("invalidJson", catalogVersion); + _logger.LogError(ex, "KEV JSON parsing failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + catch (Exception ex) + { + _diagnostics.ParseFailure("deserialize", catalogVersion); + _logger.LogError(ex, "KEV catalog deserialization failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (catalog is null) + { + _diagnostics.ParseFailure("emptyCatalog", catalogVersion); + _logger.LogWarning("KEV catalog payload was empty for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var entryCount = catalog.Vulnerabilities?.Count ?? 0; + var released = catalog.DateReleased?.ToUniversalTime(); + RecordCatalogAnomalies(catalog); + + try + { + var payloadJson = JsonSerializer.Serialize(catalog, SerializerOptions); + var payload = BsonDocument.Parse(payloadJson); + + _logger.LogInformation( + "Parsed KEV catalog document {DocumentId} (version={CatalogVersion}, released={Released}, entries={EntryCount})", + document.Id, + catalog.CatalogVersion ?? "(unknown)", + released, + entryCount); + _diagnostics.CatalogParsed(catalog.CatalogVersion, entryCount); + + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + document.Id, + SourceName, + SchemaVersion, + payload, + _timeProvider.GetUtcNow()); + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + + latestCatalogVersion = catalog.CatalogVersion ?? latestCatalogVersion; + latestCatalogReleased = catalog.DateReleased ?? latestCatalogReleased; + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV DTO persistence failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings) + .WithCatalogMetadata(latestCatalogVersion, latestCatalogReleased); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + + if (dtoRecord is null || document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + KevCatalogDto? catalog; + try + { + var dtoJson = dtoRecord.Payload.ToJson(new MongoDB.Bson.IO.JsonWriterSettings + { + OutputMode = MongoDB.Bson.IO.JsonOutputMode.RelaxedExtendedJson, + }); + + catalog = JsonSerializer.Deserialize(dtoJson, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogError(ex, "KEV mapping: failed to deserialize DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (catalog is null) + { + _logger.LogWarning("KEV mapping: DTO payload was empty for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + var feedUri = TryParseUri(document.Uri) ?? _options.FeedUri; + var advisories = KevMapper.Map(catalog, SourceName, feedUri, document.FetchedAt, dtoRecord.ValidatedAt); + var entryCount = catalog.Vulnerabilities?.Count ?? 0; + var mappedCount = advisories.Count; + var skippedCount = Math.Max(0, entryCount - mappedCount); + _logger.LogInformation( + "Mapped {MappedCount}/{EntryCount} KEV advisories from catalog version {CatalogVersion} (skipped={SkippedCount})", + mappedCount, + entryCount, + catalog.CatalogVersion ?? "(unknown)", + skippedCount); + _diagnostics.AdvisoriesMapped(catalog.CatalogVersion, mappedCount); + + foreach (var advisory in advisories) + { + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + } + + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? KevCursor.Empty : KevCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(KevCursor cursor, CancellationToken cancellationToken) + { + return _stateRepository.UpdateCursorAsync(SourceName, cursor.ToBsonDocument(), _timeProvider.GetUtcNow(), cancellationToken); + } + + private void RecordCatalogAnomalies(KevCatalogDto catalog) + { + ArgumentNullException.ThrowIfNull(catalog); + + var version = catalog.CatalogVersion; + var vulnerabilities = catalog.Vulnerabilities ?? Array.Empty(); + + if (catalog.Count != vulnerabilities.Count) + { + _diagnostics.RecordAnomaly("countMismatch", version); + } + + foreach (var entry in vulnerabilities) + { + if (entry is null) + { + _diagnostics.RecordAnomaly("nullEntry", version); + continue; + } + + if (string.IsNullOrWhiteSpace(entry.CveId)) + { + _diagnostics.RecordAnomaly("missingCveId", version); + } + } + } + + private static string? TryGetCatalogVersion(JsonElement root) + { + if (root.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (root.TryGetProperty("catalogVersion", out var versionElement) && versionElement.ValueKind == JsonValueKind.String) + { + return versionElement.GetString(); + } + + return null; + } + + private static Uri? TryParseUri(string? value) + => Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Kev/KevConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Kev/KevConnectorPlugin.cs index 0905f025..302df640 100644 --- a/src/StellaOps.Feedser.Source.Kev/KevConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Kev/KevConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Kev; - -public sealed class KevConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "kev"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Kev; + +public sealed class KevConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "kev"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Kev/KevDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Kev/KevDependencyInjectionRoutine.cs index 27e4274e..9087c10e 100644 --- a/src/StellaOps.Feedser.Source.Kev/KevDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Kev/KevDependencyInjectionRoutine.cs @@ -1,54 +1,54 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Kev.Configuration; - -namespace StellaOps.Feedser.Source.Kev; - -public sealed class KevDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:kev"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddKevConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, KevJobKinds.Fetch, typeof(KevFetchJob)); - EnsureJob(options, KevJobKinds.Parse, typeof(KevParseJob)); - EnsureJob(options, KevJobKinds.Map, typeof(KevMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Kev.Configuration; + +namespace StellaOps.Concelier.Connector.Kev; + +public sealed class KevDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:kev"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddKevConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, KevJobKinds.Fetch, typeof(KevFetchJob)); + EnsureJob(options, KevJobKinds.Parse, typeof(KevParseJob)); + EnsureJob(options, KevJobKinds.Map, typeof(KevMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Kev/KevServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Kev/KevServiceCollectionExtensions.cs index 5f0c0366..441e8caa 100644 --- a/src/StellaOps.Feedser.Source.Kev/KevServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Kev/KevServiceCollectionExtensions.cs @@ -1,38 +1,38 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Kev.Configuration; -using StellaOps.Feedser.Source.Kev.Internal; - -namespace StellaOps.Feedser.Source.Kev; - -public static class KevServiceCollectionExtensions -{ - public static IServiceCollection AddKevConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); - - services.AddSourceHttpClient(KevOptions.HttpClientName, (provider, clientOptions) => - { - var opts = provider.GetRequiredService>().Value; - clientOptions.BaseAddress = opts.FeedUri; - clientOptions.Timeout = opts.RequestTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.Kev/1.0"; - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(opts.FeedUri.Host); - clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; - }); - - services.TryAddSingleton(); - services.AddTransient(); - - return services; - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Kev.Configuration; +using StellaOps.Concelier.Connector.Kev.Internal; + +namespace StellaOps.Concelier.Connector.Kev; + +public static class KevServiceCollectionExtensions +{ + public static IServiceCollection AddKevConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(KevOptions.HttpClientName, (provider, clientOptions) => + { + var opts = provider.GetRequiredService>().Value; + clientOptions.BaseAddress = opts.FeedUri; + clientOptions.Timeout = opts.RequestTimeout; + clientOptions.UserAgent = "StellaOps.Concelier.Kev/1.0"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(opts.FeedUri.Host); + clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; + }); + + services.TryAddSingleton(); + services.AddTransient(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json b/src/StellaOps.Concelier.Connector.Kev/Schemas/kev-catalog.schema.json similarity index 95% rename from src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json rename to src/StellaOps.Concelier.Connector.Kev/Schemas/kev-catalog.schema.json index 53860c77..454d95ae 100644 --- a/src/StellaOps.Feedser.Source.Kev/Schemas/kev-catalog.schema.json +++ b/src/StellaOps.Concelier.Connector.Kev/Schemas/kev-catalog.schema.json @@ -1,80 +1,80 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "CISA Known Exploited Vulnerabilities catalog", - "type": "object", - "required": [ - "catalogVersion", - "dateReleased", - "count", - "vulnerabilities" - ], - "properties": { - "title": { - "type": "string" - }, - "catalogVersion": { - "type": "string", - "minLength": 1 - }, - "dateReleased": { - "type": "string", - "format": "date-time" - }, - "count": { - "type": "integer", - "minimum": 0 - }, - "vulnerabilities": { - "type": "array", - "items": { - "type": "object", - "required": [ - "cveID", - "vendorProject", - "product" - ], - "properties": { - "cveID": { - "type": "string", - "pattern": "^CVE-\\d{4}-\\d{4,}$" - }, - "vendorProject": { - "type": "string" - }, - "product": { - "type": "string" - }, - "vulnerabilityName": { - "type": "string" - }, - "dateAdded": { - "type": "string" - }, - "shortDescription": { - "type": "string" - }, - "requiredAction": { - "type": "string" - }, - "dueDate": { - "type": "string" - }, - "knownRansomwareCampaignUse": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "cwes": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "additionalProperties": true - } - } - }, - "additionalProperties": true -} +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "CISA Known Exploited Vulnerabilities catalog", + "type": "object", + "required": [ + "catalogVersion", + "dateReleased", + "count", + "vulnerabilities" + ], + "properties": { + "title": { + "type": "string" + }, + "catalogVersion": { + "type": "string", + "minLength": 1 + }, + "dateReleased": { + "type": "string", + "format": "date-time" + }, + "count": { + "type": "integer", + "minimum": 0 + }, + "vulnerabilities": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cveID", + "vendorProject", + "product" + ], + "properties": { + "cveID": { + "type": "string", + "pattern": "^CVE-\\d{4}-\\d{4,}$" + }, + "vendorProject": { + "type": "string" + }, + "product": { + "type": "string" + }, + "vulnerabilityName": { + "type": "string" + }, + "dateAdded": { + "type": "string" + }, + "shortDescription": { + "type": "string" + }, + "requiredAction": { + "type": "string" + }, + "dueDate": { + "type": "string" + }, + "knownRansomwareCampaignUse": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "cwes": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + } + } + }, + "additionalProperties": true +} diff --git a/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj b/src/StellaOps.Concelier.Connector.Kev/StellaOps.Concelier.Connector.Kev.csproj similarity index 57% rename from src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj rename to src/StellaOps.Concelier.Connector.Kev/StellaOps.Concelier.Connector.Kev.csproj index 84e63f07..c5f1ae2c 100644 --- a/src/StellaOps.Feedser.Source.Kev/StellaOps.Feedser.Source.Kev.csproj +++ b/src/StellaOps.Concelier.Connector.Kev/StellaOps.Concelier.Connector.Kev.csproj @@ -9,14 +9,14 @@ - - - + + + - <_Parameter1>StellaOps.Feedser.Source.Kev.Tests + <_Parameter1>StellaOps.Concelier.Connector.Kev.Tests diff --git a/src/StellaOps.Feedser.Source.Kev/TASKS.md b/src/StellaOps.Concelier.Connector.Kev/TASKS.md similarity index 86% rename from src/StellaOps.Feedser.Source.Kev/TASKS.md rename to src/StellaOps.Concelier.Connector.Kev/TASKS.md index 039870c0..98e8361c 100644 --- a/src/StellaOps.Feedser.Source.Kev/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Kev/TASKS.md @@ -1,12 +1,12 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|Review KEV JSON schema & cadence|BE-Conn-KEV|Research|**DONE** – Feed defaults lock to the public JSON catalog; AGENTS notes call out daily cadence and allowlist requirements.| -|Fetch & cursor implementation|BE-Conn-KEV|Source.Common, Storage.Mongo|**DONE** – SourceFetchService drives ETag/Last-Modified aware fetches with SourceState cursor tracking documents + catalog metadata.| -|DTO/parser implementation|BE-Conn-KEV|Source.Common|**DONE** – `KevCatalogDto`/`KevVulnerabilityDto` deserialize payloads with logging for catalog version/releases before DTO persistence.| -|Canonical mapping & range primitives|BE-Conn-KEV|Models|**DONE** – Mapper produces vendor RangePrimitives (due dates, CWE list, ransomware flag, catalog metadata) and deduplicated references.| -|Deterministic fixtures/tests|QA|Testing|**DONE** – End-to-end fetch→parse→map test with canned catalog + snapshot (`UPDATE_KEV_FIXTURES=1`) guards determinism.| -|Telemetry & docs|DevEx|Docs|**DONE** – Connector emits structured logs + meters for catalog entries/advisories and AGENTS docs cover cadence/allowlist guidance.| -|Schema validation & anomaly surfacing|BE-Conn-KEV, QA|Source.Common|**DONE (2025-10-12)** – Wired `IJsonSchemaValidator` + embedded schema, added failure reasons (`schema`, `download`, `invalidJson`, etc.), anomaly counters (`missingCveId`, `countMismatch`, `nullEntry`), and kept `dotnet test src/StellaOps.Feedser.Source.Kev.Tests` passing.| -|Metrics export wiring|DevOps, DevEx|Observability|**DONE (2025-10-12)** – Added `kev.fetch.*` counters, parse failure/anomaly tags, refreshed ops runbook + Grafana dashboard (`docs/ops/feedser-cve-kev-grafana-dashboard.json`) with PromQL guidance.| -|FEEDCONN-KEV-02-003 Normalized versions propagation|BE-Conn-KEV|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Validated catalog/date/due normalized rules emission + ordering; fixtures assert rule set and `dotnet test src/StellaOps.Feedser.Source.Kev.Tests` remains green.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Review KEV JSON schema & cadence|BE-Conn-KEV|Research|**DONE** – Feed defaults lock to the public JSON catalog; AGENTS notes call out daily cadence and allowlist requirements.| +|Fetch & cursor implementation|BE-Conn-KEV|Source.Common, Storage.Mongo|**DONE** – SourceFetchService drives ETag/Last-Modified aware fetches with SourceState cursor tracking documents + catalog metadata.| +|DTO/parser implementation|BE-Conn-KEV|Source.Common|**DONE** – `KevCatalogDto`/`KevVulnerabilityDto` deserialize payloads with logging for catalog version/releases before DTO persistence.| +|Canonical mapping & range primitives|BE-Conn-KEV|Models|**DONE** – Mapper produces vendor RangePrimitives (due dates, CWE list, ransomware flag, catalog metadata) and deduplicated references.| +|Deterministic fixtures/tests|QA|Testing|**DONE** – End-to-end fetch→parse→map test with canned catalog + snapshot (`UPDATE_KEV_FIXTURES=1`) guards determinism.| +|Telemetry & docs|DevEx|Docs|**DONE** – Connector emits structured logs + meters for catalog entries/advisories and AGENTS docs cover cadence/allowlist guidance.| +|Schema validation & anomaly surfacing|BE-Conn-KEV, QA|Source.Common|**DONE (2025-10-12)** – Wired `IJsonSchemaValidator` + embedded schema, added failure reasons (`schema`, `download`, `invalidJson`, etc.), anomaly counters (`missingCveId`, `countMismatch`, `nullEntry`), and kept `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests` passing.| +|Metrics export wiring|DevOps, DevEx|Observability|**DONE (2025-10-12)** – Added `kev.fetch.*` counters, parse failure/anomaly tags, refreshed ops runbook + Grafana dashboard (`docs/ops/concelier-cve-kev-grafana-dashboard.json`) with PromQL guidance.| +|FEEDCONN-KEV-02-003 Normalized versions propagation|BE-Conn-KEV|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-12)** – Validated catalog/date/due normalized rules emission + ordering; fixtures assert rule set and `dotnet test src/StellaOps.Concelier.Connector.Kev.Tests` remains green.| diff --git a/src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-detail.json b/src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-detail.json similarity index 96% rename from src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-detail.json rename to src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-detail.json index 6f466b0e..464a155d 100644 --- a/src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-detail.json +++ b/src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-detail.json @@ -1,25 +1,25 @@ -{ - "idx": "5868", - "title": "태그프리 제품 부적절한 권한 검증 취약점", - "summary": "태그프리사의 X-Free Uploader에서 발생하는 부적절한 권한 검증 취약점", - "contentHtml": "

      태그프리사의 X-Free Uploader에서 권한 검증이 미흡하여 임의 파일 삭제가 가능합니다.

      ", - "severity": "High", - "published": "2025-07-31T06:30:23Z", - "updated": "2025-08-01T02:15:00Z", - "cveIds": [ - "CVE-2025-29866" - ], - "references": [ - { - "url": "https://www.tagfree.com/security", - "label": "제조사 공지" - } - ], - "products": [ - { - "vendor": "태그프리", - "name": "X-Free Uploader", - "versions": "XFU 1.0.1.0084 ~ 2.0.1.0034" - } - ] -} +{ + "idx": "5868", + "title": "태그프리 제품 부적절한 권한 검증 취약점", + "summary": "태그프리사의 X-Free Uploader에서 발생하는 부적절한 권한 검증 취약점", + "contentHtml": "

      태그프리사의 X-Free Uploader에서 권한 검증이 미흡하여 임의 파일 삭제가 가능합니다.

      ", + "severity": "High", + "published": "2025-07-31T06:30:23Z", + "updated": "2025-08-01T02:15:00Z", + "cveIds": [ + "CVE-2025-29866" + ], + "references": [ + { + "url": "https://www.tagfree.com/security", + "label": "제조사 공지" + } + ], + "products": [ + { + "vendor": "태그프리", + "name": "X-Free Uploader", + "versions": "XFU 1.0.1.0084 ~ 2.0.1.0034" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-feed.xml b/src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-feed.xml similarity index 97% rename from src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-feed.xml rename to src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-feed.xml index 78494b8d..a26fb4f8 100644 --- a/src/StellaOps.Feedser.Source.Kisa.Tests/Fixtures/kisa-feed.xml +++ b/src/StellaOps.Concelier.Connector.Kisa.Tests/Fixtures/kisa-feed.xml @@ -1,15 +1,15 @@ - - - - KNVD 보안취약점 - https://knvd.krcert.or.kr/ - 테스트 피드 - ko - - 태그프리 제품 부적절한 권한 검증 취약점 - https://knvd.krcert.or.kr/detailDos.do?IDX=5868 - 취약점정보 - Thu, 31 Jul 2025 06:30:23 GMT - - - + + + + KNVD 보안취약점 + https://knvd.krcert.or.kr/ + 테스트 피드 + ko + + 태그프리 제품 부적절한 권한 검증 취약점 + https://knvd.krcert.or.kr/detailDos.do?IDX=5868 + 취약점정보 + Thu, 31 Jul 2025 06:30:23 GMT + + + diff --git a/src/StellaOps.Feedser.Source.Kisa.Tests/KisaConnectorTests.cs b/src/StellaOps.Concelier.Connector.Kisa.Tests/KisaConnectorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Kisa.Tests/KisaConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Kisa.Tests/KisaConnectorTests.cs index 5683ef86..5d29d93d 100644 --- a/src/StellaOps.Feedser.Source.Kisa.Tests/KisaConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Kisa.Tests/KisaConnectorTests.cs @@ -1,213 +1,213 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics.Metrics; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Kisa.Configuration; -using StellaOps.Feedser.Source.Kisa.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; -using System.Linq; - -namespace StellaOps.Feedser.Source.Kisa.Tests; - -[Collection("mongo-fixture")] -public sealed class KisaConnectorTests : IAsyncLifetime -{ - private static readonly Uri FeedUri = new("https://test.local/rss/securityInfo.do"); - private static readonly Uri DetailApiUri = new("https://test.local/rssDetailData.do?IDX=5868"); - private static readonly Uri DetailPageUri = new("https://test.local/detailDos.do?IDX=5868"); - - private readonly MongoIntegrationFixture _fixture; - private readonly CannedHttpMessageHandler _handler; - - public KisaConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesCanonicalAdvisory() - { - await using var provider = await BuildServiceProviderAsync(); - SeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); - advisories.Should().HaveCount(1); - - var advisory = advisories[0]; - advisory.AdvisoryKey.Should().Be("5868"); - advisory.Language.Should().Be("ko"); - advisory.Aliases.Should().Contain("CVE-2025-29866"); - advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("태그프리")); - advisory.References.Should().Contain(reference => reference.Url == DetailPageUri.ToString()); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(KisaConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.Should().NotBeNull(); - state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); - pendingDocs!.AsBsonArray.Should().BeEmpty(); - state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); - pendingMappings!.AsBsonArray.Should().BeEmpty(); - } - - [Fact] - public async Task Telemetry_RecordsMetrics() - { - await using var provider = await BuildServiceProviderAsync(); - SeedResponses(); - - using var metrics = new KisaMetricCollector(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - Sum(metrics.Measurements, "kisa.feed.success").Should().Be(1); - Sum(metrics.Measurements, "kisa.feed.items").Should().BeGreaterThan(0); - Sum(metrics.Measurements, "kisa.detail.success").Should().Be(1); - Sum(metrics.Measurements, "kisa.detail.failures").Should().Be(0); - Sum(metrics.Measurements, "kisa.parse.success").Should().Be(1); - Sum(metrics.Measurements, "kisa.parse.failures").Should().Be(0); - Sum(metrics.Measurements, "kisa.map.success").Should().Be(1); - Sum(metrics.Measurements, "kisa.map.failures").Should().Be(0); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddKisaConnector(options => - { - options.FeedUri = FeedUri; - options.DetailApiUri = new Uri("https://test.local/rssDetailData.do"); - options.DetailPageUri = new Uri("https://test.local/detailDos.do"); - options.RequestDelay = TimeSpan.Zero; - options.MaxAdvisoriesPerFetch = 10; - options.MaxKnownAdvisories = 32; - }); - - services.Configure(KisaOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedResponses() - { - AddXmlResponse(FeedUri, ReadFixture("kisa-feed.xml")); - AddJsonResponse(DetailApiUri, ReadFixture("kisa-detail.json")); - } - - private void AddXmlResponse(Uri uri, string xml) - { - _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(xml, Encoding.UTF8, "application/rss+xml"), - }); - } - - private void AddJsonResponse(Uri uri, string json) - { - _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) - { - Content = new StringContent(json, Encoding.UTF8, "application/json"), - }); - } - - private static string ReadFixture(string fileName) - => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); - - private static long Sum(IEnumerable measurements, string name) - => measurements.Where(m => m.Name == name).Sum(m => m.Value); - - private sealed class KisaMetricCollector : IDisposable - { - private readonly MeterListener _listener; - private readonly ConcurrentBag _measurements = new(); - - public KisaMetricCollector() - { - _listener = new MeterListener - { - InstrumentPublished = (instrument, listener) => - { - if (instrument.Meter.Name == KisaDiagnostics.MeterName) - { - listener.EnableMeasurementEvents(instrument); - } - }, - }; - - _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => - { - var tagList = new List>(tags.Length); - foreach (var tag in tags) - { - tagList.Add(tag); - } - - _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList)); - }); - - _listener.Start(); - } - - public IReadOnlyCollection Measurements => _measurements; - - public void Dispose() => _listener.Dispose(); - - internal sealed record MetricMeasurement(string Name, long Value, IReadOnlyList> Tags); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; -} +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Kisa.Configuration; +using StellaOps.Concelier.Connector.Kisa.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; +using System.Linq; + +namespace StellaOps.Concelier.Connector.Kisa.Tests; + +[Collection("mongo-fixture")] +public sealed class KisaConnectorTests : IAsyncLifetime +{ + private static readonly Uri FeedUri = new("https://test.local/rss/securityInfo.do"); + private static readonly Uri DetailApiUri = new("https://test.local/rssDetailData.do?IDX=5868"); + private static readonly Uri DetailPageUri = new("https://test.local/detailDos.do?IDX=5868"); + + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler; + + public KisaConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + advisories.Should().HaveCount(1); + + var advisory = advisories[0]; + advisory.AdvisoryKey.Should().Be("5868"); + advisory.Language.Should().Be("ko"); + advisory.Aliases.Should().Contain("CVE-2025-29866"); + advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("태그프리")); + advisory.References.Should().Contain(reference => reference.Url == DetailPageUri.ToString()); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(KisaConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.Should().NotBeNull(); + state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); + pendingDocs!.AsBsonArray.Should().BeEmpty(); + state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue(); + pendingMappings!.AsBsonArray.Should().BeEmpty(); + } + + [Fact] + public async Task Telemetry_RecordsMetrics() + { + await using var provider = await BuildServiceProviderAsync(); + SeedResponses(); + + using var metrics = new KisaMetricCollector(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + Sum(metrics.Measurements, "kisa.feed.success").Should().Be(1); + Sum(metrics.Measurements, "kisa.feed.items").Should().BeGreaterThan(0); + Sum(metrics.Measurements, "kisa.detail.success").Should().Be(1); + Sum(metrics.Measurements, "kisa.detail.failures").Should().Be(0); + Sum(metrics.Measurements, "kisa.parse.success").Should().Be(1); + Sum(metrics.Measurements, "kisa.parse.failures").Should().Be(0); + Sum(metrics.Measurements, "kisa.map.success").Should().Be(1); + Sum(metrics.Measurements, "kisa.map.failures").Should().Be(0); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddKisaConnector(options => + { + options.FeedUri = FeedUri; + options.DetailApiUri = new Uri("https://test.local/rssDetailData.do"); + options.DetailPageUri = new Uri("https://test.local/detailDos.do"); + options.RequestDelay = TimeSpan.Zero; + options.MaxAdvisoriesPerFetch = 10; + options.MaxKnownAdvisories = 32; + }); + + services.Configure(KisaOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedResponses() + { + AddXmlResponse(FeedUri, ReadFixture("kisa-feed.xml")); + AddJsonResponse(DetailApiUri, ReadFixture("kisa-detail.json")); + } + + private void AddXmlResponse(Uri uri, string xml) + { + _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(xml, Encoding.UTF8, "application/rss+xml"), + }); + } + + private void AddJsonResponse(Uri uri, string json) + { + _handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }); + } + + private static string ReadFixture(string fileName) + => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); + + private static long Sum(IEnumerable measurements, string name) + => measurements.Where(m => m.Name == name).Sum(m => m.Value); + + private sealed class KisaMetricCollector : IDisposable + { + private readonly MeterListener _listener; + private readonly ConcurrentBag _measurements = new(); + + public KisaMetricCollector() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == KisaDiagnostics.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + }, + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagList = new List>(tags.Length); + foreach (var tag in tags) + { + tagList.Add(tag); + } + + _measurements.Add(new MetricMeasurement(instrument.Name, measurement, tagList)); + }); + + _listener.Start(); + } + + public IReadOnlyCollection Measurements => _measurements; + + public void Dispose() => _listener.Dispose(); + + internal sealed record MetricMeasurement(string Name, long Value, IReadOnlyList> Tags); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj b/src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj similarity index 54% rename from src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj rename to src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj index 9bbb9390..c08ff53d 100644 --- a/src/StellaOps.Feedser.Source.Kisa.Tests/StellaOps.Feedser.Source.Kisa.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.Kisa.Tests/StellaOps.Concelier.Connector.Kisa.Tests.csproj @@ -1,24 +1,24 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - - - + + + net10.0 + enable + enable + + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Kisa/AGENTS.md b/src/StellaOps.Concelier.Connector.Kisa/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Source.Kisa/AGENTS.md rename to src/StellaOps.Concelier.Connector.Kisa/AGENTS.md index 8efa4065..71465b95 100644 --- a/src/StellaOps.Feedser.Source.Kisa/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Kisa/AGENTS.md @@ -1,38 +1,38 @@ -# AGENTS -## Role -Deliver the KISA (Korea Internet & Security Agency) advisory connector to ingest Korean vulnerability alerts for Feedser’s regional coverage. - -## Scope -- Identify KISA’s advisory feeds (RSS/Atom, JSON, HTML) and determine localisation requirements (Korean language parsing). -- Implement fetch/cursor logic with retry/backoff, handling authentication if required. -- Parse advisory content to extract summary, affected vendors/products, mitigation steps, CVEs, references. -- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (including vendor/language metadata). -- Provide deterministic fixtures and regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). -- `Feedser.Models` (canonical data structures). -- `Feedser.Testing` (integration fixtures and snapshots). - -## Interfaces & Contracts -- Job kinds: `kisa:fetch`, `kisa:parse`, `kisa:map`. -- Persist upstream caching metadata (e.g., ETag/Last-Modified) when available. -- Alias set should include KISA advisory identifiers and CVE IDs. - -## In/Out of scope -In scope: -- Advisory ingestion, translation/normalisation, range primitives. - -Out of scope: -- Automated Korean↔English translations beyond summary normalization (unless required for canonical fields). - -## Observability & Security Expectations -- Log fetch and mapping metrics; record failures with backoff. -- Sanitise HTML, removing scripts/styles. -- Handle character encoding (UTF-8/Korean) correctly. - -## Tests -- Add `StellaOps.Feedser.Source.Kisa.Tests` covering fetch/parse/map with Korean-language fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Deliver the KISA (Korea Internet & Security Agency) advisory connector to ingest Korean vulnerability alerts for Concelier’s regional coverage. + +## Scope +- Identify KISA’s advisory feeds (RSS/Atom, JSON, HTML) and determine localisation requirements (Korean language parsing). +- Implement fetch/cursor logic with retry/backoff, handling authentication if required. +- Parse advisory content to extract summary, affected vendors/products, mitigation steps, CVEs, references. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (including vendor/language metadata). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Concelier.Models` (canonical data structures). +- `Concelier.Testing` (integration fixtures and snapshots). + +## Interfaces & Contracts +- Job kinds: `kisa:fetch`, `kisa:parse`, `kisa:map`. +- Persist upstream caching metadata (e.g., ETag/Last-Modified) when available. +- Alias set should include KISA advisory identifiers and CVE IDs. + +## In/Out of scope +In scope: +- Advisory ingestion, translation/normalisation, range primitives. + +Out of scope: +- Automated Korean↔English translations beyond summary normalization (unless required for canonical fields). + +## Observability & Security Expectations +- Log fetch and mapping metrics; record failures with backoff. +- Sanitise HTML, removing scripts/styles. +- Handle character encoding (UTF-8/Korean) correctly. + +## Tests +- Add `StellaOps.Concelier.Connector.Kisa.Tests` covering fetch/parse/map with Korean-language fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Kisa/Configuration/KisaOptions.cs b/src/StellaOps.Concelier.Connector.Kisa/Configuration/KisaOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Kisa/Configuration/KisaOptions.cs rename to src/StellaOps.Concelier.Connector.Kisa/Configuration/KisaOptions.cs index c2da1e3a..afcbe6d9 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Configuration/KisaOptions.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Configuration/KisaOptions.cs @@ -1,97 +1,97 @@ -using System; - -namespace StellaOps.Feedser.Source.Kisa.Configuration; - -public sealed class KisaOptions -{ - public const string HttpClientName = "feedser.source.kisa"; - - /// - /// Primary RSS feed for security advisories. - /// - public Uri FeedUri { get; set; } = new("https://knvd.krcert.or.kr/rss/securityInfo.do"); - - /// - /// Detail API endpoint template; `IDX` query parameter identifies the advisory. - /// - public Uri DetailApiUri { get; set; } = new("https://knvd.krcert.or.kr/rssDetailData.do"); - - /// - /// Optional HTML detail URI template for provenance. - /// - public Uri DetailPageUri { get; set; } = new("https://knvd.krcert.or.kr/detailDos.do"); - - public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); - - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - public int MaxAdvisoriesPerFetch { get; set; } = 20; - - public int MaxKnownAdvisories { get; set; } = 256; - - public void Validate() - { - if (FeedUri is null || !FeedUri.IsAbsoluteUri) - { - throw new InvalidOperationException("KISA feed URI must be an absolute URI."); - } - - if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri) - { - throw new InvalidOperationException("KISA detail API URI must be an absolute URI."); - } - - if (DetailPageUri is null || !DetailPageUri.IsAbsoluteUri) - { - throw new InvalidOperationException("KISA detail page URI must be an absolute URI."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("RequestTimeout must be positive."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("RequestDelay cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("FailureBackoff must be positive."); - } - - if (MaxAdvisoriesPerFetch <= 0) - { - throw new InvalidOperationException("MaxAdvisoriesPerFetch must be greater than zero."); - } - - if (MaxKnownAdvisories <= 0) - { - throw new InvalidOperationException("MaxKnownAdvisories must be greater than zero."); - } - } - - public Uri BuildDetailApiUri(string idx) - { - if (string.IsNullOrWhiteSpace(idx)) - { - throw new ArgumentException("IDX must not be empty", nameof(idx)); - } - - var builder = new UriBuilder(DetailApiUri); - var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; - builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; - return builder.Uri; - } - - public Uri BuildDetailPageUri(string idx) - { - var builder = new UriBuilder(DetailPageUri); - var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; - builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; - return builder.Uri; - } -} +using System; + +namespace StellaOps.Concelier.Connector.Kisa.Configuration; + +public sealed class KisaOptions +{ + public const string HttpClientName = "concelier.source.kisa"; + + /// + /// Primary RSS feed for security advisories. + /// + public Uri FeedUri { get; set; } = new("https://knvd.krcert.or.kr/rss/securityInfo.do"); + + /// + /// Detail API endpoint template; `IDX` query parameter identifies the advisory. + /// + public Uri DetailApiUri { get; set; } = new("https://knvd.krcert.or.kr/rssDetailData.do"); + + /// + /// Optional HTML detail URI template for provenance. + /// + public Uri DetailPageUri { get; set; } = new("https://knvd.krcert.or.kr/detailDos.do"); + + public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); + + public int MaxAdvisoriesPerFetch { get; set; } = 20; + + public int MaxKnownAdvisories { get; set; } = 256; + + public void Validate() + { + if (FeedUri is null || !FeedUri.IsAbsoluteUri) + { + throw new InvalidOperationException("KISA feed URI must be an absolute URI."); + } + + if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri) + { + throw new InvalidOperationException("KISA detail API URI must be an absolute URI."); + } + + if (DetailPageUri is null || !DetailPageUri.IsAbsoluteUri) + { + throw new InvalidOperationException("KISA detail page URI must be an absolute URI."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RequestTimeout must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (FailureBackoff <= TimeSpan.Zero) + { + throw new InvalidOperationException("FailureBackoff must be positive."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException("MaxAdvisoriesPerFetch must be greater than zero."); + } + + if (MaxKnownAdvisories <= 0) + { + throw new InvalidOperationException("MaxKnownAdvisories must be greater than zero."); + } + } + + public Uri BuildDetailApiUri(string idx) + { + if (string.IsNullOrWhiteSpace(idx)) + { + throw new ArgumentException("IDX must not be empty", nameof(idx)); + } + + var builder = new UriBuilder(DetailApiUri); + var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; + builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; + return builder.Uri; + } + + public Uri BuildDetailPageUri(string idx) + { + var builder = new UriBuilder(DetailPageUri); + var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; + builder.Query = $"{queryPrefix}IDX={Uri.EscapeDataString(idx)}"; + return builder.Uri; + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaCursor.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaCursor.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaCursor.cs index 30c31adb..ed704e73 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaCursor.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaCursor.cs @@ -1,120 +1,120 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -internal sealed record KisaCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - IReadOnlyCollection KnownIds, - DateTimeOffset? LastPublished, - DateTimeOffset? LastFetchAt) -{ - private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); - private static readonly IReadOnlyCollection EmptyStrings = Array.Empty(); - - public static KisaCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null); - - public KisaCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = Distinct(documents) }; - - public KisaCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = Distinct(mappings) }; - - public KisaCursor WithKnownIds(IEnumerable ids) - => this with { KnownIds = ids?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings }; - - public KisaCursor WithLastPublished(DateTimeOffset? published) - => this with { LastPublished = published }; - - public KisaCursor WithLastFetch(DateTimeOffset? timestamp) - => this with { LastFetchAt = timestamp }; - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - ["knownIds"] = new BsonArray(KnownIds), - }; - - if (LastPublished.HasValue) - { - document["lastPublished"] = LastPublished.Value.UtcDateTime; - } - - if (LastFetchAt.HasValue) - { - document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; - } - - return document; - } - - public static KisaCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var knownIds = ReadStringArray(document, "knownIds"); - var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) - ? ParseDate(publishedValue) - : null; - var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue) - ? ParseDate(fetchValue) - : null; - - return new KisaCursor(pendingDocuments, pendingMappings, knownIds, lastPublished, lastFetch); - } - - private static IReadOnlyCollection Distinct(IEnumerable? values) - => values?.Distinct().ToArray() ?? EmptyGuids; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuids; - } - - var items = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var id)) - { - items.Add(id); - } - } - - return items; - } - - private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyStrings; - } - - return array - .Select(element => element?.ToString() ?? string.Empty) - .Where(static s => !string.IsNullOrWhiteSpace(s)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +internal sealed record KisaCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyCollection KnownIds, + DateTimeOffset? LastPublished, + DateTimeOffset? LastFetchAt) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStrings = Array.Empty(); + + public static KisaCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null); + + public KisaCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = Distinct(documents) }; + + public KisaCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = Distinct(mappings) }; + + public KisaCursor WithKnownIds(IEnumerable ids) + => this with { KnownIds = ids?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings }; + + public KisaCursor WithLastPublished(DateTimeOffset? published) + => this with { LastPublished = published }; + + public KisaCursor WithLastFetch(DateTimeOffset? timestamp) + => this with { LastFetchAt = timestamp }; + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + ["knownIds"] = new BsonArray(KnownIds), + }; + + if (LastPublished.HasValue) + { + document["lastPublished"] = LastPublished.Value.UtcDateTime; + } + + if (LastFetchAt.HasValue) + { + document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; + } + + return document; + } + + public static KisaCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var knownIds = ReadStringArray(document, "knownIds"); + var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) + ? ParseDate(publishedValue) + : null; + var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue) + ? ParseDate(fetchValue) + : null; + + return new KisaCursor(pendingDocuments, pendingMappings, knownIds, lastPublished, lastFetch); + } + + private static IReadOnlyCollection Distinct(IEnumerable? values) + => values?.Distinct().ToArray() ?? EmptyGuids; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; + } + + var items = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var id)) + { + items.Add(id); + } + } + + return items; + } + + private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyStrings; + } + + return array + .Select(element => element?.ToString() ?? string.Empty) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailParser.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailParser.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailParser.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailParser.cs index 61b0ba23..e1772214 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailParser.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailParser.cs @@ -1,114 +1,114 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -public sealed class KisaDetailParser -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private readonly HtmlContentSanitizer _sanitizer; - - public KisaDetailParser(HtmlContentSanitizer sanitizer) - => _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); - - public KisaParsedAdvisory Parse(Uri detailApiUri, Uri detailPageUri, byte[] payload) - { - var response = JsonSerializer.Deserialize(payload, SerializerOptions) - ?? throw new InvalidOperationException("KISA detail payload deserialized to null"); - - var idx = response.Idx ?? throw new InvalidOperationException("KISA detail missing IDX"); - var contentHtml = _sanitizer.Sanitize(response.ContentHtml ?? string.Empty, detailPageUri); - - return new KisaParsedAdvisory( - idx, - Normalize(response.Title) ?? idx, - Normalize(response.Summary), - contentHtml, - Normalize(response.Severity), - response.Published, - response.Updated ?? response.Published, - detailApiUri, - detailPageUri, - NormalizeArray(response.CveIds), - MapReferences(response.References), - MapProducts(response.Products)); - } - - private static IReadOnlyList NormalizeArray(string[]? values) - { - if (values is null || values.Length == 0) - { - return Array.Empty(); - } - - return values - .Select(Normalize) - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) - .ToArray()!; - } - - private static IReadOnlyList MapReferences(KisaReferenceDto[]? references) - { - if (references is null || references.Length == 0) - { - return Array.Empty(); - } - - return references - .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) - .Select(reference => new KisaParsedReference(reference.Url!, Normalize(reference.Label))) - .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList MapProducts(KisaProductDto[]? products) - { - if (products is null || products.Length == 0) - { - return Array.Empty(); - } - - return products - .Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name)) - .Select(product => new KisaParsedProduct( - Normalize(product.Vendor), - Normalize(product.Name), - Normalize(product.Versions))) - .ToArray(); - } - - private static string? Normalize(string? value) - => string.IsNullOrWhiteSpace(value) - ? null - : value.Normalize(NormalizationForm.FormC).Trim(); -} - -public sealed record KisaParsedAdvisory( - string AdvisoryId, - string Title, - string? Summary, - string ContentHtml, - string? Severity, - DateTimeOffset? Published, - DateTimeOffset? Modified, - Uri DetailApiUri, - Uri DetailPageUri, - IReadOnlyList CveIds, - IReadOnlyList References, - IReadOnlyList Products); - -public sealed record KisaParsedReference(string Url, string? Label); - -public sealed record KisaParsedProduct(string? Vendor, string? Name, string? Versions); +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Concelier.Connector.Common.Html; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +public sealed class KisaDetailParser +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly HtmlContentSanitizer _sanitizer; + + public KisaDetailParser(HtmlContentSanitizer sanitizer) + => _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); + + public KisaParsedAdvisory Parse(Uri detailApiUri, Uri detailPageUri, byte[] payload) + { + var response = JsonSerializer.Deserialize(payload, SerializerOptions) + ?? throw new InvalidOperationException("KISA detail payload deserialized to null"); + + var idx = response.Idx ?? throw new InvalidOperationException("KISA detail missing IDX"); + var contentHtml = _sanitizer.Sanitize(response.ContentHtml ?? string.Empty, detailPageUri); + + return new KisaParsedAdvisory( + idx, + Normalize(response.Title) ?? idx, + Normalize(response.Summary), + contentHtml, + Normalize(response.Severity), + response.Published, + response.Updated ?? response.Published, + detailApiUri, + detailPageUri, + NormalizeArray(response.CveIds), + MapReferences(response.References), + MapProducts(response.Products)); + } + + private static IReadOnlyList NormalizeArray(string[]? values) + { + if (values is null || values.Length == 0) + { + return Array.Empty(); + } + + return values + .Select(Normalize) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToArray()!; + } + + private static IReadOnlyList MapReferences(KisaReferenceDto[]? references) + { + if (references is null || references.Length == 0) + { + return Array.Empty(); + } + + return references + .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) + .Select(reference => new KisaParsedReference(reference.Url!, Normalize(reference.Label))) + .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList MapProducts(KisaProductDto[]? products) + { + if (products is null || products.Length == 0) + { + return Array.Empty(); + } + + return products + .Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name)) + .Select(product => new KisaParsedProduct( + Normalize(product.Vendor), + Normalize(product.Name), + Normalize(product.Versions))) + .ToArray(); + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) + ? null + : value.Normalize(NormalizationForm.FormC).Trim(); +} + +public sealed record KisaParsedAdvisory( + string AdvisoryId, + string Title, + string? Summary, + string ContentHtml, + string? Severity, + DateTimeOffset? Published, + DateTimeOffset? Modified, + Uri DetailApiUri, + Uri DetailPageUri, + IReadOnlyList CveIds, + IReadOnlyList References, + IReadOnlyList Products); + +public sealed record KisaParsedReference(string Url, string? Label); + +public sealed record KisaParsedProduct(string? Vendor, string? Name, string? Versions); diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailResponse.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailResponse.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailResponse.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailResponse.cs index f6f392f9..ed5c0bbe 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDetailResponse.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDetailResponse.cs @@ -1,58 +1,58 @@ -using System; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -internal sealed class KisaDetailResponse -{ - [JsonPropertyName("idx")] - public string? Idx { get; init; } - - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("contentHtml")] - public string? ContentHtml { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("updated")] - public DateTimeOffset? Updated { get; init; } - - [JsonPropertyName("cveIds")] - public string[]? CveIds { get; init; } - - [JsonPropertyName("references")] - public KisaReferenceDto[]? References { get; init; } - - [JsonPropertyName("products")] - public KisaProductDto[]? Products { get; init; } -} - -internal sealed class KisaReferenceDto -{ - [JsonPropertyName("url")] - public string? Url { get; init; } - - [JsonPropertyName("label")] - public string? Label { get; init; } -} - -internal sealed class KisaProductDto -{ - [JsonPropertyName("vendor")] - public string? Vendor { get; init; } - - [JsonPropertyName("name")] - public string? Name { get; init; } - - [JsonPropertyName("versions")] - public string? Versions { get; init; } -} +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +internal sealed class KisaDetailResponse +{ + [JsonPropertyName("idx")] + public string? Idx { get; init; } + + [JsonPropertyName("title")] + public string? Title { get; init; } + + [JsonPropertyName("summary")] + public string? Summary { get; init; } + + [JsonPropertyName("contentHtml")] + public string? ContentHtml { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("published")] + public DateTimeOffset? Published { get; init; } + + [JsonPropertyName("updated")] + public DateTimeOffset? Updated { get; init; } + + [JsonPropertyName("cveIds")] + public string[]? CveIds { get; init; } + + [JsonPropertyName("references")] + public KisaReferenceDto[]? References { get; init; } + + [JsonPropertyName("products")] + public KisaProductDto[]? Products { get; init; } +} + +internal sealed class KisaReferenceDto +{ + [JsonPropertyName("url")] + public string? Url { get; init; } + + [JsonPropertyName("label")] + public string? Label { get; init; } +} + +internal sealed class KisaProductDto +{ + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("versions")] + public string? Versions { get; init; } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDiagnostics.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDiagnostics.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDiagnostics.cs index ea165857..f7e2c854 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDiagnostics.cs @@ -1,169 +1,169 @@ -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -public sealed class KisaDiagnostics : IDisposable -{ - public const string MeterName = "StellaOps.Feedser.Source.Kisa"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _feedAttempts; - private readonly Counter _feedSuccess; - private readonly Counter _feedFailures; - private readonly Counter _feedItems; - private readonly Counter _detailAttempts; - private readonly Counter _detailSuccess; - private readonly Counter _detailUnchanged; - private readonly Counter _detailFailures; - private readonly Counter _parseAttempts; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Counter _cursorUpdates; - - public KisaDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _feedAttempts = _meter.CreateCounter( - name: "kisa.feed.attempts", - unit: "operations", - description: "Number of RSS fetch attempts performed for the KISA connector."); - _feedSuccess = _meter.CreateCounter( - name: "kisa.feed.success", - unit: "operations", - description: "Number of RSS fetch attempts that completed successfully."); - _feedFailures = _meter.CreateCounter( - name: "kisa.feed.failures", - unit: "operations", - description: "Number of RSS fetch attempts that failed."); - _feedItems = _meter.CreateCounter( - name: "kisa.feed.items", - unit: "items", - description: "Number of feed items returned by successful RSS fetches."); - _detailAttempts = _meter.CreateCounter( - name: "kisa.detail.attempts", - unit: "documents", - description: "Number of advisory detail fetch attempts."); - _detailSuccess = _meter.CreateCounter( - name: "kisa.detail.success", - unit: "documents", - description: "Number of advisory detail documents fetched successfully."); - _detailUnchanged = _meter.CreateCounter( - name: "kisa.detail.unchanged", - unit: "documents", - description: "Number of advisory detail fetches that returned HTTP 304 (no change)."); - _detailFailures = _meter.CreateCounter( - name: "kisa.detail.failures", - unit: "documents", - description: "Number of advisory detail fetch attempts that failed."); - _parseAttempts = _meter.CreateCounter( - name: "kisa.parse.attempts", - unit: "documents", - description: "Number of advisory documents queued for parsing."); - _parseSuccess = _meter.CreateCounter( - name: "kisa.parse.success", - unit: "documents", - description: "Number of advisory documents parsed successfully into DTOs."); - _parseFailures = _meter.CreateCounter( - name: "kisa.parse.failures", - unit: "documents", - description: "Number of advisory documents that failed parsing."); - _mapSuccess = _meter.CreateCounter( - name: "kisa.map.success", - unit: "advisories", - description: "Number of canonical advisories produced by the mapper."); - _mapFailures = _meter.CreateCounter( - name: "kisa.map.failures", - unit: "advisories", - description: "Number of advisories that failed to map."); - _cursorUpdates = _meter.CreateCounter( - name: "kisa.cursor.updates", - unit: "updates", - description: "Number of times the published cursor advanced."); - } - - public void FeedAttempt() => _feedAttempts.Add(1); - - public void FeedSuccess(int itemCount) - { - _feedSuccess.Add(1); - if (itemCount > 0) - { - _feedItems.Add(itemCount); - } - } - - public void FeedFailure(string reason) - => _feedFailures.Add(1, GetReasonTags(reason)); - - public void DetailAttempt(string? category) - => _detailAttempts.Add(1, GetCategoryTags(category)); - - public void DetailSuccess(string? category) - => _detailSuccess.Add(1, GetCategoryTags(category)); - - public void DetailUnchanged(string? category) - => _detailUnchanged.Add(1, GetCategoryTags(category)); - - public void DetailFailure(string? category, string reason) - => _detailFailures.Add(1, GetCategoryReasonTags(category, reason)); - - public void ParseAttempt(string? category) - => _parseAttempts.Add(1, GetCategoryTags(category)); - - public void ParseSuccess(string? category) - => _parseSuccess.Add(1, GetCategoryTags(category)); - - public void ParseFailure(string? category, string reason) - => _parseFailures.Add(1, GetCategoryReasonTags(category, reason)); - - public void MapSuccess(string? severity) - => _mapSuccess.Add(1, GetSeverityTags(severity)); - - public void MapFailure(string? severity, string reason) - => _mapFailures.Add(1, GetSeverityReasonTags(severity, reason)); - - public void CursorAdvanced() - => _cursorUpdates.Add(1); - - public Meter Meter => _meter; - - public void Dispose() => _meter.Dispose(); - - private static KeyValuePair[] GetCategoryTags(string? category) - => new[] - { - new KeyValuePair("category", Normalize(category)) - }; - - private static KeyValuePair[] GetCategoryReasonTags(string? category, string reason) - => new[] - { - new KeyValuePair("category", Normalize(category)), - new KeyValuePair("reason", Normalize(reason)), - }; - - private static KeyValuePair[] GetSeverityTags(string? severity) - => new[] - { - new KeyValuePair("severity", Normalize(severity)), - }; - - private static KeyValuePair[] GetSeverityReasonTags(string? severity, string reason) - => new[] - { - new KeyValuePair("severity", Normalize(severity)), - new KeyValuePair("reason", Normalize(reason)), - }; - - private static KeyValuePair[] GetReasonTags(string reason) - => new[] - { - new KeyValuePair("reason", Normalize(reason)), - }; - - private static string Normalize(string? value) - => string.IsNullOrWhiteSpace(value) ? "unknown" : value!; -} +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +public sealed class KisaDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Concelier.Connector.Kisa"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _feedAttempts; + private readonly Counter _feedSuccess; + private readonly Counter _feedFailures; + private readonly Counter _feedItems; + private readonly Counter _detailAttempts; + private readonly Counter _detailSuccess; + private readonly Counter _detailUnchanged; + private readonly Counter _detailFailures; + private readonly Counter _parseAttempts; + private readonly Counter _parseSuccess; + private readonly Counter _parseFailures; + private readonly Counter _mapSuccess; + private readonly Counter _mapFailures; + private readonly Counter _cursorUpdates; + + public KisaDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _feedAttempts = _meter.CreateCounter( + name: "kisa.feed.attempts", + unit: "operations", + description: "Number of RSS fetch attempts performed for the KISA connector."); + _feedSuccess = _meter.CreateCounter( + name: "kisa.feed.success", + unit: "operations", + description: "Number of RSS fetch attempts that completed successfully."); + _feedFailures = _meter.CreateCounter( + name: "kisa.feed.failures", + unit: "operations", + description: "Number of RSS fetch attempts that failed."); + _feedItems = _meter.CreateCounter( + name: "kisa.feed.items", + unit: "items", + description: "Number of feed items returned by successful RSS fetches."); + _detailAttempts = _meter.CreateCounter( + name: "kisa.detail.attempts", + unit: "documents", + description: "Number of advisory detail fetch attempts."); + _detailSuccess = _meter.CreateCounter( + name: "kisa.detail.success", + unit: "documents", + description: "Number of advisory detail documents fetched successfully."); + _detailUnchanged = _meter.CreateCounter( + name: "kisa.detail.unchanged", + unit: "documents", + description: "Number of advisory detail fetches that returned HTTP 304 (no change)."); + _detailFailures = _meter.CreateCounter( + name: "kisa.detail.failures", + unit: "documents", + description: "Number of advisory detail fetch attempts that failed."); + _parseAttempts = _meter.CreateCounter( + name: "kisa.parse.attempts", + unit: "documents", + description: "Number of advisory documents queued for parsing."); + _parseSuccess = _meter.CreateCounter( + name: "kisa.parse.success", + unit: "documents", + description: "Number of advisory documents parsed successfully into DTOs."); + _parseFailures = _meter.CreateCounter( + name: "kisa.parse.failures", + unit: "documents", + description: "Number of advisory documents that failed parsing."); + _mapSuccess = _meter.CreateCounter( + name: "kisa.map.success", + unit: "advisories", + description: "Number of canonical advisories produced by the mapper."); + _mapFailures = _meter.CreateCounter( + name: "kisa.map.failures", + unit: "advisories", + description: "Number of advisories that failed to map."); + _cursorUpdates = _meter.CreateCounter( + name: "kisa.cursor.updates", + unit: "updates", + description: "Number of times the published cursor advanced."); + } + + public void FeedAttempt() => _feedAttempts.Add(1); + + public void FeedSuccess(int itemCount) + { + _feedSuccess.Add(1); + if (itemCount > 0) + { + _feedItems.Add(itemCount); + } + } + + public void FeedFailure(string reason) + => _feedFailures.Add(1, GetReasonTags(reason)); + + public void DetailAttempt(string? category) + => _detailAttempts.Add(1, GetCategoryTags(category)); + + public void DetailSuccess(string? category) + => _detailSuccess.Add(1, GetCategoryTags(category)); + + public void DetailUnchanged(string? category) + => _detailUnchanged.Add(1, GetCategoryTags(category)); + + public void DetailFailure(string? category, string reason) + => _detailFailures.Add(1, GetCategoryReasonTags(category, reason)); + + public void ParseAttempt(string? category) + => _parseAttempts.Add(1, GetCategoryTags(category)); + + public void ParseSuccess(string? category) + => _parseSuccess.Add(1, GetCategoryTags(category)); + + public void ParseFailure(string? category, string reason) + => _parseFailures.Add(1, GetCategoryReasonTags(category, reason)); + + public void MapSuccess(string? severity) + => _mapSuccess.Add(1, GetSeverityTags(severity)); + + public void MapFailure(string? severity, string reason) + => _mapFailures.Add(1, GetSeverityReasonTags(severity, reason)); + + public void CursorAdvanced() + => _cursorUpdates.Add(1); + + public Meter Meter => _meter; + + public void Dispose() => _meter.Dispose(); + + private static KeyValuePair[] GetCategoryTags(string? category) + => new[] + { + new KeyValuePair("category", Normalize(category)) + }; + + private static KeyValuePair[] GetCategoryReasonTags(string? category, string reason) + => new[] + { + new KeyValuePair("category", Normalize(category)), + new KeyValuePair("reason", Normalize(reason)), + }; + + private static KeyValuePair[] GetSeverityTags(string? severity) + => new[] + { + new KeyValuePair("severity", Normalize(severity)), + }; + + private static KeyValuePair[] GetSeverityReasonTags(string? severity, string reason) + => new[] + { + new KeyValuePair("severity", Normalize(severity)), + new KeyValuePair("reason", Normalize(reason)), + }; + + private static KeyValuePair[] GetReasonTags(string reason) + => new[] + { + new KeyValuePair("reason", Normalize(reason)), + }; + + private static string Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? "unknown" : value!; +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDocumentMetadata.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDocumentMetadata.cs index 8ac285d3..62564bf5 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaDocumentMetadata.cs @@ -1,29 +1,29 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -internal static class KisaDocumentMetadata -{ - public static Dictionary CreateMetadata(KisaFeedItem item) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["kisa.idx"] = item.AdvisoryId, - ["kisa.detailPage"] = item.DetailPageUri.ToString(), - ["kisa.published"] = item.Published.ToString("O"), - }; - - if (!string.IsNullOrWhiteSpace(item.Title)) - { - metadata["kisa.title"] = item.Title!; - } - - if (!string.IsNullOrWhiteSpace(item.Category)) - { - metadata["kisa.category"] = item.Category!; - } - - return metadata; - } -} +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +internal static class KisaDocumentMetadata +{ + public static Dictionary CreateMetadata(KisaFeedItem item) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["kisa.idx"] = item.AdvisoryId, + ["kisa.detailPage"] = item.DetailPageUri.ToString(), + ["kisa.published"] = item.Published.ToString("O"), + }; + + if (!string.IsNullOrWhiteSpace(item.Title)) + { + metadata["kisa.title"] = item.Title!; + } + + if (!string.IsNullOrWhiteSpace(item.Category)) + { + metadata["kisa.category"] = item.Category!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedClient.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedClient.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedClient.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedClient.cs index 413b1983..0b75719f 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedClient.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedClient.cs @@ -1,116 +1,116 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using System.Xml.Linq; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Kisa.Configuration; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -public sealed class KisaFeedClient -{ - private readonly IHttpClientFactory _httpClientFactory; - private readonly KisaOptions _options; - private readonly ILogger _logger; - - public KisaFeedClient( - IHttpClientFactory httpClientFactory, - IOptions options, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task> LoadAsync(CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(KisaOptions.HttpClientName); - - using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri); - request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8"); - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - var document = XDocument.Load(stream); - - var items = new List(); - foreach (var element in document.Descendants("item")) - { - cancellationToken.ThrowIfCancellationRequested(); - - var link = element.Element("link")?.Value?.Trim(); - if (string.IsNullOrWhiteSpace(link)) - { - continue; - } - - if (!TryExtractIdx(link, out var idx)) - { - continue; - } - - var title = element.Element("title")?.Value?.Trim(); - var category = element.Element("category")?.Value?.Trim(); - var published = ParseDate(element.Element("pubDate")?.Value); - var detailApiUri = _options.BuildDetailApiUri(idx); - var detailPageUri = _options.BuildDetailPageUri(idx); - - items.Add(new KisaFeedItem(idx, detailApiUri, detailPageUri, published, title, category)); - } - - return items; - } - - private static bool TryExtractIdx(string link, out string idx) - { - idx = string.Empty; - if (string.IsNullOrWhiteSpace(link)) - { - return false; - } - - if (!Uri.TryCreate(link, UriKind.Absolute, out var uri)) - { - return false; - } - - var query = uri.Query?.TrimStart('?'); - if (string.IsNullOrEmpty(query)) - { - return false; - } - - foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) - { - var separatorIndex = pair.IndexOf('='); - if (separatorIndex <= 0) - { - continue; - } - - var key = pair[..separatorIndex].Trim(); - if (!key.Equals("IDX", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - idx = Uri.UnescapeDataString(pair[(separatorIndex + 1)..]); - return !string.IsNullOrWhiteSpace(idx); - } - - return false; - } - - private static DateTimeOffset ParseDate(string? value) - => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) - ? parsed - : DateTimeOffset.UtcNow; -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Kisa.Configuration; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +public sealed class KisaFeedClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly KisaOptions _options; + private readonly ILogger _logger; + + public KisaFeedClient( + IHttpClientFactory httpClientFactory, + IOptions options, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task> LoadAsync(CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(KisaOptions.HttpClientName); + + using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri); + request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8"); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + + var items = new List(); + foreach (var element in document.Descendants("item")) + { + cancellationToken.ThrowIfCancellationRequested(); + + var link = element.Element("link")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(link)) + { + continue; + } + + if (!TryExtractIdx(link, out var idx)) + { + continue; + } + + var title = element.Element("title")?.Value?.Trim(); + var category = element.Element("category")?.Value?.Trim(); + var published = ParseDate(element.Element("pubDate")?.Value); + var detailApiUri = _options.BuildDetailApiUri(idx); + var detailPageUri = _options.BuildDetailPageUri(idx); + + items.Add(new KisaFeedItem(idx, detailApiUri, detailPageUri, published, title, category)); + } + + return items; + } + + private static bool TryExtractIdx(string link, out string idx) + { + idx = string.Empty; + if (string.IsNullOrWhiteSpace(link)) + { + return false; + } + + if (!Uri.TryCreate(link, UriKind.Absolute, out var uri)) + { + return false; + } + + var query = uri.Query?.TrimStart('?'); + if (string.IsNullOrEmpty(query)) + { + return false; + } + + foreach (var pair in query.Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var separatorIndex = pair.IndexOf('='); + if (separatorIndex <= 0) + { + continue; + } + + var key = pair[..separatorIndex].Trim(); + if (!key.Equals("IDX", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + idx = Uri.UnescapeDataString(pair[(separatorIndex + 1)..]); + return !string.IsNullOrWhiteSpace(idx); + } + + return false; + } + + private static DateTimeOffset ParseDate(string? value) + => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) + ? parsed + : DateTimeOffset.UtcNow; +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedItem.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedItem.cs similarity index 74% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedItem.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedItem.cs index 3cc74e91..9c46469d 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaFeedItem.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaFeedItem.cs @@ -1,11 +1,11 @@ -using System; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -public sealed record KisaFeedItem( - string AdvisoryId, - Uri DetailApiUri, - Uri DetailPageUri, - DateTimeOffset Published, - string? Title, - string? Category); +using System; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +public sealed record KisaFeedItem( + string AdvisoryId, + Uri DetailApiUri, + Uri DetailPageUri, + DateTimeOffset Published, + string? Title, + string? Category); diff --git a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaMapper.cs b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kisa/Internal/KisaMapper.cs rename to src/StellaOps.Concelier.Connector.Kisa/Internal/KisaMapper.cs index 0fb1592a..47b989e3 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Internal/KisaMapper.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Internal/KisaMapper.cs @@ -1,145 +1,145 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Kisa.Internal; - -internal static class KisaMapper -{ - public static Advisory Map(KisaParsedAdvisory dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var packages = BuildPackages(dto, recordedAt); - var provenance = new AdvisoryProvenance( - KisaConnectorPlugin.SourceName, - "advisory", - dto.AdvisoryId, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - advisoryKey: dto.AdvisoryId, - title: dto.Title, - summary: dto.Summary, - language: "ko", - published: dto.Published, - modified: dto.Modified, - severity: dto.Severity?.ToLowerInvariant(), - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: packages, - cvssMetrics: Array.Empty(), - provenance: new[] { provenance }); - } - - private static IReadOnlyList BuildAliases(KisaParsedAdvisory dto) - { - var aliases = new List(capacity: dto.CveIds.Count + 1) { dto.AdvisoryId }; - aliases.AddRange(dto.CveIds); - return aliases - .Where(static alias => !string.IsNullOrWhiteSpace(alias)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildReferences(KisaParsedAdvisory dto, DateTimeOffset recordedAt) - { - var references = new List - { - new(dto.DetailPageUri.ToString(), "details", "kisa", null, new AdvisoryProvenance( - KisaConnectorPlugin.SourceName, - "reference", - dto.DetailPageUri.ToString(), - recordedAt, - new[] { ProvenanceFieldMasks.References })) - }; - - foreach (var reference in dto.References) - { - if (string.IsNullOrWhiteSpace(reference.Url)) - { - continue; - } - - references.Add(new AdvisoryReference( - reference.Url, - kind: "reference", - sourceTag: "kisa", - summary: reference.Label, - provenance: new AdvisoryProvenance( - KisaConnectorPlugin.SourceName, - "reference", - reference.Url, - recordedAt, - new[] { ProvenanceFieldMasks.References }))); - } - - return references - .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildPackages(KisaParsedAdvisory dto, DateTimeOffset recordedAt) - { - if (dto.Products.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Products.Count); - foreach (var product in dto.Products) - { - var vendor = string.IsNullOrWhiteSpace(product.Vendor) ? "Unknown" : product.Vendor!; - var name = product.Name; - var identifier = string.IsNullOrWhiteSpace(name) ? vendor : $"{vendor} {name}"; - - var provenance = new AdvisoryProvenance( - KisaConnectorPlugin.SourceName, - "package", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.AffectedPackages }); - - var versionRanges = string.IsNullOrWhiteSpace(product.Versions) - ? Array.Empty() - : new[] - { - new AffectedVersionRange( - rangeKind: "string", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: product.Versions, - provenance: new AdvisoryProvenance( - KisaConnectorPlugin.SourceName, - "package-range", - product.Versions, - recordedAt, - new[] { ProvenanceFieldMasks.VersionRanges })) - }; - - packages.Add(new AffectedPackage( - AffectedPackageTypes.Vendor, - identifier, - platform: null, - versionRanges: versionRanges, - statuses: Array.Empty(), - provenance: new[] { provenance }, - normalizedVersions: Array.Empty())); - } - - return packages - .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Kisa.Internal; + +internal static class KisaMapper +{ + public static Advisory Map(KisaParsedAdvisory dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var packages = BuildPackages(dto, recordedAt); + var provenance = new AdvisoryProvenance( + KisaConnectorPlugin.SourceName, + "advisory", + dto.AdvisoryId, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title, + summary: dto.Summary, + language: "ko", + published: dto.Published, + modified: dto.Modified, + severity: dto.Severity?.ToLowerInvariant(), + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static IReadOnlyList BuildAliases(KisaParsedAdvisory dto) + { + var aliases = new List(capacity: dto.CveIds.Count + 1) { dto.AdvisoryId }; + aliases.AddRange(dto.CveIds); + return aliases + .Where(static alias => !string.IsNullOrWhiteSpace(alias)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildReferences(KisaParsedAdvisory dto, DateTimeOffset recordedAt) + { + var references = new List + { + new(dto.DetailPageUri.ToString(), "details", "kisa", null, new AdvisoryProvenance( + KisaConnectorPlugin.SourceName, + "reference", + dto.DetailPageUri.ToString(), + recordedAt, + new[] { ProvenanceFieldMasks.References })) + }; + + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + references.Add(new AdvisoryReference( + reference.Url, + kind: "reference", + sourceTag: "kisa", + summary: reference.Label, + provenance: new AdvisoryProvenance( + KisaConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + return references + .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + private static IReadOnlyList BuildPackages(KisaParsedAdvisory dto, DateTimeOffset recordedAt) + { + if (dto.Products.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Products.Count); + foreach (var product in dto.Products) + { + var vendor = string.IsNullOrWhiteSpace(product.Vendor) ? "Unknown" : product.Vendor!; + var name = product.Name; + var identifier = string.IsNullOrWhiteSpace(name) ? vendor : $"{vendor} {name}"; + + var provenance = new AdvisoryProvenance( + KisaConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + var versionRanges = string.IsNullOrWhiteSpace(product.Versions) + ? Array.Empty() + : new[] + { + new AffectedVersionRange( + rangeKind: "string", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: product.Versions, + provenance: new AdvisoryProvenance( + KisaConnectorPlugin.SourceName, + "package-range", + product.Versions, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges })) + }; + + packages.Add(new AffectedPackage( + AffectedPackageTypes.Vendor, + identifier, + platform: null, + versionRanges: versionRanges, + statuses: Array.Empty(), + provenance: new[] { provenance }, + normalizedVersions: Array.Empty())); + } + + return packages + .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/Jobs.cs b/src/StellaOps.Concelier.Connector.Kisa/Jobs.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Kisa/Jobs.cs rename to src/StellaOps.Concelier.Connector.Kisa/Jobs.cs index 99e2cacc..9583043e 100644 --- a/src/StellaOps.Feedser.Source.Kisa/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/Jobs.cs @@ -1,22 +1,22 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Kisa; - -internal static class KisaJobKinds -{ - public const string Fetch = "source:kisa:fetch"; -} - -internal sealed class KisaFetchJob : IJob -{ - private readonly KisaConnector _connector; - - public KisaFetchJob(KisaConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Kisa; + +internal static class KisaJobKinds +{ + public const string Fetch = "source:kisa:fetch"; +} + +internal sealed class KisaFetchJob : IJob +{ + private readonly KisaConnector _connector; + + public KisaFetchJob(KisaConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Feedser.Source.Kisa/KisaConnector.cs b/src/StellaOps.Concelier.Connector.Kisa/KisaConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Kisa/KisaConnector.cs rename to src/StellaOps.Concelier.Connector.Kisa/KisaConnector.cs index 2cd61d53..9d6b09a2 100644 --- a/src/StellaOps.Feedser.Source.Kisa/KisaConnector.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/KisaConnector.cs @@ -1,404 +1,404 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Kisa.Configuration; -using StellaOps.Feedser.Source.Kisa.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Kisa; - -public sealed class KisaConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, - }; - - private readonly KisaFeedClient _feedClient; - private readonly KisaDetailParser _detailParser; - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly KisaOptions _options; - private readonly KisaDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - - public KisaConnector( - KisaFeedClient feedClient, - KisaDetailParser detailParser, - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - KisaDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); - _detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser)); - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => KisaConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var now = _timeProvider.GetUtcNow(); - _diagnostics.FeedAttempt(); - IReadOnlyList items; - - try - { - items = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false); - _diagnostics.FeedSuccess(items.Count); - - if (items.Count > 0) - { - _logger.LogInformation("KISA feed returned {ItemCount} advisories", items.Count); - } - else - { - _logger.LogDebug("KISA feed returned no advisories"); - } - } - catch (Exception ex) - { - _diagnostics.FeedFailure(ex.GetType().Name); - _logger.LogError(ex, "KISA feed fetch failed"); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (items.Count == 0) - { - await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false); - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var knownIds = new HashSet(cursor.KnownIds, StringComparer.OrdinalIgnoreCase); - var processed = 0; - var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; - - foreach (var item in items.OrderByDescending(static i => i.Published)) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (knownIds.Contains(item.AdvisoryId)) - { - continue; - } - - if (processed >= _options.MaxAdvisoriesPerFetch) - { - break; - } - - var category = item.Category; - _diagnostics.DetailAttempt(category); - - try - { - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailApiUri.ToString(), cancellationToken).ConfigureAwait(false); - var request = new SourceFetchRequest(KisaOptions.HttpClientName, SourceName, item.DetailApiUri) - { - Metadata = KisaDocumentMetadata.CreateMetadata(item), - AcceptHeaders = new[] { "application/json", "text/json" }, - ETag = existing?.Etag, - LastModified = existing?.LastModified, - TimeoutOverride = _options.RequestTimeout, - }; - - var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); - if (result.IsNotModified) - { - _diagnostics.DetailUnchanged(category); - _logger.LogDebug("KISA detail {Idx} unchanged ({Category})", item.AdvisoryId, category ?? "unknown"); - knownIds.Add(item.AdvisoryId); - continue; - } - - if (!result.IsSuccess || result.Document is null) - { - _diagnostics.DetailFailure(category, "empty-document"); - _logger.LogWarning("KISA detail fetch returned no document for {Idx}", item.AdvisoryId); - continue; - } - - pendingDocuments.Add(result.Document.Id); - pendingMappings.Remove(result.Document.Id); - knownIds.Add(item.AdvisoryId); - processed++; - _diagnostics.DetailSuccess(category); - _logger.LogInformation( - "KISA fetched detail for {Idx} (documentId={DocumentId}, category={Category})", - item.AdvisoryId, - result.Document.Id, - category ?? "unknown"); - - if (_options.RequestDelay > TimeSpan.Zero) - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - } - catch (Exception ex) - { - _diagnostics.DetailFailure(category, ex.GetType().Name); - _logger.LogError(ex, "KISA detail fetch failed for {Idx}", item.AdvisoryId); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (item.Published > latestPublished) - { - latestPublished = item.Published; - _diagnostics.CursorAdvanced(); - _logger.LogDebug("KISA advanced published cursor to {Published:O}", latestPublished); - } - } - - var trimmedKnown = knownIds.Count > _options.MaxKnownAdvisories - ? knownIds.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase) - .Take(_options.MaxKnownAdvisories) - .ToArray() - : knownIds.ToArray(); - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownIds(trimmedKnown) - .WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished) - .WithLastFetch(now); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - _logger.LogInformation("KISA fetch stored {Processed} new documents (knownIds={KnownCount})", processed, trimmedKnown.Length); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var now = _timeProvider.GetUtcNow(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - _diagnostics.ParseFailure(null, "document-missing"); - _logger.LogWarning("KISA document {DocumentId} missing during parse", documentId); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var category = GetCategory(document); - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(category, "missing-gridfs"); - _logger.LogWarning("KISA document {DocumentId} missing GridFS payload", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - _diagnostics.ParseAttempt(category); - - byte[] payload; - try - { - payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(category, "download"); - _logger.LogError(ex, "KISA unable to download document {DocumentId}", document.Id); - throw; - } - - KisaParsedAdvisory parsed; - try - { - var apiUri = new Uri(document.Uri); - var pageUri = document.Metadata is not null && document.Metadata.TryGetValue("kisa.detailPage", out var pageValue) - ? new Uri(pageValue) - : apiUri; - parsed = _detailParser.Parse(apiUri, pageUri, payload); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(category, "parse"); - _logger.LogError(ex, "KISA failed to parse detail {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - _diagnostics.ParseSuccess(category); - _logger.LogDebug("KISA parsed detail for {DocumentId} ({Category})", document.Id, category ?? "unknown"); - - var dtoBson = BsonDocument.Parse(JsonSerializer.Serialize(parsed, SerializerOptions)); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "kisa.detail.v1", dtoBson, now); - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - pendingMappings.Add(document.Id); - } - - var updatedCursor = cursor - .WithPendingDocuments(remainingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - _diagnostics.MapFailure(null, "document-missing"); - _logger.LogWarning("KISA document {DocumentId} missing during map", documentId); - pendingMappings.Remove(documentId); - continue; - } - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - _diagnostics.MapFailure(null, "dto-missing"); - _logger.LogWarning("KISA DTO missing for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - KisaParsedAdvisory? parsed; - try - { - parsed = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions); - } - catch (Exception ex) - { - _diagnostics.MapFailure(null, "dto-deserialize"); - _logger.LogError(ex, "KISA failed to deserialize DTO for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - if (parsed is null) - { - _diagnostics.MapFailure(null, "dto-null"); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - try - { - var advisory = KisaMapper.Map(parsed, document, dtoRecord.ValidatedAt); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - _diagnostics.MapSuccess(parsed.Severity); - _logger.LogInformation("KISA mapped advisory {AdvisoryId} (severity={Severity})", parsed.AdvisoryId, parsed.Severity ?? "unknown"); - } - catch (Exception ex) - { - _diagnostics.MapFailure(parsed.Severity, "map"); - _logger.LogError(ex, "KISA mapping failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private static string? GetCategory(DocumentRecord document) - { - if (document.Metadata is null) - { - return null; - } - - return document.Metadata.TryGetValue("kisa.category", out var category) - ? category - : null; - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? KisaCursor.Empty : KisaCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(KisaCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBsonDocument(); - var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow(); - return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Kisa.Configuration; +using StellaOps.Concelier.Connector.Kisa.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Kisa; + +public sealed class KisaConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + }; + + private readonly KisaFeedClient _feedClient; + private readonly KisaDetailParser _detailParser; + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly KisaOptions _options; + private readonly KisaDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public KisaConnector( + KisaFeedClient feedClient, + KisaDetailParser detailParser, + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + KisaDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient)); + _detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser)); + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => KisaConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + _diagnostics.FeedAttempt(); + IReadOnlyList items; + + try + { + items = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false); + _diagnostics.FeedSuccess(items.Count); + + if (items.Count > 0) + { + _logger.LogInformation("KISA feed returned {ItemCount} advisories", items.Count); + } + else + { + _logger.LogDebug("KISA feed returned no advisories"); + } + } + catch (Exception ex) + { + _diagnostics.FeedFailure(ex.GetType().Name); + _logger.LogError(ex, "KISA feed fetch failed"); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (items.Count == 0) + { + await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false); + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var knownIds = new HashSet(cursor.KnownIds, StringComparer.OrdinalIgnoreCase); + var processed = 0; + var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue; + + foreach (var item in items.OrderByDescending(static i => i.Published)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (knownIds.Contains(item.AdvisoryId)) + { + continue; + } + + if (processed >= _options.MaxAdvisoriesPerFetch) + { + break; + } + + var category = item.Category; + _diagnostics.DetailAttempt(category); + + try + { + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailApiUri.ToString(), cancellationToken).ConfigureAwait(false); + var request = new SourceFetchRequest(KisaOptions.HttpClientName, SourceName, item.DetailApiUri) + { + Metadata = KisaDocumentMetadata.CreateMetadata(item), + AcceptHeaders = new[] { "application/json", "text/json" }, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + TimeoutOverride = _options.RequestTimeout, + }; + + var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false); + if (result.IsNotModified) + { + _diagnostics.DetailUnchanged(category); + _logger.LogDebug("KISA detail {Idx} unchanged ({Category})", item.AdvisoryId, category ?? "unknown"); + knownIds.Add(item.AdvisoryId); + continue; + } + + if (!result.IsSuccess || result.Document is null) + { + _diagnostics.DetailFailure(category, "empty-document"); + _logger.LogWarning("KISA detail fetch returned no document for {Idx}", item.AdvisoryId); + continue; + } + + pendingDocuments.Add(result.Document.Id); + pendingMappings.Remove(result.Document.Id); + knownIds.Add(item.AdvisoryId); + processed++; + _diagnostics.DetailSuccess(category); + _logger.LogInformation( + "KISA fetched detail for {Idx} (documentId={DocumentId}, category={Category})", + item.AdvisoryId, + result.Document.Id, + category ?? "unknown"); + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + catch (Exception ex) + { + _diagnostics.DetailFailure(category, ex.GetType().Name); + _logger.LogError(ex, "KISA detail fetch failed for {Idx}", item.AdvisoryId); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (item.Published > latestPublished) + { + latestPublished = item.Published; + _diagnostics.CursorAdvanced(); + _logger.LogDebug("KISA advanced published cursor to {Published:O}", latestPublished); + } + } + + var trimmedKnown = knownIds.Count > _options.MaxKnownAdvisories + ? knownIds.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxKnownAdvisories) + .ToArray() + : knownIds.ToArray(); + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownIds(trimmedKnown) + .WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished) + .WithLastFetch(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("KISA fetch stored {Processed} new documents (knownIds={KnownCount})", processed, trimmedKnown.Length); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var now = _timeProvider.GetUtcNow(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.ParseFailure(null, "document-missing"); + _logger.LogWarning("KISA document {DocumentId} missing during parse", documentId); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var category = GetCategory(document); + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(category, "missing-gridfs"); + _logger.LogWarning("KISA document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + _diagnostics.ParseAttempt(category); + + byte[] payload; + try + { + payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(category, "download"); + _logger.LogError(ex, "KISA unable to download document {DocumentId}", document.Id); + throw; + } + + KisaParsedAdvisory parsed; + try + { + var apiUri = new Uri(document.Uri); + var pageUri = document.Metadata is not null && document.Metadata.TryGetValue("kisa.detailPage", out var pageValue) + ? new Uri(pageValue) + : apiUri; + parsed = _detailParser.Parse(apiUri, pageUri, payload); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(category, "parse"); + _logger.LogError(ex, "KISA failed to parse detail {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + _diagnostics.ParseSuccess(category); + _logger.LogDebug("KISA parsed detail for {DocumentId} ({Category})", document.Id, category ?? "unknown"); + + var dtoBson = BsonDocument.Parse(JsonSerializer.Serialize(parsed, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "kisa.detail.v1", dtoBson, now); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.MapFailure(null, "document-missing"); + _logger.LogWarning("KISA document {DocumentId} missing during map", documentId); + pendingMappings.Remove(documentId); + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + _diagnostics.MapFailure(null, "dto-missing"); + _logger.LogWarning("KISA DTO missing for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + KisaParsedAdvisory? parsed; + try + { + parsed = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions); + } + catch (Exception ex) + { + _diagnostics.MapFailure(null, "dto-deserialize"); + _logger.LogError(ex, "KISA failed to deserialize DTO for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + if (parsed is null) + { + _diagnostics.MapFailure(null, "dto-null"); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + try + { + var advisory = KisaMapper.Map(parsed, document, dtoRecord.ValidatedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + _diagnostics.MapSuccess(parsed.Severity); + _logger.LogInformation("KISA mapped advisory {AdvisoryId} (severity={Severity})", parsed.AdvisoryId, parsed.Severity ?? "unknown"); + } + catch (Exception ex) + { + _diagnostics.MapFailure(parsed.Severity, "map"); + _logger.LogError(ex, "KISA mapping failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private static string? GetCategory(DocumentRecord document) + { + if (document.Metadata is null) + { + return null; + } + + return document.Metadata.TryGetValue("kisa.category", out var category) + ? category + : null; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? KisaCursor.Empty : KisaCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(KisaCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow(); + return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/KisaConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Kisa/KisaConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs index cd7f8882..c6398c47 100644 --- a/src/StellaOps.Feedser.Source.Kisa/KisaConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/KisaConnectorPlugin.cs @@ -1,21 +1,21 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Kisa; - -public sealed class KisaConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "kisa"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) - => services.GetService() is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetRequiredService(); - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Kisa; + +public sealed class KisaConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "kisa"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + => services.GetService() is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/KisaDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Kisa/KisaDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs index ad805e69..d63af8e0 100644 --- a/src/StellaOps.Feedser.Source.Kisa/KisaDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/KisaDependencyInjectionRoutine.cs @@ -1,50 +1,50 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Kisa.Configuration; - -namespace StellaOps.Feedser.Source.Kisa; - -public sealed class KisaDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:kisa"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddKisaConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, KisaJobKinds.Fetch, typeof(KisaFetchJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Kisa.Configuration; + +namespace StellaOps.Concelier.Connector.Kisa; + +public sealed class KisaDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:kisa"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddKisaConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, KisaJobKinds.Fetch, typeof(KisaFetchJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Kisa/KisaServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Kisa/KisaServiceCollectionExtensions.cs similarity index 81% rename from src/StellaOps.Feedser.Source.Kisa/KisaServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Kisa/KisaServiceCollectionExtensions.cs index 06fc6106..0cdb4bd8 100644 --- a/src/StellaOps.Feedser.Source.Kisa/KisaServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Kisa/KisaServiceCollectionExtensions.cs @@ -1,47 +1,47 @@ -using System; -using System.Net; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Html; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Kisa.Configuration; -using StellaOps.Feedser.Source.Kisa.Internal; - -namespace StellaOps.Feedser.Source.Kisa; - -public static class KisaServiceCollectionExtensions -{ - public static IServiceCollection AddKisaConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static options => options.Validate()); - - services.AddSourceHttpClient(KisaOptions.HttpClientName, static (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.Timeout = options.RequestTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.Kisa/1.0"; - clientOptions.DefaultRequestHeaders["Accept-Language"] = "ko-KR"; - clientOptions.AllowedHosts.Clear(); - clientOptions.AllowedHosts.Add(options.FeedUri.Host); - clientOptions.AllowedHosts.Add(options.DetailApiUri.Host); - clientOptions.ConfigureHandler = handler => - { - handler.AutomaticDecompression = DecompressionMethods.All; - handler.AllowAutoRedirect = true; - }; - }); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddSingleton(); - services.AddSingleton(); - services.AddTransient(); - return services; - } -} +using System; +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Html; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Kisa.Configuration; +using StellaOps.Concelier.Connector.Kisa.Internal; + +namespace StellaOps.Concelier.Connector.Kisa; + +public static class KisaServiceCollectionExtensions +{ + public static IServiceCollection AddKisaConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static options => options.Validate()); + + services.AddSourceHttpClient(KisaOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = options.RequestTimeout; + clientOptions.UserAgent = "StellaOps.Concelier.Kisa/1.0"; + clientOptions.DefaultRequestHeaders["Accept-Language"] = "ko-KR"; + clientOptions.AllowedHosts.Clear(); + clientOptions.AllowedHosts.Add(options.FeedUri.Host); + clientOptions.AllowedHosts.Add(options.DetailApiUri.Host); + clientOptions.ConfigureHandler = handler => + { + handler.AutomaticDecompression = DecompressionMethods.All; + handler.AllowAutoRedirect = true; + }; + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Concelier.Connector.Kisa/StellaOps.Concelier.Connector.Kisa.csproj b/src/StellaOps.Concelier.Connector.Kisa/StellaOps.Concelier.Connector.Kisa.csproj new file mode 100644 index 00000000..48e91447 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Kisa/StellaOps.Concelier.Connector.Kisa.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Kisa/TASKS.md b/src/StellaOps.Concelier.Connector.Kisa/TASKS.md similarity index 73% rename from src/StellaOps.Feedser.Source.Kisa/TASKS.md rename to src/StellaOps.Concelier.Connector.Kisa/TASKS.md index bb67ce07..8abdbdf1 100644 --- a/src/StellaOps.Feedser.Source.Kisa/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Kisa/TASKS.md @@ -1,10 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-KISA-02-001 Research KISA advisory feeds|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Located public RSS endpoints (`https://knvd.krcert.or.kr/rss/securityInfo.do`, `.../securityNotice.do`) returning UTF-8 XML with 10-item windows and canonical `detailDos.do?IDX=` links. Logged output structure + header profile in `docs/feedser-connector-research-20251011.md`; outstanding work is parsing the SPA detail payload.| -|FEEDCONN-KISA-02-002 Fetch pipeline & source state|BE-Conn-KISA|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `KisaConnector.FetchAsync` pulls RSS, sets `Accept-Language: ko-KR`, persists detail JSON with IDX metadata, throttles requests, and tracks cursor state (pending docs/mappings, known IDs, published timestamp).| -|FEEDCONN-KISA-02-003 Parser & DTO implementation|BE-Conn-KISA|Source.Common|**DONE (2025-10-14)** – Detail API parsed via `KisaDetailParser` (Hangul NFC normalisation, sanitised HTML, CVE extraction, references/products captured into DTO `kisa.detail.v1`).| -|FEEDCONN-KISA-02-004 Canonical mapping & range primitives|BE-Conn-KISA|Models|**DONE (2025-10-14)** – `KisaMapper` emits vendor packages with range strings, aliases (IDX/CVEs), references, and provenance; advisories default to `ko` language and normalised severity.| -|FEEDCONN-KISA-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Feedser.Source.Kisa.Tests` with Korean fixtures and fetch→parse→map regression; fixtures regenerate via `UPDATE_KISA_FIXTURES=1`.| -|FEEDCONN-KISA-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Added diagnostics-backed telemetry, structured logs, regression coverage, and published localisation notes in `docs/dev/kisa_connector_notes.md` + fixture guidance for Docs/QA.| -|FEEDCONN-KISA-02-007 RSS contract & localisation brief|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Documented RSS URLs, confirmed UTF-8 payload (no additional cookies required), and drafted localisation plan (Hangul glossary + optional MT plugin). Remaining open item: capture SPA detail API contract for full-text translations.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-KISA-02-001 Research KISA advisory feeds|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Located public RSS endpoints (`https://knvd.krcert.or.kr/rss/securityInfo.do`, `.../securityNotice.do`) returning UTF-8 XML with 10-item windows and canonical `detailDos.do?IDX=` links. Logged output structure + header profile in `docs/concelier-connector-research-20251011.md`; outstanding work is parsing the SPA detail payload.| +|FEEDCONN-KISA-02-002 Fetch pipeline & source state|BE-Conn-KISA|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `KisaConnector.FetchAsync` pulls RSS, sets `Accept-Language: ko-KR`, persists detail JSON with IDX metadata, throttles requests, and tracks cursor state (pending docs/mappings, known IDs, published timestamp).| +|FEEDCONN-KISA-02-003 Parser & DTO implementation|BE-Conn-KISA|Source.Common|**DONE (2025-10-14)** – Detail API parsed via `KisaDetailParser` (Hangul NFC normalisation, sanitised HTML, CVE extraction, references/products captured into DTO `kisa.detail.v1`).| +|FEEDCONN-KISA-02-004 Canonical mapping & range primitives|BE-Conn-KISA|Models|**DONE (2025-10-14)** – `KisaMapper` emits vendor packages with range strings, aliases (IDX/CVEs), references, and provenance; advisories default to `ko` language and normalised severity.| +|FEEDCONN-KISA-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added `StellaOps.Concelier.Connector.Kisa.Tests` with Korean fixtures and fetch→parse→map regression; fixtures regenerate via `UPDATE_KISA_FIXTURES=1`.| +|FEEDCONN-KISA-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Added diagnostics-backed telemetry, structured logs, regression coverage, and published localisation notes in `docs/dev/kisa_connector_notes.md` + fixture guidance for Docs/QA.| +|FEEDCONN-KISA-02-007 RSS contract & localisation brief|BE-Conn-KISA|Research|**DONE (2025-10-11)** – Documented RSS URLs, confirmed UTF-8 payload (no additional cookies required), and drafted localisation plan (Hangul glossary + optional MT plugin). Remaining open item: capture SPA detail API contract for full-text translations.| +|FEEDCONN-KISA-02-008 Firmware scheme proposal|BE-Conn-KISA, Models|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-24)** – Define transformation for Hangul-labelled firmware ranges (`XFU 1.0.1.0084 ~ 2.0.1.0034`), propose `kisa.build` (or equivalent) scheme to Models, implement normalized rule emission/tests once scheme approved, and update localisation notes.| diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json index fd60dfb7..d89b3546 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/conflict-nvd.canonical.json @@ -1,182 +1,182 @@ -{ - "advisoryKey": "CVE-2025-4242", - "affectedPackages": [ - { - "type": "cpe", - "identifier": "cpe:2.3:a:conflict:package:1.0:*:*:*:*:*:*:*", - "platform": null, - "versionRanges": [ - { - "fixedVersion": "1.4", - "introducedVersion": "1.0", - "lastAffectedVersion": "1.0", - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": ">=1.0 <1.4 ==1.0", - "exactValue": "1.0.0", - "fixed": "1.4.0", - "fixedInclusive": false, - "introduced": "1.0.0", - "introducedInclusive": true, - "lastAffected": "1.0.0", - "lastAffectedInclusive": true, - "style": "exact" - }, - "vendorExtensions": { - "versionStartIncluding": "1.0", - "versionEndExcluding": "1.4", - "version": "1.0" - } - }, - "provenance": { - "source": "nvd", - "kind": "cpe", - "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": ">=1.0 <1.4 ==1.0", - "rangeKind": "cpe" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "exact", - "min": null, - "minInclusive": null, - "max": null, - "maxInclusive": null, - "value": "1.0.0", - "notes": "nvd:CVE-2025-4242" - } - ], - "statuses": [], - "provenance": [ - { - "source": "nvd", - "kind": "cpe", - "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2025-4242" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "credits": [], - "cvssMetrics": [ - { - "baseScore": 9.8, - "baseSeverity": "critical", - "provenance": { - "source": "nvd", - "kind": "cvss", - "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "cvssmetrics[]" - ] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-269", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/269.html", - "provenance": [ - { - "source": "nvd", - "kind": "weakness", - "value": "CWE-269", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - } - ], - "description": "NVD baseline summary for conflict-package allowing container escape.", - "exploitKnown": false, - "language": "en", - "modified": "2025-03-03T09:45:00+00:00", - "provenance": [ - { - "source": "nvd", - "kind": "document", - "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-03T10:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "nvd", - "kind": "mapping", - "value": "CVE-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-03-01T10:15:00+00:00", - "references": [ - { - "kind": "weakness", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://cwe.mitre.org/data/definitions/269.html", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "CWE-269", - "summary": null, - "url": "https://cwe.mitre.org/data/definitions/269.html" - }, - { - "kind": "vendor advisory", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-04T02:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "NVD", - "summary": null, - "url": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242" - } - ], - "severity": "critical", - "summary": "NVD baseline summary for conflict-package allowing container escape.", - "title": "CVE-2025-4242" +{ + "advisoryKey": "CVE-2025-4242", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:2.3:a:conflict:package:1.0:*:*:*:*:*:*:*", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.4", + "introducedVersion": "1.0", + "lastAffectedVersion": "1.0", + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": ">=1.0 <1.4 ==1.0", + "exactValue": "1.0.0", + "fixed": "1.4.0", + "fixedInclusive": false, + "introduced": "1.0.0", + "introducedInclusive": true, + "lastAffected": "1.0.0", + "lastAffectedInclusive": true, + "style": "exact" + }, + "vendorExtensions": { + "versionStartIncluding": "1.0", + "versionEndExcluding": "1.4", + "version": "1.0" + } + }, + "provenance": { + "source": "nvd", + "kind": "cpe", + "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": ">=1.0 <1.4 ==1.0", + "rangeKind": "cpe" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "exact", + "min": null, + "minInclusive": null, + "max": null, + "maxInclusive": null, + "value": "1.0.0", + "notes": "nvd:CVE-2025-4242" + } + ], + "statuses": [], + "provenance": [ + { + "source": "nvd", + "kind": "cpe", + "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-4242" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "nvd", + "kind": "cvss", + "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "cvssmetrics[]" + ] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-269", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/269.html", + "provenance": [ + { + "source": "nvd", + "kind": "weakness", + "value": "CWE-269", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "NVD baseline summary for conflict-package allowing container escape.", + "exploitKnown": false, + "language": "en", + "modified": "2025-03-03T09:45:00+00:00", + "provenance": [ + { + "source": "nvd", + "kind": "document", + "value": "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-03T10:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "nvd", + "kind": "mapping", + "value": "CVE-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-03-01T10:15:00+00:00", + "references": [ + { + "kind": "weakness", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://cwe.mitre.org/data/definitions/269.html", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "CWE-269", + "summary": null, + "url": "https://cwe.mitre.org/data/definitions/269.html" + }, + { + "kind": "vendor advisory", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-04T02:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "NVD", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242" + } + ], + "severity": "critical", + "summary": "NVD baseline summary for conflict-package allowing container escape.", + "title": "CVE-2025-4242" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json index 5cd5ddf8..fc6096fc 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.ghsa.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "GHSA-credit-parity", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "ghsa:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "ghsa", - "kind": "credit", - "value": "ghsa:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "ghsa", - "kind": "document", - "value": "security/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "ghsa", - "kind": "mapping", - "value": "GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "patch", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://example.com/ghsa/patch", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://example.com/ghsa/patch" - }, - { - "kind": "advisory", - "provenance": { - "source": "ghsa", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-credit-parity" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "GHSA-credit-parity", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "ghsa:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "ghsa", + "kind": "credit", + "value": "ghsa:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "document", + "value": "security/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "ghsa", + "kind": "mapping", + "value": "GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://example.com/ghsa/patch", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://example.com/ghsa/patch" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-credit-parity" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.nvd.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json similarity index 96% rename from src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.nvd.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json index 4c75a6cc..da751c42 100644 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/Fixtures/credit-parity.nvd.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.nvd.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "CVE-2025-5555", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "nvd", - "kind": "credit", - "value": "nvd:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "nvd", - "kind": "credit", - "value": "nvd:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "nvd", - "kind": "document", - "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "nvd", - "kind": "mapping", - "value": "CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "report", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://example.com/nvd/reference", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://example.com/nvd/reference" - }, - { - "kind": "advisory", - "provenance": { - "source": "nvd", - "kind": "reference", - "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "CVE-2025-5555", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "nvd", + "kind": "credit", + "value": "nvd:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "nvd", + "kind": "credit", + "value": "nvd:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "nvd", + "kind": "document", + "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "nvd", + "kind": "mapping", + "value": "CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "report", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://example.com/nvd/reference", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://example.com/nvd/reference" + }, + { + "kind": "advisory", + "provenance": { + "source": "nvd", + "kind": "reference", + "value": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json index 878c01b6..f7d1d602 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/credit-parity.osv.json @@ -1,108 +1,108 @@ -{ - "advisoryKey": "GHSA-credit-parity", - "affectedPackages": [], - "aliases": [ - "CVE-2025-5555", - "GHSA-credit-parity" - ], - "credits": [ - { - "displayName": "Bob Maintainer", - "role": "remediation_developer", - "contacts": [ - "https://github.com/acme/bob-maintainer" - ], - "provenance": { - "source": "osv", - "kind": "credit", - "value": "osv:bob-maintainer", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - }, - { - "displayName": "Alice Researcher", - "role": "reporter", - "contacts": [ - "mailto:alice.researcher@example.com" - ], - "provenance": { - "source": "osv", - "kind": "credit", - "value": "osv:alice-researcher", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2025-10-10T12:00:00+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T18:30:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://github.com/advisories/GHSA-credit-parity" - }, - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://osv.dev/vulnerability/GHSA-credit-parity", - "decisionReason": null, - "recordedAt": "2025-10-10T15:00:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": null, - "summary": null, - "url": "https://osv.dev/vulnerability/GHSA-credit-parity" - } - ], - "severity": "medium", - "summary": "Credit parity regression fixture", - "title": "Credit parity regression fixture" +{ + "advisoryKey": "GHSA-credit-parity", + "affectedPackages": [], + "aliases": [ + "CVE-2025-5555", + "GHSA-credit-parity" + ], + "credits": [ + { + "displayName": "Bob Maintainer", + "role": "remediation_developer", + "contacts": [ + "https://github.com/acme/bob-maintainer" + ], + "provenance": { + "source": "osv", + "kind": "credit", + "value": "osv:bob-maintainer", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + }, + { + "displayName": "Alice Researcher", + "role": "reporter", + "contacts": [ + "mailto:alice.researcher@example.com" + ], + "provenance": { + "source": "osv", + "kind": "credit", + "value": "osv:alice-researcher", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2025-10-10T12:00:00+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T18:30:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://github.com/advisories/GHSA-credit-parity" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://osv.dev/vulnerability/GHSA-credit-parity", + "decisionReason": null, + "recordedAt": "2025-10-10T15:00:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": null, + "summary": null, + "url": "https://osv.dev/vulnerability/GHSA-credit-parity" + } + ], + "severity": "medium", + "summary": "Credit parity regression fixture", + "title": "Credit parity regression fixture" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json similarity index 95% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json index cefb4b2f..02b611d5 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-invalid-schema.json @@ -1,6 +1,6 @@ -{ - "resultsPerPage": 1, - "startIndex": 0, - "totalResults": 1, - "vulnerabilities": "this-should-be-an-array" -} +{ + "resultsPerPage": 1, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": "this-should-be-an-array" +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json index ed90665d..e6a45070 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-1.json @@ -1,69 +1,69 @@ -{ - "resultsPerPage": 2, - "startIndex": 0, - "totalResults": 5, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-1000", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-02-01T10:00:00Z", - "lastModified": "2024-02-02T10:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Multipage vulnerability one." } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "baseScore": 9.8, - "baseSeverity": "CRITICAL" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_a:1.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - }, - { - "cve": { - "id": "CVE-2024-1001", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-02-01T11:00:00Z", - "lastModified": "2024-02-02T11:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Multipage vulnerability two." } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:P/AC:L/PR:L/UI:R/S:U/C:L/I:L/A:L", - "baseScore": 5.1, - "baseSeverity": "MEDIUM" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_b:2.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2, + "startIndex": 0, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1000", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T10:00:00Z", + "lastModified": "2024-02-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability one." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_a:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T11:00:00Z", + "lastModified": "2024-02-02T11:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability two." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:P/AC:L/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 5.1, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_b:2.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json index 530ecdf3..0270b45d 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-2.json @@ -1,69 +1,69 @@ -{ - "resultsPerPage": 2, - "startIndex": 2, - "totalResults": 5, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-1002", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-02-01T12:00:00Z", - "lastModified": "2024-02-02T12:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Multipage vulnerability three." } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N", - "baseScore": 3.1, - "baseSeverity": "LOW" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_c:3.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - }, - { - "cve": { - "id": "CVE-2024-1003", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-02-01T13:00:00Z", - "lastModified": "2024-02-02T13:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Multipage vulnerability four." } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:A/AC:L/PR:N/UI:N/S:U/C:M/I:L/A:L", - "baseScore": 7.4, - "baseSeverity": "HIGH" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_d:4.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2, + "startIndex": 2, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T12:00:00Z", + "lastModified": "2024-02-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability three." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:L/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N", + "baseScore": 3.1, + "baseSeverity": "LOW" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_c:3.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-1003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T13:00:00Z", + "lastModified": "2024-02-02T13:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability four." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:A/AC:L/PR:N/UI:N/S:U/C:M/I:L/A:L", + "baseScore": 7.4, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_d:4.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json index 42cf57dc..9b0df922 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-multipage-3.json @@ -1,38 +1,38 @@ -{ - "resultsPerPage": 2, - "startIndex": 4, - "totalResults": 5, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-1004", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-02-01T14:00:00Z", - "lastModified": "2024-02-02T14:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Multipage vulnerability five." } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:L/I:H/A:L", - "baseScore": 7.9, - "baseSeverity": "HIGH" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_e:5.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2, + "startIndex": 4, + "totalResults": 5, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-1004", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-02-01T14:00:00Z", + "lastModified": "2024-02-02T14:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Multipage vulnerability five." } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:L/I:H/A:L", + "baseScore": 7.9, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_e:5.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json index 0890cb62..c4fbf44c 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-1.json @@ -1,101 +1,101 @@ -{ - "resultsPerPage": 2000, - "startIndex": 0, - "totalResults": 2, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-0001", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-01-01T10:00:00Z", - "lastModified": "2024-01-02T10:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Example vulnerability one." } - ], - "references": [ - { - "url": "https://vendor.example.com/advisories/0001", - "source": "Vendor", - "tags": ["Vendor Advisory"] - } - ], - "weaknesses": [ - { - "description": [ - { "lang": "en", "value": "CWE-79" }, - { "lang": "en", "value": "Improper Neutralization of Input" } - ] - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "baseScore": 9.8, - "baseSeverity": "CRITICAL" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" } - ] - } - ] - } - } - }, - { - "cve": { - "id": "CVE-2024-0002", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-01-01T11:00:00Z", - "lastModified": "2024-01-02T11:00:00Z", - "descriptions": [ - { "lang": "fr", "value": "Description française" }, - { "lang": "en", "value": "Example vulnerability two." } - ], - "references": [ - { - "url": "https://cisa.example.gov/alerts/0002", - "source": "CISA", - "tags": ["US Government Resource"] - } - ], - "weaknesses": [ - { - "description": [ - { "lang": "en", "value": "CWE-89" }, - { "lang": "en", "value": "SQL Injection" } - ] - } - ], - "metrics": { - "cvssMetricV30": [ - { - "cvssData": { - "vectorString": "CVSS:3.0/AV:L/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", - "baseScore": 4.6, - "baseSeverity": "MEDIUM" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_two:2.0:*:*:*:*:*:*:*" }, - { "vulnerable": false, "criteria": "cpe:2.3:a:example:product_two:2.1:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 2, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-02T10:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + } + ], + "weaknesses": [ + { + "description": [ + { "lang": "en", "value": "CWE-79" }, + { "lang": "en", "value": "Improper Neutralization of Input" } + ] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" } + ] + } + ] + } + } + }, + { + "cve": { + "id": "CVE-2024-0002", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T11:00:00Z", + "lastModified": "2024-01-02T11:00:00Z", + "descriptions": [ + { "lang": "fr", "value": "Description française" }, + { "lang": "en", "value": "Example vulnerability two." } + ], + "references": [ + { + "url": "https://cisa.example.gov/alerts/0002", + "source": "CISA", + "tags": ["US Government Resource"] + } + ], + "weaknesses": [ + { + "description": [ + { "lang": "en", "value": "CWE-89" }, + { "lang": "en", "value": "SQL Injection" } + ] + } + ], + "metrics": { + "cvssMetricV30": [ + { + "cvssData": { + "vectorString": "CVSS:3.0/AV:L/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", + "baseScore": 4.6, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_two:2.0:*:*:*:*:*:*:*" }, + { "vulnerable": false, "criteria": "cpe:2.3:a:example:product_two:2.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json index bf68d9b9..6220fe5a 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-2.json @@ -1,45 +1,45 @@ -{ - "resultsPerPage": 2000, - "startIndex": 0, - "totalResults": 1, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-0003", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-01-01T12:00:00Z", - "lastModified": "2024-01-02T12:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Example vulnerability three." } - ], - "references": [ - { - "url": "https://example.org/patches/0003", - "source": "Vendor", - "tags": ["Patch"] - } - ], - "metrics": { - "cvssMetricV2": [ - { - "cvssData": { - "vectorString": "AV:N/AC:M/Au:N/C:P/I:P/A:P", - "baseScore": 6.8, - "baseSeverity": "MEDIUM" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_three:3.5:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0003", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T12:00:00Z", + "lastModified": "2024-01-02T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability three." } + ], + "references": [ + { + "url": "https://example.org/patches/0003", + "source": "Vendor", + "tags": ["Patch"] + } + ], + "metrics": { + "cvssMetricV2": [ + { + "cvssData": { + "vectorString": "AV:N/AC:M/Au:N/C:P/I:P/A:P", + "baseScore": 6.8, + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_three:3.5:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json similarity index 96% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json index f7be7b3a..65ad963f 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/Fixtures/nvd-window-update.json @@ -1,51 +1,51 @@ -{ - "resultsPerPage": 2000, - "startIndex": 0, - "totalResults": 1, - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2024-0001", - "sourceIdentifier": "nvd@nist.gov", - "published": "2024-01-01T10:00:00Z", - "lastModified": "2024-01-03T12:00:00Z", - "descriptions": [ - { "lang": "en", "value": "Example vulnerability one updated." } - ], - "references": [ - { - "url": "https://vendor.example.com/advisories/0001", - "source": "Vendor", - "tags": ["Vendor Advisory"] - }, - { - "url": "https://kb.example.com/articles/0001", - "source": "KnowledgeBase", - "tags": ["Third Party Advisory"] - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", - "baseScore": 8.8, - "baseSeverity": "HIGH" - } - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" }, - { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.1:*:*:*:*:*:*:*" } - ] - } - ] - } - } - } - ] -} +{ + "resultsPerPage": 2000, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2024-0001", + "sourceIdentifier": "nvd@nist.gov", + "published": "2024-01-01T10:00:00Z", + "lastModified": "2024-01-03T12:00:00Z", + "descriptions": [ + { "lang": "en", "value": "Example vulnerability one updated." } + ], + "references": [ + { + "url": "https://vendor.example.com/advisories/0001", + "source": "Vendor", + "tags": ["Vendor Advisory"] + }, + { + "url": "https://kb.example.com/articles/0001", + "source": "KnowledgeBase", + "tags": ["Third Party Advisory"] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "baseScore": 8.8, + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.0:*:*:*:*:*:*:*" }, + { "vulnerable": true, "criteria": "cpe:2.3:a:example:product_one:1.1:*:*:*:*:*:*:*" } + ] + } + ] + } + } + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs index 68383899..13728e86 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConflictFixtureTests.cs @@ -1,103 +1,103 @@ -using System.Text.Json; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Nvd.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Nvd.Tests; - -public sealed class NvdConflictFixtureTests -{ - [Fact] - public void ConflictFixture_MatchesSnapshot() - { - const string payload = """ - { - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2025-4242", - "published": "2025-03-01T10:15:00Z", - "lastModified": "2025-03-03T09:45:00Z", - "descriptions": [ - { "lang": "en", "value": "NVD baseline summary for conflict-package allowing container escape." } - ], - "references": [ - { - "url": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242", - "source": "NVD", - "tags": ["Vendor Advisory"] - } - ], - "weaknesses": [ - { - "description": [ - { "lang": "en", "value": "CWE-269" } - ] - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "baseScore": 9.8, - "baseSeverity": "CRITICAL" - }, - "exploitabilityScore": 3.9, - "impactScore": 5.9 - } - ] - }, - "configurations": { - "nodes": [ - { - "cpeMatch": [ - { - "criteria": "cpe:2.3:a:conflict:package:1.0:*:*:*:*:*:*:*", - "vulnerable": true, - "versionStartIncluding": "1.0", - "versionEndExcluding": "1.4" - } - ] - } - ] - } - } - } - ] - } - """; - - using var document = JsonDocument.Parse(payload); - - var sourceDocument = new DocumentRecord( - Id: Guid.Parse("1a6a0700-2dd0-4f69-bb37-64ca77e51c91"), - SourceName: NvdConnectorPlugin.SourceName, - Uri: "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", - FetchedAt: new DateTimeOffset(2025, 3, 3, 10, 0, 0, TimeSpan.Zero), - Sha256: "sha256-nvd-conflict-fixture", - Status: "completed", - ContentType: "application/json", - Headers: null, - Metadata: null, - Etag: "\"etag-nvd-conflict\"", - LastModified: new DateTimeOffset(2025, 3, 3, 9, 45, 0, TimeSpan.Zero), - GridFsId: null); - - var advisories = NvdMapper.Map(document, sourceDocument, new DateTimeOffset(2025, 3, 4, 2, 0, 0, TimeSpan.Zero)); - var advisory = Assert.Single(advisories); - - var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); - var expectedPath = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", "conflict-nvd.canonical.json"); - var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); - - if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", "conflict-nvd.canonical.actual.json"); - Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(expected, snapshot); - } -} +using System.Text.Json; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Nvd.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Nvd.Tests; + +public sealed class NvdConflictFixtureTests +{ + [Fact] + public void ConflictFixture_MatchesSnapshot() + { + const string payload = """ + { + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2025-4242", + "published": "2025-03-01T10:15:00Z", + "lastModified": "2025-03-03T09:45:00Z", + "descriptions": [ + { "lang": "en", "value": "NVD baseline summary for conflict-package allowing container escape." } + ], + "references": [ + { + "url": "https://nvd.nist.gov/vuln/detail/CVE-2025-4242", + "source": "NVD", + "tags": ["Vendor Advisory"] + } + ], + "weaknesses": [ + { + "description": [ + { "lang": "en", "value": "CWE-269" } + ] + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "baseScore": 9.8, + "baseSeverity": "CRITICAL" + }, + "exploitabilityScore": 3.9, + "impactScore": 5.9 + } + ] + }, + "configurations": { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:conflict:package:1.0:*:*:*:*:*:*:*", + "vulnerable": true, + "versionStartIncluding": "1.0", + "versionEndExcluding": "1.4" + } + ] + } + ] + } + } + } + ] + } + """; + + using var document = JsonDocument.Parse(payload); + + var sourceDocument = new DocumentRecord( + Id: Guid.Parse("1a6a0700-2dd0-4f69-bb37-64ca77e51c91"), + SourceName: NvdConnectorPlugin.SourceName, + Uri: "https://services.nvd.nist.gov/rest/json/cve/2.0?cveId=CVE-2025-4242", + FetchedAt: new DateTimeOffset(2025, 3, 3, 10, 0, 0, TimeSpan.Zero), + Sha256: "sha256-nvd-conflict-fixture", + Status: "completed", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: "\"etag-nvd-conflict\"", + LastModified: new DateTimeOffset(2025, 3, 3, 9, 45, 0, TimeSpan.Zero), + GridFsId: null); + + var advisories = NvdMapper.Map(document, sourceDocument, new DateTimeOffset(2025, 3, 4, 2, 0, 0, TimeSpan.Zero)); + var advisory = Assert.Single(advisories); + + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); + var expectedPath = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", "conflict-nvd.canonical.json"); + var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", "conflict-nvd.canonical.actual.json"); + Directory.CreateDirectory(Path.GetDirectoryName(actualPath)!); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + } +} diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs index 66ded4b4..03cd4754 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorHarnessTests.cs @@ -5,17 +5,17 @@ using System.IO; using System.Linq; using Microsoft.Extensions.DependencyInjection; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Nvd; -using StellaOps.Feedser.Source.Nvd.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Nvd; +using StellaOps.Concelier.Connector.Nvd.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Testing; using System.Net; -namespace StellaOps.Feedser.Source.Nvd.Tests; +namespace StellaOps.Concelier.Connector.Nvd.Tests; [Collection("mongo-fixture")] public sealed class NvdConnectorHarnessTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorTests.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorTests.cs index 14554ef3..c2f1e276 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdConnectorTests.cs @@ -12,22 +12,22 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Nvd; -using StellaOps.Feedser.Source.Nvd.Configuration; -using StellaOps.Feedser.Source.Nvd.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.ChangeHistory; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Nvd; +using StellaOps.Concelier.Connector.Nvd.Configuration; +using StellaOps.Concelier.Connector.Nvd.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.ChangeHistory; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Nvd.Tests; +namespace StellaOps.Concelier.Connector.Nvd.Tests; [Collection("mongo-fixture")] public sealed class NvdConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs rename to src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs index 1e5c9944..bf58ce21 100644 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/Nvd/NvdMergeExportParityTests.cs @@ -1,98 +1,98 @@ -using System; -using System.IO; -using System.Linq; -using System.Text.Json; -using System.Threading.Tasks; -using StellaOps.Feedser.Core; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Source.Nvd.Tests.Nvd; - -public sealed class NvdMergeExportParityTests -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); - - [Fact] - public async Task CanonicalMerge_PreservesCreditsAndReferences_ExporterMaintainsParity() - { - var ghsa = LoadFixture("credit-parity.ghsa.json"); - var osv = LoadFixture("credit-parity.osv.json"); - var nvd = LoadFixture("credit-parity.nvd.json"); - - var merger = new CanonicalMerger(); - var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv); - var merged = result.Advisory; - - Assert.NotNull(merged); - var creditKeys = merged!.Credits - .Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}") - .ToHashSet(StringComparer.Ordinal); - - Assert.Equal(2, creditKeys.Count); - Assert.Contains("reporter|Alice Researcher|mailto:alice.researcher@example.com", creditKeys); - Assert.Contains("remediation_developer|Bob Maintainer|https://github.com/acme/bob-maintainer", creditKeys); - - var referenceUrls = merged.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase); - Assert.Equal(5, referenceUrls.Count); - Assert.Contains($"https://github.com/advisories/GHSA-credit-parity", referenceUrls); - Assert.Contains("https://example.com/ghsa/patch", referenceUrls); - Assert.Contains($"https://osv.dev/vulnerability/GHSA-credit-parity", referenceUrls); - Assert.Contains($"https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", referenceUrls); - Assert.Contains("https://example.com/nvd/reference", referenceUrls); - - using var tempDirectory = new TempDirectory(); - var options = new JsonExportOptions { OutputRoot = tempDirectory.Path }; - var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); - var exportResult = await builder.WriteAsync(new[] { merged }, new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero)); - - Assert.Single(exportResult.Files); - var exportFile = exportResult.Files[0]; - var exportPath = Path.Combine(exportResult.ExportDirectory, exportFile.RelativePath.Replace('/', Path.DirectorySeparatorChar)); - Assert.True(File.Exists(exportPath)); - - var exported = JsonSerializer.Deserialize(await File.ReadAllTextAsync(exportPath), SerializerOptions); - Assert.NotNull(exported); - - var exportedCredits = exported!.Credits - .Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}") - .ToHashSet(StringComparer.Ordinal); - Assert.Equal(creditKeys, exportedCredits); - - var exportedReferences = exported.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase); - Assert.Equal(referenceUrls, exportedReferences); - } - - private static Advisory LoadFixture(string fileName) - { - var path = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", fileName); - return JsonSerializer.Deserialize(File.ReadAllText(path), SerializerOptions) - ?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'."); - } - - private sealed class TempDirectory : IDisposable - { - public TempDirectory() - { - Path = Directory.CreateTempSubdirectory("nvd-merge-export").FullName; - } - - public string Path { get; } - - public void Dispose() - { - try - { - if (Directory.Exists(Path)) - { - Directory.Delete(Path, recursive: true); - } - } - catch - { - // best effort cleanup - } - } - } -} +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading.Tasks; +using StellaOps.Concelier.Core; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Connector.Nvd.Tests.Nvd; + +public sealed class NvdMergeExportParityTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public async Task CanonicalMerge_PreservesCreditsAndReferences_ExporterMaintainsParity() + { + var ghsa = LoadFixture("credit-parity.ghsa.json"); + var osv = LoadFixture("credit-parity.osv.json"); + var nvd = LoadFixture("credit-parity.nvd.json"); + + var merger = new CanonicalMerger(); + var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv); + var merged = result.Advisory; + + Assert.NotNull(merged); + var creditKeys = merged!.Credits + .Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}") + .ToHashSet(StringComparer.Ordinal); + + Assert.Equal(2, creditKeys.Count); + Assert.Contains("reporter|Alice Researcher|mailto:alice.researcher@example.com", creditKeys); + Assert.Contains("remediation_developer|Bob Maintainer|https://github.com/acme/bob-maintainer", creditKeys); + + var referenceUrls = merged.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase); + Assert.Equal(5, referenceUrls.Count); + Assert.Contains($"https://github.com/advisories/GHSA-credit-parity", referenceUrls); + Assert.Contains("https://example.com/ghsa/patch", referenceUrls); + Assert.Contains($"https://osv.dev/vulnerability/GHSA-credit-parity", referenceUrls); + Assert.Contains($"https://services.nvd.nist.gov/vuln/detail/CVE-2025-5555", referenceUrls); + Assert.Contains("https://example.com/nvd/reference", referenceUrls); + + using var tempDirectory = new TempDirectory(); + var options = new JsonExportOptions { OutputRoot = tempDirectory.Path }; + var builder = new JsonExportSnapshotBuilder(options, new VulnListJsonExportPathResolver()); + var exportResult = await builder.WriteAsync(new[] { merged }, new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero)); + + Assert.Single(exportResult.Files); + var exportFile = exportResult.Files[0]; + var exportPath = Path.Combine(exportResult.ExportDirectory, exportFile.RelativePath.Replace('/', Path.DirectorySeparatorChar)); + Assert.True(File.Exists(exportPath)); + + var exported = JsonSerializer.Deserialize(await File.ReadAllTextAsync(exportPath), SerializerOptions); + Assert.NotNull(exported); + + var exportedCredits = exported!.Credits + .Select(static credit => $"{credit.Role}|{credit.DisplayName}|{string.Join("|", credit.Contacts.OrderBy(static c => c, StringComparer.Ordinal))}") + .ToHashSet(StringComparer.Ordinal); + Assert.Equal(creditKeys, exportedCredits); + + var exportedReferences = exported.References.Select(static reference => reference.Url).ToHashSet(StringComparer.OrdinalIgnoreCase); + Assert.Equal(referenceUrls, exportedReferences); + } + + private static Advisory LoadFixture(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, "Nvd", "Fixtures", fileName); + return JsonSerializer.Deserialize(File.ReadAllText(path), SerializerOptions) + ?? throw new InvalidOperationException($"Failed to deserialize fixture '{fileName}'."); + } + + private sealed class TempDirectory : IDisposable + { + public TempDirectory() + { + Path = Directory.CreateTempSubdirectory("nvd-merge-export").FullName; + } + + public string Path { get; } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + } +} diff --git a/src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj b/src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj new file mode 100644 index 00000000..960ed780 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md b/src/StellaOps.Concelier.Connector.Nvd/AGENTS.md similarity index 73% rename from src/StellaOps.Feedser.Source.Nvd/AGENTS.md rename to src/StellaOps.Concelier.Connector.Nvd/AGENTS.md index a22b7a18..16f53c22 100644 --- a/src/StellaOps.Feedser.Source.Nvd/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Nvd/AGENTS.md @@ -13,14 +13,14 @@ Connector for NVD API v2: fetch, validate, map CVE items to canonical advisories - Exporters consume canonical advisories. ## Interfaces & contracts - Job kinds: nvd:fetch, nvd:parse, nvd:map. -- Input params: windowHours, since, until; safe defaults in FeedserOptions. +- Input params: windowHours, since, until; safe defaults in ConcelierOptions. - Output: raw documents, sanitized DTOs, mapped advisories + provenance (document, parser). ## In/Out of scope In: registry-level data, references, generic CPEs. Out: authoritative distro package ranges; vendor patch states. ## Observability & security expectations -- Metrics: SourceDiagnostics publishes `feedser.source.http.*` counters/histograms tagged `feedser.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits. +- Metrics: SourceDiagnostics publishes `concelier.source.http.*` counters/histograms tagged `concelier.source=nvd`; dashboards slice on the tag to track page counts, schema failures, map throughput, and window advancement. Structured logs include window bounds and etag hits. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Nvd.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Nvd.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs b/src/StellaOps.Concelier.Connector.Nvd/Configuration/NvdOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs rename to src/StellaOps.Concelier.Connector.Nvd/Configuration/NvdOptions.cs index 6d26c6df..cb88d3d1 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Configuration/NvdOptions.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Configuration/NvdOptions.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Nvd.Configuration; +namespace StellaOps.Concelier.Connector.Nvd.Configuration; public sealed class NvdOptions { diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdCursor.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs rename to src/StellaOps.Concelier.Connector.Nvd/Internal/NvdCursor.cs index 01cb5e6c..235985e2 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdCursor.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdCursor.cs @@ -1,8 +1,8 @@ using System.Linq; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Cursors; +using StellaOps.Concelier.Connector.Common.Cursors; -namespace StellaOps.Feedser.Source.Nvd.Internal; +namespace StellaOps.Concelier.Connector.Nvd.Internal; internal sealed record NvdCursor( TimeWindowCursorState Window, diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdDiagnostics.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Nvd/Internal/NvdDiagnostics.cs index 1487b3ab..503d9ce4 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdDiagnostics.cs @@ -1,10 +1,10 @@ using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Nvd.Internal; +namespace StellaOps.Concelier.Connector.Nvd.Internal; public sealed class NvdDiagnostics : IDisposable { - public const string MeterName = "StellaOps.Feedser.Source.Nvd"; + public const string MeterName = "StellaOps.Concelier.Connector.Nvd"; public const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdMapper.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs rename to src/StellaOps.Concelier.Connector.Nvd/Internal/NvdMapper.cs index 9f127854..57127ed4 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdMapper.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdMapper.cs @@ -4,13 +4,13 @@ using System.Linq; using System.Text; using System.Text.Json; using NuGet.Versioning; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Identifiers; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.Text; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.Text; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Nvd.Internal; +namespace StellaOps.Concelier.Connector.Nvd.Internal; internal static class NvdMapper { diff --git a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdSchemaProvider.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs rename to src/StellaOps.Concelier.Connector.Nvd/Internal/NvdSchemaProvider.cs index e7e9e54f..e5079306 100644 --- a/src/StellaOps.Feedser.Source.Nvd/Internal/NvdSchemaProvider.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Internal/NvdSchemaProvider.cs @@ -3,7 +3,7 @@ using System.Reflection; using System.Threading; using Json.Schema; -namespace StellaOps.Feedser.Source.Nvd.Internal; +namespace StellaOps.Concelier.Connector.Nvd.Internal; internal static class NvdSchemaProvider { @@ -14,7 +14,7 @@ internal static class NvdSchemaProvider private static JsonSchema LoadSchema() { var assembly = typeof(NvdSchemaProvider).GetTypeInfo().Assembly; - const string resourceName = "StellaOps.Feedser.Source.Nvd.Schemas.nvd-vulnerability.schema.json"; + const string resourceName = "StellaOps.Concelier.Connector.Nvd.Schemas.nvd-vulnerability.schema.json"; using var stream = assembly.GetManifestResourceStream(resourceName) ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs b/src/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs rename to src/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs index f315b3ef..04bf0f68 100644 --- a/src/StellaOps.Feedser.Source.Nvd/NvdConnector.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/NvdConnector.cs @@ -5,22 +5,22 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Json; -using StellaOps.Feedser.Source.Common.Cursors; -using StellaOps.Feedser.Source.Nvd.Configuration; -using StellaOps.Feedser.Source.Nvd.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.ChangeHistory; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Json; +using StellaOps.Concelier.Connector.Common.Cursors; +using StellaOps.Concelier.Connector.Nvd.Configuration; +using StellaOps.Concelier.Connector.Nvd.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.ChangeHistory; using StellaOps.Plugin; using Json.Schema; -namespace StellaOps.Feedser.Source.Nvd; +namespace StellaOps.Concelier.Connector.Nvd; public sealed class NvdConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Nvd/NvdConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Nvd/NvdConnectorPlugin.cs index 27be1e07..7726a1aa 100644 --- a/src/StellaOps.Feedser.Source.Nvd/NvdConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/NvdConnectorPlugin.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Nvd; +namespace StellaOps.Concelier.Connector.Nvd; public sealed class NvdConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Nvd/NvdServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Nvd/NvdServiceCollectionExtensions.cs index f6d1620e..828fa6d6 100644 --- a/src/StellaOps.Feedser.Source.Nvd/NvdServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/NvdServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Nvd.Configuration; -using StellaOps.Feedser.Source.Nvd.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Nvd.Configuration; +using StellaOps.Concelier.Connector.Nvd.Internal; -namespace StellaOps.Feedser.Source.Nvd; +namespace StellaOps.Concelier.Connector.Nvd; public static class NvdServiceCollectionExtensions { @@ -22,7 +22,7 @@ public static class NvdServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = options.BaseEndpoint; clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Nvd/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.Nvd/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.BaseEndpoint.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; diff --git a/src/StellaOps.Feedser.Source.Acsc/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Nvd/Properties/AssemblyInfo.cs similarity index 53% rename from src/StellaOps.Feedser.Source.Acsc/Properties/AssemblyInfo.cs rename to src/StellaOps.Concelier.Connector.Nvd/Properties/AssemblyInfo.cs index b495b842..7413c0ba 100644 --- a/src/StellaOps.Feedser.Source.Acsc/Properties/AssemblyInfo.cs +++ b/src/StellaOps.Concelier.Connector.Nvd/Properties/AssemblyInfo.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("FixtureUpdater")] -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Acsc.Tests")] +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Nvd.Tests")] +[assembly: InternalsVisibleTo("FixtureUpdater")] diff --git a/src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json b/src/StellaOps.Concelier.Connector.Nvd/Schemas/nvd-vulnerability.schema.json similarity index 100% rename from src/StellaOps.Feedser.Source.Nvd/Schemas/nvd-vulnerability.schema.json rename to src/StellaOps.Concelier.Connector.Nvd/Schemas/nvd-vulnerability.schema.json diff --git a/src/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj b/src/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj new file mode 100644 index 00000000..797a67a3 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Nvd/TASKS.md b/src/StellaOps.Concelier.Connector.Nvd/TASKS.md similarity index 94% rename from src/StellaOps.Feedser.Source.Nvd/TASKS.md rename to src/StellaOps.Concelier.Connector.Nvd/TASKS.md index 9caff6c4..bad8737c 100644 --- a/src/StellaOps.Feedser.Source.Nvd/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Nvd/TASKS.md @@ -11,7 +11,7 @@ |Change history snapshotting|BE-Conn-Nvd|Storage.Mongo|DONE – connector now records per-CVE snapshots with top-level diff metadata whenever canonical advisories change.| |Pagination for windows over page limit|BE-Conn-Nvd|Source.Common|**DONE** – additional page fetcher honors `startIndex`; covered by multipage tests.| |Schema validation quarantine path|BE-Conn-Nvd|Storage.Mongo|**DONE** – schema failures mark documents failed and metrics assert quarantine.| -|FEEDCONN-NVD-04-002 Conflict regression fixtures|BE-Conn-Nvd, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Published `conflict-nvd.canonical.json` + mapper test; includes CVSS 3.1 + CWE reference and normalized CPE range feeding the conflict triple. Validation: `dotnet test src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj --filter NvdConflictFixtureTests`.| +|FEEDCONN-NVD-04-002 Conflict regression fixtures|BE-Conn-Nvd, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Published `conflict-nvd.canonical.json` + mapper test; includes CVSS 3.1 + CWE reference and normalized CPE range feeding the conflict triple. Validation: `dotnet test src/StellaOps.Concelier.Connector.Nvd.Tests/StellaOps.Concelier.Connector.Nvd.Tests.csproj --filter NvdConflictFixtureTests`.| |FEEDCONN-NVD-02-004 NVD CVSS & CWE precedence payloads|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – CVSS metrics now carry provenance masks, CWE weaknesses emit normalized references, and fixtures cover the additional precedence data.| |FEEDCONN-NVD-02-005 NVD merge/export parity regression|BE-Conn-Nvd, BE-Merge|Merge `FEEDMERGE-ENGINE-04-003`|**DONE (2025-10-12)** – Canonical merge parity fixtures captured, regression test validates credit/reference union, and exporter snapshot check guarantees parity through JSON exports.| |FEEDCONN-NVD-02-002 Normalized versions rollout|BE-Conn-Nvd|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** – SemVer primitives + normalized rules emitting for parseable ranges, fixtures/tests refreshed, coordination pinged via FEEDMERGE-COORD-02-900.| diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/conflict-osv.canonical.json similarity index 96% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/conflict-osv.canonical.json index fc30c63a..be4b758b 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/conflict-osv.canonical.json +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/conflict-osv.canonical.json @@ -1,177 +1,177 @@ -{ - "advisoryKey": "OSV-2025-4242", - "affectedPackages": [ - { - "type": "semver", - "identifier": "npm:conflict/package", - "platform": "npm", - "versionRanges": [ - { - "fixedVersion": "1.5.0", - "introducedVersion": "1.0.0", - "lastAffectedVersion": "1.4.2", - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "1.5.0", - "fixedInclusive": false, - "introduced": "1.0.0", - "introducedInclusive": true, - "lastAffected": "1.4.2", - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "npm:conflict/package", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "1.0.0", - "minInclusive": true, - "max": "1.5.0", - "maxInclusive": false, - "value": null, - "notes": "osv:npm:OSV-2025-4242:npm:conflict/package" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "npm:conflict/package", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2025-4242", - "GHSA-qqqq-wwww-eeee", - "OSV-2025-4242" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", - "credits": [ - { - "displayName": "osv-reporter", - "role": "reporter", - "contacts": [ - "mailto:osv-reporter@example.com" - ], - "provenance": { - "source": "osv", - "kind": "credit", - "value": "osv-reporter", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "credits[]" - ] - } - } - ], - "cvssMetrics": [ - { - "baseScore": 4.6, - "baseSeverity": "medium", - "provenance": { - "source": "osv", - "kind": "cvss", - "value": "CVSS_V3", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", - "version": "3.1" - } - ], - "cwes": [], - "description": "OSV captures the latest container escape details including patched version metadata.", - "exploitKnown": false, - "language": "en", - "modified": "2025-03-06T12:00:00+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://api.osv.dev/v1/vulns/OSV-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-06T11:30:00+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "OSV-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-02-28T00:00:00+00:00", - "references": [ - { - "kind": "patch", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/conflict/package/commit/abcdef1234567890", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "FIX", - "summary": null, - "url": "https://github.com/conflict/package/commit/abcdef1234567890" - }, - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://osv.dev/vulnerability/OSV-2025-4242", - "decisionReason": null, - "recordedAt": "2025-03-06T12:05:00+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "ADVISORY", - "summary": null, - "url": "https://osv.dev/vulnerability/OSV-2025-4242" - } - ], - "severity": "medium", - "summary": "OSV captures the latest container escape details including patched version metadata.", - "title": "Container escape for conflict-package" +{ + "advisoryKey": "OSV-2025-4242", + "affectedPackages": [ + { + "type": "semver", + "identifier": "npm:conflict/package", + "platform": "npm", + "versionRanges": [ + { + "fixedVersion": "1.5.0", + "introducedVersion": "1.0.0", + "lastAffectedVersion": "1.4.2", + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "1.5.0", + "fixedInclusive": false, + "introduced": "1.0.0", + "introducedInclusive": true, + "lastAffected": "1.4.2", + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "npm:conflict/package", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.0.0", + "minInclusive": true, + "max": "1.5.0", + "maxInclusive": false, + "value": null, + "notes": "osv:npm:OSV-2025-4242:npm:conflict/package" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "npm:conflict/package", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-4242", + "GHSA-qqqq-wwww-eeee", + "OSV-2025-4242" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", + "credits": [ + { + "displayName": "osv-reporter", + "role": "reporter", + "contacts": [ + "mailto:osv-reporter@example.com" + ], + "provenance": { + "source": "osv", + "kind": "credit", + "value": "osv-reporter", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "credits[]" + ] + } + } + ], + "cvssMetrics": [ + { + "baseScore": 4.6, + "baseSeverity": "medium", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L", + "version": "3.1" + } + ], + "cwes": [], + "description": "OSV captures the latest container escape details including patched version metadata.", + "exploitKnown": false, + "language": "en", + "modified": "2025-03-06T12:00:00+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://api.osv.dev/v1/vulns/OSV-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-06T11:30:00+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "OSV-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-02-28T00:00:00+00:00", + "references": [ + { + "kind": "patch", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/conflict/package/commit/abcdef1234567890", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "FIX", + "summary": null, + "url": "https://github.com/conflict/package/commit/abcdef1234567890" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://osv.dev/vulnerability/OSV-2025-4242", + "decisionReason": null, + "recordedAt": "2025-03-06T12:05:00+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://osv.dev/vulnerability/OSV-2025-4242" + } + ], + "severity": "medium", + "summary": "OSV captures the latest container escape details including patched version metadata.", + "title": "Container escape for conflict-package" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.ghsa.json similarity index 100% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.ghsa.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.ghsa.json diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.osv.json similarity index 97% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.osv.json index 724f6d31..826fbd3b 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.osv.json +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.osv.json @@ -1,1609 +1,1609 @@ -[ - { - "advisoryKey": "GHSA-77vh-xpmg-72qh", - "affectedPackages": [ - { - "type": "semver", - "identifier": "pkg:golang/github.com/opencontainers/image-spec", - "platform": "Go", - "versionRanges": [ - { - "fixedVersion": "1.0.2", - "introducedVersion": "0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "1.0.2", - "fixedInclusive": false, - "introduced": "0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:golang/github.com/opencontainers/image-spec", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "0", - "minInclusive": true, - "max": "1.0.2", - "maxInclusive": false, - "value": null, - "notes": "osv:Go:GHSA-77vh-xpmg-72qh:pkg:golang/github.com/opencontainers/image-spec" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:golang/github.com/opencontainers/image-spec", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CGA-j36r-723f-8c29", - "GHSA-77vh-xpmg-72qh" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", - "credits": [], - "cvssMetrics": [ - { - "baseScore": 3, - "baseSeverity": "low", - "provenance": { - "source": "osv", - "kind": "cvss", - "value": "CVSS_V3", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", - "version": "3.1" - } - ], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-843", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/843.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-843", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - } - ], - "description": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2", - "exploitKnown": false, - "language": "en", - "modified": "2021-11-24T19:43:35+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-77vh-xpmg-72qh", - "decisionReason": null, - "recordedAt": "2021-11-18T16:02:41+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-77vh-xpmg-72qh", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2021-11-18T16:02:41+00:00", - "references": [ - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/opencontainers/image-spec", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "PACKAGE", - "summary": null, - "url": "https://github.com/opencontainers/image-spec" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9970795+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" - } - ], - "severity": "low", - "summary": "### Impact In the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index. ### Patches The Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document. Release [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates. ### Workarounds Software attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields. ### References https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m ### For more information If you have any questions or comments about this advisory: * Open an issue in https://github.com/opencontainers/image-spec * Email us at [security@opencontainers.org](mailto:security@opencontainers.org) * https://github.com/opencontainers/image-spec/commits/v1.0.2", - "title": "Clarify `mediaType` handling" - }, - { - "advisoryKey": "GHSA-7rjr-3q55-vv33", - "affectedPackages": [ - { - "type": "semver", - "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "2.16.0", - "introducedVersion": "2.13.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "2.16.0", - "fixedInclusive": false, - "introduced": "2.13.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.apache.logging.log4j/log4j-core", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "2.13.0", - "minInclusive": true, - "max": "2.16.0", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.apache.logging.log4j/log4j-core", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - }, - { - "type": "semver", - "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "2.12.2", - "introducedVersion": "0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "2.12.2", - "fixedInclusive": false, - "introduced": "0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.apache.logging.log4j/log4j-core", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "0", - "minInclusive": true, - "max": "2.12.2", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.apache.logging.log4j/log4j-core", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - }, - { - "type": "semver", - "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "1.9.2", - "introducedVersion": "1.8.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "1.9.2", - "fixedInclusive": false, - "introduced": "1.8.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "1.8.0", - "minInclusive": true, - "max": "1.9.2", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - }, - { - "type": "semver", - "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "1.10.8", - "introducedVersion": "1.10.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "1.10.8", - "fixedInclusive": false, - "introduced": "1.10.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "1.10.0", - "minInclusive": true, - "max": "1.10.8", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - }, - { - "type": "semver", - "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "1.11.11", - "introducedVersion": "1.11.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "1.11.11", - "fixedInclusive": false, - "introduced": "1.11.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "1.11.0", - "minInclusive": true, - "max": "1.11.11", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - }, - { - "type": "semver", - "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "platform": "Maven", - "versionRanges": [ - { - "fixedVersion": "2.0.12", - "introducedVersion": "2.0.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "2.0.12", - "fixedInclusive": false, - "introduced": "2.0.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "2.0.0", - "minInclusive": true, - "max": "2.0.12", - "maxInclusive": false, - "value": null, - "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2021-45046", - "GHSA-7rjr-3q55-vv33" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", - "credits": [], - "cvssMetrics": [ - { - "baseScore": 9, - "baseSeverity": "critical", - "provenance": { - "source": "osv", - "kind": "cvss", - "value": "CVSS_V3", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", - "version": "3.1" - } - ], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-502", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/502.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-502", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - }, - { - "taxonomy": "cwe", - "identifier": "CWE-917", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/917.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-917", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - } - ], - "description": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", - "exploitKnown": false, - "language": "en", - "modified": "2025-05-09T13:13:16.169374+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-7rjr-3q55-vv33", - "decisionReason": null, - "recordedAt": "2021-12-14T18:01:28+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-7rjr-3q55-vv33", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2021-12-14T18:01:28+00:00", - "references": [ - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "http://www.openwall.com/lists/oss-security/2021/12/14/4", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "http://www.openwall.com/lists/oss-security/2021/12/15/3", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "http://www.openwall.com/lists/oss-security/2021/12/18/1", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" - }, - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "ADVISORY", - "summary": null, - "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://logging.apache.org/log4j/2.x/security.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://logging.apache.org/log4j/2.x/security.html" - }, - { - "kind": "advisory", - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "ADVISORY", - "summary": null, - "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://security.gentoo.org/glsa/202310-16", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://security.gentoo.org/glsa/202310-16" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.cve.org/CVERecord?id=CVE-2021-44228", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.debian.org/security/2021/dsa-5022", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.debian.org/security/2021/dsa-5022" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.kb.cert.org/vuls/id/930724", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.kb.cert.org/vuls/id/930724" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.openwall.com/lists/oss-security/2021/12/14/4", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.oracle.com/security-alerts/cpuapr2022.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.oracle.com/security-alerts/cpujan2022.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.oracle.com/security-alerts/cpujan2022.html" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://www.oracle.com/security-alerts/cpujul2022.html", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9980643+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://www.oracle.com/security-alerts/cpujul2022.html" - } - ], - "severity": "critical", - "summary": "# Impact The fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. ## Affected packages Only the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use. # Mitigation Log4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class). Log4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", - "title": "Incomplete fix for Apache Log4j vulnerability" - }, - { - "advisoryKey": "GHSA-cjjf-27cc-pvmv", - "affectedPackages": [ - { - "type": "semver", - "identifier": "pkg:pypi/pyload-ng", - "platform": "PyPI", - "versionRanges": [ - { - "fixedVersion": "0.5.0b3.dev91", - "introducedVersion": "0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "0.5.0b3.dev91", - "fixedInclusive": false, - "introduced": "0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:pypi/pyload-ng", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "0", - "minInclusive": true, - "max": "0.5.0b3.dev91", - "maxInclusive": false, - "value": null, - "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:pypi/pyload-ng", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2025-61773", - "GHSA-cjjf-27cc-pvmv" - ], - "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", - "credits": [], - "cvssMetrics": [ - { - "baseScore": 8.1, - "baseSeverity": "high", - "provenance": { - "source": "osv", - "kind": "cvss", - "value": "CVSS_V3", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", - "version": "3.1" - } - ], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-116", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/116.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-116", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - }, - { - "taxonomy": "cwe", - "identifier": "CWE-74", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/74.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-74", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - }, - { - "taxonomy": "cwe", - "identifier": "CWE-79", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/79.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-79", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - }, - { - "taxonomy": "cwe", - "identifier": "CWE-94", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/94.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-94", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - } - ], - "description": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`).\n4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n```http\nGET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n```\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", - "exploitKnown": false, - "language": "en", - "modified": "2025-10-09T15:59:13.250015+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-cjjf-27cc-pvmv", - "decisionReason": null, - "recordedAt": "2025-10-09T15:19:48+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-cjjf-27cc-pvmv", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T15:19:48+00:00", - "references": [ - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/pyload/pyload", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "PACKAGE", - "summary": null, - "url": "https://github.com/pyload/pyload" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/pyload/pyload/pull/4624", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/pyload/pyload/pull/4624" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.995174+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" - } - ], - "severity": "high", - "summary": "### Summary pyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted. user-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow. CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests. ### PoC 1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624). 2. Start the web UI and access the Captcha or CNL endpoints. 3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`). 4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS). Example request: ```http GET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1 Host: 127.0.0.1:8000 Content-Type: application/x-www-form-urlencoded Content-Length: 107 ``` ### Impact Exploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", - "title": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters" - }, - { - "advisoryKey": "GHSA-wv4w-6qv2-qqfg", - "affectedPackages": [ - { - "type": "semver", - "identifier": "pkg:pypi/social-auth-app-django", - "platform": "PyPI", - "versionRanges": [ - { - "fixedVersion": "5.6.0", - "introducedVersion": "0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": false, - "nevra": null, - "semVer": { - "constraintExpression": null, - "exactValue": null, - "fixed": "5.6.0", - "fixedInclusive": false, - "introduced": "0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": true, - "style": "range" - }, - "vendorExtensions": null - }, - "provenance": { - "source": "osv", - "kind": "range", - "value": "pkg:pypi/social-auth-app-django", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "affectedpackages[].versionranges[]" - ] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [ - { - "scheme": "semver", - "type": "range", - "min": "0", - "minInclusive": true, - "max": "5.6.0", - "maxInclusive": false, - "value": null, - "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" - } - ], - "statuses": [], - "provenance": [ - { - "source": "osv", - "kind": "affected", - "value": "pkg:pypi/social-auth-app-django", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "affectedpackages[]" - ] - } - ] - } - ], - "aliases": [ - "CVE-2025-61783", - "GHSA-wv4w-6qv2-qqfg" - ], - "canonicalMetricId": "osv:severity/medium", - "credits": [], - "cvssMetrics": [], - "cwes": [ - { - "taxonomy": "cwe", - "identifier": "CWE-290", - "name": null, - "uri": "https://cwe.mitre.org/data/definitions/290.html", - "provenance": [ - { - "source": "osv", - "kind": "weakness", - "value": "CWE-290", - "decisionReason": "database_specific.cwe_ids", - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "cwes[]" - ] - } - ] - } - ], - "description": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", - "exploitKnown": false, - "language": "en", - "modified": "2025-10-09T17:57:29.916841+00:00", - "provenance": [ - { - "source": "osv", - "kind": "document", - "value": "https://osv.dev/vulnerability/GHSA-wv4w-6qv2-qqfg", - "decisionReason": null, - "recordedAt": "2025-10-09T17:08:05+00:00", - "fieldMask": [ - "advisory" - ] - }, - { - "source": "osv", - "kind": "mapping", - "value": "GHSA-wv4w-6qv2-qqfg", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "advisory" - ] - } - ], - "published": "2025-10-09T17:08:05+00:00", - "references": [ - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "PACKAGE", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/issues/220", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/issues/220" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/issues/231", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/issues/231" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/issues/634", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/issues/634" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/pull/803", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/pull/803" - }, - { - "kind": null, - "provenance": { - "source": "osv", - "kind": "reference", - "value": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", - "decisionReason": null, - "recordedAt": "2025-10-15T14:48:57.9927932+00:00", - "fieldMask": [ - "references[]" - ] - }, - "sourceTag": "WEB", - "summary": null, - "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" - } - ], - "severity": "medium", - "summary": "### Impact Upon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses. ### Patches * https://github.com/python-social-auth/social-app-django/pull/803 ### Workarounds Review the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", - "title": "Python Social Auth - Django has unsafe account association" - } -] +[ + { + "advisoryKey": "GHSA-77vh-xpmg-72qh", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:golang/github.com/opencontainers/image-spec", + "platform": "Go", + "versionRanges": [ + { + "fixedVersion": "1.0.2", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "1.0.2", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "1.0.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Go:GHSA-77vh-xpmg-72qh:pkg:golang/github.com/opencontainers/image-spec" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:golang/github.com/opencontainers/image-spec", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CGA-j36r-723f-8c29", + "GHSA-77vh-xpmg-72qh" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "credits": [], + "cvssMetrics": [ + { + "baseScore": 3, + "baseSeverity": "low", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-843", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/843.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-843", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2", + "exploitKnown": false, + "language": "en", + "modified": "2021-11-24T19:43:35+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2021-11-18T16:02:41+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2021-11-18T16:02:41+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/opencontainers/image-spec" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9970795+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" + } + ], + "severity": "low", + "summary": "### Impact In the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index. ### Patches The Image Specification will be updated to recommend that both manifest and index documents contain a `mediaType` field to identify the type of document. Release [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates. ### Workarounds Software attempting to deserialize an ambiguous document may reject the document if it contains both “manifests” and “layers” fields or “manifests” and “config” fields. ### References https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m ### For more information If you have any questions or comments about this advisory: * Open an issue in https://github.com/opencontainers/image-spec * Email us at [security@opencontainers.org](mailto:security@opencontainers.org) * https://github.com/opencontainers/image-spec/commits/v1.0.2", + "title": "Clarify `mediaType` handling" + }, + { + "advisoryKey": "GHSA-7rjr-3q55-vv33", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.16.0", + "introducedVersion": "2.13.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "2.16.0", + "fixedInclusive": false, + "introduced": "2.13.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "2.13.0", + "minInclusive": true, + "max": "2.16.0", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.apache.logging.log4j/log4j-core", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.12.2", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "2.12.2", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "2.12.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.apache.logging.log4j/log4j-core" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.apache.logging.log4j/log4j-core", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.9.2", + "introducedVersion": "1.8.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "1.9.2", + "fixedInclusive": false, + "introduced": "1.8.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.8.0", + "minInclusive": true, + "max": "1.9.2", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.10.8", + "introducedVersion": "1.10.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "1.10.8", + "fixedInclusive": false, + "introduced": "1.10.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.10.0", + "minInclusive": true, + "max": "1.10.8", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "1.11.11", + "introducedVersion": "1.11.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "1.11.11", + "fixedInclusive": false, + "introduced": "1.11.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "1.11.0", + "minInclusive": true, + "max": "1.11.11", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + }, + { + "type": "semver", + "identifier": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "platform": "Maven", + "versionRanges": [ + { + "fixedVersion": "2.0.12", + "introducedVersion": "2.0.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "2.0.12", + "fixedInclusive": false, + "introduced": "2.0.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "2.0.0", + "minInclusive": true, + "max": "2.0.12", + "maxInclusive": false, + "value": null, + "notes": "osv:Maven:GHSA-7rjr-3q55-vv33:pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2021-45046", + "GHSA-7rjr-3q55-vv33" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9, + "baseSeverity": "critical", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-502", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/502.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-502", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + }, + { + "taxonomy": "cwe", + "identifier": "CWE-917", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/917.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-917", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", + "exploitKnown": false, + "language": "en", + "modified": "2025-05-09T13:13:16.169374+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-7rjr-3q55-vv33", + "decisionReason": null, + "recordedAt": "2021-12-14T18:01:28+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-7rjr-3q55-vv33", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2021-12-14T18:01:28+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/15/3", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "http://www.openwall.com/lists/oss-security/2021/12/18/1", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://logging.apache.org/log4j/2.x/security.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "kind": "advisory", + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "ADVISORY", + "summary": null, + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://security.gentoo.org/glsa/202310-16", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://security.gentoo.org/glsa/202310-16" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.cve.org/CVERecord?id=CVE-2021-44228", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.debian.org/security/2021/dsa-5022", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.debian.org/security/2021/dsa-5022" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.kb.cert.org/vuls/id/930724", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.openwall.com/lists/oss-security/2021/12/14/4", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpuapr2022.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujan2022.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://www.oracle.com/security-alerts/cpujul2022.html", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9980643+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://www.oracle.com/security-alerts/cpujul2022.html" + } + ], + "severity": "critical", + "summary": "# Impact The fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. ## Affected packages Only the `org.apache.logging.log4j:log4j-core` package is directly affected by this vulnerability. The `org.apache.logging.log4j:log4j-api` should be kept at the same version as the `org.apache.logging.log4j:log4j-core` package to ensure compatability if in use. # Mitigation Log4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (< 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class). Log4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property `log4j2.formatMsgNoLookups` to `true` do NOT mitigate this specific vulnerability.", + "title": "Incomplete fix for Apache Log4j vulnerability" + }, + { + "advisoryKey": "GHSA-cjjf-27cc-pvmv", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:pypi/pyload-ng", + "platform": "PyPI", + "versionRanges": [ + { + "fixedVersion": "0.5.0b3.dev91", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "0.5.0b3.dev91", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:pypi/pyload-ng", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "0.5.0b3.dev91", + "maxInclusive": false, + "value": null, + "notes": "osv:PyPI:GHSA-cjjf-27cc-pvmv:pkg:pypi/pyload-ng" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:pypi/pyload-ng", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-61773", + "GHSA-cjjf-27cc-pvmv" + ], + "canonicalMetricId": "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "credits": [], + "cvssMetrics": [ + { + "baseScore": 8.1, + "baseSeverity": "high", + "provenance": { + "source": "osv", + "kind": "cvss", + "value": "CVSS_V3", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "version": "3.1" + } + ], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-116", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/116.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-116", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + }, + { + "taxonomy": "cwe", + "identifier": "CWE-74", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/74.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-74", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + }, + { + "taxonomy": "cwe", + "identifier": "CWE-79", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/79.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-79", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + }, + { + "taxonomy": "cwe", + "identifier": "CWE-94", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/94.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-94", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`).\n4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n```http\nGET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n```\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T15:59:13.250015+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-09T15:19:48+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T15:19:48+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/pyload/pyload" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/pull/4624", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/pull/4624" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.995174+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" + } + ], + "severity": "high", + "summary": "### Summary pyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click'N'Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted. user-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow. CNL (Click'N'Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests. ### PoC 1. Run a vulnerable version of pyLoad prior to commit [`f9d27f2`](https://github.com/pyload/pyload/pull/4624). 2. Start the web UI and access the Captcha or CNL endpoints. 3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (`/flash/addcrypted2?jk=function(){alert(1)}&crypted=12345`). 4. Observe that the payload is reflected and executed in the client’s browser, demonstrating cross-site scripting (XSS). Example request: ```http GET /flash/addcrypted2?jk=function(){alert(1)}&crypted=12345 HTTP/1.1 Host: 127.0.0.1:8000 Content-Type: application/x-www-form-urlencoded Content-Length: 107 ``` ### Impact Exploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "title": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters" + }, + { + "advisoryKey": "GHSA-wv4w-6qv2-qqfg", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:pypi/social-auth-app-django", + "platform": "PyPI", + "versionRanges": [ + { + "fixedVersion": "5.6.0", + "introducedVersion": "0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": false, + "nevra": null, + "semVer": { + "constraintExpression": null, + "exactValue": null, + "fixed": "5.6.0", + "fixedInclusive": false, + "introduced": "0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": true, + "style": "range" + }, + "vendorExtensions": null + }, + "provenance": { + "source": "osv", + "kind": "range", + "value": "pkg:pypi/social-auth-app-django", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "affectedpackages[].versionranges[]" + ] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [ + { + "scheme": "semver", + "type": "range", + "min": "0", + "minInclusive": true, + "max": "5.6.0", + "maxInclusive": false, + "value": null, + "notes": "osv:PyPI:GHSA-wv4w-6qv2-qqfg:pkg:pypi/social-auth-app-django" + } + ], + "statuses": [], + "provenance": [ + { + "source": "osv", + "kind": "affected", + "value": "pkg:pypi/social-auth-app-django", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "affectedpackages[]" + ] + } + ] + } + ], + "aliases": [ + "CVE-2025-61783", + "GHSA-wv4w-6qv2-qqfg" + ], + "canonicalMetricId": "osv:severity/medium", + "credits": [], + "cvssMetrics": [], + "cwes": [ + { + "taxonomy": "cwe", + "identifier": "CWE-290", + "name": null, + "uri": "https://cwe.mitre.org/data/definitions/290.html", + "provenance": [ + { + "source": "osv", + "kind": "weakness", + "value": "CWE-290", + "decisionReason": "database_specific.cwe_ids", + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "cwes[]" + ] + } + ] + } + ], + "description": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "exploitKnown": false, + "language": "en", + "modified": "2025-10-09T17:57:29.916841+00:00", + "provenance": [ + { + "source": "osv", + "kind": "document", + "value": "https://osv.dev/vulnerability/GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-09T17:08:05+00:00", + "fieldMask": [ + "advisory" + ] + }, + { + "source": "osv", + "kind": "mapping", + "value": "GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "advisory" + ] + } + ], + "published": "2025-10-09T17:08:05+00:00", + "references": [ + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "PACKAGE", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/220", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/220" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/231", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/231" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/issues/634", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/issues/634" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/pull/803", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/pull/803" + }, + { + "kind": null, + "provenance": { + "source": "osv", + "kind": "reference", + "value": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", + "decisionReason": null, + "recordedAt": "2025-10-15T14:48:57.9927932+00:00", + "fieldMask": [ + "references[]" + ] + }, + "sourceTag": "WEB", + "summary": null, + "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" + } + ], + "severity": "medium", + "summary": "### Impact Upon authentication, the user could be associated by e-mail even if the `associate_by_email` pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn't require unique e-mail addresses. ### Patches * https://github.com/python-social-auth/social-app-django/pull/803 ### Workarounds Review the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "title": "Python Social Auth - Django has unsafe account association" + } +] diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json similarity index 98% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json index 34893ac5..4d2d84b1 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-ghsa.json @@ -1,519 +1,519 @@ -[ - { - "ghsa_id": "GHSA-wv4w-6qv2-qqfg", - "cve_id": "CVE-2025-61783", - "url": "https://api.github.com/advisories/GHSA-wv4w-6qv2-qqfg", - "html_url": "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg", - "summary": "Python Social Auth - Django has unsafe account association ", - "description": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", - "type": "reviewed", - "severity": "medium", - "repository_advisory_url": "https://api.github.com/repos/python-social-auth/social-app-django/security-advisories/GHSA-wv4w-6qv2-qqfg", - "source_code_location": "https://github.com/python-social-auth/social-app-django", - "identifiers": [ - { - "value": "GHSA-wv4w-6qv2-qqfg", - "type": "GHSA" - }, - { - "value": "CVE-2025-61783", - "type": "CVE" - } - ], - "references": [ - "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", - "https://github.com/python-social-auth/social-app-django/issues/220", - "https://github.com/python-social-auth/social-app-django/issues/231", - "https://github.com/python-social-auth/social-app-django/issues/634", - "https://github.com/python-social-auth/social-app-django/pull/803", - "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", - "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg" - ], - "published_at": "2025-10-09T17:08:05Z", - "updated_at": "2025-10-09T17:08:06Z", - "github_reviewed_at": "2025-10-09T17:08:05Z", - "nvd_published_at": null, - "withdrawn_at": null, - "vulnerabilities": [ - { - "package": { - "ecosystem": "pip", - "name": "social-auth-app-django" - }, - "vulnerable_version_range": "\u003C 5.6.0", - "first_patched_version": "5.6.0", - "vulnerable_functions": [] - } - ], - "cvss_severities": { - "cvss_v3": { - "vector_string": null, - "score": 0.0 - }, - "cvss_v4": { - "vector_string": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N", - "score": 6.3 - } - }, - "cwes": [ - { - "cwe_id": "CWE-290", - "name": "Authentication Bypass by Spoofing" - } - ], - "credits": [ - { - "user": { - "login": "mel-mason", - "id": 19391457, - "node_id": "MDQ6VXNlcjE5MzkxNDU3", - "avatar_url": "https://avatars.githubusercontent.com/u/19391457?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/mel-mason", - "html_url": "https://github.com/mel-mason", - "followers_url": "https://api.github.com/users/mel-mason/followers", - "following_url": "https://api.github.com/users/mel-mason/following{/other_user}", - "gists_url": "https://api.github.com/users/mel-mason/gists{/gist_id}", - "starred_url": "https://api.github.com/users/mel-mason/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/mel-mason/subscriptions", - "organizations_url": "https://api.github.com/users/mel-mason/orgs", - "repos_url": "https://api.github.com/users/mel-mason/repos", - "events_url": "https://api.github.com/users/mel-mason/events{/privacy}", - "received_events_url": "https://api.github.com/users/mel-mason/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "reporter" - }, - { - "user": { - "login": "vanya909", - "id": 53380238, - "node_id": "MDQ6VXNlcjUzMzgwMjM4", - "avatar_url": "https://avatars.githubusercontent.com/u/53380238?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/vanya909", - "html_url": "https://github.com/vanya909", - "followers_url": "https://api.github.com/users/vanya909/followers", - "following_url": "https://api.github.com/users/vanya909/following{/other_user}", - "gists_url": "https://api.github.com/users/vanya909/gists{/gist_id}", - "starred_url": "https://api.github.com/users/vanya909/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/vanya909/subscriptions", - "organizations_url": "https://api.github.com/users/vanya909/orgs", - "repos_url": "https://api.github.com/users/vanya909/repos", - "events_url": "https://api.github.com/users/vanya909/events{/privacy}", - "received_events_url": "https://api.github.com/users/vanya909/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "reporter" - }, - { - "user": { - "login": "nijel", - "id": 212189, - "node_id": "MDQ6VXNlcjIxMjE4OQ==", - "avatar_url": "https://avatars.githubusercontent.com/u/212189?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/nijel", - "html_url": "https://github.com/nijel", - "followers_url": "https://api.github.com/users/nijel/followers", - "following_url": "https://api.github.com/users/nijel/following{/other_user}", - "gists_url": "https://api.github.com/users/nijel/gists{/gist_id}", - "starred_url": "https://api.github.com/users/nijel/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/nijel/subscriptions", - "organizations_url": "https://api.github.com/users/nijel/orgs", - "repos_url": "https://api.github.com/users/nijel/repos", - "events_url": "https://api.github.com/users/nijel/events{/privacy}", - "received_events_url": "https://api.github.com/users/nijel/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "remediation_developer" - } - ], - "cvss": { - "vector_string": null, - "score": null - } - }, - { - "ghsa_id": "GHSA-cjjf-27cc-pvmv", - "cve_id": "CVE-2025-61773", - "url": "https://api.github.com/advisories/GHSA-cjjf-27cc-pvmv", - "html_url": "https://github.com/advisories/GHSA-cjjf-27cc-pvmv", - "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", - "description": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", - "type": "reviewed", - "severity": "high", - "repository_advisory_url": "https://api.github.com/repos/pyload/pyload/security-advisories/GHSA-cjjf-27cc-pvmv", - "source_code_location": "https://github.com/pyload/pyload", - "identifiers": [ - { - "value": "GHSA-cjjf-27cc-pvmv", - "type": "GHSA" - }, - { - "value": "CVE-2025-61773", - "type": "CVE" - } - ], - "references": [ - "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", - "https://github.com/pyload/pyload/pull/4624", - "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", - "https://github.com/advisories/GHSA-cjjf-27cc-pvmv" - ], - "published_at": "2025-10-09T15:19:48Z", - "updated_at": "2025-10-09T15:19:48Z", - "github_reviewed_at": "2025-10-09T15:19:48Z", - "nvd_published_at": null, - "withdrawn_at": null, - "vulnerabilities": [ - { - "package": { - "ecosystem": "pip", - "name": "pyload-ng" - }, - "vulnerable_version_range": "\u003C 0.5.0b3.dev91", - "first_patched_version": "0.5.0b3.dev91", - "vulnerable_functions": [] - } - ], - "cvss_severities": { - "cvss_v3": { - "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", - "score": 8.1 - }, - "cvss_v4": { - "vector_string": null, - "score": 0.0 - } - }, - "cwes": [ - { - "cwe_id": "CWE-74", - "name": "Improper Neutralization of Special Elements in Output Used by a Downstream Component (\u0027Injection\u0027)" - }, - { - "cwe_id": "CWE-79", - "name": "Improper Neutralization of Input During Web Page Generation (\u0027Cross-site Scripting\u0027)" - }, - { - "cwe_id": "CWE-94", - "name": "Improper Control of Generation of Code (\u0027Code Injection\u0027)" - }, - { - "cwe_id": "CWE-116", - "name": "Improper Encoding or Escaping of Output" - } - ], - "credits": [ - { - "user": { - "login": "odaysec", - "id": 47859767, - "node_id": "MDQ6VXNlcjQ3ODU5NzY3", - "avatar_url": "https://avatars.githubusercontent.com/u/47859767?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/odaysec", - "html_url": "https://github.com/odaysec", - "followers_url": "https://api.github.com/users/odaysec/followers", - "following_url": "https://api.github.com/users/odaysec/following{/other_user}", - "gists_url": "https://api.github.com/users/odaysec/gists{/gist_id}", - "starred_url": "https://api.github.com/users/odaysec/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/odaysec/subscriptions", - "organizations_url": "https://api.github.com/users/odaysec/orgs", - "repos_url": "https://api.github.com/users/odaysec/repos", - "events_url": "https://api.github.com/users/odaysec/events{/privacy}", - "received_events_url": "https://api.github.com/users/odaysec/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "reporter" - } - ], - "cvss": { - "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", - "score": 8.1 - } - }, - { - "ghsa_id": "GHSA-77vh-xpmg-72qh", - "cve_id": null, - "url": "https://api.github.com/advisories/GHSA-77vh-xpmg-72qh", - "html_url": "https://github.com/advisories/GHSA-77vh-xpmg-72qh", - "summary": "Clarify \u0060mediaType\u0060 handling", - "description": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", - "type": "reviewed", - "severity": "low", - "repository_advisory_url": "https://api.github.com/repos/opencontainers/image-spec/security-advisories/GHSA-77vh-xpmg-72qh", - "source_code_location": "https://github.com/opencontainers/image-spec", - "identifiers": [ - { - "value": "GHSA-77vh-xpmg-72qh", - "type": "GHSA" - } - ], - "references": [ - "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", - "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", - "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", - "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", - "https://github.com/advisories/GHSA-77vh-xpmg-72qh" - ], - "published_at": "2021-11-18T16:02:41Z", - "updated_at": "2023-01-09T05:05:32Z", - "github_reviewed_at": "2021-11-17T23:13:41Z", - "nvd_published_at": null, - "withdrawn_at": null, - "vulnerabilities": [ - { - "package": { - "ecosystem": "go", - "name": "github.com/opencontainers/image-spec" - }, - "vulnerable_version_range": "\u003C 1.0.2", - "first_patched_version": "1.0.2", - "vulnerable_functions": [] - } - ], - "cvss_severities": { - "cvss_v3": { - "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", - "score": 3.0 - }, - "cvss_v4": { - "vector_string": null, - "score": 0.0 - } - }, - "cwes": [ - { - "cwe_id": "CWE-843", - "name": "Access of Resource Using Incompatible Type (\u0027Type Confusion\u0027)" - } - ], - "credits": [], - "cvss": { - "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", - "score": 3.0 - } - }, - { - "ghsa_id": "GHSA-7rjr-3q55-vv33", - "cve_id": "CVE-2021-45046", - "url": "https://api.github.com/advisories/GHSA-7rjr-3q55-vv33", - "html_url": "https://github.com/advisories/GHSA-7rjr-3q55-vv33", - "summary": "Incomplete fix for Apache Log4j vulnerability", - "description": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", - "type": "reviewed", - "severity": "critical", - "repository_advisory_url": null, - "source_code_location": "", - "identifiers": [ - { - "value": "GHSA-7rjr-3q55-vv33", - "type": "GHSA" - }, - { - "value": "CVE-2021-45046", - "type": "CVE" - } - ], - "references": [ - "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", - "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", - "https://logging.apache.org/log4j/2.x/security.html", - "https://www.openwall.com/lists/oss-security/2021/12/14/4", - "https://www.cve.org/CVERecord?id=CVE-2021-44228", - "http://www.openwall.com/lists/oss-security/2021/12/14/4", - "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", - "http://www.openwall.com/lists/oss-security/2021/12/15/3", - "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", - "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", - "https://www.kb.cert.org/vuls/id/930724", - "https://www.debian.org/security/2021/dsa-5022", - "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", - "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", - "http://www.openwall.com/lists/oss-security/2021/12/18/1", - "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", - "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", - "https://www.oracle.com/security-alerts/cpujan2022.html", - "https://www.oracle.com/security-alerts/cpuapr2022.html", - "https://www.oracle.com/security-alerts/cpujul2022.html", - "https://security.gentoo.org/glsa/202310-16", - "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", - "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", - "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", - "https://github.com/advisories/GHSA-7rjr-3q55-vv33" - ], - "published_at": "2021-12-14T18:01:28Z", - "updated_at": "2025-05-09T12:28:41Z", - "github_reviewed_at": "2021-12-14T17:55:00Z", - "nvd_published_at": "2021-12-14T19:15:00Z", - "withdrawn_at": null, - "vulnerabilities": [ - { - "package": { - "ecosystem": "maven", - "name": "org.apache.logging.log4j:log4j-core" - }, - "vulnerable_version_range": "\u003E= 2.13.0, \u003C 2.16.0", - "first_patched_version": "2.16.0", - "vulnerable_functions": [] - }, - { - "package": { - "ecosystem": "maven", - "name": "org.apache.logging.log4j:log4j-core" - }, - "vulnerable_version_range": "\u003C 2.12.2", - "first_patched_version": "2.12.2", - "vulnerable_functions": [] - }, - { - "package": { - "ecosystem": "maven", - "name": "org.ops4j.pax.logging:pax-logging-log4j2" - }, - "vulnerable_version_range": "\u003E= 1.8.0, \u003C 1.9.2", - "first_patched_version": "1.9.2", - "vulnerable_functions": [] - }, - { - "package": { - "ecosystem": "maven", - "name": "org.ops4j.pax.logging:pax-logging-log4j2" - }, - "vulnerable_version_range": "\u003E= 1.10.0, \u003C 1.10.8", - "first_patched_version": "1.10.8", - "vulnerable_functions": [] - }, - { - "package": { - "ecosystem": "maven", - "name": "org.ops4j.pax.logging:pax-logging-log4j2" - }, - "vulnerable_version_range": "\u003E= 1.11.0, \u003C 1.11.11", - "first_patched_version": "1.11.11", - "vulnerable_functions": [] - }, - { - "package": { - "ecosystem": "maven", - "name": "org.ops4j.pax.logging:pax-logging-log4j2" - }, - "vulnerable_version_range": "\u003E= 2.0.0, \u003C 2.0.12", - "first_patched_version": "2.0.12", - "vulnerable_functions": [] - } - ], - "cvss_severities": { - "cvss_v3": { - "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", - "score": 9.1 - }, - "cvss_v4": { - "vector_string": null, - "score": 0.0 - } - }, - "cwes": [ - { - "cwe_id": "CWE-502", - "name": "Deserialization of Untrusted Data" - }, - { - "cwe_id": "CWE-917", - "name": "Improper Neutralization of Special Elements used in an Expression Language Statement (\u0027Expression Language Injection\u0027)" - } - ], - "credits": [ - { - "user": { - "login": "mrjonstrong", - "id": 42520909, - "node_id": "MDQ6VXNlcjQyNTIwOTA5", - "avatar_url": "https://avatars.githubusercontent.com/u/42520909?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/mrjonstrong", - "html_url": "https://github.com/mrjonstrong", - "followers_url": "https://api.github.com/users/mrjonstrong/followers", - "following_url": "https://api.github.com/users/mrjonstrong/following{/other_user}", - "gists_url": "https://api.github.com/users/mrjonstrong/gists{/gist_id}", - "starred_url": "https://api.github.com/users/mrjonstrong/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/mrjonstrong/subscriptions", - "organizations_url": "https://api.github.com/users/mrjonstrong/orgs", - "repos_url": "https://api.github.com/users/mrjonstrong/repos", - "events_url": "https://api.github.com/users/mrjonstrong/events{/privacy}", - "received_events_url": "https://api.github.com/users/mrjonstrong/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "analyst" - }, - { - "user": { - "login": "afdesk", - "id": 19297627, - "node_id": "MDQ6VXNlcjE5Mjk3NjI3", - "avatar_url": "https://avatars.githubusercontent.com/u/19297627?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/afdesk", - "html_url": "https://github.com/afdesk", - "followers_url": "https://api.github.com/users/afdesk/followers", - "following_url": "https://api.github.com/users/afdesk/following{/other_user}", - "gists_url": "https://api.github.com/users/afdesk/gists{/gist_id}", - "starred_url": "https://api.github.com/users/afdesk/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/afdesk/subscriptions", - "organizations_url": "https://api.github.com/users/afdesk/orgs", - "repos_url": "https://api.github.com/users/afdesk/repos", - "events_url": "https://api.github.com/users/afdesk/events{/privacy}", - "received_events_url": "https://api.github.com/users/afdesk/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "analyst" - }, - { - "user": { - "login": "ppkarwasz", - "id": 12533274, - "node_id": "MDQ6VXNlcjEyNTMzMjc0", - "avatar_url": "https://avatars.githubusercontent.com/u/12533274?v=4", - "gravatar_id": "", - "url": "https://api.github.com/users/ppkarwasz", - "html_url": "https://github.com/ppkarwasz", - "followers_url": "https://api.github.com/users/ppkarwasz/followers", - "following_url": "https://api.github.com/users/ppkarwasz/following{/other_user}", - "gists_url": "https://api.github.com/users/ppkarwasz/gists{/gist_id}", - "starred_url": "https://api.github.com/users/ppkarwasz/starred{/owner}{/repo}", - "subscriptions_url": "https://api.github.com/users/ppkarwasz/subscriptions", - "organizations_url": "https://api.github.com/users/ppkarwasz/orgs", - "repos_url": "https://api.github.com/users/ppkarwasz/repos", - "events_url": "https://api.github.com/users/ppkarwasz/events{/privacy}", - "received_events_url": "https://api.github.com/users/ppkarwasz/received_events", - "type": "User", - "user_view_type": "public", - "site_admin": false - }, - "type": "analyst" - } - ], - "cvss": { - "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", - "score": 9.1 - }, - "epss": { - "percentage": 0.9434, - "percentile": 0.9995 - } - } +[ + { + "ghsa_id": "GHSA-wv4w-6qv2-qqfg", + "cve_id": "CVE-2025-61783", + "url": "https://api.github.com/advisories/GHSA-wv4w-6qv2-qqfg", + "html_url": "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg", + "summary": "Python Social Auth - Django has unsafe account association ", + "description": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "type": "reviewed", + "severity": "medium", + "repository_advisory_url": "https://api.github.com/repos/python-social-auth/social-app-django/security-advisories/GHSA-wv4w-6qv2-qqfg", + "source_code_location": "https://github.com/python-social-auth/social-app-django", + "identifiers": [ + { + "value": "GHSA-wv4w-6qv2-qqfg", + "type": "GHSA" + }, + { + "value": "CVE-2025-61783", + "type": "CVE" + } + ], + "references": [ + "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg", + "https://github.com/python-social-auth/social-app-django/issues/220", + "https://github.com/python-social-auth/social-app-django/issues/231", + "https://github.com/python-social-auth/social-app-django/issues/634", + "https://github.com/python-social-auth/social-app-django/pull/803", + "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c", + "https://github.com/advisories/GHSA-wv4w-6qv2-qqfg" + ], + "published_at": "2025-10-09T17:08:05Z", + "updated_at": "2025-10-09T17:08:06Z", + "github_reviewed_at": "2025-10-09T17:08:05Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "social-auth-app-django" + }, + "vulnerable_version_range": "\u003C 5.6.0", + "first_patched_version": "5.6.0", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": null, + "score": 0.0 + }, + "cvss_v4": { + "vector_string": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N", + "score": 6.3 + } + }, + "cwes": [ + { + "cwe_id": "CWE-290", + "name": "Authentication Bypass by Spoofing" + } + ], + "credits": [ + { + "user": { + "login": "mel-mason", + "id": 19391457, + "node_id": "MDQ6VXNlcjE5MzkxNDU3", + "avatar_url": "https://avatars.githubusercontent.com/u/19391457?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mel-mason", + "html_url": "https://github.com/mel-mason", + "followers_url": "https://api.github.com/users/mel-mason/followers", + "following_url": "https://api.github.com/users/mel-mason/following{/other_user}", + "gists_url": "https://api.github.com/users/mel-mason/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mel-mason/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mel-mason/subscriptions", + "organizations_url": "https://api.github.com/users/mel-mason/orgs", + "repos_url": "https://api.github.com/users/mel-mason/repos", + "events_url": "https://api.github.com/users/mel-mason/events{/privacy}", + "received_events_url": "https://api.github.com/users/mel-mason/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + }, + { + "user": { + "login": "vanya909", + "id": 53380238, + "node_id": "MDQ6VXNlcjUzMzgwMjM4", + "avatar_url": "https://avatars.githubusercontent.com/u/53380238?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/vanya909", + "html_url": "https://github.com/vanya909", + "followers_url": "https://api.github.com/users/vanya909/followers", + "following_url": "https://api.github.com/users/vanya909/following{/other_user}", + "gists_url": "https://api.github.com/users/vanya909/gists{/gist_id}", + "starred_url": "https://api.github.com/users/vanya909/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/vanya909/subscriptions", + "organizations_url": "https://api.github.com/users/vanya909/orgs", + "repos_url": "https://api.github.com/users/vanya909/repos", + "events_url": "https://api.github.com/users/vanya909/events{/privacy}", + "received_events_url": "https://api.github.com/users/vanya909/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + }, + { + "user": { + "login": "nijel", + "id": 212189, + "node_id": "MDQ6VXNlcjIxMjE4OQ==", + "avatar_url": "https://avatars.githubusercontent.com/u/212189?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/nijel", + "html_url": "https://github.com/nijel", + "followers_url": "https://api.github.com/users/nijel/followers", + "following_url": "https://api.github.com/users/nijel/following{/other_user}", + "gists_url": "https://api.github.com/users/nijel/gists{/gist_id}", + "starred_url": "https://api.github.com/users/nijel/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/nijel/subscriptions", + "organizations_url": "https://api.github.com/users/nijel/orgs", + "repos_url": "https://api.github.com/users/nijel/repos", + "events_url": "https://api.github.com/users/nijel/events{/privacy}", + "received_events_url": "https://api.github.com/users/nijel/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "remediation_developer" + } + ], + "cvss": { + "vector_string": null, + "score": null + } + }, + { + "ghsa_id": "GHSA-cjjf-27cc-pvmv", + "cve_id": "CVE-2025-61773", + "url": "https://api.github.com/advisories/GHSA-cjjf-27cc-pvmv", + "html_url": "https://github.com/advisories/GHSA-cjjf-27cc-pvmv", + "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", + "description": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "type": "reviewed", + "severity": "high", + "repository_advisory_url": "https://api.github.com/repos/pyload/pyload/security-advisories/GHSA-cjjf-27cc-pvmv", + "source_code_location": "https://github.com/pyload/pyload", + "identifiers": [ + { + "value": "GHSA-cjjf-27cc-pvmv", + "type": "GHSA" + }, + { + "value": "CVE-2025-61773", + "type": "CVE" + } + ], + "references": [ + "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv", + "https://github.com/pyload/pyload/pull/4624", + "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca", + "https://github.com/advisories/GHSA-cjjf-27cc-pvmv" + ], + "published_at": "2025-10-09T15:19:48Z", + "updated_at": "2025-10-09T15:19:48Z", + "github_reviewed_at": "2025-10-09T15:19:48Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "pip", + "name": "pyload-ng" + }, + "vulnerable_version_range": "\u003C 0.5.0b3.dev91", + "first_patched_version": "0.5.0b3.dev91", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "score": 8.1 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-74", + "name": "Improper Neutralization of Special Elements in Output Used by a Downstream Component (\u0027Injection\u0027)" + }, + { + "cwe_id": "CWE-79", + "name": "Improper Neutralization of Input During Web Page Generation (\u0027Cross-site Scripting\u0027)" + }, + { + "cwe_id": "CWE-94", + "name": "Improper Control of Generation of Code (\u0027Code Injection\u0027)" + }, + { + "cwe_id": "CWE-116", + "name": "Improper Encoding or Escaping of Output" + } + ], + "credits": [ + { + "user": { + "login": "odaysec", + "id": 47859767, + "node_id": "MDQ6VXNlcjQ3ODU5NzY3", + "avatar_url": "https://avatars.githubusercontent.com/u/47859767?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/odaysec", + "html_url": "https://github.com/odaysec", + "followers_url": "https://api.github.com/users/odaysec/followers", + "following_url": "https://api.github.com/users/odaysec/following{/other_user}", + "gists_url": "https://api.github.com/users/odaysec/gists{/gist_id}", + "starred_url": "https://api.github.com/users/odaysec/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/odaysec/subscriptions", + "organizations_url": "https://api.github.com/users/odaysec/orgs", + "repos_url": "https://api.github.com/users/odaysec/repos", + "events_url": "https://api.github.com/users/odaysec/events{/privacy}", + "received_events_url": "https://api.github.com/users/odaysec/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "reporter" + } + ], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N", + "score": 8.1 + } + }, + { + "ghsa_id": "GHSA-77vh-xpmg-72qh", + "cve_id": null, + "url": "https://api.github.com/advisories/GHSA-77vh-xpmg-72qh", + "html_url": "https://github.com/advisories/GHSA-77vh-xpmg-72qh", + "summary": "Clarify \u0060mediaType\u0060 handling", + "description": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", + "type": "reviewed", + "severity": "low", + "repository_advisory_url": "https://api.github.com/repos/opencontainers/image-spec/security-advisories/GHSA-77vh-xpmg-72qh", + "source_code_location": "https://github.com/opencontainers/image-spec", + "identifiers": [ + { + "value": "GHSA-77vh-xpmg-72qh", + "type": "GHSA" + } + ], + "references": [ + "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m", + "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh", + "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c", + "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2", + "https://github.com/advisories/GHSA-77vh-xpmg-72qh" + ], + "published_at": "2021-11-18T16:02:41Z", + "updated_at": "2023-01-09T05:05:32Z", + "github_reviewed_at": "2021-11-17T23:13:41Z", + "nvd_published_at": null, + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "go", + "name": "github.com/opencontainers/image-spec" + }, + "vulnerable_version_range": "\u003C 1.0.2", + "first_patched_version": "1.0.2", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "score": 3.0 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-843", + "name": "Access of Resource Using Incompatible Type (\u0027Type Confusion\u0027)" + } + ], + "credits": [], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N", + "score": 3.0 + } + }, + { + "ghsa_id": "GHSA-7rjr-3q55-vv33", + "cve_id": "CVE-2021-45046", + "url": "https://api.github.com/advisories/GHSA-7rjr-3q55-vv33", + "html_url": "https://github.com/advisories/GHSA-7rjr-3q55-vv33", + "summary": "Incomplete fix for Apache Log4j vulnerability", + "description": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", + "type": "reviewed", + "severity": "critical", + "repository_advisory_url": null, + "source_code_location": "", + "identifiers": [ + { + "value": "GHSA-7rjr-3q55-vv33", + "type": "GHSA" + }, + { + "value": "CVE-2021-45046", + "type": "CVE" + } + ], + "references": [ + "https://nvd.nist.gov/vuln/detail/CVE-2021-45046", + "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q", + "https://logging.apache.org/log4j/2.x/security.html", + "https://www.openwall.com/lists/oss-security/2021/12/14/4", + "https://www.cve.org/CVERecord?id=CVE-2021-44228", + "http://www.openwall.com/lists/oss-security/2021/12/14/4", + "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html", + "http://www.openwall.com/lists/oss-security/2021/12/15/3", + "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf", + "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf", + "https://www.kb.cert.org/vuls/id/930724", + "https://www.debian.org/security/2021/dsa-5022", + "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032", + "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html", + "http://www.openwall.com/lists/oss-security/2021/12/18/1", + "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf", + "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf", + "https://www.oracle.com/security-alerts/cpujan2022.html", + "https://www.oracle.com/security-alerts/cpuapr2022.html", + "https://www.oracle.com/security-alerts/cpujul2022.html", + "https://security.gentoo.org/glsa/202310-16", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ", + "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY", + "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd", + "https://github.com/advisories/GHSA-7rjr-3q55-vv33" + ], + "published_at": "2021-12-14T18:01:28Z", + "updated_at": "2025-05-09T12:28:41Z", + "github_reviewed_at": "2021-12-14T17:55:00Z", + "nvd_published_at": "2021-12-14T19:15:00Z", + "withdrawn_at": null, + "vulnerabilities": [ + { + "package": { + "ecosystem": "maven", + "name": "org.apache.logging.log4j:log4j-core" + }, + "vulnerable_version_range": "\u003E= 2.13.0, \u003C 2.16.0", + "first_patched_version": "2.16.0", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.apache.logging.log4j:log4j-core" + }, + "vulnerable_version_range": "\u003C 2.12.2", + "first_patched_version": "2.12.2", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.8.0, \u003C 1.9.2", + "first_patched_version": "1.9.2", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.10.0, \u003C 1.10.8", + "first_patched_version": "1.10.8", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 1.11.0, \u003C 1.11.11", + "first_patched_version": "1.11.11", + "vulnerable_functions": [] + }, + { + "package": { + "ecosystem": "maven", + "name": "org.ops4j.pax.logging:pax-logging-log4j2" + }, + "vulnerable_version_range": "\u003E= 2.0.0, \u003C 2.0.12", + "first_patched_version": "2.0.12", + "vulnerable_functions": [] + } + ], + "cvss_severities": { + "cvss_v3": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "score": 9.1 + }, + "cvss_v4": { + "vector_string": null, + "score": 0.0 + } + }, + "cwes": [ + { + "cwe_id": "CWE-502", + "name": "Deserialization of Untrusted Data" + }, + { + "cwe_id": "CWE-917", + "name": "Improper Neutralization of Special Elements used in an Expression Language Statement (\u0027Expression Language Injection\u0027)" + } + ], + "credits": [ + { + "user": { + "login": "mrjonstrong", + "id": 42520909, + "node_id": "MDQ6VXNlcjQyNTIwOTA5", + "avatar_url": "https://avatars.githubusercontent.com/u/42520909?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/mrjonstrong", + "html_url": "https://github.com/mrjonstrong", + "followers_url": "https://api.github.com/users/mrjonstrong/followers", + "following_url": "https://api.github.com/users/mrjonstrong/following{/other_user}", + "gists_url": "https://api.github.com/users/mrjonstrong/gists{/gist_id}", + "starred_url": "https://api.github.com/users/mrjonstrong/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/mrjonstrong/subscriptions", + "organizations_url": "https://api.github.com/users/mrjonstrong/orgs", + "repos_url": "https://api.github.com/users/mrjonstrong/repos", + "events_url": "https://api.github.com/users/mrjonstrong/events{/privacy}", + "received_events_url": "https://api.github.com/users/mrjonstrong/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + }, + { + "user": { + "login": "afdesk", + "id": 19297627, + "node_id": "MDQ6VXNlcjE5Mjk3NjI3", + "avatar_url": "https://avatars.githubusercontent.com/u/19297627?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/afdesk", + "html_url": "https://github.com/afdesk", + "followers_url": "https://api.github.com/users/afdesk/followers", + "following_url": "https://api.github.com/users/afdesk/following{/other_user}", + "gists_url": "https://api.github.com/users/afdesk/gists{/gist_id}", + "starred_url": "https://api.github.com/users/afdesk/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/afdesk/subscriptions", + "organizations_url": "https://api.github.com/users/afdesk/orgs", + "repos_url": "https://api.github.com/users/afdesk/repos", + "events_url": "https://api.github.com/users/afdesk/events{/privacy}", + "received_events_url": "https://api.github.com/users/afdesk/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + }, + { + "user": { + "login": "ppkarwasz", + "id": 12533274, + "node_id": "MDQ6VXNlcjEyNTMzMjc0", + "avatar_url": "https://avatars.githubusercontent.com/u/12533274?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/ppkarwasz", + "html_url": "https://github.com/ppkarwasz", + "followers_url": "https://api.github.com/users/ppkarwasz/followers", + "following_url": "https://api.github.com/users/ppkarwasz/following{/other_user}", + "gists_url": "https://api.github.com/users/ppkarwasz/gists{/gist_id}", + "starred_url": "https://api.github.com/users/ppkarwasz/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/ppkarwasz/subscriptions", + "organizations_url": "https://api.github.com/users/ppkarwasz/orgs", + "repos_url": "https://api.github.com/users/ppkarwasz/repos", + "events_url": "https://api.github.com/users/ppkarwasz/events{/privacy}", + "received_events_url": "https://api.github.com/users/ppkarwasz/received_events", + "type": "User", + "user_view_type": "public", + "site_admin": false + }, + "type": "analyst" + } + ], + "cvss": { + "vector_string": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H", + "score": 9.1 + }, + "epss": { + "percentage": 0.9434, + "percentile": 0.9995 + } + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json similarity index 97% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json index bea1df2e..b48295ab 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-ghsa.raw-osv.json @@ -1,714 +1,714 @@ -[ - { - "id": "GHSA-wv4w-6qv2-qqfg", - "summary": "Python Social Auth - Django has unsafe account association ", - "details": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", - "aliases": [ - "CVE-2025-61783" - ], - "modified": "2025-10-09T17:57:29.916841Z", - "published": "2025-10-09T17:08:05Z", - "database_specific": { - "github_reviewed_at": "2025-10-09T17:08:05Z", - "severity": "MODERATE", - "cwe_ids": [ - "CWE-290" - ], - "github_reviewed": true, - "nvd_published_at": null - }, - "references": [ - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" - }, - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/issues/220" - }, - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/issues/231" - }, - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/issues/634" - }, - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/pull/803" - }, - { - "type": "WEB", - "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" - }, - { - "type": "PACKAGE", - "url": "https://github.com/python-social-auth/social-app-django" - } - ], - "affected": [ - { - "package": { - "name": "social-auth-app-django", - "ecosystem": "PyPI", - "purl": "pkg:pypi/social-auth-app-django" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "5.6.0" - } - ] - } - ], - "versions": [ - "0.0.1", - "0.1.0", - "1.0.0", - "1.0.1", - "1.1.0", - "1.2.0", - "2.0.0", - "2.1.0", - "3.0.0", - "3.1.0", - "3.3.0", - "3.4.0", - "4.0.0", - "5.0.0", - "5.1.0", - "5.2.0", - "5.3.0", - "5.4.0", - "5.4.1", - "5.4.2", - "5.4.3", - "5.5.0", - "5.5.1" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-wv4w-6qv2-qqfg/GHSA-wv4w-6qv2-qqfg.json" - } - } - ], - "schema_version": "1.7.3", - "severity": [ - { - "type": "CVSS_V4", - "score": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N" - } - ] - }, - { - "id": "GHSA-cjjf-27cc-pvmv", - "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", - "details": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", - "aliases": [ - "CVE-2025-61773" - ], - "modified": "2025-10-09T15:59:13.250015Z", - "published": "2025-10-09T15:19:48Z", - "database_specific": { - "github_reviewed_at": "2025-10-09T15:19:48Z", - "github_reviewed": true, - "cwe_ids": [ - "CWE-116", - "CWE-74", - "CWE-79", - "CWE-94" - ], - "severity": "HIGH", - "nvd_published_at": null - }, - "references": [ - { - "type": "WEB", - "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" - }, - { - "type": "WEB", - "url": "https://github.com/pyload/pyload/pull/4624" - }, - { - "type": "WEB", - "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" - }, - { - "type": "PACKAGE", - "url": "https://github.com/pyload/pyload" - } - ], - "affected": [ - { - "package": { - "name": "pyload-ng", - "ecosystem": "PyPI", - "purl": "pkg:pypi/pyload-ng" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "0.5.0b3.dev91" - } - ] - } - ], - "versions": [ - "0.5.0a5.dev528", - "0.5.0a5.dev532", - "0.5.0a5.dev535", - "0.5.0a5.dev536", - "0.5.0a5.dev537", - "0.5.0a5.dev539", - "0.5.0a5.dev540", - "0.5.0a5.dev545", - "0.5.0a5.dev562", - "0.5.0a5.dev564", - "0.5.0a5.dev565", - "0.5.0a6.dev570", - "0.5.0a6.dev578", - "0.5.0a6.dev587", - "0.5.0a7.dev596", - "0.5.0a8.dev602", - "0.5.0a9.dev615", - "0.5.0a9.dev629", - "0.5.0a9.dev632", - "0.5.0a9.dev641", - "0.5.0a9.dev643", - "0.5.0a9.dev655", - "0.5.0a9.dev806", - "0.5.0b1.dev1", - "0.5.0b1.dev2", - "0.5.0b1.dev3", - "0.5.0b1.dev4", - "0.5.0b1.dev5", - "0.5.0b2.dev10", - "0.5.0b2.dev11", - "0.5.0b2.dev12", - "0.5.0b2.dev9", - "0.5.0b3.dev13", - "0.5.0b3.dev14", - "0.5.0b3.dev17", - "0.5.0b3.dev18", - "0.5.0b3.dev19", - "0.5.0b3.dev20", - "0.5.0b3.dev21", - "0.5.0b3.dev22", - "0.5.0b3.dev24", - "0.5.0b3.dev26", - "0.5.0b3.dev27", - "0.5.0b3.dev28", - "0.5.0b3.dev29", - "0.5.0b3.dev30", - "0.5.0b3.dev31", - "0.5.0b3.dev32", - "0.5.0b3.dev33", - "0.5.0b3.dev34", - "0.5.0b3.dev35", - "0.5.0b3.dev38", - "0.5.0b3.dev39", - "0.5.0b3.dev40", - "0.5.0b3.dev41", - "0.5.0b3.dev42", - "0.5.0b3.dev43", - "0.5.0b3.dev44", - "0.5.0b3.dev45", - "0.5.0b3.dev46", - "0.5.0b3.dev47", - "0.5.0b3.dev48", - "0.5.0b3.dev49", - "0.5.0b3.dev50", - "0.5.0b3.dev51", - "0.5.0b3.dev52", - "0.5.0b3.dev53", - "0.5.0b3.dev54", - "0.5.0b3.dev57", - "0.5.0b3.dev60", - "0.5.0b3.dev62", - "0.5.0b3.dev64", - "0.5.0b3.dev65", - "0.5.0b3.dev66", - "0.5.0b3.dev67", - "0.5.0b3.dev68", - "0.5.0b3.dev69", - "0.5.0b3.dev70", - "0.5.0b3.dev71", - "0.5.0b3.dev72", - "0.5.0b3.dev73", - "0.5.0b3.dev74", - "0.5.0b3.dev75", - "0.5.0b3.dev76", - "0.5.0b3.dev77", - "0.5.0b3.dev78", - "0.5.0b3.dev79", - "0.5.0b3.dev80", - "0.5.0b3.dev81", - "0.5.0b3.dev82", - "0.5.0b3.dev85", - "0.5.0b3.dev87", - "0.5.0b3.dev88", - "0.5.0b3.dev89", - "0.5.0b3.dev90" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-cjjf-27cc-pvmv/GHSA-cjjf-27cc-pvmv.json" - } - } - ], - "schema_version": "1.7.3", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N" - } - ] - }, - { - "id": "GHSA-77vh-xpmg-72qh", - "summary": "Clarify \u0060mediaType\u0060 handling", - "details": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", - "modified": "2021-11-24T19:43:35Z", - "published": "2021-11-18T16:02:41Z", - "related": [ - "CGA-j36r-723f-8c29" - ], - "database_specific": { - "github_reviewed": true, - "nvd_published_at": null, - "github_reviewed_at": "2021-11-17T23:13:41Z", - "cwe_ids": [ - "CWE-843" - ], - "severity": "LOW" - }, - "references": [ - { - "type": "WEB", - "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" - }, - { - "type": "WEB", - "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" - }, - { - "type": "WEB", - "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" - }, - { - "type": "PACKAGE", - "url": "https://github.com/opencontainers/image-spec" - }, - { - "type": "WEB", - "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" - } - ], - "affected": [ - { - "package": { - "name": "github.com/opencontainers/image-spec", - "ecosystem": "Go", - "purl": "pkg:golang/github.com/opencontainers/image-spec" - }, - "ranges": [ - { - "type": "SEMVER", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "1.0.2" - } - ] - } - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/11/GHSA-77vh-xpmg-72qh/GHSA-77vh-xpmg-72qh.json" - } - } - ], - "schema_version": "1.7.3", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N" - } - ] - }, - { - "id": "GHSA-7rjr-3q55-vv33", - "summary": "Incomplete fix for Apache Log4j vulnerability", - "details": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", - "aliases": [ - "CVE-2021-45046" - ], - "modified": "2025-05-09T13:13:16.169374Z", - "published": "2021-12-14T18:01:28Z", - "database_specific": { - "github_reviewed_at": "2021-12-14T17:55:00Z", - "cwe_ids": [ - "CWE-502", - "CWE-917" - ], - "github_reviewed": true, - "severity": "CRITICAL", - "nvd_published_at": "2021-12-14T19:15:00Z" - }, - "references": [ - { - "type": "ADVISORY", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/cpujul2022.html" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/cpujan2022.html" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" - }, - { - "type": "WEB", - "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" - }, - { - "type": "WEB", - "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" - }, - { - "type": "WEB", - "url": "https://www.kb.cert.org/vuls/id/930724" - }, - { - "type": "WEB", - "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" - }, - { - "type": "WEB", - "url": "https://www.debian.org/security/2021/dsa-5022" - }, - { - "type": "WEB", - "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" - }, - { - "type": "WEB", - "url": "https://security.gentoo.org/glsa/202310-16" - }, - { - "type": "WEB", - "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" - }, - { - "type": "WEB", - "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" - }, - { - "type": "WEB", - "url": "https://logging.apache.org/log4j/2.x/security.html" - }, - { - "type": "WEB", - "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" - }, - { - "type": "WEB", - "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" - }, - { - "type": "ADVISORY", - "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" - }, - { - "type": "WEB", - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" - }, - { - "type": "WEB", - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" - }, - { - "type": "WEB", - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" - }, - { - "type": "WEB", - "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" - }, - { - "type": "WEB", - "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" - }, - { - "type": "WEB", - "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" - }, - { - "type": "WEB", - "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" - } - ], - "affected": [ - { - "package": { - "name": "org.apache.logging.log4j:log4j-core", - "ecosystem": "Maven", - "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "2.13.0" - }, - { - "fixed": "2.16.0" - } - ] - } - ], - "versions": [ - "2.13.0", - "2.13.1", - "2.13.2", - "2.13.3", - "2.14.0", - "2.14.1", - "2.15.0" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - }, - { - "package": { - "name": "org.apache.logging.log4j:log4j-core", - "ecosystem": "Maven", - "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "0" - }, - { - "fixed": "2.12.2" - } - ] - } - ], - "versions": [ - "2.0", - "2.0-alpha1", - "2.0-alpha2", - "2.0-beta1", - "2.0-beta2", - "2.0-beta3", - "2.0-beta4", - "2.0-beta5", - "2.0-beta6", - "2.0-beta7", - "2.0-beta8", - "2.0-beta9", - "2.0-rc1", - "2.0-rc2", - "2.0.1", - "2.0.2", - "2.1", - "2.10.0", - "2.11.0", - "2.11.1", - "2.11.2", - "2.12.0", - "2.12.1", - "2.2", - "2.3", - "2.3.1", - "2.3.2", - "2.4", - "2.4.1", - "2.5", - "2.6", - "2.6.1", - "2.6.2", - "2.7", - "2.8", - "2.8.1", - "2.8.2", - "2.9.0", - "2.9.1" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - }, - { - "package": { - "name": "org.ops4j.pax.logging:pax-logging-log4j2", - "ecosystem": "Maven", - "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "1.8.0" - }, - { - "fixed": "1.9.2" - } - ] - } - ], - "versions": [ - "1.8.0", - "1.8.1", - "1.8.2", - "1.8.3", - "1.8.4", - "1.8.5", - "1.8.6", - "1.8.7", - "1.9.0", - "1.9.1" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - }, - { - "package": { - "name": "org.ops4j.pax.logging:pax-logging-log4j2", - "ecosystem": "Maven", - "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "1.10.0" - }, - { - "fixed": "1.10.8" - } - ] - } - ], - "versions": [ - "1.10.0", - "1.10.1", - "1.10.2", - "1.10.3", - "1.10.4", - "1.10.5", - "1.10.6", - "1.10.7" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - }, - { - "package": { - "name": "org.ops4j.pax.logging:pax-logging-log4j2", - "ecosystem": "Maven", - "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "1.11.0" - }, - { - "fixed": "1.11.11" - } - ] - } - ], - "versions": [ - "1.11.0", - "1.11.1", - "1.11.10", - "1.11.2", - "1.11.3", - "1.11.4", - "1.11.5", - "1.11.6", - "1.11.7", - "1.11.8", - "1.11.9" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - }, - { - "package": { - "name": "org.ops4j.pax.logging:pax-logging-log4j2", - "ecosystem": "Maven", - "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" - }, - "ranges": [ - { - "type": "ECOSYSTEM", - "events": [ - { - "introduced": "2.0.0" - }, - { - "fixed": "2.0.12" - } - ] - } - ], - "versions": [ - "2.0.0", - "2.0.1", - "2.0.10", - "2.0.11", - "2.0.2", - "2.0.3", - "2.0.4", - "2.0.5", - "2.0.6", - "2.0.7", - "2.0.8", - "2.0.9" - ], - "database_specific": { - "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" - } - } - ], - "schema_version": "1.7.3", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H" - } - ] - } +[ + { + "id": "GHSA-wv4w-6qv2-qqfg", + "summary": "Python Social Auth - Django has unsafe account association ", + "details": "### Impact\n\nUpon authentication, the user could be associated by e-mail even if the \u0060associate_by_email\u0060 pipeline was not included. This could lead to account compromise when a third-party authentication service does not validate provided e-mail addresses or doesn\u0027t require unique e-mail addresses.\n\n### Patches\n\n* https://github.com/python-social-auth/social-app-django/pull/803\n\n### Workarounds\n\nReview the authentication service policy on e-mail addresses; many will not allow exploiting this vulnerability.", + "aliases": [ + "CVE-2025-61783" + ], + "modified": "2025-10-09T17:57:29.916841Z", + "published": "2025-10-09T17:08:05Z", + "database_specific": { + "github_reviewed_at": "2025-10-09T17:08:05Z", + "severity": "MODERATE", + "cwe_ids": [ + "CWE-290" + ], + "github_reviewed": true, + "nvd_published_at": null + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/security/advisories/GHSA-wv4w-6qv2-qqfg" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/220" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/231" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/issues/634" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/pull/803" + }, + { + "type": "WEB", + "url": "https://github.com/python-social-auth/social-app-django/commit/10c80e2ebabeccd4e9c84ad0e16e1db74148ed4c" + }, + { + "type": "PACKAGE", + "url": "https://github.com/python-social-auth/social-app-django" + } + ], + "affected": [ + { + "package": { + "name": "social-auth-app-django", + "ecosystem": "PyPI", + "purl": "pkg:pypi/social-auth-app-django" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "5.6.0" + } + ] + } + ], + "versions": [ + "0.0.1", + "0.1.0", + "1.0.0", + "1.0.1", + "1.1.0", + "1.2.0", + "2.0.0", + "2.1.0", + "3.0.0", + "3.1.0", + "3.3.0", + "3.4.0", + "4.0.0", + "5.0.0", + "5.1.0", + "5.2.0", + "5.3.0", + "5.4.0", + "5.4.1", + "5.4.2", + "5.4.3", + "5.5.0", + "5.5.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-wv4w-6qv2-qqfg/GHSA-wv4w-6qv2-qqfg.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V4", + "score": "CVSS:4.0/AV:N/AC:H/AT:N/PR:N/UI:N/VC:L/VI:L/VA:N/SC:N/SI:N/SA:N" + } + ] + }, + { + "id": "GHSA-cjjf-27cc-pvmv", + "summary": "pyLoad CNL and captcha handlers allow Code Injection via unsanitized parameters", + "details": "### Summary\npyLoad web interface contained insufficient input validation in both the Captcha script endpoint and the Click\u0027N\u0027Load (CNL) Blueprint. This flaw allowed untrusted user input to be processed unsafely, which could be exploited by an attacker to inject arbitrary content into the web UI or manipulate request handling. The vulnerability could lead to client-side code execution (XSS) or other unintended behaviors when a malicious payload is submitted.\n\nuser-supplied parameters from HTTP requests were not adequately validated or sanitized before being passed into the application logic and response generation. This allowed crafted input to alter the expected execution flow.\n CNL (Click\u0027N\u0027Load) blueprint exposed unsafe handling of untrusted parameters in HTTP requests. The application did not consistently enforce input validation or encoding, making it possible for an attacker to craft malicious requests.\n\n### PoC\n\n1. Run a vulnerable version of pyLoad prior to commit [\u0060f9d27f2\u0060](https://github.com/pyload/pyload/pull/4624).\n2. Start the web UI and access the Captcha or CNL endpoints.\n3. Submit a crafted request containing malicious JavaScript payloads in unvalidated parameters (\u0060/flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345\u0060).\n4. Observe that the payload is reflected and executed in the client\u2019s browser, demonstrating cross-site scripting (XSS).\n\nExample request:\n\n\u0060\u0060\u0060http\nGET /flash/addcrypted2?jk=function(){alert(1)}\u0026crypted=12345 HTTP/1.1\nHost: 127.0.0.1:8000\nContent-Type: application/x-www-form-urlencoded\nContent-Length: 107\n\u0060\u0060\u0060\n\n### Impact\n\nExploiting this vulnerability allows an attacker to inject and execute arbitrary JavaScript within the browser session of a user accessing the pyLoad Web UI. In practice, this means an attacker could impersonate an administrator, steal authentication cookies or tokens, and perform unauthorized actions on behalf of the victim. Because the affected endpoints are part of the core interface, a successful attack undermines the trust and security of the entire application, potentially leading to a full compromise of the management interface and the data it controls. The impact is particularly severe in cases where the Web UI is exposed over a network without additional access restrictions, as it enables remote attackers to directly target users with crafted links or requests that trigger the vulnerability.", + "aliases": [ + "CVE-2025-61773" + ], + "modified": "2025-10-09T15:59:13.250015Z", + "published": "2025-10-09T15:19:48Z", + "database_specific": { + "github_reviewed_at": "2025-10-09T15:19:48Z", + "github_reviewed": true, + "cwe_ids": [ + "CWE-116", + "CWE-74", + "CWE-79", + "CWE-94" + ], + "severity": "HIGH", + "nvd_published_at": null + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/security/advisories/GHSA-cjjf-27cc-pvmv" + }, + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/pull/4624" + }, + { + "type": "WEB", + "url": "https://github.com/pyload/pyload/commit/5823327d0b797161c7195a1f660266d30a69f0ca" + }, + { + "type": "PACKAGE", + "url": "https://github.com/pyload/pyload" + } + ], + "affected": [ + { + "package": { + "name": "pyload-ng", + "ecosystem": "PyPI", + "purl": "pkg:pypi/pyload-ng" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "0.5.0b3.dev91" + } + ] + } + ], + "versions": [ + "0.5.0a5.dev528", + "0.5.0a5.dev532", + "0.5.0a5.dev535", + "0.5.0a5.dev536", + "0.5.0a5.dev537", + "0.5.0a5.dev539", + "0.5.0a5.dev540", + "0.5.0a5.dev545", + "0.5.0a5.dev562", + "0.5.0a5.dev564", + "0.5.0a5.dev565", + "0.5.0a6.dev570", + "0.5.0a6.dev578", + "0.5.0a6.dev587", + "0.5.0a7.dev596", + "0.5.0a8.dev602", + "0.5.0a9.dev615", + "0.5.0a9.dev629", + "0.5.0a9.dev632", + "0.5.0a9.dev641", + "0.5.0a9.dev643", + "0.5.0a9.dev655", + "0.5.0a9.dev806", + "0.5.0b1.dev1", + "0.5.0b1.dev2", + "0.5.0b1.dev3", + "0.5.0b1.dev4", + "0.5.0b1.dev5", + "0.5.0b2.dev10", + "0.5.0b2.dev11", + "0.5.0b2.dev12", + "0.5.0b2.dev9", + "0.5.0b3.dev13", + "0.5.0b3.dev14", + "0.5.0b3.dev17", + "0.5.0b3.dev18", + "0.5.0b3.dev19", + "0.5.0b3.dev20", + "0.5.0b3.dev21", + "0.5.0b3.dev22", + "0.5.0b3.dev24", + "0.5.0b3.dev26", + "0.5.0b3.dev27", + "0.5.0b3.dev28", + "0.5.0b3.dev29", + "0.5.0b3.dev30", + "0.5.0b3.dev31", + "0.5.0b3.dev32", + "0.5.0b3.dev33", + "0.5.0b3.dev34", + "0.5.0b3.dev35", + "0.5.0b3.dev38", + "0.5.0b3.dev39", + "0.5.0b3.dev40", + "0.5.0b3.dev41", + "0.5.0b3.dev42", + "0.5.0b3.dev43", + "0.5.0b3.dev44", + "0.5.0b3.dev45", + "0.5.0b3.dev46", + "0.5.0b3.dev47", + "0.5.0b3.dev48", + "0.5.0b3.dev49", + "0.5.0b3.dev50", + "0.5.0b3.dev51", + "0.5.0b3.dev52", + "0.5.0b3.dev53", + "0.5.0b3.dev54", + "0.5.0b3.dev57", + "0.5.0b3.dev60", + "0.5.0b3.dev62", + "0.5.0b3.dev64", + "0.5.0b3.dev65", + "0.5.0b3.dev66", + "0.5.0b3.dev67", + "0.5.0b3.dev68", + "0.5.0b3.dev69", + "0.5.0b3.dev70", + "0.5.0b3.dev71", + "0.5.0b3.dev72", + "0.5.0b3.dev73", + "0.5.0b3.dev74", + "0.5.0b3.dev75", + "0.5.0b3.dev76", + "0.5.0b3.dev77", + "0.5.0b3.dev78", + "0.5.0b3.dev79", + "0.5.0b3.dev80", + "0.5.0b3.dev81", + "0.5.0b3.dev82", + "0.5.0b3.dev85", + "0.5.0b3.dev87", + "0.5.0b3.dev88", + "0.5.0b3.dev89", + "0.5.0b3.dev90" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2025/10/GHSA-cjjf-27cc-pvmv/GHSA-cjjf-27cc-pvmv.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:N" + } + ] + }, + { + "id": "GHSA-77vh-xpmg-72qh", + "summary": "Clarify \u0060mediaType\u0060 handling", + "details": "### Impact\nIn the OCI Image Specification version 1.0.1 and prior, manifest and index documents are not self-describing and documents with a single digest could be interpreted as either a manifest or an index.\n\n### Patches\nThe Image Specification will be updated to recommend that both manifest and index documents contain a \u0060mediaType\u0060 field to identify the type of document.\nRelease [v1.0.2](https://github.com/opencontainers/image-spec/releases/tag/v1.0.2) includes these updates.\n\n### Workarounds\nSoftware attempting to deserialize an ambiguous document may reject the document if it contains both \u201Cmanifests\u201D and \u201Clayers\u201D fields or \u201Cmanifests\u201D and \u201Cconfig\u201D fields.\n\n### References\nhttps://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m\n\n### For more information\nIf you have any questions or comments about this advisory:\n* Open an issue in https://github.com/opencontainers/image-spec\n* Email us at [security@opencontainers.org](mailto:security@opencontainers.org)\n* https://github.com/opencontainers/image-spec/commits/v1.0.2\n", + "modified": "2021-11-24T19:43:35Z", + "published": "2021-11-18T16:02:41Z", + "related": [ + "CGA-j36r-723f-8c29" + ], + "database_specific": { + "github_reviewed": true, + "nvd_published_at": null, + "github_reviewed_at": "2021-11-17T23:13:41Z", + "cwe_ids": [ + "CWE-843" + ], + "severity": "LOW" + }, + "references": [ + { + "type": "WEB", + "url": "https://github.com/opencontainers/distribution-spec/security/advisories/GHSA-mc8v-mgrf-8f4m" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/commit/693428a734f5bab1a84bd2f990d92ef1111cd60c" + }, + { + "type": "PACKAGE", + "url": "https://github.com/opencontainers/image-spec" + }, + { + "type": "WEB", + "url": "https://github.com/opencontainers/image-spec/releases/tag/v1.0.2" + } + ], + "affected": [ + { + "package": { + "name": "github.com/opencontainers/image-spec", + "ecosystem": "Go", + "purl": "pkg:golang/github.com/opencontainers/image-spec" + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.0.2" + } + ] + } + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/11/GHSA-77vh-xpmg-72qh/GHSA-77vh-xpmg-72qh.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:C/C:N/I:L/A:N" + } + ] + }, + { + "id": "GHSA-7rjr-3q55-vv33", + "summary": "Incomplete fix for Apache Log4j vulnerability", + "details": "# Impact\n\nThe fix to address [CVE-2021-44228](https://nvd.nist.gov/vuln/detail/CVE-2021-44228) in Apache Log4j 2.15.0 was incomplete in certain non-default configurations. This could allow attackers with control over Thread Context Map (MDC) input data when the logging configuration uses a non-default Pattern Layout with either a Context Lookup (for example, $${ctx:loginId}) or a Thread Context Map pattern (%X, %mdc, or %MDC) to craft malicious input data using a JNDI Lookup pattern resulting in a remote code execution (RCE) attack. \n\n## Affected packages\nOnly the \u0060org.apache.logging.log4j:log4j-core\u0060 package is directly affected by this vulnerability. The \u0060org.apache.logging.log4j:log4j-api\u0060 should be kept at the same version as the \u0060org.apache.logging.log4j:log4j-core\u0060 package to ensure compatability if in use.\n\n# Mitigation\n\nLog4j 2.16.0 fixes this issue by removing support for message lookup patterns and disabling JNDI functionality by default. This issue can be mitigated in prior releases (\u003C 2.16.0) by removing the JndiLookup class from the classpath (example: zip -q -d log4j-core-*.jar org/apache/logging/log4j/core/lookup/JndiLookup.class).\n\nLog4j 2.15.0 restricts JNDI LDAP lookups to localhost by default. Note that previous mitigations involving configuration such as to set the system property \u0060log4j2.formatMsgNoLookups\u0060 to \u0060true\u0060 do NOT mitigate this specific vulnerability.", + "aliases": [ + "CVE-2021-45046" + ], + "modified": "2025-05-09T13:13:16.169374Z", + "published": "2021-12-14T18:01:28Z", + "database_specific": { + "github_reviewed_at": "2021-12-14T17:55:00Z", + "cwe_ids": [ + "CWE-502", + "CWE-917" + ], + "github_reviewed": true, + "severity": "CRITICAL", + "nvd_published_at": "2021-12-14T19:15:00Z" + }, + "references": [ + { + "type": "ADVISORY", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-45046" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujul2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpujan2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/cpuapr2022.html" + }, + { + "type": "WEB", + "url": "https://www.oracle.com/security-alerts/alert-cve-2021-44228.html" + }, + { + "type": "WEB", + "url": "https://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "type": "WEB", + "url": "https://www.kb.cert.org/vuls/id/930724" + }, + { + "type": "WEB", + "url": "https://www.intel.com/content/www/us/en/security-center/advisory/intel-sa-00646.html" + }, + { + "type": "WEB", + "url": "https://www.debian.org/security/2021/dsa-5022" + }, + { + "type": "WEB", + "url": "https://www.cve.org/CVERecord?id=CVE-2021-44228" + }, + { + "type": "WEB", + "url": "https://security.gentoo.org/glsa/202310-16" + }, + { + "type": "WEB", + "url": "https://sec.cloudapps.cisco.com/security/center/content/CiscoSecurityAdvisory/cisco-sa-apache-log4j-qRuKNEbd" + }, + { + "type": "WEB", + "url": "https://psirt.global.sonicwall.com/vuln-detail/SNWLID-2021-0032" + }, + { + "type": "WEB", + "url": "https://logging.apache.org/log4j/2.x/security.html" + }, + { + "type": "WEB", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/SIG7FZULMNK2XF6FZRU4VWYDQXNMUGAJ" + }, + { + "type": "WEB", + "url": "https://lists.fedoraproject.org/archives/list/package-announce@lists.fedoraproject.org/message/EOKPQGV24RRBBI4TBZUDQMM4MEH7MXCY" + }, + { + "type": "ADVISORY", + "url": "https://github.com/advisories/GHSA-jfh8-c2jp-5v3q" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-714170.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-661247.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-479842.pdf" + }, + { + "type": "WEB", + "url": "https://cert-portal.siemens.com/productcert/pdf/ssa-397453.pdf" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/14/4" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/15/3" + }, + { + "type": "WEB", + "url": "http://www.openwall.com/lists/oss-security/2021/12/18/1" + } + ], + "affected": [ + { + "package": { + "name": "org.apache.logging.log4j:log4j-core", + "ecosystem": "Maven", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "2.13.0" + }, + { + "fixed": "2.16.0" + } + ] + } + ], + "versions": [ + "2.13.0", + "2.13.1", + "2.13.2", + "2.13.3", + "2.14.0", + "2.14.1", + "2.15.0" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.apache.logging.log4j:log4j-core", + "ecosystem": "Maven", + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "2.12.2" + } + ] + } + ], + "versions": [ + "2.0", + "2.0-alpha1", + "2.0-alpha2", + "2.0-beta1", + "2.0-beta2", + "2.0-beta3", + "2.0-beta4", + "2.0-beta5", + "2.0-beta6", + "2.0-beta7", + "2.0-beta8", + "2.0-beta9", + "2.0-rc1", + "2.0-rc2", + "2.0.1", + "2.0.2", + "2.1", + "2.10.0", + "2.11.0", + "2.11.1", + "2.11.2", + "2.12.0", + "2.12.1", + "2.2", + "2.3", + "2.3.1", + "2.3.2", + "2.4", + "2.4.1", + "2.5", + "2.6", + "2.6.1", + "2.6.2", + "2.7", + "2.8", + "2.8.1", + "2.8.2", + "2.9.0", + "2.9.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.8.0" + }, + { + "fixed": "1.9.2" + } + ] + } + ], + "versions": [ + "1.8.0", + "1.8.1", + "1.8.2", + "1.8.3", + "1.8.4", + "1.8.5", + "1.8.6", + "1.8.7", + "1.9.0", + "1.9.1" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.10.0" + }, + { + "fixed": "1.10.8" + } + ] + } + ], + "versions": [ + "1.10.0", + "1.10.1", + "1.10.2", + "1.10.3", + "1.10.4", + "1.10.5", + "1.10.6", + "1.10.7" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "1.11.0" + }, + { + "fixed": "1.11.11" + } + ] + } + ], + "versions": [ + "1.11.0", + "1.11.1", + "1.11.10", + "1.11.2", + "1.11.3", + "1.11.4", + "1.11.5", + "1.11.6", + "1.11.7", + "1.11.8", + "1.11.9" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + }, + { + "package": { + "name": "org.ops4j.pax.logging:pax-logging-log4j2", + "ecosystem": "Maven", + "purl": "pkg:maven/org.ops4j.pax.logging/pax-logging-log4j2" + }, + "ranges": [ + { + "type": "ECOSYSTEM", + "events": [ + { + "introduced": "2.0.0" + }, + { + "fixed": "2.0.12" + } + ] + } + ], + "versions": [ + "2.0.0", + "2.0.1", + "2.0.10", + "2.0.11", + "2.0.2", + "2.0.3", + "2.0.4", + "2.0.5", + "2.0.6", + "2.0.7", + "2.0.8", + "2.0.9" + ], + "database_specific": { + "source": "https://github.com/github/advisory-database/blob/main/advisories/github-reviewed/2021/12/GHSA-7rjr-3q55-vv33/GHSA-7rjr-3q55-vv33.json" + } + } + ], + "schema_version": "1.7.3", + "severity": [ + { + "type": "CVSS_V3", + "score": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:C/C:H/I:H/A:H" + } + ] + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-npm.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-npm.snapshot.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-npm.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json b/src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-pypi.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Osv.Tests/Fixtures/osv-pypi.snapshot.json rename to src/StellaOps.Concelier.Connector.Osv.Tests/Fixtures/osv-pypi.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvConflictFixtureTests.cs b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConflictFixtureTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvConflictFixtureTests.cs rename to src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConflictFixtureTests.cs index c0570590..79d6c69f 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvConflictFixtureTests.cs +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvConflictFixtureTests.cs @@ -1,118 +1,118 @@ -using System.Text.Json; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; - -namespace StellaOps.Feedser.Source.Osv.Tests; - -public sealed class OsvConflictFixtureTests -{ - [Fact] - public void ConflictFixture_MatchesSnapshot() - { - using var databaseSpecificDoc = JsonDocument.Parse("""{"severity":"medium"}"""); - - var dto = new OsvVulnerabilityDto - { - Id = "OSV-2025-4242", - Summary = "Container escape for conflict-package", - Details = "OSV captures the latest container escape details including patched version metadata.", - Aliases = new[] { "CVE-2025-4242", "GHSA-qqqq-wwww-eeee" }, - Published = new DateTimeOffset(2025, 2, 28, 0, 0, 0, TimeSpan.Zero), - Modified = new DateTimeOffset(2025, 3, 6, 12, 0, 0, TimeSpan.Zero), - Severity = new[] - { - new OsvSeverityDto - { - Type = "CVSS_V3", - Score = "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L" - } - }, - References = new[] - { - new OsvReferenceDto - { - Type = "ADVISORY", - Url = "https://osv.dev/vulnerability/OSV-2025-4242" - }, - new OsvReferenceDto - { - Type = "FIX", - Url = "https://github.com/conflict/package/commit/abcdef1234567890" - } - }, - Credits = new[] - { - new OsvCreditDto - { - Name = "osv-reporter", - Type = "reporter", - Contact = new[] { "mailto:osv-reporter@example.com" } - } - }, - Affected = new[] - { - new OsvAffectedPackageDto - { - Package = new OsvPackageDto - { - Ecosystem = "npm", - Name = "conflict/package" - }, - Ranges = new[] - { - new OsvRangeDto - { - Type = "SEMVER", - Events = new[] - { - new OsvEventDto { Introduced = "1.0.0" }, - new OsvEventDto { LastAffected = "1.4.2" }, - new OsvEventDto { Fixed = "1.5.0" } - } - } - } - } - }, - DatabaseSpecific = databaseSpecificDoc.RootElement.Clone() - }; - - var document = new DocumentRecord( - Id: Guid.Parse("8dd2b0fe-a5f5-4b3b-9f5c-0f3aad6fb6ce"), - SourceName: OsvConnectorPlugin.SourceName, - Uri: "https://api.osv.dev/v1/vulns/OSV-2025-4242", - FetchedAt: new DateTimeOffset(2025, 3, 6, 11, 30, 0, TimeSpan.Zero), - Sha256: "sha256-osv-conflict-fixture", - Status: "completed", - ContentType: "application/json", - Headers: null, - Metadata: null, - Etag: "\"etag-osv-conflict\"", - LastModified: new DateTimeOffset(2025, 3, 6, 12, 0, 0, TimeSpan.Zero), - GridFsId: null); - - var dtoRecord = new DtoRecord( - Id: Guid.Parse("6f7d5ce7-cb47-40a5-8b41-8ad022b5fd5c"), - DocumentId: document.Id, - SourceName: OsvConnectorPlugin.SourceName, - SchemaVersion: "osv.v1", - Payload: new BsonDocument("id", dto.Id), - ValidatedAt: new DateTimeOffset(2025, 3, 6, 12, 5, 0, TimeSpan.Zero)); - - var advisory = OsvMapper.Map(dto, document, dtoRecord, "npm"); - var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); - - var expectedPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-osv.canonical.json"); - var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); - - if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) - { - var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-osv.canonical.actual.json"); - File.WriteAllText(actualPath, snapshot); - } - - Assert.Equal(expected, snapshot); - } -} +using System.Text.Json; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; + +namespace StellaOps.Concelier.Connector.Osv.Tests; + +public sealed class OsvConflictFixtureTests +{ + [Fact] + public void ConflictFixture_MatchesSnapshot() + { + using var databaseSpecificDoc = JsonDocument.Parse("""{"severity":"medium"}"""); + + var dto = new OsvVulnerabilityDto + { + Id = "OSV-2025-4242", + Summary = "Container escape for conflict-package", + Details = "OSV captures the latest container escape details including patched version metadata.", + Aliases = new[] { "CVE-2025-4242", "GHSA-qqqq-wwww-eeee" }, + Published = new DateTimeOffset(2025, 2, 28, 0, 0, 0, TimeSpan.Zero), + Modified = new DateTimeOffset(2025, 3, 6, 12, 0, 0, TimeSpan.Zero), + Severity = new[] + { + new OsvSeverityDto + { + Type = "CVSS_V3", + Score = "CVSS:3.1/AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:L" + } + }, + References = new[] + { + new OsvReferenceDto + { + Type = "ADVISORY", + Url = "https://osv.dev/vulnerability/OSV-2025-4242" + }, + new OsvReferenceDto + { + Type = "FIX", + Url = "https://github.com/conflict/package/commit/abcdef1234567890" + } + }, + Credits = new[] + { + new OsvCreditDto + { + Name = "osv-reporter", + Type = "reporter", + Contact = new[] { "mailto:osv-reporter@example.com" } + } + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = "npm", + Name = "conflict/package" + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "1.0.0" }, + new OsvEventDto { LastAffected = "1.4.2" }, + new OsvEventDto { Fixed = "1.5.0" } + } + } + } + } + }, + DatabaseSpecific = databaseSpecificDoc.RootElement.Clone() + }; + + var document = new DocumentRecord( + Id: Guid.Parse("8dd2b0fe-a5f5-4b3b-9f5c-0f3aad6fb6ce"), + SourceName: OsvConnectorPlugin.SourceName, + Uri: "https://api.osv.dev/v1/vulns/OSV-2025-4242", + FetchedAt: new DateTimeOffset(2025, 3, 6, 11, 30, 0, TimeSpan.Zero), + Sha256: "sha256-osv-conflict-fixture", + Status: "completed", + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: "\"etag-osv-conflict\"", + LastModified: new DateTimeOffset(2025, 3, 6, 12, 0, 0, TimeSpan.Zero), + GridFsId: null); + + var dtoRecord = new DtoRecord( + Id: Guid.Parse("6f7d5ce7-cb47-40a5-8b41-8ad022b5fd5c"), + DocumentId: document.Id, + SourceName: OsvConnectorPlugin.SourceName, + SchemaVersion: "osv.v1", + Payload: new BsonDocument("id", dto.Id), + ValidatedAt: new DateTimeOffset(2025, 3, 6, 12, 5, 0, TimeSpan.Zero)); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, "npm"); + var snapshot = SnapshotSerializer.ToSnapshot(advisory).Replace("\r\n", "\n").TrimEnd(); + + var expectedPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-osv.canonical.json"); + var expected = File.ReadAllText(expectedPath).Replace("\r\n", "\n").TrimEnd(); + + if (!string.Equals(expected, snapshot, StringComparison.Ordinal)) + { + var actualPath = Path.Combine(AppContext.BaseDirectory, "Fixtures", "conflict-osv.canonical.actual.json"); + File.WriteAllText(actualPath, snapshot); + } + + Assert.Equal(expected, snapshot); + } +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs rename to src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs index b47ae735..61ca75be 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvGhsaParityRegressionTests.cs @@ -1,572 +1,572 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text.RegularExpressions; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Osv; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using Xunit; - -namespace StellaOps.Feedser.Source.Osv.Tests; - -public sealed class OsvGhsaParityRegressionTests -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); - - // Curated GHSA identifiers spanning multiple ecosystems (PyPI, npm/go, Maven) for parity coverage. - private static readonly string[] GhsaIds = - { - "GHSA-wv4w-6qv2-qqfg", // PyPI – social-auth-app-django - "GHSA-cjjf-27cc-pvmv", // PyPI – pyload-ng - "GHSA-77vh-xpmg-72qh", // Go – opencontainers/image-spec - "GHSA-7rjr-3q55-vv33" // Maven – log4j-core / pax-logging - }; - - [Fact] - public void FixtureParity_NoIssues_EmitsMetrics() - { - RegenerateFixturesIfRequested(); - - var osvAdvisories = LoadOsvAdvisories(); - var ghsaAdvisories = LoadGhsaAdvisories(); - - if (File.Exists(RebuildSentinelPath)) - { - WriteFixture("osv-ghsa.osv.json", osvAdvisories); - WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); - File.Delete(RebuildSentinelPath); - } - - AssertSnapshot("osv-ghsa.osv.json", osvAdvisories); - AssertSnapshot("osv-ghsa.ghsa.json", ghsaAdvisories); - - var measurements = new List(); - using var listener = CreateListener(measurements); - - var report = OsvGhsaParityInspector.Compare(osvAdvisories, ghsaAdvisories); - - if (report.HasIssues) - { - foreach (var issue in report.Issues) - { - Console.WriteLine($"[Parity] Issue: {issue.GhsaId} {issue.IssueKind} {issue.Detail}"); - } - } - - Assert.False(report.HasIssues); - Assert.Equal(GhsaIds.Length, report.TotalGhsaIds); - - OsvGhsaParityDiagnostics.RecordReport(report, "fixtures"); - listener.Dispose(); - - var total = Assert.Single(measurements, entry => string.Equals(entry.Instrument, "feedser.osv_ghsa.total", StringComparison.Ordinal)); - Assert.Equal(GhsaIds.Length, total.Value); - Assert.Equal("fixtures", Assert.IsType(total.Tags["dataset"])); - - Assert.DoesNotContain(measurements, entry => string.Equals(entry.Instrument, "feedser.osv_ghsa.issues", StringComparison.Ordinal)); - } - - private static MeterListener CreateListener(List buffer) - { - var listener = new MeterListener - { - InstrumentPublished = (instrument, l) => - { - if (instrument.Meter.Name.StartsWith("StellaOps.Feedser.Models.OsvGhsaParity", StringComparison.Ordinal)) - { - l.EnableMeasurementEvents(instrument); - } - } - }; - - listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => - { - var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var tag in tags) - { - dict[tag.Key] = tag.Value; - } - - buffer.Add(new MeasurementRecord(instrument.Name, measurement, dict)); - }); - - listener.Start(); - return listener; - } - - private static IReadOnlyList LoadOsvAdvisories() - { - var path = ResolveFixturePath("osv-ghsa.raw-osv.json"); - using var document = JsonDocument.Parse(File.ReadAllText(path)); - var advisories = new List(); - foreach (var element in document.RootElement.EnumerateArray()) - { - advisories.Add(MapOsvAdvisory(element.GetRawText())); - } - advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); - return advisories; - } - - private static IReadOnlyList LoadGhsaAdvisories() - { - var path = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); - using var document = JsonDocument.Parse(File.ReadAllText(path)); - var advisories = new List(); - foreach (var element in document.RootElement.EnumerateArray()) - { - advisories.Add(MapGhsaAdvisory(element.GetRawText())); - } - advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); - return advisories; - } - - private static void RegenerateFixturesIfRequested() - { - var flag = Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"); - Console.WriteLine($"[Parity] UPDATE_PARITY_FIXTURES={flag ?? "(null)"}"); - - var rawOsvPath = ResolveFixturePath("osv-ghsa.raw-osv.json"); - var rawGhsaPath = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); - var shouldBootstrap = !File.Exists(rawOsvPath) || !File.Exists(rawGhsaPath); - - if (!string.Equals(flag, "1", StringComparison.Ordinal) && !shouldBootstrap) - { - return; - } - - // regeneration trigger - Console.WriteLine(shouldBootstrap - ? $"[Parity] Raw fixtures missing – regenerating OSV/GHSA snapshots for {GhsaIds.Length} advisories." - : $"[Parity] Regenerating OSV/GHSA fixtures for {GhsaIds.Length} advisories."); - - using var client = new HttpClient(); - client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsParityFixtures", "1.0")); - - var osvAdvisories = new List(GhsaIds.Length); - var ghsaAdvisories = new List(GhsaIds.Length); - var rawOsv = new List(GhsaIds.Length); - var rawGhsa = new List(GhsaIds.Length); - - foreach (var ghsaId in GhsaIds) - { - var osvJson = FetchJson(client, $"https://api.osv.dev/v1/vulns/{ghsaId}"); - var ghsaJson = FetchJson(client, $"https://api.github.com/advisories/{ghsaId}"); - - using (var osvDocument = JsonDocument.Parse(osvJson)) - { - rawOsv.Add(osvDocument.RootElement.Clone()); - } - using (var ghsaDocument = JsonDocument.Parse(ghsaJson)) - { - rawGhsa.Add(ghsaDocument.RootElement.Clone()); - } - - var osv = MapOsvAdvisory(osvJson); - var ghsa = MapGhsaAdvisory(ghsaJson); - - osvAdvisories.Add(osv); - ghsaAdvisories.Add(ghsa); - } - - osvAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); - ghsaAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); - - WriteRawFixture("osv-ghsa.raw-osv.json", rawOsv); - WriteRawFixture("osv-ghsa.raw-ghsa.json", rawGhsa); - WriteFixture("osv-ghsa.osv.json", osvAdvisories); - WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); - } - - private static string FetchJson(HttpClient client, string uri) - { - try - { - return client.GetStringAsync(uri).GetAwaiter().GetResult(); - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to download '{uri}'.", ex); - } - } - - private static Advisory MapOsvAdvisory(string json) - { - var dto = JsonSerializer.Deserialize(json, SerializerOptions) - ?? throw new InvalidOperationException("Unable to deserialize OSV payload."); - - var documentId = Guid.NewGuid(); - var identifier = dto.Id ?? throw new InvalidOperationException("OSV payload missing id."); - var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "osv"; - var fetchedAt = dto.Published ?? dto.Modified ?? DateTimeOffset.UtcNow; - var sha = ComputeSha256Hex(json); - - var document = new DocumentRecord( - documentId, - OsvConnectorPlugin.SourceName, - $"https://osv.dev/vulnerability/{identifier}", - fetchedAt, - sha, - DocumentStatuses.PendingMap, - "application/json", - null, - new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, - null, - dto.Modified, - null); - - var payload = BsonDocument.Parse(json); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); - - return OsvMapper.Map(dto, document, dtoRecord, ecosystem); - } - - private static Advisory MapGhsaAdvisory(string json) - { - using var document = JsonDocument.Parse(json); - var root = document.RootElement; - - var ghsaId = GetString(root, "ghsa_id"); - if (string.IsNullOrWhiteSpace(ghsaId)) - { - throw new InvalidOperationException("GHSA payload missing ghsa_id."); - } - - var summary = GetString(root, "summary"); - var description = GetString(root, "description"); - var severity = GetString(root, "severity")?.ToLowerInvariant(); - var published = GetDateTime(root, "published_at"); - var updated = GetDateTime(root, "updated_at"); - var recordedAt = updated ?? DateTimeOffset.UtcNow; - - var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { ghsaId }; - if (root.TryGetProperty("identifiers", out var identifiers) && identifiers.ValueKind == JsonValueKind.Array) - { - foreach (var identifier in identifiers.EnumerateArray()) - { - var value = identifier.TryGetProperty("value", out var valueElem) ? valueElem.GetString() : null; - if (!string.IsNullOrWhiteSpace(value)) - { - aliases.Add(value); - } - } - } - - var references = new List(); - if (root.TryGetProperty("references", out var referencesElem) && referencesElem.ValueKind == JsonValueKind.Array) - { - foreach (var referenceElem in referencesElem.EnumerateArray()) - { - var url = referenceElem.GetString(); - if (string.IsNullOrWhiteSpace(url)) - { - continue; - } - - var provenance = new AdvisoryProvenance("ghsa", "reference", url, recordedAt, new[] { ProvenanceFieldMasks.References }); - references.Add(new AdvisoryReference(url, DetermineReferenceKind(url), DetermineSourceTag(url), null, provenance)); - } - } - - references = references - .DistinctBy(reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .OrderBy(reference => reference.Url, StringComparer.Ordinal) - .ToList(); - - var affectedPackages = BuildGhsaPackages(root, recordedAt); - var cvssMetrics = BuildGhsaCvss(root, recordedAt); - - var advisoryProvenance = new AdvisoryProvenance("ghsa", "map", ghsaId, recordedAt, new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - ghsaId, - string.IsNullOrWhiteSpace(summary) ? ghsaId : summary!, - string.IsNullOrWhiteSpace(description) ? summary : description, - language: "en", - published, - updated, - severity, - exploitKnown: false, - aliases, - references, - affectedPackages, - cvssMetrics, - new[] { advisoryProvenance }); - } - - private static IReadOnlyList BuildGhsaPackages(JsonElement root, DateTimeOffset recordedAt) - { - if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElem) || vulnerabilitiesElem.ValueKind != JsonValueKind.Array) - { - return Array.Empty(); - } - - var packages = new List(); - foreach (var entry in vulnerabilitiesElem.EnumerateArray()) - { - if (!entry.TryGetProperty("package", out var packageElem) || packageElem.ValueKind != JsonValueKind.Object) - { - continue; - } - - var ecosystem = GetString(packageElem, "ecosystem"); - var name = GetString(packageElem, "name"); - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - - var identifier = BuildIdentifier(ecosystem, name); - var packageProvenance = new AdvisoryProvenance("ghsa", "package", identifier, recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }); - - var rangeExpression = GetString(entry, "vulnerable_version_range"); - string? firstPatched = null; - if (entry.TryGetProperty("first_patched_version", out var firstPatchedElem) && firstPatchedElem.ValueKind == JsonValueKind.Object) - { - firstPatched = GetString(firstPatchedElem, "identifier"); - } - - var ranges = ParseVersionRanges(rangeExpression, firstPatched, identifier, recordedAt); - - packages.Add(new AffectedPackage( - AffectedPackageTypes.SemVer, - identifier, - ecosystem, - ranges, - Array.Empty(), - new[] { packageProvenance })); - } - - return packages.OrderBy(package => package.Identifier, StringComparer.Ordinal).ToArray(); - } - - private static IReadOnlyList ParseVersionRanges(string? vulnerableVersionRange, string? firstPatchedVersion, string identifier, DateTimeOffset recordedAt) - { - if (string.IsNullOrWhiteSpace(vulnerableVersionRange) && string.IsNullOrWhiteSpace(firstPatchedVersion)) - { - return Array.Empty(); - } - - var ranges = new List(); - - var expressions = vulnerableVersionRange? - .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) - .ToArray() ?? Array.Empty(); - - string? introduced = null; - string? fixedVersion = firstPatchedVersion; - string? lastAffected = null; - - foreach (var expression in expressions) - { - if (expression.StartsWith(">=", StringComparison.Ordinal)) - { - introduced = expression[(expression.IndexOf('=') + 1)..].Trim(); - } - else if (expression.StartsWith(">", StringComparison.Ordinal)) - { - introduced = expression[1..].Trim(); - } - else if (expression.StartsWith("<=", StringComparison.Ordinal)) - { - lastAffected = expression[(expression.IndexOf('=') + 1)..].Trim(); - } - else if (expression.StartsWith("<", StringComparison.Ordinal)) - { - fixedVersion = expression[1..].Trim(); - } - } - - var provenance = new AdvisoryProvenance("ghsa", "range", identifier, recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }); - ranges.Add(new AffectedVersionRange("semver", NullIfWhitespace(introduced), NullIfWhitespace(fixedVersion), NullIfWhitespace(lastAffected), vulnerableVersionRange, provenance)); - - return ranges; - } - - private static IReadOnlyList BuildGhsaCvss(JsonElement root, DateTimeOffset recordedAt) - { - if (!root.TryGetProperty("cvss_severities", out var severitiesElem) || severitiesElem.ValueKind != JsonValueKind.Object) - { - return Array.Empty(); - } - - var metrics = new List(); - if (severitiesElem.TryGetProperty("cvss_v3", out var cvssElem) && cvssElem.ValueKind == JsonValueKind.Object) - { - var vector = GetString(cvssElem, "vector_string"); - if (!string.IsNullOrWhiteSpace(vector)) - { - var score = cvssElem.TryGetProperty("score", out var scoreElem) && scoreElem.ValueKind == JsonValueKind.Number - ? scoreElem.GetDouble() - : 0d; - var provenance = new AdvisoryProvenance("ghsa", "cvss", vector, recordedAt, new[] { ProvenanceFieldMasks.CvssMetrics }); - var version = vector.StartsWith("CVSS:4.0", StringComparison.OrdinalIgnoreCase) ? "4.0" : "3.1"; - var severity = GetString(root, "severity")?.ToLowerInvariant() ?? "unknown"; - metrics.Add(new CvssMetric(version, vector, score, severity, provenance)); - } - } - - return metrics; - } - - private static string BuildIdentifier(string? ecosystem, string name) - { - if (string.IsNullOrWhiteSpace(ecosystem)) - { - return name; - } - - var key = ecosystem.Trim().ToLowerInvariant(); - return key switch - { - "pypi" => $"pkg:pypi/{name.Replace('_', '-').ToLowerInvariant()}", - "npm" => $"pkg:npm/{name.ToLowerInvariant()}", - "maven" => $"pkg:maven/{name.Replace(':', '/')}", - "go" or "golang" => $"pkg:golang/{name}", - _ => name - }; - } - - private static string? DetermineReferenceKind(string url) - { - if (url.Contains("/commit/", StringComparison.OrdinalIgnoreCase) || - url.Contains("/pull/", StringComparison.OrdinalIgnoreCase) || - url.Contains("/releases/tag/", StringComparison.OrdinalIgnoreCase) || - url.Contains("/pull-requests/", StringComparison.OrdinalIgnoreCase)) - { - return "patch"; - } - - if (url.Contains("advisories", StringComparison.OrdinalIgnoreCase) || - url.Contains("security", StringComparison.OrdinalIgnoreCase) || - url.Contains("cve", StringComparison.OrdinalIgnoreCase)) - { - return "advisory"; - } - - return null; - } - - private static string? DetermineSourceTag(string url) - { - if (Uri.TryCreate(url, UriKind.Absolute, out var uri)) - { - return uri.Host; - } - - return null; - } - - private static string? GetString(JsonElement element, string propertyName) - { - if (element.TryGetProperty(propertyName, out var property)) - { - if (property.ValueKind == JsonValueKind.String) - { - return property.GetString(); - } - } - - return null; - } - - private static DateTimeOffset? GetDateTime(JsonElement element, string propertyName) - { - if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) - { - if (property.TryGetDateTimeOffset(out var value)) - { - return value; - } - } - - return null; - } - - private static void WriteFixture(string filename, IReadOnlyList advisories) - { - var path = ResolveFixturePath(filename); - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - } - - var snapshot = SnapshotSerializer.ToSnapshot(advisories); - File.WriteAllText(path, snapshot); - } - - private static void WriteRawFixture(string filename, IReadOnlyList elements) - { - var path = ResolveFixturePath(filename); - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - } - - var json = JsonSerializer.Serialize(elements, new JsonSerializerOptions - { - WriteIndented = true - }); - File.WriteAllText(path, json); - } - - private static void AssertSnapshot(string filename, IReadOnlyList advisories) - { - var path = ResolveFixturePath(filename); - var actual = File.ReadAllText(path).Trim().ReplaceLineEndings("\n"); - var expected = SnapshotSerializer.ToSnapshot(advisories).Trim().ReplaceLineEndings("\n"); - - var normalizedActual = NormalizeRecordedAt(actual); - var normalizedExpected = NormalizeRecordedAt(expected); - - if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) - { - var shouldUpdate = string.Equals(Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"), "1", StringComparison.Ordinal); - if (shouldUpdate) - { - var normalized = expected.Replace("\n", Environment.NewLine, StringComparison.Ordinal); - File.WriteAllText(path, normalized); - actual = expected; - normalizedActual = normalizedExpected; - } - } - - Assert.Equal(normalizedExpected, normalizedActual); - } - - private static string ResolveFixturePath(string filename) - => Path.Combine(ProjectFixtureDirectory, filename); - - private static string NormalizeRecordedAt(string input) - => RecordedAtRegex.Replace(input, "\"recordedAt\": \"#normalized#\""); - - private static string ProjectFixtureDirectory { get; } = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures")); - - private static string RebuildSentinelPath => Path.Combine(ProjectFixtureDirectory, ".rebuild"); - - private static readonly Regex RecordedAtRegex = new("\"recordedAt\": \"[^\"]+\"", RegexOptions.CultureInvariant | RegexOptions.Compiled); - - private static string ComputeSha256Hex(string payload) - { - var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(payload)); - return Convert.ToHexString(bytes); - } - - private static string? NullIfWhitespace(string? value) - => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); - - private sealed record MeasurementRecord(string Instrument, long Value, IReadOnlyDictionary Tags); - -} +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Osv; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Concelier.Connector.Osv.Tests; + +public sealed class OsvGhsaParityRegressionTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + // Curated GHSA identifiers spanning multiple ecosystems (PyPI, npm/go, Maven) for parity coverage. + private static readonly string[] GhsaIds = + { + "GHSA-wv4w-6qv2-qqfg", // PyPI – social-auth-app-django + "GHSA-cjjf-27cc-pvmv", // PyPI – pyload-ng + "GHSA-77vh-xpmg-72qh", // Go – opencontainers/image-spec + "GHSA-7rjr-3q55-vv33" // Maven – log4j-core / pax-logging + }; + + [Fact] + public void FixtureParity_NoIssues_EmitsMetrics() + { + RegenerateFixturesIfRequested(); + + var osvAdvisories = LoadOsvAdvisories(); + var ghsaAdvisories = LoadGhsaAdvisories(); + + if (File.Exists(RebuildSentinelPath)) + { + WriteFixture("osv-ghsa.osv.json", osvAdvisories); + WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); + File.Delete(RebuildSentinelPath); + } + + AssertSnapshot("osv-ghsa.osv.json", osvAdvisories); + AssertSnapshot("osv-ghsa.ghsa.json", ghsaAdvisories); + + var measurements = new List(); + using var listener = CreateListener(measurements); + + var report = OsvGhsaParityInspector.Compare(osvAdvisories, ghsaAdvisories); + + if (report.HasIssues) + { + foreach (var issue in report.Issues) + { + Console.WriteLine($"[Parity] Issue: {issue.GhsaId} {issue.IssueKind} {issue.Detail}"); + } + } + + Assert.False(report.HasIssues); + Assert.Equal(GhsaIds.Length, report.TotalGhsaIds); + + OsvGhsaParityDiagnostics.RecordReport(report, "fixtures"); + listener.Dispose(); + + var total = Assert.Single(measurements, entry => string.Equals(entry.Instrument, "concelier.osv_ghsa.total", StringComparison.Ordinal)); + Assert.Equal(GhsaIds.Length, total.Value); + Assert.Equal("fixtures", Assert.IsType(total.Tags["dataset"])); + + Assert.DoesNotContain(measurements, entry => string.Equals(entry.Instrument, "concelier.osv_ghsa.issues", StringComparison.Ordinal)); + } + + private static MeterListener CreateListener(List buffer) + { + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name.StartsWith("StellaOps.Concelier.Models.OsvGhsaParity", StringComparison.Ordinal)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + buffer.Add(new MeasurementRecord(instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } + + private static IReadOnlyList LoadOsvAdvisories() + { + var path = ResolveFixturePath("osv-ghsa.raw-osv.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + advisories.Add(MapOsvAdvisory(element.GetRawText())); + } + advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + return advisories; + } + + private static IReadOnlyList LoadGhsaAdvisories() + { + var path = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + advisories.Add(MapGhsaAdvisory(element.GetRawText())); + } + advisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + return advisories; + } + + private static void RegenerateFixturesIfRequested() + { + var flag = Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"); + Console.WriteLine($"[Parity] UPDATE_PARITY_FIXTURES={flag ?? "(null)"}"); + + var rawOsvPath = ResolveFixturePath("osv-ghsa.raw-osv.json"); + var rawGhsaPath = ResolveFixturePath("osv-ghsa.raw-ghsa.json"); + var shouldBootstrap = !File.Exists(rawOsvPath) || !File.Exists(rawGhsaPath); + + if (!string.Equals(flag, "1", StringComparison.Ordinal) && !shouldBootstrap) + { + return; + } + + // regeneration trigger + Console.WriteLine(shouldBootstrap + ? $"[Parity] Raw fixtures missing – regenerating OSV/GHSA snapshots for {GhsaIds.Length} advisories." + : $"[Parity] Regenerating OSV/GHSA fixtures for {GhsaIds.Length} advisories."); + + using var client = new HttpClient(); + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsParityFixtures", "1.0")); + + var osvAdvisories = new List(GhsaIds.Length); + var ghsaAdvisories = new List(GhsaIds.Length); + var rawOsv = new List(GhsaIds.Length); + var rawGhsa = new List(GhsaIds.Length); + + foreach (var ghsaId in GhsaIds) + { + var osvJson = FetchJson(client, $"https://api.osv.dev/v1/vulns/{ghsaId}"); + var ghsaJson = FetchJson(client, $"https://api.github.com/advisories/{ghsaId}"); + + using (var osvDocument = JsonDocument.Parse(osvJson)) + { + rawOsv.Add(osvDocument.RootElement.Clone()); + } + using (var ghsaDocument = JsonDocument.Parse(ghsaJson)) + { + rawGhsa.Add(ghsaDocument.RootElement.Clone()); + } + + var osv = MapOsvAdvisory(osvJson); + var ghsa = MapGhsaAdvisory(ghsaJson); + + osvAdvisories.Add(osv); + ghsaAdvisories.Add(ghsa); + } + + osvAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + ghsaAdvisories.Sort((a, b) => string.Compare(a.AdvisoryKey, b.AdvisoryKey, StringComparison.OrdinalIgnoreCase)); + + WriteRawFixture("osv-ghsa.raw-osv.json", rawOsv); + WriteRawFixture("osv-ghsa.raw-ghsa.json", rawGhsa); + WriteFixture("osv-ghsa.osv.json", osvAdvisories); + WriteFixture("osv-ghsa.ghsa.json", ghsaAdvisories); + } + + private static string FetchJson(HttpClient client, string uri) + { + try + { + return client.GetStringAsync(uri).GetAwaiter().GetResult(); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to download '{uri}'.", ex); + } + } + + private static Advisory MapOsvAdvisory(string json) + { + var dto = JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize OSV payload."); + + var documentId = Guid.NewGuid(); + var identifier = dto.Id ?? throw new InvalidOperationException("OSV payload missing id."); + var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "osv"; + var fetchedAt = dto.Published ?? dto.Modified ?? DateTimeOffset.UtcNow; + var sha = ComputeSha256Hex(json); + + var document = new DocumentRecord( + documentId, + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/{identifier}", + fetchedAt, + sha, + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, + null, + dto.Modified, + null); + + var payload = BsonDocument.Parse(json); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, DateTimeOffset.UtcNow); + + return OsvMapper.Map(dto, document, dtoRecord, ecosystem); + } + + private static Advisory MapGhsaAdvisory(string json) + { + using var document = JsonDocument.Parse(json); + var root = document.RootElement; + + var ghsaId = GetString(root, "ghsa_id"); + if (string.IsNullOrWhiteSpace(ghsaId)) + { + throw new InvalidOperationException("GHSA payload missing ghsa_id."); + } + + var summary = GetString(root, "summary"); + var description = GetString(root, "description"); + var severity = GetString(root, "severity")?.ToLowerInvariant(); + var published = GetDateTime(root, "published_at"); + var updated = GetDateTime(root, "updated_at"); + var recordedAt = updated ?? DateTimeOffset.UtcNow; + + var aliases = new HashSet(StringComparer.OrdinalIgnoreCase) { ghsaId }; + if (root.TryGetProperty("identifiers", out var identifiers) && identifiers.ValueKind == JsonValueKind.Array) + { + foreach (var identifier in identifiers.EnumerateArray()) + { + var value = identifier.TryGetProperty("value", out var valueElem) ? valueElem.GetString() : null; + if (!string.IsNullOrWhiteSpace(value)) + { + aliases.Add(value); + } + } + } + + var references = new List(); + if (root.TryGetProperty("references", out var referencesElem) && referencesElem.ValueKind == JsonValueKind.Array) + { + foreach (var referenceElem in referencesElem.EnumerateArray()) + { + var url = referenceElem.GetString(); + if (string.IsNullOrWhiteSpace(url)) + { + continue; + } + + var provenance = new AdvisoryProvenance("ghsa", "reference", url, recordedAt, new[] { ProvenanceFieldMasks.References }); + references.Add(new AdvisoryReference(url, DetermineReferenceKind(url), DetermineSourceTag(url), null, provenance)); + } + } + + references = references + .DistinctBy(reference => reference.Url, StringComparer.OrdinalIgnoreCase) + .OrderBy(reference => reference.Url, StringComparer.Ordinal) + .ToList(); + + var affectedPackages = BuildGhsaPackages(root, recordedAt); + var cvssMetrics = BuildGhsaCvss(root, recordedAt); + + var advisoryProvenance = new AdvisoryProvenance("ghsa", "map", ghsaId, recordedAt, new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + ghsaId, + string.IsNullOrWhiteSpace(summary) ? ghsaId : summary!, + string.IsNullOrWhiteSpace(description) ? summary : description, + language: "en", + published, + updated, + severity, + exploitKnown: false, + aliases, + references, + affectedPackages, + cvssMetrics, + new[] { advisoryProvenance }); + } + + private static IReadOnlyList BuildGhsaPackages(JsonElement root, DateTimeOffset recordedAt) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElem) || vulnerabilitiesElem.ValueKind != JsonValueKind.Array) + { + return Array.Empty(); + } + + var packages = new List(); + foreach (var entry in vulnerabilitiesElem.EnumerateArray()) + { + if (!entry.TryGetProperty("package", out var packageElem) || packageElem.ValueKind != JsonValueKind.Object) + { + continue; + } + + var ecosystem = GetString(packageElem, "ecosystem"); + var name = GetString(packageElem, "name"); + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + var identifier = BuildIdentifier(ecosystem, name); + var packageProvenance = new AdvisoryProvenance("ghsa", "package", identifier, recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }); + + var rangeExpression = GetString(entry, "vulnerable_version_range"); + string? firstPatched = null; + if (entry.TryGetProperty("first_patched_version", out var firstPatchedElem) && firstPatchedElem.ValueKind == JsonValueKind.Object) + { + firstPatched = GetString(firstPatchedElem, "identifier"); + } + + var ranges = ParseVersionRanges(rangeExpression, firstPatched, identifier, recordedAt); + + packages.Add(new AffectedPackage( + AffectedPackageTypes.SemVer, + identifier, + ecosystem, + ranges, + Array.Empty(), + new[] { packageProvenance })); + } + + return packages.OrderBy(package => package.Identifier, StringComparer.Ordinal).ToArray(); + } + + private static IReadOnlyList ParseVersionRanges(string? vulnerableVersionRange, string? firstPatchedVersion, string identifier, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(vulnerableVersionRange) && string.IsNullOrWhiteSpace(firstPatchedVersion)) + { + return Array.Empty(); + } + + var ranges = new List(); + + var expressions = vulnerableVersionRange? + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .ToArray() ?? Array.Empty(); + + string? introduced = null; + string? fixedVersion = firstPatchedVersion; + string? lastAffected = null; + + foreach (var expression in expressions) + { + if (expression.StartsWith(">=", StringComparison.Ordinal)) + { + introduced = expression[(expression.IndexOf('=') + 1)..].Trim(); + } + else if (expression.StartsWith(">", StringComparison.Ordinal)) + { + introduced = expression[1..].Trim(); + } + else if (expression.StartsWith("<=", StringComparison.Ordinal)) + { + lastAffected = expression[(expression.IndexOf('=') + 1)..].Trim(); + } + else if (expression.StartsWith("<", StringComparison.Ordinal)) + { + fixedVersion = expression[1..].Trim(); + } + } + + var provenance = new AdvisoryProvenance("ghsa", "range", identifier, recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }); + ranges.Add(new AffectedVersionRange("semver", NullIfWhitespace(introduced), NullIfWhitespace(fixedVersion), NullIfWhitespace(lastAffected), vulnerableVersionRange, provenance)); + + return ranges; + } + + private static IReadOnlyList BuildGhsaCvss(JsonElement root, DateTimeOffset recordedAt) + { + if (!root.TryGetProperty("cvss_severities", out var severitiesElem) || severitiesElem.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var metrics = new List(); + if (severitiesElem.TryGetProperty("cvss_v3", out var cvssElem) && cvssElem.ValueKind == JsonValueKind.Object) + { + var vector = GetString(cvssElem, "vector_string"); + if (!string.IsNullOrWhiteSpace(vector)) + { + var score = cvssElem.TryGetProperty("score", out var scoreElem) && scoreElem.ValueKind == JsonValueKind.Number + ? scoreElem.GetDouble() + : 0d; + var provenance = new AdvisoryProvenance("ghsa", "cvss", vector, recordedAt, new[] { ProvenanceFieldMasks.CvssMetrics }); + var version = vector.StartsWith("CVSS:4.0", StringComparison.OrdinalIgnoreCase) ? "4.0" : "3.1"; + var severity = GetString(root, "severity")?.ToLowerInvariant() ?? "unknown"; + metrics.Add(new CvssMetric(version, vector, score, severity, provenance)); + } + } + + return metrics; + } + + private static string BuildIdentifier(string? ecosystem, string name) + { + if (string.IsNullOrWhiteSpace(ecosystem)) + { + return name; + } + + var key = ecosystem.Trim().ToLowerInvariant(); + return key switch + { + "pypi" => $"pkg:pypi/{name.Replace('_', '-').ToLowerInvariant()}", + "npm" => $"pkg:npm/{name.ToLowerInvariant()}", + "maven" => $"pkg:maven/{name.Replace(':', '/')}", + "go" or "golang" => $"pkg:golang/{name}", + _ => name + }; + } + + private static string? DetermineReferenceKind(string url) + { + if (url.Contains("/commit/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/pull/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/releases/tag/", StringComparison.OrdinalIgnoreCase) || + url.Contains("/pull-requests/", StringComparison.OrdinalIgnoreCase)) + { + return "patch"; + } + + if (url.Contains("advisories", StringComparison.OrdinalIgnoreCase) || + url.Contains("security", StringComparison.OrdinalIgnoreCase) || + url.Contains("cve", StringComparison.OrdinalIgnoreCase)) + { + return "advisory"; + } + + return null; + } + + private static string? DetermineSourceTag(string url) + { + if (Uri.TryCreate(url, UriKind.Absolute, out var uri)) + { + return uri.Host; + } + + return null; + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property)) + { + if (property.ValueKind == JsonValueKind.String) + { + return property.GetString(); + } + } + + return null; + } + + private static DateTimeOffset? GetDateTime(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + if (property.TryGetDateTimeOffset(out var value)) + { + return value; + } + } + + return null; + } + + private static void WriteFixture(string filename, IReadOnlyList advisories) + { + var path = ResolveFixturePath(filename); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(path, snapshot); + } + + private static void WriteRawFixture(string filename, IReadOnlyList elements) + { + var path = ResolveFixturePath(filename); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + var json = JsonSerializer.Serialize(elements, new JsonSerializerOptions + { + WriteIndented = true + }); + File.WriteAllText(path, json); + } + + private static void AssertSnapshot(string filename, IReadOnlyList advisories) + { + var path = ResolveFixturePath(filename); + var actual = File.ReadAllText(path).Trim().ReplaceLineEndings("\n"); + var expected = SnapshotSerializer.ToSnapshot(advisories).Trim().ReplaceLineEndings("\n"); + + var normalizedActual = NormalizeRecordedAt(actual); + var normalizedExpected = NormalizeRecordedAt(expected); + + if (!string.Equals(normalizedExpected, normalizedActual, StringComparison.Ordinal)) + { + var shouldUpdate = string.Equals(Environment.GetEnvironmentVariable("UPDATE_PARITY_FIXTURES"), "1", StringComparison.Ordinal); + if (shouldUpdate) + { + var normalized = expected.Replace("\n", Environment.NewLine, StringComparison.Ordinal); + File.WriteAllText(path, normalized); + actual = expected; + normalizedActual = normalizedExpected; + } + } + + Assert.Equal(normalizedExpected, normalizedActual); + } + + private static string ResolveFixturePath(string filename) + => Path.Combine(ProjectFixtureDirectory, filename); + + private static string NormalizeRecordedAt(string input) + => RecordedAtRegex.Replace(input, "\"recordedAt\": \"#normalized#\""); + + private static string ProjectFixtureDirectory { get; } = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures")); + + private static string RebuildSentinelPath => Path.Combine(ProjectFixtureDirectory, ".rebuild"); + + private static readonly Regex RecordedAtRegex = new("\"recordedAt\": \"[^\"]+\"", RegexOptions.CultureInvariant | RegexOptions.Compiled); + + private static string ComputeSha256Hex(string payload) + { + var bytes = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(payload)); + return Convert.ToHexString(bytes); + } + + private static string? NullIfWhitespace(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private sealed record MeasurementRecord(string Instrument, long Value, IReadOnlyDictionary Tags); + +} diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvMapperTests.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs rename to src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvMapperTests.cs index e479579b..682a1109 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvMapperTests.cs @@ -4,16 +4,16 @@ using System.Text.Json; using System.Text.Json.Serialization; using System.Reflection; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Osv; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Normalization.Identifiers; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Osv; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Normalization.Identifiers; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using Xunit; -namespace StellaOps.Feedser.Source.Osv.Tests; +namespace StellaOps.Concelier.Connector.Osv.Tests; public sealed class OsvMapperTests { diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvSnapshotTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs rename to src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvSnapshotTests.cs index 6ddfcd47..9a5d7936 100644 --- a/src/StellaOps.Feedser.Source.Osv.Tests/Osv/OsvSnapshotTests.cs +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/Osv/OsvSnapshotTests.cs @@ -3,16 +3,16 @@ using System.Collections.Generic; using System.IO; using System.Text.Json; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Osv; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Source.Common; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Osv; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Connector.Common; using Xunit; using Xunit.Abstractions; -namespace StellaOps.Feedser.Source.Osv.Tests; +namespace StellaOps.Concelier.Connector.Osv.Tests; public sealed class OsvSnapshotTests { diff --git a/src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj b/src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj new file mode 100644 index 00000000..6d8232bd --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Osv/AGENTS.md b/src/StellaOps.Concelier.Connector.Osv/AGENTS.md similarity index 76% rename from src/StellaOps.Feedser.Source.Osv/AGENTS.md rename to src/StellaOps.Concelier.Connector.Osv/AGENTS.md index 40f0ddf0..14834498 100644 --- a/src/StellaOps.Feedser.Source.Osv/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Osv/AGENTS.md @@ -19,8 +19,8 @@ Connector for OSV.dev across ecosystems; authoritative SemVer/PURL ranges for OS In: SemVer+PURL accuracy for OSS ecosystems. Out: vendor PSIRT and distro OVAL specifics. ## Observability & security expectations -- Metrics: SourceDiagnostics exposes the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=osv`; observability dashboards slice on the tag to monitor item volume, schema failures, range counts, and ecosystem coverage. Logs include ecosystem and cursor values. +- Metrics: SourceDiagnostics exposes the shared `concelier.source.http.*` counters/histograms tagged `concelier.source=osv`; observability dashboards slice on the tag to monitor item volume, schema failures, range counts, and ecosystem coverage. Logs include ecosystem and cursor values. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Osv.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Osv.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs b/src/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs rename to src/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs index c6d5f333..eb6d4216 100644 --- a/src/StellaOps.Feedser.Source.Osv/Configuration/OsvOptions.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Configuration/OsvOptions.cs @@ -1,6 +1,6 @@ using System.Diagnostics.CodeAnalysis; -namespace StellaOps.Feedser.Source.Osv.Configuration; +namespace StellaOps.Concelier.Connector.Osv.Configuration; public sealed class OsvOptions { diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs rename to src/StellaOps.Concelier.Connector.Osv/Internal/OsvCursor.cs index a11cf28a..b9420f71 100644 --- a/src/StellaOps.Feedser.Source.Osv/Internal/OsvCursor.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Osv.Internal; +namespace StellaOps.Concelier.Connector.Osv.Internal; internal sealed record OsvCursor( IReadOnlyDictionary LastModifiedByEcosystem, diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvDiagnostics.cs b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvDiagnostics.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Osv/Internal/OsvDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Osv/Internal/OsvDiagnostics.cs index 28bfa958..5d1052f4 100644 --- a/src/StellaOps.Feedser.Source.Osv/Internal/OsvDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvDiagnostics.cs @@ -2,14 +2,14 @@ using System; using System.Collections.Generic; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Osv.Internal; +namespace StellaOps.Concelier.Connector.Osv.Internal; /// /// Connector-specific diagnostics for OSV mapping. /// public sealed class OsvDiagnostics : IDisposable { - private const string MeterName = "StellaOps.Feedser.Source.Osv"; + private const string MeterName = "StellaOps.Concelier.Connector.Osv"; private const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvMapper.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs rename to src/StellaOps.Concelier.Connector.Osv/Internal/OsvMapper.cs index 88625b0d..8d216eec 100644 --- a/src/StellaOps.Feedser.Source.Osv/Internal/OsvMapper.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvMapper.cs @@ -4,15 +4,15 @@ using System.Collections.Immutable; using System.Linq; using System.Text; using System.Text.Json; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.Identifiers; -using StellaOps.Feedser.Normalization.Text; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Text; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; -namespace StellaOps.Feedser.Source.Osv.Internal; +namespace StellaOps.Concelier.Connector.Osv.Internal; internal static class OsvMapper { diff --git a/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvVulnerabilityDto.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs rename to src/StellaOps.Concelier.Connector.Osv/Internal/OsvVulnerabilityDto.cs index 26f4ce12..32d260b7 100644 --- a/src/StellaOps.Feedser.Source.Osv/Internal/OsvVulnerabilityDto.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Internal/OsvVulnerabilityDto.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Osv.Internal; +namespace StellaOps.Concelier.Connector.Osv.Internal; internal sealed record OsvVulnerabilityDto { diff --git a/src/StellaOps.Feedser.Source.Osv/Jobs.cs b/src/StellaOps.Concelier.Connector.Osv/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Osv/Jobs.cs rename to src/StellaOps.Concelier.Connector.Osv/Jobs.cs index 14e395d4..1b4b606f 100644 --- a/src/StellaOps.Feedser.Source.Osv/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Osv; +namespace StellaOps.Concelier.Connector.Osv; internal static class OsvJobKinds { diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs b/src/StellaOps.Concelier.Connector.Osv/OsvConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Osv/OsvConnector.cs rename to src/StellaOps.Concelier.Connector.Osv/OsvConnector.cs index 7690b135..cdaefce2 100644 --- a/src/StellaOps.Feedser.Source.Osv/OsvConnector.cs +++ b/src/StellaOps.Concelier.Connector.Osv/OsvConnector.cs @@ -14,18 +14,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Osv.Configuration; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Osv.Configuration; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Osv; +namespace StellaOps.Concelier.Connector.Osv; public sealed class OsvConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Osv/OsvConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Osv/OsvConnectorPlugin.cs index a5beba0f..624d5586 100644 --- a/src/StellaOps.Feedser.Source.Osv/OsvConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Osv/OsvConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Osv; +namespace StellaOps.Concelier.Connector.Osv; public sealed class OsvConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Osv/OsvDependencyInjectionRoutine.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Osv/OsvDependencyInjectionRoutine.cs index c401aaa2..d99798d4 100644 --- a/src/StellaOps.Feedser.Source.Osv/OsvDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Osv/OsvDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Osv.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Osv.Configuration; -namespace StellaOps.Feedser.Source.Osv; +namespace StellaOps.Concelier.Connector.Osv; public sealed class OsvDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:osv"; + private const string ConfigurationSection = "concelier:sources:osv"; private const string FetchCron = "0,20,40 * * * *"; private const string ParseCron = "5,25,45 * * * *"; private const string MapCron = "10,30,50 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Osv/OsvServiceCollectionExtensions.cs similarity index 81% rename from src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Osv/OsvServiceCollectionExtensions.cs index 34740db5..15bbbcd1 100644 --- a/src/StellaOps.Feedser.Source.Osv/OsvServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Osv/OsvServiceCollectionExtensions.cs @@ -1,11 +1,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Osv.Configuration; -using StellaOps.Feedser.Source.Osv.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Osv.Configuration; +using StellaOps.Concelier.Connector.Osv.Internal; -namespace StellaOps.Feedser.Source.Osv; +namespace StellaOps.Concelier.Connector.Osv; public static class OsvServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class OsvServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = options.BaseUri; clientOptions.Timeout = options.HttpTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.OSV/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.OSV/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.BaseUri.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/zip"; diff --git a/src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Osv/Properties/AssemblyInfo.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs rename to src/StellaOps.Concelier.Connector.Osv/Properties/AssemblyInfo.cs index 150bab12..c08abd4b 100644 --- a/src/StellaOps.Feedser.Source.Osv/Properties/AssemblyInfo.cs +++ b/src/StellaOps.Concelier.Connector.Osv/Properties/AssemblyInfo.cs @@ -1,3 +1,3 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("FixtureUpdater")] +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("FixtureUpdater")] diff --git a/src/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj b/src/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj new file mode 100644 index 00000000..dec35b89 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + <_Parameter1>StellaOps.Concelier.Tests + + + <_Parameter1>StellaOps.Concelier.Connector.Osv.Tests + + + diff --git a/src/StellaOps.Feedser.Source.Osv/TASKS.md b/src/StellaOps.Concelier.Connector.Osv/TASKS.md similarity index 90% rename from src/StellaOps.Feedser.Source.Osv/TASKS.md rename to src/StellaOps.Concelier.Connector.Osv/TASKS.md index 1d35c28a..b17f04cb 100644 --- a/src/StellaOps.Feedser.Source.Osv/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Osv/TASKS.md @@ -14,7 +14,7 @@ |FEEDCONN-OSV-02-004 OSV references & credits alignment|BE-Conn-OSV|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-11)** – Mapper normalizes references with provenance masks, emits advisory credits, and regression fixtures/assertions cover the new fields.| |FEEDCONN-OSV-02-005 Fixture updater workflow|BE-Conn-OSV, QA|Docs|**DONE (2025-10-12)** – Canonical PURL derivation now covers Go + scoped npm advisories without upstream `purl`; legacy invalid npm names still fall back to `ecosystem:name`. OSV/GHSA/NVD suites and normalization/storage tests rerun clean.| |FEEDCONN-OSV-02-003 Normalized versions rollout|BE-Conn-OSV|Models `FEEDMODELS-SCHEMA-01-003`, Normalization playbook|**DONE (2025-10-11)** – `OsvMapper` now emits SemVer primitives + normalized rules with `osv:{ecosystem}:{advisoryId}:{identifier}` notes; npm/PyPI/Parity fixtures refreshed; merge coordination pinged (OSV handoff).| -|FEEDCONN-OSV-04-003 Parity fixture refresh|QA, BE-Conn-OSV|Normalized versions rollout, GHSA parity tests|**DONE (2025-10-12)** – Parity fixtures include normalizedVersions notes (`osv:::`); regression math rerun via `dotnet test src/StellaOps.Feedser.Source.Osv.Tests` and docs flagged for workflow sync.| -|FEEDCONN-OSV-04-002 Conflict regression fixtures|BE-Conn-OSV, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Added `conflict-osv.canonical.json` + regression asserting SemVer range + CVSS medium severity; dataset matches GHSA/NVD fixtures for merge tests. Validation: `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj --filter OsvConflictFixtureTests`.| +|FEEDCONN-OSV-04-003 Parity fixture refresh|QA, BE-Conn-OSV|Normalized versions rollout, GHSA parity tests|**DONE (2025-10-12)** – Parity fixtures include normalizedVersions notes (`osv:::`); regression math rerun via `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests` and docs flagged for workflow sync.| +|FEEDCONN-OSV-04-002 Conflict regression fixtures|BE-Conn-OSV, QA|Merge `FEEDMERGE-ENGINE-04-001`|**DONE (2025-10-12)** – Added `conflict-osv.canonical.json` + regression asserting SemVer range + CVSS medium severity; dataset matches GHSA/NVD fixtures for merge tests. Validation: `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj --filter OsvConflictFixtureTests`.| |FEEDCONN-OSV-04-004 Description/CWE/metric parity rollout|BE-Conn-OSV|Models, Core|**DONE (2025-10-15)** – OSV mapper writes advisory descriptions, `database_specific.cwe_ids` weaknesses, and canonical CVSS metric id. Parity fixtures (`osv-ghsa.*`, `osv-npm.snapshot.json`, `osv-pypi.snapshot.json`) refreshed and status communicated to Merge coordination.| -|FEEDCONN-OSV-04-005 Canonical metric fallbacks & CWE notes|BE-Conn-OSV|Models, Merge|**DONE (2025-10-16)** – Add fallback logic and metrics for advisories lacking CVSS vectors, enrich CWE provenance notes, and document merge/export expectations; refresh parity fixtures accordingly.
      2025-10-16: Mapper now emits `osv:severity/` canonical ids for severity-only advisories, weakness provenance carries `database_specific.cwe_ids`, diagnostics expose `osv.map.canonical_metric_fallbacks`, parity fixtures regenerated, and ops notes added in `docs/ops/feedser-osv-operations.md`. Tests: `dotnet test src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj`.| +|FEEDCONN-OSV-04-005 Canonical metric fallbacks & CWE notes|BE-Conn-OSV|Models, Merge|**DONE (2025-10-16)** – Add fallback logic and metrics for advisories lacking CVSS vectors, enrich CWE provenance notes, and document merge/export expectations; refresh parity fixtures accordingly.
      2025-10-16: Mapper now emits `osv:severity/` canonical ids for severity-only advisories, weakness provenance carries `database_specific.cwe_ids`, diagnostics expose `osv.map.canonical_metric_fallbacks`, parity fixtures regenerated, and ops notes added in `docs/ops/concelier-osv-operations.md`. Tests: `dotnet test src/StellaOps.Concelier.Connector.Osv.Tests/StellaOps.Concelier.Connector.Osv.Tests.csproj`.| diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/export-sample.xml b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/export-sample.xml similarity index 97% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/export-sample.xml rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/export-sample.xml index fb9e7806..b94fad82 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/export-sample.xml +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/export-sample.xml @@ -1,118 +1,118 @@ - - - - BDU:2025-00001 - Множественные уязвимости криптопровайдера - Удалённый злоумышленник может вызвать отказ в обслуживании или получить доступ к данным. - Установить обновление 8.2.19.116 защищённого комплекса. - 01.12.2013 - Высокий уровень опасности (базовая оценка CVSS 2.0 составляет 7,5) - Существует в открытом доступе - Уязвимость устранена - Подтверждена производителем - 0 - - AV:N/AC:L/Au:N/C:P/I:P/A:P - - - AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H - - - - ООО «1С-Софт» - 1С:Предприятие - 8.2.18.96 - Windows - - Прикладное ПО информационных систем - - - - ООО «1С-Софт» - 1С:Предприятие - 8.2.19.116 - Не указана - - Прикладное ПО информационных систем - - - - - - Microsoft Corp - Windows - - - 64-bit - - - Microsoft Corp - Windows - - - 32-bit - - - - - CWE-310 - Проблемы использования криптографии - - - - https://advisories.example/BDU-2025-00001 - http://mirror.example/ru-bdu/BDU-2025-00001 - - - CVE-2015-0206 - CVE-2009-3555 - PT-2015-0206 - - Язык разработки ПО – С - Уязвимость кода - Опубликована - - - BDU:2025-00002 - Уязвимость контроллера АСУ ТП - Локальный злоумышленник может повысить привилегии в контроллере. - Производитель готовит обновление микропрограммы. - 15.10.2024 - Средний уровень опасности - Данные уточняются - Информация об устранении отсутствует - Потенциальная уязвимость - 2 - - AV:L/AC:H/Au:S/C:P/I:P/A:P - - - - АО «Системы Управления» - SCADA Controller - 1.0.0;1.0.1 - - - - ПО программно-аппаратного средства АСУ ТП - - - - - - CWE-269 - Неправильное управление привилегиями - - - CWE-287 - Недостаточная аутентификация - - - - www.vendor.example/security/advisories/ctl-2025-01 - - - ICSA-25-123-01 - - Поставщик сообщает об ограниченном наличии эксплойтов. - Уязвимость архитектуры - Опубликована - - + + + + BDU:2025-00001 + Множественные уязвимости криптопровайдера + Удалённый злоумышленник может вызвать отказ в обслуживании или получить доступ к данным. + Установить обновление 8.2.19.116 защищённого комплекса. + 01.12.2013 + Высокий уровень опасности (базовая оценка CVSS 2.0 составляет 7,5) + Существует в открытом доступе + Уязвимость устранена + Подтверждена производителем + 0 + + AV:N/AC:L/Au:N/C:P/I:P/A:P + + + AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H + + + + ООО «1С-Софт» + 1С:Предприятие + 8.2.18.96 + Windows + + Прикладное ПО информационных систем + + + + ООО «1С-Софт» + 1С:Предприятие + 8.2.19.116 + Не указана + + Прикладное ПО информационных систем + + + + + + Microsoft Corp + Windows + - + 64-bit + + + Microsoft Corp + Windows + - + 32-bit + + + + + CWE-310 + Проблемы использования криптографии + + + + https://advisories.example/BDU-2025-00001 + http://mirror.example/ru-bdu/BDU-2025-00001 + + + CVE-2015-0206 + CVE-2009-3555 + PT-2015-0206 + + Язык разработки ПО – С + Уязвимость кода + Опубликована + + + BDU:2025-00002 + Уязвимость контроллера АСУ ТП + Локальный злоумышленник может повысить привилегии в контроллере. + Производитель готовит обновление микропрограммы. + 15.10.2024 + Средний уровень опасности + Данные уточняются + Информация об устранении отсутствует + Потенциальная уязвимость + 2 + + AV:L/AC:H/Au:S/C:P/I:P/A:P + + + + АО «Системы Управления» + SCADA Controller + 1.0.0;1.0.1 + - + + ПО программно-аппаратного средства АСУ ТП + + + + + + CWE-269 + Неправильное управление привилегиями + + + CWE-287 + Недостаточная аутентификация + + + + www.vendor.example/security/advisories/ctl-2025-01 + + + ICSA-25-123-01 + + Поставщик сообщает об ограниченном наличии эксплойтов. + Уязвимость архитектуры + Опубликована + + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-advisories.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-advisories.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-documents.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-documents.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-documents.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-documents.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-dtos.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-dtos.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-dtos.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-dtos.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json similarity index 77% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json index b9882642..aa6f9c3b 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-requests.snapshot.json @@ -3,7 +3,7 @@ "headers": { "accept": "application/zip,application/octet-stream,application/x-zip-compressed", "accept-Language": "ru-RU,ru; q=0.9,en-US; q=0.6,en; q=0.4", - "user-Agent": "StellaOps/Feedser,(+https://stella-ops.org)" + "user-Agent": "StellaOps/Concelier,(+https://stella-ops.org)" }, "method": "GET", "uri": "https://bdu.fstec.ru/files/documents/vulxml.zip" diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-state.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-state.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures/ru-bdu-state.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures/ru-bdu-state.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs index 618bf863..730b479c 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduConnectorSnapshotTests.cs @@ -1,303 +1,303 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using MongoDB.Bson; -using MongoDB.Bson.Serialization; -using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Ru.Bdu; -using StellaOps.Feedser.Source.Ru.Bdu.Configuration; -using StellaOps.Feedser.Source.Ru.Bdu.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; -using Xunit.Sdk; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Tests; - -[Collection("mongo-fixture")] -public sealed class RuBduConnectorSnapshotTests : IAsyncLifetime -{ - private const string UpdateFixturesVariable = "UPDATE_BDU_FIXTURES"; - private static readonly Uri ArchiveUri = new("https://bdu.fstec.ru/files/documents/vulxml.zip"); - - private readonly MongoIntegrationFixture _fixture; - private ConnectorTestHarness? _harness; - - public RuBduConnectorSnapshotTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - } - - [Fact] - public async Task FetchParseMap_ProducesDeterministicSnapshots() - { - var harness = await EnsureHarnessAsync(); - harness.Handler.AddResponse(ArchiveUri, BuildArchiveResponse); - - var connector = harness.ServiceProvider.GetRequiredService(); - await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); - - var stateRepository = harness.ServiceProvider.GetRequiredService(); - var initialState = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(initialState); - var cursorBeforeParse = initialState!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(initialState.Cursor); - Assert.NotEmpty(cursorBeforeParse.PendingDocuments); - var expectedDocumentIds = cursorBeforeParse.PendingDocuments.ToArray(); - - await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); - await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); - - var documentsCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Document); - var documentCount = await documentsCollection.CountDocumentsAsync(Builders.Filter.Empty); - Assert.True(documentCount > 0, "Expected persisted documents after map stage"); - - var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider, expectedDocumentIds); - WriteOrAssertSnapshot(documentsSnapshot, "ru-bdu-documents.snapshot.json"); - - var dtoSnapshot = await BuildDtoSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(dtoSnapshot, "ru-bdu-dtos.snapshot.json"); - - var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(advisoriesSnapshot, "ru-bdu-advisories.snapshot.json"); - - var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); - WriteOrAssertSnapshot(stateSnapshot, "ru-bdu-state.snapshot.json"); - - var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); - WriteOrAssertSnapshot(requestsSnapshot, "ru-bdu-requests.snapshot.json"); - - harness.Handler.AssertNoPendingResponses(); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public async Task DisposeAsync() - { - if (_harness is not null) - { - await _harness.DisposeAsync(); - _harness = null; - } - } - - private async Task EnsureHarnessAsync() - { - if (_harness is not null) - { - return _harness; - } - - var initialTime = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero); - var harness = new ConnectorTestHarness(_fixture, initialTime, RuBduOptions.HttpClientName); - await harness.EnsureServiceProviderAsync(services => - { - services.AddLogging(builder => - { - builder.ClearProviders(); - builder.AddProvider(NullLoggerProvider.Instance); - }); - - services.AddRuBduConnector(options => - { - options.BaseAddress = new Uri("https://bdu.fstec.ru/"); - options.DataArchivePath = "files/documents/vulxml.zip"; - options.MaxVulnerabilitiesPerFetch = 25; - options.RequestTimeout = TimeSpan.FromSeconds(30); - var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName, "ru-bdu"); - Directory.CreateDirectory(cacheRoot); - options.CacheDirectory = cacheRoot; - }); - - services.Configure(RuBduOptions.HttpClientName, options => - { - options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); - }); - }); - - _harness = harness; - return harness; - } - - private static HttpResponseMessage BuildArchiveResponse() - { - var archiveBytes = CreateArchiveBytes(); - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new ByteArrayContent(archiveBytes), - }; - response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); - response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero); - response.Content.Headers.ContentLength = archiveBytes.Length; - return response; - } - - private async Task BuildDocumentsSnapshotAsync(IServiceProvider provider, IReadOnlyCollection documentIds) - { - var documentStore = provider.GetRequiredService(); - var records = new List(documentIds.Count); - - foreach (var documentId in documentIds) - { - var record = await documentStore.FindAsync(documentId, CancellationToken.None); - if (record is null) - { - var existing = await _fixture.Database - .GetCollection("documents") - .Find(Builders.Filter.Empty) - .Project(Builders.Projection.Include("Uri")) - .ToListAsync(CancellationToken.None); - var uris = existing - .Select(document => document.GetValue("Uri", BsonValue.Create(string.Empty)).AsString) - .ToArray(); - throw new XunitException($"Document id not found: {documentId}. Known URIs: {string.Join(", ", uris)}"); - } - - records.Add(new - { - record.Uri, - record.Status, - record.Sha256, - Metadata = record.Metadata is null - ? null - : record.Metadata - .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) - .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase) - }); - } - - var ordered = records - .OrderBy(static entry => entry?.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) - .ToArray(); - - return SnapshotSerializer.ToSnapshot(ordered); - } - - private async Task BuildDtoSnapshotAsync(IServiceProvider provider) - { - var dtoStore = provider.GetRequiredService(); - var documentStore = provider.GetRequiredService(); - var records = await dtoStore.GetBySourceAsync(RuBduConnectorPlugin.SourceName, 25, CancellationToken.None); - - var entries = new List(records.Count); - foreach (var record in records.OrderBy(static r => r.DocumentId)) - { - var document = await documentStore.FindAsync(record.DocumentId, CancellationToken.None); - Assert.NotNull(document); - - var payload = BsonTypeMapper.MapToDotNetValue(record.Payload); - entries.Add(new - { - DocumentUri = document!.Uri, - record.SchemaVersion, - Payload = payload, - }); - } - - return SnapshotSerializer.ToSnapshot(entries.OrderBy(static entry => entry.GetType().GetProperty("DocumentUri")!.GetValue(entry)?.ToString(), StringComparer.Ordinal).ToArray()); - } - - private async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) - { - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(25, CancellationToken.None); - var ordered = advisories - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) - .ToArray(); - return SnapshotSerializer.ToSnapshot(ordered); - } - - private async Task BuildStateSnapshotAsync(IServiceProvider provider) - { - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - - var cursor = state!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor); - var snapshot = new - { - PendingDocuments = cursor.PendingDocuments.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), - PendingMappings = cursor.PendingMappings.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), - LastSuccessfulFetch = cursor.LastSuccessfulFetch?.ToUniversalTime().ToString("O"), - }; - - return SnapshotSerializer.ToSnapshot(snapshot); - } - - private static string BuildRequestsSnapshot(IReadOnlyCollection requests) - { - var ordered = requests - .Select(record => new - { - Method = record.Method.Method, - Uri = record.Uri.ToString(), - Headers = record.Headers - .OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase) - .ToDictionary(static kvp => kvp.Key, static kvp => kvp.Value, StringComparer.OrdinalIgnoreCase), - }) - .OrderBy(static entry => entry.Uri, StringComparer.Ordinal) - .ToArray(); - - return SnapshotSerializer.ToSnapshot(ordered); - } - - private static string ReadFixtureText(string filename) - { - var path = GetSourceFixturePath(filename); - return File.ReadAllText(path, Encoding.UTF8); - } - - private static byte[] CreateArchiveBytes() - { - var xml = ReadFixtureText("export-sample.xml"); - using var buffer = new MemoryStream(); - using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) - { - var entry = archive.CreateEntry("export/export.xml", CompressionLevel.NoCompression); - entry.LastWriteTime = new DateTimeOffset(2025, 10, 14, 9, 0, 0, TimeSpan.Zero); - using var entryStream = entry.Open(); - using var writer = new StreamWriter(entryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); - writer.Write(xml); - } - - return buffer.ToArray(); - } - - private static bool ShouldUpdateFixtures() - => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(UpdateFixturesVariable)); - - private static void WriteOrAssertSnapshot(string content, string filename) - { - var path = GetSourceFixturePath(filename); - if (ShouldUpdateFixtures()) - { - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, content, Encoding.UTF8); - } - else - { - Assert.True(File.Exists(path), $"Snapshot '{filename}' is missing. Run {UpdateFixturesVariable}=1 dotnet test to regenerate fixtures."); - var expected = File.ReadAllText(path, Encoding.UTF8); - Assert.Equal(expected, content); - } - } - - private static string GetSourceFixturePath(string relativeName) - => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", relativeName)); -} +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Ru.Bdu; +using StellaOps.Concelier.Connector.Ru.Bdu.Configuration; +using StellaOps.Concelier.Connector.Ru.Bdu.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; +using Xunit.Sdk; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests; + +[Collection("mongo-fixture")] +public sealed class RuBduConnectorSnapshotTests : IAsyncLifetime +{ + private const string UpdateFixturesVariable = "UPDATE_BDU_FIXTURES"; + private static readonly Uri ArchiveUri = new("https://bdu.fstec.ru/files/documents/vulxml.zip"); + + private readonly MongoIntegrationFixture _fixture; + private ConnectorTestHarness? _harness; + + public RuBduConnectorSnapshotTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task FetchParseMap_ProducesDeterministicSnapshots() + { + var harness = await EnsureHarnessAsync(); + harness.Handler.AddResponse(ArchiveUri, BuildArchiveResponse); + + var connector = harness.ServiceProvider.GetRequiredService(); + await connector.FetchAsync(harness.ServiceProvider, CancellationToken.None); + + var stateRepository = harness.ServiceProvider.GetRequiredService(); + var initialState = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(initialState); + var cursorBeforeParse = initialState!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(initialState.Cursor); + Assert.NotEmpty(cursorBeforeParse.PendingDocuments); + var expectedDocumentIds = cursorBeforeParse.PendingDocuments.ToArray(); + + await connector.ParseAsync(harness.ServiceProvider, CancellationToken.None); + await connector.MapAsync(harness.ServiceProvider, CancellationToken.None); + + var documentsCollection = _fixture.Database.GetCollection(MongoStorageDefaults.Collections.Document); + var documentCount = await documentsCollection.CountDocumentsAsync(Builders.Filter.Empty); + Assert.True(documentCount > 0, "Expected persisted documents after map stage"); + + var documentsSnapshot = await BuildDocumentsSnapshotAsync(harness.ServiceProvider, expectedDocumentIds); + WriteOrAssertSnapshot(documentsSnapshot, "ru-bdu-documents.snapshot.json"); + + var dtoSnapshot = await BuildDtoSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(dtoSnapshot, "ru-bdu-dtos.snapshot.json"); + + var advisoriesSnapshot = await BuildAdvisoriesSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(advisoriesSnapshot, "ru-bdu-advisories.snapshot.json"); + + var stateSnapshot = await BuildStateSnapshotAsync(harness.ServiceProvider); + WriteOrAssertSnapshot(stateSnapshot, "ru-bdu-state.snapshot.json"); + + var requestsSnapshot = BuildRequestsSnapshot(harness.Handler.Requests); + WriteOrAssertSnapshot(requestsSnapshot, "ru-bdu-requests.snapshot.json"); + + harness.Handler.AssertNoPendingResponses(); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + { + if (_harness is not null) + { + await _harness.DisposeAsync(); + _harness = null; + } + } + + private async Task EnsureHarnessAsync() + { + if (_harness is not null) + { + return _harness; + } + + var initialTime = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero); + var harness = new ConnectorTestHarness(_fixture, initialTime, RuBduOptions.HttpClientName); + await harness.EnsureServiceProviderAsync(services => + { + services.AddLogging(builder => + { + builder.ClearProviders(); + builder.AddProvider(NullLoggerProvider.Instance); + }); + + services.AddRuBduConnector(options => + { + options.BaseAddress = new Uri("https://bdu.fstec.ru/"); + options.DataArchivePath = "files/documents/vulxml.zip"; + options.MaxVulnerabilitiesPerFetch = 25; + options.RequestTimeout = TimeSpan.FromSeconds(30); + var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName, "ru-bdu"); + Directory.CreateDirectory(cacheRoot); + options.CacheDirectory = cacheRoot; + }); + + services.Configure(RuBduOptions.HttpClientName, options => + { + options.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = harness.Handler); + }); + }); + + _harness = harness; + return harness; + } + + private static HttpResponseMessage BuildArchiveResponse() + { + var archiveBytes = CreateArchiveBytes(); + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(archiveBytes), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero); + response.Content.Headers.ContentLength = archiveBytes.Length; + return response; + } + + private async Task BuildDocumentsSnapshotAsync(IServiceProvider provider, IReadOnlyCollection documentIds) + { + var documentStore = provider.GetRequiredService(); + var records = new List(documentIds.Count); + + foreach (var documentId in documentIds) + { + var record = await documentStore.FindAsync(documentId, CancellationToken.None); + if (record is null) + { + var existing = await _fixture.Database + .GetCollection("documents") + .Find(Builders.Filter.Empty) + .Project(Builders.Projection.Include("Uri")) + .ToListAsync(CancellationToken.None); + var uris = existing + .Select(document => document.GetValue("Uri", BsonValue.Create(string.Empty)).AsString) + .ToArray(); + throw new XunitException($"Document id not found: {documentId}. Known URIs: {string.Join(", ", uris)}"); + } + + records.Add(new + { + record.Uri, + record.Status, + record.Sha256, + Metadata = record.Metadata is null + ? null + : record.Metadata + .OrderBy(static pair => pair.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.OrdinalIgnoreCase) + }); + } + + var ordered = records + .OrderBy(static entry => entry?.GetType().GetProperty("Uri")?.GetValue(entry)?.ToString(), StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private async Task BuildDtoSnapshotAsync(IServiceProvider provider) + { + var dtoStore = provider.GetRequiredService(); + var documentStore = provider.GetRequiredService(); + var records = await dtoStore.GetBySourceAsync(RuBduConnectorPlugin.SourceName, 25, CancellationToken.None); + + var entries = new List(records.Count); + foreach (var record in records.OrderBy(static r => r.DocumentId)) + { + var document = await documentStore.FindAsync(record.DocumentId, CancellationToken.None); + Assert.NotNull(document); + + var payload = BsonTypeMapper.MapToDotNetValue(record.Payload); + entries.Add(new + { + DocumentUri = document!.Uri, + record.SchemaVersion, + Payload = payload, + }); + } + + return SnapshotSerializer.ToSnapshot(entries.OrderBy(static entry => entry.GetType().GetProperty("DocumentUri")!.GetValue(entry)?.ToString(), StringComparer.Ordinal).ToArray()); + } + + private async Task BuildAdvisoriesSnapshotAsync(IServiceProvider provider) + { + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(25, CancellationToken.None); + var ordered = advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + return SnapshotSerializer.ToSnapshot(ordered); + } + + private async Task BuildStateSnapshotAsync(IServiceProvider provider) + { + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(RuBduConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + + var cursor = state!.Cursor is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor); + var snapshot = new + { + PendingDocuments = cursor.PendingDocuments.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), + PendingMappings = cursor.PendingMappings.Select(static guid => guid.ToString()).OrderBy(static id => id, StringComparer.Ordinal).ToArray(), + LastSuccessfulFetch = cursor.LastSuccessfulFetch?.ToUniversalTime().ToString("O"), + }; + + return SnapshotSerializer.ToSnapshot(snapshot); + } + + private static string BuildRequestsSnapshot(IReadOnlyCollection requests) + { + var ordered = requests + .Select(record => new + { + Method = record.Method.Method, + Uri = record.Uri.ToString(), + Headers = record.Headers + .OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase) + .ToDictionary(static kvp => kvp.Key, static kvp => kvp.Value, StringComparer.OrdinalIgnoreCase), + }) + .OrderBy(static entry => entry.Uri, StringComparer.Ordinal) + .ToArray(); + + return SnapshotSerializer.ToSnapshot(ordered); + } + + private static string ReadFixtureText(string filename) + { + var path = GetSourceFixturePath(filename); + return File.ReadAllText(path, Encoding.UTF8); + } + + private static byte[] CreateArchiveBytes() + { + var xml = ReadFixtureText("export-sample.xml"); + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry("export/export.xml", CompressionLevel.NoCompression); + entry.LastWriteTime = new DateTimeOffset(2025, 10, 14, 9, 0, 0, TimeSpan.Zero); + using var entryStream = entry.Open(); + using var writer = new StreamWriter(entryStream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + writer.Write(xml); + } + + return buffer.ToArray(); + } + + private static bool ShouldUpdateFixtures() + => !string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable(UpdateFixturesVariable)); + + private static void WriteOrAssertSnapshot(string content, string filename) + { + var path = GetSourceFixturePath(filename); + if (ShouldUpdateFixtures()) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content, Encoding.UTF8); + } + else + { + Assert.True(File.Exists(path), $"Snapshot '{filename}' is missing. Run {UpdateFixturesVariable}=1 dotnet test to regenerate fixtures."); + var expected = File.ReadAllText(path, Encoding.UTF8); + Assert.Equal(expected, content); + } + } + + private static string GetSourceFixturePath(string relativeName) + => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", relativeName)); +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduMapperTests.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduMapperTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduMapperTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduMapperTests.cs index d10ae342..34086751 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduMapperTests.cs @@ -1,95 +1,95 @@ -using System.Collections.Immutable; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Ru.Bdu.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using Xunit; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Tests; - -public sealed class RuBduMapperTests -{ - [Fact] - public void Map_ConstructsCanonicalAdvisory() - { - var dto = new RuBduVulnerabilityDto( - Identifier: "BDU:2025-12345", - Name: "Уязвимость тестового продукта", - Description: "Описание", - Solution: "Обновить", - IdentifyDate: new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero), - SeverityText: "Высокий уровень опасности", - CvssVector: "AV:N/AC:L/Au:N/C:P/I:P/A:P", - CvssScore: 7.5, - Cvss3Vector: "AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - Cvss3Score: 9.8, - ExploitStatus: "Существует", - IncidentCount: 2, - FixStatus: "Уязвимость устранена", - VulStatus: "Подтверждена производителем", - VulClass: null, - VulState: null, - Other: null, - Software: new[] - { - new RuBduSoftwareDto( - "ООО Вендор", - "Продукт", - "1.2.3;1.2.4", - "Windows", - new[] { "ПО программно-аппаратного средства АСУ ТП" }.ToImmutableArray()) - }.ToImmutableArray(), - Environment: ImmutableArray.Empty, - Cwes: new[] { new RuBduCweDto("CWE-79", "XSS"), new RuBduCweDto("CWE-89", "SQL Injection") }.ToImmutableArray(), - Sources: new[] - { - "https://advisories.example/BDU-2025-12345", - "www.example.com/ru-bdu/BDU-2025-12345" - }.ToImmutableArray(), - Identifiers: new[] - { - new RuBduExternalIdentifierDto("CVE", "CVE-2025-12345", "https://nvd.nist.gov/vuln/detail/CVE-2025-12345"), - new RuBduExternalIdentifierDto("Positive Technologies Advisory", "PT-2025-001", "https://ptsecurity.com/PT-2025-001") - }.ToImmutableArray()); - - var document = new DocumentRecord( - Guid.NewGuid(), - RuBduConnectorPlugin.SourceName, - "https://bdu.fstec.ru/vul/2025-12345", - DateTimeOffset.UtcNow, - "abc", - DocumentStatuses.PendingMap, - "application/json", - null, - null, - null, - dto.IdentifyDate, - ObjectId.GenerateNewId()); - - var advisory = RuBduMapper.Map(dto, document, dto.IdentifyDate!.Value); - - Assert.Equal("BDU:2025-12345", advisory.AdvisoryKey); - Assert.Contains("BDU:2025-12345", advisory.Aliases); - Assert.Contains("CVE-2025-12345", advisory.Aliases); - Assert.Equal("critical", advisory.Severity); - Assert.True(advisory.ExploitKnown); - - var package = Assert.Single(advisory.AffectedPackages); - Assert.Equal(AffectedPackageTypes.IcsVendor, package.Type); - Assert.Equal(2, package.VersionRanges.Length); - Assert.Equal(2, package.NormalizedVersions.Length); - Assert.All(package.NormalizedVersions, rule => Assert.Equal("ru-bdu.raw", rule.Scheme)); - Assert.Contains(package.NormalizedVersions, rule => rule.Value == "1.2.3"); - Assert.Contains(package.NormalizedVersions, rule => rule.Value == "1.2.4"); - Assert.Contains(package.Statuses, status => status.Status == AffectedPackageStatusCatalog.Affected); - Assert.Contains(package.Statuses, status => status.Status == AffectedPackageStatusCatalog.Fixed); - - Assert.Equal(2, advisory.CvssMetrics.Length); - Assert.Contains(advisory.References, reference => reference.Url == "https://bdu.fstec.ru/vul/2025-12345" && reference.Kind == "details"); - Assert.Contains(advisory.References, reference => reference.Url == "https://nvd.nist.gov/vuln/detail/CVE-2025-12345" && reference.Kind == "cve"); - Assert.Contains(advisory.References, reference => reference.Url == "https://advisories.example/BDU-2025-12345" && reference.Kind == "source"); - Assert.Contains(advisory.References, reference => reference.Url == "https://www.example.com/ru-bdu/BDU-2025-12345" && reference.Kind == "source"); - Assert.Contains(advisory.References, reference => reference.SourceTag == "positivetechnologiesadvisory"); - } -} +using System.Collections.Immutable; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Ru.Bdu.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests; + +public sealed class RuBduMapperTests +{ + [Fact] + public void Map_ConstructsCanonicalAdvisory() + { + var dto = new RuBduVulnerabilityDto( + Identifier: "BDU:2025-12345", + Name: "Уязвимость тестового продукта", + Description: "Описание", + Solution: "Обновить", + IdentifyDate: new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero), + SeverityText: "Высокий уровень опасности", + CvssVector: "AV:N/AC:L/Au:N/C:P/I:P/A:P", + CvssScore: 7.5, + Cvss3Vector: "AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + Cvss3Score: 9.8, + ExploitStatus: "Существует", + IncidentCount: 2, + FixStatus: "Уязвимость устранена", + VulStatus: "Подтверждена производителем", + VulClass: null, + VulState: null, + Other: null, + Software: new[] + { + new RuBduSoftwareDto( + "ООО Вендор", + "Продукт", + "1.2.3;1.2.4", + "Windows", + new[] { "ПО программно-аппаратного средства АСУ ТП" }.ToImmutableArray()) + }.ToImmutableArray(), + Environment: ImmutableArray.Empty, + Cwes: new[] { new RuBduCweDto("CWE-79", "XSS"), new RuBduCweDto("CWE-89", "SQL Injection") }.ToImmutableArray(), + Sources: new[] + { + "https://advisories.example/BDU-2025-12345", + "www.example.com/ru-bdu/BDU-2025-12345" + }.ToImmutableArray(), + Identifiers: new[] + { + new RuBduExternalIdentifierDto("CVE", "CVE-2025-12345", "https://nvd.nist.gov/vuln/detail/CVE-2025-12345"), + new RuBduExternalIdentifierDto("Positive Technologies Advisory", "PT-2025-001", "https://ptsecurity.com/PT-2025-001") + }.ToImmutableArray()); + + var document = new DocumentRecord( + Guid.NewGuid(), + RuBduConnectorPlugin.SourceName, + "https://bdu.fstec.ru/vul/2025-12345", + DateTimeOffset.UtcNow, + "abc", + DocumentStatuses.PendingMap, + "application/json", + null, + null, + null, + dto.IdentifyDate, + ObjectId.GenerateNewId()); + + var advisory = RuBduMapper.Map(dto, document, dto.IdentifyDate!.Value); + + Assert.Equal("BDU:2025-12345", advisory.AdvisoryKey); + Assert.Contains("BDU:2025-12345", advisory.Aliases); + Assert.Contains("CVE-2025-12345", advisory.Aliases); + Assert.Equal("critical", advisory.Severity); + Assert.True(advisory.ExploitKnown); + + var package = Assert.Single(advisory.AffectedPackages); + Assert.Equal(AffectedPackageTypes.IcsVendor, package.Type); + Assert.Equal(2, package.VersionRanges.Length); + Assert.Equal(2, package.NormalizedVersions.Length); + Assert.All(package.NormalizedVersions, rule => Assert.Equal("ru-bdu.raw", rule.Scheme)); + Assert.Contains(package.NormalizedVersions, rule => rule.Value == "1.2.3"); + Assert.Contains(package.NormalizedVersions, rule => rule.Value == "1.2.4"); + Assert.Contains(package.Statuses, status => status.Status == AffectedPackageStatusCatalog.Affected); + Assert.Contains(package.Statuses, status => status.Status == AffectedPackageStatusCatalog.Fixed); + + Assert.Equal(2, advisory.CvssMetrics.Length); + Assert.Contains(advisory.References, reference => reference.Url == "https://bdu.fstec.ru/vul/2025-12345" && reference.Kind == "details"); + Assert.Contains(advisory.References, reference => reference.Url == "https://nvd.nist.gov/vuln/detail/CVE-2025-12345" && reference.Kind == "cve"); + Assert.Contains(advisory.References, reference => reference.Url == "https://advisories.example/BDU-2025-12345" && reference.Kind == "source"); + Assert.Contains(advisory.References, reference => reference.Url == "https://www.example.com/ru-bdu/BDU-2025-12345" && reference.Kind == "source"); + Assert.Contains(advisory.References, reference => reference.SourceTag == "positivetechnologiesadvisory"); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduXmlParserTests.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduXmlParserTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduXmlParserTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduXmlParserTests.cs index 903ef628..966700f1 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/RuBduXmlParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/RuBduXmlParserTests.cs @@ -1,93 +1,93 @@ -using System.IO; -using System.Xml.Linq; -using StellaOps.Feedser.Source.Ru.Bdu.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Tests; - -public sealed class RuBduXmlParserTests -{ - [Fact] - public void TryParse_ValidElement_ReturnsDto() - { - const string xml = """ - - BDU:2025-12345 - Уязвимость тестового продукта - Описание уязвимости - Обновить продукт - 2025-10-10 - Высокий уровень опасности - Существует эксплойт - Устранена - Подтверждена производителем - 1 - - AV:N/AC:L/Au:N/C:P/I:P/A:P - - - AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H - - - - ООО «Вендор» - Продукт - 1.2.3 - Windows - - ics - - - - - https://advisories.example/BDU-2025-12345 - https://mirror.example/ru-bdu/BDU-2025-12345 - - - CVE-2025-12345 - GHSA-xxxx-yyyy-zzzz - - - - CWE-79 - XSS - - - -"""; - - var element = XElement.Parse(xml); - var dto = RuBduXmlParser.TryParse(element); - - Assert.NotNull(dto); - Assert.Equal("BDU:2025-12345", dto!.Identifier); - Assert.Equal("Уязвимость тестового продукта", dto.Name); - Assert.Equal("AV:N/AC:L/Au:N/C:P/I:P/A:P", dto.CvssVector); - Assert.Equal(7.5, dto.CvssScore); - Assert.Equal("AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", dto.Cvss3Vector); - Assert.Equal(9.8, dto.Cvss3Score); - Assert.Single(dto.Software); - Assert.Single(dto.Cwes); - Assert.Equal(2, dto.Sources.Length); - Assert.Contains("https://advisories.example/BDU-2025-12345", dto.Sources); - Assert.Equal(2, dto.Identifiers.Length); - Assert.Contains(dto.Identifiers, identifier => identifier.Type == "CVE" && identifier.Value == "CVE-2025-12345"); - Assert.Contains(dto.Identifiers, identifier => identifier.Type == "GHSA" && identifier.Link == "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz"); - } - - [Fact] - public void TryParse_SampleArchiveEntries_ReturnDtos() - { - var path = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "export-sample.xml")); - var document = XDocument.Load(path); - var vulnerabilities = document.Root?.Elements("vul"); - Assert.NotNull(vulnerabilities); - - foreach (var element in vulnerabilities!) - { - var dto = RuBduXmlParser.TryParse(element); - Assert.NotNull(dto); - Assert.False(string.IsNullOrWhiteSpace(dto!.Identifier)); - } - } -} +using System.IO; +using System.Xml.Linq; +using StellaOps.Concelier.Connector.Ru.Bdu.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Tests; + +public sealed class RuBduXmlParserTests +{ + [Fact] + public void TryParse_ValidElement_ReturnsDto() + { + const string xml = """ + + BDU:2025-12345 + Уязвимость тестового продукта + Описание уязвимости + Обновить продукт + 2025-10-10 + Высокий уровень опасности + Существует эксплойт + Устранена + Подтверждена производителем + 1 + + AV:N/AC:L/Au:N/C:P/I:P/A:P + + + AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H + + + + ООО «Вендор» + Продукт + 1.2.3 + Windows + + ics + + + + + https://advisories.example/BDU-2025-12345 + https://mirror.example/ru-bdu/BDU-2025-12345 + + + CVE-2025-12345 + GHSA-xxxx-yyyy-zzzz + + + + CWE-79 + XSS + + + +"""; + + var element = XElement.Parse(xml); + var dto = RuBduXmlParser.TryParse(element); + + Assert.NotNull(dto); + Assert.Equal("BDU:2025-12345", dto!.Identifier); + Assert.Equal("Уязвимость тестового продукта", dto.Name); + Assert.Equal("AV:N/AC:L/Au:N/C:P/I:P/A:P", dto.CvssVector); + Assert.Equal(7.5, dto.CvssScore); + Assert.Equal("AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", dto.Cvss3Vector); + Assert.Equal(9.8, dto.Cvss3Score); + Assert.Single(dto.Software); + Assert.Single(dto.Cwes); + Assert.Equal(2, dto.Sources.Length); + Assert.Contains("https://advisories.example/BDU-2025-12345", dto.Sources); + Assert.Equal(2, dto.Identifiers.Length); + Assert.Contains(dto.Identifiers, identifier => identifier.Type == "CVE" && identifier.Value == "CVE-2025-12345"); + Assert.Contains(dto.Identifiers, identifier => identifier.Type == "GHSA" && identifier.Link == "https://github.com/advisories/GHSA-xxxx-yyyy-zzzz"); + } + + [Fact] + public void TryParse_SampleArchiveEntries_ReturnDtos() + { + var path = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "export-sample.xml")); + var document = XDocument.Load(path); + var vulnerabilities = document.Root?.Elements("vul"); + Assert.NotNull(vulnerabilities); + + foreach (var element in vulnerabilities!) + { + var dto = RuBduXmlParser.TryParse(element); + Assert.NotNull(dto); + Assert.False(string.IsNullOrWhiteSpace(dto!.Identifier)); + } + } +} diff --git a/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/StellaOps.Concelier.Connector.Ru.Bdu.Tests.csproj b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/StellaOps.Concelier.Connector.Ru.Bdu.Tests.csproj new file mode 100644 index 00000000..8cb9bc6d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/StellaOps.Concelier.Connector.Ru.Bdu.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md b/src/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md similarity index 86% rename from src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md rename to src/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md index 17a9d357..3dbe3c19 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/AGENTS.md @@ -1,38 +1,38 @@ -# AGENTS -## Role -Implement the Russian BDU (Vulnerability Database) connector to ingest advisories published by FSTEC’s BDU catalogue. - -## Scope -- Determine accessible BDU feeds/APIs (HTML listings, downloadable CSV, SOAP/REST) and access constraints. -- Build fetch/cursor pipeline with dedupe, retries, and backoff appropriate for the data source. -- Parse advisory records to extract summary, affected vendors/products, mitigation recommendations, CVE IDs. -- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and range primitives. -- Provide deterministic fixtures and regression tests for the connector lifecycle. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). -- `Feedser.Models` (canonical data structures). -- `Feedser.Testing` (integration harness, snapshot utilities). - -## Interfaces & Contracts -- Job kinds: `bdu:fetch`, `bdu:parse`, `bdu:map`. -- Persist upstream metadata (e.g., record modification timestamp) to drive incremental updates. -- Alias set should include BDU identifiers and CVE IDs when present. - -## In/Out of scope -In scope: -- Core ingestion/mapping of BDU vulnerability records. - -Out of scope: -- Translation beyond normalising required canonical fields. - -## Observability & Security Expectations -- Log fetch/mapping statistics and failure details. -- Sanitize source payloads, handling Cyrillic text/encodings correctly. -- Respect upstream rate limits and mark failures with backoff. - -## Tests -- Add `StellaOps.Feedser.Source.Ru.Bdu.Tests` covering fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Implement the Russian BDU (Vulnerability Database) connector to ingest advisories published by FSTEC’s BDU catalogue. + +## Scope +- Determine accessible BDU feeds/APIs (HTML listings, downloadable CSV, SOAP/REST) and access constraints. +- Build fetch/cursor pipeline with dedupe, retries, and backoff appropriate for the data source. +- Parse advisory records to extract summary, affected vendors/products, mitigation recommendations, CVE IDs. +- Map advisories into canonical `Advisory` objects including aliases, references, affected packages, and range primitives. +- Provide deterministic fixtures and regression tests for the connector lifecycle. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores + source state). +- `Concelier.Models` (canonical data structures). +- `Concelier.Testing` (integration harness, snapshot utilities). + +## Interfaces & Contracts +- Job kinds: `bdu:fetch`, `bdu:parse`, `bdu:map`. +- Persist upstream metadata (e.g., record modification timestamp) to drive incremental updates. +- Alias set should include BDU identifiers and CVE IDs when present. + +## In/Out of scope +In scope: +- Core ingestion/mapping of BDU vulnerability records. + +Out of scope: +- Translation beyond normalising required canonical fields. + +## Observability & Security Expectations +- Log fetch/mapping statistics and failure details. +- Sanitize source payloads, handling Cyrillic text/encodings correctly. +- Respect upstream rate limits and mark failures with backoff. + +## Tests +- Add `StellaOps.Concelier.Connector.Ru.Bdu.Tests` covering fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Configuration/RuBduOptions.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Configuration/RuBduOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Configuration/RuBduOptions.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Configuration/RuBduOptions.cs index 6c93d189..e3867402 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Configuration/RuBduOptions.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Configuration/RuBduOptions.cs @@ -1,102 +1,102 @@ -using System.Net; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Configuration; - -/// -/// Connector options for the Russian BDU archive ingestion pipeline. -/// -public sealed class RuBduOptions -{ - public const string HttpClientName = "ru-bdu"; - - private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromMinutes(2); - private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(30); - - /// - /// Base endpoint used for resolving relative resource paths. - /// - public Uri BaseAddress { get; set; } = new("https://bdu.fstec.ru/", UriKind.Absolute); - - /// - /// Relative path to the zipped vulnerability dataset. - /// - public string DataArchivePath { get; set; } = "files/documents/vulxml.zip"; - - /// - /// HTTP timeout applied when downloading the archive. - /// - public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; - - /// - /// Backoff applied when the remote endpoint fails to serve the archive. - /// - public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; - - /// - /// User-Agent header used for outbound requests. - /// - public string UserAgent { get; set; } = "StellaOps/Feedser (+https://stella-ops.org)"; - - /// - /// Accept-Language preference sent with outbound requests. - /// - public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4"; - - /// - /// Maximum number of vulnerabilities ingested per fetch cycle. - /// - public int MaxVulnerabilitiesPerFetch { get; set; } = 500; - - /// - /// Returns the absolute URI for the archive download. - /// - public Uri DataArchiveUri => new(BaseAddress, DataArchivePath); - - /// - /// Optional directory for caching the most recent archive (relative paths resolve under the content root). - /// - public string? CacheDirectory { get; set; } = null; - - public void Validate() - { - if (BaseAddress is null || !BaseAddress.IsAbsoluteUri) - { - throw new InvalidOperationException("RuBdu BaseAddress must be an absolute URI."); - } - - if (string.IsNullOrWhiteSpace(DataArchivePath)) - { - throw new InvalidOperationException("RuBdu DataArchivePath must be provided."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("RuBdu RequestTimeout must be positive."); - } - - if (FailureBackoff < TimeSpan.Zero) - { - throw new InvalidOperationException("RuBdu FailureBackoff cannot be negative."); - } - - if (string.IsNullOrWhiteSpace(UserAgent)) - { - throw new InvalidOperationException("RuBdu UserAgent cannot be empty."); - } - - if (string.IsNullOrWhiteSpace(AcceptLanguage)) - { - throw new InvalidOperationException("RuBdu AcceptLanguage cannot be empty."); - } - - if (MaxVulnerabilitiesPerFetch <= 0) - { - throw new InvalidOperationException("RuBdu MaxVulnerabilitiesPerFetch must be greater than zero."); - } - - if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0) - { - throw new InvalidOperationException("RuBdu CacheDirectory cannot be whitespace."); - } - } -} +using System.Net; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Configuration; + +/// +/// Connector options for the Russian BDU archive ingestion pipeline. +/// +public sealed class RuBduOptions +{ + public const string HttpClientName = "ru-bdu"; + + private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromMinutes(2); + private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(30); + + /// + /// Base endpoint used for resolving relative resource paths. + /// + public Uri BaseAddress { get; set; } = new("https://bdu.fstec.ru/", UriKind.Absolute); + + /// + /// Relative path to the zipped vulnerability dataset. + /// + public string DataArchivePath { get; set; } = "files/documents/vulxml.zip"; + + /// + /// HTTP timeout applied when downloading the archive. + /// + public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; + + /// + /// Backoff applied when the remote endpoint fails to serve the archive. + /// + public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; + + /// + /// User-Agent header used for outbound requests. + /// + public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)"; + + /// + /// Accept-Language preference sent with outbound requests. + /// + public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4"; + + /// + /// Maximum number of vulnerabilities ingested per fetch cycle. + /// + public int MaxVulnerabilitiesPerFetch { get; set; } = 500; + + /// + /// Returns the absolute URI for the archive download. + /// + public Uri DataArchiveUri => new(BaseAddress, DataArchivePath); + + /// + /// Optional directory for caching the most recent archive (relative paths resolve under the content root). + /// + public string? CacheDirectory { get; set; } = null; + + public void Validate() + { + if (BaseAddress is null || !BaseAddress.IsAbsoluteUri) + { + throw new InvalidOperationException("RuBdu BaseAddress must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(DataArchivePath)) + { + throw new InvalidOperationException("RuBdu DataArchivePath must be provided."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RuBdu RequestTimeout must be positive."); + } + + if (FailureBackoff < TimeSpan.Zero) + { + throw new InvalidOperationException("RuBdu FailureBackoff cannot be negative."); + } + + if (string.IsNullOrWhiteSpace(UserAgent)) + { + throw new InvalidOperationException("RuBdu UserAgent cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(AcceptLanguage)) + { + throw new InvalidOperationException("RuBdu AcceptLanguage cannot be empty."); + } + + if (MaxVulnerabilitiesPerFetch <= 0) + { + throw new InvalidOperationException("RuBdu MaxVulnerabilitiesPerFetch must be greater than zero."); + } + + if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0) + { + throw new InvalidOperationException("RuBdu CacheDirectory cannot be whitespace."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduCursor.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduCursor.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduCursor.cs index fea10cb8..975e132c 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduCursor.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduCursor.cs @@ -1,81 +1,81 @@ -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Internal; - -internal sealed record RuBduCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - DateTimeOffset? LastSuccessfulFetch) -{ - private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); - - public static RuBduCursor Empty { get; } = new(EmptyGuids, EmptyGuids, null); - - public RuBduCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = (documents ?? Enumerable.Empty()).Distinct().ToArray() }; - - public RuBduCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = (mappings ?? Enumerable.Empty()).Distinct().ToArray() }; - - public RuBduCursor WithLastSuccessfulFetch(DateTimeOffset? timestamp) - => this with { LastSuccessfulFetch = timestamp }; - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastSuccessfulFetch.HasValue) - { - document["lastSuccessfulFetch"] = LastSuccessfulFetch.Value.UtcDateTime; - } - - return document; - } - - public static RuBduCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var lastFetch = document.TryGetValue("lastSuccessfulFetch", out var fetchValue) - ? ParseDate(fetchValue) - : null; - - return new RuBduCursor(pendingDocuments, pendingMappings, lastFetch); - } - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuids; - } - - var result = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var guid)) - { - result.Add(guid); - } - } - - return result; - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal; + +internal sealed record RuBduCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + DateTimeOffset? LastSuccessfulFetch) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + + public static RuBduCursor Empty { get; } = new(EmptyGuids, EmptyGuids, null); + + public RuBduCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = (documents ?? Enumerable.Empty()).Distinct().ToArray() }; + + public RuBduCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = (mappings ?? Enumerable.Empty()).Distinct().ToArray() }; + + public RuBduCursor WithLastSuccessfulFetch(DateTimeOffset? timestamp) + => this with { LastSuccessfulFetch = timestamp }; + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastSuccessfulFetch.HasValue) + { + document["lastSuccessfulFetch"] = LastSuccessfulFetch.Value.UtcDateTime; + } + + return document; + } + + public static RuBduCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var lastFetch = document.TryGetValue("lastSuccessfulFetch", out var fetchValue) + ? ParseDate(fetchValue) + : null; + + return new RuBduCursor(pendingDocuments, pendingMappings, lastFetch); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduDiagnostics.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduDiagnostics.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduDiagnostics.cs index c39d8963..181a6c45 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduDiagnostics.cs @@ -1,15 +1,15 @@ using System; using System.Diagnostics.Metrics; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Source.Ru.Bdu.Internal; +namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal; /// /// Emits RU-BDU specific OpenTelemetry metrics for fetch/parse/map stages. /// public sealed class RuBduDiagnostics : IDisposable { - private const string MeterName = "StellaOps.Feedser.Source.Ru.Bdu"; + private const string MeterName = "StellaOps.Concelier.Connector.Ru.Bdu"; private const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduMapper.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduMapper.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduMapper.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduMapper.cs index eacb61c8..2d8151b3 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduMapper.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduMapper.cs @@ -1,554 +1,554 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using System.Linq; -using System.Text; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Internal; - -internal static class RuBduMapper -{ - private const string RawVersionScheme = "ru-bdu.raw"; - - public static Advisory Map(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var advisoryProvenance = new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "advisory", - dto.Identifier, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - - var aliases = BuildAliases(dto); - var packages = BuildPackages(dto, recordedAt); - var references = BuildReferences(dto, document, recordedAt); - var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss); - var severity = severityFromCvss ?? NormalizeSeverity(dto.SeverityText); - var exploitKnown = DetermineExploitKnown(dto); - - return new Advisory( - advisoryKey: dto.Identifier, - title: dto.Name ?? dto.Identifier, - summary: dto.Description, - language: "ru", - published: dto.IdentifyDate, - modified: dto.IdentifyDate, - severity: severity, - exploitKnown: exploitKnown, - aliases: aliases, - references: references, - affectedPackages: packages, - cvssMetrics: cvssMetrics, - provenance: new[] { advisoryProvenance }); - } - - private static IReadOnlyList BuildAliases(RuBduVulnerabilityDto dto) - { - var aliases = new HashSet(StringComparer.Ordinal); - - if (!string.IsNullOrWhiteSpace(dto.Identifier)) - { - aliases.Add(dto.Identifier.Trim()); - } - - foreach (var identifier in dto.Identifiers) - { - if (string.IsNullOrWhiteSpace(identifier.Value)) - { - continue; - } - - aliases.Add(identifier.Value.Trim()); - } - - return aliases.Count == 0 ? Array.Empty() : aliases.ToArray(); - } - - private static IReadOnlyList BuildPackages(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt) - { - if (dto.Software.IsDefaultOrEmpty) - { - return Array.Empty(); - } - - var packages = new List(dto.Software.Length); - foreach (var software in dto.Software) - { - if (string.IsNullOrWhiteSpace(software.Name) && string.IsNullOrWhiteSpace(software.Vendor)) - { - continue; - } - - var identifier = BuildPackageIdentifier(dto.Identifier, software); - var packageProvenance = new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "package", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.AffectedPackages }); - - var statuses = BuildPackageStatuses(dto, recordedAt); - var ranges = BuildVersionRanges(software, recordedAt); - var normalizedVersions = BuildNormalizedVersions(software); - - packages.Add(new AffectedPackage( - DeterminePackageType(software.Types), - identifier, - platform: NormalizePlatform(software.Platform), - versionRanges: ranges, - statuses: statuses, - provenance: new[] { packageProvenance }, - normalizedVersions: normalizedVersions)); - } - - return packages; - } - - private static string BuildPackageIdentifier(string fallbackIdentifier, RuBduSoftwareDto software) - { - var parts = new[] { software.Vendor, software.Name } - .Where(static part => !string.IsNullOrWhiteSpace(part)) - .Select(static part => part!.Trim()) - .ToArray(); - - if (parts.Length == 0) - { - return software.Name ?? software.Vendor ?? fallbackIdentifier; - } - - return string.Join(" ", parts); - } - - private static IReadOnlyList BuildPackageStatuses(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt) - { - var statuses = new List(capacity: 2); - - if (TryNormalizeVulnerabilityStatus(dto.VulStatus, out var vulnerabilityStatus)) - { - statuses.Add(new AffectedPackageStatus( - vulnerabilityStatus!, - new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "package-status", - dto.VulStatus!, - recordedAt, - new[] { ProvenanceFieldMasks.PackageStatuses }))); - } - - if (TryNormalizeFixStatus(dto.FixStatus, out var fixStatus)) - { - statuses.Add(new AffectedPackageStatus( - fixStatus!, - new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "package-fix-status", - dto.FixStatus!, - recordedAt, - new[] { ProvenanceFieldMasks.PackageStatuses }))); - } - - return statuses.Count == 0 ? Array.Empty() : statuses; - } - - private static bool TryNormalizeVulnerabilityStatus(string? status, out string? normalized) - { - normalized = null; - if (string.IsNullOrWhiteSpace(status)) - { - return false; - } - - var token = status.Trim().ToLowerInvariant(); - if (token.Contains("потенциал", StringComparison.Ordinal)) - { - normalized = AffectedPackageStatusCatalog.UnderInvestigation; - return true; - } - - if (token.Contains("подтвержд", StringComparison.Ordinal)) - { - normalized = AffectedPackageStatusCatalog.Affected; - return true; - } - - if (token.Contains("актуал", StringComparison.Ordinal)) - { - normalized = AffectedPackageStatusCatalog.Affected; - return true; - } - - return false; - } - - private static bool TryNormalizeFixStatus(string? status, out string? normalized) - { - normalized = null; - if (string.IsNullOrWhiteSpace(status)) - { - return false; - } - - var token = status.Trim().ToLowerInvariant(); - if (token.Contains("устранена", StringComparison.Ordinal)) - { - normalized = AffectedPackageStatusCatalog.Fixed; - return true; - } - - if (token.Contains("информация об устранении отсутствует", StringComparison.Ordinal)) - { - normalized = AffectedPackageStatusCatalog.Unknown; - return true; - } - - return false; - } - - private static IReadOnlyList BuildVersionRanges(RuBduSoftwareDto software, DateTimeOffset recordedAt) - { - var tokens = SplitVersionTokens(software.Version).ToArray(); - if (tokens.Length == 0) - { - return Array.Empty(); - } - - var ranges = new List(tokens.Length); - foreach (var token in tokens) - { - ranges.Add(new AffectedVersionRange( - rangeKind: "string", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: token, - provenance: new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "package-range", - token, - recordedAt, - new[] { ProvenanceFieldMasks.VersionRanges }))); - } - - return ranges; - } - - private static IReadOnlyList BuildNormalizedVersions(RuBduSoftwareDto software) - { - var tokens = SplitVersionTokens(software.Version).ToArray(); - if (tokens.Length == 0) - { - return Array.Empty(); - } - - var rules = new List(tokens.Length); - foreach (var token in tokens) - { - rules.Add(new NormalizedVersionRule( - RawVersionScheme, - NormalizedVersionRuleTypes.Exact, - value: token)); - } - - return rules; - } - - private static IEnumerable SplitVersionTokens(string? version) - { - if (string.IsNullOrWhiteSpace(version)) - { - yield break; - } - - var raw = version.Trim(); - if (raw.Length == 0 || string.Equals(raw, "-", StringComparison.Ordinal)) - { - yield break; - } - - var tokens = raw.Split(VersionSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (tokens.Length == 0) - { - yield return raw; - yield break; - } - - foreach (var token in tokens) - { - if (string.IsNullOrWhiteSpace(token) || string.Equals(token, "-", StringComparison.Ordinal)) - { - continue; - } - - if (token.Equals("не указано", StringComparison.OrdinalIgnoreCase) - || token.Equals("не указана", StringComparison.OrdinalIgnoreCase) - || token.Equals("не определено", StringComparison.OrdinalIgnoreCase) - || token.Equals("не определена", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - yield return token; - } - } - - private static string? NormalizePlatform(string? platform) - { - if (string.IsNullOrWhiteSpace(platform)) - { - return null; - } - - var trimmed = platform.Trim(); - if (trimmed.Length == 0) - { - return null; - } - - if (trimmed.Equals("-", StringComparison.Ordinal) - || trimmed.Equals("не указана", StringComparison.OrdinalIgnoreCase) - || trimmed.Equals("не указано", StringComparison.OrdinalIgnoreCase)) - { - return null; - } - - return trimmed; - } - - private static string DeterminePackageType(ImmutableArray types) - => IsIcsSoftware(types) ? AffectedPackageTypes.IcsVendor : AffectedPackageTypes.Vendor; - - private static bool IsIcsSoftware(ImmutableArray types) - { - if (types.IsDefaultOrEmpty) - { - return false; - } - - foreach (var type in types) - { - if (string.IsNullOrWhiteSpace(type)) - { - continue; - } - - var token = type.Trim(); - if (token.Contains("АСУ", StringComparison.OrdinalIgnoreCase) - || token.Contains("SCADA", StringComparison.OrdinalIgnoreCase) - || token.Contains("ICS", StringComparison.OrdinalIgnoreCase) - || token.Contains("промыш", StringComparison.OrdinalIgnoreCase) - || token.Contains("industrial", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - } - - return false; - } - - private static IReadOnlyList BuildReferences(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - var references = new List(); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - void AddReference(string? url, string kind, string sourceTag, string? summary = null) - { - if (string.IsNullOrWhiteSpace(url)) - { - return; - } - - var trimmed = url.Trim(); - if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)) - { - if (trimmed.StartsWith("www.", StringComparison.OrdinalIgnoreCase) - && Uri.TryCreate($"https://{trimmed}", UriKind.Absolute, out var prefixed)) - { - uri = prefixed; - } - else - { - return; - } - } - - var canonical = uri.ToString(); - if (!seen.Add(canonical)) - { - return; - } - - references.Add(new AdvisoryReference( - canonical, - kind, - sourceTag, - summary, - new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "reference", - canonical, - recordedAt, - new[] { ProvenanceFieldMasks.References }))); - } - - AddReference(document.Uri, "details", RuBduConnectorPlugin.SourceName); - - foreach (var source in dto.Sources) - { - AddReference(source, "source", RuBduConnectorPlugin.SourceName); - } - - foreach (var identifier in dto.Identifiers) - { - if (string.IsNullOrWhiteSpace(identifier.Link)) - { - continue; - } - - var sourceTag = NormalizeIdentifierType(identifier.Type); - var kind = string.Equals(sourceTag, "cve", StringComparison.Ordinal) ? "cve" : "external"; - AddReference(identifier.Link, kind, sourceTag, identifier.Value); - } - - foreach (var cwe in dto.Cwes) - { - if (string.IsNullOrWhiteSpace(cwe.Identifier)) - { - continue; - } - - var slug = cwe.Identifier.ToUpperInvariant().Replace("CWE-", string.Empty, StringComparison.OrdinalIgnoreCase); - if (!slug.All(char.IsDigit)) - { - continue; - } - - var url = $"https://cwe.mitre.org/data/definitions/{slug}.html"; - AddReference(url, "cwe", "cwe", cwe.Name); - } - - return references; - } - - private static string NormalizeIdentifierType(string? type) - { - if (string.IsNullOrWhiteSpace(type)) - { - return RuBduConnectorPlugin.SourceName; - } - - var builder = new StringBuilder(type.Length); - foreach (var ch in type) - { - if (char.IsLetterOrDigit(ch)) - { - builder.Append(char.ToLowerInvariant(ch)); - } - else if (ch is '-' or '_' or '.') - { - builder.Append(ch); - } - } - - return builder.Length == 0 ? RuBduConnectorPlugin.SourceName : builder.ToString(); - } - - private static IReadOnlyList BuildCvssMetrics(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity) - { - severity = null; - var metrics = new List(); - - if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize("2.0", dto.CvssVector, dto.CvssScore, null, out var normalized)) - { - var provenance = new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "cvss", - normalized.Vector, - recordedAt, - new[] { ProvenanceFieldMasks.CvssMetrics }); - var metric = normalized.ToModel(provenance); - metrics.Add(metric); - } - - if (!string.IsNullOrWhiteSpace(dto.Cvss3Vector) && CvssMetricNormalizer.TryNormalize("3.1", dto.Cvss3Vector, dto.Cvss3Score, null, out var normalized3)) - { - var provenance = new AdvisoryProvenance( - RuBduConnectorPlugin.SourceName, - "cvss", - normalized3.Vector, - recordedAt, - new[] { ProvenanceFieldMasks.CvssMetrics }); - var metric = normalized3.ToModel(provenance); - metrics.Add(metric); - } - - if (metrics.Count > 1) - { - metrics = metrics - .OrderByDescending(static metric => metric.BaseScore) - .ThenBy(static metric => metric.Version, StringComparer.Ordinal) - .ToList(); - } - - severity = metrics.Count > 0 ? metrics[0].BaseSeverity : severity; - return metrics; - } - - private static string? NormalizeSeverity(string? severityText) - { - if (string.IsNullOrWhiteSpace(severityText)) - { - return null; - } - - var token = severityText.Trim().ToLowerInvariant(); - if (token.Contains("критич", StringComparison.Ordinal)) - { - return "critical"; - } - - if (token.Contains("высок", StringComparison.Ordinal)) - { - return "high"; - } - - if (token.Contains("средн", StringComparison.Ordinal) || token.Contains("умер", StringComparison.Ordinal)) - { - return "medium"; - } - - if (token.Contains("низк", StringComparison.Ordinal)) - { - return "low"; - } - - return null; - } - - private static bool DetermineExploitKnown(RuBduVulnerabilityDto dto) - { - if (dto.IncidentCount.HasValue && dto.IncidentCount.Value > 0) - { - return true; - } - - if (!string.IsNullOrWhiteSpace(dto.ExploitStatus)) - { - var status = dto.ExploitStatus.Trim().ToLowerInvariant(); - if (status.Contains("существ", StringComparison.Ordinal) || status.Contains("использ", StringComparison.Ordinal)) - { - return true; - } - } - - return false; - } - - private static readonly char[] VersionSeparators = { ',', ';', '\r', '\n', '\t' }; -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Storage.Mongo.Documents; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal; + +internal static class RuBduMapper +{ + private const string RawVersionScheme = "ru-bdu.raw"; + + public static Advisory Map(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + + var advisoryProvenance = new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "advisory", + dto.Identifier, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + var aliases = BuildAliases(dto); + var packages = BuildPackages(dto, recordedAt); + var references = BuildReferences(dto, document, recordedAt); + var cvssMetrics = BuildCvssMetrics(dto, recordedAt, out var severityFromCvss); + var severity = severityFromCvss ?? NormalizeSeverity(dto.SeverityText); + var exploitKnown = DetermineExploitKnown(dto); + + return new Advisory( + advisoryKey: dto.Identifier, + title: dto.Name ?? dto.Identifier, + summary: dto.Description, + language: "ru", + published: dto.IdentifyDate, + modified: dto.IdentifyDate, + severity: severity, + exploitKnown: exploitKnown, + aliases: aliases, + references: references, + affectedPackages: packages, + cvssMetrics: cvssMetrics, + provenance: new[] { advisoryProvenance }); + } + + private static IReadOnlyList BuildAliases(RuBduVulnerabilityDto dto) + { + var aliases = new HashSet(StringComparer.Ordinal); + + if (!string.IsNullOrWhiteSpace(dto.Identifier)) + { + aliases.Add(dto.Identifier.Trim()); + } + + foreach (var identifier in dto.Identifiers) + { + if (string.IsNullOrWhiteSpace(identifier.Value)) + { + continue; + } + + aliases.Add(identifier.Value.Trim()); + } + + return aliases.Count == 0 ? Array.Empty() : aliases.ToArray(); + } + + private static IReadOnlyList BuildPackages(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt) + { + if (dto.Software.IsDefaultOrEmpty) + { + return Array.Empty(); + } + + var packages = new List(dto.Software.Length); + foreach (var software in dto.Software) + { + if (string.IsNullOrWhiteSpace(software.Name) && string.IsNullOrWhiteSpace(software.Vendor)) + { + continue; + } + + var identifier = BuildPackageIdentifier(dto.Identifier, software); + var packageProvenance = new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "package", + identifier, + recordedAt, + new[] { ProvenanceFieldMasks.AffectedPackages }); + + var statuses = BuildPackageStatuses(dto, recordedAt); + var ranges = BuildVersionRanges(software, recordedAt); + var normalizedVersions = BuildNormalizedVersions(software); + + packages.Add(new AffectedPackage( + DeterminePackageType(software.Types), + identifier, + platform: NormalizePlatform(software.Platform), + versionRanges: ranges, + statuses: statuses, + provenance: new[] { packageProvenance }, + normalizedVersions: normalizedVersions)); + } + + return packages; + } + + private static string BuildPackageIdentifier(string fallbackIdentifier, RuBduSoftwareDto software) + { + var parts = new[] { software.Vendor, software.Name } + .Where(static part => !string.IsNullOrWhiteSpace(part)) + .Select(static part => part!.Trim()) + .ToArray(); + + if (parts.Length == 0) + { + return software.Name ?? software.Vendor ?? fallbackIdentifier; + } + + return string.Join(" ", parts); + } + + private static IReadOnlyList BuildPackageStatuses(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt) + { + var statuses = new List(capacity: 2); + + if (TryNormalizeVulnerabilityStatus(dto.VulStatus, out var vulnerabilityStatus)) + { + statuses.Add(new AffectedPackageStatus( + vulnerabilityStatus!, + new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "package-status", + dto.VulStatus!, + recordedAt, + new[] { ProvenanceFieldMasks.PackageStatuses }))); + } + + if (TryNormalizeFixStatus(dto.FixStatus, out var fixStatus)) + { + statuses.Add(new AffectedPackageStatus( + fixStatus!, + new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "package-fix-status", + dto.FixStatus!, + recordedAt, + new[] { ProvenanceFieldMasks.PackageStatuses }))); + } + + return statuses.Count == 0 ? Array.Empty() : statuses; + } + + private static bool TryNormalizeVulnerabilityStatus(string? status, out string? normalized) + { + normalized = null; + if (string.IsNullOrWhiteSpace(status)) + { + return false; + } + + var token = status.Trim().ToLowerInvariant(); + if (token.Contains("потенциал", StringComparison.Ordinal)) + { + normalized = AffectedPackageStatusCatalog.UnderInvestigation; + return true; + } + + if (token.Contains("подтвержд", StringComparison.Ordinal)) + { + normalized = AffectedPackageStatusCatalog.Affected; + return true; + } + + if (token.Contains("актуал", StringComparison.Ordinal)) + { + normalized = AffectedPackageStatusCatalog.Affected; + return true; + } + + return false; + } + + private static bool TryNormalizeFixStatus(string? status, out string? normalized) + { + normalized = null; + if (string.IsNullOrWhiteSpace(status)) + { + return false; + } + + var token = status.Trim().ToLowerInvariant(); + if (token.Contains("устранена", StringComparison.Ordinal)) + { + normalized = AffectedPackageStatusCatalog.Fixed; + return true; + } + + if (token.Contains("информация об устранении отсутствует", StringComparison.Ordinal)) + { + normalized = AffectedPackageStatusCatalog.Unknown; + return true; + } + + return false; + } + + private static IReadOnlyList BuildVersionRanges(RuBduSoftwareDto software, DateTimeOffset recordedAt) + { + var tokens = SplitVersionTokens(software.Version).ToArray(); + if (tokens.Length == 0) + { + return Array.Empty(); + } + + var ranges = new List(tokens.Length); + foreach (var token in tokens) + { + ranges.Add(new AffectedVersionRange( + rangeKind: "string", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: token, + provenance: new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "package-range", + token, + recordedAt, + new[] { ProvenanceFieldMasks.VersionRanges }))); + } + + return ranges; + } + + private static IReadOnlyList BuildNormalizedVersions(RuBduSoftwareDto software) + { + var tokens = SplitVersionTokens(software.Version).ToArray(); + if (tokens.Length == 0) + { + return Array.Empty(); + } + + var rules = new List(tokens.Length); + foreach (var token in tokens) + { + rules.Add(new NormalizedVersionRule( + RawVersionScheme, + NormalizedVersionRuleTypes.Exact, + value: token)); + } + + return rules; + } + + private static IEnumerable SplitVersionTokens(string? version) + { + if (string.IsNullOrWhiteSpace(version)) + { + yield break; + } + + var raw = version.Trim(); + if (raw.Length == 0 || string.Equals(raw, "-", StringComparison.Ordinal)) + { + yield break; + } + + var tokens = raw.Split(VersionSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (tokens.Length == 0) + { + yield return raw; + yield break; + } + + foreach (var token in tokens) + { + if (string.IsNullOrWhiteSpace(token) || string.Equals(token, "-", StringComparison.Ordinal)) + { + continue; + } + + if (token.Equals("не указано", StringComparison.OrdinalIgnoreCase) + || token.Equals("не указана", StringComparison.OrdinalIgnoreCase) + || token.Equals("не определено", StringComparison.OrdinalIgnoreCase) + || token.Equals("не определена", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + yield return token; + } + } + + private static string? NormalizePlatform(string? platform) + { + if (string.IsNullOrWhiteSpace(platform)) + { + return null; + } + + var trimmed = platform.Trim(); + if (trimmed.Length == 0) + { + return null; + } + + if (trimmed.Equals("-", StringComparison.Ordinal) + || trimmed.Equals("не указана", StringComparison.OrdinalIgnoreCase) + || trimmed.Equals("не указано", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + return trimmed; + } + + private static string DeterminePackageType(ImmutableArray types) + => IsIcsSoftware(types) ? AffectedPackageTypes.IcsVendor : AffectedPackageTypes.Vendor; + + private static bool IsIcsSoftware(ImmutableArray types) + { + if (types.IsDefaultOrEmpty) + { + return false; + } + + foreach (var type in types) + { + if (string.IsNullOrWhiteSpace(type)) + { + continue; + } + + var token = type.Trim(); + if (token.Contains("АСУ", StringComparison.OrdinalIgnoreCase) + || token.Contains("SCADA", StringComparison.OrdinalIgnoreCase) + || token.Contains("ICS", StringComparison.OrdinalIgnoreCase) + || token.Contains("промыш", StringComparison.OrdinalIgnoreCase) + || token.Contains("industrial", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static IReadOnlyList BuildReferences(RuBduVulnerabilityDto dto, DocumentRecord document, DateTimeOffset recordedAt) + { + var references = new List(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + void AddReference(string? url, string kind, string sourceTag, string? summary = null) + { + if (string.IsNullOrWhiteSpace(url)) + { + return; + } + + var trimmed = url.Trim(); + if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)) + { + if (trimmed.StartsWith("www.", StringComparison.OrdinalIgnoreCase) + && Uri.TryCreate($"https://{trimmed}", UriKind.Absolute, out var prefixed)) + { + uri = prefixed; + } + else + { + return; + } + } + + var canonical = uri.ToString(); + if (!seen.Add(canonical)) + { + return; + } + + references.Add(new AdvisoryReference( + canonical, + kind, + sourceTag, + summary, + new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "reference", + canonical, + recordedAt, + new[] { ProvenanceFieldMasks.References }))); + } + + AddReference(document.Uri, "details", RuBduConnectorPlugin.SourceName); + + foreach (var source in dto.Sources) + { + AddReference(source, "source", RuBduConnectorPlugin.SourceName); + } + + foreach (var identifier in dto.Identifiers) + { + if (string.IsNullOrWhiteSpace(identifier.Link)) + { + continue; + } + + var sourceTag = NormalizeIdentifierType(identifier.Type); + var kind = string.Equals(sourceTag, "cve", StringComparison.Ordinal) ? "cve" : "external"; + AddReference(identifier.Link, kind, sourceTag, identifier.Value); + } + + foreach (var cwe in dto.Cwes) + { + if (string.IsNullOrWhiteSpace(cwe.Identifier)) + { + continue; + } + + var slug = cwe.Identifier.ToUpperInvariant().Replace("CWE-", string.Empty, StringComparison.OrdinalIgnoreCase); + if (!slug.All(char.IsDigit)) + { + continue; + } + + var url = $"https://cwe.mitre.org/data/definitions/{slug}.html"; + AddReference(url, "cwe", "cwe", cwe.Name); + } + + return references; + } + + private static string NormalizeIdentifierType(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return RuBduConnectorPlugin.SourceName; + } + + var builder = new StringBuilder(type.Length); + foreach (var ch in type) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + } + else if (ch is '-' or '_' or '.') + { + builder.Append(ch); + } + } + + return builder.Length == 0 ? RuBduConnectorPlugin.SourceName : builder.ToString(); + } + + private static IReadOnlyList BuildCvssMetrics(RuBduVulnerabilityDto dto, DateTimeOffset recordedAt, out string? severity) + { + severity = null; + var metrics = new List(); + + if (!string.IsNullOrWhiteSpace(dto.CvssVector) && CvssMetricNormalizer.TryNormalize("2.0", dto.CvssVector, dto.CvssScore, null, out var normalized)) + { + var provenance = new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "cvss", + normalized.Vector, + recordedAt, + new[] { ProvenanceFieldMasks.CvssMetrics }); + var metric = normalized.ToModel(provenance); + metrics.Add(metric); + } + + if (!string.IsNullOrWhiteSpace(dto.Cvss3Vector) && CvssMetricNormalizer.TryNormalize("3.1", dto.Cvss3Vector, dto.Cvss3Score, null, out var normalized3)) + { + var provenance = new AdvisoryProvenance( + RuBduConnectorPlugin.SourceName, + "cvss", + normalized3.Vector, + recordedAt, + new[] { ProvenanceFieldMasks.CvssMetrics }); + var metric = normalized3.ToModel(provenance); + metrics.Add(metric); + } + + if (metrics.Count > 1) + { + metrics = metrics + .OrderByDescending(static metric => metric.BaseScore) + .ThenBy(static metric => metric.Version, StringComparer.Ordinal) + .ToList(); + } + + severity = metrics.Count > 0 ? metrics[0].BaseSeverity : severity; + return metrics; + } + + private static string? NormalizeSeverity(string? severityText) + { + if (string.IsNullOrWhiteSpace(severityText)) + { + return null; + } + + var token = severityText.Trim().ToLowerInvariant(); + if (token.Contains("критич", StringComparison.Ordinal)) + { + return "critical"; + } + + if (token.Contains("высок", StringComparison.Ordinal)) + { + return "high"; + } + + if (token.Contains("средн", StringComparison.Ordinal) || token.Contains("умер", StringComparison.Ordinal)) + { + return "medium"; + } + + if (token.Contains("низк", StringComparison.Ordinal)) + { + return "low"; + } + + return null; + } + + private static bool DetermineExploitKnown(RuBduVulnerabilityDto dto) + { + if (dto.IncidentCount.HasValue && dto.IncidentCount.Value > 0) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(dto.ExploitStatus)) + { + var status = dto.ExploitStatus.Trim().ToLowerInvariant(); + if (status.Contains("существ", StringComparison.Ordinal) || status.Contains("использ", StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static readonly char[] VersionSeparators = { ',', ';', '\r', '\n', '\t' }; +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs index 104ab1c4..421f6c1e 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduVulnerabilityDto.cs @@ -1,52 +1,52 @@ -using System.Collections.Immutable; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Internal; - -internal sealed record RuBduVulnerabilityDto( - string Identifier, - string? Name, - string? Description, - string? Solution, - DateTimeOffset? IdentifyDate, - string? SeverityText, - string? CvssVector, - double? CvssScore, - string? Cvss3Vector, - double? Cvss3Score, - string? ExploitStatus, - int? IncidentCount, - string? FixStatus, - string? VulStatus, - string? VulClass, - string? VulState, - string? Other, - ImmutableArray Software, - ImmutableArray Environment, - ImmutableArray Cwes, - ImmutableArray Sources, - ImmutableArray Identifiers) -{ - [JsonIgnore] - public bool HasCvss => !string.IsNullOrWhiteSpace(CvssVector) || !string.IsNullOrWhiteSpace(Cvss3Vector); -} - -internal sealed record RuBduSoftwareDto( - string? Vendor, - string? Name, - string? Version, - string? Platform, - ImmutableArray Types); - -internal sealed record RuBduEnvironmentDto( - string? Vendor, - string? Name, - string? Version, - string? Platform); - -internal sealed record RuBduCweDto(string Identifier, string? Name); - -internal sealed record RuBduExternalIdentifierDto( - string Type, - string Value, - string? Link); +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal; + +internal sealed record RuBduVulnerabilityDto( + string Identifier, + string? Name, + string? Description, + string? Solution, + DateTimeOffset? IdentifyDate, + string? SeverityText, + string? CvssVector, + double? CvssScore, + string? Cvss3Vector, + double? Cvss3Score, + string? ExploitStatus, + int? IncidentCount, + string? FixStatus, + string? VulStatus, + string? VulClass, + string? VulState, + string? Other, + ImmutableArray Software, + ImmutableArray Environment, + ImmutableArray Cwes, + ImmutableArray Sources, + ImmutableArray Identifiers) +{ + [JsonIgnore] + public bool HasCvss => !string.IsNullOrWhiteSpace(CvssVector) || !string.IsNullOrWhiteSpace(Cvss3Vector); +} + +internal sealed record RuBduSoftwareDto( + string? Vendor, + string? Name, + string? Version, + string? Platform, + ImmutableArray Types); + +internal sealed record RuBduEnvironmentDto( + string? Vendor, + string? Name, + string? Version, + string? Platform); + +internal sealed record RuBduCweDto(string Identifier, string? Name); + +internal sealed record RuBduExternalIdentifierDto( + string Type, + string Value, + string? Link); diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduXmlParser.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduXmlParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduXmlParser.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduXmlParser.cs index 424b1185..c50bfe5c 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Internal/RuBduXmlParser.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Internal/RuBduXmlParser.cs @@ -1,268 +1,268 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Globalization; -using System.Xml.Linq; - -namespace StellaOps.Feedser.Source.Ru.Bdu.Internal; - -internal static class RuBduXmlParser -{ - public static RuBduVulnerabilityDto? TryParse(XElement element) - { - ArgumentNullException.ThrowIfNull(element); - - var identifier = element.Element("identifier")?.Value?.Trim(); - if (string.IsNullOrWhiteSpace(identifier)) - { - return null; - } - - var name = Normalize(element.Element("name")?.Value); - var description = Normalize(element.Element("description")?.Value); - var solution = Normalize(element.Element("solution")?.Value); - var severity = Normalize(element.Element("severity")?.Value); - var exploitStatus = Normalize(element.Element("exploit_status")?.Value); - var fixStatus = Normalize(element.Element("fix_status")?.Value); - var vulStatus = Normalize(element.Element("vul_status")?.Value); - var vulClass = Normalize(element.Element("vul_class")?.Value); - var vulState = Normalize(element.Element("vul_state")?.Value); - var other = Normalize(element.Element("other")?.Value); - var incidentCount = ParseInt(element.Element("vul_incident")?.Value); - - var identifyDate = ParseDate(element.Element("identify_date")?.Value); - - var cvssVectorElement = element.Element("cvss")?.Element("vector"); - var cvssVector = Normalize(cvssVectorElement?.Value); - var cvssScore = ParseDouble(cvssVectorElement?.Attribute("score")?.Value); - - var cvss3VectorElement = element.Element("cvss3")?.Element("vector"); - var cvss3Vector = Normalize(cvss3VectorElement?.Value); - var cvss3Score = ParseDouble(cvss3VectorElement?.Attribute("score")?.Value); - - if (string.IsNullOrWhiteSpace(cvssVector)) - { - cvssVector = null; - cvssScore = null; - } - - if (string.IsNullOrWhiteSpace(cvss3Vector)) - { - cvss3Vector = null; - cvss3Score = null; - } - - var software = ParseSoftware(element.Element("vulnerable_software")); - var environment = ParseEnvironment(element.Element("environment")); - var cwes = ParseCwes(element.Element("cwes")); - var sources = ParseSources(element.Element("sources")); - var identifiers = ParseIdentifiers(element.Element("identifiers")); - - return new RuBduVulnerabilityDto( - identifier.Trim(), - name, - description, - solution, - identifyDate, - severity, - cvssVector, - cvssScore, - cvss3Vector, - cvss3Score, - exploitStatus, - incidentCount, - fixStatus, - vulStatus, - vulClass, - vulState, - other, - software, - environment, - cwes, - sources, - identifiers); - } - - private static ImmutableArray ParseSoftware(XElement? root) - { - if (root is null) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var soft in root.Elements("soft")) - { - var vendor = Normalize(soft.Element("vendor")?.Value); - var name = Normalize(soft.Element("name")?.Value); - var version = Normalize(soft.Element("version")?.Value); - var platform = Normalize(soft.Element("platform")?.Value); - var types = soft.Element("types") is { } typesElement - ? typesElement.Elements("type").Select(static x => Normalize(x.Value)).Where(static value => !string.IsNullOrWhiteSpace(value)).Cast().ToImmutableArray() - : ImmutableArray.Empty; - - builder.Add(new RuBduSoftwareDto(vendor, name, version, platform, types)); - } - - return builder.ToImmutable(); - } - - private static ImmutableArray ParseEnvironment(XElement? root) - { - if (root is null) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var os in root.Elements()) - { - var vendor = Normalize(os.Element("vendor")?.Value); - var name = Normalize(os.Element("name")?.Value); - var version = Normalize(os.Element("version")?.Value); - var platform = Normalize(os.Element("platform")?.Value); - builder.Add(new RuBduEnvironmentDto(vendor, name, version, platform)); - } - - return builder.ToImmutable(); - } - - private static ImmutableArray ParseCwes(XElement? root) - { - if (root is null) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var cwe in root.Elements("cwe")) - { - var identifier = Normalize(cwe.Element("identifier")?.Value); - if (string.IsNullOrWhiteSpace(identifier)) - { - continue; - } - - var name = Normalize(cwe.Element("name")?.Value); - builder.Add(new RuBduCweDto(identifier, name)); - } - - return builder.ToImmutable(); - } - - private static ImmutableArray ParseSources(XElement? root) - { - if (root is null) - { - return ImmutableArray.Empty; - } - - var raw = root.Value; - if (string.IsNullOrWhiteSpace(raw)) - { - return ImmutableArray.Empty; - } - - var tokens = raw - .Split(SourceSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Select(static token => token.Trim()) - .Where(static token => !string.IsNullOrWhiteSpace(token)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - return tokens.IsDefaultOrEmpty ? ImmutableArray.Empty : tokens; - } - - private static ImmutableArray ParseIdentifiers(XElement? root) - { - if (root is null) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var identifier in root.Elements("identifier")) - { - var value = Normalize(identifier?.Value); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - var type = identifier?.Attribute("type")?.Value?.Trim(); - var link = identifier?.Attribute("link")?.Value?.Trim(); - - if (string.IsNullOrWhiteSpace(type)) - { - type = "external"; - } - - builder.Add(new RuBduExternalIdentifierDto(type, value.Trim(), string.IsNullOrWhiteSpace(link) ? null : link)); - } - - return builder.ToImmutable(); - } - - private static readonly char[] SourceSeparators = { '\r', '\n', '\t', ' ' }; - - private static DateTimeOffset? ParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - var trimmed = value.Trim(); - if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var isoDate)) - { - return isoDate; - } - - if (DateTimeOffset.TryParseExact(trimmed, new[] { "dd.MM.yyyy", "dd.MM.yyyy HH:mm:ss" }, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal, out var ruDate)) - { - return ruDate; - } - - return null; - } - - private static double? ParseDouble(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (double.TryParse(value.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed)) - { - return parsed; - } - - return null; - } - - private static int? ParseInt(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (int.TryParse(value.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) - { - return parsed; - } - - return null; - } - - private static string? Normalize(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return value.Replace('\r', ' ').Replace('\n', ' ').Trim(); - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Globalization; +using System.Xml.Linq; + +namespace StellaOps.Concelier.Connector.Ru.Bdu.Internal; + +internal static class RuBduXmlParser +{ + public static RuBduVulnerabilityDto? TryParse(XElement element) + { + ArgumentNullException.ThrowIfNull(element); + + var identifier = element.Element("identifier")?.Value?.Trim(); + if (string.IsNullOrWhiteSpace(identifier)) + { + return null; + } + + var name = Normalize(element.Element("name")?.Value); + var description = Normalize(element.Element("description")?.Value); + var solution = Normalize(element.Element("solution")?.Value); + var severity = Normalize(element.Element("severity")?.Value); + var exploitStatus = Normalize(element.Element("exploit_status")?.Value); + var fixStatus = Normalize(element.Element("fix_status")?.Value); + var vulStatus = Normalize(element.Element("vul_status")?.Value); + var vulClass = Normalize(element.Element("vul_class")?.Value); + var vulState = Normalize(element.Element("vul_state")?.Value); + var other = Normalize(element.Element("other")?.Value); + var incidentCount = ParseInt(element.Element("vul_incident")?.Value); + + var identifyDate = ParseDate(element.Element("identify_date")?.Value); + + var cvssVectorElement = element.Element("cvss")?.Element("vector"); + var cvssVector = Normalize(cvssVectorElement?.Value); + var cvssScore = ParseDouble(cvssVectorElement?.Attribute("score")?.Value); + + var cvss3VectorElement = element.Element("cvss3")?.Element("vector"); + var cvss3Vector = Normalize(cvss3VectorElement?.Value); + var cvss3Score = ParseDouble(cvss3VectorElement?.Attribute("score")?.Value); + + if (string.IsNullOrWhiteSpace(cvssVector)) + { + cvssVector = null; + cvssScore = null; + } + + if (string.IsNullOrWhiteSpace(cvss3Vector)) + { + cvss3Vector = null; + cvss3Score = null; + } + + var software = ParseSoftware(element.Element("vulnerable_software")); + var environment = ParseEnvironment(element.Element("environment")); + var cwes = ParseCwes(element.Element("cwes")); + var sources = ParseSources(element.Element("sources")); + var identifiers = ParseIdentifiers(element.Element("identifiers")); + + return new RuBduVulnerabilityDto( + identifier.Trim(), + name, + description, + solution, + identifyDate, + severity, + cvssVector, + cvssScore, + cvss3Vector, + cvss3Score, + exploitStatus, + incidentCount, + fixStatus, + vulStatus, + vulClass, + vulState, + other, + software, + environment, + cwes, + sources, + identifiers); + } + + private static ImmutableArray ParseSoftware(XElement? root) + { + if (root is null) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var soft in root.Elements("soft")) + { + var vendor = Normalize(soft.Element("vendor")?.Value); + var name = Normalize(soft.Element("name")?.Value); + var version = Normalize(soft.Element("version")?.Value); + var platform = Normalize(soft.Element("platform")?.Value); + var types = soft.Element("types") is { } typesElement + ? typesElement.Elements("type").Select(static x => Normalize(x.Value)).Where(static value => !string.IsNullOrWhiteSpace(value)).Cast().ToImmutableArray() + : ImmutableArray.Empty; + + builder.Add(new RuBduSoftwareDto(vendor, name, version, platform, types)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray ParseEnvironment(XElement? root) + { + if (root is null) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var os in root.Elements()) + { + var vendor = Normalize(os.Element("vendor")?.Value); + var name = Normalize(os.Element("name")?.Value); + var version = Normalize(os.Element("version")?.Value); + var platform = Normalize(os.Element("platform")?.Value); + builder.Add(new RuBduEnvironmentDto(vendor, name, version, platform)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray ParseCwes(XElement? root) + { + if (root is null) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var cwe in root.Elements("cwe")) + { + var identifier = Normalize(cwe.Element("identifier")?.Value); + if (string.IsNullOrWhiteSpace(identifier)) + { + continue; + } + + var name = Normalize(cwe.Element("name")?.Value); + builder.Add(new RuBduCweDto(identifier, name)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray ParseSources(XElement? root) + { + if (root is null) + { + return ImmutableArray.Empty; + } + + var raw = root.Value; + if (string.IsNullOrWhiteSpace(raw)) + { + return ImmutableArray.Empty; + } + + var tokens = raw + .Split(SourceSeparators, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(static token => token.Trim()) + .Where(static token => !string.IsNullOrWhiteSpace(token)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return tokens.IsDefaultOrEmpty ? ImmutableArray.Empty : tokens; + } + + private static ImmutableArray ParseIdentifiers(XElement? root) + { + if (root is null) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var identifier in root.Elements("identifier")) + { + var value = Normalize(identifier?.Value); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + var type = identifier?.Attribute("type")?.Value?.Trim(); + var link = identifier?.Attribute("link")?.Value?.Trim(); + + if (string.IsNullOrWhiteSpace(type)) + { + type = "external"; + } + + builder.Add(new RuBduExternalIdentifierDto(type, value.Trim(), string.IsNullOrWhiteSpace(link) ? null : link)); + } + + return builder.ToImmutable(); + } + + private static readonly char[] SourceSeparators = { '\r', '\n', '\t', ' ' }; + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + if (DateTimeOffset.TryParse(trimmed, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var isoDate)) + { + return isoDate; + } + + if (DateTimeOffset.TryParseExact(trimmed, new[] { "dd.MM.yyyy", "dd.MM.yyyy HH:mm:ss" }, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal, out var ruDate)) + { + return ruDate; + } + + return null; + } + + private static double? ParseDouble(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (double.TryParse(value.Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return null; + } + + private static int? ParseInt(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (int.TryParse(value.Trim(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Replace('\r', ' ').Replace('\n', ' ').Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Jobs.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ru.Bdu/Jobs.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs index 934b2155..ae77a718 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Jobs.cs @@ -1,43 +1,43 @@ -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Ru.Bdu; - -internal static class RuBduJobKinds -{ - public const string Fetch = "source:ru-bdu:fetch"; - public const string Parse = "source:ru-bdu:parse"; - public const string Map = "source:ru-bdu:map"; -} - -internal sealed class RuBduFetchJob : IJob -{ - private readonly RuBduConnector _connector; - - public RuBduFetchJob(RuBduConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class RuBduParseJob : IJob -{ - private readonly RuBduConnector _connector; - - public RuBduParseJob(RuBduConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class RuBduMapJob : IJob -{ - private readonly RuBduConnector _connector; - - public RuBduMapJob(RuBduConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Ru.Bdu; + +internal static class RuBduJobKinds +{ + public const string Fetch = "source:ru-bdu:fetch"; + public const string Parse = "source:ru-bdu:parse"; + public const string Map = "source:ru-bdu:map"; +} + +internal sealed class RuBduFetchJob : IJob +{ + private readonly RuBduConnector _connector; + + public RuBduFetchJob(RuBduConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class RuBduParseJob : IJob +{ + private readonly RuBduConnector _connector; + + public RuBduParseJob(RuBduConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class RuBduMapJob : IJob +{ + private readonly RuBduConnector _connector; + + public RuBduMapJob(RuBduConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.Ru.Bdu/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..04d0ee34 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Bdu.Tests")] diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/README.md b/src/StellaOps.Concelier.Connector.Ru.Bdu/README.md similarity index 81% rename from src/StellaOps.Feedser.Source.Ru.Bdu/README.md rename to src/StellaOps.Concelier.Connector.Ru.Bdu/README.md index 2edf64b6..aa378f50 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/README.md +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/README.md @@ -6,7 +6,7 @@ - **TLS trust**: the endpoint presents certificates chained to the Russian Trusted Root/Sub CAs. Bundle the official PEMs inside the deployment (`certificates/russian_trusted_root_ca.pem`, `certificates/russian_trusted_sub_ca.pem`, or the combined `certificates/russian_trusted_bundle.pem`) and point the connector at them, e.g.: ```yaml - feedser: + concelier: httpClients: source.bdu: trustedRootPaths: @@ -15,13 +15,13 @@ timeout: 00:02:00 ``` -- **Offline Kit**: copy the PEM bundle above into the Offline Kit artefacts and set `feedser:offline:root` (or `FEEDSER_OFFLINE_ROOT`) so air‑gapped installs can resolve relative certificate paths. Package the most recent `vulxml.zip` alongside cached exports when preparing air-gap refreshes. +- **Offline Kit**: copy the PEM bundle above into the Offline Kit artefacts and set `concelier:offline:root` (or `CONCELIER_OFFLINE_ROOT`) so air‑gapped installs can resolve relative certificate paths. Package the most recent `vulxml.zip` alongside cached exports when preparing air-gap refreshes. The connector keeps a local cache (`cache/ru-bdu/vulxml.zip`) so transient fetch failures can fall back to the last successful archive without blocking the cursor. ## Telemetry -The connector publishes an OpenTelemetry meter named `StellaOps.Feedser.Source.Ru.Bdu`. Instruments include: +The connector publishes an OpenTelemetry meter named `StellaOps.Concelier.Connector.Ru.Bdu`. Instruments include: - `ru.bdu.fetch.*` – `attempts`, `success`, `failures`, `not_modified`, `cache_fallbacks`, and histogram `ru.bdu.fetch.documents`. - `ru.bdu.parse.*` – counters for success/failures plus histograms tracking vulnerable software, external identifiers, and source reference counts per DTO. @@ -31,10 +31,10 @@ Use these metrics to alert on repeated cache fallbacks, sustained parse failures ## Regression fixtures -Deterministic fixtures live under `src/StellaOps.Feedser.Source.Ru.Bdu.Tests/Fixtures`. Run +Deterministic fixtures live under `src/StellaOps.Concelier.Connector.Ru.Bdu.Tests/Fixtures`. Run ```bash -dotnet test src/StellaOps.Feedser.Source.Ru.Bdu.Tests +dotnet test src/StellaOps.Concelier.Connector.Ru.Bdu.Tests ``` to execute the RU BDU snapshot suite, and set `UPDATE_BDU_FIXTURES=1` to refresh stored snapshots when ingest logic changes. The harness records the fetch requests, documents, DTOs, advisories, and state cursor to guarantee reproducible pipelines across machines. diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnector.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnector.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnector.cs index ba709d13..7d4394fa 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnector.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnector.cs @@ -1,528 +1,528 @@ -using System.Collections.Immutable; -using System.Globalization; -using System.IO; -using System.IO.Compression; -using System.Security.Cryptography; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Xml; -using System.Xml.Linq; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Ru.Bdu.Configuration; -using StellaOps.Feedser.Source.Ru.Bdu.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ru.Bdu; - -public sealed class RuBduConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = false, - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly RuBduOptions _options; - private readonly RuBduDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - - private readonly string _cacheDirectory; - private readonly string _archiveCachePath; - - public RuBduConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - RuBduDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory); - _archiveCachePath = Path.Combine(_cacheDirectory, "vulxml.zip"); - EnsureCacheDirectory(); - } - - public string SourceName => RuBduConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - _diagnostics.FetchAttempt(); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var now = _timeProvider.GetUtcNow(); - - SourceFetchContentResult? archiveResult = null; - byte[]? archiveContent = null; - var usedCache = false; - - try - { - var request = new SourceFetchRequest(RuBduOptions.HttpClientName, SourceName, _options.DataArchiveUri) - { - AcceptHeaders = new[] - { - "application/zip", - "application/octet-stream", - "application/x-zip-compressed", - }, - TimeoutOverride = _options.RequestTimeout, - }; - - var fetchResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - archiveResult = fetchResult; - - if (fetchResult.IsNotModified) - { - _logger.LogDebug("RU-BDU archive not modified."); - _diagnostics.FetchUnchanged(); - await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false); - return; - } - - if (fetchResult.IsSuccess && fetchResult.Content is not null) - { - archiveContent = fetchResult.Content; - TryWriteCachedArchive(archiveContent); - } - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - if (TryReadCachedArchive(out var cachedFallback)) - { - _logger.LogWarning(ex, "RU-BDU archive fetch failed; using cached artefact {CachePath}", _archiveCachePath); - archiveContent = cachedFallback; - usedCache = true; - _diagnostics.FetchCacheFallback(); - } - else - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "RU-BDU archive fetch failed for {ArchiveUri}", _options.DataArchiveUri); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - } - - if (archiveContent is null) - { - if (TryReadCachedArchive(out var cachedFallback)) - { - var status = archiveResult?.StatusCode; - _logger.LogWarning("RU-BDU archive unavailable (status={Status}); using cached artefact {CachePath}", status, _archiveCachePath); - archiveContent = cachedFallback; - usedCache = true; - _diagnostics.FetchCacheFallback(); - } - else - { - var status = archiveResult?.StatusCode; - _logger.LogWarning("RU-BDU archive fetch returned no content (status={Status})", status); - _diagnostics.FetchSuccess(addedCount: 0, usedCache: false); - await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false); - return; - } - } - - var archiveLastModified = archiveResult?.LastModified; - int added; - try - { - added = await ProcessArchiveAsync(archiveContent, now, pendingDocuments, pendingMappings, archiveLastModified, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "RU-BDU archive processing failed"); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - _diagnostics.FetchSuccess(added, usedCache); - if (added > 0) - { - _logger.LogInformation("RU-BDU processed {Added} vulnerabilities (cacheUsed={CacheUsed})", added, usedCache); - } - else - { - _logger.LogDebug("RU-BDU fetch completed with no new vulnerabilities (cacheUsed={CacheUsed})", usedCache); - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastSuccessfulFetch(now); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - _diagnostics.ParseFailure(); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _logger.LogWarning("RU-BDU document {DocumentId} missing GridFS payload", documentId); - _diagnostics.ParseFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - byte[] payload; - try - { - payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogError(ex, "RU-BDU unable to download raw document {DocumentId}", documentId); - _diagnostics.ParseFailure(); - throw; - } - - RuBduVulnerabilityDto? dto; - try - { - dto = JsonSerializer.Deserialize(payload, SerializerOptions); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "RU-BDU failed to deserialize document {DocumentId}", documentId); - _diagnostics.ParseFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (dto is null) - { - _logger.LogWarning("RU-BDU document {DocumentId} produced null DTO", documentId); - _diagnostics.ParseFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-bdu.v1", bson, _timeProvider.GetUtcNow()); - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - _diagnostics.ParseSuccess( - dto.Software.IsDefaultOrEmpty ? 0 : dto.Software.Length, - dto.Identifiers.IsDefaultOrEmpty ? 0 : dto.Identifiers.Length, - dto.Sources.IsDefaultOrEmpty ? 0 : dto.Sources.Length); - - pendingDocuments.Remove(documentId); - if (!pendingMappings.Contains(documentId)) - { - pendingMappings.Add(documentId); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - _diagnostics.MapFailure(); - pendingMappings.Remove(documentId); - continue; - } - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - _logger.LogWarning("RU-BDU document {DocumentId} missing DTO payload", documentId); - _diagnostics.MapFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - RuBduVulnerabilityDto dto; - try - { - dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null"); - } - catch (Exception ex) - { - _logger.LogError(ex, "RU-BDU failed to deserialize DTO for document {DocumentId}", documentId); - _diagnostics.MapFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - try - { - var advisory = RuBduMapper.Map(dto, document, dtoRecord.ValidatedAt); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - _diagnostics.MapSuccess(advisory); - pendingMappings.Remove(documentId); - } - catch (Exception ex) - { - _logger.LogError(ex, "RU-BDU mapping failed for document {DocumentId}", documentId); - _diagnostics.MapFailure(); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task ProcessArchiveAsync( - byte[] archiveContent, - DateTimeOffset now, - HashSet pendingDocuments, - HashSet pendingMappings, - DateTimeOffset? archiveLastModified, - CancellationToken cancellationToken) - { - var added = 0; - using var archiveStream = new MemoryStream(archiveContent, writable: false); - using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); - var entry = archive.GetEntry("export/export.xml") ?? archive.Entries.FirstOrDefault(); - if (entry is null) - { - _logger.LogWarning("RU-BDU archive does not contain export/export.xml; skipping."); - return added; - } - - await using var entryStream = entry.Open(); - using var reader = XmlReader.Create(entryStream, new XmlReaderSettings - { - IgnoreComments = true, - IgnoreWhitespace = true, - DtdProcessing = DtdProcessing.Ignore, - CloseInput = false, - }); - - while (reader.Read()) - { - cancellationToken.ThrowIfCancellationRequested(); - if (reader.NodeType != XmlNodeType.Element || !reader.Name.Equals("vul", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (RuBduXmlParser.TryParse(XNode.ReadFrom(reader) as XElement ?? new XElement("vul")) is not { } dto) - { - continue; - } - - var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions); - var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); - var documentUri = BuildDocumentUri(dto.Identifier); - - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); - if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false); - - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["ru-bdu.identifier"] = dto.Identifier, - }; - - if (!string.IsNullOrWhiteSpace(dto.Name)) - { - metadata["ru-bdu.name"] = dto.Name!; - } - - var recordId = existing?.Id ?? Guid.NewGuid(); - var record = new DocumentRecord( - recordId, - SourceName, - documentUri, - now, - sha, - DocumentStatuses.PendingParse, - "application/json", - Headers: null, - Metadata: metadata, - Etag: null, - LastModified: archiveLastModified ?? dto.IdentifyDate, - GridFsId: gridFsId, - ExpiresAt: null); - - var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); - pendingDocuments.Add(upserted.Id); - pendingMappings.Remove(upserted.Id); - added++; - - if (added >= _options.MaxVulnerabilitiesPerFetch) - { - break; - } - } - - return added; - } - - private string ResolveCacheDirectory(string? configuredPath) - { - if (!string.IsNullOrWhiteSpace(configuredPath)) - { - return Path.GetFullPath(Path.IsPathRooted(configuredPath) - ? configuredPath - : Path.Combine(AppContext.BaseDirectory, configuredPath)); - } - - return Path.Combine(AppContext.BaseDirectory, "cache", RuBduConnectorPlugin.SourceName); - } - - private void EnsureCacheDirectory() - { - try - { - Directory.CreateDirectory(_cacheDirectory); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "RU-BDU unable to ensure cache directory {CachePath}", _cacheDirectory); - } - } - - private void TryWriteCachedArchive(byte[] content) - { - try - { - Directory.CreateDirectory(Path.GetDirectoryName(_archiveCachePath)!); - File.WriteAllBytes(_archiveCachePath, content); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "RU-BDU failed to write cache archive {CachePath}", _archiveCachePath); - } - } - - private bool TryReadCachedArchive(out byte[] content) - { - try - { - if (File.Exists(_archiveCachePath)) - { - content = File.ReadAllBytes(_archiveCachePath); - return true; - } - } - catch (Exception ex) - { - _logger.LogDebug(ex, "RU-BDU failed to read cache archive {CachePath}", _archiveCachePath); - } - - content = Array.Empty(); - return false; - } - - private static string BuildDocumentUri(string identifier) - { - var slug = identifier.Contains(':', StringComparison.Ordinal) - ? identifier[(identifier.IndexOf(':') + 1)..] - : identifier; - return $"https://bdu.fstec.ru/vul/{slug}"; - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(RuBduCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBsonDocument(); - var completedAt = cursor.LastSuccessfulFetch ?? _timeProvider.GetUtcNow(); - return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); - } -} +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Xml; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Ru.Bdu.Configuration; +using StellaOps.Concelier.Connector.Ru.Bdu.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ru.Bdu; + +public sealed class RuBduConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly RuBduOptions _options; + private readonly RuBduDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + private readonly string _cacheDirectory; + private readonly string _archiveCachePath; + + public RuBduConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + RuBduDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory); + _archiveCachePath = Path.Combine(_cacheDirectory, "vulxml.zip"); + EnsureCacheDirectory(); + } + + public string SourceName => RuBduConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + _diagnostics.FetchAttempt(); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var now = _timeProvider.GetUtcNow(); + + SourceFetchContentResult? archiveResult = null; + byte[]? archiveContent = null; + var usedCache = false; + + try + { + var request = new SourceFetchRequest(RuBduOptions.HttpClientName, SourceName, _options.DataArchiveUri) + { + AcceptHeaders = new[] + { + "application/zip", + "application/octet-stream", + "application/x-zip-compressed", + }, + TimeoutOverride = _options.RequestTimeout, + }; + + var fetchResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + archiveResult = fetchResult; + + if (fetchResult.IsNotModified) + { + _logger.LogDebug("RU-BDU archive not modified."); + _diagnostics.FetchUnchanged(); + await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false); + return; + } + + if (fetchResult.IsSuccess && fetchResult.Content is not null) + { + archiveContent = fetchResult.Content; + TryWriteCachedArchive(archiveContent); + } + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + if (TryReadCachedArchive(out var cachedFallback)) + { + _logger.LogWarning(ex, "RU-BDU archive fetch failed; using cached artefact {CachePath}", _archiveCachePath); + archiveContent = cachedFallback; + usedCache = true; + _diagnostics.FetchCacheFallback(); + } + else + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "RU-BDU archive fetch failed for {ArchiveUri}", _options.DataArchiveUri); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (archiveContent is null) + { + if (TryReadCachedArchive(out var cachedFallback)) + { + var status = archiveResult?.StatusCode; + _logger.LogWarning("RU-BDU archive unavailable (status={Status}); using cached artefact {CachePath}", status, _archiveCachePath); + archiveContent = cachedFallback; + usedCache = true; + _diagnostics.FetchCacheFallback(); + } + else + { + var status = archiveResult?.StatusCode; + _logger.LogWarning("RU-BDU archive fetch returned no content (status={Status})", status); + _diagnostics.FetchSuccess(addedCount: 0, usedCache: false); + await UpdateCursorAsync(cursor.WithLastSuccessfulFetch(now), cancellationToken).ConfigureAwait(false); + return; + } + } + + var archiveLastModified = archiveResult?.LastModified; + int added; + try + { + added = await ProcessArchiveAsync(archiveContent, now, pendingDocuments, pendingMappings, archiveLastModified, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "RU-BDU archive processing failed"); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + _diagnostics.FetchSuccess(added, usedCache); + if (added > 0) + { + _logger.LogInformation("RU-BDU processed {Added} vulnerabilities (cacheUsed={CacheUsed})", added, usedCache); + } + else + { + _logger.LogDebug("RU-BDU fetch completed with no new vulnerabilities (cacheUsed={CacheUsed})", usedCache); + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastSuccessfulFetch(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.ParseFailure(); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("RU-BDU document {DocumentId} missing GridFS payload", documentId); + _diagnostics.ParseFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + byte[] payload; + try + { + payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "RU-BDU unable to download raw document {DocumentId}", documentId); + _diagnostics.ParseFailure(); + throw; + } + + RuBduVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(payload, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "RU-BDU failed to deserialize document {DocumentId}", documentId); + _diagnostics.ParseFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("RU-BDU document {DocumentId} produced null DTO", documentId); + _diagnostics.ParseFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-bdu.v1", bson, _timeProvider.GetUtcNow()); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + _diagnostics.ParseSuccess( + dto.Software.IsDefaultOrEmpty ? 0 : dto.Software.Length, + dto.Identifiers.IsDefaultOrEmpty ? 0 : dto.Identifiers.Length, + dto.Sources.IsDefaultOrEmpty ? 0 : dto.Sources.Length); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + _diagnostics.MapFailure(); + pendingMappings.Remove(documentId); + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + _logger.LogWarning("RU-BDU document {DocumentId} missing DTO payload", documentId); + _diagnostics.MapFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + RuBduVulnerabilityDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null"); + } + catch (Exception ex) + { + _logger.LogError(ex, "RU-BDU failed to deserialize DTO for document {DocumentId}", documentId); + _diagnostics.MapFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + try + { + var advisory = RuBduMapper.Map(dto, document, dtoRecord.ValidatedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + _diagnostics.MapSuccess(advisory); + pendingMappings.Remove(documentId); + } + catch (Exception ex) + { + _logger.LogError(ex, "RU-BDU mapping failed for document {DocumentId}", documentId); + _diagnostics.MapFailure(); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task ProcessArchiveAsync( + byte[] archiveContent, + DateTimeOffset now, + HashSet pendingDocuments, + HashSet pendingMappings, + DateTimeOffset? archiveLastModified, + CancellationToken cancellationToken) + { + var added = 0; + using var archiveStream = new MemoryStream(archiveContent, writable: false); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); + var entry = archive.GetEntry("export/export.xml") ?? archive.Entries.FirstOrDefault(); + if (entry is null) + { + _logger.LogWarning("RU-BDU archive does not contain export/export.xml; skipping."); + return added; + } + + await using var entryStream = entry.Open(); + using var reader = XmlReader.Create(entryStream, new XmlReaderSettings + { + IgnoreComments = true, + IgnoreWhitespace = true, + DtdProcessing = DtdProcessing.Ignore, + CloseInput = false, + }); + + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (reader.NodeType != XmlNodeType.Element || !reader.Name.Equals("vul", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (RuBduXmlParser.TryParse(XNode.ReadFrom(reader) as XElement ?? new XElement("vul")) is not { } dto) + { + continue; + } + + var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions); + var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var documentUri = BuildDocumentUri(dto.Identifier); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); + if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ru-bdu.identifier"] = dto.Identifier, + }; + + if (!string.IsNullOrWhiteSpace(dto.Name)) + { + metadata["ru-bdu.name"] = dto.Name!; + } + + var recordId = existing?.Id ?? Guid.NewGuid(); + var record = new DocumentRecord( + recordId, + SourceName, + documentUri, + now, + sha, + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: archiveLastModified ?? dto.IdentifyDate, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + pendingDocuments.Add(upserted.Id); + pendingMappings.Remove(upserted.Id); + added++; + + if (added >= _options.MaxVulnerabilitiesPerFetch) + { + break; + } + } + + return added; + } + + private string ResolveCacheDirectory(string? configuredPath) + { + if (!string.IsNullOrWhiteSpace(configuredPath)) + { + return Path.GetFullPath(Path.IsPathRooted(configuredPath) + ? configuredPath + : Path.Combine(AppContext.BaseDirectory, configuredPath)); + } + + return Path.Combine(AppContext.BaseDirectory, "cache", RuBduConnectorPlugin.SourceName); + } + + private void EnsureCacheDirectory() + { + try + { + Directory.CreateDirectory(_cacheDirectory); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "RU-BDU unable to ensure cache directory {CachePath}", _cacheDirectory); + } + } + + private void TryWriteCachedArchive(byte[] content) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(_archiveCachePath)!); + File.WriteAllBytes(_archiveCachePath, content); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "RU-BDU failed to write cache archive {CachePath}", _archiveCachePath); + } + } + + private bool TryReadCachedArchive(out byte[] content) + { + try + { + if (File.Exists(_archiveCachePath)) + { + content = File.ReadAllBytes(_archiveCachePath); + return true; + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "RU-BDU failed to read cache archive {CachePath}", _archiveCachePath); + } + + content = Array.Empty(); + return false; + } + + private static string BuildDocumentUri(string identifier) + { + var slug = identifier.Contains(':', StringComparison.Ordinal) + ? identifier[(identifier.IndexOf(':') + 1)..] + : identifier; + return $"https://bdu.fstec.ru/vul/{slug}"; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? RuBduCursor.Empty : RuBduCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(RuBduCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var completedAt = cursor.LastSuccessfulFetch ?? _timeProvider.GetUtcNow(); + return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnectorPlugin.cs index 1b38a5f7..2253737a 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ru.Bdu; - -public sealed class RuBduConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "ru-bdu"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ru.Bdu; + +public sealed class RuBduConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ru-bdu"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Ru.Bdu/RuBduDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs index 1d914e32..ea958241 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduDependencyInjectionRoutine.cs @@ -1,53 +1,53 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Ru.Bdu.Configuration; - -namespace StellaOps.Feedser.Source.Ru.Bdu; - -public sealed class RuBduDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:ru-bdu"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddRuBduConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, RuBduJobKinds.Fetch, typeof(RuBduFetchJob)); - EnsureJob(options, RuBduJobKinds.Parse, typeof(RuBduParseJob)); - EnsureJob(options, RuBduJobKinds.Map, typeof(RuBduMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType) - { - if (schedulerOptions.Definitions.ContainsKey(kind)) - { - return; - } - - schedulerOptions.Definitions[kind] = new JobDefinition( - kind, - jobType, - schedulerOptions.DefaultTimeout, - schedulerOptions.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Ru.Bdu.Configuration; + +namespace StellaOps.Concelier.Connector.Ru.Bdu; + +public sealed class RuBduDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:ru-bdu"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddRuBduConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, RuBduJobKinds.Fetch, typeof(RuBduFetchJob)); + EnsureJob(options, RuBduJobKinds.Parse, typeof(RuBduParseJob)); + EnsureJob(options, RuBduJobKinds.Map, typeof(RuBduMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType) + { + if (schedulerOptions.Definitions.ContainsKey(kind)) + { + return; + } + + schedulerOptions.Definitions[kind] = new JobDefinition( + kind, + jobType, + schedulerOptions.DefaultTimeout, + schedulerOptions.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduServiceCollectionExtensions.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Ru.Bdu/RuBduServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduServiceCollectionExtensions.cs index c35b843c..ac416a3c 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/RuBduServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/RuBduServiceCollectionExtensions.cs @@ -1,11 +1,11 @@ using System.Net; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ru.Bdu.Configuration; -using StellaOps.Feedser.Source.Ru.Bdu.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ru.Bdu.Configuration; +using StellaOps.Concelier.Connector.Ru.Bdu.Internal; -namespace StellaOps.Feedser.Source.Ru.Bdu; +namespace StellaOps.Concelier.Connector.Ru.Bdu; public static class RuBduServiceCollectionExtensions { diff --git a/src/StellaOps.Concelier.Connector.Ru.Bdu/StellaOps.Concelier.Connector.Ru.Bdu.csproj b/src/StellaOps.Concelier.Connector.Ru.Bdu/StellaOps.Concelier.Connector.Ru.Bdu.csproj new file mode 100644 index 00000000..ca36c398 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/StellaOps.Concelier.Connector.Ru.Bdu.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md b/src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md similarity index 84% rename from src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md rename to src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md index 36119b42..846d1b39 100644 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Ru.Bdu/TASKS.md @@ -1,11 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-RUBDU-02-001 Identify BDU data source & schema|BE-Conn-BDU|Research|**DONE (2025-10-11)** – Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml`, `...?format=json`) return 403/404 even with `--insecure` because TLS chain requires Russian Trusted Sub CA and WAF expects referer/session headers. Documented request/response samples in `docs/feedser-connector-research-20251011.md`; blocked until trusted root + access strategy from Ops.| -|FEEDCONN-RUBDU-02-002 Fetch pipeline & cursor handling|BE-Conn-BDU|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Connector streams `vulxml.zip` through cached fetches, persists JSON payloads via `RawDocumentStorage`, and tracks cursor pending sets. Added cache fallback + deterministic SHA logging and state updates tied to `TimeProvider`.| -|FEEDCONN-RUBDU-02-003 DTO/parser implementation|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – `RuBduXmlParser` now captures identifiers, source links, CVSS 2/3 metrics, CWE arrays, and environment/software metadata with coverage for multi-entry fixtures.| -|FEEDCONN-RUBDU-02-004 Canonical mapping & range primitives|BE-Conn-BDU|Models|**DONE (2025-10-14)** – `RuBduMapper` emits vendor/ICS packages with normalized `ru-bdu.raw` rules, dual status provenance, alias/reference hydration (CVE, external, source), and CVSS severity normalisation.| -|FEEDCONN-RUBDU-02-005 Deterministic fixtures & regression tests|QA|Testing|**DONE (2025-10-14)** – Added connector harness snapshot suite with canned archive, state/documents/dtos/advisories snapshots under `Fixtures/`, gated by `UPDATE_BDU_FIXTURES`.| -|FEEDCONN-RUBDU-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Introduced `RuBduDiagnostics` meter (fetch/parse/map counters & histograms) and authored connector README covering configuration, trusted roots, telemetry, and offline behaviour.| -|FEEDCONN-RUBDU-02-007 Access & export options assessment|BE-Conn-BDU|Research|**DONE (2025-10-14)** – Documented archive access constraints, offline mirroring expectations, and export packaging in `src/StellaOps.Feedser.Source.Ru.Bdu/README.md` + flagged Offline Kit bundling requirements.| -|FEEDCONN-RUBDU-02-008 Trusted root onboarding plan|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – Validated Russian Trusted Root/Sub CA bundle wiring (`certificates/russian_trusted_bundle.pem`), updated Offline Kit guidance, and surfaced `feedser:httpClients:source.bdu:trustedRootPaths` sample configuration.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-RUBDU-02-001 Identify BDU data source & schema|BE-Conn-BDU|Research|**DONE (2025-10-11)** – Candidate endpoints (`https://bdu.fstec.ru/component/rsform/form/7-bdu?format=xml`, `...?format=json`) return 403/404 even with `--insecure` because TLS chain requires Russian Trusted Sub CA and WAF expects referer/session headers. Documented request/response samples in `docs/concelier-connector-research-20251011.md`; blocked until trusted root + access strategy from Ops.| +|FEEDCONN-RUBDU-02-002 Fetch pipeline & cursor handling|BE-Conn-BDU|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Connector streams `vulxml.zip` through cached fetches, persists JSON payloads via `RawDocumentStorage`, and tracks cursor pending sets. Added cache fallback + deterministic SHA logging and state updates tied to `TimeProvider`.| +|FEEDCONN-RUBDU-02-003 DTO/parser implementation|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – `RuBduXmlParser` now captures identifiers, source links, CVSS 2/3 metrics, CWE arrays, and environment/software metadata with coverage for multi-entry fixtures.| +|FEEDCONN-RUBDU-02-004 Canonical mapping & range primitives|BE-Conn-BDU|Models|**DONE (2025-10-14)** – `RuBduMapper` emits vendor/ICS packages with normalized `ru-bdu.raw` rules, dual status provenance, alias/reference hydration (CVE, external, source), and CVSS severity normalisation.| +|FEEDCONN-RUBDU-02-005 Deterministic fixtures & regression tests|QA|Testing|**DONE (2025-10-14)** – Added connector harness snapshot suite with canned archive, state/documents/dtos/advisories snapshots under `Fixtures/`, gated by `UPDATE_BDU_FIXTURES`.| +|FEEDCONN-RUBDU-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Introduced `RuBduDiagnostics` meter (fetch/parse/map counters & histograms) and authored connector README covering configuration, trusted roots, telemetry, and offline behaviour.| +|FEEDCONN-RUBDU-02-007 Access & export options assessment|BE-Conn-BDU|Research|**DONE (2025-10-14)** – Documented archive access constraints, offline mirroring expectations, and export packaging in `src/StellaOps.Concelier.Connector.Ru.Bdu/README.md` + flagged Offline Kit bundling requirements.| +|FEEDCONN-RUBDU-02-008 Trusted root onboarding plan|BE-Conn-BDU|Source.Common|**DONE (2025-10-14)** – Validated Russian Trusted Root/Sub CA bundle wiring (`certificates/russian_trusted_bundle.pem`), updated Offline Kit guidance, and surfaced `concelier:httpClients:source.bdu:trustedRootPaths` sample configuration.| diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-legacy.json.zip b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/bulletin-legacy.json.zip similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-legacy.json.zip rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/bulletin-legacy.json.zip diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/bulletin-sample.json.zip diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing-page2.html b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/listing-page2.html similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing-page2.html rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/listing-page2.html diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/listing.html similarity index 96% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/listing.html index abd43946..08309506 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/listing.html +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/listing.html @@ -1,10 +1,10 @@ - - - - - - + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/Fixtures/nkcki-advisories.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs index d4d4eae2..97eef970 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiConnectorTests.cs @@ -1,291 +1,291 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Ru.Nkcki; -using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Testing; -using StellaOps.Feedser.Models; -using MongoDB.Driver; -using Xunit; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; - -[Collection("mongo-fixture")] -public sealed class RuNkckiConnectorTests : IAsyncLifetime -{ - private static readonly Uri ListingUri = new("https://cert.gov.ru/materialy/uyazvimosti/"); - private static readonly Uri ListingPage2Uri = new("https://cert.gov.ru/materialy/uyazvimosti/?PAGEN_1=2"); - private static readonly Uri BulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-sample.json.zip"); - private static readonly Uri LegacyBulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-legacy.json.zip"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - private readonly CannedHttpMessageHandler _handler; - - public RuNkckiConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesExpectedSnapshot() - { - await using var provider = await BuildServiceProviderAsync(); - SeedListingAndBulletin(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - _timeProvider.Advance(TimeSpan.FromMinutes(1)); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(2, advisories.Count); - - var snapshot = SnapshotSerializer.ToSnapshot(advisories); - WriteOrAssertSnapshot(snapshot, "nkcki-advisories.snapshot.json"); - - var documentStore = provider.GetRequiredService(); - var document = await documentStore.FindBySourceAndUriAsync(RuNkckiConnectorPlugin.SourceName, "https://cert.gov.ru/materialy/uyazvimosti/2025-01001", CancellationToken.None); - Assert.NotNull(document); - Assert.Equal(DocumentStatuses.Mapped, document!.Status); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(RuNkckiConnectorPlugin.SourceName, CancellationToken.None); - Assert.NotNull(state); - Assert.True(IsEmptyArray(state!.Cursor, "pendingDocuments")); - Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); - } - - [Fact] - public async Task Fetch_ReusesCachedBulletinWhenListingFails() - { - await using var provider = await BuildServiceProviderAsync(); - SeedListingAndBulletin(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - _handler.Clear(); - for (var i = 0; i < 3; i++) - { - _handler.AddResponse(ListingUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) - { - Content = new StringContent("error", Encoding.UTF8, "text/plain"), - }); - } - - var advisoryStore = provider.GetRequiredService(); - var before = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(2, before.Count); - - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var after = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(before.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key), after.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key)); - - _handler.AssertNoPendingResponses(); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddRuNkckiConnector(options => - { - options.BaseAddress = new Uri("https://cert.gov.ru/"); - options.ListingPath = "/materialy/uyazvimosti/"; - options.MaxBulletinsPerFetch = 2; - options.MaxListingPagesPerFetch = 2; - options.MaxVulnerabilitiesPerFetch = 50; - options.ListingCacheDuration = TimeSpan.Zero; - var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName); - Directory.CreateDirectory(cacheRoot); - options.CacheDirectory = Path.Combine(cacheRoot, "ru-nkcki"); - options.RequestDelay = TimeSpan.Zero; - }); - - services.Configure(RuNkckiOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedListingAndBulletin() - { - var listingHtml = ReadFixture("listing.html"); - _handler.AddTextResponse(ListingUri, listingHtml, "text/html"); - - var listingPage2Html = ReadFixture("listing-page2.html"); - _handler.AddTextResponse(ListingPage2Uri, listingPage2Html, "text/html"); - - var bulletinBytes = ReadBulletinFixture("bulletin-sample.json.zip"); - _handler.AddResponse(BulletinUri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new ByteArrayContent(bulletinBytes), - }; - response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); - response.Content.Headers.LastModified = new DateTimeOffset(2025, 9, 22, 0, 0, 0, TimeSpan.Zero); - return response; - }); - - var legacyBytes = ReadBulletinFixture("bulletin-legacy.json.zip"); - _handler.AddResponse(LegacyBulletinUri, () => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new ByteArrayContent(legacyBytes), - }; - response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); - response.Content.Headers.LastModified = new DateTimeOffset(2024, 8, 2, 0, 0, 0, TimeSpan.Zero); - return response; - }); - } - - private static bool IsEmptyArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return false; - } - - return array.Count == 0; - } - - private static string ReadFixture(string filename) - { - var path = Path.Combine("Fixtures", filename); - var resolved = ResolveFixturePath(path); - return File.ReadAllText(resolved); - } - - private static byte[] ReadBulletinFixture(string filename) - { - var path = Path.Combine("Fixtures", filename); - var resolved = ResolveFixturePath(path); - return File.ReadAllBytes(resolved); - } - - private static string ResolveFixturePath(string relativePath) - { - var projectRoot = GetProjectRoot(); - var projectPath = Path.Combine(projectRoot, relativePath); - if (File.Exists(projectPath)) - { - return projectPath; - } - - var binaryPath = Path.Combine(AppContext.BaseDirectory, relativePath); - if (File.Exists(binaryPath)) - { - return Path.GetFullPath(binaryPath); - } - - throw new FileNotFoundException($"Fixture not found: {relativePath}"); - } - - private static void WriteOrAssertSnapshot(string snapshot, string filename) - { - if (ShouldUpdateFixtures()) - { - var path = GetWritableFixturePath(filename); - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllText(path, snapshot); - return; - } - - var expectedPath = ResolveFixturePath(Path.Combine("Fixtures", filename)); - if (!File.Exists(expectedPath)) - { - throw new FileNotFoundException($"Expected snapshot missing: {expectedPath}. Set UPDATE_NKCKI_FIXTURES=1 to generate."); - } - - var expected = File.ReadAllText(expectedPath); - Assert.Equal(Normalize(expected), Normalize(snapshot)); - } - - private static string GetWritableFixturePath(string filename) - { - var projectRoot = GetProjectRoot(); - return Path.Combine(projectRoot, "Fixtures", filename); - } - - private static bool ShouldUpdateFixtures() - { - var value = Environment.GetEnvironmentVariable("UPDATE_NKCKI_FIXTURES"); - return string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) - || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); - } - - private static string Normalize(string text) - => text.Replace("\r\n", "\n", StringComparison.Ordinal); - - private static string GetProjectRoot() - { - var current = AppContext.BaseDirectory; - while (!string.IsNullOrEmpty(current)) - { - var candidate = Path.Combine(current, "StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj"); - if (File.Exists(candidate)) - { - return current; - } - - current = Path.GetDirectoryName(current); - } - - throw new InvalidOperationException("Unable to locate project root for Ru.Nkcki tests."); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public async Task DisposeAsync() - => await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); -} +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Ru.Nkcki; +using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Testing; +using StellaOps.Concelier.Models; +using MongoDB.Driver; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Tests; + +[Collection("mongo-fixture")] +public sealed class RuNkckiConnectorTests : IAsyncLifetime +{ + private static readonly Uri ListingUri = new("https://cert.gov.ru/materialy/uyazvimosti/"); + private static readonly Uri ListingPage2Uri = new("https://cert.gov.ru/materialy/uyazvimosti/?PAGEN_1=2"); + private static readonly Uri BulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-sample.json.zip"); + private static readonly Uri LegacyBulletinUri = new("https://cert.gov.ru/materialy/uyazvimosti/bulletin-legacy.json.zip"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + private readonly CannedHttpMessageHandler _handler; + + public RuNkckiConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesExpectedSnapshot() + { + await using var provider = await BuildServiceProviderAsync(); + SeedListingAndBulletin(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, advisories.Count); + + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + WriteOrAssertSnapshot(snapshot, "nkcki-advisories.snapshot.json"); + + var documentStore = provider.GetRequiredService(); + var document = await documentStore.FindBySourceAndUriAsync(RuNkckiConnectorPlugin.SourceName, "https://cert.gov.ru/materialy/uyazvimosti/2025-01001", CancellationToken.None); + Assert.NotNull(document); + Assert.Equal(DocumentStatuses.Mapped, document!.Status); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(RuNkckiConnectorPlugin.SourceName, CancellationToken.None); + Assert.NotNull(state); + Assert.True(IsEmptyArray(state!.Cursor, "pendingDocuments")); + Assert.True(IsEmptyArray(state.Cursor, "pendingMappings")); + } + + [Fact] + public async Task Fetch_ReusesCachedBulletinWhenListingFails() + { + await using var provider = await BuildServiceProviderAsync(); + SeedListingAndBulletin(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + _handler.Clear(); + for (var i = 0; i < 3; i++) + { + _handler.AddResponse(ListingUri, () => new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("error", Encoding.UTF8, "text/plain"), + }); + } + + var advisoryStore = provider.GetRequiredService(); + var before = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(2, before.Count); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var after = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(before.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key), after.Select(advisory => advisory.AdvisoryKey).OrderBy(static key => key)); + + _handler.AssertNoPendingResponses(); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddRuNkckiConnector(options => + { + options.BaseAddress = new Uri("https://cert.gov.ru/"); + options.ListingPath = "/materialy/uyazvimosti/"; + options.MaxBulletinsPerFetch = 2; + options.MaxListingPagesPerFetch = 2; + options.MaxVulnerabilitiesPerFetch = 50; + options.ListingCacheDuration = TimeSpan.Zero; + var cacheRoot = Path.Combine(Path.GetTempPath(), "stellaops-tests", _fixture.Database.DatabaseNamespace.DatabaseName); + Directory.CreateDirectory(cacheRoot); + options.CacheDirectory = Path.Combine(cacheRoot, "ru-nkcki"); + options.RequestDelay = TimeSpan.Zero; + }); + + services.Configure(RuNkckiOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => builder.PrimaryHandler = _handler); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedListingAndBulletin() + { + var listingHtml = ReadFixture("listing.html"); + _handler.AddTextResponse(ListingUri, listingHtml, "text/html"); + + var listingPage2Html = ReadFixture("listing-page2.html"); + _handler.AddTextResponse(ListingPage2Uri, listingPage2Html, "text/html"); + + var bulletinBytes = ReadBulletinFixture("bulletin-sample.json.zip"); + _handler.AddResponse(BulletinUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(bulletinBytes), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + response.Content.Headers.LastModified = new DateTimeOffset(2025, 9, 22, 0, 0, 0, TimeSpan.Zero); + return response; + }); + + var legacyBytes = ReadBulletinFixture("bulletin-legacy.json.zip"); + _handler.AddResponse(LegacyBulletinUri, () => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(legacyBytes), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/zip"); + response.Content.Headers.LastModified = new DateTimeOffset(2024, 8, 2, 0, 0, 0, TimeSpan.Zero); + return response; + }); + } + + private static bool IsEmptyArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return false; + } + + return array.Count == 0; + } + + private static string ReadFixture(string filename) + { + var path = Path.Combine("Fixtures", filename); + var resolved = ResolveFixturePath(path); + return File.ReadAllText(resolved); + } + + private static byte[] ReadBulletinFixture(string filename) + { + var path = Path.Combine("Fixtures", filename); + var resolved = ResolveFixturePath(path); + return File.ReadAllBytes(resolved); + } + + private static string ResolveFixturePath(string relativePath) + { + var projectRoot = GetProjectRoot(); + var projectPath = Path.Combine(projectRoot, relativePath); + if (File.Exists(projectPath)) + { + return projectPath; + } + + var binaryPath = Path.Combine(AppContext.BaseDirectory, relativePath); + if (File.Exists(binaryPath)) + { + return Path.GetFullPath(binaryPath); + } + + throw new FileNotFoundException($"Fixture not found: {relativePath}"); + } + + private static void WriteOrAssertSnapshot(string snapshot, string filename) + { + if (ShouldUpdateFixtures()) + { + var path = GetWritableFixturePath(filename); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, snapshot); + return; + } + + var expectedPath = ResolveFixturePath(Path.Combine("Fixtures", filename)); + if (!File.Exists(expectedPath)) + { + throw new FileNotFoundException($"Expected snapshot missing: {expectedPath}. Set UPDATE_NKCKI_FIXTURES=1 to generate."); + } + + var expected = File.ReadAllText(expectedPath); + Assert.Equal(Normalize(expected), Normalize(snapshot)); + } + + private static string GetWritableFixturePath(string filename) + { + var projectRoot = GetProjectRoot(); + return Path.Combine(projectRoot, "Fixtures", filename); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable("UPDATE_NKCKI_FIXTURES"); + return string.Equals(value, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static string Normalize(string text) + => text.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string GetProjectRoot() + { + var current = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(current)) + { + var candidate = Path.Combine(current, "StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj"); + if (File.Exists(candidate)) + { + return current; + } + + current = Path.GetDirectoryName(current); + } + + throw new InvalidOperationException("Unable to locate project root for Ru.Nkcki tests."); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public async Task DisposeAsync() + => await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs index 1f9f18cb..c8f836ab 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiJsonParserTests.cs @@ -1,60 +1,60 @@ -using System.Text.Json; -using StellaOps.Feedser.Source.Ru.Nkcki.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; - -public sealed class RuNkckiJsonParserTests -{ - [Fact] - public void Parse_WellFormedEntry_ReturnsDto() - { - const string json = """ -{ - "vuln_id": {"MITRE": "CVE-2025-0001", "FSTEC": "BDU:2025-00001"}, - "date_published": "2025-09-01", - "date_updated": "2025-09-02", - "cvss_rating": "КРИТИЧЕСКИЙ", - "patch_available": true, - "description": "Test description", - "cwe": {"cwe_number": 79, "cwe_description": "Cross-site scripting"}, - "product_category": ["Web", "CMS"], - "mitigation": ["Apply update", "Review configuration"], - "vulnerable_software": { - "software_text": "ExampleCMS <= 1.0", - "software": [{"vendor": "Example", "name": "ExampleCMS", "version": "<= 1.0"}], - "cpe": false - }, - "cvss": { - "cvss_score": 8.8, - "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", - "cvss_score_v4": 5.5, - "cvss_vector_v4": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H" - }, - "impact": "ACE", - "method_of_exploitation": "Special request", - "user_interaction": false, - "urls": ["https://example.com/advisory", {"url": "https://cert.gov.ru/materialy/uyazvimosti/2025-00001"}], - "tags": ["cms"] -} -"""; - - using var document = JsonDocument.Parse(json); - var dto = RuNkckiJsonParser.Parse(document.RootElement); - - Assert.Equal("BDU:2025-00001", dto.FstecId); - Assert.Equal("CVE-2025-0001", dto.MitreId); - Assert.Equal(8.8, dto.CvssScore); - Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", dto.CvssVector); - Assert.True(dto.PatchAvailable); - Assert.Equal(79, dto.Cwe?.Number); - Assert.Contains("Web", dto.ProductCategories); - Assert.Contains("CMS", dto.ProductCategories); - Assert.Single(dto.VulnerableSoftwareEntries); - var entry = dto.VulnerableSoftwareEntries[0]; - Assert.Equal("Example ExampleCMS", entry.Identifier); - Assert.Contains("<= 1.0", entry.RangeExpressions); - Assert.Equal(2, dto.Urls.Length); - Assert.Contains("cms", dto.Tags); - } -} +using System.Text.Json; +using StellaOps.Concelier.Connector.Ru.Nkcki.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Tests; + +public sealed class RuNkckiJsonParserTests +{ + [Fact] + public void Parse_WellFormedEntry_ReturnsDto() + { + const string json = """ +{ + "vuln_id": {"MITRE": "CVE-2025-0001", "FSTEC": "BDU:2025-00001"}, + "date_published": "2025-09-01", + "date_updated": "2025-09-02", + "cvss_rating": "КРИТИЧЕСКИЙ", + "patch_available": true, + "description": "Test description", + "cwe": {"cwe_number": 79, "cwe_description": "Cross-site scripting"}, + "product_category": ["Web", "CMS"], + "mitigation": ["Apply update", "Review configuration"], + "vulnerable_software": { + "software_text": "ExampleCMS <= 1.0", + "software": [{"vendor": "Example", "name": "ExampleCMS", "version": "<= 1.0"}], + "cpe": false + }, + "cvss": { + "cvss_score": 8.8, + "cvss_vector": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + "cvss_score_v4": 5.5, + "cvss_vector_v4": "CVSS:4.0/AV:N/AC:L/AT:N/PR:N/UI:N/VC:H/VI:H/VA:H" + }, + "impact": "ACE", + "method_of_exploitation": "Special request", + "user_interaction": false, + "urls": ["https://example.com/advisory", {"url": "https://cert.gov.ru/materialy/uyazvimosti/2025-00001"}], + "tags": ["cms"] +} +"""; + + using var document = JsonDocument.Parse(json); + var dto = RuNkckiJsonParser.Parse(document.RootElement); + + Assert.Equal("BDU:2025-00001", dto.FstecId); + Assert.Equal("CVE-2025-0001", dto.MitreId); + Assert.Equal(8.8, dto.CvssScore); + Assert.Equal("CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", dto.CvssVector); + Assert.True(dto.PatchAvailable); + Assert.Equal(79, dto.Cwe?.Number); + Assert.Contains("Web", dto.ProductCategories); + Assert.Contains("CMS", dto.ProductCategories); + Assert.Single(dto.VulnerableSoftwareEntries); + var entry = dto.VulnerableSoftwareEntries[0]; + Assert.Equal("Example ExampleCMS", entry.Identifier); + Assert.Contains("<= 1.0", entry.RangeExpressions); + Assert.Equal(2, dto.Urls.Length); + Assert.Contains("cms", dto.Tags); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiMapperTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiMapperTests.cs index acb44e79..46ebb6c2 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/RuNkckiMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/RuNkckiMapperTests.cs @@ -1,81 +1,81 @@ -using System.Collections.Immutable; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Ru.Nkcki.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using Xunit; -using System.Reflection; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Tests; - -public sealed class RuNkckiMapperTests -{ - [Fact] - public void Map_ConstructsCanonicalAdvisory() - { - var softwareEntries = ImmutableArray.Create( - new RuNkckiSoftwareEntry( - "SampleVendor SampleSCADA", - "SampleVendor SampleSCADA <= 4.2", - ImmutableArray.Create("<= 4.2"))); - - var dto = new RuNkckiVulnerabilityDto( - FstecId: "BDU:2025-00001", - MitreId: "CVE-2025-0001", - DatePublished: new DateTimeOffset(2025, 9, 1, 0, 0, 0, TimeSpan.Zero), - DateUpdated: new DateTimeOffset(2025, 9, 2, 0, 0, 0, TimeSpan.Zero), - CvssRating: "КРИТИЧЕСКИЙ", - PatchAvailable: true, - Description: "Test NKCKI vulnerability", - Cwe: new RuNkckiCweDto(79, "Cross-site scripting"), - ProductCategories: ImmutableArray.Create("ICS", "Automation"), - Mitigation: "Apply update", - VulnerableSoftwareText: null, - VulnerableSoftwareHasCpe: false, - VulnerableSoftwareEntries: softwareEntries, - CvssScore: 8.8, - CvssVector: "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", - CvssScoreV4: 6.4, - CvssVectorV4: "CVSS:4.0/AV:N/AC:H/AT:N/PR:L/UI:N/VC:H/VI:H/VA:H", - Impact: "ACE", - MethodOfExploitation: "Special request", - UserInteraction: false, - Urls: ImmutableArray.Create("https://example.com/advisory", "https://cert.gov.ru/materialy/uyazvimosti/2025-00001"), - Tags: ImmutableArray.Create("ics")); - - var document = new DocumentRecord( - Guid.NewGuid(), - RuNkckiConnectorPlugin.SourceName, - "https://cert.gov.ru/materialy/uyazvimosti/2025-00001", - DateTimeOffset.UtcNow, - "abc", - DocumentStatuses.PendingMap, - "application/json", - null, - null, - null, - dto.DateUpdated, - ObjectId.GenerateNewId()); - - Assert.Equal("КРИТИЧЕСКИЙ", dto.CvssRating); - var normalizeSeverity = typeof(RuNkckiMapper).GetMethod("NormalizeSeverity", BindingFlags.NonPublic | BindingFlags.Static)!; - var ratingSeverity = (string?)normalizeSeverity.Invoke(null, new object?[] { dto.CvssRating }); - Assert.Equal("critical", ratingSeverity); - - var advisory = RuNkckiMapper.Map(dto, document, dto.DateUpdated!.Value); - - Assert.Contains("BDU:2025-00001", advisory.Aliases); - Assert.Contains("CVE-2025-0001", advisory.Aliases); - Assert.Equal("critical", advisory.Severity); - Assert.True(advisory.ExploitKnown); - Assert.Single(advisory.AffectedPackages); - var package = advisory.AffectedPackages[0]; - Assert.Equal(AffectedPackageTypes.IcsVendor, package.Type); - Assert.Single(package.NormalizedVersions); - Assert.Equal(2, advisory.CvssMetrics.Length); - Assert.Contains(advisory.CvssMetrics, metric => metric.Version == "4.0"); - Assert.Equal("critical", advisory.Severity); - Assert.Contains(advisory.References, reference => reference.Url.Contains("example.com", StringComparison.OrdinalIgnoreCase)); - } -} +using System.Collections.Immutable; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Ru.Nkcki.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using Xunit; +using System.Reflection; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Tests; + +public sealed class RuNkckiMapperTests +{ + [Fact] + public void Map_ConstructsCanonicalAdvisory() + { + var softwareEntries = ImmutableArray.Create( + new RuNkckiSoftwareEntry( + "SampleVendor SampleSCADA", + "SampleVendor SampleSCADA <= 4.2", + ImmutableArray.Create("<= 4.2"))); + + var dto = new RuNkckiVulnerabilityDto( + FstecId: "BDU:2025-00001", + MitreId: "CVE-2025-0001", + DatePublished: new DateTimeOffset(2025, 9, 1, 0, 0, 0, TimeSpan.Zero), + DateUpdated: new DateTimeOffset(2025, 9, 2, 0, 0, 0, TimeSpan.Zero), + CvssRating: "КРИТИЧЕСКИЙ", + PatchAvailable: true, + Description: "Test NKCKI vulnerability", + Cwe: new RuNkckiCweDto(79, "Cross-site scripting"), + ProductCategories: ImmutableArray.Create("ICS", "Automation"), + Mitigation: "Apply update", + VulnerableSoftwareText: null, + VulnerableSoftwareHasCpe: false, + VulnerableSoftwareEntries: softwareEntries, + CvssScore: 8.8, + CvssVector: "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H", + CvssScoreV4: 6.4, + CvssVectorV4: "CVSS:4.0/AV:N/AC:H/AT:N/PR:L/UI:N/VC:H/VI:H/VA:H", + Impact: "ACE", + MethodOfExploitation: "Special request", + UserInteraction: false, + Urls: ImmutableArray.Create("https://example.com/advisory", "https://cert.gov.ru/materialy/uyazvimosti/2025-00001"), + Tags: ImmutableArray.Create("ics")); + + var document = new DocumentRecord( + Guid.NewGuid(), + RuNkckiConnectorPlugin.SourceName, + "https://cert.gov.ru/materialy/uyazvimosti/2025-00001", + DateTimeOffset.UtcNow, + "abc", + DocumentStatuses.PendingMap, + "application/json", + null, + null, + null, + dto.DateUpdated, + ObjectId.GenerateNewId()); + + Assert.Equal("КРИТИЧЕСКИЙ", dto.CvssRating); + var normalizeSeverity = typeof(RuNkckiMapper).GetMethod("NormalizeSeverity", BindingFlags.NonPublic | BindingFlags.Static)!; + var ratingSeverity = (string?)normalizeSeverity.Invoke(null, new object?[] { dto.CvssRating }); + Assert.Equal("critical", ratingSeverity); + + var advisory = RuNkckiMapper.Map(dto, document, dto.DateUpdated!.Value); + + Assert.Contains("BDU:2025-00001", advisory.Aliases); + Assert.Contains("CVE-2025-0001", advisory.Aliases); + Assert.Equal("critical", advisory.Severity); + Assert.True(advisory.ExploitKnown); + Assert.Single(advisory.AffectedPackages); + var package = advisory.AffectedPackages[0]; + Assert.Equal(AffectedPackageTypes.IcsVendor, package.Type); + Assert.Single(package.NormalizedVersions); + Assert.Equal(2, advisory.CvssMetrics.Length); + Assert.Contains(advisory.CvssMetrics, metric => metric.Version == "4.0"); + Assert.Equal("critical", advisory.Severity); + Assert.Contains(advisory.References, reference => reference.Url.Contains("example.com", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj new file mode 100644 index 00000000..ba7c8b8c --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki.Tests/StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md b/src/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md similarity index 81% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md index 00b65152..96b8e81a 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/AGENTS.md @@ -1,38 +1,38 @@ -# AGENTS -## Role -Implement the Russian NKTsKI (formerly NKCKI) advisories connector to ingest NKTsKI vulnerability bulletins for Feedser’s regional coverage. - -## Scope -- Identify NKTsKI advisory feeds/APIs (HTML, RSS, CSV) and access/authentication requirements. -- Implement fetch/cursor pipeline with dedupe and failure backoff tailored to the source format. -- Parse advisories to extract summary, affected vendors/products, recommended mitigation, and CVE identifiers. -- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives. -- Create deterministic fixtures and regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). -- `Feedser.Models` (canonical data structures). -- `Feedser.Testing` (integration fixtures, snapshots). - -## Interfaces & Contracts -- Job kinds: `nkcki:fetch`, `nkcki:parse`, `nkcki:map`. -- Persist upstream modification metadata to support incremental updates. -- Alias set should include NKTsKI advisory IDs and CVEs when present. - -## In/Out of scope -In scope: -- Core ingestion/mapping pipeline with range primitives. - -Out of scope: -- Translation beyond canonical field normalisation. - -## Observability & Security Expectations -- Log fetch/mapping activity; mark failures with backoff delays. -- Handle Cyrillic text encoding and sanitise HTML safely. -- Respect upstream rate limiting/politeness. - -## Tests -- Add `StellaOps.Feedser.Source.Ru.Nkcki.Tests` for fetch/parse/map with canned fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Implement the Russian NKTsKI (formerly NKCKI) advisories connector to ingest NKTsKI vulnerability bulletins for Concelier’s regional coverage. + +## Scope +- Identify NKTsKI advisory feeds/APIs (HTML, RSS, CSV) and access/authentication requirements. +- Implement fetch/cursor pipeline with dedupe and failure backoff tailored to the source format. +- Parse advisories to extract summary, affected vendors/products, recommended mitigation, and CVE identifiers. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives. +- Create deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Concelier.Models` (canonical data structures). +- `Concelier.Testing` (integration fixtures, snapshots). + +## Interfaces & Contracts +- Job kinds: `nkcki:fetch`, `nkcki:parse`, `nkcki:map`. +- Persist upstream modification metadata to support incremental updates. +- Alias set should include NKTsKI advisory IDs and CVEs when present. + +## In/Out of scope +In scope: +- Core ingestion/mapping pipeline with range primitives. + +Out of scope: +- Translation beyond canonical field normalisation. + +## Observability & Security Expectations +- Log fetch/mapping activity; mark failures with backoff delays. +- Handle Cyrillic text encoding and sanitise HTML safely. +- Respect upstream rate limiting/politeness. + +## Tests +- Add `StellaOps.Concelier.Connector.Ru.Nkcki.Tests` for fetch/parse/map with canned fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Configuration/RuNkckiOptions.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Configuration/RuNkckiOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Configuration/RuNkckiOptions.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Configuration/RuNkckiOptions.cs index cf3a1f66..5b871912 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Configuration/RuNkckiOptions.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Configuration/RuNkckiOptions.cs @@ -1,137 +1,137 @@ -using System.Net; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Configuration; - -/// -/// Connector options for the Russian NKTsKI bulletin ingestion pipeline. -/// -public sealed class RuNkckiOptions -{ - public const string HttpClientName = "ru-nkcki"; - - private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90); - private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20); - private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10); - - /// - /// Base endpoint used for resolving relative resource links. - /// - public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute); - - /// - /// Relative path to the bulletin listing page. - /// - public string ListingPath { get; set; } = "materialy/uyazvimosti/"; - - /// - /// Timeout applied to listing and bulletin fetch requests. - /// - public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; - - /// - /// Backoff applied when the listing or attachments cannot be retrieved. - /// - public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; - - /// - /// Maximum number of bulletin attachments downloaded per fetch run. - /// - public int MaxBulletinsPerFetch { get; set; } = 5; - - /// - /// Maximum number of listing pages visited per fetch cycle. - /// - public int MaxListingPagesPerFetch { get; set; } = 3; - - /// - /// Maximum number of vulnerabilities ingested per fetch cycle across all attachments. - /// - public int MaxVulnerabilitiesPerFetch { get; set; } = 250; - - /// - /// Maximum bulletin identifiers remembered to avoid refetching historical files. - /// - public int KnownBulletinCapacity { get; set; } = 512; - - /// - /// Delay between sequential bulletin downloads. - /// - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - /// - /// Duration the HTML listing can be cached before forcing a refetch. - /// - public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache; - - public string UserAgent { get; set; } = "StellaOps/Feedser (+https://stella-ops.org)"; - - public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4"; - - /// - /// Absolute URI for the listing page. - /// - public Uri ListingUri => new(BaseAddress, ListingPath); - - /// - /// Optional directory for caching downloaded bulletins (relative paths resolve under the content root). - /// - public string? CacheDirectory { get; set; } = null; - - public void Validate() - { - if (BaseAddress is null || !BaseAddress.IsAbsoluteUri) - { - throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI."); - } - - if (string.IsNullOrWhiteSpace(ListingPath)) - { - throw new InvalidOperationException("RuNkcki ListingPath must be provided."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("RuNkcki RequestTimeout must be positive."); - } - - if (FailureBackoff < TimeSpan.Zero) - { - throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative."); - } - - if (MaxBulletinsPerFetch <= 0) - { - throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero."); - } - - if (MaxListingPagesPerFetch <= 0) - { - throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero."); - } - - if (MaxVulnerabilitiesPerFetch <= 0) - { - throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero."); - } - - if (KnownBulletinCapacity <= 0) - { - throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero."); - } - - if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0) - { - throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace."); - } - - if (string.IsNullOrWhiteSpace(UserAgent)) - { - throw new InvalidOperationException("RuNkcki UserAgent cannot be empty."); - } - - if (string.IsNullOrWhiteSpace(AcceptLanguage)) - { - throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty."); - } - } -} +using System.Net; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Configuration; + +/// +/// Connector options for the Russian NKTsKI bulletin ingestion pipeline. +/// +public sealed class RuNkckiOptions +{ + public const string HttpClientName = "ru-nkcki"; + + private static readonly TimeSpan DefaultRequestTimeout = TimeSpan.FromSeconds(90); + private static readonly TimeSpan DefaultFailureBackoff = TimeSpan.FromMinutes(20); + private static readonly TimeSpan DefaultListingCache = TimeSpan.FromMinutes(10); + + /// + /// Base endpoint used for resolving relative resource links. + /// + public Uri BaseAddress { get; set; } = new("https://cert.gov.ru/", UriKind.Absolute); + + /// + /// Relative path to the bulletin listing page. + /// + public string ListingPath { get; set; } = "materialy/uyazvimosti/"; + + /// + /// Timeout applied to listing and bulletin fetch requests. + /// + public TimeSpan RequestTimeout { get; set; } = DefaultRequestTimeout; + + /// + /// Backoff applied when the listing or attachments cannot be retrieved. + /// + public TimeSpan FailureBackoff { get; set; } = DefaultFailureBackoff; + + /// + /// Maximum number of bulletin attachments downloaded per fetch run. + /// + public int MaxBulletinsPerFetch { get; set; } = 5; + + /// + /// Maximum number of listing pages visited per fetch cycle. + /// + public int MaxListingPagesPerFetch { get; set; } = 3; + + /// + /// Maximum number of vulnerabilities ingested per fetch cycle across all attachments. + /// + public int MaxVulnerabilitiesPerFetch { get; set; } = 250; + + /// + /// Maximum bulletin identifiers remembered to avoid refetching historical files. + /// + public int KnownBulletinCapacity { get; set; } = 512; + + /// + /// Delay between sequential bulletin downloads. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Duration the HTML listing can be cached before forcing a refetch. + /// + public TimeSpan ListingCacheDuration { get; set; } = DefaultListingCache; + + public string UserAgent { get; set; } = "StellaOps/Concelier (+https://stella-ops.org)"; + + public string AcceptLanguage { get; set; } = "ru-RU,ru;q=0.9,en-US;q=0.6,en;q=0.4"; + + /// + /// Absolute URI for the listing page. + /// + public Uri ListingUri => new(BaseAddress, ListingPath); + + /// + /// Optional directory for caching downloaded bulletins (relative paths resolve under the content root). + /// + public string? CacheDirectory { get; set; } = null; + + public void Validate() + { + if (BaseAddress is null || !BaseAddress.IsAbsoluteUri) + { + throw new InvalidOperationException("RuNkcki BaseAddress must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(ListingPath)) + { + throw new InvalidOperationException("RuNkcki ListingPath must be provided."); + } + + if (RequestTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("RuNkcki RequestTimeout must be positive."); + } + + if (FailureBackoff < TimeSpan.Zero) + { + throw new InvalidOperationException("RuNkcki FailureBackoff cannot be negative."); + } + + if (MaxBulletinsPerFetch <= 0) + { + throw new InvalidOperationException("RuNkcki MaxBulletinsPerFetch must be greater than zero."); + } + + if (MaxListingPagesPerFetch <= 0) + { + throw new InvalidOperationException("RuNkcki MaxListingPagesPerFetch must be greater than zero."); + } + + if (MaxVulnerabilitiesPerFetch <= 0) + { + throw new InvalidOperationException("RuNkcki MaxVulnerabilitiesPerFetch must be greater than zero."); + } + + if (KnownBulletinCapacity <= 0) + { + throw new InvalidOperationException("RuNkcki KnownBulletinCapacity must be greater than zero."); + } + + if (CacheDirectory is not null && CacheDirectory.Trim().Length == 0) + { + throw new InvalidOperationException("RuNkcki CacheDirectory cannot be whitespace."); + } + + if (string.IsNullOrWhiteSpace(UserAgent)) + { + throw new InvalidOperationException("RuNkcki UserAgent cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(AcceptLanguage)) + { + throw new InvalidOperationException("RuNkcki AcceptLanguage cannot be empty."); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiCursor.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiCursor.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiCursor.cs index 98714af8..0051c1d8 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiCursor.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiCursor.cs @@ -1,108 +1,108 @@ -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; - -internal sealed record RuNkckiCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - IReadOnlyCollection KnownBulletins, - DateTimeOffset? LastListingFetchAt) -{ - private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); - private static readonly IReadOnlyCollection EmptyBulletins = Array.Empty(); - - public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null); - - public RuNkckiCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = (documents ?? Enumerable.Empty()).Distinct().ToArray() }; - - public RuNkckiCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = (mappings ?? Enumerable.Empty()).Distinct().ToArray() }; - - public RuNkckiCursor WithKnownBulletins(IEnumerable bulletins) - => this with { KnownBulletins = (bulletins ?? Enumerable.Empty()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() }; - - public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp) - => this with { LastListingFetchAt = timestamp }; - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - ["knownBulletins"] = new BsonArray(KnownBulletins), - }; - - if (LastListingFetchAt.HasValue) - { - document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime; - } - - return document; - } - - public static RuNkckiCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var knownBulletins = ReadStringArray(document, "knownBulletins"); - var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue) - ? ParseDate(dateValue) - : null; - - return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch); - } - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuids; - } - - var result = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var guid)) - { - result.Add(guid); - } - } - - return result; - } - - private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyBulletins; - } - - var result = new List(array.Count); - foreach (var element in array) - { - var text = element?.ToString(); - if (!string.IsNullOrWhiteSpace(text)) - { - result.Add(text); - } - } - - return result; - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal; + +internal sealed record RuNkckiCursor( + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings, + IReadOnlyCollection KnownBulletins, + DateTimeOffset? LastListingFetchAt) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + private static readonly IReadOnlyCollection EmptyBulletins = Array.Empty(); + + public static RuNkckiCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyBulletins, null); + + public RuNkckiCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = (documents ?? Enumerable.Empty()).Distinct().ToArray() }; + + public RuNkckiCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = (mappings ?? Enumerable.Empty()).Distinct().ToArray() }; + + public RuNkckiCursor WithKnownBulletins(IEnumerable bulletins) + => this with { KnownBulletins = (bulletins ?? Enumerable.Empty()).Where(static id => !string.IsNullOrWhiteSpace(id)).Distinct(StringComparer.OrdinalIgnoreCase).ToArray() }; + + public RuNkckiCursor WithLastListingFetch(DateTimeOffset? timestamp) + => this with { LastListingFetchAt = timestamp }; + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + ["knownBulletins"] = new BsonArray(KnownBulletins), + }; + + if (LastListingFetchAt.HasValue) + { + document["lastListingFetchAt"] = LastListingFetchAt.Value.UtcDateTime; + } + + return document; + } + + public static RuNkckiCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + var knownBulletins = ReadStringArray(document, "knownBulletins"); + var lastListingFetch = document.TryGetValue("lastListingFetchAt", out var dateValue) + ? ParseDate(dateValue) + : null; + + return new RuNkckiCursor(pendingDocuments, pendingMappings, knownBulletins, lastListingFetch); + } + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; + } + + var result = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element?.ToString(), out var guid)) + { + result.Add(guid); + } + } + + return result; + } + + private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyBulletins; + } + + var result = new List(array.Count); + foreach (var element in array) + { + var text = element?.ToString(); + if (!string.IsNullOrWhiteSpace(text)) + { + result.Add(text); + } + } + + return result; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs index 9c4fa9ba..c2af7c87 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiDiagnostics.cs @@ -2,14 +2,14 @@ using System; using System.Collections.Generic; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal; /// /// Emits telemetry counters for the NKCKI connector lifecycle. /// public sealed class RuNkckiDiagnostics : IDisposable { - private const string MeterName = "StellaOps.Feedser.Source.Ru.Nkcki"; + private const string MeterName = "StellaOps.Concelier.Connector.Ru.Nkcki"; private const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiJsonParser.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiJsonParser.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiJsonParser.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiJsonParser.cs index aa706f83..86f0e9a4 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiJsonParser.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiJsonParser.cs @@ -1,646 +1,646 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using System.Text.RegularExpressions; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; - -internal static class RuNkckiJsonParser -{ - private static readonly Regex ComparatorRegex = new( - @"^(?.+?)\s*(?<=|>=|<|>|==|=)\s*(?.+?)$", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - - private static readonly Regex RangeRegex = new( - @"^(?.+?)\s+(?[\p{L}\p{N}\._-]+)\s*[-–]\s*(?[\p{L}\p{N}\._-]+)$", - RegexOptions.Compiled | RegexOptions.CultureInvariant); - - private static readonly Regex QualifierRegex = new( - @"^(?.+?)\s+(?[\p{L}\p{N}\._-]+)\s+(?(and\s+earlier|and\s+later|and\s+newer|до\s+и\s+включительно|и\s+ниже|и\s+выше|и\s+старше|и\s+позже))$", - RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); - - private static readonly Regex QualifierInlineRegex = new( - @"верс(ии|ия)\s+(?[\p{L}\p{N}\._-]+)\s+(?и\s+ниже|и\s+выше|и\s+старше)", - RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); - - private static readonly Regex VersionWindowRegex = new( - @"верс(ии|ия)\s+(?[\p{L}\p{N}\._-]+)\s+по\s+(?[\p{L}\p{N}\._-]+)", - RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); - - private static readonly char[] SoftwareSplitDelimiters = { '\n', ';', '\u2022', '\u2023', '\r' }; - - private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; - - public static RuNkckiVulnerabilityDto Parse(JsonElement element) - { - var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec) - ? Normalize(fstec.GetString()) - : null; - var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre) - ? Normalize(mitre.GetString()) - : null; - - var datePublished = ParseDate(element.TryGetProperty("date_published", out var published) ? published.GetString() : null); - var dateUpdated = ParseDate(element.TryGetProperty("date_updated", out var updated) ? updated.GetString() : null); - var cvssRating = Normalize(element.TryGetProperty("cvss_rating", out var rating) ? rating.GetString() : null); - bool? patchAvailable = element.TryGetProperty("patch_available", out var patch) ? patch.ValueKind switch - { - JsonValueKind.True => true, - JsonValueKind.False => false, - _ => null, - } : null; - - var description = ReadJoinedString(element, "description"); - var mitigation = ReadJoinedString(element, "mitigation"); - var productCategories = ReadStringCollection(element, "product_category"); - var impact = ReadJoinedString(element, "impact"); - var method = ReadJoinedString(element, "method_of_exploitation"); - bool? userInteraction = element.TryGetProperty("user_interaction", out var uiElement) ? uiElement.ValueKind switch - { - JsonValueKind.True => true, - JsonValueKind.False => false, - _ => null, - } : null; - - var (softwareText, softwareHasCpe, softwareEntries) = ParseVulnerableSoftware(element); - - RuNkckiCweDto? cweDto = null; - if (element.TryGetProperty("cwe", out var cweElement)) - { - int? number = null; - if (cweElement.TryGetProperty("cwe_number", out var numberElement)) - { - if (numberElement.ValueKind == JsonValueKind.Number && numberElement.TryGetInt32(out var parsed)) - { - number = parsed; - } - else if (int.TryParse(numberElement.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedInt)) - { - number = parsedInt; - } - } - - var cweDescription = ReadJoinedString(cweElement, "cwe_description") ?? Normalize(cweElement.GetString()); - if (number.HasValue || !string.IsNullOrWhiteSpace(cweDescription)) - { - cweDto = new RuNkckiCweDto(number, cweDescription); - } - } - - double? cvssScore = element.TryGetProperty("cvss", out var cvssElement) && cvssElement.TryGetProperty("cvss_score", out var scoreElement) - ? ParseDouble(scoreElement) - : null; - var cvssVector = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector", out var vectorElement) - ? Normalize(vectorElement.GetString()) - : null; - double? cvssScoreV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_score_v4", out var scoreV4Element) - ? ParseDouble(scoreV4Element) - : null; - var cvssVectorV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector_v4", out var vectorV4Element) - ? Normalize(vectorV4Element.GetString()) - : null; - - var urls = ReadUrls(element); - var tags = ReadStringCollection(element, "tags"); - - return new RuNkckiVulnerabilityDto( - fstecId, - mitreId, - datePublished, - dateUpdated, - cvssRating, - patchAvailable, - description, - cweDto, - productCategories, - mitigation, - softwareText, - softwareHasCpe, - softwareEntries, - cvssScore, - cvssVector, - cvssScoreV4, - cvssVectorV4, - impact, - method, - userInteraction, - urls, - tags); - } - - private static ImmutableArray ReadUrls(JsonElement element) - { - if (!element.TryGetProperty("urls", out var urlsElement)) - { - return ImmutableArray.Empty; - } - - var collected = new List(); - CollectUrls(urlsElement, collected); - if (collected.Count == 0) - { - return ImmutableArray.Empty; - } - - return collected - .Select(Normalize) - .Where(static url => !string.IsNullOrWhiteSpace(url)) - .Select(static url => url!) - .Distinct(OrdinalIgnoreCase) - .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - } - - private static void CollectUrls(JsonElement element, ICollection results) - { - switch (element.ValueKind) - { - case JsonValueKind.String: - var value = element.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - results.Add(value); - } - break; - case JsonValueKind.Array: - foreach (var child in element.EnumerateArray()) - { - CollectUrls(child, results); - } - break; - case JsonValueKind.Object: - if (element.TryGetProperty("url", out var urlProperty)) - { - CollectUrls(urlProperty, results); - } - - if (element.TryGetProperty("href", out var hrefProperty)) - { - CollectUrls(hrefProperty, results); - } - - foreach (var property in element.EnumerateObject()) - { - if (property.NameEquals("value") || property.NameEquals("link")) - { - CollectUrls(property.Value, results); - } - } - break; - } - } - - private static string? ReadJoinedString(JsonElement element, string property) - { - if (!element.TryGetProperty(property, out var target)) - { - return null; - } - - var values = ReadStringCollection(target); - if (!values.IsDefaultOrEmpty) - { - return string.Join("; ", values); - } - - return Normalize(target.ValueKind == JsonValueKind.String ? target.GetString() : target.ToString()); - } - - private static ImmutableArray ReadStringCollection(JsonElement element, string property) - { - if (!element.TryGetProperty(property, out var target)) - { - return ImmutableArray.Empty; - } - - return ReadStringCollection(target); - } - - private static ImmutableArray ReadStringCollection(JsonElement element) - { - var builder = ImmutableArray.CreateBuilder(); - CollectStrings(element, builder); - return Deduplicate(builder); - } - - private static void CollectStrings(JsonElement element, ImmutableArray.Builder builder) - { - switch (element.ValueKind) - { - case JsonValueKind.String: - AddIfPresent(builder, Normalize(element.GetString())); - break; - case JsonValueKind.Number: - AddIfPresent(builder, Normalize(element.ToString())); - break; - case JsonValueKind.True: - builder.Add("true"); - break; - case JsonValueKind.False: - builder.Add("false"); - break; - case JsonValueKind.Array: - foreach (var child in element.EnumerateArray()) - { - CollectStrings(child, builder); - } - break; - case JsonValueKind.Object: - foreach (var property in element.EnumerateObject()) - { - CollectStrings(property.Value, builder); - } - break; - } - } - - private static ImmutableArray Deduplicate(ImmutableArray.Builder builder) - { - if (builder.Count == 0) - { - return ImmutableArray.Empty; - } - - return builder - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Distinct(OrdinalIgnoreCase) - .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - } - - private static void AddIfPresent(ImmutableArray.Builder builder, string? value) - { - if (!string.IsNullOrWhiteSpace(value)) - { - builder.Add(value!); - } - } - - private static (string? Text, bool? HasCpe, ImmutableArray Entries) ParseVulnerableSoftware(JsonElement element) - { - if (!element.TryGetProperty("vulnerable_software", out var softwareElement)) - { - return (null, null, ImmutableArray.Empty); - } - - string? softwareText = null; - if (softwareElement.TryGetProperty("software_text", out var textElement)) - { - softwareText = Normalize(textElement.ValueKind == JsonValueKind.String ? textElement.GetString() : textElement.ToString()); - } - - bool? softwareHasCpe = null; - if (softwareElement.TryGetProperty("cpe", out var cpeElement)) - { - softwareHasCpe = cpeElement.ValueKind switch - { - JsonValueKind.True => true, - JsonValueKind.False => false, - _ => softwareHasCpe, - }; - } - - var entries = new List(); - if (softwareElement.TryGetProperty("software", out var softwareNodes)) - { - entries.AddRange(ParseSoftwareEntries(softwareNodes)); - } - - if (entries.Count == 0 && !string.IsNullOrWhiteSpace(softwareText)) - { - entries.AddRange(SplitSoftwareTextIntoEntries(softwareText)); - } - - if (entries.Count == 0) - { - foreach (var fallbackProperty in new[] { "items", "aliases", "software_lines" }) - { - if (softwareElement.TryGetProperty(fallbackProperty, out var fallbackNodes)) - { - entries.AddRange(ParseSoftwareEntries(fallbackNodes)); - } - } - } - - if (entries.Count == 0) - { - return (softwareText, softwareHasCpe, ImmutableArray.Empty); - } - - var grouped = entries - .GroupBy(static entry => entry.Identifier, OrdinalIgnoreCase) - .Select(static group => - { - var evidence = string.Join( - "; ", - group.Select(static entry => entry.Evidence) - .Where(static evidence => !string.IsNullOrWhiteSpace(evidence)) - .Distinct(OrdinalIgnoreCase)); - - var ranges = group - .SelectMany(static entry => entry.RangeExpressions) - .Where(static range => !string.IsNullOrWhiteSpace(range)) - .Distinct(OrdinalIgnoreCase) - .OrderBy(static range => range, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - return new RuNkckiSoftwareEntry( - group.Key, - string.IsNullOrWhiteSpace(evidence) ? group.Key : evidence, - ranges); - }) - .OrderBy(static entry => entry.Identifier, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - return (softwareText, softwareHasCpe, grouped); - } - - private static IEnumerable ParseSoftwareEntries(JsonElement element) - { - switch (element.ValueKind) - { - case JsonValueKind.Array: - foreach (var child in element.EnumerateArray()) - { - foreach (var entry in ParseSoftwareEntries(child)) - { - yield return entry; - } - } - break; - case JsonValueKind.Object: - yield return CreateEntryFromObject(element); - break; - case JsonValueKind.String: - foreach (var entry in SplitSoftwareTextIntoEntries(element.GetString() ?? string.Empty)) - { - yield return entry; - } - break; - } - } - - private static RuNkckiSoftwareEntry CreateEntryFromObject(JsonElement element) - { - var vendor = ReadFirstString(element, "vendor", "manufacturer", "organisation"); - var name = ReadFirstString(element, "name", "product", "title"); - var rawVersion = ReadFirstString(element, "version", "versions", "range"); - var comment = ReadFirstString(element, "comment", "notes", "summary"); - - var identifierParts = new List(); - if (!string.IsNullOrWhiteSpace(vendor)) - { - identifierParts.Add(vendor!); - } - - if (!string.IsNullOrWhiteSpace(name)) - { - identifierParts.Add(name!); - } - - var identifier = identifierParts.Count > 0 - ? string.Join(" ", identifierParts) - : ReadFirstString(element, "identifier") ?? name ?? rawVersion ?? comment ?? "unknown"; - - var evidenceParts = new List(identifierParts); - if (!string.IsNullOrWhiteSpace(rawVersion)) - { - evidenceParts.Add(rawVersion!); - } - - if (!string.IsNullOrWhiteSpace(comment)) - { - evidenceParts.Add(comment!); - } - - var evidence = string.Join(" ", evidenceParts.Where(static part => !string.IsNullOrWhiteSpace(part))).Trim(); - - var rangeHints = new List(); - if (!string.IsNullOrWhiteSpace(rawVersion)) - { - rangeHints.Add(rawVersion); - } - - if (element.TryGetProperty("range", out var rangeElement)) - { - rangeHints.Add(Normalize(rangeElement.ToString())); - } - - return CreateSoftwareEntry(identifier!, evidence, rangeHints); - } - - private static IEnumerable SplitSoftwareTextIntoEntries(string text) - { - if (string.IsNullOrWhiteSpace(text)) - { - yield break; - } - - var segments = text.Split(SoftwareSplitDelimiters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (segments.Length == 0) - { - segments = new[] { text }; - } - - foreach (var segment in segments) - { - var normalized = Normalize(segment); - if (string.IsNullOrWhiteSpace(normalized)) - { - continue; - } - - var (identifier, hints) = ExtractIdentifierAndRangeHints(normalized!); - yield return CreateSoftwareEntry(identifier, normalized!, hints); - } - } - - private static RuNkckiSoftwareEntry CreateSoftwareEntry(string identifier, string evidence, IEnumerable hints) - { - var normalizedIdentifier = Normalize(identifier) ?? "unknown"; - var normalizedEvidence = Normalize(evidence) ?? normalizedIdentifier; - - var ranges = hints - .Select(NormalizeRangeHint) - .Where(static hint => !string.IsNullOrWhiteSpace(hint)) - .Select(static hint => hint!) - .Distinct(OrdinalIgnoreCase) - .OrderBy(static hint => hint, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - return new RuNkckiSoftwareEntry(normalizedIdentifier, normalizedEvidence!, ranges); - } - - private static string? NormalizeRangeHint(string? hint) - { - if (string.IsNullOrWhiteSpace(hint)) - { - return null; - } - - var normalized = Normalize(hint)? - .Replace("≤", "<=", StringComparison.Ordinal) - .Replace("≥", ">=", StringComparison.Ordinal) - .Replace("=>", ">=", StringComparison.Ordinal) - .Replace("=<", "<=", StringComparison.Ordinal); - - if (string.IsNullOrWhiteSpace(normalized)) - { - return null; - } - - return normalized; - } - - private static (string Identifier, IReadOnlyList RangeHints) ExtractIdentifierAndRangeHints(string value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return ("unknown", Array.Empty()); - } - - var comparatorMatch = ComparatorRegex.Match(value); - if (comparatorMatch.Success) - { - var name = Normalize(comparatorMatch.Groups["name"].Value); - var version = Normalize(comparatorMatch.Groups["version"].Value); - var op = comparatorMatch.Groups["operator"].Value; - return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $"{op} {version}" }); - } - - var rangeMatch = RangeRegex.Match(value); - if (rangeMatch.Success) - { - var name = Normalize(rangeMatch.Groups["name"].Value); - var start = Normalize(rangeMatch.Groups["start"].Value); - var end = Normalize(rangeMatch.Groups["end"].Value); - return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" }); - } - - var qualifierMatch = QualifierRegex.Match(value); - if (qualifierMatch.Success) - { - var name = Normalize(qualifierMatch.Groups["name"].Value); - var version = Normalize(qualifierMatch.Groups["version"].Value); - var qualifier = qualifierMatch.Groups["qualifier"].Value.ToLowerInvariant(); - var hint = qualifier.Contains("ниж") || qualifier.Contains("earlier") || qualifier.Contains("включ") - ? $"<= {version}" - : $">= {version}"; - return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint }); - } - - var inlineQualifierMatch = QualifierInlineRegex.Match(value); - if (inlineQualifierMatch.Success) - { - var version = Normalize(inlineQualifierMatch.Groups["version"].Value); - var qualifier = inlineQualifierMatch.Groups["qualifier"].Value.ToLowerInvariant(); - var hint = qualifier.Contains("ниж") ? $"<= {version}" : $">= {version}"; - var name = Normalize(QualifierInlineRegex.Replace(value, string.Empty)); - return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint }); - } - - var windowMatch = VersionWindowRegex.Match(value); - if (windowMatch.Success) - { - var start = Normalize(windowMatch.Groups["start"].Value); - var end = Normalize(windowMatch.Groups["end"].Value); - var name = Normalize(VersionWindowRegex.Replace(value, string.Empty)); - return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" }); - } - - return (value, Array.Empty()); - } - - private static string? ReadFirstString(JsonElement element, params string[] names) - { - foreach (var name in names) - { - if (element.TryGetProperty(name, out var property)) - { - switch (property.ValueKind) - { - case JsonValueKind.String: - { - var normalized = Normalize(property.GetString()); - if (!string.IsNullOrWhiteSpace(normalized)) - { - return normalized; - } - - break; - } - case JsonValueKind.Number: - { - var normalized = Normalize(property.ToString()); - if (!string.IsNullOrWhiteSpace(normalized)) - { - return normalized; - } - - break; - } - } - } - } - - return null; - } - - private static double? ParseDouble(JsonElement element) - { - if (element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value)) - { - return value; - } - - if (element.ValueKind == JsonValueKind.String && double.TryParse(element.GetString(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed)) - { - return parsed; - } - - return null; - } - - private static DateTimeOffset? ParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) - { - return parsed; - } - - if (DateTimeOffset.TryParse(value, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var ruParsed)) - { - return ruParsed; - } - - return null; - } - - private static string? Normalize(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - var normalized = value - .Replace('\r', ' ') - .Replace('\n', ' ') - .Trim(); - - while (normalized.Contains(" ", StringComparison.Ordinal)) - { - normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); - } - - return normalized.Length == 0 ? null : normalized; - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal; + +internal static class RuNkckiJsonParser +{ + private static readonly Regex ComparatorRegex = new( + @"^(?.+?)\s*(?<=|>=|<|>|==|=)\s*(?.+?)$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex RangeRegex = new( + @"^(?.+?)\s+(?[\p{L}\p{N}\._-]+)\s*[-–]\s*(?[\p{L}\p{N}\._-]+)$", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static readonly Regex QualifierRegex = new( + @"^(?.+?)\s+(?[\p{L}\p{N}\._-]+)\s+(?(and\s+earlier|and\s+later|and\s+newer|до\s+и\s+включительно|и\s+ниже|и\s+выше|и\s+старше|и\s+позже))$", + RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + private static readonly Regex QualifierInlineRegex = new( + @"верс(ии|ия)\s+(?[\p{L}\p{N}\._-]+)\s+(?и\s+ниже|и\s+выше|и\s+старше)", + RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + private static readonly Regex VersionWindowRegex = new( + @"верс(ии|ия)\s+(?[\p{L}\p{N}\._-]+)\s+по\s+(?[\p{L}\p{N}\._-]+)", + RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + + private static readonly char[] SoftwareSplitDelimiters = { '\n', ';', '\u2022', '\u2023', '\r' }; + + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + public static RuNkckiVulnerabilityDto Parse(JsonElement element) + { + var fstecId = element.TryGetProperty("vuln_id", out var vulnIdElement) && vulnIdElement.TryGetProperty("FSTEC", out var fstec) + ? Normalize(fstec.GetString()) + : null; + var mitreId = element.TryGetProperty("vuln_id", out vulnIdElement) && vulnIdElement.TryGetProperty("MITRE", out var mitre) + ? Normalize(mitre.GetString()) + : null; + + var datePublished = ParseDate(element.TryGetProperty("date_published", out var published) ? published.GetString() : null); + var dateUpdated = ParseDate(element.TryGetProperty("date_updated", out var updated) ? updated.GetString() : null); + var cvssRating = Normalize(element.TryGetProperty("cvss_rating", out var rating) ? rating.GetString() : null); + bool? patchAvailable = element.TryGetProperty("patch_available", out var patch) ? patch.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => null, + } : null; + + var description = ReadJoinedString(element, "description"); + var mitigation = ReadJoinedString(element, "mitigation"); + var productCategories = ReadStringCollection(element, "product_category"); + var impact = ReadJoinedString(element, "impact"); + var method = ReadJoinedString(element, "method_of_exploitation"); + bool? userInteraction = element.TryGetProperty("user_interaction", out var uiElement) ? uiElement.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => null, + } : null; + + var (softwareText, softwareHasCpe, softwareEntries) = ParseVulnerableSoftware(element); + + RuNkckiCweDto? cweDto = null; + if (element.TryGetProperty("cwe", out var cweElement)) + { + int? number = null; + if (cweElement.TryGetProperty("cwe_number", out var numberElement)) + { + if (numberElement.ValueKind == JsonValueKind.Number && numberElement.TryGetInt32(out var parsed)) + { + number = parsed; + } + else if (int.TryParse(numberElement.GetString(), NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedInt)) + { + number = parsedInt; + } + } + + var cweDescription = ReadJoinedString(cweElement, "cwe_description") ?? Normalize(cweElement.GetString()); + if (number.HasValue || !string.IsNullOrWhiteSpace(cweDescription)) + { + cweDto = new RuNkckiCweDto(number, cweDescription); + } + } + + double? cvssScore = element.TryGetProperty("cvss", out var cvssElement) && cvssElement.TryGetProperty("cvss_score", out var scoreElement) + ? ParseDouble(scoreElement) + : null; + var cvssVector = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector", out var vectorElement) + ? Normalize(vectorElement.GetString()) + : null; + double? cvssScoreV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_score_v4", out var scoreV4Element) + ? ParseDouble(scoreV4Element) + : null; + var cvssVectorV4 = element.TryGetProperty("cvss", out cvssElement) && cvssElement.TryGetProperty("cvss_vector_v4", out var vectorV4Element) + ? Normalize(vectorV4Element.GetString()) + : null; + + var urls = ReadUrls(element); + var tags = ReadStringCollection(element, "tags"); + + return new RuNkckiVulnerabilityDto( + fstecId, + mitreId, + datePublished, + dateUpdated, + cvssRating, + patchAvailable, + description, + cweDto, + productCategories, + mitigation, + softwareText, + softwareHasCpe, + softwareEntries, + cvssScore, + cvssVector, + cvssScoreV4, + cvssVectorV4, + impact, + method, + userInteraction, + urls, + tags); + } + + private static ImmutableArray ReadUrls(JsonElement element) + { + if (!element.TryGetProperty("urls", out var urlsElement)) + { + return ImmutableArray.Empty; + } + + var collected = new List(); + CollectUrls(urlsElement, collected); + if (collected.Count == 0) + { + return ImmutableArray.Empty; + } + + return collected + .Select(Normalize) + .Where(static url => !string.IsNullOrWhiteSpace(url)) + .Select(static url => url!) + .Distinct(OrdinalIgnoreCase) + .OrderBy(static url => url, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static void CollectUrls(JsonElement element, ICollection results) + { + switch (element.ValueKind) + { + case JsonValueKind.String: + var value = element.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + results.Add(value); + } + break; + case JsonValueKind.Array: + foreach (var child in element.EnumerateArray()) + { + CollectUrls(child, results); + } + break; + case JsonValueKind.Object: + if (element.TryGetProperty("url", out var urlProperty)) + { + CollectUrls(urlProperty, results); + } + + if (element.TryGetProperty("href", out var hrefProperty)) + { + CollectUrls(hrefProperty, results); + } + + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("value") || property.NameEquals("link")) + { + CollectUrls(property.Value, results); + } + } + break; + } + } + + private static string? ReadJoinedString(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var target)) + { + return null; + } + + var values = ReadStringCollection(target); + if (!values.IsDefaultOrEmpty) + { + return string.Join("; ", values); + } + + return Normalize(target.ValueKind == JsonValueKind.String ? target.GetString() : target.ToString()); + } + + private static ImmutableArray ReadStringCollection(JsonElement element, string property) + { + if (!element.TryGetProperty(property, out var target)) + { + return ImmutableArray.Empty; + } + + return ReadStringCollection(target); + } + + private static ImmutableArray ReadStringCollection(JsonElement element) + { + var builder = ImmutableArray.CreateBuilder(); + CollectStrings(element, builder); + return Deduplicate(builder); + } + + private static void CollectStrings(JsonElement element, ImmutableArray.Builder builder) + { + switch (element.ValueKind) + { + case JsonValueKind.String: + AddIfPresent(builder, Normalize(element.GetString())); + break; + case JsonValueKind.Number: + AddIfPresent(builder, Normalize(element.ToString())); + break; + case JsonValueKind.True: + builder.Add("true"); + break; + case JsonValueKind.False: + builder.Add("false"); + break; + case JsonValueKind.Array: + foreach (var child in element.EnumerateArray()) + { + CollectStrings(child, builder); + } + break; + case JsonValueKind.Object: + foreach (var property in element.EnumerateObject()) + { + CollectStrings(property.Value, builder); + } + break; + } + } + + private static ImmutableArray Deduplicate(ImmutableArray.Builder builder) + { + if (builder.Count == 0) + { + return ImmutableArray.Empty; + } + + return builder + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Distinct(OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static void AddIfPresent(ImmutableArray.Builder builder, string? value) + { + if (!string.IsNullOrWhiteSpace(value)) + { + builder.Add(value!); + } + } + + private static (string? Text, bool? HasCpe, ImmutableArray Entries) ParseVulnerableSoftware(JsonElement element) + { + if (!element.TryGetProperty("vulnerable_software", out var softwareElement)) + { + return (null, null, ImmutableArray.Empty); + } + + string? softwareText = null; + if (softwareElement.TryGetProperty("software_text", out var textElement)) + { + softwareText = Normalize(textElement.ValueKind == JsonValueKind.String ? textElement.GetString() : textElement.ToString()); + } + + bool? softwareHasCpe = null; + if (softwareElement.TryGetProperty("cpe", out var cpeElement)) + { + softwareHasCpe = cpeElement.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + _ => softwareHasCpe, + }; + } + + var entries = new List(); + if (softwareElement.TryGetProperty("software", out var softwareNodes)) + { + entries.AddRange(ParseSoftwareEntries(softwareNodes)); + } + + if (entries.Count == 0 && !string.IsNullOrWhiteSpace(softwareText)) + { + entries.AddRange(SplitSoftwareTextIntoEntries(softwareText)); + } + + if (entries.Count == 0) + { + foreach (var fallbackProperty in new[] { "items", "aliases", "software_lines" }) + { + if (softwareElement.TryGetProperty(fallbackProperty, out var fallbackNodes)) + { + entries.AddRange(ParseSoftwareEntries(fallbackNodes)); + } + } + } + + if (entries.Count == 0) + { + return (softwareText, softwareHasCpe, ImmutableArray.Empty); + } + + var grouped = entries + .GroupBy(static entry => entry.Identifier, OrdinalIgnoreCase) + .Select(static group => + { + var evidence = string.Join( + "; ", + group.Select(static entry => entry.Evidence) + .Where(static evidence => !string.IsNullOrWhiteSpace(evidence)) + .Distinct(OrdinalIgnoreCase)); + + var ranges = group + .SelectMany(static entry => entry.RangeExpressions) + .Where(static range => !string.IsNullOrWhiteSpace(range)) + .Distinct(OrdinalIgnoreCase) + .OrderBy(static range => range, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return new RuNkckiSoftwareEntry( + group.Key, + string.IsNullOrWhiteSpace(evidence) ? group.Key : evidence, + ranges); + }) + .OrderBy(static entry => entry.Identifier, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return (softwareText, softwareHasCpe, grouped); + } + + private static IEnumerable ParseSoftwareEntries(JsonElement element) + { + switch (element.ValueKind) + { + case JsonValueKind.Array: + foreach (var child in element.EnumerateArray()) + { + foreach (var entry in ParseSoftwareEntries(child)) + { + yield return entry; + } + } + break; + case JsonValueKind.Object: + yield return CreateEntryFromObject(element); + break; + case JsonValueKind.String: + foreach (var entry in SplitSoftwareTextIntoEntries(element.GetString() ?? string.Empty)) + { + yield return entry; + } + break; + } + } + + private static RuNkckiSoftwareEntry CreateEntryFromObject(JsonElement element) + { + var vendor = ReadFirstString(element, "vendor", "manufacturer", "organisation"); + var name = ReadFirstString(element, "name", "product", "title"); + var rawVersion = ReadFirstString(element, "version", "versions", "range"); + var comment = ReadFirstString(element, "comment", "notes", "summary"); + + var identifierParts = new List(); + if (!string.IsNullOrWhiteSpace(vendor)) + { + identifierParts.Add(vendor!); + } + + if (!string.IsNullOrWhiteSpace(name)) + { + identifierParts.Add(name!); + } + + var identifier = identifierParts.Count > 0 + ? string.Join(" ", identifierParts) + : ReadFirstString(element, "identifier") ?? name ?? rawVersion ?? comment ?? "unknown"; + + var evidenceParts = new List(identifierParts); + if (!string.IsNullOrWhiteSpace(rawVersion)) + { + evidenceParts.Add(rawVersion!); + } + + if (!string.IsNullOrWhiteSpace(comment)) + { + evidenceParts.Add(comment!); + } + + var evidence = string.Join(" ", evidenceParts.Where(static part => !string.IsNullOrWhiteSpace(part))).Trim(); + + var rangeHints = new List(); + if (!string.IsNullOrWhiteSpace(rawVersion)) + { + rangeHints.Add(rawVersion); + } + + if (element.TryGetProperty("range", out var rangeElement)) + { + rangeHints.Add(Normalize(rangeElement.ToString())); + } + + return CreateSoftwareEntry(identifier!, evidence, rangeHints); + } + + private static IEnumerable SplitSoftwareTextIntoEntries(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + yield break; + } + + var segments = text.Split(SoftwareSplitDelimiters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length == 0) + { + segments = new[] { text }; + } + + foreach (var segment in segments) + { + var normalized = Normalize(segment); + if (string.IsNullOrWhiteSpace(normalized)) + { + continue; + } + + var (identifier, hints) = ExtractIdentifierAndRangeHints(normalized!); + yield return CreateSoftwareEntry(identifier, normalized!, hints); + } + } + + private static RuNkckiSoftwareEntry CreateSoftwareEntry(string identifier, string evidence, IEnumerable hints) + { + var normalizedIdentifier = Normalize(identifier) ?? "unknown"; + var normalizedEvidence = Normalize(evidence) ?? normalizedIdentifier; + + var ranges = hints + .Select(NormalizeRangeHint) + .Where(static hint => !string.IsNullOrWhiteSpace(hint)) + .Select(static hint => hint!) + .Distinct(OrdinalIgnoreCase) + .OrderBy(static hint => hint, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return new RuNkckiSoftwareEntry(normalizedIdentifier, normalizedEvidence!, ranges); + } + + private static string? NormalizeRangeHint(string? hint) + { + if (string.IsNullOrWhiteSpace(hint)) + { + return null; + } + + var normalized = Normalize(hint)? + .Replace("≤", "<=", StringComparison.Ordinal) + .Replace("≥", ">=", StringComparison.Ordinal) + .Replace("=>", ">=", StringComparison.Ordinal) + .Replace("=<", "<=", StringComparison.Ordinal); + + if (string.IsNullOrWhiteSpace(normalized)) + { + return null; + } + + return normalized; + } + + private static (string Identifier, IReadOnlyList RangeHints) ExtractIdentifierAndRangeHints(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ("unknown", Array.Empty()); + } + + var comparatorMatch = ComparatorRegex.Match(value); + if (comparatorMatch.Success) + { + var name = Normalize(comparatorMatch.Groups["name"].Value); + var version = Normalize(comparatorMatch.Groups["version"].Value); + var op = comparatorMatch.Groups["operator"].Value; + return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $"{op} {version}" }); + } + + var rangeMatch = RangeRegex.Match(value); + if (rangeMatch.Success) + { + var name = Normalize(rangeMatch.Groups["name"].Value); + var start = Normalize(rangeMatch.Groups["start"].Value); + var end = Normalize(rangeMatch.Groups["end"].Value); + return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" }); + } + + var qualifierMatch = QualifierRegex.Match(value); + if (qualifierMatch.Success) + { + var name = Normalize(qualifierMatch.Groups["name"].Value); + var version = Normalize(qualifierMatch.Groups["version"].Value); + var qualifier = qualifierMatch.Groups["qualifier"].Value.ToLowerInvariant(); + var hint = qualifier.Contains("ниж") || qualifier.Contains("earlier") || qualifier.Contains("включ") + ? $"<= {version}" + : $">= {version}"; + return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint }); + } + + var inlineQualifierMatch = QualifierInlineRegex.Match(value); + if (inlineQualifierMatch.Success) + { + var version = Normalize(inlineQualifierMatch.Groups["version"].Value); + var qualifier = inlineQualifierMatch.Groups["qualifier"].Value.ToLowerInvariant(); + var hint = qualifier.Contains("ниж") ? $"<= {version}" : $">= {version}"; + var name = Normalize(QualifierInlineRegex.Replace(value, string.Empty)); + return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { hint }); + } + + var windowMatch = VersionWindowRegex.Match(value); + if (windowMatch.Success) + { + var start = Normalize(windowMatch.Groups["start"].Value); + var end = Normalize(windowMatch.Groups["end"].Value); + var name = Normalize(VersionWindowRegex.Replace(value, string.Empty)); + return (string.IsNullOrWhiteSpace(name) ? value : name!, new[] { $">= {start}", $"<= {end}" }); + } + + return (value, Array.Empty()); + } + + private static string? ReadFirstString(JsonElement element, params string[] names) + { + foreach (var name in names) + { + if (element.TryGetProperty(name, out var property)) + { + switch (property.ValueKind) + { + case JsonValueKind.String: + { + var normalized = Normalize(property.GetString()); + if (!string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + break; + } + case JsonValueKind.Number: + { + var normalized = Normalize(property.ToString()); + if (!string.IsNullOrWhiteSpace(normalized)) + { + return normalized; + } + + break; + } + } + } + } + + return null; + } + + private static double? ParseDouble(JsonElement element) + { + if (element.ValueKind == JsonValueKind.Number && element.TryGetDouble(out var value)) + { + return value; + } + + if (element.ValueKind == JsonValueKind.String && double.TryParse(element.GetString(), NumberStyles.Any, CultureInfo.InvariantCulture, out var parsed)) + { + return parsed; + } + + return null; + } + + private static DateTimeOffset? ParseDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.GetCultureInfo("ru-RU"), DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var ruParsed)) + { + return ruParsed; + } + + return null; + } + + private static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var normalized = value + .Replace('\r', ' ') + .Replace('\n', ' ') + .Trim(); + + while (normalized.Contains(" ", StringComparison.Ordinal)) + { + normalized = normalized.Replace(" ", " ", StringComparison.Ordinal); + } + + return normalized.Length == 0 ? null : normalized; + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiMapper.cs similarity index 98% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiMapper.cs index 51a57b85..a320ac79 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiMapper.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiMapper.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Globalization; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; -using StellaOps.Feedser.Normalization.SemVer; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; +using StellaOps.Concelier.Normalization.SemVer; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal; internal static class RuNkckiMapper { diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs index a4e006f1..6f563752 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Internal/RuNkckiVulnerabilityDto.cs @@ -1,40 +1,40 @@ -using System.Collections.Immutable; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Ru.Nkcki.Internal; - -internal sealed record RuNkckiVulnerabilityDto( - string? FstecId, - string? MitreId, - DateTimeOffset? DatePublished, - DateTimeOffset? DateUpdated, - string? CvssRating, - bool? PatchAvailable, - string? Description, - RuNkckiCweDto? Cwe, - ImmutableArray ProductCategories, - string? Mitigation, - string? VulnerableSoftwareText, - bool? VulnerableSoftwareHasCpe, - ImmutableArray VulnerableSoftwareEntries, - double? CvssScore, - string? CvssVector, - double? CvssScoreV4, - string? CvssVectorV4, - string? Impact, - string? MethodOfExploitation, - bool? UserInteraction, - ImmutableArray Urls, - ImmutableArray Tags) -{ - [JsonIgnore] - public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId) - ? FstecId! - : !string.IsNullOrWhiteSpace(MitreId) - ? MitreId! - : Guid.NewGuid().ToString(); -} - -internal sealed record RuNkckiCweDto(int? Number, string? Description); - -internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray RangeExpressions); +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki.Internal; + +internal sealed record RuNkckiVulnerabilityDto( + string? FstecId, + string? MitreId, + DateTimeOffset? DatePublished, + DateTimeOffset? DateUpdated, + string? CvssRating, + bool? PatchAvailable, + string? Description, + RuNkckiCweDto? Cwe, + ImmutableArray ProductCategories, + string? Mitigation, + string? VulnerableSoftwareText, + bool? VulnerableSoftwareHasCpe, + ImmutableArray VulnerableSoftwareEntries, + double? CvssScore, + string? CvssVector, + double? CvssScoreV4, + string? CvssVectorV4, + string? Impact, + string? MethodOfExploitation, + bool? UserInteraction, + ImmutableArray Urls, + ImmutableArray Tags) +{ + [JsonIgnore] + public string AdvisoryKey => !string.IsNullOrWhiteSpace(FstecId) + ? FstecId! + : !string.IsNullOrWhiteSpace(MitreId) + ? MitreId! + : Guid.NewGuid().ToString(); +} + +internal sealed record RuNkckiCweDto(int? Number, string? Description); + +internal sealed record RuNkckiSoftwareEntry(string Identifier, string Evidence, ImmutableArray RangeExpressions); diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Jobs.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/Jobs.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs index 283c711a..7e3bcc31 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Jobs.cs @@ -1,43 +1,43 @@ -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Ru.Nkcki; - -internal static class RuNkckiJobKinds -{ - public const string Fetch = "source:ru-nkcki:fetch"; - public const string Parse = "source:ru-nkcki:parse"; - public const string Map = "source:ru-nkcki:map"; -} - -internal sealed class RuNkckiFetchJob : IJob -{ - private readonly RuNkckiConnector _connector; - - public RuNkckiFetchJob(RuNkckiConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class RuNkckiParseJob : IJob -{ - private readonly RuNkckiConnector _connector; - - public RuNkckiParseJob(RuNkckiConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class RuNkckiMapJob : IJob -{ - private readonly RuNkckiConnector _connector; - - public RuNkckiMapJob(RuNkckiConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki; + +internal static class RuNkckiJobKinds +{ + public const string Fetch = "source:ru-nkcki:fetch"; + public const string Parse = "source:ru-nkcki:parse"; + public const string Map = "source:ru-nkcki:map"; +} + +internal sealed class RuNkckiFetchJob : IJob +{ + private readonly RuNkckiConnector _connector; + + public RuNkckiFetchJob(RuNkckiConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class RuNkckiParseJob : IJob +{ + private readonly RuNkckiConnector _connector; + + public RuNkckiParseJob(RuNkckiConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class RuNkckiMapJob : IJob +{ + private readonly RuNkckiConnector _connector; + + public RuNkckiMapJob(RuNkckiConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.Ru.Nkcki/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..becbbcf6 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Ru.Nkcki.Tests")] diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnector.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnector.cs index 21be40c6..4607db0c 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnector.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnector.cs @@ -1,946 +1,946 @@ -using System.Collections.Immutable; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Net; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using AngleSharp.Html.Parser; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; -using StellaOps.Feedser.Source.Ru.Nkcki.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ru.Nkcki; - -public sealed class RuNkckiConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = false, - }; - - private static readonly string[] ListingAcceptHeaders = - { - "text/html", - "application/xhtml+xml;q=0.9", - "text/plain;q=0.1", - }; - - private static readonly string[] BulletinAcceptHeaders = - { - "application/zip", - "application/octet-stream", - "application/x-zip-compressed", - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly ISourceStateRepository _stateRepository; - private readonly RuNkckiOptions _options; - private readonly TimeProvider _timeProvider; - private readonly RuNkckiDiagnostics _diagnostics; - private readonly ILogger _logger; - private readonly string _cacheDirectory; - - private readonly HtmlParser _htmlParser = new(); - - public RuNkckiConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - ISourceStateRepository stateRepository, - IOptions options, - RuNkckiDiagnostics diagnostics, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory); - EnsureCacheDirectory(); - } - - public string SourceName => RuNkckiConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var knownBulletins = cursor.KnownBulletins.ToHashSet(StringComparer.OrdinalIgnoreCase); - var now = _timeProvider.GetUtcNow(); - var processed = 0; - - if (ShouldUseListingCache(cursor, now)) - { - _logger.LogDebug( - "NKCKI listing fetch skipped (cache duration {CacheDuration:c}); processing cached bulletins only", - _options.ListingCacheDuration); - - processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); - await UpdateCursorAsync(cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownBulletins(NormalizeBulletins(knownBulletins)) - .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false); - return; - } - - ListingFetchSummary listingSummary; - try - { - listingSummary = await LoadListingAsync(cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - _logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins"); - _diagnostics.ListingFetchFailure(ex.Message); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - - processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); - await UpdateCursorAsync(cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownBulletins(NormalizeBulletins(knownBulletins)) - .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false); - return; - } - - var uniqueAttachments = listingSummary.Attachments - .GroupBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase) - .Select(static group => group.First()) - .OrderBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase) - .ToList(); - - var newAttachments = uniqueAttachments - .Where(attachment => !knownBulletins.Contains(attachment.Id)) - .Take(_options.MaxBulletinsPerFetch) - .ToList(); - - _diagnostics.ListingFetchSuccess(listingSummary.PagesVisited, uniqueAttachments.Count, newAttachments.Count); - - if (newAttachments.Count == 0) - { - _logger.LogDebug("NKCKI listing contained no new bulletin attachments"); - processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); - await UpdateCursorAsync(cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownBulletins(NormalizeBulletins(knownBulletins)) - .WithLastListingFetch(now), cancellationToken).ConfigureAwait(false); - return; - } - - var downloaded = 0; - var cachedUsed = 0; - var failures = 0; - - foreach (var attachment in newAttachments) - { - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, attachment.Uri) - { - AcceptHeaders = BulletinAcceptHeaders, - TimeoutOverride = _options.RequestTimeout, - }; - - var attachmentResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - if (!attachmentResult.IsSuccess || attachmentResult.Content is null) - { - if (TryReadCachedBulletin(attachment.Id, out var cachedBytes)) - { - _diagnostics.BulletinFetchCached(); - cachedUsed++; - _logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode); - processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); - knownBulletins.Add(attachment.Id); - } - else - { - _diagnostics.BulletinFetchFailure(attachmentResult.StatusCode.ToString()); - failures++; - _logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode); - } - - continue; - } - - _diagnostics.BulletinFetchSuccess(); - downloaded++; - TryWriteCachedBulletin(attachment.Id, attachmentResult.Content); - processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); - knownBulletins.Add(attachment.Id); - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) - { - if (TryReadCachedBulletin(attachment.Id, out var cachedBytes)) - { - _diagnostics.BulletinFetchCached(); - cachedUsed++; - _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id); - processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); - knownBulletins.Add(attachment.Id); - } - else - { - _diagnostics.BulletinFetchFailure(ex.Message); - failures++; - _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id); - await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - } - - if (processed >= _options.MaxVulnerabilitiesPerFetch) - { - break; - } - - if (_options.RequestDelay > TimeSpan.Zero) - { - try - { - await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); - } - catch (TaskCanceledException) - { - break; - } - } - } - - if (processed < _options.MaxVulnerabilitiesPerFetch) - { - processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); - } - - var normalizedBulletins = NormalizeBulletins(knownBulletins); - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithKnownBulletins(normalizedBulletins) - .WithLastListingFetch(now); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation( - "NKCKI fetch complete: new bulletins {Downloaded}, cached bulletins {Cached}, failures {Failures}, processed entries {Processed}, pending documents {PendingDocuments}, pending mappings {PendingMappings}", - downloaded, - cachedUsed, - failures, - processed, - pendingDocuments.Count, - pendingMappings.Count); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var pendingDocuments = cursor.PendingDocuments.ToList(); - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _logger.LogWarning("NKCKI document {DocumentId} missing GridFS payload", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - byte[] payload; - try - { - payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogError(ex, "NKCKI unable to download raw document {DocumentId}", documentId); - throw; - } - - RuNkckiVulnerabilityDto? dto; - try - { - dto = JsonSerializer.Deserialize(payload, SerializerOptions); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "NKCKI failed to deserialize document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (dto is null) - { - _logger.LogWarning("NKCKI document {DocumentId} produced null DTO", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow()); - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - pendingDocuments.Remove(documentId); - if (!pendingMappings.Contains(documentId)) - { - pendingMappings.Add(documentId); - } - } - - var updatedCursor = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToList(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingMappings.Remove(documentId); - continue; - } - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - _logger.LogWarning("NKCKI document {DocumentId} missing DTO payload", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - RuNkckiVulnerabilityDto dto; - try - { - dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null"); - } - catch (Exception ex) - { - _logger.LogError(ex, "NKCKI failed to deserialize DTO for document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - continue; - } - - try - { - var advisory = RuNkckiMapper.Map(dto, document, dtoRecord.ValidatedAt); - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - catch (Exception ex) - { - _logger.LogError(ex, "NKCKI mapping failed for document {DocumentId}", documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - pendingMappings.Remove(documentId); - } - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private async Task ProcessCachedBulletinsAsync( - HashSet pendingDocuments, - HashSet pendingMappings, - HashSet knownBulletins, - DateTimeOffset now, - int processed, - CancellationToken cancellationToken) - { - if (!Directory.Exists(_cacheDirectory)) - { - return processed; - } - - var updated = processed; - var cacheFiles = Directory - .EnumerateFiles(_cacheDirectory, "*.json.zip", SearchOption.TopDirectoryOnly) - .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) - .ToList(); - - foreach (var filePath in cacheFiles) - { - cancellationToken.ThrowIfCancellationRequested(); - - var bulletinId = ExtractBulletinIdFromCachePath(filePath); - if (string.IsNullOrWhiteSpace(bulletinId) || knownBulletins.Contains(bulletinId)) - { - continue; - } - - byte[] content; - try - { - content = File.ReadAllBytes(filePath); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "NKCKI failed to read cached bulletin at {CachePath}", filePath); - continue; - } - - _diagnostics.BulletinFetchCached(); - updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false); - knownBulletins.Add(bulletinId); - - if (updated >= _options.MaxVulnerabilitiesPerFetch) - { - break; - } - } - - return updated; - } - - private async Task ProcessBulletinEntriesAsync( - byte[] content, - string bulletinId, - HashSet pendingDocuments, - HashSet pendingMappings, - DateTimeOffset now, - int processed, - CancellationToken cancellationToken) - { - if (content.Length == 0) - { - return processed; - } - - var updated = processed; - using var archiveStream = new MemoryStream(content, writable: false); - using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); - - foreach (var entry in archive.Entries.OrderBy(static e => e.FullName, StringComparer.OrdinalIgnoreCase)) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - using var entryStream = entry.Open(); - using var buffer = new MemoryStream(); - await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); - - if (buffer.Length == 0) - { - continue; - } - - buffer.Position = 0; - - using var document = await JsonDocument.ParseAsync(buffer, cancellationToken: cancellationToken).ConfigureAwait(false); - updated = await ProcessBulletinJsonElementAsync(document.RootElement, entry.FullName, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false); - - if (updated >= _options.MaxVulnerabilitiesPerFetch) - { - break; - } - } - - var delta = updated - processed; - if (delta > 0) - { - _diagnostics.EntriesProcessed(delta); - } - - return updated; - } - - private async Task ProcessBulletinJsonElementAsync( - JsonElement element, - string entryName, - string bulletinId, - HashSet pendingDocuments, - HashSet pendingMappings, - DateTimeOffset now, - int processed, - CancellationToken cancellationToken) - { - var updated = processed; - - switch (element.ValueKind) - { - case JsonValueKind.Array: - foreach (var child in element.EnumerateArray()) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (updated >= _options.MaxVulnerabilitiesPerFetch) - { - break; - } - - if (child.ValueKind != JsonValueKind.Object) - { - continue; - } - - if (await ProcessVulnerabilityObjectAsync(child, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false)) - { - updated++; - } - } - - break; - - case JsonValueKind.Object: - if (await ProcessVulnerabilityObjectAsync(element, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false)) - { - updated++; - } - - break; - } - - return updated; - } - - private async Task ProcessVulnerabilityObjectAsync( - JsonElement element, - string entryName, - string bulletinId, - HashSet pendingDocuments, - HashSet pendingMappings, - DateTimeOffset now, - CancellationToken cancellationToken) - { - RuNkckiVulnerabilityDto dto; - try - { - dto = RuNkckiJsonParser.Parse(element); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "NKCKI failed to parse vulnerability in bulletin {BulletinId} entry {Entry}", bulletinId, entryName); - return false; - } - - var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions); - var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); - var documentUri = BuildDocumentUri(dto); - - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); - if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false); - - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["ru-nkcki.bulletin"] = bulletinId, - ["ru-nkcki.entry"] = entryName, - }; - - if (!string.IsNullOrWhiteSpace(dto.FstecId)) - { - metadata["ru-nkcki.fstec_id"] = dto.FstecId!; - } - - if (!string.IsNullOrWhiteSpace(dto.MitreId)) - { - metadata["ru-nkcki.mitre_id"] = dto.MitreId!; - } - - var recordId = existing?.Id ?? Guid.NewGuid(); - var lastModified = dto.DateUpdated ?? dto.DatePublished; - var record = new DocumentRecord( - recordId, - SourceName, - documentUri, - now, - sha, - DocumentStatuses.PendingParse, - "application/json", - Headers: null, - Metadata: metadata, - Etag: null, - LastModified: lastModified, - GridFsId: gridFsId, - ExpiresAt: null); - - var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); - pendingDocuments.Add(upserted.Id); - pendingMappings.Remove(upserted.Id); - return true; - } - - private Task FetchListingPageAsync(Uri pageUri, CancellationToken cancellationToken) - { - var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, pageUri) - { - AcceptHeaders = ListingAcceptHeaders, - TimeoutOverride = _options.RequestTimeout, - }; - - return _fetchService.FetchContentAsync(request, cancellationToken); - } - - private async Task ParseListingAsync(Uri pageUri, byte[] content, CancellationToken cancellationToken) - { - var html = Encoding.UTF8.GetString(content); - var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false); - var attachments = new List(); - var pagination = new List(); - - foreach (var anchor in document.QuerySelectorAll("a[href$='.json.zip']")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!Uri.TryCreate(pageUri, href, out var absoluteUri)) - { - continue; - } - - var id = DeriveBulletinId(absoluteUri); - if (string.IsNullOrWhiteSpace(id)) - { - continue; - } - - var title = anchor.GetAttribute("title"); - if (string.IsNullOrWhiteSpace(title)) - { - title = anchor.TextContent?.Trim(); - } - - attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id)); - } - - foreach (var anchor in document.QuerySelectorAll("a[href]")) - { - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!href.Contains("PAGEN", StringComparison.OrdinalIgnoreCase) - && !href.Contains("page=", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (Uri.TryCreate(pageUri, href, out var absoluteUri)) - { - pagination.Add(absoluteUri); - } - } - - var uniquePagination = pagination - .DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase) - .Take(_options.MaxListingPagesPerFetch) - .ToList(); - - return new ListingPageResult(attachments, uniquePagination); - } - - private static string DeriveBulletinId(Uri uri) - { - var fileName = Path.GetFileName(uri.AbsolutePath); - if (string.IsNullOrWhiteSpace(fileName)) - { - return Guid.NewGuid().ToString("N"); - } - - if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) - { - fileName = fileName[..^4]; - } - - if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - { - fileName = fileName[..^5]; - } - - return fileName.Replace('_', '-'); - } - - private static string BuildDocumentUri(RuNkckiVulnerabilityDto dto) - { - if (!string.IsNullOrWhiteSpace(dto.FstecId)) - { - var slug = dto.FstecId.Contains(':', StringComparison.Ordinal) - ? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..] - : dto.FstecId; - return $"https://cert.gov.ru/materialy/uyazvimosti/{slug}"; - } - - if (!string.IsNullOrWhiteSpace(dto.MitreId)) - { - return $"https://nvd.nist.gov/vuln/detail/{dto.MitreId}"; - } - - return $"https://cert.gov.ru/materialy/uyazvimosti/{Guid.NewGuid():N}"; - } - - private string ResolveCacheDirectory(string? configuredPath) - { - if (!string.IsNullOrWhiteSpace(configuredPath)) - { - return Path.GetFullPath(Path.IsPathRooted(configuredPath) - ? configuredPath - : Path.Combine(AppContext.BaseDirectory, configuredPath)); - } - - return Path.Combine(AppContext.BaseDirectory, "cache", RuNkckiConnectorPlugin.SourceName); - } - - private void EnsureCacheDirectory() - { - try - { - Directory.CreateDirectory(_cacheDirectory); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "NKCKI unable to ensure cache directory {CachePath}", _cacheDirectory); - } - } - - private string GetBulletinCachePath(string bulletinId) - { - var fileStem = string.IsNullOrWhiteSpace(bulletinId) - ? Guid.NewGuid().ToString("N") - : Uri.EscapeDataString(bulletinId); - return Path.Combine(_cacheDirectory, $"{fileStem}.json.zip"); - } - - private static string ExtractBulletinIdFromCachePath(string path) - { - if (string.IsNullOrWhiteSpace(path)) - { - return string.Empty; - } - - var fileName = Path.GetFileName(path); - if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) - { - fileName = fileName[..^4]; - } - - if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) - { - fileName = fileName[..^5]; - } - - return Uri.UnescapeDataString(fileName); - } - - private void TryWriteCachedBulletin(string bulletinId, byte[] content) - { - try - { - var cachePath = GetBulletinCachePath(bulletinId); - Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!); - File.WriteAllBytes(cachePath, content); - } - catch (Exception ex) - { - _logger.LogDebug(ex, "NKCKI failed to cache bulletin {BulletinId}", bulletinId); - } - } - - private bool TryReadCachedBulletin(string bulletinId, out byte[] content) - { - var cachePath = GetBulletinCachePath(bulletinId); - try - { - if (File.Exists(cachePath)) - { - content = File.ReadAllBytes(cachePath); - return true; - } - } - catch (Exception ex) - { - _logger.LogDebug(ex, "NKCKI failed to read cached bulletin {BulletinId}", bulletinId); - } - - content = Array.Empty(); - return false; - } - - private IReadOnlyCollection NormalizeBulletins(IEnumerable bulletins) - { - var normalized = (bulletins ?? Enumerable.Empty()) - .Where(static id => !string.IsNullOrWhiteSpace(id)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) - .ToList(); - - if (normalized.Count <= _options.KnownBulletinCapacity) - { - return normalized.ToArray(); - } - - var skip = normalized.Count - _options.KnownBulletinCapacity; - return normalized.Skip(skip).ToArray(); - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? RuNkckiCursor.Empty : RuNkckiCursor.FromBson(state.Cursor); - } - - private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBsonDocument(); - var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow(); - return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); - } - - private readonly record struct ListingFetchSummary(IReadOnlyList Attachments, int PagesVisited); - - private readonly record struct ListingPageResult(IReadOnlyList Attachments, IReadOnlyList PaginationLinks); - - private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title); - - private bool ShouldUseListingCache(RuNkckiCursor cursor, DateTimeOffset now) - { - if (!cursor.LastListingFetchAt.HasValue) - { - return false; - } - - var age = now - cursor.LastListingFetchAt.Value; - return age < _options.ListingCacheDuration; - } - - private async Task LoadListingAsync(CancellationToken cancellationToken) - { - var attachments = new List(); - var visited = 0; - var visitedUris = new HashSet(StringComparer.OrdinalIgnoreCase); - var queue = new Queue(); - queue.Enqueue(_options.ListingUri); - - while (queue.Count > 0 && visited < _options.MaxListingPagesPerFetch) - { - cancellationToken.ThrowIfCancellationRequested(); - - var pageUri = queue.Dequeue(); - if (!visitedUris.Add(pageUri.AbsoluteUri)) - { - continue; - } - - _diagnostics.ListingFetchAttempt(); - - var listingResult = await FetchListingPageAsync(pageUri, cancellationToken).ConfigureAwait(false); - if (!listingResult.IsSuccess || listingResult.Content is null) - { - _diagnostics.ListingFetchFailure(listingResult.StatusCode.ToString()); - _logger.LogWarning("NKCKI listing page {ListingUri} returned no content (status={Status})", pageUri, listingResult.StatusCode); - continue; - } - - visited++; - - var page = await ParseListingAsync(pageUri, listingResult.Content, cancellationToken).ConfigureAwait(false); - attachments.AddRange(page.Attachments); - - foreach (var link in page.PaginationLinks) - { - if (!visitedUris.Contains(link.AbsoluteUri) && queue.Count + visitedUris.Count < _options.MaxListingPagesPerFetch) - { - queue.Enqueue(link); - } - } - - if (attachments.Count >= _options.MaxBulletinsPerFetch * 2) - { - break; - } - } - - return new ListingFetchSummary(attachments, visited); - } -} +using System.Collections.Immutable; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using AngleSharp.Html.Parser; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration; +using StellaOps.Concelier.Connector.Ru.Nkcki.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki; + +public sealed class RuNkckiConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }; + + private static readonly string[] ListingAcceptHeaders = + { + "text/html", + "application/xhtml+xml;q=0.9", + "text/plain;q=0.1", + }; + + private static readonly string[] BulletinAcceptHeaders = + { + "application/zip", + "application/octet-stream", + "application/x-zip-compressed", + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly ISourceStateRepository _stateRepository; + private readonly RuNkckiOptions _options; + private readonly TimeProvider _timeProvider; + private readonly RuNkckiDiagnostics _diagnostics; + private readonly ILogger _logger; + private readonly string _cacheDirectory; + + private readonly HtmlParser _htmlParser = new(); + + public RuNkckiConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + ISourceStateRepository stateRepository, + IOptions options, + RuNkckiDiagnostics diagnostics, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _cacheDirectory = ResolveCacheDirectory(_options.CacheDirectory); + EnsureCacheDirectory(); + } + + public string SourceName => RuNkckiConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var knownBulletins = cursor.KnownBulletins.ToHashSet(StringComparer.OrdinalIgnoreCase); + var now = _timeProvider.GetUtcNow(); + var processed = 0; + + if (ShouldUseListingCache(cursor, now)) + { + _logger.LogDebug( + "NKCKI listing fetch skipped (cache duration {CacheDuration:c}); processing cached bulletins only", + _options.ListingCacheDuration); + + processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); + await UpdateCursorAsync(cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownBulletins(NormalizeBulletins(knownBulletins)) + .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false); + return; + } + + ListingFetchSummary listingSummary; + try + { + listingSummary = await LoadListingAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + _logger.LogWarning(ex, "NKCKI listing fetch failed; attempting cached bulletins"); + _diagnostics.ListingFetchFailure(ex.Message); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + + processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); + await UpdateCursorAsync(cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownBulletins(NormalizeBulletins(knownBulletins)) + .WithLastListingFetch(cursor.LastListingFetchAt ?? now), cancellationToken).ConfigureAwait(false); + return; + } + + var uniqueAttachments = listingSummary.Attachments + .GroupBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase) + .Select(static group => group.First()) + .OrderBy(static attachment => attachment.Id, StringComparer.OrdinalIgnoreCase) + .ToList(); + + var newAttachments = uniqueAttachments + .Where(attachment => !knownBulletins.Contains(attachment.Id)) + .Take(_options.MaxBulletinsPerFetch) + .ToList(); + + _diagnostics.ListingFetchSuccess(listingSummary.PagesVisited, uniqueAttachments.Count, newAttachments.Count); + + if (newAttachments.Count == 0) + { + _logger.LogDebug("NKCKI listing contained no new bulletin attachments"); + processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); + await UpdateCursorAsync(cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownBulletins(NormalizeBulletins(knownBulletins)) + .WithLastListingFetch(now), cancellationToken).ConfigureAwait(false); + return; + } + + var downloaded = 0; + var cachedUsed = 0; + var failures = 0; + + foreach (var attachment in newAttachments) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, attachment.Uri) + { + AcceptHeaders = BulletinAcceptHeaders, + TimeoutOverride = _options.RequestTimeout, + }; + + var attachmentResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + if (!attachmentResult.IsSuccess || attachmentResult.Content is null) + { + if (TryReadCachedBulletin(attachment.Id, out var cachedBytes)) + { + _diagnostics.BulletinFetchCached(); + cachedUsed++; + _logger.LogWarning("NKCKI bulletin {BulletinId} unavailable (status={Status}); using cached artefact", attachment.Id, attachmentResult.StatusCode); + processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); + knownBulletins.Add(attachment.Id); + } + else + { + _diagnostics.BulletinFetchFailure(attachmentResult.StatusCode.ToString()); + failures++; + _logger.LogWarning("NKCKI bulletin {BulletinId} returned no content (status={Status})", attachment.Id, attachmentResult.StatusCode); + } + + continue; + } + + _diagnostics.BulletinFetchSuccess(); + downloaded++; + TryWriteCachedBulletin(attachment.Id, attachmentResult.Content); + processed = await ProcessBulletinEntriesAsync(attachmentResult.Content, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); + knownBulletins.Add(attachment.Id); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + if (TryReadCachedBulletin(attachment.Id, out var cachedBytes)) + { + _diagnostics.BulletinFetchCached(); + cachedUsed++; + _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}; using cached artefact", attachment.Id); + processed = await ProcessBulletinEntriesAsync(cachedBytes, attachment.Id, pendingDocuments, pendingMappings, now, processed, cancellationToken).ConfigureAwait(false); + knownBulletins.Add(attachment.Id); + } + else + { + _diagnostics.BulletinFetchFailure(ex.Message); + failures++; + _logger.LogWarning(ex, "NKCKI bulletin fetch failed for {BulletinId}", attachment.Id); + await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + } + + if (processed >= _options.MaxVulnerabilitiesPerFetch) + { + break; + } + + if (_options.RequestDelay > TimeSpan.Zero) + { + try + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + break; + } + } + } + + if (processed < _options.MaxVulnerabilitiesPerFetch) + { + processed = await ProcessCachedBulletinsAsync(pendingDocuments, pendingMappings, knownBulletins, now, processed, cancellationToken).ConfigureAwait(false); + } + + var normalizedBulletins = NormalizeBulletins(knownBulletins); + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithKnownBulletins(normalizedBulletins) + .WithLastListingFetch(now); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "NKCKI fetch complete: new bulletins {Downloaded}, cached bulletins {Cached}, failures {Failures}, processed entries {Processed}, pending documents {PendingDocuments}, pending mappings {PendingMappings}", + downloaded, + cachedUsed, + failures, + processed, + pendingDocuments.Count, + pendingMappings.Count); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var pendingDocuments = cursor.PendingDocuments.ToList(); + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _logger.LogWarning("NKCKI document {DocumentId} missing GridFS payload", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + byte[] payload; + try + { + payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogError(ex, "NKCKI unable to download raw document {DocumentId}", documentId); + throw; + } + + RuNkckiVulnerabilityDto? dto; + try + { + dto = JsonSerializer.Deserialize(payload, SerializerOptions); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "NKCKI failed to deserialize document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (dto is null) + { + _logger.LogWarning("NKCKI document {DocumentId} produced null DTO", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var bson = MongoDB.Bson.BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "ru-nkcki.v1", bson, _timeProvider.GetUtcNow()); + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + pendingDocuments.Remove(documentId); + if (!pendingMappings.Contains(documentId)) + { + pendingMappings.Add(documentId); + } + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToList(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + _logger.LogWarning("NKCKI document {DocumentId} missing DTO payload", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + RuNkckiVulnerabilityDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToString(), SerializerOptions) ?? throw new InvalidOperationException("DTO deserialized to null"); + } + catch (Exception ex) + { + _logger.LogError(ex, "NKCKI failed to deserialize DTO for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + continue; + } + + try + { + var advisory = RuNkckiMapper.Map(dto, document, dtoRecord.ValidatedAt); + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + catch (Exception ex) + { + _logger.LogError(ex, "NKCKI mapping failed for document {DocumentId}", documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + pendingMappings.Remove(documentId); + } + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private async Task ProcessCachedBulletinsAsync( + HashSet pendingDocuments, + HashSet pendingMappings, + HashSet knownBulletins, + DateTimeOffset now, + int processed, + CancellationToken cancellationToken) + { + if (!Directory.Exists(_cacheDirectory)) + { + return processed; + } + + var updated = processed; + var cacheFiles = Directory + .EnumerateFiles(_cacheDirectory, "*.json.zip", SearchOption.TopDirectoryOnly) + .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) + .ToList(); + + foreach (var filePath in cacheFiles) + { + cancellationToken.ThrowIfCancellationRequested(); + + var bulletinId = ExtractBulletinIdFromCachePath(filePath); + if (string.IsNullOrWhiteSpace(bulletinId) || knownBulletins.Contains(bulletinId)) + { + continue; + } + + byte[] content; + try + { + content = File.ReadAllBytes(filePath); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "NKCKI failed to read cached bulletin at {CachePath}", filePath); + continue; + } + + _diagnostics.BulletinFetchCached(); + updated = await ProcessBulletinEntriesAsync(content, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false); + knownBulletins.Add(bulletinId); + + if (updated >= _options.MaxVulnerabilitiesPerFetch) + { + break; + } + } + + return updated; + } + + private async Task ProcessBulletinEntriesAsync( + byte[] content, + string bulletinId, + HashSet pendingDocuments, + HashSet pendingMappings, + DateTimeOffset now, + int processed, + CancellationToken cancellationToken) + { + if (content.Length == 0) + { + return processed; + } + + var updated = processed; + using var archiveStream = new MemoryStream(content, writable: false); + using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Read, leaveOpen: false); + + foreach (var entry in archive.Entries.OrderBy(static e => e.FullName, StringComparer.OrdinalIgnoreCase)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!entry.FullName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + using var entryStream = entry.Open(); + using var buffer = new MemoryStream(); + await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + + if (buffer.Length == 0) + { + continue; + } + + buffer.Position = 0; + + using var document = await JsonDocument.ParseAsync(buffer, cancellationToken: cancellationToken).ConfigureAwait(false); + updated = await ProcessBulletinJsonElementAsync(document.RootElement, entry.FullName, bulletinId, pendingDocuments, pendingMappings, now, updated, cancellationToken).ConfigureAwait(false); + + if (updated >= _options.MaxVulnerabilitiesPerFetch) + { + break; + } + } + + var delta = updated - processed; + if (delta > 0) + { + _diagnostics.EntriesProcessed(delta); + } + + return updated; + } + + private async Task ProcessBulletinJsonElementAsync( + JsonElement element, + string entryName, + string bulletinId, + HashSet pendingDocuments, + HashSet pendingMappings, + DateTimeOffset now, + int processed, + CancellationToken cancellationToken) + { + var updated = processed; + + switch (element.ValueKind) + { + case JsonValueKind.Array: + foreach (var child in element.EnumerateArray()) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (updated >= _options.MaxVulnerabilitiesPerFetch) + { + break; + } + + if (child.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (await ProcessVulnerabilityObjectAsync(child, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false)) + { + updated++; + } + } + + break; + + case JsonValueKind.Object: + if (await ProcessVulnerabilityObjectAsync(element, entryName, bulletinId, pendingDocuments, pendingMappings, now, cancellationToken).ConfigureAwait(false)) + { + updated++; + } + + break; + } + + return updated; + } + + private async Task ProcessVulnerabilityObjectAsync( + JsonElement element, + string entryName, + string bulletinId, + HashSet pendingDocuments, + HashSet pendingMappings, + DateTimeOffset now, + CancellationToken cancellationToken) + { + RuNkckiVulnerabilityDto dto; + try + { + dto = RuNkckiJsonParser.Parse(element); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "NKCKI failed to parse vulnerability in bulletin {BulletinId} entry {Entry}", bulletinId, entryName); + return false; + } + + var payload = JsonSerializer.SerializeToUtf8Bytes(dto, SerializerOptions); + var sha = Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant(); + var documentUri = BuildDocumentUri(dto); + + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false); + if (existing is not null && string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync(SourceName, documentUri, payload, "application/json", null, cancellationToken).ConfigureAwait(false); + + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["ru-nkcki.bulletin"] = bulletinId, + ["ru-nkcki.entry"] = entryName, + }; + + if (!string.IsNullOrWhiteSpace(dto.FstecId)) + { + metadata["ru-nkcki.fstec_id"] = dto.FstecId!; + } + + if (!string.IsNullOrWhiteSpace(dto.MitreId)) + { + metadata["ru-nkcki.mitre_id"] = dto.MitreId!; + } + + var recordId = existing?.Id ?? Guid.NewGuid(); + var lastModified = dto.DateUpdated ?? dto.DatePublished; + var record = new DocumentRecord( + recordId, + SourceName, + documentUri, + now, + sha, + DocumentStatuses.PendingParse, + "application/json", + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: lastModified, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + pendingDocuments.Add(upserted.Id); + pendingMappings.Remove(upserted.Id); + return true; + } + + private Task FetchListingPageAsync(Uri pageUri, CancellationToken cancellationToken) + { + var request = new SourceFetchRequest(RuNkckiOptions.HttpClientName, SourceName, pageUri) + { + AcceptHeaders = ListingAcceptHeaders, + TimeoutOverride = _options.RequestTimeout, + }; + + return _fetchService.FetchContentAsync(request, cancellationToken); + } + + private async Task ParseListingAsync(Uri pageUri, byte[] content, CancellationToken cancellationToken) + { + var html = Encoding.UTF8.GetString(content); + var document = await _htmlParser.ParseDocumentAsync(html, cancellationToken).ConfigureAwait(false); + var attachments = new List(); + var pagination = new List(); + + foreach (var anchor in document.QuerySelectorAll("a[href$='.json.zip']")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(pageUri, href, out var absoluteUri)) + { + continue; + } + + var id = DeriveBulletinId(absoluteUri); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + var title = anchor.GetAttribute("title"); + if (string.IsNullOrWhiteSpace(title)) + { + title = anchor.TextContent?.Trim(); + } + + attachments.Add(new BulletinAttachment(id, absoluteUri, title ?? id)); + } + + foreach (var anchor in document.QuerySelectorAll("a[href]")) + { + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!href.Contains("PAGEN", StringComparison.OrdinalIgnoreCase) + && !href.Contains("page=", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (Uri.TryCreate(pageUri, href, out var absoluteUri)) + { + pagination.Add(absoluteUri); + } + } + + var uniquePagination = pagination + .DistinctBy(static uri => uri.AbsoluteUri, StringComparer.OrdinalIgnoreCase) + .Take(_options.MaxListingPagesPerFetch) + .ToList(); + + return new ListingPageResult(attachments, uniquePagination); + } + + private static string DeriveBulletinId(Uri uri) + { + var fileName = Path.GetFileName(uri.AbsolutePath); + if (string.IsNullOrWhiteSpace(fileName)) + { + return Guid.NewGuid().ToString("N"); + } + + if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^4]; + } + + if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^5]; + } + + return fileName.Replace('_', '-'); + } + + private static string BuildDocumentUri(RuNkckiVulnerabilityDto dto) + { + if (!string.IsNullOrWhiteSpace(dto.FstecId)) + { + var slug = dto.FstecId.Contains(':', StringComparison.Ordinal) + ? dto.FstecId[(dto.FstecId.IndexOf(':') + 1)..] + : dto.FstecId; + return $"https://cert.gov.ru/materialy/uyazvimosti/{slug}"; + } + + if (!string.IsNullOrWhiteSpace(dto.MitreId)) + { + return $"https://nvd.nist.gov/vuln/detail/{dto.MitreId}"; + } + + return $"https://cert.gov.ru/materialy/uyazvimosti/{Guid.NewGuid():N}"; + } + + private string ResolveCacheDirectory(string? configuredPath) + { + if (!string.IsNullOrWhiteSpace(configuredPath)) + { + return Path.GetFullPath(Path.IsPathRooted(configuredPath) + ? configuredPath + : Path.Combine(AppContext.BaseDirectory, configuredPath)); + } + + return Path.Combine(AppContext.BaseDirectory, "cache", RuNkckiConnectorPlugin.SourceName); + } + + private void EnsureCacheDirectory() + { + try + { + Directory.CreateDirectory(_cacheDirectory); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "NKCKI unable to ensure cache directory {CachePath}", _cacheDirectory); + } + } + + private string GetBulletinCachePath(string bulletinId) + { + var fileStem = string.IsNullOrWhiteSpace(bulletinId) + ? Guid.NewGuid().ToString("N") + : Uri.EscapeDataString(bulletinId); + return Path.Combine(_cacheDirectory, $"{fileStem}.json.zip"); + } + + private static string ExtractBulletinIdFromCachePath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return string.Empty; + } + + var fileName = Path.GetFileName(path); + if (fileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^4]; + } + + if (fileName.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + fileName = fileName[..^5]; + } + + return Uri.UnescapeDataString(fileName); + } + + private void TryWriteCachedBulletin(string bulletinId, byte[] content) + { + try + { + var cachePath = GetBulletinCachePath(bulletinId); + Directory.CreateDirectory(Path.GetDirectoryName(cachePath)!); + File.WriteAllBytes(cachePath, content); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "NKCKI failed to cache bulletin {BulletinId}", bulletinId); + } + } + + private bool TryReadCachedBulletin(string bulletinId, out byte[] content) + { + var cachePath = GetBulletinCachePath(bulletinId); + try + { + if (File.Exists(cachePath)) + { + content = File.ReadAllBytes(cachePath); + return true; + } + } + catch (Exception ex) + { + _logger.LogDebug(ex, "NKCKI failed to read cached bulletin {BulletinId}", bulletinId); + } + + content = Array.Empty(); + return false; + } + + private IReadOnlyCollection NormalizeBulletins(IEnumerable bulletins) + { + var normalized = (bulletins ?? Enumerable.Empty()) + .Where(static id => !string.IsNullOrWhiteSpace(id)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (normalized.Count <= _options.KnownBulletinCapacity) + { + return normalized.ToArray(); + } + + var skip = normalized.Count - _options.KnownBulletinCapacity; + return normalized.Skip(skip).ToArray(); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? RuNkckiCursor.Empty : RuNkckiCursor.FromBson(state.Cursor); + } + + private Task UpdateCursorAsync(RuNkckiCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var completedAt = cursor.LastListingFetchAt ?? _timeProvider.GetUtcNow(); + return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken); + } + + private readonly record struct ListingFetchSummary(IReadOnlyList Attachments, int PagesVisited); + + private readonly record struct ListingPageResult(IReadOnlyList Attachments, IReadOnlyList PaginationLinks); + + private readonly record struct BulletinAttachment(string Id, Uri Uri, string Title); + + private bool ShouldUseListingCache(RuNkckiCursor cursor, DateTimeOffset now) + { + if (!cursor.LastListingFetchAt.HasValue) + { + return false; + } + + var age = now - cursor.LastListingFetchAt.Value; + return age < _options.ListingCacheDuration; + } + + private async Task LoadListingAsync(CancellationToken cancellationToken) + { + var attachments = new List(); + var visited = 0; + var visitedUris = new HashSet(StringComparer.OrdinalIgnoreCase); + var queue = new Queue(); + queue.Enqueue(_options.ListingUri); + + while (queue.Count > 0 && visited < _options.MaxListingPagesPerFetch) + { + cancellationToken.ThrowIfCancellationRequested(); + + var pageUri = queue.Dequeue(); + if (!visitedUris.Add(pageUri.AbsoluteUri)) + { + continue; + } + + _diagnostics.ListingFetchAttempt(); + + var listingResult = await FetchListingPageAsync(pageUri, cancellationToken).ConfigureAwait(false); + if (!listingResult.IsSuccess || listingResult.Content is null) + { + _diagnostics.ListingFetchFailure(listingResult.StatusCode.ToString()); + _logger.LogWarning("NKCKI listing page {ListingUri} returned no content (status={Status})", pageUri, listingResult.StatusCode); + continue; + } + + visited++; + + var page = await ParseListingAsync(pageUri, listingResult.Content, cancellationToken).ConfigureAwait(false); + attachments.AddRange(page.Attachments); + + foreach (var link in page.PaginationLinks) + { + if (!visitedUris.Contains(link.AbsoluteUri) && queue.Count + visitedUris.Count < _options.MaxListingPagesPerFetch) + { + queue.Enqueue(link); + } + } + + if (attachments.Count >= _options.MaxBulletinsPerFetch * 2) + { + break; + } + } + + return new ListingFetchSummary(attachments, visited); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnectorPlugin.cs index 525f7d4e..eea20fed 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiConnectorPlugin.cs @@ -1,19 +1,19 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Ru.Nkcki; - -public sealed class RuNkckiConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "ru-nkcki"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) => services is not null; - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki; + +public sealed class RuNkckiConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "ru-nkcki"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs index c198e204..94e73bd9 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiDependencyInjectionRoutine.cs @@ -1,53 +1,53 @@ -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; - -namespace StellaOps.Feedser.Source.Ru.Nkcki; - -public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:ru-nkcki"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddRuNkckiConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob)); - EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob)); - EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType) - { - if (schedulerOptions.Definitions.ContainsKey(kind)) - { - return; - } - - schedulerOptions.Definitions[kind] = new JobDefinition( - kind, - jobType, - schedulerOptions.DefaultTimeout, - schedulerOptions.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration; + +namespace StellaOps.Concelier.Connector.Ru.Nkcki; + +public sealed class RuNkckiDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:ru-nkcki"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddRuNkckiConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, RuNkckiJobKinds.Fetch, typeof(RuNkckiFetchJob)); + EnsureJob(options, RuNkckiJobKinds.Parse, typeof(RuNkckiParseJob)); + EnsureJob(options, RuNkckiJobKinds.Map, typeof(RuNkckiMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions schedulerOptions, string kind, Type jobType) + { + if (schedulerOptions.Definitions.ContainsKey(kind)) + { + return; + } + + schedulerOptions.Definitions[kind] = new JobDefinition( + kind, + jobType, + schedulerOptions.DefaultTimeout, + schedulerOptions.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs index 4c7d58d8..9c11d66a 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/RuNkckiServiceCollectionExtensions.cs @@ -2,11 +2,11 @@ using System.Net; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Ru.Nkcki.Configuration; -using StellaOps.Feedser.Source.Ru.Nkcki.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Ru.Nkcki.Configuration; +using StellaOps.Concelier.Connector.Ru.Nkcki.Internal; -namespace StellaOps.Feedser.Source.Ru.Nkcki; +namespace StellaOps.Concelier.Connector.Ru.Nkcki; public static class RuNkckiServiceCollectionExtensions { diff --git a/src/StellaOps.Concelier.Connector.Ru.Nkcki/StellaOps.Concelier.Connector.Ru.Nkcki.csproj b/src/StellaOps.Concelier.Connector.Ru.Nkcki/StellaOps.Concelier.Connector.Ru.Nkcki.csproj new file mode 100644 index 00000000..b10cc4d0 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/StellaOps.Concelier.Connector.Ru.Nkcki.csproj @@ -0,0 +1,22 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md b/src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md similarity index 88% rename from src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md rename to src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md index 976bd795..d41ef558 100644 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Ru.Nkcki/TASKS.md @@ -1,11 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/feedser-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.| -|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DONE (2025-10-13)** – Listing fetch now honours `maxListingPagesPerFetch`, persists cache hits when listing access fails, and records telemetry via `RuNkckiDiagnostics`. Cursor tracking covers pending documents/mappings and the known bulletin ring buffer.| -|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-13)** – Parser normalises nested arrays (ICS categories, vulnerable software lists, optional tags), flattens multiline `software_text`, and guarantees deterministic ordering for URLs and tags.| -|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DONE (2025-10-13)** – Mapper splits structured software entries, emits SemVer range primitives + normalized rules, deduplicates references, and surfaces CVSS v4 metadata alongside existing metrics.| -|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-13)** – Fixtures refreshed with multi-page pagination + multi-entry bulletins. Tests exercise cache replay and rely on bundled OpenSSL 1.1 libs in `tools/openssl/linux-x64` to keep Mongo2Go green on modern distros.| -|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-13)** – Added connector-specific metrics (`nkcki.*`) and documented configuration/operational guidance in `docs/ops/feedser-nkcki-operations.md`.| -|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**DONE (2025-10-13)** – Documented Bitrix pagination/backfill plan (cache-first, offline replay, HTML/PDF capture) in `docs/ops/feedser-nkcki-operations.md`.| -|FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`feedser:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-NKCKI-02-001 Research NKTsKI advisory feeds|BE-Conn-Nkcki|Research|**DONE (2025-10-11)** – Candidate RSS locations (`https://cert.gov.ru/rss/advisories.xml`, `https://www.cert.gov.ru/...`) return 403/404 even with `Accept-Language: ru-RU` and `--insecure`; site is Bitrix-backed and expects Russian Trusted Sub CA plus session cookies. Logged packet captures + needed cert list in `docs/concelier-connector-research-20251011.md`; waiting on Ops for sanctioned trust bundle.| +|FEEDCONN-NKCKI-02-002 Fetch pipeline & state persistence|BE-Conn-Nkcki|Source.Common, Storage.Mongo|**DONE (2025-10-13)** – Listing fetch now honours `maxListingPagesPerFetch`, persists cache hits when listing access fails, and records telemetry via `RuNkckiDiagnostics`. Cursor tracking covers pending documents/mappings and the known bulletin ring buffer.| +|FEEDCONN-NKCKI-02-003 DTO & parser implementation|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-13)** – Parser normalises nested arrays (ICS categories, vulnerable software lists, optional tags), flattens multiline `software_text`, and guarantees deterministic ordering for URLs and tags.| +|FEEDCONN-NKCKI-02-004 Canonical mapping & range primitives|BE-Conn-Nkcki|Models|**DONE (2025-10-13)** – Mapper splits structured software entries, emits SemVer range primitives + normalized rules, deduplicates references, and surfaces CVSS v4 metadata alongside existing metrics.| +|FEEDCONN-NKCKI-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-13)** – Fixtures refreshed with multi-page pagination + multi-entry bulletins. Tests exercise cache replay and rely on bundled OpenSSL 1.1 libs in `tools/openssl/linux-x64` to keep Mongo2Go green on modern distros.| +|FEEDCONN-NKCKI-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-13)** – Added connector-specific metrics (`nkcki.*`) and documented configuration/operational guidance in `docs/ops/concelier-nkcki-operations.md`.| +|FEEDCONN-NKCKI-02-007 Archive ingestion strategy|BE-Conn-Nkcki|Research|**DONE (2025-10-13)** – Documented Bitrix pagination/backfill plan (cache-first, offline replay, HTML/PDF capture) in `docs/ops/concelier-nkcki-operations.md`.| +|FEEDCONN-NKCKI-02-008 Access enablement plan|BE-Conn-Nkcki|Source.Common|**DONE (2025-10-11)** – Documented trust-store requirement, optional SOCKS proxy fallback, and monitoring plan; shared TLS support now available via `SourceHttpClientOptions.TrustedRootCertificates` (`concelier:httpClients:source.nkcki:*`), awaiting Ops-sourced cert bundle before fetch implementation.| diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs new file mode 100644 index 00000000..1cd7a6ea --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/MirrorSignatureVerifierTests.cs @@ -0,0 +1,95 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Concelier.Connector.StellaOpsMirror.Security; +using StellaOps.Cryptography; +using Xunit; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; + +public sealed class MirrorSignatureVerifierTests +{ + [Fact] + public async Task VerifyAsync_ValidSignaturePasses() + { + var provider = new DefaultCryptoProvider(); + var key = CreateSigningKey("mirror-key"); + provider.UpsertSigningKey(key); + + var registry = new CryptoProviderRegistry(new[] { provider }); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + + var payload = "{\"advisories\":[]}\"u8".ToUtf8Bytes(); + var (signature, _) = await CreateDetachedJwsAsync(provider, key.Reference.KeyId, payload); + + await verifier.VerifyAsync(payload, signature, CancellationToken.None); + } + + [Fact] + public async Task VerifyAsync_InvalidSignatureThrows() + { + var provider = new DefaultCryptoProvider(); + var key = CreateSigningKey("mirror-key"); + provider.UpsertSigningKey(key); + + var registry = new CryptoProviderRegistry(new[] { provider }); + var verifier = new MirrorSignatureVerifier(registry, NullLogger.Instance); + + var payload = "{\"advisories\":[]}\"u8".ToUtf8Bytes(); + var (signature, _) = await CreateDetachedJwsAsync(provider, key.Reference.KeyId, payload); + + var tampered = signature.Replace("a", "b", StringComparison.Ordinal); + + await Assert.ThrowsAsync(() => verifier.VerifyAsync(payload, tampered, CancellationToken.None)); + } + + private static CryptoSigningKey CreateSigningKey(string keyId) + { + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var parameters = ecdsa.ExportParameters(includePrivateParameters: true); + return new CryptoSigningKey(new CryptoKeyReference(keyId), SignatureAlgorithms.Es256, in parameters, DateTimeOffset.UtcNow); + } + + private static async Task<(string Signature, DateTimeOffset SignedAt)> CreateDetachedJwsAsync( + DefaultCryptoProvider provider, + string keyId, + ReadOnlyMemory payload) + { + var signer = provider.GetSigner(SignatureAlgorithms.Es256, new CryptoKeyReference(keyId)); + var header = new Dictionary + { + ["alg"] = SignatureAlgorithms.Es256, + ["kid"] = keyId, + ["provider"] = provider.Name, + ["typ"] = "application/vnd.stellaops.concelier.mirror-bundle+jws", + ["b64"] = false, + ["crit"] = new[] { "b64" } + }; + + var headerJson = System.Text.Json.JsonSerializer.Serialize(header); + var protectedHeader = Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(headerJson); + + var signingInput = BuildSigningInput(protectedHeader, payload.Span); + var signatureBytes = await signer.SignAsync(signingInput, CancellationToken.None).ConfigureAwait(false); + var encodedSignature = Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(signatureBytes); + + return (string.Concat(protectedHeader, "..", encodedSignature), DateTimeOffset.UtcNow); + } + + private static ReadOnlyMemory BuildSigningInput(string encodedHeader, ReadOnlySpan payload) + { + var headerBytes = System.Text.Encoding.ASCII.GetBytes(encodedHeader); + var buffer = new byte[headerBytes.Length + 1 + payload.Length]; + headerBytes.CopyTo(buffer.AsSpan()); + buffer[headerBytes.Length] = (byte)'.'; + payload.CopyTo(buffer.AsSpan(headerBytes.Length + 1)); + return buffer; + } +} + +file static class Utf8Extensions +{ + public static ReadOnlyMemory ToUtf8Bytes(this string value) + => System.Text.Encoding.UTF8.GetBytes(value); +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj new file mode 100644 index 00000000..339c2eb1 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOps.Concelier.Connector.StellaOpsMirror.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs new file mode 100644 index 00000000..c1920156 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror.Tests/StellaOpsMirrorConnectorTests.cs @@ -0,0 +1,319 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.SourceState; +using StellaOps.Concelier.Testing; +using StellaOps.Cryptography; +using Xunit; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Tests; + +[Collection("mongo-fixture")] +public sealed class StellaOpsMirrorConnectorTests : IAsyncLifetime +{ + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler; + + public StellaOpsMirrorConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchAsync_PersistsMirrorArtifacts() + { + var manifestContent = "{\"domain\":\"primary\",\"files\":[]}"; + var bundleContent = "{\"advisories\":[{\"id\":\"CVE-2025-0001\"}]}"; + + var manifestDigest = ComputeDigest(manifestContent); + var bundleDigest = ComputeDigest(bundleContent); + + var index = BuildIndex(manifestDigest, Encoding.UTF8.GetByteCount(manifestContent), bundleDigest, Encoding.UTF8.GetByteCount(bundleContent), includeSignature: false); + + await using var provider = await BuildServiceProviderAsync(); + + SeedResponses(index, manifestContent, bundleContent, signature: null); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + + var documentStore = provider.GetRequiredService(); + var manifestUri = "https://mirror.test/mirror/primary/manifest.json"; + var bundleUri = "https://mirror.test/mirror/primary/bundle.json"; + + var manifestDocument = await documentStore.FindBySourceAndUriAsync(StellaOpsMirrorConnector.Source, manifestUri, CancellationToken.None); + Assert.NotNull(manifestDocument); + Assert.Equal(DocumentStatuses.Mapped, manifestDocument!.Status); + Assert.Equal(NormalizeDigest(manifestDigest), manifestDocument.Sha256); + + var bundleDocument = await documentStore.FindBySourceAndUriAsync(StellaOpsMirrorConnector.Source, bundleUri, CancellationToken.None); + Assert.NotNull(bundleDocument); + Assert.Equal(DocumentStatuses.PendingParse, bundleDocument!.Status); + Assert.Equal(NormalizeDigest(bundleDigest), bundleDocument.Sha256); + + var rawStorage = provider.GetRequiredService(); + Assert.NotNull(manifestDocument.GridFsId); + Assert.NotNull(bundleDocument.GridFsId); + + var manifestBytes = await rawStorage.DownloadAsync(manifestDocument.GridFsId!.Value, CancellationToken.None); + var bundleBytes = await rawStorage.DownloadAsync(bundleDocument.GridFsId!.Value, CancellationToken.None); + Assert.Equal(manifestContent, Encoding.UTF8.GetString(manifestBytes)); + Assert.Equal(bundleContent, Encoding.UTF8.GetString(bundleBytes)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(StellaOpsMirrorConnector.Source, CancellationToken.None); + Assert.NotNull(state); + + var cursorDocument = state!.Cursor ?? new BsonDocument(); + var digestValue = cursorDocument.TryGetValue("bundleDigest", out var digestBson) ? digestBson.AsString : string.Empty; + Assert.Equal(NormalizeDigest(bundleDigest), NormalizeDigest(digestValue)); + + var pendingDocumentsArray = cursorDocument.TryGetValue("pendingDocuments", out var pendingDocsBson) && pendingDocsBson is BsonArray pendingArray + ? pendingArray + : new BsonArray(); + Assert.Single(pendingDocumentsArray); + var pendingDocumentId = Guid.Parse(pendingDocumentsArray[0].AsString); + Assert.Equal(bundleDocument.Id, pendingDocumentId); + + var pendingMappingsArray = cursorDocument.TryGetValue("pendingMappings", out var pendingMappingsBson) && pendingMappingsBson is BsonArray mappingsArray + ? mappingsArray + : new BsonArray(); + Assert.Empty(pendingMappingsArray); + } + + [Fact] + public async Task FetchAsync_TamperedSignatureThrows() + { + var manifestContent = "{\"domain\":\"primary\"}"; + var bundleContent = "{\"advisories\":[{\"id\":\"CVE-2025-0002\"}]}"; + + var manifestDigest = ComputeDigest(manifestContent); + var bundleDigest = ComputeDigest(bundleContent); + var index = BuildIndex(manifestDigest, Encoding.UTF8.GetByteCount(manifestContent), bundleDigest, Encoding.UTF8.GetByteCount(bundleContent), includeSignature: true); + + await using var provider = await BuildServiceProviderAsync(options => + { + options.Signature.Enabled = true; + options.Signature.KeyId = "mirror-key"; + options.Signature.Provider = "default"; + }); + + var defaultProvider = provider.GetRequiredService(); + var signingKey = CreateSigningKey("mirror-key"); + defaultProvider.UpsertSigningKey(signingKey); + + var (signatureValue, _) = CreateDetachedJws(signingKey, bundleContent); + // Tamper with signature so verification fails. + var tamperedSignature = signatureValue.Replace('a', 'b'); + + SeedResponses(index, manifestContent, bundleContent, tamperedSignature); + + var connector = provider.GetRequiredService(); + await Assert.ThrowsAsync(() => connector.FetchAsync(provider, CancellationToken.None)); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(StellaOpsMirrorConnector.Source, CancellationToken.None); + Assert.NotNull(state); + Assert.True(state!.FailCount >= 1); + Assert.False(state.Cursor.TryGetValue("bundleDigest", out _)); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _handler.Clear(); + return Task.CompletedTask; + } + + private async Task BuildServiceProviderAsync(Action? configureOptions = null) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + services.AddSingleton(TimeProvider.System); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => new CryptoProviderRegistry(sp.GetServices())); + + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["concelier:sources:stellaopsMirror:baseAddress"] = "https://mirror.test/", + ["concelier:sources:stellaopsMirror:domainId"] = "primary", + ["concelier:sources:stellaopsMirror:indexPath"] = "/concelier/exports/index.json", + }) + .Build(); + + var routine = new StellaOpsMirrorDependencyInjectionRoutine(); + routine.Register(services, configuration); + + if (configureOptions is not null) + { + services.PostConfigure(configureOptions); + } + + services.Configure("stellaops-mirror", builder => + { + builder.HttpMessageHandlerBuilderActions.Add(options => + { + options.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedResponses(string indexJson, string manifestContent, string bundleContent, string? signature) + { + var baseUri = new Uri("https://mirror.test"); + _handler.AddResponse(HttpMethod.Get, new Uri(baseUri, "/concelier/exports/index.json"), () => CreateJsonResponse(indexJson)); + _handler.AddResponse(HttpMethod.Get, new Uri(baseUri, "mirror/primary/manifest.json"), () => CreateJsonResponse(manifestContent)); + _handler.AddResponse(HttpMethod.Get, new Uri(baseUri, "mirror/primary/bundle.json"), () => CreateJsonResponse(bundleContent)); + + if (signature is not null) + { + _handler.AddResponse(HttpMethod.Get, new Uri(baseUri, "mirror/primary/bundle.json.jws"), () => new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(signature, Encoding.UTF8, "application/jose+json"), + }); + } + } + + private static HttpResponseMessage CreateJsonResponse(string content) + => new(HttpStatusCode.OK) + { + Content = new StringContent(content, Encoding.UTF8, "application/json"), + }; + + private static string BuildIndex(string manifestDigest, int manifestBytes, string bundleDigest, int bundleBytes, bool includeSignature) + { + var index = new + { + schemaVersion = 1, + generatedAt = new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero), + targetRepository = "repo", + domains = new[] + { + new + { + domainId = "primary", + displayName = "Primary", + advisoryCount = 1, + manifest = new + { + path = "mirror/primary/manifest.json", + sizeBytes = manifestBytes, + digest = manifestDigest, + signature = (object?)null, + }, + bundle = new + { + path = "mirror/primary/bundle.json", + sizeBytes = bundleBytes, + digest = bundleDigest, + signature = includeSignature + ? new + { + path = "mirror/primary/bundle.json.jws", + algorithm = "ES256", + keyId = "mirror-key", + provider = "default", + signedAt = new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero), + } + : null, + }, + sources = Array.Empty(), + } + } + }; + + return JsonSerializer.Serialize(index, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + }); + } + + private static string ComputeDigest(string content) + { + var bytes = Encoding.UTF8.GetBytes(content); + var hash = SHA256.HashData(bytes); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string NormalizeDigest(string digest) + => digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) ? digest[7..] : digest; + + private static CryptoSigningKey CreateSigningKey(string keyId) + { + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var parameters = ecdsa.ExportParameters(includePrivateParameters: true); + return new CryptoSigningKey(new CryptoKeyReference(keyId), SignatureAlgorithms.Es256, in parameters, DateTimeOffset.UtcNow); + } + + private static (string Signature, DateTimeOffset SignedAt) CreateDetachedJws(CryptoSigningKey signingKey, string payload) + { + using var provider = new DefaultCryptoProvider(); + provider.UpsertSigningKey(signingKey); + var signer = provider.GetSigner(SignatureAlgorithms.Es256, signingKey.Reference); + var header = new Dictionary + { + ["alg"] = SignatureAlgorithms.Es256, + ["kid"] = signingKey.Reference.KeyId, + ["provider"] = provider.Name, + ["typ"] = "application/vnd.stellaops.concelier.mirror-bundle+jws", + ["b64"] = false, + ["crit"] = new[] { "b64" } + }; + + var headerJson = JsonSerializer.Serialize(header); + var encodedHeader = Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(headerJson); + var payloadBytes = Encoding.UTF8.GetBytes(payload); + var signingInput = BuildSigningInput(encodedHeader, payloadBytes); + var signatureBytes = signer.SignAsync(signingInput, CancellationToken.None).GetAwaiter().GetResult(); + var encodedSignature = Microsoft.IdentityModel.Tokens.Base64UrlEncoder.Encode(signatureBytes); + return (string.Concat(encodedHeader, "..", encodedSignature), DateTimeOffset.UtcNow); + } + + private static ReadOnlyMemory BuildSigningInput(string encodedHeader, ReadOnlySpan payload) + { + var headerBytes = Encoding.ASCII.GetBytes(encodedHeader); + var buffer = new byte[headerBytes.Length + 1 + payload.Length]; + headerBytes.CopyTo(buffer, 0); + buffer[headerBytes.Length] = (byte)'.'; + payload.CopyTo(buffer.AsSpan(headerBytes.Length + 1)); + return buffer; + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Client/MirrorManifestClient.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Client/MirrorManifestClient.cs new file mode 100644 index 00000000..7ef0acae --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Client/MirrorManifestClient.cs @@ -0,0 +1,89 @@ +using System; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Client; + +/// +/// Lightweight HTTP client for retrieving mirror index and domain artefacts. +/// +public sealed class MirrorManifestClient +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip + }; + + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public MirrorManifestClient(HttpClient httpClient, ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task GetIndexAsync(string indexPath, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(indexPath)) + { + throw new ArgumentException("Index path must be provided.", nameof(indexPath)); + } + + using var request = new HttpRequestMessage(HttpMethod.Get, indexPath); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + await EnsureSuccessAsync(response, indexPath, cancellationToken).ConfigureAwait(false); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = await JsonSerializer.DeserializeAsync(stream, JsonOptions, cancellationToken).ConfigureAwait(false); + if (document is null) + { + throw new InvalidOperationException("Mirror index payload was empty."); + } + + return document; + } + + public async Task DownloadAsync(string path, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Path must be provided.", nameof(path)); + } + + using var request = new HttpRequestMessage(HttpMethod.Get, path); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + await EnsureSuccessAsync(response, path, cancellationToken).ConfigureAwait(false); + return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task EnsureSuccessAsync(HttpResponseMessage response, string path, CancellationToken cancellationToken) + { + if (response.IsSuccessStatusCode) + { + return; + } + + var status = (int)response.StatusCode; + var body = string.Empty; + + if (response.Content.Headers.ContentLength is long length && length > 0) + { + body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + + _logger.LogWarning( + "Mirror request to {Path} failed with {StatusCode}. Body: {Body}", + path, + status, + string.IsNullOrEmpty(body) ? "" : body); + + throw new HttpRequestException($"Mirror request to '{path}' failed with status {(HttpStatusCode)status} ({status}).", null, response.StatusCode); + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorIndexDocument.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorIndexDocument.cs new file mode 100644 index 00000000..130f3ecd --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/MirrorIndexDocument.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal; + +public sealed record MirrorIndexDocument( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("targetRepository")] string? TargetRepository, + [property: JsonPropertyName("domains")] IReadOnlyList Domains); + +public sealed record MirrorIndexDomainEntry( + [property: JsonPropertyName("domainId")] string DomainId, + [property: JsonPropertyName("displayName")] string DisplayName, + [property: JsonPropertyName("advisoryCount")] int AdvisoryCount, + [property: JsonPropertyName("manifest")] MirrorFileDescriptor Manifest, + [property: JsonPropertyName("bundle")] MirrorFileDescriptor Bundle, + [property: JsonPropertyName("sources")] IReadOnlyList Sources); + +public sealed record MirrorFileDescriptor( + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("sizeBytes")] long SizeBytes, + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("signature")] MirrorSignatureDescriptor? Signature); + +public sealed record MirrorSignatureDescriptor( + [property: JsonPropertyName("path")] string Path, + [property: JsonPropertyName("algorithm")] string Algorithm, + [property: JsonPropertyName("keyId")] string KeyId, + [property: JsonPropertyName("provider")] string Provider, + [property: JsonPropertyName("signedAt")] DateTimeOffset SignedAt); + +public sealed record MirrorSourceSummary( + [property: JsonPropertyName("source")] string Source, + [property: JsonPropertyName("firstRecordedAt")] DateTimeOffset? FirstRecordedAt, + [property: JsonPropertyName("lastRecordedAt")] DateTimeOffset? LastRecordedAt, + [property: JsonPropertyName("advisoryCount")] int AdvisoryCount); diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs new file mode 100644 index 00000000..47c26135 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Internal/StellaOpsMirrorCursor.cs @@ -0,0 +1,111 @@ +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Internal; + +internal sealed record StellaOpsMirrorCursor( + string? ExportId, + string? BundleDigest, + DateTimeOffset? GeneratedAt, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); + + public static StellaOpsMirrorCursor Empty { get; } = new( + ExportId: null, + BundleDigest: null, + GeneratedAt: null, + PendingDocuments: EmptyGuids, + PendingMappings: EmptyGuids); + + public BsonDocument ToBsonDocument() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (!string.IsNullOrWhiteSpace(ExportId)) + { + document["exportId"] = ExportId; + } + + if (!string.IsNullOrWhiteSpace(BundleDigest)) + { + document["bundleDigest"] = BundleDigest; + } + + if (GeneratedAt.HasValue) + { + document["generatedAt"] = GeneratedAt.Value.UtcDateTime; + } + + return document; + } + + public static StellaOpsMirrorCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var exportId = document.TryGetValue("exportId", out var exportValue) && exportValue.IsString ? exportValue.AsString : null; + var digest = document.TryGetValue("bundleDigest", out var digestValue) && digestValue.IsString ? digestValue.AsString : null; + DateTimeOffset? generatedAt = null; + if (document.TryGetValue("generatedAt", out var generatedValue)) + { + generatedAt = generatedValue.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(generatedValue.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(generatedValue.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; + } + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new StellaOpsMirrorCursor(exportId, digest, generatedAt, pendingDocuments, pendingMappings); + } + + public StellaOpsMirrorCursor WithPendingDocuments(IEnumerable documents) + => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuids }; + + public StellaOpsMirrorCursor WithPendingMappings(IEnumerable mappings) + => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuids }; + + public StellaOpsMirrorCursor WithBundleSnapshot(string? exportId, string? digest, DateTimeOffset generatedAt) + => this with + { + ExportId = string.IsNullOrWhiteSpace(exportId) ? ExportId : exportId, + BundleDigest = digest, + GeneratedAt = generatedAt, + }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) + { + if (!document.TryGetValue(field, out var value) || value is not BsonArray array) + { + return EmptyGuids; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (element is null) + { + continue; + } + + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Jobs.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Jobs.cs new file mode 100644 index 00000000..b0991016 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Jobs.cs @@ -0,0 +1,43 @@ +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror; + +internal static class StellaOpsMirrorJobKinds +{ + public const string Fetch = "source:stellaops-mirror:fetch"; + public const string Parse = "source:stellaops-mirror:parse"; + public const string Map = "source:stellaops-mirror:map"; +} + +internal sealed class StellaOpsMirrorFetchJob : IJob +{ + private readonly StellaOpsMirrorConnector _connector; + + public StellaOpsMirrorFetchJob(StellaOpsMirrorConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class StellaOpsMirrorParseJob : IJob +{ + private readonly StellaOpsMirrorConnector _connector; + + public StellaOpsMirrorParseJob(StellaOpsMirrorConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class StellaOpsMirrorMapJob : IJob +{ + private readonly StellaOpsMirrorConnector _connector; + + public StellaOpsMirrorMapJob(StellaOpsMirrorConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs new file mode 100644 index 00000000..789f1439 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Security/MirrorSignatureVerifier.cs @@ -0,0 +1,121 @@ +using System; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Cryptography; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Security; + +/// +/// Validates detached JWS signatures emitted by mirror bundles. +/// +public sealed class MirrorSignatureVerifier +{ + private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true + }; + + private readonly ICryptoProviderRegistry _providerRegistry; + private readonly ILogger _logger; + + public MirrorSignatureVerifier(ICryptoProviderRegistry providerRegistry, ILogger logger) + { + _providerRegistry = providerRegistry ?? throw new ArgumentNullException(nameof(providerRegistry)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task VerifyAsync(ReadOnlyMemory payload, string signatureValue, CancellationToken cancellationToken) + { + if (payload.IsEmpty) + { + throw new ArgumentException("Payload must not be empty.", nameof(payload)); + } + + if (string.IsNullOrWhiteSpace(signatureValue)) + { + throw new ArgumentException("Signature value must be provided.", nameof(signatureValue)); + } + + if (!TryParseDetachedJws(signatureValue, out var encodedHeader, out var encodedSignature)) + { + throw new InvalidOperationException("Detached JWS signature is malformed."); + } + + var headerJson = Encoding.UTF8.GetString(Base64UrlEncoder.DecodeBytes(encodedHeader)); + var header = JsonSerializer.Deserialize(headerJson, HeaderSerializerOptions) + ?? throw new InvalidOperationException("Detached JWS header could not be parsed."); + + if (!header.Critical.Contains("b64", StringComparer.Ordinal)) + { + throw new InvalidOperationException("Detached JWS header is missing required 'b64' critical parameter."); + } + + if (header.Base64Payload) + { + throw new InvalidOperationException("Detached JWS header sets b64=true; expected unencoded payload."); + } + + if (string.IsNullOrWhiteSpace(header.KeyId)) + { + throw new InvalidOperationException("Detached JWS header missing key identifier."); + } + + if (string.IsNullOrWhiteSpace(header.Algorithm)) + { + throw new InvalidOperationException("Detached JWS header missing algorithm identifier."); + } + + var signingInput = BuildSigningInput(encodedHeader, payload.Span); + var signatureBytes = Base64UrlEncoder.DecodeBytes(encodedSignature); + + var keyReference = new CryptoKeyReference(header.KeyId, header.Provider); + var resolution = _providerRegistry.ResolveSigner( + CryptoCapability.Verification, + header.Algorithm, + keyReference, + header.Provider); + + var verified = await resolution.Signer.VerifyAsync(signingInput, signatureBytes, cancellationToken).ConfigureAwait(false); + if (!verified) + { + _logger.LogWarning("Detached JWS verification failed for key {KeyId} via provider {Provider}.", header.KeyId, resolution.ProviderName); + throw new InvalidOperationException("Detached JWS signature verification failed."); + } + } + + private static bool TryParseDetachedJws(string value, out string encodedHeader, out string encodedSignature) + { + var parts = value.Split("..", StringSplitOptions.None); + if (parts.Length != 2 || string.IsNullOrEmpty(parts[0]) || string.IsNullOrEmpty(parts[1])) + { + encodedHeader = string.Empty; + encodedSignature = string.Empty; + return false; + } + + encodedHeader = parts[0]; + encodedSignature = parts[1]; + return true; + } + + private static ReadOnlyMemory BuildSigningInput(string encodedHeader, ReadOnlySpan payload) + { + var headerBytes = Encoding.ASCII.GetBytes(encodedHeader); + var buffer = new byte[headerBytes.Length + 1 + payload.Length]; + headerBytes.CopyTo(buffer.AsSpan()); + buffer[headerBytes.Length] = (byte)'.'; + payload.CopyTo(buffer.AsSpan(headerBytes.Length + 1)); + return buffer; + } + + private sealed record MirrorSignatureHeader( + [property: JsonPropertyName("alg")] string Algorithm, + [property: JsonPropertyName("kid")] string KeyId, + [property: JsonPropertyName("provider")] string? Provider, + [property: JsonPropertyName("typ")] string? Type, + [property: JsonPropertyName("b64")] bool Base64Payload, + [property: JsonPropertyName("crit")] string[] Critical); +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/Settings/StellaOpsMirrorConnectorOptions.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Settings/StellaOpsMirrorConnectorOptions.cs new file mode 100644 index 00000000..49058f95 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/Settings/StellaOpsMirrorConnectorOptions.cs @@ -0,0 +1,61 @@ +using System; +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror.Settings; + +/// +/// Configuration for the StellaOps mirror connector HTTP client. +/// +public sealed class StellaOpsMirrorConnectorOptions +{ + /// + /// Base address of the mirror distribution endpoint (e.g., https://mirror.stella-ops.org). + /// + [Required] + public Uri BaseAddress { get; set; } = new("https://mirror.stella-ops.org", UriKind.Absolute); + + /// + /// Relative path to the mirror index document. Defaults to /concelier/exports/index.json. + /// + [Required] + public string IndexPath { get; set; } = "/concelier/exports/index.json"; + + /// + /// Preferred mirror domain identifier when multiple domains are published in the index. + /// + [Required] + public string DomainId { get; set; } = "primary"; + + /// + /// Maximum duration to wait on HTTP requests. + /// + public TimeSpan HttpTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// + /// Signature verification configuration for downloaded bundles. + /// + public SignatureOptions Signature { get; set; } = new(); + + public sealed class SignatureOptions + { + /// + /// When true, downloaded bundles must include a detached JWS that validates successfully. + /// + public bool Enabled { get; set; } = false; + + /// + /// Expected signing key identifier (kid) emitted in the detached JWS header. + /// + public string KeyId { get; set; } = string.Empty; + + /// + /// Optional crypto provider hint used to resolve verification keys. + /// + public string? Provider { get; set; } + + /// + /// Optional path to a PEM-encoded EC public key used to verify signatures when registry resolution fails. + /// + public string? PublicKeyPath { get; set; } + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOps.Concelier.Connector.StellaOpsMirror.csproj b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOps.Concelier.Connector.StellaOpsMirror.csproj new file mode 100644 index 00000000..adf69977 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOps.Concelier.Connector.StellaOpsMirror.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs new file mode 100644 index 00000000..45e236b2 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnector.cs @@ -0,0 +1,288 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.StellaOpsMirror.Client; +using StellaOps.Concelier.Connector.StellaOpsMirror.Internal; +using StellaOps.Concelier.Connector.StellaOpsMirror.Security; +using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror; + +public sealed class StellaOpsMirrorConnector : IFeedConnector +{ + public const string Source = "stellaops-mirror"; + + private readonly MirrorManifestClient _client; + private readonly MirrorSignatureVerifier _signatureVerifier; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly ISourceStateRepository _stateRepository; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly StellaOpsMirrorConnectorOptions _options; + + public StellaOpsMirrorConnector( + MirrorManifestClient client, + MirrorSignatureVerifier signatureVerifier, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + ISourceStateRepository stateRepository, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + ValidateOptions(_options); + } + + public string SourceName => Source; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + _ = services ?? throw new ArgumentNullException(nameof(services)); + + var now = _timeProvider.GetUtcNow(); + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + MirrorIndexDocument index; + try + { + index = await _client.GetIndexAsync(_options.IndexPath, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(15), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var domain = index.Domains.FirstOrDefault(entry => + string.Equals(entry.DomainId, _options.DomainId, StringComparison.OrdinalIgnoreCase)); + + if (domain is null) + { + var message = $"Mirror domain '{_options.DomainId}' not present in index."; + await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(30), message, cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException(message); + } + + if (string.Equals(domain.Bundle.Digest, cursor.BundleDigest, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogInformation("Mirror bundle digest {Digest} unchanged; skipping fetch.", domain.Bundle.Digest); + return; + } + + try + { + await ProcessDomainAsync(index, domain, pendingDocuments, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + await _stateRepository.MarkFailureAsync(Source, now, TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + var updatedCursor = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithBundleSnapshot(domain.Bundle.Path, domain.Bundle.Digest, index.GeneratedAt); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + => Task.CompletedTask; + + private async Task ProcessDomainAsync( + MirrorIndexDocument index, + MirrorIndexDomainEntry domain, + HashSet pendingDocuments, + CancellationToken cancellationToken) + { + var manifestBytes = await _client.DownloadAsync(domain.Manifest.Path, cancellationToken).ConfigureAwait(false); + var bundleBytes = await _client.DownloadAsync(domain.Bundle.Path, cancellationToken).ConfigureAwait(false); + + VerifyDigest(domain.Manifest.Digest, manifestBytes, domain.Manifest.Path); + VerifyDigest(domain.Bundle.Digest, bundleBytes, domain.Bundle.Path); + + if (_options.Signature.Enabled) + { + if (domain.Bundle.Signature is null) + { + throw new InvalidOperationException("Mirror bundle did not include a signature descriptor while verification is enabled."); + } + + var signatureBytes = await _client.DownloadAsync(domain.Bundle.Signature.Path, cancellationToken).ConfigureAwait(false); + var signatureValue = Encoding.UTF8.GetString(signatureBytes); + await _signatureVerifier.VerifyAsync(bundleBytes, signatureValue, cancellationToken).ConfigureAwait(false); + } + + await StoreAsync(domain, index.GeneratedAt, domain.Manifest, manifestBytes, "application/json", DocumentStatuses.Mapped, addToPending: false, pendingDocuments, cancellationToken).ConfigureAwait(false); + var bundleRecord = await StoreAsync(domain, index.GeneratedAt, domain.Bundle, bundleBytes, "application/json", DocumentStatuses.PendingParse, addToPending: true, pendingDocuments, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Stored mirror bundle {Uri} as document {DocumentId} with digest {Digest}.", + bundleRecord.Uri, + bundleRecord.Id, + bundleRecord.Sha256); + } + + private async Task StoreAsync( + MirrorIndexDomainEntry domain, + DateTimeOffset generatedAt, + MirrorFileDescriptor descriptor, + byte[] payload, + string contentType, + string status, + bool addToPending, + HashSet pendingDocuments, + CancellationToken cancellationToken) + { + var absolute = ResolveAbsolutePath(descriptor.Path); + + var existing = await _documentStore.FindBySourceAndUriAsync(Source, absolute, cancellationToken).ConfigureAwait(false); + if (existing is not null && string.Equals(existing.Sha256, NormalizeDigest(descriptor.Digest), StringComparison.OrdinalIgnoreCase)) + { + if (addToPending) + { + pendingDocuments.Add(existing.Id); + } + + return existing; + } + + var gridFsId = await _rawDocumentStorage.UploadAsync(Source, absolute, payload, contentType, cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + var sha = ComputeSha256(payload); + + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["mirror.domainId"] = domain.DomainId, + ["mirror.displayName"] = domain.DisplayName, + ["mirror.path"] = descriptor.Path, + ["mirror.digest"] = NormalizeDigest(descriptor.Digest), + ["mirror.type"] = ReferenceEquals(descriptor, domain.Bundle) ? "bundle" : "manifest", + }; + + var record = new DocumentRecord( + existing?.Id ?? Guid.NewGuid(), + Source, + absolute, + now, + sha, + status, + contentType, + Headers: null, + Metadata: metadata, + Etag: null, + LastModified: generatedAt, + GridFsId: gridFsId, + ExpiresAt: null); + + var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); + + if (addToPending) + { + pendingDocuments.Add(upserted.Id); + } + + return upserted; + } + + private string ResolveAbsolutePath(string path) + { + var uri = new Uri(_options.BaseAddress, path); + return uri.ToString(); + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(Source, cancellationToken).ConfigureAwait(false); + return state is null ? StellaOpsMirrorCursor.Empty : StellaOpsMirrorCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(StellaOpsMirrorCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBsonDocument(); + var now = _timeProvider.GetUtcNow(); + await _stateRepository.UpdateCursorAsync(Source, document, now, cancellationToken).ConfigureAwait(false); + } + + private static void VerifyDigest(string expected, ReadOnlySpan payload, string path) + { + if (string.IsNullOrWhiteSpace(expected)) + { + return; + } + + if (!expected.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Unsupported digest '{expected}' for '{path}'."); + } + + var actualHash = SHA256.HashData(payload); + var actual = "sha256:" + Convert.ToHexString(actualHash).ToLowerInvariant(); + if (!string.Equals(actual, expected, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Digest mismatch for '{path}'. Expected {expected}, computed {actual}."); + } + } + + private static string ComputeSha256(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string NormalizeDigest(string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return string.Empty; + } + + return digest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) + ? digest[7..] + : digest.ToLowerInvariant(); + } + + private static void ValidateOptions(StellaOpsMirrorConnectorOptions options) + { + if (options.BaseAddress is null || !options.BaseAddress.IsAbsoluteUri) + { + throw new InvalidOperationException("Mirror connector requires an absolute baseAddress."); + } + + if (string.IsNullOrWhiteSpace(options.DomainId)) + { + throw new InvalidOperationException("Mirror connector requires domainId to be specified."); + } + } +} + +file static class UriExtensions +{ + public static Uri Combine(this Uri baseUri, string relative) + => new(baseUri, relative); +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnectorPlugin.cs new file mode 100644 index 00000000..ba3c8bbf --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorConnectorPlugin.cs @@ -0,0 +1,19 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror; + +public sealed class StellaOpsMirrorConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = StellaOpsMirrorConnector.Source; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs new file mode 100644 index 00000000..91ef741d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/StellaOpsMirrorDependencyInjectionRoutine.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.StellaOpsMirror.Client; +using StellaOps.Concelier.Connector.StellaOpsMirror.Security; +using StellaOps.Concelier.Connector.StellaOpsMirror.Settings; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.DependencyInjection; + +namespace StellaOps.Concelier.Connector.StellaOpsMirror; + +public sealed class StellaOpsMirrorDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:stellaopsMirror"; + private const string HttpClientName = "stellaops-mirror"; + private static readonly TimeSpan FetchTimeout = TimeSpan.FromMinutes(5); + private static readonly TimeSpan LeaseDuration = TimeSpan.FromMinutes(4); + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(options => + { + if (options.BaseAddress is null) + { + throw new InvalidOperationException("stellaopsMirror.baseAddress must be configured."); + } + }) + .ValidateOnStart(); + + services.AddSourceCommon(); + + services.AddHttpClient(HttpClientName, (sp, client) => + { + var options = sp.GetRequiredService>().Value; + client.BaseAddress = options.BaseAddress; + client.Timeout = options.HttpTimeout; + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.Accept.Add(new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json")); + }); + + services.AddTransient(sp => + { + var factory = sp.GetRequiredService(); + var httpClient = factory.CreateClient(HttpClientName); + return ActivatorUtilities.CreateInstance(sp, httpClient); + }); + + services.TryAddSingleton(); + services.AddTransient(); + + var scheduler = new JobSchedulerBuilder(services); + scheduler.AddJob( + StellaOpsMirrorJobKinds.Fetch, + cronExpression: "*/15 * * * *", + timeout: FetchTimeout, + leaseDuration: LeaseDuration); + scheduler.AddJob( + StellaOpsMirrorJobKinds.Parse, + cronExpression: null, + timeout: TimeSpan.FromMinutes(5), + leaseDuration: LeaseDuration, + enabled: false); + scheduler.AddJob( + StellaOpsMirrorJobKinds.Map, + cronExpression: null, + timeout: TimeSpan.FromMinutes(5), + leaseDuration: LeaseDuration, + enabled: false); + + return services; + } +} diff --git a/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md b/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md new file mode 100644 index 00000000..7df65210 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.StellaOpsMirror/TASKS.md @@ -0,0 +1,7 @@ +# StellaOps Mirror Connector Task Board (Sprint 8) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| FEEDCONN-STELLA-08-001 | DOING (2025-10-19) | BE-Conn-Stella | CONCELIER-EXPORT-08-201 | Implement Concelier mirror fetcher hitting `https://.stella-ops.org/concelier/exports/index.json`, verify signatures/digests, and persist raw documents with provenance. | Fetch job downloads mirror manifest, verifies digest/signature, stores raw docs with tests covering happy-path + tampered manifest. *(In progress: HTTP client + detached JWS verifier scaffolding landed.)* | +| FEEDCONN-STELLA-08-002 | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-001 | Map mirror payloads into canonical advisory DTOs with provenance referencing mirror domain + original source metadata. | Mapper produces advisories/aliases/affected with mirror provenance; fixtures assert canonical parity with upstream JSON exporters. | +| FEEDCONN-STELLA-08-003 | TODO | BE-Conn-Stella | FEEDCONN-STELLA-08-002 | Add incremental cursor + resume support (per-export fingerprint) and document configuration for downstream Concelier instances. | Connector resumes from last export, handles deletion/delta cases, docs updated with config sample; integration test covers resume + new export scenario. | diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs index d3f798b1..3c7b58a2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/AdobeConnectorFetchTests.cs @@ -16,20 +16,20 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Vndr.Adobe; -using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Vndr.Adobe; +using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Tests; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Tests; [Collection("mongo-fixture")] public sealed class AdobeConnectorFetchTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-advisories.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-85.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-detail-apsb25-87.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/Adobe/Fixtures/adobe-index.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests.csproj similarity index 53% rename from src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj rename to src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests.csproj index 2446f91a..801f1a05 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe.Tests/StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe.Tests/StellaOps.Concelier.Connector.Vndr.Adobe.Tests.csproj @@ -5,10 +5,10 @@ enable - - - - + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AGENTS.md similarity index 81% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/AGENTS.md index 227fc8b1..0e8afc6a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AGENTS.md @@ -20,9 +20,9 @@ Adobe PSIRT connector ingesting APSB/APA advisories; authoritative for Adobe pro In: PSIRT ingestion, aliases, affected plus fixedBy, psirt_flags, watermark/resume. Out: signing, package artifact downloads, non-Adobe product truth. ## Observability & security expectations -- Metrics: SourceDiagnostics produces `feedser.source.http.*` counters/histograms tagged `feedser.source=adobe`; operators filter on that tag to monitor fetch counts, parse failures, map affected counts, and cursor movement without bespoke metric names. +- Metrics: SourceDiagnostics produces `concelier.source.http.*` counters/histograms tagged `concelier.source=adobe`; operators filter on that tag to monitor fetch counts, parse failures, map affected counts, and cursor movement without bespoke metric names. - Logs: advisory ids, product counts, extraction timings; hosts allowlisted; no secret logging. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Adobe.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Adobe.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs index e684a057..566952bc 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnector.cs @@ -9,21 +9,21 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Json.Schema; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Json; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; -using StellaOps.Feedser.Source.Vndr.Adobe.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Json; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; +using StellaOps.Concelier.Connector.Vndr.Adobe.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Adobe; +namespace StellaOps.Concelier.Connector.Vndr.Adobe; public sealed class AdobeConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnectorPlugin.cs index 5f3fe442..e6f13dcb 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Adobe; +namespace StellaOps.Concelier.Connector.Vndr.Adobe; public sealed class VndrAdobeConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDiagnostics.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDiagnostics.cs index a6bcdacf..ba57ac44 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeDiagnostics.cs @@ -1,11 +1,11 @@ using System; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Vndr.Adobe; +namespace StellaOps.Concelier.Connector.Vndr.Adobe; public sealed class AdobeDiagnostics : IDisposable { - public const string MeterName = "StellaOps.Feedser.Source.Vndr.Adobe"; + public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Adobe"; private static readonly string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeServiceCollectionExtensions.cs similarity index 82% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeServiceCollectionExtensions.cs index 77f3f581..8398ccd1 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/AdobeServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/AdobeServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Adobe; +namespace StellaOps.Concelier.Connector.Vndr.Adobe; public static class AdobeServiceCollectionExtensions { @@ -21,7 +21,7 @@ public static class AdobeServiceCollectionExtensions { var adobeOptions = sp.GetRequiredService>().Value; options.BaseAddress = adobeOptions.IndexUri; - options.UserAgent = "StellaOps.Feedser.VndrAdobe/1.0"; + options.UserAgent = "StellaOps.Concelier.VndrAdobe/1.0"; options.Timeout = TimeSpan.FromSeconds(20); options.AllowedHosts.Clear(); options.AllowedHosts.Add(adobeOptions.IndexUri.Host); diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Configuration/AdobeOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Configuration/AdobeOptions.cs index d92a02ee..627a1c77 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Configuration/AdobeOptions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Configuration/AdobeOptions.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Configuration; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Configuration; public sealed class AdobeOptions { diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeBulletinDto.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeBulletinDto.cs index 6afd0658..95e581dc 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeBulletinDto.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeBulletinDto.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal sealed record AdobeBulletinDto( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeCursor.cs index e24f17ab..508aa83e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeCursor.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal sealed record AdobeCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDetailParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDetailParser.cs index fdc83bec..a2694c9b 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDetailParser.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDetailParser.cs @@ -7,7 +7,7 @@ using AngleSharp.Dom; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal static class AdobeDetailParser { diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs index 616afe37..e5981ed9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeDocumentMetadata.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal sealed record AdobeDocumentMetadata( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexEntry.cs similarity index 67% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexEntry.cs index baebbff7..ddcaa615 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexEntry.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexEntry.cs @@ -1,5 +1,5 @@ using System; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal sealed record AdobeIndexEntry(string AdvisoryId, Uri DetailUri, DateTimeOffset PublishedUtc, string? Title); diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexParser.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexParser.cs index 738e1044..6802dcfa 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeIndexParser.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeIndexParser.cs @@ -6,7 +6,7 @@ using System.Text.RegularExpressions; using AngleSharp.Dom; using AngleSharp.Html.Parser; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal static class AdobeIndexParser { diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeSchemaProvider.cs similarity index 78% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeSchemaProvider.cs index 6f599163..a33e2d89 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/Internal/AdobeSchemaProvider.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Internal/AdobeSchemaProvider.cs @@ -3,7 +3,7 @@ using System.Reflection; using System.Threading; using Json.Schema; -namespace StellaOps.Feedser.Source.Vndr.Adobe.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Adobe.Internal; internal static class AdobeSchemaProvider { @@ -14,7 +14,7 @@ internal static class AdobeSchemaProvider private static JsonSchema Load() { var assembly = typeof(AdobeSchemaProvider).GetTypeInfo().Assembly; - const string resourceName = "StellaOps.Feedser.Source.Vndr.Adobe.Schemas.adobe-bulletin.schema.json"; + const string resourceName = "StellaOps.Concelier.Connector.Vndr.Adobe.Schemas.adobe-bulletin.schema.json"; using var stream = assembly.GetManifestResourceStream(resourceName) ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json b/src/StellaOps.Concelier.Connector.Vndr.Adobe/Schemas/adobe-bulletin.schema.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/Schemas/adobe-bulletin.schema.json rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/Schemas/adobe-bulletin.schema.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj b/src/StellaOps.Concelier.Connector.Vndr.Adobe/StellaOps.Concelier.Connector.Vndr.Adobe.csproj similarity index 58% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/StellaOps.Concelier.Connector.Vndr.Adobe.csproj index 5304f8a0..bf9c782a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Adobe/StellaOps.Feedser.Source.Vndr.Adobe.csproj +++ b/src/StellaOps.Concelier.Connector.Vndr.Adobe/StellaOps.Concelier.Connector.Vndr.Adobe.csproj @@ -17,9 +17,9 @@ - - - + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Adobe/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Adobe/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs index fd8b0eb6..f288e6df 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleConnectorTests.cs @@ -1,259 +1,259 @@ -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Source.Vndr.Apple; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Feedser.Testing; -using Xunit; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Tests; - -[Collection("mongo-fixture")] -public sealed class AppleConnectorTests : IAsyncLifetime -{ - private static readonly Uri IndexUri = new("https://support.example.com/index.json"); - private static readonly Uri DetailBaseUri = new("https://support.example.com/en-us/"); - - private readonly MongoIntegrationFixture _fixture; - private readonly FakeTimeProvider _timeProvider; - - public AppleConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); - } - - [Fact] - public async Task FetchParseMap_EndToEnd_ProducesCanonicalAdvisories() - { - var handler = new CannedHttpMessageHandler(); - SeedIndex(handler); - SeedDetail(handler); - - await using var provider = await BuildServiceProviderAsync(handler); - var connector = provider.GetRequiredService(); - - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); - Assert.Equal(5, advisories.Count); - - var advisoriesByKey = advisories.ToDictionary(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal); - - var iosBaseline = advisoriesByKey["125326"]; - Assert.Contains("CVE-2025-43400", iosBaseline.Aliases, StringComparer.OrdinalIgnoreCase); - Assert.Equal(3, iosBaseline.AffectedPackages.Count()); - AssertPackageDetails(iosBaseline, "iPhone 16 Pro", "26.0.1", "24A341", "iOS"); - AssertPackageDetails(iosBaseline, "iPhone 16 Pro", "26.0.1 (a)", "24A341a", "iOS"); - AssertPackageDetails(iosBaseline, "iPad Pro (M4)", "26", "24B120", "iPadOS"); - - var macBaseline = advisoriesByKey["125328"]; - Assert.Contains("CVE-2025-43400", macBaseline.Aliases, StringComparer.OrdinalIgnoreCase); - Assert.Equal(2, macBaseline.AffectedPackages.Count()); - AssertPackageDetails(macBaseline, "MacBook Pro (M4)", "26.0.1", "26A123", "macOS"); - AssertPackageDetails(macBaseline, "Mac Studio", "26", "26A120b", "macOS"); - - var venturaRsr = advisoriesByKey["106355"]; - Assert.Contains("CVE-2023-37450", venturaRsr.Aliases, StringComparer.OrdinalIgnoreCase); - Assert.Equal(2, venturaRsr.AffectedPackages.Count()); - AssertPackageDetails(venturaRsr, "macOS Ventura", string.Empty, "22F400", "macOS Ventura"); - AssertPackageDetails(venturaRsr, "macOS Ventura (Intel)", string.Empty, "22F400a", "macOS Ventura"); - - var visionOs = advisoriesByKey["HT214108"]; - Assert.Contains("CVE-2024-27800", visionOs.Aliases, StringComparer.OrdinalIgnoreCase); - Assert.False(visionOs.AffectedPackages.Any()); - - var rsrAdvisory = advisoriesByKey["HT215500"]; - Assert.Contains("CVE-2025-2468", rsrAdvisory.Aliases, StringComparer.OrdinalIgnoreCase); - var rsrPackage = AssertPackageDetails(rsrAdvisory, "iPhone 15 Pro", "18.0.1 (c)", "22A123c", "iOS"); - Assert.True(rsrPackage.NormalizedVersions.IsDefaultOrEmpty || rsrPackage.NormalizedVersions.Length == 0); - - var flagStore = provider.GetRequiredService(); - var rsrFlag = await flagStore.FindAsync("HT215500", CancellationToken.None); - Assert.NotNull(rsrFlag); - } - - - - private static AffectedPackage AssertPackageDetails( - Advisory advisory, - string identifier, - string expectedRangeExpression, - string? expectedBuild, - string expectedPlatform) - { - var candidates = advisory.AffectedPackages - .Where(package => string.Equals(package.Identifier, identifier, StringComparison.Ordinal)) - .ToList(); - Assert.NotEmpty(candidates); - - var package = Assert.Single(candidates, candidate => - { - var rangeCandidate = candidate.VersionRanges.SingleOrDefault(); - return rangeCandidate is not null - && string.Equals(rangeCandidate.RangeExpression ?? string.Empty, expectedRangeExpression, StringComparison.Ordinal); - }); - - Assert.Equal(expectedPlatform, package.Platform); - - var range = Assert.Single(package.VersionRanges); - Assert.Equal(expectedRangeExpression, range.RangeExpression ?? string.Empty); - Assert.Equal("vendor", range.RangeKind); - - Assert.NotNull(range.Primitives); - Assert.NotNull(range.Primitives!.VendorExtensions); - var vendorExtensions = range.Primitives.VendorExtensions; - - if (!string.IsNullOrWhiteSpace(expectedRangeExpression)) - { - if (!vendorExtensions.TryGetValue("apple.version.raw", out var rawVersion)) - { - throw new Xunit.Sdk.XunitException($"Missing apple.version.raw for {identifier}; available keys: {string.Join(", ", vendorExtensions.Keys)}"); - } - - Assert.Equal(expectedRangeExpression, rawVersion); - } - else - { - Assert.False(vendorExtensions.ContainsKey("apple.version.raw")); - } - - if (!string.IsNullOrWhiteSpace(expectedPlatform)) - { - if (vendorExtensions.TryGetValue("apple.platform", out var platformExtension)) - { - Assert.Equal(expectedPlatform, platformExtension); - } - else - { - throw new Xunit.Sdk.XunitException($"Missing apple.platform extension for {identifier}; available keys: {string.Join(", ", vendorExtensions.Keys)}"); - } - } - - if (!string.IsNullOrWhiteSpace(expectedBuild)) - { - Assert.True(vendorExtensions.TryGetValue("apple.build", out var buildExtension)); - Assert.Equal(expectedBuild, buildExtension); - } - else - { - Assert.False(vendorExtensions.ContainsKey("apple.build")); - } - - if (PackageCoordinateHelper.TryParseSemVer(expectedRangeExpression, out _, out var normalized)) - { - var normalizedRule = Assert.Single(package.NormalizedVersions); - Assert.Equal(NormalizedVersionSchemes.SemVer, normalizedRule.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, normalizedRule.Type); - Assert.Equal(normalized, normalizedRule.Max); - Assert.Equal($"apple:{expectedPlatform}:{identifier}", normalizedRule.Notes); - } - else - { - Assert.True(package.NormalizedVersions.IsDefaultOrEmpty || package.NormalizedVersions.Length == 0); - } - - return package; - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; - - private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler) - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_timeProvider); - services.AddSingleton(handler); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddAppleConnector(opts => - { - opts.SoftwareLookupUri = IndexUri; - opts.AdvisoryBaseUri = DetailBaseUri; - opts.LocaleSegment = "en-us"; - opts.InitialBackfill = TimeSpan.FromDays(120); - opts.ModifiedTolerance = TimeSpan.FromHours(2); - opts.MaxAdvisoriesPerFetch = 10; - }); - - services.Configure(AppleOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private static void SeedIndex(CannedHttpMessageHandler handler) - { - handler.AddJsonResponse(IndexUri, ReadFixture("index.json")); - } - - private static void SeedDetail(CannedHttpMessageHandler handler) - { - AddHtmlResponse(handler, new Uri(DetailBaseUri, "125326"), "125326.html"); - AddHtmlResponse(handler, new Uri(DetailBaseUri, "125328"), "125328.html"); - AddHtmlResponse(handler, new Uri(DetailBaseUri, "106355"), "106355.html"); - AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT214108"), "ht214108.html"); - AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT215500"), "ht215500.html"); - } - - private static void AddHtmlResponse(CannedHttpMessageHandler handler, Uri uri, string fixture) - { - handler.AddResponse(uri, () => - { - var content = ReadFixture(fixture); - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(content, Encoding.UTF8, "text/html"), - }; - }); - } - - private static string ReadFixture(string name) - { - var path = Path.Combine( - AppContext.BaseDirectory, - "Source", - "Vndr", - "Apple", - "Fixtures", - name); - return File.ReadAllText(path); - } -} +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Connector.Vndr.Apple; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Tests; + +[Collection("mongo-fixture")] +public sealed class AppleConnectorTests : IAsyncLifetime +{ + private static readonly Uri IndexUri = new("https://support.example.com/index.json"); + private static readonly Uri DetailBaseUri = new("https://support.example.com/en-us/"); + + private readonly MongoIntegrationFixture _fixture; + private readonly FakeTimeProvider _timeProvider; + + public AppleConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + } + + [Fact] + public async Task FetchParseMap_EndToEnd_ProducesCanonicalAdvisories() + { + var handler = new CannedHttpMessageHandler(); + SeedIndex(handler); + SeedDetail(handler); + + await using var provider = await BuildServiceProviderAsync(handler); + var connector = provider.GetRequiredService(); + + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None); + Assert.Equal(5, advisories.Count); + + var advisoriesByKey = advisories.ToDictionary(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal); + + var iosBaseline = advisoriesByKey["125326"]; + Assert.Contains("CVE-2025-43400", iosBaseline.Aliases, StringComparer.OrdinalIgnoreCase); + Assert.Equal(3, iosBaseline.AffectedPackages.Count()); + AssertPackageDetails(iosBaseline, "iPhone 16 Pro", "26.0.1", "24A341", "iOS"); + AssertPackageDetails(iosBaseline, "iPhone 16 Pro", "26.0.1 (a)", "24A341a", "iOS"); + AssertPackageDetails(iosBaseline, "iPad Pro (M4)", "26", "24B120", "iPadOS"); + + var macBaseline = advisoriesByKey["125328"]; + Assert.Contains("CVE-2025-43400", macBaseline.Aliases, StringComparer.OrdinalIgnoreCase); + Assert.Equal(2, macBaseline.AffectedPackages.Count()); + AssertPackageDetails(macBaseline, "MacBook Pro (M4)", "26.0.1", "26A123", "macOS"); + AssertPackageDetails(macBaseline, "Mac Studio", "26", "26A120b", "macOS"); + + var venturaRsr = advisoriesByKey["106355"]; + Assert.Contains("CVE-2023-37450", venturaRsr.Aliases, StringComparer.OrdinalIgnoreCase); + Assert.Equal(2, venturaRsr.AffectedPackages.Count()); + AssertPackageDetails(venturaRsr, "macOS Ventura", string.Empty, "22F400", "macOS Ventura"); + AssertPackageDetails(venturaRsr, "macOS Ventura (Intel)", string.Empty, "22F400a", "macOS Ventura"); + + var visionOs = advisoriesByKey["HT214108"]; + Assert.Contains("CVE-2024-27800", visionOs.Aliases, StringComparer.OrdinalIgnoreCase); + Assert.False(visionOs.AffectedPackages.Any()); + + var rsrAdvisory = advisoriesByKey["HT215500"]; + Assert.Contains("CVE-2025-2468", rsrAdvisory.Aliases, StringComparer.OrdinalIgnoreCase); + var rsrPackage = AssertPackageDetails(rsrAdvisory, "iPhone 15 Pro", "18.0.1 (c)", "22A123c", "iOS"); + Assert.True(rsrPackage.NormalizedVersions.IsDefaultOrEmpty || rsrPackage.NormalizedVersions.Length == 0); + + var flagStore = provider.GetRequiredService(); + var rsrFlag = await flagStore.FindAsync("HT215500", CancellationToken.None); + Assert.NotNull(rsrFlag); + } + + + + private static AffectedPackage AssertPackageDetails( + Advisory advisory, + string identifier, + string expectedRangeExpression, + string? expectedBuild, + string expectedPlatform) + { + var candidates = advisory.AffectedPackages + .Where(package => string.Equals(package.Identifier, identifier, StringComparison.Ordinal)) + .ToList(); + Assert.NotEmpty(candidates); + + var package = Assert.Single(candidates, candidate => + { + var rangeCandidate = candidate.VersionRanges.SingleOrDefault(); + return rangeCandidate is not null + && string.Equals(rangeCandidate.RangeExpression ?? string.Empty, expectedRangeExpression, StringComparison.Ordinal); + }); + + Assert.Equal(expectedPlatform, package.Platform); + + var range = Assert.Single(package.VersionRanges); + Assert.Equal(expectedRangeExpression, range.RangeExpression ?? string.Empty); + Assert.Equal("vendor", range.RangeKind); + + Assert.NotNull(range.Primitives); + Assert.NotNull(range.Primitives!.VendorExtensions); + var vendorExtensions = range.Primitives.VendorExtensions; + + if (!string.IsNullOrWhiteSpace(expectedRangeExpression)) + { + if (!vendorExtensions.TryGetValue("apple.version.raw", out var rawVersion)) + { + throw new Xunit.Sdk.XunitException($"Missing apple.version.raw for {identifier}; available keys: {string.Join(", ", vendorExtensions.Keys)}"); + } + + Assert.Equal(expectedRangeExpression, rawVersion); + } + else + { + Assert.False(vendorExtensions.ContainsKey("apple.version.raw")); + } + + if (!string.IsNullOrWhiteSpace(expectedPlatform)) + { + if (vendorExtensions.TryGetValue("apple.platform", out var platformExtension)) + { + Assert.Equal(expectedPlatform, platformExtension); + } + else + { + throw new Xunit.Sdk.XunitException($"Missing apple.platform extension for {identifier}; available keys: {string.Join(", ", vendorExtensions.Keys)}"); + } + } + + if (!string.IsNullOrWhiteSpace(expectedBuild)) + { + Assert.True(vendorExtensions.TryGetValue("apple.build", out var buildExtension)); + Assert.Equal(expectedBuild, buildExtension); + } + else + { + Assert.False(vendorExtensions.ContainsKey("apple.build")); + } + + if (PackageCoordinateHelper.TryParseSemVer(expectedRangeExpression, out _, out var normalized)) + { + var normalizedRule = Assert.Single(package.NormalizedVersions); + Assert.Equal(NormalizedVersionSchemes.SemVer, normalizedRule.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, normalizedRule.Type); + Assert.Equal(normalized, normalizedRule.Max); + Assert.Equal($"apple:{expectedPlatform}:{identifier}", normalizedRule.Notes); + } + else + { + Assert.True(package.NormalizedVersions.IsDefaultOrEmpty || package.NormalizedVersions.Length == 0); + } + + return package; + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + private async Task BuildServiceProviderAsync(CannedHttpMessageHandler handler) + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_timeProvider); + services.AddSingleton(handler); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddAppleConnector(opts => + { + opts.SoftwareLookupUri = IndexUri; + opts.AdvisoryBaseUri = DetailBaseUri; + opts.LocaleSegment = "en-us"; + opts.InitialBackfill = TimeSpan.FromDays(120); + opts.ModifiedTolerance = TimeSpan.FromHours(2); + opts.MaxAdvisoriesPerFetch = 10; + }); + + services.Configure(AppleOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private static void SeedIndex(CannedHttpMessageHandler handler) + { + handler.AddJsonResponse(IndexUri, ReadFixture("index.json")); + } + + private static void SeedDetail(CannedHttpMessageHandler handler) + { + AddHtmlResponse(handler, new Uri(DetailBaseUri, "125326"), "125326.html"); + AddHtmlResponse(handler, new Uri(DetailBaseUri, "125328"), "125328.html"); + AddHtmlResponse(handler, new Uri(DetailBaseUri, "106355"), "106355.html"); + AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT214108"), "ht214108.html"); + AddHtmlResponse(handler, new Uri(DetailBaseUri, "HT215500"), "ht215500.html"); + } + + private static void AddHtmlResponse(CannedHttpMessageHandler handler, Uri uri, string fixture) + { + handler.AddResponse(uri, () => + { + var content = ReadFixture(fixture); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(content, Encoding.UTF8, "text/html"), + }; + }); + } + + private static string ReadFixture(string name) + { + var path = Path.Combine( + AppContext.BaseDirectory, + "Source", + "Vndr", + "Apple", + "Fixtures", + name); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs index 87e580ea..97466dff 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleFixtureManager.cs @@ -1,342 +1,342 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Source.Vndr.Apple.Internal; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Tests.Apple; - -internal static class AppleFixtureManager -{ - private const string UpdateEnvVar = "UPDATE_APPLE_FIXTURES"; - private const string UpdateSentinelFileName = ".update-apple-fixtures"; - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - WriteIndented = true, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private static readonly Uri ExampleDetailBaseUri = new("https://support.example.com/en-us/"); - - private static readonly AppleFixtureDefinition[] Definitions = - { - new( - "125326", - new Uri("https://support.apple.com/en-us/125326"), - Products: new[] - { - new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1", "24A341"), - new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1 (a)", "24A341a"), - new AppleFixtureProduct("iPadOS", "iPad Pro (M4)", "26", "24B120"), - }), - new( - "125328", - new Uri("https://support.apple.com/en-us/125328"), - Products: new[] - { - new AppleFixtureProduct("macOS", "MacBook Pro (M4)", "26.0.1", "26A123"), - new AppleFixtureProduct("macOS", "Mac Studio", "26", "26A120b"), - }), - new( - "106355", - new Uri("https://support.apple.com/en-us/106355"), - ForceRapidSecurityResponse: true, - Products: new[] - { - new AppleFixtureProduct("macOS Ventura", "macOS Ventura", string.Empty, "22F400"), - new AppleFixtureProduct("macOS Ventura", "macOS Ventura (Intel)", string.Empty, "22F400a"), - }), - new( - "HT214108", - new Uri("https://support.apple.com/en-us/HT214108")), - new( - "HT215500", - new Uri("https://support.apple.com/en-us/HT215500"), - ForceRapidSecurityResponse: true), - }; - - private static readonly Lazy UpdateTask = new( - () => UpdateFixturesAsync(CancellationToken.None), - LazyThreadSafetyMode.ExecutionAndPublication); - - public static IReadOnlyList Fixtures => Definitions; - - public static Task EnsureUpdatedAsync(CancellationToken cancellationToken = default) - { - if (!ShouldUpdateFixtures()) - { - return Task.CompletedTask; - } - - Console.WriteLine("[AppleFixtures] UPDATE flag detected; refreshing fixtures"); - return UpdateTask.Value; - } - - public static string ReadFixture(string name) - { - var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); - return File.ReadAllText(path); - } - - public static AppleDetailDto ReadExpectedDto(string articleId) - { - var json = ReadFixture($"{articleId}.expected.json"); - return JsonSerializer.Deserialize(json, SerializerOptions) - ?? throw new InvalidOperationException($"Unable to deserialize expected DTO for {articleId}."); - } - - private static bool ShouldUpdateFixtures() - { - var value = Environment.GetEnvironmentVariable(UpdateEnvVar)?.Trim(); - if (string.IsNullOrEmpty(value)) - { - var sentinelPath = Path.Combine(ResolveFixtureRoot(), UpdateSentinelFileName); - return File.Exists(sentinelPath); - } - - if (string.Equals(value, "0", StringComparison.Ordinal) - || string.Equals(value, "false", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - return true; - } - - private static async Task UpdateFixturesAsync(CancellationToken cancellationToken) - { - var handler = new HttpClientHandler - { - AutomaticDecompression = System.Net.DecompressionMethods.All, - ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator, - }; - - using var httpClient = new HttpClient(handler); - httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsFixtureUpdater", "1.0")); - httpClient.Timeout = TimeSpan.FromSeconds(30); - - var indexEntries = new List(Definitions.Length); - - try - { - foreach (var definition in Definitions) - { - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var html = await httpClient.GetStringAsync(definition.DetailUri, cancellationToken).ConfigureAwait(false); - var entryProducts = definition.Products? - .Select(product => new AppleIndexProduct( - product.Platform, - product.Name, - product.Version, - product.Build)) - .ToArray() ?? Array.Empty(); - - var entry = new AppleIndexEntry( - UpdateId: definition.ArticleId, - ArticleId: definition.ArticleId, - Title: definition.ArticleId, - PostingDate: DateTimeOffset.UtcNow, - DetailUri: definition.DetailUri, - Products: entryProducts, - IsRapidSecurityResponse: definition.ForceRapidSecurityResponse ?? false); - - var dto = AppleDetailParser.Parse(html, entry); - var sanitizedHtml = BuildSanitizedHtml(dto); - - WriteFixture(definition.HtmlFixtureName, sanitizedHtml); - WriteFixture(definition.ExpectedFixtureName, JsonSerializer.Serialize(dto, SerializerOptions)); - - var exampleDetailUri = new Uri(ExampleDetailBaseUri, definition.ArticleId); - indexEntries.Add(new - { - id = definition.ArticleId, - articleId = definition.ArticleId, - title = dto.Title, - postingDate = dto.Published.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture), - detailUrl = exampleDetailUri.ToString(), - rapidSecurityResponse = definition.ForceRapidSecurityResponse - ?? dto.Title.Contains("Rapid Security Response", StringComparison.OrdinalIgnoreCase), - products = dto.Affected - .OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase) - .Select(p => new - { - platform = p.Platform, - name = p.Name, - version = p.Version, - build = p.Build, - }) - .ToArray(), - }); - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException) - { - Console.WriteLine($"[AppleFixtures] Skipped {definition.ArticleId}: {ex.Message}"); - } - } - } - finally - { - var sentinelPath = Path.Combine(ResolveFixtureRoot(), UpdateSentinelFileName); - if (File.Exists(sentinelPath)) - { - try - { - File.Delete(sentinelPath); - } - catch (IOException) - { - // best effort - } - } - } - - var indexDocument = new { updates = indexEntries }; - WriteFixture("index.json", JsonSerializer.Serialize(indexDocument, SerializerOptions)); - } - - private static string BuildSanitizedHtml(AppleDetailDto dto) - { - var builder = new StringBuilder(); - builder.AppendLine(""); - builder.AppendLine(""); - builder.AppendLine(""); - builder.AppendLine("
      "); - - builder.AppendLine($"

      {Escape(dto.Title)}

      "); - - if (!string.IsNullOrWhiteSpace(dto.Summary)) - { - builder.AppendLine($"

      {Escape(dto.Summary)}

      "); - } - - var publishedDisplay = dto.Published.ToString("MMMM d, yyyy", CultureInfo.InvariantCulture); - builder.AppendLine($" "); - - if (dto.Updated.HasValue) - { - var updatedDisplay = dto.Updated.Value.ToString("MMMM d, yyyy", CultureInfo.InvariantCulture); - builder.AppendLine($" "); - } - - if (dto.CveIds.Count > 0) - { - builder.AppendLine("
      "); - builder.AppendLine("

      Security Issues

      "); - builder.AppendLine("
        "); - foreach (var cve in dto.CveIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)) - { - builder.AppendLine($"
      • {Escape(cve)}
        • "); - } - builder.AppendLine("
      "); - builder.AppendLine("
      "); - } - - if (dto.Affected.Count > 0) - { - builder.AppendLine(" "); - builder.AppendLine(" "); - foreach (var product in dto.Affected - .OrderBy(p => p.Platform ?? string.Empty, StringComparer.OrdinalIgnoreCase) - .ThenBy(p => p.Name, StringComparer.OrdinalIgnoreCase) - .ThenBy(p => p.Version ?? string.Empty, StringComparer.OrdinalIgnoreCase) - .ThenBy(p => p.Build ?? string.Empty, StringComparer.OrdinalIgnoreCase)) - { - var platform = product.Platform is null ? string.Empty : Escape(product.Platform); - var name = Escape(product.Name); - var version = product.Version is null ? string.Empty : Escape(product.Version); - var build = product.Build is null ? string.Empty : Escape(product.Build); - - builder.Append(" "); - builder.AppendLine(); - builder.AppendLine($" "); - builder.AppendLine($" "); - builder.AppendLine($" "); - builder.AppendLine(" "); - } - - builder.AppendLine(" "); - builder.AppendLine("
      {name}{version}{build}
      "); - } - - if (dto.References.Count > 0) - { - builder.AppendLine("
      "); - builder.AppendLine("

      References

      "); - foreach (var reference in dto.References - .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase)) - { - var title = reference.Title ?? string.Empty; - builder.AppendLine($" {Escape(title)}"); - } - builder.AppendLine("
      "); - } - - builder.AppendLine("
      "); - builder.AppendLine(""); - builder.AppendLine(""); - return builder.ToString(); - } - - private static void WriteFixture(string name, string contents) - { - var root = ResolveFixtureRoot(); - Directory.CreateDirectory(root); - var normalized = NormalizeLineEndings(contents); - - var sourcePath = Path.Combine(root, name); - File.WriteAllText(sourcePath, normalized); - - var outputPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); - Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); - File.WriteAllText(outputPath, normalized); - - Console.WriteLine($"[AppleFixtures] Wrote {name}"); - } - - private static string ResolveFixtureRoot() - { - var baseDir = AppContext.BaseDirectory; - // bin/Debug/net10.0/ -> project -> src -> repo root - var root = Path.GetFullPath(Path.Combine(baseDir, "..", "..", "..", "..", "..")); - return Path.Combine(root, "src", "StellaOps.Feedser.Source.Vndr.Apple.Tests", "Apple", "Fixtures"); - } - - private static string NormalizeLineEndings(string value) - => value.Replace("\r\n", "\n", StringComparison.Ordinal); - - private static string Escape(string value) - => System.Net.WebUtility.HtmlEncode(value); -} - -internal sealed record AppleFixtureDefinition( - string ArticleId, - Uri DetailUri, - bool? ForceRapidSecurityResponse = null, - IReadOnlyList? Products = null) -{ - public string HtmlFixtureName => $"{ArticleId}.html"; - public string ExpectedFixtureName => $"{ArticleId}.expected.json"; -} - -internal sealed record AppleFixtureProduct( - string Platform, - string Name, - string Version, - string Build); +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Tests.Apple; + +internal static class AppleFixtureManager +{ + private const string UpdateEnvVar = "UPDATE_APPLE_FIXTURES"; + private const string UpdateSentinelFileName = ".update-apple-fixtures"; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + private static readonly Uri ExampleDetailBaseUri = new("https://support.example.com/en-us/"); + + private static readonly AppleFixtureDefinition[] Definitions = + { + new( + "125326", + new Uri("https://support.apple.com/en-us/125326"), + Products: new[] + { + new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1", "24A341"), + new AppleFixtureProduct("iOS", "iPhone 16 Pro", "26.0.1 (a)", "24A341a"), + new AppleFixtureProduct("iPadOS", "iPad Pro (M4)", "26", "24B120"), + }), + new( + "125328", + new Uri("https://support.apple.com/en-us/125328"), + Products: new[] + { + new AppleFixtureProduct("macOS", "MacBook Pro (M4)", "26.0.1", "26A123"), + new AppleFixtureProduct("macOS", "Mac Studio", "26", "26A120b"), + }), + new( + "106355", + new Uri("https://support.apple.com/en-us/106355"), + ForceRapidSecurityResponse: true, + Products: new[] + { + new AppleFixtureProduct("macOS Ventura", "macOS Ventura", string.Empty, "22F400"), + new AppleFixtureProduct("macOS Ventura", "macOS Ventura (Intel)", string.Empty, "22F400a"), + }), + new( + "HT214108", + new Uri("https://support.apple.com/en-us/HT214108")), + new( + "HT215500", + new Uri("https://support.apple.com/en-us/HT215500"), + ForceRapidSecurityResponse: true), + }; + + private static readonly Lazy UpdateTask = new( + () => UpdateFixturesAsync(CancellationToken.None), + LazyThreadSafetyMode.ExecutionAndPublication); + + public static IReadOnlyList Fixtures => Definitions; + + public static Task EnsureUpdatedAsync(CancellationToken cancellationToken = default) + { + if (!ShouldUpdateFixtures()) + { + return Task.CompletedTask; + } + + Console.WriteLine("[AppleFixtures] UPDATE flag detected; refreshing fixtures"); + return UpdateTask.Value; + } + + public static string ReadFixture(string name) + { + var path = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); + return File.ReadAllText(path); + } + + public static AppleDetailDto ReadExpectedDto(string articleId) + { + var json = ReadFixture($"{articleId}.expected.json"); + return JsonSerializer.Deserialize(json, SerializerOptions) + ?? throw new InvalidOperationException($"Unable to deserialize expected DTO for {articleId}."); + } + + private static bool ShouldUpdateFixtures() + { + var value = Environment.GetEnvironmentVariable(UpdateEnvVar)?.Trim(); + if (string.IsNullOrEmpty(value)) + { + var sentinelPath = Path.Combine(ResolveFixtureRoot(), UpdateSentinelFileName); + return File.Exists(sentinelPath); + } + + if (string.Equals(value, "0", StringComparison.Ordinal) + || string.Equals(value, "false", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return true; + } + + private static async Task UpdateFixturesAsync(CancellationToken cancellationToken) + { + var handler = new HttpClientHandler + { + AutomaticDecompression = System.Net.DecompressionMethods.All, + ServerCertificateCustomValidationCallback = HttpClientHandler.DangerousAcceptAnyServerCertificateValidator, + }; + + using var httpClient = new HttpClient(handler); + httpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("StellaOpsFixtureUpdater", "1.0")); + httpClient.Timeout = TimeSpan.FromSeconds(30); + + var indexEntries = new List(Definitions.Length); + + try + { + foreach (var definition in Definitions) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var html = await httpClient.GetStringAsync(definition.DetailUri, cancellationToken).ConfigureAwait(false); + var entryProducts = definition.Products? + .Select(product => new AppleIndexProduct( + product.Platform, + product.Name, + product.Version, + product.Build)) + .ToArray() ?? Array.Empty(); + + var entry = new AppleIndexEntry( + UpdateId: definition.ArticleId, + ArticleId: definition.ArticleId, + Title: definition.ArticleId, + PostingDate: DateTimeOffset.UtcNow, + DetailUri: definition.DetailUri, + Products: entryProducts, + IsRapidSecurityResponse: definition.ForceRapidSecurityResponse ?? false); + + var dto = AppleDetailParser.Parse(html, entry); + var sanitizedHtml = BuildSanitizedHtml(dto); + + WriteFixture(definition.HtmlFixtureName, sanitizedHtml); + WriteFixture(definition.ExpectedFixtureName, JsonSerializer.Serialize(dto, SerializerOptions)); + + var exampleDetailUri = new Uri(ExampleDetailBaseUri, definition.ArticleId); + indexEntries.Add(new + { + id = definition.ArticleId, + articleId = definition.ArticleId, + title = dto.Title, + postingDate = dto.Published.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture), + detailUrl = exampleDetailUri.ToString(), + rapidSecurityResponse = definition.ForceRapidSecurityResponse + ?? dto.Title.Contains("Rapid Security Response", StringComparison.OrdinalIgnoreCase), + products = dto.Affected + .OrderBy(p => p.Name, StringComparer.OrdinalIgnoreCase) + .Select(p => new + { + platform = p.Platform, + name = p.Name, + version = p.Version, + build = p.Build, + }) + .ToArray(), + }); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException) + { + Console.WriteLine($"[AppleFixtures] Skipped {definition.ArticleId}: {ex.Message}"); + } + } + } + finally + { + var sentinelPath = Path.Combine(ResolveFixtureRoot(), UpdateSentinelFileName); + if (File.Exists(sentinelPath)) + { + try + { + File.Delete(sentinelPath); + } + catch (IOException) + { + // best effort + } + } + } + + var indexDocument = new { updates = indexEntries }; + WriteFixture("index.json", JsonSerializer.Serialize(indexDocument, SerializerOptions)); + } + + private static string BuildSanitizedHtml(AppleDetailDto dto) + { + var builder = new StringBuilder(); + builder.AppendLine(""); + builder.AppendLine(""); + builder.AppendLine(""); + builder.AppendLine("
      "); + + builder.AppendLine($"

      {Escape(dto.Title)}

      "); + + if (!string.IsNullOrWhiteSpace(dto.Summary)) + { + builder.AppendLine($"

      {Escape(dto.Summary)}

      "); + } + + var publishedDisplay = dto.Published.ToString("MMMM d, yyyy", CultureInfo.InvariantCulture); + builder.AppendLine($" "); + + if (dto.Updated.HasValue) + { + var updatedDisplay = dto.Updated.Value.ToString("MMMM d, yyyy", CultureInfo.InvariantCulture); + builder.AppendLine($" "); + } + + if (dto.CveIds.Count > 0) + { + builder.AppendLine("
      "); + builder.AppendLine("

      Security Issues

      "); + builder.AppendLine("
        "); + foreach (var cve in dto.CveIds.OrderBy(id => id, StringComparer.OrdinalIgnoreCase)) + { + builder.AppendLine($"
      • {Escape(cve)}
        • "); + } + builder.AppendLine("
      "); + builder.AppendLine("
      "); + } + + if (dto.Affected.Count > 0) + { + builder.AppendLine(" "); + builder.AppendLine(" "); + foreach (var product in dto.Affected + .OrderBy(p => p.Platform ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Version ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(p => p.Build ?? string.Empty, StringComparer.OrdinalIgnoreCase)) + { + var platform = product.Platform is null ? string.Empty : Escape(product.Platform); + var name = Escape(product.Name); + var version = product.Version is null ? string.Empty : Escape(product.Version); + var build = product.Build is null ? string.Empty : Escape(product.Build); + + builder.Append(" "); + builder.AppendLine(); + builder.AppendLine($" "); + builder.AppendLine($" "); + builder.AppendLine($" "); + builder.AppendLine(" "); + } + + builder.AppendLine(" "); + builder.AppendLine("
      {name}{version}{build}
      "); + } + + if (dto.References.Count > 0) + { + builder.AppendLine("
      "); + builder.AppendLine("

      References

      "); + foreach (var reference in dto.References + .OrderBy(r => r.Url, StringComparer.OrdinalIgnoreCase)) + { + var title = reference.Title ?? string.Empty; + builder.AppendLine($" {Escape(title)}"); + } + builder.AppendLine("
      "); + } + + builder.AppendLine("
      "); + builder.AppendLine(""); + builder.AppendLine(""); + return builder.ToString(); + } + + private static void WriteFixture(string name, string contents) + { + var root = ResolveFixtureRoot(); + Directory.CreateDirectory(root); + var normalized = NormalizeLineEndings(contents); + + var sourcePath = Path.Combine(root, name); + File.WriteAllText(sourcePath, normalized); + + var outputPath = Path.Combine(AppContext.BaseDirectory, "Source", "Vndr", "Apple", "Fixtures", name); + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + File.WriteAllText(outputPath, normalized); + + Console.WriteLine($"[AppleFixtures] Wrote {name}"); + } + + private static string ResolveFixtureRoot() + { + var baseDir = AppContext.BaseDirectory; + // bin/Debug/net10.0/ -> project -> src -> repo root + var root = Path.GetFullPath(Path.Combine(baseDir, "..", "..", "..", "..", "..")); + return Path.Combine(root, "src", "StellaOps.Concelier.Connector.Vndr.Apple.Tests", "Apple", "Fixtures"); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); + + private static string Escape(string value) + => System.Net.WebUtility.HtmlEncode(value); +} + +internal sealed record AppleFixtureDefinition( + string ArticleId, + Uri DetailUri, + bool? ForceRapidSecurityResponse = null, + IReadOnlyList? Products = null) +{ + public string HtmlFixtureName => $"{ArticleId}.html"; + public string ExpectedFixtureName => $"{ArticleId}.expected.json"; +} + +internal sealed record AppleFixtureProduct( + string Platform, + string Name, + string Version, + string Build); diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs index 53026658..8cdaedc2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/AppleLiveRegressionTests.cs @@ -1,60 +1,60 @@ -using System; -using System.Collections.Generic; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading.Tasks; -using StellaOps.Feedser.Source.Vndr.Apple.Internal; -using StellaOps.Feedser.Source.Vndr.Apple.Tests.Apple; -using Xunit; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Tests; - -public sealed class AppleLiveRegressionTests -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - WriteIndented = false, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - - public static IEnumerable FixtureCases - { - get - { - foreach (var definition in AppleFixtureManager.Fixtures) - { - yield return new object[] { definition.ArticleId }; - } - } - } - - [Theory] - [MemberData(nameof(FixtureCases))] - public async Task Parser_SanitizedFixture_MatchesExpectedDto(string articleId) - { - var updateFlag = Environment.GetEnvironmentVariable("UPDATE_APPLE_FIXTURES"); - if (!string.IsNullOrEmpty(updateFlag)) - { - Console.WriteLine($"[AppleFixtures] UPDATE_APPLE_FIXTURES={updateFlag}"); - } - await AppleFixtureManager.EnsureUpdatedAsync(); - var expected = AppleFixtureManager.ReadExpectedDto(articleId); - var html = AppleFixtureManager.ReadFixture($"{articleId}.html"); - - var entry = new AppleIndexEntry( - UpdateId: articleId, - ArticleId: articleId, - Title: expected.Title, - PostingDate: expected.Published, - DetailUri: new Uri($"https://support.apple.com/en-us/{articleId}"), - Products: Array.Empty(), - IsRapidSecurityResponse: expected.RapidSecurityResponse); - - var dto = AppleDetailParser.Parse(html, entry); - var actualJson = JsonSerializer.Serialize(dto, SerializerOptions); - var expectedJson = JsonSerializer.Serialize(expected, SerializerOptions); - Assert.Equal(expectedJson, actualJson); - } -} +using System; +using System.Collections.Generic; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using StellaOps.Concelier.Connector.Vndr.Apple.Internal; +using StellaOps.Concelier.Connector.Vndr.Apple.Tests.Apple; +using Xunit; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Tests; + +public sealed class AppleLiveRegressionTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + public static IEnumerable FixtureCases + { + get + { + foreach (var definition in AppleFixtureManager.Fixtures) + { + yield return new object[] { definition.ArticleId }; + } + } + } + + [Theory] + [MemberData(nameof(FixtureCases))] + public async Task Parser_SanitizedFixture_MatchesExpectedDto(string articleId) + { + var updateFlag = Environment.GetEnvironmentVariable("UPDATE_APPLE_FIXTURES"); + if (!string.IsNullOrEmpty(updateFlag)) + { + Console.WriteLine($"[AppleFixtures] UPDATE_APPLE_FIXTURES={updateFlag}"); + } + await AppleFixtureManager.EnsureUpdatedAsync(); + var expected = AppleFixtureManager.ReadExpectedDto(articleId); + var html = AppleFixtureManager.ReadFixture($"{articleId}.html"); + + var entry = new AppleIndexEntry( + UpdateId: articleId, + ArticleId: articleId, + Title: expected.Title, + PostingDate: expected.Published, + DetailUri: new Uri($"https://support.apple.com/en-us/{articleId}"), + Products: Array.Empty(), + IsRapidSecurityResponse: expected.RapidSecurityResponse); + + var dto = AppleDetailParser.Parse(html, entry); + var actualJson = JsonSerializer.Serialize(dto, SerializerOptions); + var expectedJson = JsonSerializer.Serialize(expected, SerializerOptions); + Assert.Equal(expectedJson, actualJson); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json index f577ddc8..b21f717c 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.expected.json @@ -1,51 +1,51 @@ -{ - "advisoryId": "106355", - "articleId": "106355", - "title": "About the security content of Rapid Security Responses for macOS Ventura 13.4.1", - "summary": "This document describes the content of Rapid Security Responses.", - "published": "2025-10-12T13:19:10.4446382+00:00", - "cveIds": [ - "CVE-2023-37450" - ], - "affected": [ - { - "platform": "macOS Ventura", - "name": "macOS Ventura", - "version": "", - "build": "22F400" - }, - { - "platform": "macOS Ventura", - "name": "macOS Ventura (Intel)", - "version": "", - "build": "22F400a" - } - ], - "references": [ - { - "url": "https://support.apple.com/103190", - "title": "Contact the vendor", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/106355/localeselector", - "title": "United States", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/kb/HT201222", - "title": "Apple security releases", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/kb/HT201224", - "title": "Rapid Security Responses", - "kind": "advisory" - }, - { - "url": "https://www.cve.org/About/Overview", - "title": "CVE-ID" - } - ], - "rapidSecurityResponse": true +{ + "advisoryId": "106355", + "articleId": "106355", + "title": "About the security content of Rapid Security Responses for macOS Ventura 13.4.1", + "summary": "This document describes the content of Rapid Security Responses.", + "published": "2025-10-12T13:19:10.4446382+00:00", + "cveIds": [ + "CVE-2023-37450" + ], + "affected": [ + { + "platform": "macOS Ventura", + "name": "macOS Ventura", + "version": "", + "build": "22F400" + }, + { + "platform": "macOS Ventura", + "name": "macOS Ventura (Intel)", + "version": "", + "build": "22F400a" + } + ], + "references": [ + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/106355/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201222", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201224", + "title": "Rapid Security Responses", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": true } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.html similarity index 97% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.html index c985c0a6..7d7b7adf 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/106355.html +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/106355.html @@ -1,38 +1,38 @@ - - - - - - + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json index 3bd204ba..f63c470f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.expected.json @@ -1,57 +1,57 @@ -{ - "advisoryId": "125326", - "articleId": "125326", - "title": "About the security content of iOS 26.0.1 and iPadOS 26.0.1", - "summary": "This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.", - "published": "2025-10-12T13:19:09.732004+00:00", - "cveIds": [ - "CVE-2025-43400" - ], - "affected": [ - { - "platform": "iOS", - "name": "iPhone 16 Pro", - "version": "26.0.1", - "build": "24A341" - }, - { - "platform": "iOS", - "name": "iPhone 16 Pro", - "version": "26.0.1 (a)", - "build": "24A341a" - }, - { - "platform": "iPadOS", - "name": "iPad Pro (M4)", - "version": "26", - "build": "24B120" - } - ], - "references": [ - { - "url": "https://support.apple.com/103190", - "title": "Contact the vendor", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/100100", - "title": "Apple security releases", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/102549", - "title": "Apple Product Security", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/125326/localeselector", - "title": "United States", - "kind": "advisory" - }, - { - "url": "https://www.cve.org/About/Overview", - "title": "CVE-ID" - } - ], - "rapidSecurityResponse": false +{ + "advisoryId": "125326", + "articleId": "125326", + "title": "About the security content of iOS 26.0.1 and iPadOS 26.0.1", + "summary": "This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.", + "published": "2025-10-12T13:19:09.732004+00:00", + "cveIds": [ + "CVE-2025-43400" + ], + "affected": [ + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1", + "build": "24A341" + }, + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1 (a)", + "build": "24A341a" + }, + { + "platform": "iPadOS", + "name": "iPad Pro (M4)", + "version": "26", + "build": "24B120" + } + ], + "references": [ + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/100100", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/102549", + "title": "Apple Product Security", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/125326/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": false } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.html similarity index 97% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.html index 5ed77341..2f0e5d92 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125326.html +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125326.html @@ -1,43 +1,43 @@ - - - -
      -

      About the security content of iOS 26.0.1 and iPadOS 26.0.1

      -

      This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.

      - -
      -

      Security Issues

      -
        -
      • CVE-2025-43400
        • -
      -
      - - - - - - - - - - - - - - - - - - -
      iPhone 16 Pro26.0.124A341
      iPhone 16 Pro26.0.1 (a)24A341a
      iPad Pro (M4)2624B120
      -
      -

      References

      - Contact the vendor - Apple security releases - Apple Product Security - United States - CVE-ID -
      -
      - - + + + +
      +

      About the security content of iOS 26.0.1 and iPadOS 26.0.1

      +

      This document describes the security content of iOS 26.0.1 and iPadOS 26.0.1.

      + +
      +

      Security Issues

      +
        +
      • CVE-2025-43400
        • +
      +
      + + + + + + + + + + + + + + + + + + +
      iPhone 16 Pro26.0.124A341
      iPhone 16 Pro26.0.1 (a)24A341a
      iPad Pro (M4)2624B120
      +
      +

      References

      + Contact the vendor + Apple security releases + Apple Product Security + United States + CVE-ID +
      +
      + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json index 19fcc64b..5cfcc78a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.expected.json @@ -1,51 +1,51 @@ -{ - "advisoryId": "125328", - "articleId": "125328", - "title": "About the security content of macOS Tahoe 26.0.1", - "summary": "This document describes the security content of macOS Tahoe 26.0.1.", - "published": "2025-10-12T13:19:10.3703446+00:00", - "cveIds": [ - "CVE-2025-43400" - ], - "affected": [ - { - "platform": "macOS", - "name": "MacBook Pro (M4)", - "version": "26.0.1", - "build": "26A123" - }, - { - "platform": "macOS", - "name": "Mac Studio", - "version": "26", - "build": "26A120b" - } - ], - "references": [ - { - "url": "https://support.apple.com/103190", - "title": "Contact the vendor", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/100100", - "title": "Apple security releases", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/102549", - "title": "Apple Product Security", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/125328/localeselector", - "title": "United States", - "kind": "advisory" - }, - { - "url": "https://www.cve.org/About/Overview", - "title": "CVE-ID" - } - ], - "rapidSecurityResponse": false +{ + "advisoryId": "125328", + "articleId": "125328", + "title": "About the security content of macOS Tahoe 26.0.1", + "summary": "This document describes the security content of macOS Tahoe 26.0.1.", + "published": "2025-10-12T13:19:10.3703446+00:00", + "cveIds": [ + "CVE-2025-43400" + ], + "affected": [ + { + "platform": "macOS", + "name": "MacBook Pro (M4)", + "version": "26.0.1", + "build": "26A123" + }, + { + "platform": "macOS", + "name": "Mac Studio", + "version": "26", + "build": "26A120b" + } + ], + "references": [ + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/100100", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/102549", + "title": "Apple Product Security", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/125328/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": false } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.html similarity index 97% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.html index 601058eb..09a7575f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/125328.html +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/125328.html @@ -1,38 +1,38 @@ - - - - - - + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json index f7320a06..49f298b7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT214108.expected.json @@ -1,61 +1,61 @@ -{ - "advisoryId": "HT214108", - "articleId": "HT214108", - "title": "About the security content of visionOS 1.2", - "summary": "This document describes the security content of visionOS 1.2.", - "published": "2025-10-12T13:19:10.5262006+00:00", - "cveIds": [ - "CVE-2024-27800", - "CVE-2024-27801", - "CVE-2024-27802", - "CVE-2024-27808", - "CVE-2024-27811", - "CVE-2024-27812", - "CVE-2024-27815", - "CVE-2024-27817", - "CVE-2024-27820", - "CVE-2024-27828", - "CVE-2024-27830", - "CVE-2024-27831", - "CVE-2024-27832", - "CVE-2024-27833", - "CVE-2024-27836", - "CVE-2024-27838", - "CVE-2024-27840", - "CVE-2024-27844", - "CVE-2024-27850", - "CVE-2024-27851", - "CVE-2024-27856", - "CVE-2024-27857", - "CVE-2024-27884", - "CVE-2024-40771" - ], - "affected": [], - "references": [ - { - "url": "https://support.apple.com/103190", - "title": "Contact the vendor", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/en-us/120906/localeselector", - "title": "United States", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/kb/HT201220", - "title": "Apple Product Security", - "kind": "advisory" - }, - { - "url": "https://support.apple.com/kb/HT201222", - "title": "Apple security releases", - "kind": "advisory" - }, - { - "url": "https://www.cve.org/About/Overview", - "title": "CVE-ID" - } - ], - "rapidSecurityResponse": false +{ + "advisoryId": "HT214108", + "articleId": "HT214108", + "title": "About the security content of visionOS 1.2", + "summary": "This document describes the security content of visionOS 1.2.", + "published": "2025-10-12T13:19:10.5262006+00:00", + "cveIds": [ + "CVE-2024-27800", + "CVE-2024-27801", + "CVE-2024-27802", + "CVE-2024-27808", + "CVE-2024-27811", + "CVE-2024-27812", + "CVE-2024-27815", + "CVE-2024-27817", + "CVE-2024-27820", + "CVE-2024-27828", + "CVE-2024-27830", + "CVE-2024-27831", + "CVE-2024-27832", + "CVE-2024-27833", + "CVE-2024-27836", + "CVE-2024-27838", + "CVE-2024-27840", + "CVE-2024-27844", + "CVE-2024-27850", + "CVE-2024-27851", + "CVE-2024-27856", + "CVE-2024-27857", + "CVE-2024-27884", + "CVE-2024-40771" + ], + "affected": [], + "references": [ + { + "url": "https://support.apple.com/103190", + "title": "Contact the vendor", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/en-us/120906/localeselector", + "title": "United States", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201220", + "title": "Apple Product Security", + "kind": "advisory" + }, + { + "url": "https://support.apple.com/kb/HT201222", + "title": "Apple security releases", + "kind": "advisory" + }, + { + "url": "https://www.cve.org/About/Overview", + "title": "CVE-ID" + } + ], + "rapidSecurityResponse": false } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json index 82dc9c9e..7579a3e0 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/HT215500.expected.json @@ -1,20 +1,20 @@ -{ - "advisoryId": "HT215500", - "articleId": "HT215500", - "title": "Rapid Security Response iOS 18.0.1 (c)", - "summary": "Rapid Security Response provides important security fixes between software updates.", - "published": "2025-10-02T15:30:00+00:00", - "cveIds": [ - "CVE-2025-2468" - ], - "affected": [ - { - "platform": "iOS", - "name": "iPhone 15 Pro", - "version": "18.0.1 (c)", - "build": "22A123c" - } - ], - "references": [], - "rapidSecurityResponse": true -} +{ + "advisoryId": "HT215500", + "articleId": "HT215500", + "title": "Rapid Security Response iOS 18.0.1 (c)", + "summary": "Rapid Security Response provides important security fixes between software updates.", + "published": "2025-10-02T15:30:00+00:00", + "cveIds": [ + "CVE-2025-2468" + ], + "affected": [ + { + "platform": "iOS", + "name": "iPhone 15 Pro", + "version": "18.0.1 (c)", + "build": "22A123c" + } + ], + "references": [], + "rapidSecurityResponse": true +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html similarity index 97% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html index 2a58120c..0edc9317 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht214108.html @@ -1,47 +1,47 @@ - - - -
      -

      About the security content of visionOS 1.2

      -

      This document describes the security content of visionOS 1.2.

      - -
      -

      Security Issues

      -
        -
      • CVE-2024-27800
        • -
      • CVE-2024-27801
        • -
      • CVE-2024-27802
        • -
      • CVE-2024-27808
        • -
      • CVE-2024-27811
        • -
      • CVE-2024-27812
        • -
      • CVE-2024-27815
        • -
      • CVE-2024-27817
        • -
      • CVE-2024-27820
        • -
      • CVE-2024-27828
        • -
      • CVE-2024-27830
        • -
      • CVE-2024-27831
        • -
      • CVE-2024-27832
        • -
      • CVE-2024-27833
        • -
      • CVE-2024-27836
        • -
      • CVE-2024-27838
        • -
      • CVE-2024-27840
        • -
      • CVE-2024-27844
        • -
      • CVE-2024-27850
        • -
      • CVE-2024-27851
        • -
      • CVE-2024-27856
        • -
      • CVE-2024-27857
        • -
      • CVE-2024-27884
        • -
      • CVE-2024-40771
        • -
      -
      -
      -

      References

      - Contact the vendor - United States - Apple Product Security - Apple security releases - CVE-ID -
      -
      - - + + + +
      +

      About the security content of visionOS 1.2

      +

      This document describes the security content of visionOS 1.2.

      + +
      +

      Security Issues

      +
        +
      • CVE-2024-27800
        • +
      • CVE-2024-27801
        • +
      • CVE-2024-27802
        • +
      • CVE-2024-27808
        • +
      • CVE-2024-27811
        • +
      • CVE-2024-27812
        • +
      • CVE-2024-27815
        • +
      • CVE-2024-27817
        • +
      • CVE-2024-27820
        • +
      • CVE-2024-27828
        • +
      • CVE-2024-27830
        • +
      • CVE-2024-27831
        • +
      • CVE-2024-27832
        • +
      • CVE-2024-27833
        • +
      • CVE-2024-27836
        • +
      • CVE-2024-27838
        • +
      • CVE-2024-27840
        • +
      • CVE-2024-27844
        • +
      • CVE-2024-27850
        • +
      • CVE-2024-27851
        • +
      • CVE-2024-27856
        • +
      • CVE-2024-27857
        • +
      • CVE-2024-27884
        • +
      • CVE-2024-40771
        • +
      +
      +
      +

      References

      + Contact the vendor + United States + Apple Product Security + Apple security releases + CVE-ID +
      +
      + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html index f53e3097..351213f2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/ht215500.html @@ -1,25 +1,25 @@ - - - -
      -

      Rapid Security Response iOS 18.0.1 (c)

      -

      Rapid Security Response provides important security fixes between software updates.

      - -
      -

      Security Issues

      -
        -
      • CVE-2025-2468
        • -
      -
      - - - - - - - - -
      iPhone 15 Pro18.0.1 (c)22A123c
      -
      - - + + + +
      +

      Rapid Security Response iOS 18.0.1 (c)

      +

      Rapid Security Response provides important security fixes between software updates.

      + +
      +

      Security Issues

      +
        +
      • CVE-2025-2468
        • +
      +
      + + + + + + + + +
      iPhone 15 Pro18.0.1 (c)22A123c
      +
      + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/index.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json rename to src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/index.json index a09f4f56..c8b00696 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures/index.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures/index.json @@ -1,101 +1,101 @@ -{ - "updates": [ - { - "id": "125326", - "articleId": "125326", - "title": "About the security content of iOS 26.0.1 and iPadOS 26.0.1", - "postingDate": "2025-10-12T13:19:09.7320040+00:00", - "detailUrl": "https://support.example.com/en-us/125326", - "rapidSecurityResponse": false, - "products": [ - { - "platform": "iPadOS", - "name": "iPad Pro (M4)", - "version": "26", - "build": "24B120" - }, - { - "platform": "iOS", - "name": "iPhone 16 Pro", - "version": "26.0.1", - "build": "24A341" - }, - { - "platform": "iOS", - "name": "iPhone 16 Pro", - "version": "26.0.1 (a)", - "build": "24A341a" - } - ] - }, - { - "id": "125328", - "articleId": "125328", - "title": "About the security content of macOS Tahoe 26.0.1", - "postingDate": "2025-10-12T13:19:10.3703446+00:00", - "detailUrl": "https://support.example.com/en-us/125328", - "rapidSecurityResponse": false, - "products": [ - { - "platform": "macOS", - "name": "Mac Studio", - "version": "26", - "build": "26A120b" - }, - { - "platform": "macOS", - "name": "MacBook Pro (M4)", - "version": "26.0.1", - "build": "26A123" - } - ] - }, - { - "id": "106355", - "articleId": "106355", - "title": "About the security content of Rapid Security Responses for macOS Ventura 13.4.1", - "postingDate": "2025-10-12T13:19:10.4446382+00:00", - "detailUrl": "https://support.example.com/en-us/106355", - "rapidSecurityResponse": true, - "products": [ - { - "platform": "macOS Ventura", - "name": "macOS Ventura", - "version": "", - "build": "22F400" - }, - { - "platform": "macOS Ventura", - "name": "macOS Ventura (Intel)", - "version": "", - "build": "22F400a" - } - ] - }, - { - "id": "HT214108", - "articleId": "HT214108", - "title": "About the security content of visionOS 1.2", - "postingDate": "2025-10-12T13:19:10.5262006+00:00", - "detailUrl": "https://support.example.com/en-us/HT214108", - "rapidSecurityResponse": false, - "products": [] - }, - { - "id": "RSR-iOS-18.0.1-c", - "articleId": "HT215500", - "title": "Rapid Security Response iOS 18.0.1 (c)", - "postingDate": "2025-10-02T15:30:00+00:00", - "detailUrl": "https://support.example.com/en-us/HT215500", - "rapidSecurityResponse": true, - "products": [ - { - "platform": "iOS", - "name": "iPhone 15 Pro", - "version": "18.0.1 (c)", - "build": "22A123c" - } - ] - } - ] -} +{ + "updates": [ + { + "id": "125326", + "articleId": "125326", + "title": "About the security content of iOS 26.0.1 and iPadOS 26.0.1", + "postingDate": "2025-10-12T13:19:09.7320040+00:00", + "detailUrl": "https://support.example.com/en-us/125326", + "rapidSecurityResponse": false, + "products": [ + { + "platform": "iPadOS", + "name": "iPad Pro (M4)", + "version": "26", + "build": "24B120" + }, + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1", + "build": "24A341" + }, + { + "platform": "iOS", + "name": "iPhone 16 Pro", + "version": "26.0.1 (a)", + "build": "24A341a" + } + ] + }, + { + "id": "125328", + "articleId": "125328", + "title": "About the security content of macOS Tahoe 26.0.1", + "postingDate": "2025-10-12T13:19:10.3703446+00:00", + "detailUrl": "https://support.example.com/en-us/125328", + "rapidSecurityResponse": false, + "products": [ + { + "platform": "macOS", + "name": "Mac Studio", + "version": "26", + "build": "26A120b" + }, + { + "platform": "macOS", + "name": "MacBook Pro (M4)", + "version": "26.0.1", + "build": "26A123" + } + ] + }, + { + "id": "106355", + "articleId": "106355", + "title": "About the security content of Rapid Security Responses for macOS Ventura 13.4.1", + "postingDate": "2025-10-12T13:19:10.4446382+00:00", + "detailUrl": "https://support.example.com/en-us/106355", + "rapidSecurityResponse": true, + "products": [ + { + "platform": "macOS Ventura", + "name": "macOS Ventura", + "version": "", + "build": "22F400" + }, + { + "platform": "macOS Ventura", + "name": "macOS Ventura (Intel)", + "version": "", + "build": "22F400a" + } + ] + }, + { + "id": "HT214108", + "articleId": "HT214108", + "title": "About the security content of visionOS 1.2", + "postingDate": "2025-10-12T13:19:10.5262006+00:00", + "detailUrl": "https://support.example.com/en-us/HT214108", + "rapidSecurityResponse": false, + "products": [] + }, + { + "id": "RSR-iOS-18.0.1-c", + "articleId": "HT215500", + "title": "Rapid Security Response iOS 18.0.1 (c)", + "postingDate": "2025-10-02T15:30:00+00:00", + "detailUrl": "https://support.example.com/en-us/HT215500", + "rapidSecurityResponse": true, + "products": [ + { + "platform": "iOS", + "name": "iPhone 15 Pro", + "version": "18.0.1 (c)", + "build": "22A123c" + } + ] + } + ] +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj new file mode 100644 index 00000000..629b25a2 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md similarity index 85% rename from src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md index 9bb992e0..3a218151 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/AGENTS.md @@ -1,39 +1,39 @@ -# AGENTS -## Role -Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS. - -## Scope -- Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy. -- Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks. -- Parse advisories to extract summary, affected products/versions, mitigation, CVEs. -- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions). -- Produce deterministic fixtures and regression tests. - -## Participants -- `Source.Common` (HTTP/fetch utilities, DTO storage). -- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). -- `Feedser.Models` (canonical structures + range primitives). -- `Feedser.Testing` (integration fixtures/snapshots). - -## Interfaces & Contracts -- Job kinds: `apple:fetch`, `apple:parse`, `apple:map`. -- Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates. -- Alias set should include Apple HT IDs and CVE IDs. - -## In/Out of scope -In scope: -- Security advisories covering Apple OS/app updates. -- Range primitives capturing device/OS version ranges. - -Out of scope: -- Release notes unrelated to security. - -## Observability & Security Expectations -- Log fetch/mapping statistics and failure details. -- Sanitize HTML while preserving structured data tables. -- Respect upstream rate limits; record failures with backoff. - -## Tests -- Add `StellaOps.Feedser.Source.Vndr.Apple.Tests` covering fetch/parse/map with fixtures. -- Snapshot canonical advisories; support fixture regeneration via env flag. -- Ensure deterministic ordering/time normalisation. +# AGENTS +## Role +Implement the Apple security advisories connector to ingest Apple HT/HT2 security bulletins for macOS/iOS/tvOS/visionOS. + +## Scope +- Identify canonical Apple security bulletin feeds (HTML, RSS, JSON) and change detection strategy. +- Implement fetch/cursor pipeline with retry/backoff, handling localisation/HTML quirks. +- Parse advisories to extract summary, affected products/versions, mitigation, CVEs. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (SemVer + vendor extensions). +- Produce deterministic fixtures and regression tests. + +## Participants +- `Source.Common` (HTTP/fetch utilities, DTO storage). +- `Storage.Mongo` (raw/document/DTO/advisory stores, source state). +- `Concelier.Models` (canonical structures + range primitives). +- `Concelier.Testing` (integration fixtures/snapshots). + +## Interfaces & Contracts +- Job kinds: `apple:fetch`, `apple:parse`, `apple:map`. +- Persist upstream metadata (ETag/Last-Modified or revision IDs) for incremental updates. +- Alias set should include Apple HT IDs and CVE IDs. + +## In/Out of scope +In scope: +- Security advisories covering Apple OS/app updates. +- Range primitives capturing device/OS version ranges. + +Out of scope: +- Release notes unrelated to security. + +## Observability & Security Expectations +- Log fetch/mapping statistics and failure details. +- Sanitize HTML while preserving structured data tables. +- Respect upstream rate limits; record failures with backoff. + +## Tests +- Add `StellaOps.Concelier.Connector.Vndr.Apple.Tests` covering fetch/parse/map with fixtures. +- Snapshot canonical advisories; support fixture regeneration via env flag. +- Ensure deterministic ordering/time normalisation. diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs index 01828324..d8ff245f 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleConnector.cs @@ -1,439 +1,439 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Apple.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -public sealed class AppleConnector : IFeedConnector -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - PropertyNameCaseInsensitive = true, - }; - - private readonly SourceFetchService _fetchService; - private readonly RawDocumentStorage _rawDocumentStorage; - private readonly IDocumentStore _documentStore; - private readonly IDtoStore _dtoStore; - private readonly IAdvisoryStore _advisoryStore; - private readonly IPsirtFlagStore _psirtFlagStore; - private readonly ISourceStateRepository _stateRepository; - private readonly AppleOptions _options; - private readonly AppleDiagnostics _diagnostics; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - - public AppleConnector( - SourceFetchService fetchService, - RawDocumentStorage rawDocumentStorage, - IDocumentStore documentStore, - IDtoStore dtoStore, - IAdvisoryStore advisoryStore, - IPsirtFlagStore psirtFlagStore, - ISourceStateRepository stateRepository, - AppleDiagnostics diagnostics, - IOptions options, - TimeProvider? timeProvider, - ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); - _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); - _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string SourceName => VndrAppleConnectorPlugin.SourceName; - - public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - var pendingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase); - var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue; - var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill); - - SourceFetchContentResult indexResult; - try - { - var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!) - { - AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" }, - }; - - indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri); - await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (!indexResult.IsSuccess || indexResult.Content is null) - { - if (indexResult.IsNotModified) - { - _diagnostics.FetchUnchanged(); - } - - await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); - return; - } - - var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!); - if (indexEntries.Count == 0) - { - await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); - return; - } - - var allowlist = _options.AdvisoryAllowlist; - var blocklist = _options.AdvisoryBlocklist; - - var ordered = indexEntries - .Where(entry => ShouldInclude(entry, allowlist, blocklist)) - .OrderBy(entry => entry.PostingDate) - .ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - foreach (var entry in ordered) - { - cancellationToken.ThrowIfCancellationRequested(); - - if (entry.PostingDate < baseline) - { - continue; - } - - if (cursor.LastPosted.HasValue - && entry.PostingDate <= cursor.LastPosted.Value - && processedIds.Contains(entry.UpdateId)) - { - continue; - } - - var metadata = BuildMetadata(entry); - var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); - - SourceFetchResult result; - try - { - result = await _fetchService.FetchAsync( - new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri) - { - Metadata = metadata, - ETag = existing?.Etag, - LastModified = existing?.LastModified, - AcceptHeaders = new[] - { - "text/html", - "application/xhtml+xml", - "text/plain;q=0.5" - }, - }, - cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - _diagnostics.FetchFailure(); - _logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri); - await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); - throw; - } - - if (result.StatusCode == HttpStatusCode.NotModified) - { - _diagnostics.FetchUnchanged(); - } - - if (!result.IsSuccess || result.Document is null) - { - continue; - } - - _diagnostics.FetchItem(); - - pendingDocuments.Add(result.Document.Id); - processedIds.Add(entry.UpdateId); - - if (entry.PostingDate > maxPosted) - { - maxPosted = entry.PostingDate; - } - } - - var updated = cursor - .WithPendingDocuments(pendingDocuments) - .WithPendingMappings(pendingMappings) - .WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds); - - await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); - } - - public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingDocuments.Count == 0) - { - return; - } - - var remainingDocuments = cursor.PendingDocuments.ToHashSet(); - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingDocuments) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - if (!document.GridFsId.HasValue) - { - _diagnostics.ParseFailure(); - _logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - AppleDetailDto dto; - try - { - var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); - var html = System.Text.Encoding.UTF8.GetString(content); - var entry = RehydrateIndexEntry(document); - dto = AppleDetailParser.Parse(html, entry); - } - catch (Exception ex) - { - _diagnostics.ParseFailure(); - _logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - remainingDocuments.Remove(documentId); - pendingMappings.Remove(documentId); - continue; - } - - var json = JsonSerializer.Serialize(dto, SerializerOptions); - var payload = BsonDocument.Parse(json); - var validatedAt = _timeProvider.GetUtcNow(); - - var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); - var dtoRecord = existingDto is null - ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt) - : existingDto with - { - Payload = payload, - SchemaVersion = "apple.security.update.v1", - ValidatedAt = validatedAt, - }; - - await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); - - remainingDocuments.Remove(documentId); - pendingMappings.Add(document.Id); - } - - var updatedCursor = cursor - .WithPendingDocuments(remainingDocuments) - .WithPendingMappings(pendingMappings); - - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(services); - - var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); - if (cursor.PendingMappings.Count == 0) - { - return; - } - - var pendingMappings = cursor.PendingMappings.ToHashSet(); - - foreach (var documentId in cursor.PendingMappings) - { - cancellationToken.ThrowIfCancellationRequested(); - - var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); - if (document is null) - { - pendingMappings.Remove(documentId); - continue; - } - - var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); - if (dtoRecord is null) - { - pendingMappings.Remove(documentId); - continue; - } - - AppleDetailDto dto; - try - { - dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions) - ?? throw new InvalidOperationException("Unable to deserialize Apple DTO."); - } - catch (Exception ex) - { - _logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id); - pendingMappings.Remove(documentId); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); - continue; - } - - var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord); - _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); - - await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); - await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); - - if (flag is not null) - { - await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); - } - - pendingMappings.Remove(documentId); - } - - var updatedCursor = cursor.WithPendingMappings(pendingMappings); - await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); - } - - private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document) - { - var metadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); - metadata.TryGetValue("apple.articleId", out var articleId); - metadata.TryGetValue("apple.updateId", out var updateId); - metadata.TryGetValue("apple.title", out var title); - metadata.TryGetValue("apple.postingDate", out var postingDateRaw); - metadata.TryGetValue("apple.detailUri", out var detailUriRaw); - metadata.TryGetValue("apple.rapidResponse", out var rapidRaw); - metadata.TryGetValue("apple.products", out var productsJson); - - if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate)) - { - postingDate = document.FetchedAt; - } - - var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri) - ? parsedUri - : new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri); - - var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase); - var products = DeserializeProducts(productsJson); - - return new AppleIndexEntry( - UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId, - ArticleId: articleId ?? document.Uri, - Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update", - PostingDate: postingDate.ToUniversalTime(), - DetailUri: detailUri, - Products: products, - IsRapidSecurityResponse: rapid); - } - - private static IReadOnlyList DeserializeProducts(string? json) - { - if (string.IsNullOrWhiteSpace(json)) - { - return Array.Empty(); - } - - try - { - var products = JsonSerializer.Deserialize>(json, SerializerOptions); - return products is { Count: > 0 } ? products : Array.Empty(); - } - catch (JsonException) - { - return Array.Empty(); - } - } - - private static Dictionary BuildMetadata(AppleIndexEntry entry) - { - var metadata = new Dictionary(StringComparer.Ordinal) - { - ["apple.articleId"] = entry.ArticleId, - ["apple.updateId"] = entry.UpdateId, - ["apple.title"] = entry.Title, - ["apple.postingDate"] = entry.PostingDate.ToString("O"), - ["apple.detailUri"] = entry.DetailUri.ToString(), - ["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false", - ["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions), - }; - - return metadata; - } - - private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection allowlist, IReadOnlyCollection blocklist) - { - if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId)) - { - return false; - } - - if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId)) - { - return false; - } - - return true; - } - - private async Task GetCursorAsync(CancellationToken cancellationToken) - { - var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); - return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor); - } - - private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken) - { - var document = cursor.ToBson(); - await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Apple.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +public sealed class AppleConnector : IFeedConnector +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + PropertyNameCaseInsensitive = true, + }; + + private readonly SourceFetchService _fetchService; + private readonly RawDocumentStorage _rawDocumentStorage; + private readonly IDocumentStore _documentStore; + private readonly IDtoStore _dtoStore; + private readonly IAdvisoryStore _advisoryStore; + private readonly IPsirtFlagStore _psirtFlagStore; + private readonly ISourceStateRepository _stateRepository; + private readonly AppleOptions _options; + private readonly AppleDiagnostics _diagnostics; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public AppleConnector( + SourceFetchService fetchService, + RawDocumentStorage rawDocumentStorage, + IDocumentStore documentStore, + IDtoStore dtoStore, + IAdvisoryStore advisoryStore, + IPsirtFlagStore psirtFlagStore, + ISourceStateRepository stateRepository, + AppleDiagnostics diagnostics, + IOptions options, + TimeProvider? timeProvider, + ILogger logger) + { + _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); + _rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage)); + _documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore)); + _dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _psirtFlagStore = psirtFlagStore ?? throw new ArgumentNullException(nameof(psirtFlagStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string SourceName => VndrAppleConnectorPlugin.SourceName; + + public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + var pendingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + var processedIds = cursor.ProcessedIds.ToHashSet(StringComparer.OrdinalIgnoreCase); + var maxPosted = cursor.LastPosted ?? DateTimeOffset.MinValue; + var baseline = cursor.LastPosted?.Add(-_options.ModifiedTolerance) ?? _timeProvider.GetUtcNow().Add(-_options.InitialBackfill); + + SourceFetchContentResult indexResult; + try + { + var request = new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, _options.SoftwareLookupUri!) + { + AcceptHeaders = new[] { "application/json", "application/vnd.apple.security+json;q=0.9" }, + }; + + indexResult = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Apple software index fetch failed from {Uri}", _options.SoftwareLookupUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(10), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (!indexResult.IsSuccess || indexResult.Content is null) + { + if (indexResult.IsNotModified) + { + _diagnostics.FetchUnchanged(); + } + + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + var indexEntries = AppleIndexParser.Parse(indexResult.Content, _options.AdvisoryBaseUri!); + if (indexEntries.Count == 0) + { + await UpdateCursorAsync(cursor, cancellationToken).ConfigureAwait(false); + return; + } + + var allowlist = _options.AdvisoryAllowlist; + var blocklist = _options.AdvisoryBlocklist; + + var ordered = indexEntries + .Where(entry => ShouldInclude(entry, allowlist, blocklist)) + .OrderBy(entry => entry.PostingDate) + .ThenBy(entry => entry.ArticleId, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + foreach (var entry in ordered) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (entry.PostingDate < baseline) + { + continue; + } + + if (cursor.LastPosted.HasValue + && entry.PostingDate <= cursor.LastPosted.Value + && processedIds.Contains(entry.UpdateId)) + { + continue; + } + + var metadata = BuildMetadata(entry); + var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, entry.DetailUri.ToString(), cancellationToken).ConfigureAwait(false); + + SourceFetchResult result; + try + { + result = await _fetchService.FetchAsync( + new SourceFetchRequest(AppleOptions.HttpClientName, SourceName, entry.DetailUri) + { + Metadata = metadata, + ETag = existing?.Etag, + LastModified = existing?.LastModified, + AcceptHeaders = new[] + { + "text/html", + "application/xhtml+xml", + "text/plain;q=0.5" + }, + }, + cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _diagnostics.FetchFailure(); + _logger.LogError(ex, "Apple advisory fetch failed for {Uri}", entry.DetailUri); + await _stateRepository.MarkFailureAsync(SourceName, _timeProvider.GetUtcNow(), TimeSpan.FromMinutes(5), ex.Message, cancellationToken).ConfigureAwait(false); + throw; + } + + if (result.StatusCode == HttpStatusCode.NotModified) + { + _diagnostics.FetchUnchanged(); + } + + if (!result.IsSuccess || result.Document is null) + { + continue; + } + + _diagnostics.FetchItem(); + + pendingDocuments.Add(result.Document.Id); + processedIds.Add(entry.UpdateId); + + if (entry.PostingDate > maxPosted) + { + maxPosted = entry.PostingDate; + } + } + + var updated = cursor + .WithPendingDocuments(pendingDocuments) + .WithPendingMappings(pendingMappings) + .WithLastPosted(maxPosted == DateTimeOffset.MinValue ? cursor.LastPosted ?? DateTimeOffset.MinValue : maxPosted, processedIds); + + await UpdateCursorAsync(updated, cancellationToken).ConfigureAwait(false); + } + + public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingDocuments.Count == 0) + { + return; + } + + var remainingDocuments = cursor.PendingDocuments.ToHashSet(); + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingDocuments) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + if (!document.GridFsId.HasValue) + { + _diagnostics.ParseFailure(); + _logger.LogWarning("Apple document {DocumentId} missing GridFS payload", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + AppleDetailDto dto; + try + { + var content = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false); + var html = System.Text.Encoding.UTF8.GetString(content); + var entry = RehydrateIndexEntry(document); + dto = AppleDetailParser.Parse(html, entry); + } + catch (Exception ex) + { + _diagnostics.ParseFailure(); + _logger.LogError(ex, "Apple parse failed for document {DocumentId}", document.Id); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + remainingDocuments.Remove(documentId); + pendingMappings.Remove(documentId); + continue; + } + + var json = JsonSerializer.Serialize(dto, SerializerOptions); + var payload = BsonDocument.Parse(json); + var validatedAt = _timeProvider.GetUtcNow(); + + var existingDto = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + var dtoRecord = existingDto is null + ? new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "apple.security.update.v1", payload, validatedAt) + : existingDto with + { + Payload = payload, + SchemaVersion = "apple.security.update.v1", + ValidatedAt = validatedAt, + }; + + await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false); + + remainingDocuments.Remove(documentId); + pendingMappings.Add(document.Id); + } + + var updatedCursor = cursor + .WithPendingDocuments(remainingDocuments) + .WithPendingMappings(pendingMappings); + + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(services); + + var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false); + if (cursor.PendingMappings.Count == 0) + { + return; + } + + var pendingMappings = cursor.PendingMappings.ToHashSet(); + + foreach (var documentId in cursor.PendingMappings) + { + cancellationToken.ThrowIfCancellationRequested(); + + var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false); + if (document is null) + { + pendingMappings.Remove(documentId); + continue; + } + + var dtoRecord = await _dtoStore.FindByDocumentIdAsync(document.Id, cancellationToken).ConfigureAwait(false); + if (dtoRecord is null) + { + pendingMappings.Remove(documentId); + continue; + } + + AppleDetailDto dto; + try + { + dto = JsonSerializer.Deserialize(dtoRecord.Payload.ToJson(), SerializerOptions) + ?? throw new InvalidOperationException("Unable to deserialize Apple DTO."); + } + catch (Exception ex) + { + _logger.LogError(ex, "Apple DTO deserialization failed for document {DocumentId}", document.Id); + pendingMappings.Remove(documentId); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); + continue; + } + + var (advisory, flag) = AppleMapper.Map(dto, document, dtoRecord); + _diagnostics.MapAffectedCount(advisory.AffectedPackages.Length); + + await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false); + await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false); + + if (flag is not null) + { + await _psirtFlagStore.UpsertAsync(flag, cancellationToken).ConfigureAwait(false); + } + + pendingMappings.Remove(documentId); + } + + var updatedCursor = cursor.WithPendingMappings(pendingMappings); + await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false); + } + + private AppleIndexEntry RehydrateIndexEntry(DocumentRecord document) + { + var metadata = document.Metadata ?? new Dictionary(StringComparer.Ordinal); + metadata.TryGetValue("apple.articleId", out var articleId); + metadata.TryGetValue("apple.updateId", out var updateId); + metadata.TryGetValue("apple.title", out var title); + metadata.TryGetValue("apple.postingDate", out var postingDateRaw); + metadata.TryGetValue("apple.detailUri", out var detailUriRaw); + metadata.TryGetValue("apple.rapidResponse", out var rapidRaw); + metadata.TryGetValue("apple.products", out var productsJson); + + if (!DateTimeOffset.TryParse(postingDateRaw, out var postingDate)) + { + postingDate = document.FetchedAt; + } + + var detailUri = !string.IsNullOrWhiteSpace(detailUriRaw) && Uri.TryCreate(detailUriRaw, UriKind.Absolute, out var parsedUri) + ? parsedUri + : new Uri(_options.AdvisoryBaseUri!, articleId ?? document.Uri); + + var rapid = string.Equals(rapidRaw, "true", StringComparison.OrdinalIgnoreCase); + var products = DeserializeProducts(productsJson); + + return new AppleIndexEntry( + UpdateId: string.IsNullOrWhiteSpace(updateId) ? articleId ?? document.Uri : updateId, + ArticleId: articleId ?? document.Uri, + Title: title ?? document.Metadata?["apple.originalTitle"] ?? "Apple Security Update", + PostingDate: postingDate.ToUniversalTime(), + DetailUri: detailUri, + Products: products, + IsRapidSecurityResponse: rapid); + } + + private static IReadOnlyList DeserializeProducts(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return Array.Empty(); + } + + try + { + var products = JsonSerializer.Deserialize>(json, SerializerOptions); + return products is { Count: > 0 } ? products : Array.Empty(); + } + catch (JsonException) + { + return Array.Empty(); + } + } + + private static Dictionary BuildMetadata(AppleIndexEntry entry) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["apple.articleId"] = entry.ArticleId, + ["apple.updateId"] = entry.UpdateId, + ["apple.title"] = entry.Title, + ["apple.postingDate"] = entry.PostingDate.ToString("O"), + ["apple.detailUri"] = entry.DetailUri.ToString(), + ["apple.rapidResponse"] = entry.IsRapidSecurityResponse ? "true" : "false", + ["apple.products"] = JsonSerializer.Serialize(entry.Products, SerializerOptions), + }; + + return metadata; + } + + private static bool ShouldInclude(AppleIndexEntry entry, IReadOnlyCollection allowlist, IReadOnlyCollection blocklist) + { + if (allowlist.Count > 0 && !allowlist.Contains(entry.ArticleId)) + { + return false; + } + + if (blocklist.Count > 0 && blocklist.Contains(entry.ArticleId)) + { + return false; + } + + return true; + } + + private async Task GetCursorAsync(CancellationToken cancellationToken) + { + var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false); + return state is null ? AppleCursor.Empty : AppleCursor.FromBson(state.Cursor); + } + + private async Task UpdateCursorAsync(AppleCursor cursor, CancellationToken cancellationToken) + { + var document = cursor.ToBson(); + await _stateRepository.UpdateCursorAsync(SourceName, document, _timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleDependencyInjectionRoutine.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/AppleDependencyInjectionRoutine.cs index 35924cf6..92aaf31d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleDependencyInjectionRoutine.cs @@ -1,53 +1,53 @@ -using System; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:sources:apple"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.AddAppleConnector(options => - { - configuration.GetSection(ConfigurationSection).Bind(options); - options.Validate(); - }); - - services.AddTransient(); - services.AddTransient(); - services.AddTransient(); - - services.PostConfigure(options => - { - EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob)); - EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob)); - EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob)); - }); - - return services; - } - - private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) - { - if (options.Definitions.ContainsKey(kind)) - { - return; - } - - options.Definitions[kind] = new JobDefinition( - kind, - jobType, - options.DefaultTimeout, - options.DefaultLeaseDuration, - CronExpression: null, - Enabled: true); - } -} +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +public sealed class AppleDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:sources:apple"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddAppleConnector(options => + { + configuration.GetSection(ConfigurationSection).Bind(options); + options.Validate(); + }); + + services.AddTransient(); + services.AddTransient(); + services.AddTransient(); + + services.PostConfigure(options => + { + EnsureJob(options, AppleJobKinds.Fetch, typeof(AppleFetchJob)); + EnsureJob(options, AppleJobKinds.Parse, typeof(AppleParseJob)); + EnsureJob(options, AppleJobKinds.Map, typeof(AppleMapJob)); + }); + + return services; + } + + private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType) + { + if (options.Definitions.ContainsKey(kind)) + { + return; + } + + options.Definitions[kind] = new JobDefinition( + kind, + jobType, + options.DefaultTimeout, + options.DefaultLeaseDuration, + CronExpression: null, + Enabled: true); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs index 558188d0..126f2ad4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleOptions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleOptions.cs @@ -1,101 +1,101 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -public sealed class AppleOptions : IValidatableObject -{ - public const string HttpClientName = "feedser-vndr-apple"; - - /// - /// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service). - /// - public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv"); - - /// - /// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required. - /// - public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/"); - - /// - /// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us". - /// - public string LocaleSegment { get; set; } = "en-us"; - - /// - /// Maximum advisories to fetch per run; defaults to 50. - /// - public int MaxAdvisoriesPerFetch { get; set; } = 50; - - /// - /// Sliding backfill window for initial sync (defaults to 90 days). - /// - public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); - - /// - /// Tolerance added to the modified timestamp comparisons during resume. - /// - public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1); - - /// - /// Optional allowlist of HT identifiers to include; empty means include all. - /// - public HashSet AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase); - - /// - /// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed). - /// - public HashSet AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase); - - public IEnumerable Validate(ValidationContext validationContext) - { - if (SoftwareLookupUri is null) - { - yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) }); - } - else if (!SoftwareLookupUri.IsAbsoluteUri) - { - yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) }); - } - - if (AdvisoryBaseUri is null) - { - yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) }); - } - else if (!AdvisoryBaseUri.IsAbsoluteUri) - { - yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) }); - } - - if (string.IsNullOrWhiteSpace(LocaleSegment)) - { - yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) }); - } - - if (MaxAdvisoriesPerFetch <= 0) - { - yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) }); - } - - if (InitialBackfill <= TimeSpan.Zero) - { - yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) }); - } - - if (ModifiedTolerance < TimeSpan.Zero) - { - yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) }); - } - } - - public void Validate() - { - var context = new ValidationContext(this); - var results = new List(); - if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true)) - { - throw new ValidationException(string.Join("; ", results)); - } - } -} +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +public sealed class AppleOptions : IValidatableObject +{ + public const string HttpClientName = "concelier-vndr-apple"; + + /// + /// Gets or sets the JSON endpoint that lists software metadata (defaults to Apple Software Lookup Service). + /// + public Uri? SoftwareLookupUri { get; set; } = new("https://gdmf.apple.com/v2/pmv"); + + /// + /// Gets or sets the base URI for HT advisory pages (locale neutral); trailing slash required. + /// + public Uri? AdvisoryBaseUri { get; set; } = new("https://support.apple.com/"); + + /// + /// Gets or sets the locale segment inserted between the base URI and HT identifier, e.g. "en-us". + /// + public string LocaleSegment { get; set; } = "en-us"; + + /// + /// Maximum advisories to fetch per run; defaults to 50. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 50; + + /// + /// Sliding backfill window for initial sync (defaults to 90 days). + /// + public TimeSpan InitialBackfill { get; set; } = TimeSpan.FromDays(90); + + /// + /// Tolerance added to the modified timestamp comparisons during resume. + /// + public TimeSpan ModifiedTolerance { get; set; } = TimeSpan.FromHours(1); + + /// + /// Optional allowlist of HT identifiers to include; empty means include all. + /// + public HashSet AdvisoryAllowlist { get; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Optional blocklist of HT identifiers to skip (e.g. non-security bulletins that share the feed). + /// + public HashSet AdvisoryBlocklist { get; } = new(StringComparer.OrdinalIgnoreCase); + + public IEnumerable Validate(ValidationContext validationContext) + { + if (SoftwareLookupUri is null) + { + yield return new ValidationResult("SoftwareLookupUri must be provided.", new[] { nameof(SoftwareLookupUri) }); + } + else if (!SoftwareLookupUri.IsAbsoluteUri) + { + yield return new ValidationResult("SoftwareLookupUri must be absolute.", new[] { nameof(SoftwareLookupUri) }); + } + + if (AdvisoryBaseUri is null) + { + yield return new ValidationResult("AdvisoryBaseUri must be provided.", new[] { nameof(AdvisoryBaseUri) }); + } + else if (!AdvisoryBaseUri.IsAbsoluteUri) + { + yield return new ValidationResult("AdvisoryBaseUri must be absolute.", new[] { nameof(AdvisoryBaseUri) }); + } + + if (string.IsNullOrWhiteSpace(LocaleSegment)) + { + yield return new ValidationResult("LocaleSegment must be specified.", new[] { nameof(LocaleSegment) }); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + yield return new ValidationResult("MaxAdvisoriesPerFetch must be greater than zero.", new[] { nameof(MaxAdvisoriesPerFetch) }); + } + + if (InitialBackfill <= TimeSpan.Zero) + { + yield return new ValidationResult("InitialBackfill must be positive.", new[] { nameof(InitialBackfill) }); + } + + if (ModifiedTolerance < TimeSpan.Zero) + { + yield return new ValidationResult("ModifiedTolerance cannot be negative.", new[] { nameof(ModifiedTolerance) }); + } + } + + public void Validate() + { + var context = new ValidationContext(this); + var results = new List(); + if (!Validator.TryValidateObject(this, context, results, validateAllProperties: true)) + { + throw new ValidationException(string.Join("; ", results)); + } + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleServiceCollectionExtensions.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/AppleServiceCollectionExtensions.cs index 7807b3f5..9402f28e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/AppleServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/AppleServiceCollectionExtensions.cs @@ -1,44 +1,44 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Apple.Internal; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -public static class AppleServiceCollectionExtensions -{ - public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configure); - - services.AddOptions() - .Configure(configure) - .PostConfigure(static opts => opts.Validate()) - .ValidateOnStart(); - - services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) => - { - var options = sp.GetRequiredService>().Value; - clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Apple/1.0"; - clientOptions.AllowedHosts.Clear(); - if (options.SoftwareLookupUri is not null) - { - clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host); - } - - if (options.AdvisoryBaseUri is not null) - { - clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); - } - }); - - services.TryAddSingleton(_ => TimeProvider.System); - services.AddSingleton(); - services.AddTransient(); - return services; - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +public static class AppleServiceCollectionExtensions +{ + public static IServiceCollection AddAppleConnector(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions() + .Configure(configure) + .PostConfigure(static opts => opts.Validate()) + .ValidateOnStart(); + + services.AddSourceHttpClient(AppleOptions.HttpClientName, static (sp, clientOptions) => + { + var options = sp.GetRequiredService>().Value; + clientOptions.Timeout = TimeSpan.FromSeconds(30); + clientOptions.UserAgent = "StellaOps.Concelier.Apple/1.0"; + clientOptions.AllowedHosts.Clear(); + if (options.SoftwareLookupUri is not null) + { + clientOptions.AllowedHosts.Add(options.SoftwareLookupUri.Host); + } + + if (options.AdvisoryBaseUri is not null) + { + clientOptions.AllowedHosts.Add(options.AdvisoryBaseUri.Host); + } + }); + + services.TryAddSingleton(_ => TimeProvider.System); + services.AddSingleton(); + services.AddTransient(); + return services; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleCursor.cs index 839b0a44..e2e4b11a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleCursor.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleCursor.cs @@ -1,114 +1,114 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -internal sealed record AppleCursor( - DateTimeOffset? LastPosted, - IReadOnlyCollection ProcessedIds, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); - private static readonly IReadOnlyCollection EmptyStringCollection = Array.Empty(); - - public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection); - - public BsonDocument ToBson() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastPosted.HasValue) - { - document["lastPosted"] = LastPosted.Value.UtcDateTime; - } - - if (ProcessedIds.Count > 0) - { - document["processedIds"] = new BsonArray(ProcessedIds); - } - - return document; - } - - public static AppleCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue) - ? ParseDate(lastPostedValue) - : null; - - var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray - ? processedArray.OfType() - .Where(static value => value.BsonType == BsonType.String) - .Select(static value => value.AsString.Trim()) - .Where(static value => value.Length > 0) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray() - : EmptyStringCollection; - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings); - } - - public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable? processedIds = null) - { - var ids = processedIds is null - ? ProcessedIds - : processedIds.Where(static id => !string.IsNullOrWhiteSpace(id)) - .Select(static id => id.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - - return this with - { - LastPosted = timestamp.ToUniversalTime(), - ProcessedIds = ids, - }; - } - - public AppleCursor WithPendingDocuments(IEnumerable? ids) - => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; - - public AppleCursor WithPendingMappings(IEnumerable? ids) - => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string key) - { - if (!document.TryGetValue(key, out var value) || value is not BsonArray array) - { - return EmptyGuidCollection; - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +internal sealed record AppleCursor( + DateTimeOffset? LastPosted, + IReadOnlyCollection ProcessedIds, + IReadOnlyCollection PendingDocuments, + IReadOnlyCollection PendingMappings) +{ + private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); + private static readonly IReadOnlyCollection EmptyStringCollection = Array.Empty(); + + public static AppleCursor Empty { get; } = new(null, EmptyStringCollection, EmptyGuidCollection, EmptyGuidCollection); + + public BsonDocument ToBson() + { + var document = new BsonDocument + { + ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), + ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), + }; + + if (LastPosted.HasValue) + { + document["lastPosted"] = LastPosted.Value.UtcDateTime; + } + + if (ProcessedIds.Count > 0) + { + document["processedIds"] = new BsonArray(ProcessedIds); + } + + return document; + } + + public static AppleCursor FromBson(BsonDocument? document) + { + if (document is null || document.ElementCount == 0) + { + return Empty; + } + + var lastPosted = document.TryGetValue("lastPosted", out var lastPostedValue) + ? ParseDate(lastPostedValue) + : null; + + var processedIds = document.TryGetValue("processedIds", out var processedValue) && processedValue is BsonArray processedArray + ? processedArray.OfType() + .Where(static value => value.BsonType == BsonType.String) + .Select(static value => value.AsString.Trim()) + .Where(static value => value.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray() + : EmptyStringCollection; + + var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); + var pendingMappings = ReadGuidArray(document, "pendingMappings"); + + return new AppleCursor(lastPosted, processedIds, pendingDocuments, pendingMappings); + } + + public AppleCursor WithLastPosted(DateTimeOffset timestamp, IEnumerable? processedIds = null) + { + var ids = processedIds is null + ? ProcessedIds + : processedIds.Where(static id => !string.IsNullOrWhiteSpace(id)) + .Select(static id => id.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + return this with + { + LastPosted = timestamp.ToUniversalTime(), + ProcessedIds = ids, + }; + } + + public AppleCursor WithPendingDocuments(IEnumerable? ids) + => this with { PendingDocuments = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + public AppleCursor WithPendingMappings(IEnumerable? ids) + => this with { PendingMappings = ids?.Distinct().ToArray() ?? EmptyGuidCollection }; + + private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string key) + { + if (!document.TryGetValue(key, out var value) || value is not BsonArray array) + { + return EmptyGuidCollection; + } + + var results = new List(array.Count); + foreach (var element in array) + { + if (Guid.TryParse(element.ToString(), out var guid)) + { + results.Add(guid); + } + } + + return results; + } + + private static DateTimeOffset? ParseDate(BsonValue value) + => value.BsonType switch + { + BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), + BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), + _ => null, + }; +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailDto.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailDto.cs index 50bc23a9..2d0adb61 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailDto.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailDto.cs @@ -1,78 +1,78 @@ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -internal sealed record AppleDetailDto( - string AdvisoryId, - string ArticleId, - string Title, - string Summary, - DateTimeOffset Published, - DateTimeOffset? Updated, - IReadOnlyList CveIds, - IReadOnlyList Affected, - IReadOnlyList References, - bool RapidSecurityResponse); - -internal sealed record AppleAffectedProductDto( - string Platform, - string Name, - string Version, - string Build); - -internal sealed record AppleReferenceDto( - string Url, - string? Title, - string? Kind); - -internal static class AppleDetailDtoExtensions -{ - public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable products) - { - if (dto.Affected.Count > 0) - { - return dto; - } - - var fallback = products - .Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build)) - .Select(static product => new AppleAffectedProductDto( - product.Platform, - product.Name, - product.Version, - product.Build)) - .OrderBy(static product => NormalizeSortKey(product.Platform)) - .ThenBy(static product => NormalizeSortKey(product.Name)) - .ThenBy(static product => NormalizeSortKey(product.Version)) - .ThenBy(static product => NormalizeSortKey(product.Build)) - .ToArray(); - - return fallback.Length == 0 ? dto : dto with { Affected = fallback }; - } - - private static string NormalizeSortKey(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var span = value.AsSpan(); - var buffer = new char[span.Length]; - var index = 0; - - foreach (var ch in span) - { - if (char.IsWhiteSpace(ch)) - { - continue; - } - - buffer[index++] = char.ToUpperInvariant(ch); - } - - return new string(buffer, 0, index); - } -} +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +internal sealed record AppleDetailDto( + string AdvisoryId, + string ArticleId, + string Title, + string Summary, + DateTimeOffset Published, + DateTimeOffset? Updated, + IReadOnlyList CveIds, + IReadOnlyList Affected, + IReadOnlyList References, + bool RapidSecurityResponse); + +internal sealed record AppleAffectedProductDto( + string Platform, + string Name, + string Version, + string Build); + +internal sealed record AppleReferenceDto( + string Url, + string? Title, + string? Kind); + +internal static class AppleDetailDtoExtensions +{ + public static AppleDetailDto WithAffectedFallback(this AppleDetailDto dto, IEnumerable products) + { + if (dto.Affected.Count > 0) + { + return dto; + } + + var fallback = products + .Where(static product => !string.IsNullOrWhiteSpace(product.Version) || !string.IsNullOrWhiteSpace(product.Build)) + .Select(static product => new AppleAffectedProductDto( + product.Platform, + product.Name, + product.Version, + product.Build)) + .OrderBy(static product => NormalizeSortKey(product.Platform)) + .ThenBy(static product => NormalizeSortKey(product.Name)) + .ThenBy(static product => NormalizeSortKey(product.Version)) + .ThenBy(static product => NormalizeSortKey(product.Build)) + .ToArray(); + + return fallback.Length == 0 ? dto : dto with { Affected = fallback }; + } + + private static string NormalizeSortKey(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + + foreach (var ch in span) + { + if (char.IsWhiteSpace(ch)) + { + continue; + } + + buffer[index++] = char.ToUpperInvariant(ch); + } + + return new string(buffer, 0, index); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailParser.cs index eb44ecd3..f84bd8d2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDetailParser.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDetailParser.cs @@ -1,460 +1,460 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using AngleSharp.Dom; -using AngleSharp.Html.Dom; -using AngleSharp.Html.Parser; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -internal static class AppleDetailParser -{ - private static readonly HtmlParser Parser = new(); - private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase); - - public static AppleDetailDto Parse(string html, AppleIndexEntry entry) - { - if (string.IsNullOrWhiteSpace(html)) - { - throw new ArgumentException("HTML content must not be empty.", nameof(html)); - } - - var document = Parser.ParseDocument(html); - var title = ResolveTitle(document, entry.Title); - var summary = ResolveSummary(document); - var (published, updated) = ResolveTimestamps(document, entry.PostingDate); - var cves = ExtractCves(document); - var affected = NormalizeAffectedProducts(ExtractProducts(document)); - var references = ExtractReferences(document, entry.DetailUri); - - var dto = new AppleDetailDto( - entry.ArticleId, - entry.ArticleId, - title, - summary, - published, - updated, - cves, - affected, - references, - entry.IsRapidSecurityResponse); - - return dto.WithAffectedFallback(entry.Products); - } - - private static IReadOnlyList NormalizeAffectedProducts(IReadOnlyList affected) - { - if (affected.Count <= 1) - { - return affected; - } - - return affected - .OrderBy(static product => NormalizeSortKey(product.Platform)) - .ThenBy(static product => NormalizeSortKey(product.Name)) - .ThenBy(static product => NormalizeSortKey(product.Version)) - .ThenBy(static product => NormalizeSortKey(product.Build)) - .ToArray(); - } - - private static string NormalizeSortKey(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var span = value.AsSpan(); - var buffer = new char[span.Length]; - var index = 0; - - foreach (var ch in span) - { - if (char.IsWhiteSpace(ch)) - { - continue; - } - - buffer[index++] = char.ToUpperInvariant(ch); - } - - return new string(buffer, 0, index); - } - - private static string ResolveTitle(IHtmlDocument document, string fallback) - { - var title = document.QuerySelector("[data-testid='update-title']")?.TextContent - ?? document.QuerySelector("h1, h2")?.TextContent - ?? document.Title; - - title = title?.Trim(); - return string.IsNullOrEmpty(title) ? fallback : title; - } - - private static string ResolveSummary(IHtmlDocument document) - { - var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent - ?? document.QuerySelector("meta[name='description']")?.GetAttribute("content") - ?? document.QuerySelector("p")?.TextContent - ?? string.Empty; - - return CleanWhitespace(summary); - } - - private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback) - { - DateTimeOffset published = postingFallback; - DateTimeOffset? updated = null; - - foreach (var time in document.QuerySelectorAll("time")) - { - var raw = time.GetAttribute("datetime") ?? time.TextContent; - if (string.IsNullOrWhiteSpace(raw)) - { - continue; - } - - if (!DateTimeOffset.TryParse(raw, out var parsed)) - { - continue; - } - - parsed = parsed.ToUniversalTime(); - - var itemProp = time.GetAttribute("itemprop") ?? string.Empty; - var dataTestId = time.GetAttribute("data-testid") ?? string.Empty; - - if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase) - || dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase)) - { - published = parsed; - } - else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase) - || dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase)) - { - updated = parsed; - } - else if (updated is null && parsed > published) - { - updated = parsed; - } - } - - return (published, updated); - } - - private static IReadOnlyList ExtractCves(IHtmlDocument document) - { - var set = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var node in document.All) - { - if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element) - { - continue; - } - - var text = node.TextContent; - if (string.IsNullOrWhiteSpace(text)) - { - continue; - } - - foreach (Match match in CveRegex.Matches(text)) - { - if (match.Success) - { - set.Add(match.Value.ToUpperInvariant()); - } - } - } - - if (set.Count == 0) - { - return Array.Empty(); - } - - var list = set.ToList(); - list.Sort(StringComparer.OrdinalIgnoreCase); - return list; - } - - private static IReadOnlyList ExtractProducts(IHtmlDocument document) - { - var rows = new List(); - - foreach (var element in document.QuerySelectorAll("[data-testid='product-row']")) - { - var platform = element.GetAttribute("data-platform") ?? string.Empty; - var name = element.GetAttribute("data-product") ?? platform; - var version = element.GetAttribute("data-version") ?? string.Empty; - var build = element.GetAttribute("data-build") ?? string.Empty; - - if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow) - { - var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); - if (cells.Length >= 1) - { - name = cells[0]; - } - - if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version)) - { - version = cells[1]; - } - - if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build)) - { - build = cells[2]; - } - } - - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - - rows.Add(new AppleAffectedProductDto(platform, name, version, build)); - } - - if (rows.Count > 0) - { - return rows; - } - - // fallback for generic tables without data attributes - foreach (var table in document.QuerySelectorAll("table")) - { - var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray(); - if (headers.Length == 0) - { - continue; - } - - var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase) - || header.Contains("device", StringComparison.OrdinalIgnoreCase)); - var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase)); - var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase) - || header.Contains("release", StringComparison.OrdinalIgnoreCase)); - - if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1) - { - continue; - } - - foreach (var row in table.QuerySelectorAll("tr")) - { - var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); - if (cells.Length == 0) - { - continue; - } - - string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0]; - string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty; - string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty; - - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - - rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build)); - } - - if (rows.Count > 0) - { - break; - } - } - - return rows.Count == 0 ? Array.Empty() : rows; - } - - private static IReadOnlyList ExtractReferences(IHtmlDocument document, Uri detailUri) - { - var scope = document.QuerySelector("article") - ?? document.QuerySelector("main") - ?? (IElement?)document.Body; - - if (scope is null) - { - return Array.Empty(); - } - - var anchors = scope.QuerySelectorAll("a[href]"); - if (anchors.Length == 0) - { - return Array.Empty(); - } - - var references = new List(anchors.Length); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var element in anchors) - { - if (element is not IHtmlAnchorElement anchor) - { - continue; - } - - if (anchor.HasAttribute("data-globalnav-item-name") - || anchor.HasAttribute("data-analytics-title")) - { - continue; - } - - var href = anchor.GetAttribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (!Uri.TryCreate(detailUri, href, out var uri)) - { - continue; - } - - if (!IsRelevantReference(uri)) - { - continue; - } - - var normalized = uri.ToString(); - if (!seen.Add(normalized)) - { - continue; - } - - var title = CleanWhitespace(anchor.TextContent); - var kind = ResolveReferenceKind(uri); - references.Add(new AppleReferenceDto(normalized, title, kind)); - } - - if (references.Count == 0) - { - return Array.Empty(); - } - - references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); - return references; - } - - private static bool IsRelevantReference(Uri uri) - { - if (!uri.IsAbsoluteUri) - { - return false; - } - - if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase) - && !string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - - if (string.IsNullOrWhiteSpace(uri.Host)) - { - return false; - } - - if (uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase)) - { - if (!uri.AbsolutePath.Contains("/support/", StringComparison.OrdinalIgnoreCase) - && !uri.AbsolutePath.Contains("/security", StringComparison.OrdinalIgnoreCase) - && !uri.AbsolutePath.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase)) - { - return false; - } - } - - if (uri.Host.EndsWith(".apple.com", StringComparison.OrdinalIgnoreCase)) - { - var supported = - uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) - || uri.Host.StartsWith("developer.", StringComparison.OrdinalIgnoreCase) - || uri.Host.StartsWith("download.", StringComparison.OrdinalIgnoreCase) - || uri.Host.StartsWith("updates.", StringComparison.OrdinalIgnoreCase) - || uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase); - - if (!supported) - { - return false; - } - - if (uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) - && uri.AbsolutePath == "/" - && (string.IsNullOrEmpty(uri.Query) - || uri.Query.Contains("cid=", StringComparison.OrdinalIgnoreCase))) - { - return false; - } - } - - return true; - } - - private static string? ResolveReferenceKind(Uri uri) - { - if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase)) - { - if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase)) - { - return "download"; - } - - if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase)) - { - return "document"; - } - - return "advisory"; - } - - if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase)) - { - return "nvd"; - } - - if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase)) - { - return "kb"; - } - - return null; - } - - private static string CleanWhitespace(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return string.Empty; - } - - var span = value.AsSpan(); - var buffer = new char[span.Length]; - var index = 0; - var previousWhitespace = false; - - foreach (var ch in span) - { - if (char.IsWhiteSpace(ch)) - { - if (previousWhitespace) - { - continue; - } - - buffer[index++] = ' '; - previousWhitespace = true; - } - else - { - buffer[index++] = ch; - previousWhitespace = false; - } - } - - return new string(buffer, 0, index).Trim(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Dom; +using AngleSharp.Html.Parser; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +internal static class AppleDetailParser +{ + private static readonly HtmlParser Parser = new(); + private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + public static AppleDetailDto Parse(string html, AppleIndexEntry entry) + { + if (string.IsNullOrWhiteSpace(html)) + { + throw new ArgumentException("HTML content must not be empty.", nameof(html)); + } + + var document = Parser.ParseDocument(html); + var title = ResolveTitle(document, entry.Title); + var summary = ResolveSummary(document); + var (published, updated) = ResolveTimestamps(document, entry.PostingDate); + var cves = ExtractCves(document); + var affected = NormalizeAffectedProducts(ExtractProducts(document)); + var references = ExtractReferences(document, entry.DetailUri); + + var dto = new AppleDetailDto( + entry.ArticleId, + entry.ArticleId, + title, + summary, + published, + updated, + cves, + affected, + references, + entry.IsRapidSecurityResponse); + + return dto.WithAffectedFallback(entry.Products); + } + + private static IReadOnlyList NormalizeAffectedProducts(IReadOnlyList affected) + { + if (affected.Count <= 1) + { + return affected; + } + + return affected + .OrderBy(static product => NormalizeSortKey(product.Platform)) + .ThenBy(static product => NormalizeSortKey(product.Name)) + .ThenBy(static product => NormalizeSortKey(product.Version)) + .ThenBy(static product => NormalizeSortKey(product.Build)) + .ToArray(); + } + + private static string NormalizeSortKey(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + + foreach (var ch in span) + { + if (char.IsWhiteSpace(ch)) + { + continue; + } + + buffer[index++] = char.ToUpperInvariant(ch); + } + + return new string(buffer, 0, index); + } + + private static string ResolveTitle(IHtmlDocument document, string fallback) + { + var title = document.QuerySelector("[data-testid='update-title']")?.TextContent + ?? document.QuerySelector("h1, h2")?.TextContent + ?? document.Title; + + title = title?.Trim(); + return string.IsNullOrEmpty(title) ? fallback : title; + } + + private static string ResolveSummary(IHtmlDocument document) + { + var summary = document.QuerySelector("[data-testid='update-summary']")?.TextContent + ?? document.QuerySelector("meta[name='description']")?.GetAttribute("content") + ?? document.QuerySelector("p")?.TextContent + ?? string.Empty; + + return CleanWhitespace(summary); + } + + private static (DateTimeOffset Published, DateTimeOffset? Updated) ResolveTimestamps(IHtmlDocument document, DateTimeOffset postingFallback) + { + DateTimeOffset published = postingFallback; + DateTimeOffset? updated = null; + + foreach (var time in document.QuerySelectorAll("time")) + { + var raw = time.GetAttribute("datetime") ?? time.TextContent; + if (string.IsNullOrWhiteSpace(raw)) + { + continue; + } + + if (!DateTimeOffset.TryParse(raw, out var parsed)) + { + continue; + } + + parsed = parsed.ToUniversalTime(); + + var itemProp = time.GetAttribute("itemprop") ?? string.Empty; + var dataTestId = time.GetAttribute("data-testid") ?? string.Empty; + + if (itemProp.Equals("datePublished", StringComparison.OrdinalIgnoreCase) + || dataTestId.Equals("published", StringComparison.OrdinalIgnoreCase)) + { + published = parsed; + } + else if (itemProp.Equals("dateModified", StringComparison.OrdinalIgnoreCase) + || dataTestId.Equals("updated", StringComparison.OrdinalIgnoreCase)) + { + updated = parsed; + } + else if (updated is null && parsed > published) + { + updated = parsed; + } + } + + return (published, updated); + } + + private static IReadOnlyList ExtractCves(IHtmlDocument document) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var node in document.All) + { + if (node.NodeType != NodeType.Text && node.NodeType != NodeType.Element) + { + continue; + } + + var text = node.TextContent; + if (string.IsNullOrWhiteSpace(text)) + { + continue; + } + + foreach (Match match in CveRegex.Matches(text)) + { + if (match.Success) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + } + + if (set.Count == 0) + { + return Array.Empty(); + } + + var list = set.ToList(); + list.Sort(StringComparer.OrdinalIgnoreCase); + return list; + } + + private static IReadOnlyList ExtractProducts(IHtmlDocument document) + { + var rows = new List(); + + foreach (var element in document.QuerySelectorAll("[data-testid='product-row']")) + { + var platform = element.GetAttribute("data-platform") ?? string.Empty; + var name = element.GetAttribute("data-product") ?? platform; + var version = element.GetAttribute("data-version") ?? string.Empty; + var build = element.GetAttribute("data-build") ?? string.Empty; + + if (string.IsNullOrWhiteSpace(name) && element is IHtmlTableRowElement tableRow) + { + var cells = tableRow.Cells.Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); + if (cells.Length >= 1) + { + name = cells[0]; + } + + if (cells.Length >= 2 && string.IsNullOrWhiteSpace(version)) + { + version = cells[1]; + } + + if (cells.Length >= 3 && string.IsNullOrWhiteSpace(build)) + { + build = cells[2]; + } + } + + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + rows.Add(new AppleAffectedProductDto(platform, name, version, build)); + } + + if (rows.Count > 0) + { + return rows; + } + + // fallback for generic tables without data attributes + foreach (var table in document.QuerySelectorAll("table")) + { + var headers = table.QuerySelectorAll("th").Select(static th => CleanWhitespace(th.TextContent)).ToArray(); + if (headers.Length == 0) + { + continue; + } + + var nameIndex = Array.FindIndex(headers, static header => header.Contains("product", StringComparison.OrdinalIgnoreCase) + || header.Contains("device", StringComparison.OrdinalIgnoreCase)); + var versionIndex = Array.FindIndex(headers, static header => header.Contains("version", StringComparison.OrdinalIgnoreCase)); + var buildIndex = Array.FindIndex(headers, static header => header.Contains("build", StringComparison.OrdinalIgnoreCase) + || header.Contains("release", StringComparison.OrdinalIgnoreCase)); + + if (nameIndex == -1 && versionIndex == -1 && buildIndex == -1) + { + continue; + } + + foreach (var row in table.QuerySelectorAll("tr")) + { + var cells = row.QuerySelectorAll("td").Select(static cell => CleanWhitespace(cell.TextContent)).ToArray(); + if (cells.Length == 0) + { + continue; + } + + string name = nameIndex >= 0 && nameIndex < cells.Length ? cells[nameIndex] : cells[0]; + string version = versionIndex >= 0 && versionIndex < cells.Length ? cells[versionIndex] : string.Empty; + string build = buildIndex >= 0 && buildIndex < cells.Length ? cells[buildIndex] : string.Empty; + + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + rows.Add(new AppleAffectedProductDto(string.Empty, name, version, build)); + } + + if (rows.Count > 0) + { + break; + } + } + + return rows.Count == 0 ? Array.Empty() : rows; + } + + private static IReadOnlyList ExtractReferences(IHtmlDocument document, Uri detailUri) + { + var scope = document.QuerySelector("article") + ?? document.QuerySelector("main") + ?? (IElement?)document.Body; + + if (scope is null) + { + return Array.Empty(); + } + + var anchors = scope.QuerySelectorAll("a[href]"); + if (anchors.Length == 0) + { + return Array.Empty(); + } + + var references = new List(anchors.Length); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var element in anchors) + { + if (element is not IHtmlAnchorElement anchor) + { + continue; + } + + if (anchor.HasAttribute("data-globalnav-item-name") + || anchor.HasAttribute("data-analytics-title")) + { + continue; + } + + var href = anchor.GetAttribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (!Uri.TryCreate(detailUri, href, out var uri)) + { + continue; + } + + if (!IsRelevantReference(uri)) + { + continue; + } + + var normalized = uri.ToString(); + if (!seen.Add(normalized)) + { + continue; + } + + var title = CleanWhitespace(anchor.TextContent); + var kind = ResolveReferenceKind(uri); + references.Add(new AppleReferenceDto(normalized, title, kind)); + } + + if (references.Count == 0) + { + return Array.Empty(); + } + + references.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return references; + } + + private static bool IsRelevantReference(Uri uri) + { + if (!uri.IsAbsoluteUri) + { + return false; + } + + if (!string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase) + && !string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (string.IsNullOrWhiteSpace(uri.Host)) + { + return false; + } + + if (uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase)) + { + if (!uri.AbsolutePath.Contains("/support/", StringComparison.OrdinalIgnoreCase) + && !uri.AbsolutePath.Contains("/security", StringComparison.OrdinalIgnoreCase) + && !uri.AbsolutePath.EndsWith(".pdf", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + } + + if (uri.Host.EndsWith(".apple.com", StringComparison.OrdinalIgnoreCase)) + { + var supported = + uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) + || uri.Host.StartsWith("developer.", StringComparison.OrdinalIgnoreCase) + || uri.Host.StartsWith("download.", StringComparison.OrdinalIgnoreCase) + || uri.Host.StartsWith("updates.", StringComparison.OrdinalIgnoreCase) + || uri.Host.Equals("www.apple.com", StringComparison.OrdinalIgnoreCase); + + if (!supported) + { + return false; + } + + if (uri.Host.StartsWith("support.", StringComparison.OrdinalIgnoreCase) + && uri.AbsolutePath == "/" + && (string.IsNullOrEmpty(uri.Query) + || uri.Query.Contains("cid=", StringComparison.OrdinalIgnoreCase))) + { + return false; + } + } + + return true; + } + + private static string? ResolveReferenceKind(Uri uri) + { + if (uri.Host.Contains("apple.com", StringComparison.OrdinalIgnoreCase)) + { + if (uri.AbsolutePath.Contains("download", StringComparison.OrdinalIgnoreCase)) + { + return "download"; + } + + if (uri.AbsolutePath.Contains(".pdf", StringComparison.OrdinalIgnoreCase)) + { + return "document"; + } + + return "advisory"; + } + + if (uri.Host.Contains("nvd.nist.gov", StringComparison.OrdinalIgnoreCase)) + { + return "nvd"; + } + + if (uri.Host.Contains("support", StringComparison.OrdinalIgnoreCase)) + { + return "kb"; + } + + return null; + } + + private static string CleanWhitespace(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return string.Empty; + } + + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + var previousWhitespace = false; + + foreach (var ch in span) + { + if (char.IsWhiteSpace(ch)) + { + if (previousWhitespace) + { + continue; + } + + buffer[index++] = ' '; + previousWhitespace = true; + } + else + { + buffer[index++] = ch; + previousWhitespace = false; + } + } + + return new string(buffer, 0, index).Trim(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDiagnostics.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDiagnostics.cs index caa94508..ba37f651 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleDiagnostics.cs @@ -1,62 +1,62 @@ -using System; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -public sealed class AppleDiagnostics : IDisposable -{ - public const string MeterName = "StellaOps.Feedser.Source.Vndr.Apple"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _fetchItems; - private readonly Counter _fetchFailures; - private readonly Counter _fetchUnchanged; - private readonly Counter _parseFailures; - private readonly Histogram _mapAffected; - - public AppleDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchItems = _meter.CreateCounter( - name: "apple.fetch.items", - unit: "documents", - description: "Number of Apple advisories fetched."); - _fetchFailures = _meter.CreateCounter( - name: "apple.fetch.failures", - unit: "operations", - description: "Number of Apple fetch failures."); - _fetchUnchanged = _meter.CreateCounter( - name: "apple.fetch.unchanged", - unit: "documents", - description: "Number of Apple advisories skipped due to 304 responses."); - _parseFailures = _meter.CreateCounter( - name: "apple.parse.failures", - unit: "documents", - description: "Number of Apple documents that failed to parse."); - _mapAffected = _meter.CreateHistogram( - name: "apple.map.affected.count", - unit: "packages", - description: "Distribution of affected package counts emitted per Apple advisory."); - } - - public Meter Meter => _meter; - - public void FetchItem() => _fetchItems.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void ParseFailure() => _parseFailures.Add(1); - - public void MapAffectedCount(int count) - { - if (count >= 0) - { - _mapAffected.Record(count); - } - } - - public void Dispose() => _meter.Dispose(); -} +using System; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +public sealed class AppleDiagnostics : IDisposable +{ + public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Apple"; + private const string MeterVersion = "1.0.0"; + + private readonly Meter _meter; + private readonly Counter _fetchItems; + private readonly Counter _fetchFailures; + private readonly Counter _fetchUnchanged; + private readonly Counter _parseFailures; + private readonly Histogram _mapAffected; + + public AppleDiagnostics() + { + _meter = new Meter(MeterName, MeterVersion); + _fetchItems = _meter.CreateCounter( + name: "apple.fetch.items", + unit: "documents", + description: "Number of Apple advisories fetched."); + _fetchFailures = _meter.CreateCounter( + name: "apple.fetch.failures", + unit: "operations", + description: "Number of Apple fetch failures."); + _fetchUnchanged = _meter.CreateCounter( + name: "apple.fetch.unchanged", + unit: "documents", + description: "Number of Apple advisories skipped due to 304 responses."); + _parseFailures = _meter.CreateCounter( + name: "apple.parse.failures", + unit: "documents", + description: "Number of Apple documents that failed to parse."); + _mapAffected = _meter.CreateHistogram( + name: "apple.map.affected.count", + unit: "packages", + description: "Distribution of affected package counts emitted per Apple advisory."); + } + + public Meter Meter => _meter; + + public void FetchItem() => _fetchItems.Add(1); + + public void FetchFailure() => _fetchFailures.Add(1); + + public void FetchUnchanged() => _fetchUnchanged.Add(1); + + public void ParseFailure() => _parseFailures.Add(1); + + public void MapAffectedCount(int count) + { + if (count >= 0) + { + _mapAffected.Record(count); + } + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleIndexEntry.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleIndexEntry.cs index 387769f1..2dc8fbf7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleIndexEntry.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleIndexEntry.cs @@ -1,144 +1,144 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -internal sealed record AppleIndexEntry( - string UpdateId, - string ArticleId, - string Title, - DateTimeOffset PostingDate, - Uri DetailUri, - IReadOnlyList Products, - bool IsRapidSecurityResponse); - -internal sealed record AppleIndexProduct( - string Platform, - string Name, - string Version, - string Build); - -internal static class AppleIndexParser -{ - private sealed record AppleIndexDocument( - [property: JsonPropertyName("updates")] IReadOnlyList? Updates); - - private sealed record AppleIndexEntryDto( - [property: JsonPropertyName("id")] string? Id, - [property: JsonPropertyName("articleId")] string? ArticleId, - [property: JsonPropertyName("title")] string? Title, - [property: JsonPropertyName("postingDate")] string? PostingDate, - [property: JsonPropertyName("detailUrl")] string? DetailUrl, - [property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse, - [property: JsonPropertyName("products")] IReadOnlyList? Products); - - private sealed record AppleIndexProductDto( - [property: JsonPropertyName("platform")] string? Platform, - [property: JsonPropertyName("name")] string? Name, - [property: JsonPropertyName("version")] string? Version, - [property: JsonPropertyName("build")] string? Build); - - public static IReadOnlyList Parse(ReadOnlySpan payload, Uri baseUri) - { - if (payload.IsEmpty) - { - return Array.Empty(); - } - - var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - }; - - AppleIndexDocument? document; - try - { - document = JsonSerializer.Deserialize(payload, options); - } - catch (JsonException) - { - return Array.Empty(); - } - - if (document?.Updates is null || document.Updates.Count == 0) - { - return Array.Empty(); - } - - var entries = new List(document.Updates.Count); - foreach (var dto in document.Updates) - { - if (dto is null) - { - continue; - } - - var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id; - if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId)) - { - continue; - } - - if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate)) - { - continue; - } - - if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate)) - { - continue; - } - - if (!TryResolveDetailUri(dto, baseUri, out var detailUri)) - { - continue; - } - - var products = dto.Products?.Select(static product => new AppleIndexProduct( - product.Platform ?? string.Empty, - product.Name ?? product.Platform ?? string.Empty, - product.Version ?? string.Empty, - product.Build ?? string.Empty)) - .ToArray() ?? Array.Empty(); - - entries.Add(new AppleIndexEntry( - id.Trim(), - dto.ArticleId!.Trim(), - dto.Title!.Trim(), - postingDate.ToUniversalTime(), - detailUri, - products, - dto.RapidSecurityResponse ?? false)); - } - - return entries.Count == 0 ? Array.Empty() : entries; - } - - private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri) - { - if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri)) - { - return true; - } - - if (string.IsNullOrWhiteSpace(dto.ArticleId)) - { - uri = default!; - return false; - } - - var article = dto.ArticleId.Trim(); - if (article.Length == 0) - { - uri = default!; - return false; - } - - var combined = new Uri(baseUri, article); - uri = combined; - return true; - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +internal sealed record AppleIndexEntry( + string UpdateId, + string ArticleId, + string Title, + DateTimeOffset PostingDate, + Uri DetailUri, + IReadOnlyList Products, + bool IsRapidSecurityResponse); + +internal sealed record AppleIndexProduct( + string Platform, + string Name, + string Version, + string Build); + +internal static class AppleIndexParser +{ + private sealed record AppleIndexDocument( + [property: JsonPropertyName("updates")] IReadOnlyList? Updates); + + private sealed record AppleIndexEntryDto( + [property: JsonPropertyName("id")] string? Id, + [property: JsonPropertyName("articleId")] string? ArticleId, + [property: JsonPropertyName("title")] string? Title, + [property: JsonPropertyName("postingDate")] string? PostingDate, + [property: JsonPropertyName("detailUrl")] string? DetailUrl, + [property: JsonPropertyName("rapidSecurityResponse")] bool? RapidSecurityResponse, + [property: JsonPropertyName("products")] IReadOnlyList? Products); + + private sealed record AppleIndexProductDto( + [property: JsonPropertyName("platform")] string? Platform, + [property: JsonPropertyName("name")] string? Name, + [property: JsonPropertyName("version")] string? Version, + [property: JsonPropertyName("build")] string? Build); + + public static IReadOnlyList Parse(ReadOnlySpan payload, Uri baseUri) + { + if (payload.IsEmpty) + { + return Array.Empty(); + } + + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + }; + + AppleIndexDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, options); + } + catch (JsonException) + { + return Array.Empty(); + } + + if (document?.Updates is null || document.Updates.Count == 0) + { + return Array.Empty(); + } + + var entries = new List(document.Updates.Count); + foreach (var dto in document.Updates) + { + if (dto is null) + { + continue; + } + + var id = string.IsNullOrWhiteSpace(dto.Id) ? dto.ArticleId : dto.Id; + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(dto.ArticleId)) + { + continue; + } + + if (string.IsNullOrWhiteSpace(dto.Title) || string.IsNullOrWhiteSpace(dto.PostingDate)) + { + continue; + } + + if (!DateTimeOffset.TryParse(dto.PostingDate, out var postingDate)) + { + continue; + } + + if (!TryResolveDetailUri(dto, baseUri, out var detailUri)) + { + continue; + } + + var products = dto.Products?.Select(static product => new AppleIndexProduct( + product.Platform ?? string.Empty, + product.Name ?? product.Platform ?? string.Empty, + product.Version ?? string.Empty, + product.Build ?? string.Empty)) + .ToArray() ?? Array.Empty(); + + entries.Add(new AppleIndexEntry( + id.Trim(), + dto.ArticleId!.Trim(), + dto.Title!.Trim(), + postingDate.ToUniversalTime(), + detailUri, + products, + dto.RapidSecurityResponse ?? false)); + } + + return entries.Count == 0 ? Array.Empty() : entries; + } + + private static bool TryResolveDetailUri(AppleIndexEntryDto dto, Uri baseUri, out Uri uri) + { + if (!string.IsNullOrWhiteSpace(dto.DetailUrl) && Uri.TryCreate(dto.DetailUrl, UriKind.Absolute, out uri)) + { + return true; + } + + if (string.IsNullOrWhiteSpace(dto.ArticleId)) + { + uri = default!; + return false; + } + + var article = dto.ArticleId.Trim(); + if (article.Length == 0) + { + uri = default!; + return false; + } + + var combined = new Uri(baseUri, article); + uri = combined; + return true; + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleMapper.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleMapper.cs index 240b05bd..7501556e 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Internal/AppleMapper.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Internal/AppleMapper.cs @@ -1,281 +1,281 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; - -namespace StellaOps.Feedser.Source.Vndr.Apple.Internal; - -internal static class AppleMapper -{ - public static (Advisory Advisory, PsirtFlagRecord? Flag) Map( - AppleDetailDto dto, - DocumentRecord document, - DtoRecord dtoRecord) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(dtoRecord); - - var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); - - var fetchProvenance = new AdvisoryProvenance( - VndrAppleConnectorPlugin.SourceName, - "document", - document.Uri, - document.FetchedAt.ToUniversalTime()); - - var mapProvenance = new AdvisoryProvenance( - VndrAppleConnectorPlugin.SourceName, - "map", - dto.AdvisoryId, - recordedAt); - - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var affected = BuildAffected(dto, recordedAt); - - var advisory = new Advisory( - advisoryKey: dto.AdvisoryId, - title: dto.Title, - summary: dto.Summary, - language: "en", - published: dto.Published.ToUniversalTime(), - modified: dto.Updated?.ToUniversalTime(), - severity: null, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affected, - cvssMetrics: Array.Empty(), - provenance: new[] { fetchProvenance, mapProvenance }); - - PsirtFlagRecord? flag = dto.RapidSecurityResponse - ? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt) - : null; - - return (advisory, flag); - } - - private static IReadOnlyList BuildAliases(AppleDetailDto dto) - { - var set = new HashSet(StringComparer.OrdinalIgnoreCase) - { - dto.AdvisoryId, - dto.ArticleId, - }; - - foreach (var cve in dto.CveIds) - { - if (!string.IsNullOrWhiteSpace(cve)) - { - set.Add(cve.Trim()); - } - } - - var aliases = set.ToList(); - aliases.Sort(StringComparer.OrdinalIgnoreCase); - return aliases; - } - - private static IReadOnlyList BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt) - { - if (dto.References.Count == 0) - { - return Array.Empty(); - } - - var list = new List(dto.References.Count); - foreach (var reference in dto.References) - { - if (string.IsNullOrWhiteSpace(reference.Url)) - { - continue; - } - - try - { - var provenance = new AdvisoryProvenance( - VndrAppleConnectorPlugin.SourceName, - "reference", - reference.Url, - recordedAt); - - list.Add(new AdvisoryReference( - url: reference.Url, - kind: reference.Kind, - sourceTag: null, - summary: reference.Title, - provenance: provenance)); - } - catch (ArgumentException) - { - // ignore invalid URLs - } - } - - if (list.Count == 0) - { - return Array.Empty(); - } - - list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); - return list; - } - - private static IReadOnlyList BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt) - { - if (dto.Affected.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Affected.Count); - foreach (var product in dto.Affected) - { - if (string.IsNullOrWhiteSpace(product.Name)) - { - continue; - } - - var provenance = new[] - { - new AdvisoryProvenance( - VndrAppleConnectorPlugin.SourceName, - "affected", - product.Name, - recordedAt), - }; - - var ranges = BuildRanges(product, recordedAt); - var normalizedVersions = BuildNormalizedVersions(product, ranges); - - packages.Add(new AffectedPackage( - type: AffectedPackageTypes.Vendor, - identifier: product.Name, - platform: product.Platform, - versionRanges: ranges, - statuses: Array.Empty(), - provenance: provenance, - normalizedVersions: normalizedVersions)); - } - - return packages.Count == 0 ? Array.Empty() : packages; - } - - private static IReadOnlyList BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt) - { - if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build)) - { - return Array.Empty(); - } - - var provenance = new AdvisoryProvenance( - VndrAppleConnectorPlugin.SourceName, - "range", - product.Name, - recordedAt); - - var extensions = new Dictionary(StringComparer.Ordinal); - if (!string.IsNullOrWhiteSpace(product.Version)) - { - extensions["apple.version.raw"] = product.Version; - } - - if (!string.IsNullOrWhiteSpace(product.Build)) - { - extensions["apple.build"] = product.Build; - } - - if (!string.IsNullOrWhiteSpace(product.Platform)) - { - extensions["apple.platform"] = product.Platform; - } - - var primitives = extensions.Count == 0 - ? null - : new RangePrimitives( - SemVer: TryCreateSemVerPrimitive(product.Version), - Nevra: null, - Evr: null, - VendorExtensions: extensions); - - var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion) - ? normalizedVersion - : product.Version; - - return new[] - { - new AffectedVersionRange( - rangeKind: "vendor", - introducedVersion: null, - fixedVersion: sanitizedVersion, - lastAffectedVersion: null, - rangeExpression: product.Version, - provenance: provenance, - primitives: primitives), - }; - } - - private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version) - { - if (string.IsNullOrWhiteSpace(version)) - { - return null; - } - - if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) - { - return null; - } - - // treat as fixed version, unknown introduced/last affected - return new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: true, - Fixed: normalized, - FixedInclusive: true, - LastAffected: null, - LastAffectedInclusive: true, - ConstraintExpression: null); - } - - private static IReadOnlyList BuildNormalizedVersions( - AppleAffectedProductDto product, - IReadOnlyList ranges) - { - if (ranges.Count == 0) - { - return Array.Empty(); - } - - var segments = new List(); - if (!string.IsNullOrWhiteSpace(product.Platform)) - { - segments.Add(product.Platform.Trim()); - } - - if (!string.IsNullOrWhiteSpace(product.Name)) - { - segments.Add(product.Name.Trim()); - } - - var note = segments.Count == 0 ? null : $"apple:{string.Join(':', segments)}"; - - var rules = new List(ranges.Count); - foreach (var range in ranges) - { - var rule = range.ToNormalizedVersionRule(note); - if (rule is not null) - { - rules.Add(rule); - } - } - - return rules.Count == 0 ? Array.Empty() : rules.ToArray(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; + +namespace StellaOps.Concelier.Connector.Vndr.Apple.Internal; + +internal static class AppleMapper +{ + public static (Advisory Advisory, PsirtFlagRecord? Flag) Map( + AppleDetailDto dto, + DocumentRecord document, + DtoRecord dtoRecord) + { + ArgumentNullException.ThrowIfNull(dto); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(dtoRecord); + + var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); + + var fetchProvenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "document", + document.Uri, + document.FetchedAt.ToUniversalTime()); + + var mapProvenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "map", + dto.AdvisoryId, + recordedAt); + + var aliases = BuildAliases(dto); + var references = BuildReferences(dto, recordedAt); + var affected = BuildAffected(dto, recordedAt); + + var advisory = new Advisory( + advisoryKey: dto.AdvisoryId, + title: dto.Title, + summary: dto.Summary, + language: "en", + published: dto.Published.ToUniversalTime(), + modified: dto.Updated?.ToUniversalTime(), + severity: null, + exploitKnown: false, + aliases: aliases, + references: references, + affectedPackages: affected, + cvssMetrics: Array.Empty(), + provenance: new[] { fetchProvenance, mapProvenance }); + + PsirtFlagRecord? flag = dto.RapidSecurityResponse + ? new PsirtFlagRecord(dto.AdvisoryId, "Apple", VndrAppleConnectorPlugin.SourceName, dto.ArticleId, recordedAt) + : null; + + return (advisory, flag); + } + + private static IReadOnlyList BuildAliases(AppleDetailDto dto) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase) + { + dto.AdvisoryId, + dto.ArticleId, + }; + + foreach (var cve in dto.CveIds) + { + if (!string.IsNullOrWhiteSpace(cve)) + { + set.Add(cve.Trim()); + } + } + + var aliases = set.ToList(); + aliases.Sort(StringComparer.OrdinalIgnoreCase); + return aliases; + } + + private static IReadOnlyList BuildReferences(AppleDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.References.Count == 0) + { + return Array.Empty(); + } + + var list = new List(dto.References.Count); + foreach (var reference in dto.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + try + { + var provenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "reference", + reference.Url, + recordedAt); + + list.Add(new AdvisoryReference( + url: reference.Url, + kind: reference.Kind, + sourceTag: null, + summary: reference.Title, + provenance: provenance)); + } + catch (ArgumentException) + { + // ignore invalid URLs + } + } + + if (list.Count == 0) + { + return Array.Empty(); + } + + list.Sort(static (left, right) => StringComparer.OrdinalIgnoreCase.Compare(left.Url, right.Url)); + return list; + } + + private static IReadOnlyList BuildAffected(AppleDetailDto dto, DateTimeOffset recordedAt) + { + if (dto.Affected.Count == 0) + { + return Array.Empty(); + } + + var packages = new List(dto.Affected.Count); + foreach (var product in dto.Affected) + { + if (string.IsNullOrWhiteSpace(product.Name)) + { + continue; + } + + var provenance = new[] + { + new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "affected", + product.Name, + recordedAt), + }; + + var ranges = BuildRanges(product, recordedAt); + var normalizedVersions = BuildNormalizedVersions(product, ranges); + + packages.Add(new AffectedPackage( + type: AffectedPackageTypes.Vendor, + identifier: product.Name, + platform: product.Platform, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: provenance, + normalizedVersions: normalizedVersions)); + } + + return packages.Count == 0 ? Array.Empty() : packages; + } + + private static IReadOnlyList BuildRanges(AppleAffectedProductDto product, DateTimeOffset recordedAt) + { + if (string.IsNullOrWhiteSpace(product.Version) && string.IsNullOrWhiteSpace(product.Build)) + { + return Array.Empty(); + } + + var provenance = new AdvisoryProvenance( + VndrAppleConnectorPlugin.SourceName, + "range", + product.Name, + recordedAt); + + var extensions = new Dictionary(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(product.Version)) + { + extensions["apple.version.raw"] = product.Version; + } + + if (!string.IsNullOrWhiteSpace(product.Build)) + { + extensions["apple.build"] = product.Build; + } + + if (!string.IsNullOrWhiteSpace(product.Platform)) + { + extensions["apple.platform"] = product.Platform; + } + + var primitives = extensions.Count == 0 + ? null + : new RangePrimitives( + SemVer: TryCreateSemVerPrimitive(product.Version), + Nevra: null, + Evr: null, + VendorExtensions: extensions); + + var sanitizedVersion = PackageCoordinateHelper.TryParseSemVer(product.Version, out _, out var normalizedVersion) + ? normalizedVersion + : product.Version; + + return new[] + { + new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: sanitizedVersion, + lastAffectedVersion: null, + rangeExpression: product.Version, + provenance: provenance, + primitives: primitives), + }; + } + + private static SemVerPrimitive? TryCreateSemVerPrimitive(string? version) + { + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + if (!PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) + { + return null; + } + + // treat as fixed version, unknown introduced/last affected + return new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: normalized, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: true, + ConstraintExpression: null); + } + + private static IReadOnlyList BuildNormalizedVersions( + AppleAffectedProductDto product, + IReadOnlyList ranges) + { + if (ranges.Count == 0) + { + return Array.Empty(); + } + + var segments = new List(); + if (!string.IsNullOrWhiteSpace(product.Platform)) + { + segments.Add(product.Platform.Trim()); + } + + if (!string.IsNullOrWhiteSpace(product.Name)) + { + segments.Add(product.Name.Trim()); + } + + var note = segments.Count == 0 ? null : $"apple:{string.Join(':', segments)}"; + + var rules = new List(ranges.Count); + foreach (var range in ranges) + { + var rule = range.ToNormalizedVersionRule(note); + if (rule is not null) + { + rules.Add(rule); + } + } + + return rules.Count == 0 ? Array.Empty() : rules.ToArray(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs index 92381260..67efbe0b 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Jobs.cs @@ -1,46 +1,46 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -internal static class AppleJobKinds -{ - public const string Fetch = "source:vndr-apple:fetch"; - public const string Parse = "source:vndr-apple:parse"; - public const string Map = "source:vndr-apple:map"; -} - -internal sealed class AppleFetchJob : IJob -{ - private readonly AppleConnector _connector; - - public AppleFetchJob(AppleConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.FetchAsync(context.Services, cancellationToken); -} - -internal sealed class AppleParseJob : IJob -{ - private readonly AppleConnector _connector; - - public AppleParseJob(AppleConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.ParseAsync(context.Services, cancellationToken); -} - -internal sealed class AppleMapJob : IJob -{ - private readonly AppleConnector _connector; - - public AppleMapJob(AppleConnector connector) - => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); - - public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - => _connector.MapAsync(context.Services, cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +internal static class AppleJobKinds +{ + public const string Fetch = "source:vndr-apple:fetch"; + public const string Parse = "source:vndr-apple:parse"; + public const string Map = "source:vndr-apple:map"; +} + +internal sealed class AppleFetchJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleFetchJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.FetchAsync(context.Services, cancellationToken); +} + +internal sealed class AppleParseJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleParseJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.ParseAsync(context.Services, cancellationToken); +} + +internal sealed class AppleMapJob : IJob +{ + private readonly AppleConnector _connector; + + public AppleMapJob(AppleConnector connector) + => _connector = connector ?? throw new ArgumentNullException(nameof(connector)); + + public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + => _connector.MapAsync(context.Services, cancellationToken); +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Apple/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..bc993aee --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Apple.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/README.md b/src/StellaOps.Concelier.Connector.Vndr.Apple/README.md similarity index 89% rename from src/StellaOps.Feedser.Source.Vndr.Apple/README.md rename to src/StellaOps.Concelier.Connector.Vndr.Apple/README.md index 5a61da3f..3aeafca7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/README.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/README.md @@ -1,49 +1,49 @@ -# Apple Security Updates Connector - -## Feed contract - -The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes: - -- `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`) -- Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline` -- Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS) -- Metadata for download packages, release notes, and signing assets - -The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8 - -Apple’s new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists: - -- CVE identifiers with Apple’s internal tracking IDs -- Product version/build applicability tables -- Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6 - -Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `
      ` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries. - -## Change detection strategy - -1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches. -2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`). -3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only. - -Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3 - -## Parsing & mapping notes - -- CVE lists live inside `
        ` items; each `
      • ` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apple’s component name as the package `name` and the OS build as the range primitive seed. -- Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`). -- Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`. - -## Outstanding questions - -- Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`. -- Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured. - -## Fixture maintenance - -Deterministic regression coverage lives in `src/StellaOps.Feedser.Source.Vndr.Apple.Tests/Apple/Fixtures`. When Apple publishes new advisories the fixtures must be refreshed using the provided helper scripts: - -- Bash: `./scripts/update-apple-fixtures.sh` -- PowerShell: `./scripts/update-apple-fixtures.ps1` - -Both scripts set `UPDATE_APPLE_FIXTURES=1`, touch a `.update-apple-fixtures` sentinel so test runs inside WSL propagate the flag, fetch the live HT articles referenced in `AppleFixtureManager`, sanitise the HTML, and rewrite the paired `.expected.json` DTO snapshots. Always inspect the resulting diff and re-run `dotnet test src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj` without the environment variable to ensure deterministic output before committing. - +# Apple Security Updates Connector + +## Feed contract + +The Apple Software Lookup Service (`https://gdmf.apple.com/v2/pmv`) publishes JSON payloads describing every public software release Apple has shipped. Each `AssetSet` entry exposes: + +- `ProductBuildVersion`, `ProductVersion`, and channel flags (e.g., `RapidSecurityResponse`) +- Timestamps for `PostingDate`, `ExpirationDate`, and `PreInstallDeadline` +- Associated product families/devices (Mac, iPhone, iPad, Apple TV, Apple Watch, VisionOS) +- Metadata for download packages, release notes, and signing assets + +The service supports delta polling by filtering on `PostingDate` and `ReleaseType`; responses are gzip-compressed and require a standard HTTPS client.citeturn3search8 + +Apple’s new security updates landing hub (`https://support.apple.com/100100`) consolidates bulletin detail pages (HT articles). Each update is linked via an `HT` identifier such as `HT214108` and lists: + +- CVE identifiers with Apple’s internal tracking IDs +- Product version/build applicability tables +- Mitigation guidance, acknowledgements, and update packaging notesciteturn1search6 + +Historical advisories redirect to per-platform pages (e.g., macOS, iOS, visionOS). The HTML structure uses `
        ` blocks with nested tables for affected products. CVE rows include disclosure dates and impact text that we can normalise into canonical `AffectedPackage` entries. + +## Change detection strategy + +1. Poll the Software Lookup Service for updates where `PostingDate` is within the sliding window (`lastModified - tolerance`). Cache `ProductID` + `PostingDate` to avoid duplicate fetches. +2. For each candidate, derive the HT article URL from `DocumentationURL` or by combining the `HT` identifier with the base path (`https://support.apple.com/{locale}/`). Fetch with conditional headers (`If-None-Match`, `If-Modified-Since`). +3. On HTTP `200`, store the raw HTML + metadata (HT id, posting date, product identifiers). On `304`, re-queue existing documents for mapping only. + +Unofficial Apple documentation warns that the Software Lookup Service rate-limits clients after repeated unauthenticated bursts; respect 5 requests/second and honour `Retry-After` headers on `403/429` responses.citeturn3search3 + +## Parsing & mapping notes + +- CVE lists live inside `
          ` items; each `
        • ` contains CVE, impact, and credit text. Parse these into canonical `Alias` + `AffectedPackage` records, using Apple’s component name as the package `name` and the OS build as the range primitive seed. +- Product/version tables have headers for platform (`Platform`, `Version`, `Build`). Map the OS name into our vendor range primitive namespace (`apple.platform`, `apple.build`). +- Rapid Security Response advisories include an `Rapid Security Responses` badge; emit `psirt_flags` with `apple.rapid_security_response = true`. + +## Outstanding questions + +- Some HT pages embed downloadable PDFs for supplemental mitigations. Confirm whether to persist PDF text via the shared `PdfTextExtractor`. +- Vision Pro updates include `deviceFamily` identifiers not yet mapped in `RangePrimitives`. Extend the model with `apple.deviceFamily` once sample fixtures are captured. + +## Fixture maintenance + +Deterministic regression coverage lives in `src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/Apple/Fixtures`. When Apple publishes new advisories the fixtures must be refreshed using the provided helper scripts: + +- Bash: `./scripts/update-apple-fixtures.sh` +- PowerShell: `./scripts/update-apple-fixtures.ps1` + +Both scripts set `UPDATE_APPLE_FIXTURES=1`, touch a `.update-apple-fixtures` sentinel so test runs inside WSL propagate the flag, fetch the live HT articles referenced in `AppleFixtureManager`, sanitise the HTML, and rewrite the paired `.expected.json` DTO snapshots. Always inspect the resulting diff and re-run `dotnet test src/StellaOps.Concelier.Connector.Vndr.Apple.Tests/StellaOps.Concelier.Connector.Vndr.Apple.Tests.csproj` without the environment variable to ensure deterministic output before committing. + diff --git a/src/StellaOps.Concelier.Connector.Vndr.Apple/StellaOps.Concelier.Connector.Vndr.Apple.csproj b/src/StellaOps.Concelier.Connector.Vndr.Apple/StellaOps.Concelier.Connector.Vndr.Apple.csproj new file mode 100644 index 00000000..6f59477f --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/StellaOps.Concelier.Connector.Vndr.Apple.csproj @@ -0,0 +1,18 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md index 44002d20..90245bc7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/TASKS.md @@ -1,11 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** – Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.| -|Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** – Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.| -|Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** – AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.| -|Canonical mapping & range primitives|BE-Conn-Apple|Models|**DONE** – Mapper now emits SemVer-derived normalizedVersions with `apple::` notes; fixtures updated to assert canonical rules while we continue tracking multi-device coverage in follow-up tasks.
          2025-10-11 research trail: confirmed payload aligns with `[{"scheme":"semver","type":"range","min":"","minInclusive":true,"max":"","maxInclusive":false,"notes":"apple:ios:17.1"}]`; continue using `notes` to surface build identifiers for storage provenance.| -|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-12)** – Parser now scopes references to article content, sorts affected rows deterministically, and regenerated fixtures (125326/125328/106355/HT214108/HT215500) produce stable JSON + sanitizer HTML in English.| -|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – OpenTelemetry pipeline exports `StellaOps.Feedser.Source.Vndr.Apple`; runbook `docs/ops/feedser-apple-operations.md` added with metrics + monitoring guidance.| -|Live HTML regression sweep|QA|Source.Common|**DONE (2025-10-12)** – Captured latest support.apple.com articles for 125326/125328/106355/HT214108/HT215500, trimmed nav noise, and committed sanitized HTML + expected DTOs with invariant timestamps.| -|Fixture regeneration tooling|DevEx|Testing|**DONE (2025-10-12)** – `scripts/update-apple-fixtures.(sh|ps1)` set the env flag + sentinel, forward through WSLENV, and clean up after regeneration; README references updated usage.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Catalogue Apple security bulletin sources|BE-Conn-Apple|Research|**DONE** – Feed contract documented in README (Software Lookup Service JSON + HT article hub) with rate-limit notes.| +|Fetch pipeline & state persistence|BE-Conn-Apple|Source.Common, Storage.Mongo|**DONE** – Index fetch + detail ingestion with SourceState cursoring/allowlists committed; awaiting live smoke run before enabling in scheduler defaults.| +|Parser & DTO implementation|BE-Conn-Apple|Source.Common|**DONE** – AngleSharp detail parser produces canonical DTO payloads (CVE list, timestamps, affected tables) persisted via DTO store.| +|Canonical mapping & range primitives|BE-Conn-Apple|Models|**DONE** – Mapper now emits SemVer-derived normalizedVersions with `apple::` notes; fixtures updated to assert canonical rules while we continue tracking multi-device coverage in follow-up tasks.
          2025-10-11 research trail: confirmed payload aligns with `[{"scheme":"semver","type":"range","min":"","minInclusive":true,"max":"","maxInclusive":false,"notes":"apple:ios:17.1"}]`; continue using `notes` to surface build identifiers for storage provenance.| +|Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-12)** – Parser now scopes references to article content, sorts affected rows deterministically, and regenerated fixtures (125326/125328/106355/HT214108/HT215500) produce stable JSON + sanitizer HTML in English.| +|Telemetry & documentation|DevEx|Docs|**DONE (2025-10-12)** – OpenTelemetry pipeline exports `StellaOps.Concelier.Connector.Vndr.Apple`; runbook `docs/ops/concelier-apple-operations.md` added with metrics + monitoring guidance.| +|Live HTML regression sweep|QA|Source.Common|**DONE (2025-10-12)** – Captured latest support.apple.com articles for 125326/125328/106355/HT214108/HT215500, trimmed nav noise, and committed sanitized HTML + expected DTOs with invariant timestamps.| +|Fixture regeneration tooling|DevEx|Testing|**DONE (2025-10-12)** – `scripts/update-apple-fixtures.(sh|ps1)` set the env flag + sentinel, forward through WSLENV, and clean up after regeneration; README references updated usage.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Apple/VndrAppleConnectorPlugin.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Apple/VndrAppleConnectorPlugin.cs index afa445f1..c51ca479 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/VndrAppleConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Apple/VndrAppleConnectorPlugin.cs @@ -1,24 +1,24 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Source.Vndr.Apple; - -public sealed class VndrAppleConnectorPlugin : IConnectorPlugin -{ - public const string SourceName = "vndr-apple"; - - public string Name => SourceName; - - public bool IsAvailable(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetService() is not null; - } - - public IFeedConnector Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetRequiredService(); - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Connector.Vndr.Apple; + +public sealed class VndrAppleConnectorPlugin : IConnectorPlugin +{ + public const string SourceName = "vndr-apple"; + + public string Name => SourceName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedConnector Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetRequiredService(); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs index 89fc0504..cde86889 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumConnectorTests.cs @@ -10,20 +10,20 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Json; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Vndr.Chromium; -using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Json; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Vndr.Chromium; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Tests; [Collection("mongo-fixture")] public sealed class ChromiumConnectorTests : IAsyncLifetime @@ -56,7 +56,7 @@ public sealed class ChromiumConnectorTests : IAsyncLifetime { await connector.ParseAsync(provider, CancellationToken.None); } - catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException) + catch (StellaOps.Concelier.Connector.Common.Json.JsonSchemaValidationException) { // Parsing should flag document as failed even when schema validation rejects payloads. } diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs index ca26fa24..3f6398a1 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/ChromiumMapperTests.cs @@ -1,10 +1,10 @@ using System; using System.Linq; -using StellaOps.Feedser.Source.Vndr.Chromium; -using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using StellaOps.Concelier.Connector.Vndr.Chromium; +using StellaOps.Concelier.Connector.Vndr.Chromium.Internal; using Xunit; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Tests; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Tests; public sealed class ChromiumMapperTests { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-advisory.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-detail.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/Chromium/Fixtures/chromium-feed.xml diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj similarity index 50% rename from src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj rename to src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj index 887bec82..2a091c7a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium.Tests/StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium.Tests/StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj @@ -5,10 +5,10 @@ enable - - - - + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Chromium/AGENTS.md similarity index 81% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/AGENTS.md index a87e00fb..bf628ae5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/AGENTS.md @@ -20,9 +20,9 @@ Chromium/Chrome vendor feed connector parsing Stable Channel Update posts; autho In: vendor advisory mapping, fixed version emission per platform, psirt_flags vendor context. Out: OS distro packaging semantics; bug bounty details beyond references. ## Observability & security expectations -- Metrics: SourceDiagnostics exports the shared `feedser.source.http.*` counters/histograms tagged `feedser.source=chromium`, enabling dashboards to observe fetch volumes, parse failures, and map affected counts via tag filters. +- Metrics: SourceDiagnostics exports the shared `concelier.source.http.*` counters/histograms tagged `concelier.source=chromium`, enabling dashboards to observe fetch volumes, parse failures, and map affected counts via tag filters. - Logs: post slugs, version extracted, platform coverage, timing; allowlist blog host. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Chromium.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Chromium.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnector.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnector.cs index 2e432fc3..8a4bed41 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnector.cs @@ -6,21 +6,21 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Json; -using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; -using StellaOps.Feedser.Source.Vndr.Chromium.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Json; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; using StellaOps.Plugin; using Json.Schema; -namespace StellaOps.Feedser.Source.Vndr.Chromium; +namespace StellaOps.Concelier.Connector.Vndr.Chromium; public sealed class ChromiumConnector : IFeedConnector { @@ -247,7 +247,7 @@ public sealed class ChromiumConnector : IFeedConnector { _schemaValidator.Validate(jsonDocument, Schema, dto.PostId); } - catch (StellaOps.Feedser.Source.Common.Json.JsonSchemaValidationException ex) + catch (StellaOps.Concelier.Connector.Common.Json.JsonSchemaValidationException ex) { _logger.LogError(ex, "Chromium schema validation failed for {DocumentId}", document.Id); await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false); diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnectorPlugin.cs index 8cb72d05..f2eee705 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Chromium; +namespace StellaOps.Concelier.Connector.Vndr.Chromium; public sealed class VndrChromiumConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDiagnostics.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDiagnostics.cs index cf5c1de6..9f07a5f6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumDiagnostics.cs @@ -1,10 +1,10 @@ using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Vndr.Chromium; +namespace StellaOps.Concelier.Connector.Vndr.Chromium; public sealed class ChromiumDiagnostics : IDisposable { - public const string MeterName = "StellaOps.Feedser.Source.Vndr.Chromium"; + public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Chromium"; public const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs index 16f7351d..7c990017 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/ChromiumServiceCollectionExtensions.cs @@ -1,10 +1,10 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; -using StellaOps.Feedser.Source.Vndr.Chromium.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium.Internal; -namespace StellaOps.Feedser.Source.Vndr.Chromium; +namespace StellaOps.Concelier.Connector.Vndr.Chromium; public static class ChromiumServiceCollectionExtensions { @@ -24,7 +24,7 @@ public static class ChromiumServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = new Uri(options.FeedUri.GetLeftPart(UriPartial.Authority)); clientOptions.Timeout = TimeSpan.FromSeconds(20); - clientOptions.UserAgent = "StellaOps.Feedser.VndrChromium/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.VndrChromium/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.FeedUri.Host); }); diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Configuration/ChromiumOptions.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Configuration/ChromiumOptions.cs index 08619672..70cd14d5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Configuration/ChromiumOptions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Configuration/ChromiumOptions.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; public sealed class ChromiumOptions { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumCursor.cs index 622ae605..a4f1ea3c 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumCursor.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumCursor.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal sealed record ChromiumCursor( DateTimeOffset? LastPublished, diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs index c1c8cd36..0a3d65a6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDocumentMetadata.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal sealed record ChromiumDocumentMetadata( string PostId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDto.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDto.cs index 6dacc7a7..8863a656 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumDto.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumDto.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal sealed record ChromiumDto( [property: JsonPropertyName("postId")] string PostId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedEntry.cs similarity index 89% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedEntry.cs index 81017793..f28c6f94 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedEntry.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedEntry.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; public sealed record ChromiumFeedEntry( string EntryId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedLoader.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedLoader.cs index 84888297..7da157d5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumFeedLoader.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumFeedLoader.cs @@ -2,9 +2,9 @@ using System.ServiceModel.Syndication; using System.Xml; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Vndr.Chromium.Configuration; +using StellaOps.Concelier.Connector.Vndr.Chromium.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; public sealed class ChromiumFeedLoader { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumMapper.cs index a534c38d..333dfeb5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumMapper.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumMapper.cs @@ -2,10 +2,10 @@ using System; using System.Collections.Generic; using System.Linq; using System.Globalization; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal static class ChromiumMapper { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumParser.cs index cd5f70ec..523621b6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumParser.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumParser.cs @@ -2,7 +2,7 @@ using System.Text.RegularExpressions; using AngleSharp.Dom; using AngleSharp.Html.Parser; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal static class ChromiumParser { diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs similarity index 78% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs index 33854ffb..77ef6d03 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Internal/ChromiumSchemaProvider.cs @@ -3,7 +3,7 @@ using System.Reflection; using System.Threading; using Json.Schema; -namespace StellaOps.Feedser.Source.Vndr.Chromium.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Chromium.Internal; internal static class ChromiumSchemaProvider { @@ -14,7 +14,7 @@ internal static class ChromiumSchemaProvider private static JsonSchema Load() { var assembly = typeof(ChromiumSchemaProvider).GetTypeInfo().Assembly; - const string resourceName = "StellaOps.Feedser.Source.Vndr.Chromium.Schemas.chromium-post.schema.json"; + const string resourceName = "StellaOps.Concelier.Connector.Vndr.Chromium.Schemas.chromium-post.schema.json"; using var stream = assembly.GetManifestResourceStream(resourceName) ?? throw new InvalidOperationException($"Embedded schema '{resourceName}' not found."); diff --git a/src/StellaOps.Concelier.Connector.Vndr.Chromium/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..9bfffcf6 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Chromium.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json b/src/StellaOps.Concelier.Connector.Vndr.Chromium/Schemas/chromium-post.schema.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/Schemas/chromium-post.schema.json rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/Schemas/chromium-post.schema.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj b/src/StellaOps.Concelier.Connector.Vndr.Chromium/StellaOps.Concelier.Connector.Vndr.Chromium.csproj similarity index 62% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/StellaOps.Concelier.Connector.Vndr.Chromium.csproj index 31406c32..e1f0543a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/StellaOps.Feedser.Source.Vndr.Chromium.csproj +++ b/src/StellaOps.Concelier.Connector.Vndr.Chromium/StellaOps.Concelier.Connector.Vndr.Chromium.csproj @@ -18,14 +18,14 @@ - - - + + + - <_Parameter1>StellaOps.Feedser.Source.Vndr.Chromium.Tests + <_Parameter1>StellaOps.Concelier.Connector.Vndr.Chromium.Tests diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Chromium/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Chromium/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs index e0384e04..7bcd51d2 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoDtoFactoryTests.cs @@ -1,73 +1,73 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Source.Vndr.Cisco.Internal; -using Xunit; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Tests; - -public sealed class CiscoDtoFactoryTests -{ - [Fact] - public async Task CreateAsync_MergesRawAndCsafProducts() - { - const string CsafPayload = @" -{ - ""product_tree"": { - ""full_product_names"": [ - { ""product_id"": ""PID-1"", ""name"": ""Cisco Widget"" } - ] - }, - ""vulnerabilities"": [ - { - ""product_status"": { - ""known_affected"": [""PID-1""] - } - } - ] -}"; - - var csafClient = new StubCsafClient(CsafPayload); - var factory = new CiscoDtoFactory(csafClient, NullLogger.Instance); - - var raw = new CiscoRawAdvisory - { - AdvisoryId = "CISCO-SA-TEST", - AdvisoryTitle = "Test Advisory", - Summary = "Summary", - Sir = "High", - FirstPublished = "2025-10-01T00:00:00Z", - LastUpdated = "2025-10-02T00:00:00Z", - PublicationUrl = "https://example.com/advisory", - CsafUrl = "https://sec.cloudapps.cisco.com/csaf/test.json", - Cves = new List { "CVE-2024-0001" }, - BugIds = new List { "BUG123" }, - ProductNames = new List { "Cisco Widget" }, - Version = "1.2.3", - CvssBaseScore = "9.8" - }; - - var dto = await factory.CreateAsync(raw, CancellationToken.None); - - dto.Should().NotBeNull(); - dto.Severity.Should().Be("high"); - dto.CvssBaseScore.Should().Be(9.8); - dto.Products.Should().HaveCount(1); - var product = dto.Products[0]; - product.Name.Should().Be("Cisco Widget"); - product.ProductId.Should().Be("PID-1"); - product.Statuses.Should().Contain("known_affected"); - } - - private sealed class StubCsafClient : ICiscoCsafClient - { - private readonly string? _payload; - - public StubCsafClient(string? payload) => _payload = payload; - - public Task TryFetchAsync(string? url, CancellationToken cancellationToken) - => Task.FromResult(_payload); - } -} +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Concelier.Connector.Vndr.Cisco.Internal; +using Xunit; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Tests; + +public sealed class CiscoDtoFactoryTests +{ + [Fact] + public async Task CreateAsync_MergesRawAndCsafProducts() + { + const string CsafPayload = @" +{ + ""product_tree"": { + ""full_product_names"": [ + { ""product_id"": ""PID-1"", ""name"": ""Cisco Widget"" } + ] + }, + ""vulnerabilities"": [ + { + ""product_status"": { + ""known_affected"": [""PID-1""] + } + } + ] +}"; + + var csafClient = new StubCsafClient(CsafPayload); + var factory = new CiscoDtoFactory(csafClient, NullLogger.Instance); + + var raw = new CiscoRawAdvisory + { + AdvisoryId = "CISCO-SA-TEST", + AdvisoryTitle = "Test Advisory", + Summary = "Summary", + Sir = "High", + FirstPublished = "2025-10-01T00:00:00Z", + LastUpdated = "2025-10-02T00:00:00Z", + PublicationUrl = "https://example.com/advisory", + CsafUrl = "https://sec.cloudapps.cisco.com/csaf/test.json", + Cves = new List { "CVE-2024-0001" }, + BugIds = new List { "BUG123" }, + ProductNames = new List { "Cisco Widget" }, + Version = "1.2.3", + CvssBaseScore = "9.8" + }; + + var dto = await factory.CreateAsync(raw, CancellationToken.None); + + dto.Should().NotBeNull(); + dto.Severity.Should().Be("high"); + dto.CvssBaseScore.Should().Be(9.8); + dto.Products.Should().HaveCount(1); + var product = dto.Products[0]; + product.Name.Should().Be("Cisco Widget"); + product.ProductId.Should().Be("PID-1"); + product.Statuses.Should().Contain("known_affected"); + } + + private sealed class StubCsafClient : ICiscoCsafClient + { + private readonly string? _payload; + + public StubCsafClient(string? payload) => _payload = payload; + + public Task TryFetchAsync(string? url, CancellationToken cancellationToken) + => Task.FromResult(_payload); + } +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoMapperTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoMapperTests.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoMapperTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoMapperTests.cs index b117ce25..e8eea6cd 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/CiscoMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/CiscoMapperTests.cs @@ -1,79 +1,79 @@ -using System; -using System.Collections.Generic; -using FluentAssertions; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Vndr.Cisco; -using StellaOps.Feedser.Source.Vndr.Cisco.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using Xunit; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Tests; - -public sealed class CiscoMapperTests -{ - [Fact] - public void Map_ProducesCanonicalAdvisory() - { - var published = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero); - var updated = published.AddDays(1); - - var dto = new CiscoAdvisoryDto( - AdvisoryId: "CISCO-SA-TEST", - Title: "Test Advisory", - Summary: "Sample summary", - Severity: "High", - Published: published, - Updated: updated, - PublicationUrl: "https://example.com/advisory", - CsafUrl: "https://sec.cloudapps.cisco.com/csaf/test.json", - CvrfUrl: "https://example.com/cvrf.xml", - CvssBaseScore: 9.8, - Cves: new List { "CVE-2024-0001" }, - BugIds: new List { "BUG123" }, - Products: new List - { - new("Cisco Widget", "PID-1", "1.2.3", new [] { AffectedPackageStatusCatalog.KnownAffected }) - }); - - var document = new DocumentRecord( - Id: Guid.NewGuid(), - SourceName: VndrCiscoConnectorPlugin.SourceName, - Uri: "https://api.cisco.com/security/advisories/v2/advisories/CISCO-SA-TEST", - FetchedAt: published, - Sha256: "abc123", - Status: DocumentStatuses.PendingMap, - ContentType: "application/json", - Headers: null, - Metadata: null, - Etag: null, - LastModified: updated, - GridFsId: null); - - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, VndrCiscoConnectorPlugin.SourceName, "cisco.dto.test", new BsonDocument(), updated); - - var advisory = CiscoMapper.Map(dto, document, dtoRecord); - - advisory.AdvisoryKey.Should().Be("CISCO-SA-TEST"); - advisory.Title.Should().Be("Test Advisory"); - advisory.Severity.Should().Be("high"); - advisory.Aliases.Should().Contain(new[] { "CISCO-SA-TEST", "CVE-2024-0001", "BUG123" }); - advisory.References.Should().Contain(reference => reference.Url == "https://example.com/advisory"); - advisory.References.Should().Contain(reference => reference.Url == "https://sec.cloudapps.cisco.com/csaf/test.json"); - advisory.AffectedPackages.Should().HaveCount(1); - - var package = advisory.AffectedPackages[0]; - package.Type.Should().Be(AffectedPackageTypes.Vendor); - package.Identifier.Should().Be("Cisco Widget"); - package.Statuses.Should().ContainSingle(status => status.Status == AffectedPackageStatusCatalog.KnownAffected); - package.VersionRanges.Should().ContainSingle(); - var range = package.VersionRanges[0]; - range.RangeKind.Should().Be("semver"); - range.Provenance.Source.Should().Be(VndrCiscoConnectorPlugin.SourceName); - range.Primitives.Should().NotBeNull(); - range.Primitives!.SemVer.Should().NotBeNull(); - range.Primitives.SemVer!.ExactValue.Should().Be("1.2.3"); - } -} +using System; +using System.Collections.Generic; +using FluentAssertions; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Vndr.Cisco; +using StellaOps.Concelier.Connector.Vndr.Cisco.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using Xunit; + +namespace StellaOps.Concelier.Connector.Vndr.Cisco.Tests; + +public sealed class CiscoMapperTests +{ + [Fact] + public void Map_ProducesCanonicalAdvisory() + { + var published = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero); + var updated = published.AddDays(1); + + var dto = new CiscoAdvisoryDto( + AdvisoryId: "CISCO-SA-TEST", + Title: "Test Advisory", + Summary: "Sample summary", + Severity: "High", + Published: published, + Updated: updated, + PublicationUrl: "https://example.com/advisory", + CsafUrl: "https://sec.cloudapps.cisco.com/csaf/test.json", + CvrfUrl: "https://example.com/cvrf.xml", + CvssBaseScore: 9.8, + Cves: new List { "CVE-2024-0001" }, + BugIds: new List { "BUG123" }, + Products: new List + { + new("Cisco Widget", "PID-1", "1.2.3", new [] { AffectedPackageStatusCatalog.KnownAffected }) + }); + + var document = new DocumentRecord( + Id: Guid.NewGuid(), + SourceName: VndrCiscoConnectorPlugin.SourceName, + Uri: "https://api.cisco.com/security/advisories/v2/advisories/CISCO-SA-TEST", + FetchedAt: published, + Sha256: "abc123", + Status: DocumentStatuses.PendingMap, + ContentType: "application/json", + Headers: null, + Metadata: null, + Etag: null, + LastModified: updated, + GridFsId: null); + + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, VndrCiscoConnectorPlugin.SourceName, "cisco.dto.test", new BsonDocument(), updated); + + var advisory = CiscoMapper.Map(dto, document, dtoRecord); + + advisory.AdvisoryKey.Should().Be("CISCO-SA-TEST"); + advisory.Title.Should().Be("Test Advisory"); + advisory.Severity.Should().Be("high"); + advisory.Aliases.Should().Contain(new[] { "CISCO-SA-TEST", "CVE-2024-0001", "BUG123" }); + advisory.References.Should().Contain(reference => reference.Url == "https://example.com/advisory"); + advisory.References.Should().Contain(reference => reference.Url == "https://sec.cloudapps.cisco.com/csaf/test.json"); + advisory.AffectedPackages.Should().HaveCount(1); + + var package = advisory.AffectedPackages[0]; + package.Type.Should().Be(AffectedPackageTypes.Vendor); + package.Identifier.Should().Be("Cisco Widget"); + package.Statuses.Should().ContainSingle(status => status.Status == AffectedPackageStatusCatalog.KnownAffected); + package.VersionRanges.Should().ContainSingle(); + var range = package.VersionRanges[0]; + range.RangeKind.Should().Be("semver"); + range.Provenance.Source.Should().Be(VndrCiscoConnectorPlugin.SourceName); + range.Primitives.Should().NotBeNull(); + range.Primitives!.SemVer.Should().NotBeNull(); + range.Primitives.SemVer!.ExactValue.Should().Be("1.2.3"); + } +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj new file mode 100644 index 00000000..a45633fe --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco.Tests/StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Cisco/AGENTS.md similarity index 83% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/AGENTS.md index 08334c1d..99565a05 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/AGENTS.md @@ -1,30 +1,30 @@ -# AGENTS -## Role -Implement the Cisco security advisory connector to ingest Cisco PSIRT bulletins for Feedser. - -## Scope -- Identify Cisco advisory feeds/APIs (XML, HTML, JSON) and define incremental fetch strategy. -- Implement fetch/cursor pipeline with retry/backoff and document dedupe. -- Parse advisories to extract summary, affected products, Cisco bug IDs, CVEs, mitigation guidance. -- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (e.g., SemVer/IOS version metadata). -- Provide deterministic fixtures and regression tests. - -## Participants -- `Source.Common`, `Storage.Mongo`, `Feedser.Models`, `Feedser.Testing`. - -## Interfaces & Contracts -- Job kinds: `cisco:fetch`, `cisco:parse`, `cisco:map`. -- Persist upstream metadata (e.g., `Last-Modified`, `advisoryId`). -- Alias set should include Cisco advisory IDs, bug IDs, and CVEs. - -## In/Out of scope -In scope: Cisco PSIRT advisories, range primitive coverage. -Out of scope: Non-security Cisco release notes. - -## Observability & Security Expectations -- Log fetch/mapping statistics, respect Cisco API rate limits, sanitise HTML. -- Handle authentication tokens if API requires them. - -## Tests -- Add `StellaOps.Feedser.Source.Vndr.Cisco.Tests` with canned fixtures for fetch/parse/map. -- Snapshot canonical advisories and support fixture regeneration. +# AGENTS +## Role +Implement the Cisco security advisory connector to ingest Cisco PSIRT bulletins for Concelier. + +## Scope +- Identify Cisco advisory feeds/APIs (XML, HTML, JSON) and define incremental fetch strategy. +- Implement fetch/cursor pipeline with retry/backoff and document dedupe. +- Parse advisories to extract summary, affected products, Cisco bug IDs, CVEs, mitigation guidance. +- Map advisories into canonical `Advisory` records with aliases, references, affected packages, and range primitives (e.g., SemVer/IOS version metadata). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common`, `Storage.Mongo`, `Concelier.Models`, `Concelier.Testing`. + +## Interfaces & Contracts +- Job kinds: `cisco:fetch`, `cisco:parse`, `cisco:map`. +- Persist upstream metadata (e.g., `Last-Modified`, `advisoryId`). +- Alias set should include Cisco advisory IDs, bug IDs, and CVEs. + +## In/Out of scope +In scope: Cisco PSIRT advisories, range primitive coverage. +Out of scope: Non-security Cisco release notes. + +## Observability & Security Expectations +- Log fetch/mapping statistics, respect Cisco API rate limits, sanitise HTML. +- Handle authentication tokens if API requires them. + +## Tests +- Add `StellaOps.Concelier.Connector.Vndr.Cisco.Tests` with canned fixtures for fetch/parse/map. +- Snapshot canonical advisories and support fixture regeneration. diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs similarity index 98% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs index 52b800db..2ef7b35c 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoConnector.cs @@ -7,17 +7,17 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; -using StellaOps.Feedser.Source.Vndr.Cisco.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Cisco; +namespace StellaOps.Concelier.Connector.Vndr.Cisco; public sealed class CiscoConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs index e532a40d..59cbb383 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoDependencyInjectionRoutine.cs @@ -2,13 +2,13 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Vndr.Cisco; +namespace StellaOps.Concelier.Connector.Vndr.Cisco; public sealed class CiscoDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:cisco"; + private const string ConfigurationSection = "concelier:sources:cisco"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs index d38e2b06..8d6b1905 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/CiscoServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/CiscoServiceCollectionExtensions.cs @@ -2,12 +2,12 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; -using StellaOps.Feedser.Source.Vndr.Cisco.Internal; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Cisco.Configuration; +using StellaOps.Concelier.Connector.Vndr.Cisco.Internal; -namespace StellaOps.Feedser.Source.Vndr.Cisco; +namespace StellaOps.Concelier.Connector.Vndr.Cisco; public static class CiscoServiceCollectionExtensions { @@ -31,7 +31,7 @@ public static class CiscoServiceCollectionExtensions { var options = sp.GetRequiredService>().Value; client.Timeout = options.RequestTimeout; - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Feedser.Cisco/1.0"); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Concelier.Cisco/1.0"); client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); if (options.TokenEndpoint is not null) { @@ -43,7 +43,7 @@ public static class CiscoServiceCollectionExtensions { var options = sp.GetRequiredService>().Value; clientOptions.Timeout = options.RequestTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.Cisco/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.Cisco/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.BaseUri.Host); clientOptions.AllowedHosts.Add("sec.cloudapps.cisco.com"); diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Jobs.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/Jobs.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs index ddaa298e..f77721d7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Vndr.Cisco; +namespace StellaOps.Concelier.Connector.Vndr.Cisco; internal static class CiscoJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Vndr.Cisco/StellaOps.Concelier.Connector.Vndr.Cisco.csproj b/src/StellaOps.Concelier.Connector.Vndr.Cisco/StellaOps.Concelier.Connector.Vndr.Cisco.csproj new file mode 100644 index 00000000..4eea5e86 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/StellaOps.Concelier.Connector.Vndr.Cisco.csproj @@ -0,0 +1,16 @@ + + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md similarity index 68% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md index 3df21054..475e95fd 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/TASKS.md @@ -1,11 +1,12 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-CISCO-02-001 Confirm Cisco PSIRT data source|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Selected openVuln REST API (`https://apix.cisco.com/security/advisories/v2/…`) as primary (structured JSON, CSAF/CVRF links) with RSS as fallback. Documented OAuth2 client-credentials flow (`cloudsso.cisco.com/as/token.oauth2`), baseline quotas (5 req/s, 30 req/min, 5 000 req/day), and pagination contract (`pageIndex`, `pageSize≤100`) in `docs/feedser-connector-research-20251011.md`.| -|FEEDCONN-CISCO-02-002 Fetch pipeline & state persistence|BE-Conn-Cisco|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Fetch job now streams openVuln pages with OAuth bearer handler, honours 429 `Retry-After`, persists per-advisory JSON + metadata into GridFS, and updates cursor (`lastModified`, advisory ID, pending docs).| -|FEEDCONN-CISCO-02-003 Parser & DTO implementation|BE-Conn-Cisco|Source.Common|**DONE (2025-10-14)** – DTO factory normalizes SIR, folds CSAF product statuses, and persists `cisco.dto.v1` payloads (see `CiscoDtoFactory`).| -|FEEDCONN-CISCO-02-004 Canonical mapping & range primitives|BE-Conn-Cisco|Models|**DONE (2025-10-14)** – `CiscoMapper` emits canonical advisories with vendor + SemVer primitives, provenance, and status tags.| -|FEEDCONN-CISCO-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added unit tests (`StellaOps.Feedser.Source.Vndr.Cisco.Tests`) exercising DTO/mapper pipelines; `dotnet test` validated.| -|FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Cisco diagnostics counters exposed and ops runbook updated with telemetry guidance (`docs/ops/feedser-cisco-operations.md`).| -|FEEDCONN-CISCO-02-007 API selection decision memo|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Drafted decision matrix: openVuln (structured/delta filters, OAuth throttle) vs RSS (delayed/minimal metadata). Pending OAuth onboarding (`FEEDCONN-CISCO-02-008`) before final recommendation circulated.| -|FEEDCONN-CISCO-02-008 OAuth client provisioning|Ops, BE-Conn-Cisco|Ops|**DONE (2025-10-14)** – `docs/ops/feedser-cisco-operations.md` documents OAuth provisioning/rotation, quotas, and Offline Kit distribution guidance.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-CISCO-02-001 Confirm Cisco PSIRT data source|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Selected openVuln REST API (`https://apix.cisco.com/security/advisories/v2/…`) as primary (structured JSON, CSAF/CVRF links) with RSS as fallback. Documented OAuth2 client-credentials flow (`cloudsso.cisco.com/as/token.oauth2`), baseline quotas (5 req/s, 30 req/min, 5 000 req/day), and pagination contract (`pageIndex`, `pageSize≤100`) in `docs/concelier-connector-research-20251011.md`.| +|FEEDCONN-CISCO-02-002 Fetch pipeline & state persistence|BE-Conn-Cisco|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – Fetch job now streams openVuln pages with OAuth bearer handler, honours 429 `Retry-After`, persists per-advisory JSON + metadata into GridFS, and updates cursor (`lastModified`, advisory ID, pending docs).| +|FEEDCONN-CISCO-02-003 Parser & DTO implementation|BE-Conn-Cisco|Source.Common|**DONE (2025-10-14)** – DTO factory normalizes SIR, folds CSAF product statuses, and persists `cisco.dto.v1` payloads (see `CiscoDtoFactory`).| +|FEEDCONN-CISCO-02-004 Canonical mapping & range primitives|BE-Conn-Cisco|Models|**DONE (2025-10-14)** – `CiscoMapper` emits canonical advisories with vendor + SemVer primitives, provenance, and status tags.| +|FEEDCONN-CISCO-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added unit tests (`StellaOps.Concelier.Connector.Vndr.Cisco.Tests`) exercising DTO/mapper pipelines; `dotnet test` validated.| +|FEEDCONN-CISCO-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-14)** – Cisco diagnostics counters exposed and ops runbook updated with telemetry guidance (`docs/ops/concelier-cisco-operations.md`).| +|FEEDCONN-CISCO-02-007 API selection decision memo|BE-Conn-Cisco|Research|**DONE (2025-10-11)** – Drafted decision matrix: openVuln (structured/delta filters, OAuth throttle) vs RSS (delayed/minimal metadata). Pending OAuth onboarding (`FEEDCONN-CISCO-02-008`) before final recommendation circulated.| +|FEEDCONN-CISCO-02-008 OAuth client provisioning|Ops, BE-Conn-Cisco|Ops|**DONE (2025-10-14)** – `docs/ops/concelier-cisco-operations.md` documents OAuth provisioning/rotation, quotas, and Offline Kit distribution guidance.| +|FEEDCONN-CISCO-02-009 Normalized SemVer promotion|BE-Conn-Cisco|Merge coordination (`FEEDMERGE-COORD-02-900`)|**TODO (due 2025-10-21)** – Use helper from `../Merge/RANGE_PRIMITIVES_COORDINATION.md` to convert `SemVerPrimitive` outputs into `NormalizedVersionRule` with provenance (`cisco:{productId}`), update mapper/tests, and confirm merge normalized-rule counters drop.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/VndrCiscoConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Cisco/VndrCiscoConnectorPlugin.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Cisco/VndrCiscoConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Cisco/VndrCiscoConnectorPlugin.cs index 204da255..a9c3561b 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/VndrCiscoConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Cisco/VndrCiscoConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Cisco; +namespace StellaOps.Concelier.Connector.Vndr.Cisco; public sealed class VndrCiscoConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-detail.json b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-detail.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-detail.json rename to src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-detail.json index b2463f28..78ee3b87 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-detail.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-detail.json @@ -1,44 +1,44 @@ -{ - "id": "7a760e58-bd5f-4f37-8b87-1b61f2deb001", - "vulnerabilityId": "ADV123456", - "cveNumbers": [ - "CVE-2025-0001" - ], - "title": "Windows Kernel Elevation of Privilege Vulnerability", - "description": "An elevation of privilege vulnerability exists in the Windows kernel.", - "releaseDate": "2025-10-10T10:00:00Z", - "lastModifiedDate": "2025-10-14T11:00:00Z", - "severity": "Critical", - "threats": [ - { - "type": "Impact", - "description": "Elevation of Privilege", - "severity": "Important" - } - ], - "remediations": [ - { - "id": "1", - "type": "Security Update", - "description": "Install KB5031234 to address this vulnerability.", - "url": "https://support.microsoft.com/help/5031234", - "kbNumber": "KB5031234" - } - ], - "affectedProducts": [ - { - "productId": "Windows11-23H2-x64", - "productName": "Windows 11 Version 23H2 for x64-based Systems", - "platform": "Windows", - "architecture": "x64", - "buildNumber": "22631.3520", - "cpe": "cpe:/o:microsoft:windows_11:23H2" - } - ], - "cvssV3": { - "baseScore": 8.1, - "vectorString": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H" - }, - "releaseNoteUrl": "https://msrc.microsoft.com/update-guide/vulnerability/ADV123456", - "cvrfUrl": "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip" -} +{ + "id": "7a760e58-bd5f-4f37-8b87-1b61f2deb001", + "vulnerabilityId": "ADV123456", + "cveNumbers": [ + "CVE-2025-0001" + ], + "title": "Windows Kernel Elevation of Privilege Vulnerability", + "description": "An elevation of privilege vulnerability exists in the Windows kernel.", + "releaseDate": "2025-10-10T10:00:00Z", + "lastModifiedDate": "2025-10-14T11:00:00Z", + "severity": "Critical", + "threats": [ + { + "type": "Impact", + "description": "Elevation of Privilege", + "severity": "Important" + } + ], + "remediations": [ + { + "id": "1", + "type": "Security Update", + "description": "Install KB5031234 to address this vulnerability.", + "url": "https://support.microsoft.com/help/5031234", + "kbNumber": "KB5031234" + } + ], + "affectedProducts": [ + { + "productId": "Windows11-23H2-x64", + "productName": "Windows 11 Version 23H2 for x64-based Systems", + "platform": "Windows", + "architecture": "x64", + "buildNumber": "22631.3520", + "cpe": "cpe:/o:microsoft:windows_11:23H2" + } + ], + "cvssV3": { + "baseScore": 8.1, + "vectorString": "CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H" + }, + "releaseNoteUrl": "https://msrc.microsoft.com/update-guide/vulnerability/ADV123456", + "cvrfUrl": "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip" +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-summary.json b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-summary.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-summary.json rename to src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-summary.json index a49fefdf..0fe34e32 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/Fixtures/msrc-summary.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/Fixtures/msrc-summary.json @@ -1,17 +1,17 @@ -{ - "value": [ - { - "id": "7a760e58-bd5f-4f37-8b87-1b61f2deb001", - "vulnerabilityId": "ADV123456", - "cveNumbers": [ - "CVE-2025-0001" - ], - "title": "Windows Kernel Elevation of Privilege Vulnerability", - "description": "An elevation of privilege vulnerability exists in the Windows kernel.", - "releaseDate": "2025-10-10T10:00:00Z", - "lastModifiedDate": "2025-10-14T11:00:00Z", - "severity": "Critical", - "cvrfUrl": "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip" - } - ] -} +{ + "value": [ + { + "id": "7a760e58-bd5f-4f37-8b87-1b61f2deb001", + "vulnerabilityId": "ADV123456", + "cveNumbers": [ + "CVE-2025-0001" + ], + "title": "Windows Kernel Elevation of Privilege Vulnerability", + "description": "An elevation of privilege vulnerability exists in the Windows kernel.", + "releaseDate": "2025-10-10T10:00:00Z", + "lastModifiedDate": "2025-10-14T11:00:00Z", + "severity": "Critical", + "cvrfUrl": "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip" + } + ] +} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/MsrcConnectorTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/MsrcConnectorTests.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/MsrcConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/MsrcConnectorTests.cs index 134d0261..41587ff6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/MsrcConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/MsrcConnectorTests.cs @@ -1,200 +1,200 @@ -using System; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; -using StellaOps.Feedser.Source.Vndr.Msrc.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; -using Xunit; -using StellaOps.Feedser.Source.Common.Http; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Tests; - -[Collection("mongo-fixture")] -public sealed class MsrcConnectorTests : IAsyncLifetime -{ - private static readonly Uri TokenUri = new("https://login.microsoftonline.com/11111111-1111-1111-1111-111111111111/oauth2/v2.0/token"); - private static readonly Uri SummaryUri = new("https://api.msrc.microsoft.com/sug/v2.0/vulnerabilities"); - private static readonly Uri DetailUri = new("https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV123456"); - - private readonly MongoIntegrationFixture _fixture; - private readonly CannedHttpMessageHandler _handler; - - public MsrcConnectorTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - _handler = new CannedHttpMessageHandler(); - } - - [Fact] - public async Task FetchParseMap_ProducesCanonicalAdvisory() - { - await using var provider = await BuildServiceProviderAsync(); - SeedResponses(); - - var connector = provider.GetRequiredService(); - await connector.FetchAsync(provider, CancellationToken.None); - await connector.ParseAsync(provider, CancellationToken.None); - await connector.MapAsync(provider, CancellationToken.None); - - var advisoryStore = provider.GetRequiredService(); - var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); - advisories.Should().HaveCount(1); - - var advisory = advisories[0]; - advisory.AdvisoryKey.Should().Be("ADV123456"); - advisory.Severity.Should().Be("critical"); - advisory.Aliases.Should().Contain("CVE-2025-0001"); - advisory.Aliases.Should().Contain("KB5031234"); - advisory.References.Should().Contain(reference => reference.Url == "https://msrc.microsoft.com/update-guide/vulnerability/ADV123456"); - advisory.References.Should().Contain(reference => reference.Url == "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip"); - advisory.AffectedPackages.Should().HaveCount(1); - advisory.AffectedPackages[0].NormalizedVersions.Should().Contain(rule => rule.Scheme == "msrc.build" && rule.Value == "22631.3520"); - advisory.CvssMetrics.Should().Contain(metric => metric.BaseScore == 8.1); - - var stateRepository = provider.GetRequiredService(); - var state = await stateRepository.TryGetAsync(MsrcConnectorPlugin.SourceName, CancellationToken.None); - state.Should().NotBeNull(); - state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); - pendingDocs!.AsBsonArray.Should().BeEmpty(); - - var documentStore = provider.GetRequiredService(); - var cvrfDocument = await documentStore.FindBySourceAndUriAsync(MsrcConnectorPlugin.SourceName, "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip", CancellationToken.None); - cvrfDocument.Should().NotBeNull(); - cvrfDocument!.Status.Should().Be(DocumentStatuses.Mapped); - } - - private async Task BuildServiceProviderAsync() - { - await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); - _handler.Clear(); - - var services = new ServiceCollection(); - services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); - services.AddSingleton(_handler); - services.AddSingleton(TimeProvider.System); - - services.AddMongoStorage(options => - { - options.ConnectionString = _fixture.Runner.ConnectionString; - options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; - options.CommandTimeout = TimeSpan.FromSeconds(5); - }); - - services.AddSourceCommon(); - services.AddMsrcConnector(options => - { - options.TenantId = "11111111-1111-1111-1111-111111111111"; - options.ClientId = "client-id"; - options.ClientSecret = "secret"; - options.InitialLastModified = new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero); - options.RequestDelay = TimeSpan.Zero; - options.MaxAdvisoriesPerFetch = 10; - options.CursorOverlap = TimeSpan.FromMinutes(1); - options.DownloadCvrf = true; - }); - - services.Configure(MsrcOptions.HttpClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - services.Configure(MsrcOptions.TokenClientName, builderOptions => - { - builderOptions.HttpMessageHandlerBuilderActions.Add(builder => - { - builder.PrimaryHandler = _handler; - }); - }); - - var provider = services.BuildServiceProvider(); - var bootstrapper = provider.GetRequiredService(); - await bootstrapper.InitializeAsync(CancellationToken.None); - return provider; - } - - private void SeedResponses() - { - var summaryJson = ReadFixture("msrc-summary.json"); - var detailJson = ReadFixture("msrc-detail.json"); - var tokenJson = """{"token_type":"Bearer","expires_in":3600,"access_token":"fake-token"}"""; - var cvrfBytes = Encoding.UTF8.GetBytes("PK\x03\x04FAKECVRF"); - - _handler.SetFallback(request => - { - if (request.RequestUri is null) - { - return new HttpResponseMessage(HttpStatusCode.BadRequest); - } - - if (request.RequestUri.Host.Contains("login.microsoftonline.com", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(tokenJson, Encoding.UTF8, "application/json"), - }; - } - - if (request.RequestUri.AbsolutePath.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(summaryJson, Encoding.UTF8, "application/json"), - }; - } - - if (request.RequestUri.AbsolutePath.Contains("/vulnerability/ADV123456", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(detailJson, Encoding.UTF8, "application/json"), - }; - } - - if (request.RequestUri.Host.Contains("download.microsoft.com", StringComparison.OrdinalIgnoreCase)) - { - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new ByteArrayContent(cvrfBytes) - { - Headers = - { - ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/zip"), - }, - }, - }; - } - - return new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent($"No canned response for {request.RequestUri}", Encoding.UTF8), - }; - }); - } - - private static string ReadFixture(string fileName) - => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() => Task.CompletedTask; -} +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; +using Xunit; +using StellaOps.Concelier.Connector.Common.Http; + +namespace StellaOps.Concelier.Connector.Vndr.Msrc.Tests; + +[Collection("mongo-fixture")] +public sealed class MsrcConnectorTests : IAsyncLifetime +{ + private static readonly Uri TokenUri = new("https://login.microsoftonline.com/11111111-1111-1111-1111-111111111111/oauth2/v2.0/token"); + private static readonly Uri SummaryUri = new("https://api.msrc.microsoft.com/sug/v2.0/vulnerabilities"); + private static readonly Uri DetailUri = new("https://api.msrc.microsoft.com/sug/v2.0/vulnerability/ADV123456"); + + private readonly MongoIntegrationFixture _fixture; + private readonly CannedHttpMessageHandler _handler; + + public MsrcConnectorTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + _handler = new CannedHttpMessageHandler(); + } + + [Fact] + public async Task FetchParseMap_ProducesCanonicalAdvisory() + { + await using var provider = await BuildServiceProviderAsync(); + SeedResponses(); + + var connector = provider.GetRequiredService(); + await connector.FetchAsync(provider, CancellationToken.None); + await connector.ParseAsync(provider, CancellationToken.None); + await connector.MapAsync(provider, CancellationToken.None); + + var advisoryStore = provider.GetRequiredService(); + var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None); + advisories.Should().HaveCount(1); + + var advisory = advisories[0]; + advisory.AdvisoryKey.Should().Be("ADV123456"); + advisory.Severity.Should().Be("critical"); + advisory.Aliases.Should().Contain("CVE-2025-0001"); + advisory.Aliases.Should().Contain("KB5031234"); + advisory.References.Should().Contain(reference => reference.Url == "https://msrc.microsoft.com/update-guide/vulnerability/ADV123456"); + advisory.References.Should().Contain(reference => reference.Url == "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip"); + advisory.AffectedPackages.Should().HaveCount(1); + advisory.AffectedPackages[0].NormalizedVersions.Should().Contain(rule => rule.Scheme == "msrc.build" && rule.Value == "22631.3520"); + advisory.CvssMetrics.Should().Contain(metric => metric.BaseScore == 8.1); + + var stateRepository = provider.GetRequiredService(); + var state = await stateRepository.TryGetAsync(MsrcConnectorPlugin.SourceName, CancellationToken.None); + state.Should().NotBeNull(); + state!.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue(); + pendingDocs!.AsBsonArray.Should().BeEmpty(); + + var documentStore = provider.GetRequiredService(); + var cvrfDocument = await documentStore.FindBySourceAndUriAsync(MsrcConnectorPlugin.SourceName, "https://download.microsoft.com/msrc/2025/ADV123456.cvrf.zip", CancellationToken.None); + cvrfDocument.Should().NotBeNull(); + cvrfDocument!.Status.Should().Be(DocumentStatuses.Mapped); + } + + private async Task BuildServiceProviderAsync() + { + await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName); + _handler.Clear(); + + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance)); + services.AddSingleton(_handler); + services.AddSingleton(TimeProvider.System); + + services.AddMongoStorage(options => + { + options.ConnectionString = _fixture.Runner.ConnectionString; + options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName; + options.CommandTimeout = TimeSpan.FromSeconds(5); + }); + + services.AddSourceCommon(); + services.AddMsrcConnector(options => + { + options.TenantId = "11111111-1111-1111-1111-111111111111"; + options.ClientId = "client-id"; + options.ClientSecret = "secret"; + options.InitialLastModified = new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero); + options.RequestDelay = TimeSpan.Zero; + options.MaxAdvisoriesPerFetch = 10; + options.CursorOverlap = TimeSpan.FromMinutes(1); + options.DownloadCvrf = true; + }); + + services.Configure(MsrcOptions.HttpClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + services.Configure(MsrcOptions.TokenClientName, builderOptions => + { + builderOptions.HttpMessageHandlerBuilderActions.Add(builder => + { + builder.PrimaryHandler = _handler; + }); + }); + + var provider = services.BuildServiceProvider(); + var bootstrapper = provider.GetRequiredService(); + await bootstrapper.InitializeAsync(CancellationToken.None); + return provider; + } + + private void SeedResponses() + { + var summaryJson = ReadFixture("msrc-summary.json"); + var detailJson = ReadFixture("msrc-detail.json"); + var tokenJson = """{"token_type":"Bearer","expires_in":3600,"access_token":"fake-token"}"""; + var cvrfBytes = Encoding.UTF8.GetBytes("PK\x03\x04FAKECVRF"); + + _handler.SetFallback(request => + { + if (request.RequestUri is null) + { + return new HttpResponseMessage(HttpStatusCode.BadRequest); + } + + if (request.RequestUri.Host.Contains("login.microsoftonline.com", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(tokenJson, Encoding.UTF8, "application/json"), + }; + } + + if (request.RequestUri.AbsolutePath.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(summaryJson, Encoding.UTF8, "application/json"), + }; + } + + if (request.RequestUri.AbsolutePath.Contains("/vulnerability/ADV123456", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(detailJson, Encoding.UTF8, "application/json"), + }; + } + + if (request.RequestUri.Host.Contains("download.microsoft.com", StringComparison.OrdinalIgnoreCase)) + { + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(cvrfBytes) + { + Headers = + { + ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/zip"), + }, + }, + }; + } + + return new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No canned response for {request.RequestUri}", Encoding.UTF8), + }; + }); + } + + private static string ReadFixture(string fileName) + => System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName)); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; +} diff --git a/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/StellaOps.Concelier.Connector.Vndr.Msrc.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/StellaOps.Concelier.Connector.Vndr.Msrc.Tests.csproj new file mode 100644 index 00000000..78191fde --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc.Tests/StellaOps.Concelier.Connector.Vndr.Msrc.Tests.csproj @@ -0,0 +1,24 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md similarity index 86% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md index 289819ad..e2f82ea5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/AGENTS.md @@ -1,30 +1,30 @@ -# AGENTS -## Role -Implement the Microsoft Security Response Center (MSRC) connector to ingest Microsoft security updates (Security Updates API / CVRF). - -## Scope -- Identify MSRC data sources (Security Update Guide API, CVRF downloads) and incremental update strategy. -- Implement fetch/cursor pipeline with retry/backoff, handling API keys if required. -- Parse advisories to extract summary, affected products, KBs, CVEs, severities, mitigations. -- Map entries into canonical `Advisory` objects with aliases, references, affected packages, and range primitives (e.g., Windows build numbers, SemVer). -- Provide deterministic fixtures and regression tests. - -## Participants -- `Source.Common`, `Storage.Mongo`, `Feedser.Models`, `Feedser.Testing`. - -## Interfaces & Contracts -- Job kinds: `msrc:fetch`, `msrc:parse`, `msrc:map`. -- Persist upstream metadata (e.g., `lastModified`, `releaseDate`). -- Alias set should include MSRC ID, CVEs, and KB identifiers. - -## In/Out of scope -In scope: Microsoft Security Update Guide advisories. -Out of scope: Non-security Microsoft release notes. - -## Observability & Security Expectations -- Log fetch/mapping stats, respect API rate limits, handle authentication securely. -- Sanitize payloads; validate JSON/CVRF before persistence. - -## Tests -- Add `StellaOps.Feedser.Source.Vndr.Msrc.Tests` with fixtures covering fetch/parse/map. -- Snapshot canonical advisories; support fixture regeneration. +# AGENTS +## Role +Implement the Microsoft Security Response Center (MSRC) connector to ingest Microsoft security updates (Security Updates API / CVRF). + +## Scope +- Identify MSRC data sources (Security Update Guide API, CVRF downloads) and incremental update strategy. +- Implement fetch/cursor pipeline with retry/backoff, handling API keys if required. +- Parse advisories to extract summary, affected products, KBs, CVEs, severities, mitigations. +- Map entries into canonical `Advisory` objects with aliases, references, affected packages, and range primitives (e.g., Windows build numbers, SemVer). +- Provide deterministic fixtures and regression tests. + +## Participants +- `Source.Common`, `Storage.Mongo`, `Concelier.Models`, `Concelier.Testing`. + +## Interfaces & Contracts +- Job kinds: `msrc:fetch`, `msrc:parse`, `msrc:map`. +- Persist upstream metadata (e.g., `lastModified`, `releaseDate`). +- Alias set should include MSRC ID, CVEs, and KB identifiers. + +## In/Out of scope +In scope: Microsoft Security Update Guide advisories. +Out of scope: Non-security Microsoft release notes. + +## Observability & Security Expectations +- Log fetch/mapping stats, respect API rate limits, handle authentication securely. +- Sanitize payloads; validate JSON/CVRF before persistence. + +## Tests +- Add `StellaOps.Concelier.Connector.Vndr.Msrc.Tests` with fixtures covering fetch/parse/map. +- Snapshot canonical advisories; support fixture regeneration. diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Jobs.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/Jobs.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs index 9e618b6c..3101773a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Vndr.Msrc; +namespace StellaOps.Concelier.Connector.Vndr.Msrc; internal static class MsrcJobKinds { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs similarity index 97% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs index 33114ff5..c4fd943d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnector.cs @@ -9,18 +9,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; -using StellaOps.Feedser.Source.Vndr.Msrc.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Msrc; +namespace StellaOps.Concelier.Connector.Vndr.Msrc; public sealed class MsrcConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnectorPlugin.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnectorPlugin.cs index 61176fd9..654011b4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Msrc; +namespace StellaOps.Concelier.Connector.Vndr.Msrc; public sealed class MsrcConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs index 685941e8..033ad3e4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Msrc; +namespace StellaOps.Concelier.Connector.Vndr.Msrc; public sealed class MsrcDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:vndr:msrc"; + private const string ConfigurationSection = "concelier:sources:vndr:msrc"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs index d02b321b..f5aed6e3 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/MsrcServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/MsrcServiceCollectionExtensions.cs @@ -3,11 +3,11 @@ using System.Net; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; -using StellaOps.Feedser.Source.Vndr.Msrc.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Msrc.Configuration; +using StellaOps.Concelier.Connector.Vndr.Msrc.Internal; -namespace StellaOps.Feedser.Source.Vndr.Msrc; +namespace StellaOps.Concelier.Connector.Vndr.Msrc; public static class MsrcServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/README.md b/src/StellaOps.Concelier.Connector.Vndr.Msrc/README.md similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/README.md rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/README.md index 6b45656e..45e65e81 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/README.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/README.md @@ -12,7 +12,7 @@ ## Authentication - Uses Azure AD client credential flow against `https://login.microsoftonline.com/{tenantId}/oauth2/v2.0/token` with scope `api://api.msrc.microsoft.com/.default`. - Token refresh happens lazily and is cached until 60 seconds before expiry. -- Configuration values (`tenantId`, `clientId`, `clientSecret`) must be supplied via `feedser:sources:vndr:msrc`. +- Configuration values (`tenantId`, `clientId`, `clientSecret`) must be supplied via `concelier:sources:vndr:msrc`. ## CVRF handling - Detail payload is persisted with the `cvrfUrl` in metadata (`msrc.cvrfUrl`). diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj b/src/StellaOps.Concelier.Connector.Vndr.Msrc/StellaOps.Concelier.Connector.Vndr.Msrc.csproj similarity index 57% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/StellaOps.Concelier.Connector.Vndr.Msrc.csproj index f7f2c154..d97e92d3 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/StellaOps.Feedser.Source.Vndr.Msrc.csproj +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/StellaOps.Concelier.Connector.Vndr.Msrc.csproj @@ -9,8 +9,8 @@ - - + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md similarity index 78% rename from src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md index 8139644e..067596e1 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/TASKS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Msrc/TASKS.md @@ -1,11 +1,11 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Confirmed REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) + CVRF ZIP download flow, required Azure AD client-credentials scope (`api://api.msrc.microsoft.com/.default`), mandatory `api-version=2024-08-01` header, and delta params (`lastModifiedStartDateTime`, `lastModifiedEndDateTime`). Findings recorded in `docs/feedser-connector-research-20251011.md`.| -|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**DONE (2025-10-15)** – Added `MsrcApiClient` + token provider, cursor overlap handling, and detail persistence via GridFS (metadata carries CVRF URL + timestamps). State tracks `lastModifiedCursor` with configurable overlap/backoff. **Next:** coordinate with Tools on shared state-seeding helper once CVRF download flag stabilises.| -|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**DONE (2025-10-15)** – Implemented `MsrcDetailParser`/DTOs capturing threats, remediations, KB IDs, CVEs, CVSS, and affected products (build/platform metadata preserved).| -|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**DONE (2025-10-15)** – `MsrcMapper` emits aliases (MSRC ID/CVE/KB), references (release notes + CVRF), vendor packages with `msrc.build` normalized rules, and CVSS provenance.| -|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-15)** – Added `StellaOps.Feedser.Source.Vndr.Msrc.Tests` with canned token/summary/detail responses and snapshot assertions via Mongo2Go. Fixtures regenerate via `UPDATE_MSRC_FIXTURES`.| -|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Introduced `MsrcDiagnostics` meter (summary/detail/parse/map metrics), structured fetch logs, README updates, and Ops brief `docs/ops/feedser-msrc-operations.md` covering AAD onboarding + CVRF handling.| -|FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Completed memo outline recommending dual-path (REST for incremental, CVRF for offline); implementation hinges on `FEEDCONN-MSRC-02-008` AAD onboarding for token acquisition.| -|FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**DONE (2025-10-15)** – Coordinated Ops handoff; drafted AAD onboarding brief (`docs/ops/feedser-msrc-operations.md`) with app registration requirements, secret rotation policy, sample configuration, and CVRF mirroring guidance for Offline Kit.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDCONN-MSRC-02-001 Document MSRC Security Update Guide API|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Confirmed REST endpoint (`https://api.msrc.microsoft.com/sug/v2.0/en-US/vulnerabilities`) + CVRF ZIP download flow, required Azure AD client-credentials scope (`api://api.msrc.microsoft.com/.default`), mandatory `api-version=2024-08-01` header, and delta params (`lastModifiedStartDateTime`, `lastModifiedEndDateTime`). Findings recorded in `docs/concelier-connector-research-20251011.md`.| +|FEEDCONN-MSRC-02-002 Fetch pipeline & source state|BE-Conn-MSRC|Source.Common, Storage.Mongo|**DONE (2025-10-15)** – Added `MsrcApiClient` + token provider, cursor overlap handling, and detail persistence via GridFS (metadata carries CVRF URL + timestamps). State tracks `lastModifiedCursor` with configurable overlap/backoff. **Next:** coordinate with Tools on shared state-seeding helper once CVRF download flag stabilises.| +|FEEDCONN-MSRC-02-003 Parser & DTO implementation|BE-Conn-MSRC|Source.Common|**DONE (2025-10-15)** – Implemented `MsrcDetailParser`/DTOs capturing threats, remediations, KB IDs, CVEs, CVSS, and affected products (build/platform metadata preserved).| +|FEEDCONN-MSRC-02-004 Canonical mapping & range primitives|BE-Conn-MSRC|Models|**DONE (2025-10-15)** – `MsrcMapper` emits aliases (MSRC ID/CVE/KB), references (release notes + CVRF), vendor packages with `msrc.build` normalized rules, and CVSS provenance.| +|FEEDCONN-MSRC-02-005 Deterministic fixtures/tests|QA|Testing|**DONE (2025-10-15)** – Added `StellaOps.Concelier.Connector.Vndr.Msrc.Tests` with canned token/summary/detail responses and snapshot assertions via Mongo2Go. Fixtures regenerate via `UPDATE_MSRC_FIXTURES`.| +|FEEDCONN-MSRC-02-006 Telemetry & documentation|DevEx|Docs|**DONE (2025-10-15)** – Introduced `MsrcDiagnostics` meter (summary/detail/parse/map metrics), structured fetch logs, README updates, and Ops brief `docs/ops/concelier-msrc-operations.md` covering AAD onboarding + CVRF handling.| +|FEEDCONN-MSRC-02-007 API contract comparison memo|BE-Conn-MSRC|Research|**DONE (2025-10-11)** – Completed memo outline recommending dual-path (REST for incremental, CVRF for offline); implementation hinges on `FEEDCONN-MSRC-02-008` AAD onboarding for token acquisition.| +|FEEDCONN-MSRC-02-008 Azure AD application onboarding|Ops, BE-Conn-MSRC|Ops|**DONE (2025-10-15)** – Coordinated Ops handoff; drafted AAD onboarding brief (`docs/ops/concelier-msrc-operations.md`) with app registration requirements, secret rotation policy, sample configuration, and CVRF mirroring guidance for Offline Kit.| diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-advisories.snapshot.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024-single.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-calendar-cpuapr2024.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-01.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-cpuapr2024-02.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/Fixtures/oracle-detail-invalid.html diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs index 3a8e175c..fac8670a 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/Oracle/OracleConnectorTests.cs @@ -18,21 +18,21 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Serializers; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Vndr.Oracle; -using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; -using StellaOps.Feedser.Source.Vndr.Oracle.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Vndr.Oracle; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; using Xunit.Abstractions; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Tests; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Tests; [Collection("mongo-fixture")] public sealed class OracleConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests.csproj new file mode 100644 index 00000000..0487a6c6 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle.Tests/StellaOps.Concelier.Connector.Vndr.Oracle.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Oracle/AGENTS.md similarity index 80% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/AGENTS.md index 0145dc63..f17567a5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/AGENTS.md @@ -19,9 +19,9 @@ Oracle PSIRT connector for Critical Patch Updates (CPU) and Security Alerts; aut In: PSIRT authoritative mapping, cycles handling, precedence signaling. Out: signing or patch artifact downloads. ## Observability & security expectations -- Metrics: SourceDiagnostics emits `feedser.source.http.*` counters/histograms tagged `feedser.source=oracle`, so observability dashboards slice on that tag to monitor fetch pages, CPU cycle coverage, parse failures, and map affected counts. +- Metrics: SourceDiagnostics emits `concelier.source.http.*` counters/histograms tagged `concelier.source=oracle`, so observability dashboards slice on that tag to monitor fetch pages, CPU cycle coverage, parse failures, and map affected counts. - Logs: cycle tags, advisory ids, extraction timings; redact nothing sensitive. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Oracle.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Oracle.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs index 4a336c7a..edfb95d9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Configuration/OracleOptions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Configuration/OracleOptions.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; public sealed class OracleOptions { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleAffectedEntry.cs similarity index 85% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleAffectedEntry.cs index 07804c01..44bc48ef 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleAffectedEntry.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleAffectedEntry.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal sealed record OracleAffectedEntry( [property: JsonPropertyName("product")] string Product, diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs index 3dc5406e..e5b91633 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCalendarFetcher.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCalendarFetcher.cs @@ -7,9 +7,9 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; public sealed class OracleCalendarFetcher { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCursor.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCursor.cs index 72e9b1b4..4c462f91 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleCursor.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleCursor.cs @@ -2,9 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal sealed record OracleCursor( DateTimeOffset? LastProcessed, diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDocumentMetadata.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDocumentMetadata.cs index b609d5fe..dccac7b5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDocumentMetadata.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDocumentMetadata.cs @@ -1,8 +1,8 @@ using System; using System.Collections.Generic; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal sealed record OracleDocumentMetadata( string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDto.cs similarity index 90% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDto.cs index 074db653..8ec923d6 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDto.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDto.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal sealed record OracleDto( [property: JsonPropertyName("advisoryId")] string AdvisoryId, diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDtoValidator.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDtoValidator.cs index f782e96b..4ca15bf3 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleDtoValidator.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleDtoValidator.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal static class OracleDtoValidator { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleMapper.cs similarity index 95% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleMapper.cs index 0a805658..418a31fa 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleMapper.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleMapper.cs @@ -2,13 +2,13 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text.RegularExpressions; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal static class OracleMapper { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleParser.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleParser.cs index 6cd27135..5ea69512 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OracleParser.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OracleParser.cs @@ -6,7 +6,7 @@ using System.Text.RegularExpressions; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal static class OracleParser { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OraclePatchDocument.cs similarity index 78% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OraclePatchDocument.cs index 42787aeb..18638e13 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Internal/OraclePatchDocument.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Internal/OraclePatchDocument.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Oracle.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Oracle.Internal; internal sealed record OraclePatchDocument( [property: JsonPropertyName("product")] string Product, diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs index ba7602be..2bcdc478 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Vndr.Oracle; +namespace StellaOps.Concelier.Connector.Vndr.Oracle; internal static class OracleJobKinds { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs index 54e24a8f..912a6707 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleConnector.cs @@ -7,18 +7,18 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; -using StellaOps.Feedser.Source.Vndr.Oracle.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Oracle; +namespace StellaOps.Concelier.Connector.Vndr.Oracle; public sealed class OracleConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs similarity index 84% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs index 1a1f42a2..41539b95 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Oracle; +namespace StellaOps.Concelier.Connector.Vndr.Oracle; public sealed class OracleDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:oracle"; + private const string ConfigurationSection = "concelier:sources:oracle"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs index 1acaaca8..a6911e55 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/OracleServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/OracleServiceCollectionExtensions.cs @@ -2,11 +2,11 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Oracle.Configuration; -using StellaOps.Feedser.Source.Vndr.Oracle.Internal; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Oracle.Configuration; +using StellaOps.Concelier.Connector.Vndr.Oracle.Internal; -namespace StellaOps.Feedser.Source.Vndr.Oracle; +namespace StellaOps.Concelier.Connector.Vndr.Oracle; public static class OracleServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class OracleServiceCollectionExtensions { var options = sp.GetRequiredService>().Value; clientOptions.Timeout = TimeSpan.FromSeconds(30); - clientOptions.UserAgent = "StellaOps.Feedser.Oracle/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.Oracle/1.0"; clientOptions.AllowedHosts.Clear(); foreach (var uri in options.AdvisoryUris) { diff --git a/src/StellaOps.Concelier.Connector.Vndr.Oracle/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..d040d4b0 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Oracle.Tests")] diff --git a/src/StellaOps.Concelier.Connector.Vndr.Oracle/StellaOps.Concelier.Connector.Vndr.Oracle.csproj b/src/StellaOps.Concelier.Connector.Vndr.Oracle/StellaOps.Concelier.Connector.Vndr.Oracle.csproj new file mode 100644 index 00000000..092f2073 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/StellaOps.Concelier.Connector.Vndr.Oracle.csproj @@ -0,0 +1,17 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Oracle/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Oracle/VndrOracleConnectorPlugin.cs similarity index 88% rename from src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Oracle/VndrOracleConnectorPlugin.cs index 0ec2ee31..2430edcf 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/VndrOracleConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Oracle/VndrOracleConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Oracle; +namespace StellaOps.Concelier.Connector.Vndr.Oracle; public sealed class VndrOracleConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests.csproj b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests.csproj new file mode 100644 index 00000000..da4707d3 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/StellaOps.Concelier.Connector.Vndr.Vmware.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json index 361d8f63..c10778ed 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-advisories.snapshot.json @@ -1,275 +1,275 @@ -[ - { - "advisoryKey": "VMSA-2024-0001", - "affectedPackages": [ - { - "identifier": "VMware ESXi 7.0", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "affected", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware ESXi 7.0" - } - ], - "statuses": [], - "type": "vendor", - "versionRanges": [ - { - "fixedVersion": "7.0u3f", - "introducedVersion": "7.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": null, - "fixed": null, - "fixedInclusive": false, - "introduced": "7.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": false - }, - "vendorExtensions": { - "vmware.product": "VMware ESXi 7.0", - "vmware.version.raw": "7.0", - "vmware.fixedVersion.raw": "7.0u3f" - } - }, - "provenance": { - "fieldMask": [], - "kind": "range", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware ESXi 7.0" - }, - "rangeExpression": "7.0", - "rangeKind": "vendor" - } - ] - }, - { - "identifier": "VMware vCenter Server 8.0", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "affected", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware vCenter Server 8.0" - } - ], - "statuses": [], - "type": "vendor", - "versionRanges": [ - { - "fixedVersion": "8.0a", - "introducedVersion": "8.0", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": null, - "fixed": null, - "fixedInclusive": false, - "introduced": "8.0", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": false - }, - "vendorExtensions": { - "vmware.product": "VMware vCenter Server 8.0", - "vmware.version.raw": "8.0", - "vmware.fixedVersion.raw": "8.0a" - } - }, - "provenance": { - "fieldMask": [], - "kind": "range", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware vCenter Server 8.0" - }, - "rangeExpression": "8.0", - "rangeKind": "vendor" - } - ] - } - ], - "aliases": [ - "CVE-2024-1000", - "CVE-2024-1001", - "VMSA-2024-0001" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2024-04-01T10:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://vmware.example/api/vmsa/VMSA-2024-0001.json" - }, - { - "fieldMask": [], - "kind": "mapping", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMSA-2024-0001" - } - ], - "published": "2024-04-01T10:00:00+00:00", - "references": [ - { - "kind": "kb", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://kb.vmware.example/90234" - }, - "sourceTag": "kb", - "summary": null, - "url": "https://kb.vmware.example/90234" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" - }, - "sourceTag": "advisory", - "summary": null, - "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" - } - ], - "severity": null, - "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", - "title": "VMware ESXi and vCenter Server updates address vulnerabilities" - }, - { - "advisoryKey": "VMSA-2024-0002", - "affectedPackages": [ - { - "identifier": "VMware Cloud Foundation 5.x", - "platform": null, - "provenance": [ - { - "fieldMask": [], - "kind": "affected", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware Cloud Foundation 5.x" - } - ], - "statuses": [], - "type": "vendor", - "versionRanges": [ - { - "fixedVersion": "5.1.1", - "introducedVersion": "5.1", - "lastAffectedVersion": null, - "primitives": { - "evr": null, - "hasVendorExtensions": true, - "nevra": null, - "semVer": { - "constraintExpression": null, - "fixed": "5.1.1", - "fixedInclusive": false, - "introduced": "5.1", - "introducedInclusive": true, - "lastAffected": null, - "lastAffectedInclusive": false - }, - "vendorExtensions": { - "vmware.product": "VMware Cloud Foundation 5.x", - "vmware.version.raw": "5.1", - "vmware.fixedVersion.raw": "5.1.1" - } - }, - "provenance": { - "fieldMask": [], - "kind": "range", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMware Cloud Foundation 5.x" - }, - "rangeExpression": "5.1", - "rangeKind": "vendor" - } - ] - } - ], - "aliases": [ - "CVE-2024-2000", - "VMSA-2024-0002" - ], - "cvssMetrics": [], - "exploitKnown": false, - "language": "en", - "modified": "2024-04-02T09:00:00+00:00", - "provenance": [ - { - "fieldMask": [], - "kind": "document", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://vmware.example/api/vmsa/VMSA-2024-0002.json" - }, - { - "fieldMask": [], - "kind": "mapping", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "VMSA-2024-0002" - } - ], - "published": "2024-04-02T09:00:00+00:00", - "references": [ - { - "kind": "kb", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://kb.vmware.example/91234" - }, - "sourceTag": "kb", - "summary": null, - "url": "https://kb.vmware.example/91234" - }, - { - "kind": "advisory", - "provenance": { - "fieldMask": [], - "kind": "reference", - "recordedAt": "2024-04-05T00:00:00+00:00", - "source": "vmware", - "value": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" - }, - "sourceTag": "advisory", - "summary": null, - "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" - } - ], - "severity": null, - "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", - "title": "VMware Cloud Foundation remote code execution vulnerability" - } +[ + { + "advisoryKey": "VMSA-2024-0001", + "affectedPackages": [ + { + "identifier": "VMware ESXi 7.0", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "7.0u3f", + "introducedVersion": "7.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "7.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware ESXi 7.0", + "vmware.version.raw": "7.0", + "vmware.fixedVersion.raw": "7.0u3f" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware ESXi 7.0" + }, + "rangeExpression": "7.0", + "rangeKind": "vendor" + } + ] + }, + { + "identifier": "VMware vCenter Server 8.0", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "8.0a", + "introducedVersion": "8.0", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": null, + "fixedInclusive": false, + "introduced": "8.0", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware vCenter Server 8.0", + "vmware.version.raw": "8.0", + "vmware.fixedVersion.raw": "8.0a" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware vCenter Server 8.0" + }, + "rangeExpression": "8.0", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-1000", + "CVE-2024-1001", + "VMSA-2024-0001" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-01T10:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0001.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0001" + } + ], + "published": "2024-04-01T10:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/90234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/90234" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0001.html" + } + ], + "severity": null, + "summary": "Security updates for VMware ESXi 7.0 and vCenter Server 8.0 resolve multiple vulnerabilities.", + "title": "VMware ESXi and vCenter Server updates address vulnerabilities" + }, + { + "advisoryKey": "VMSA-2024-0002", + "affectedPackages": [ + { + "identifier": "VMware Cloud Foundation 5.x", + "platform": null, + "provenance": [ + { + "fieldMask": [], + "kind": "affected", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + } + ], + "statuses": [], + "type": "vendor", + "versionRanges": [ + { + "fixedVersion": "5.1.1", + "introducedVersion": "5.1", + "lastAffectedVersion": null, + "primitives": { + "evr": null, + "hasVendorExtensions": true, + "nevra": null, + "semVer": { + "constraintExpression": null, + "fixed": "5.1.1", + "fixedInclusive": false, + "introduced": "5.1", + "introducedInclusive": true, + "lastAffected": null, + "lastAffectedInclusive": false + }, + "vendorExtensions": { + "vmware.product": "VMware Cloud Foundation 5.x", + "vmware.version.raw": "5.1", + "vmware.fixedVersion.raw": "5.1.1" + } + }, + "provenance": { + "fieldMask": [], + "kind": "range", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMware Cloud Foundation 5.x" + }, + "rangeExpression": "5.1", + "rangeKind": "vendor" + } + ] + } + ], + "aliases": [ + "CVE-2024-2000", + "VMSA-2024-0002" + ], + "cvssMetrics": [], + "exploitKnown": false, + "language": "en", + "modified": "2024-04-02T09:00:00+00:00", + "provenance": [ + { + "fieldMask": [], + "kind": "document", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://vmware.example/api/vmsa/VMSA-2024-0002.json" + }, + { + "fieldMask": [], + "kind": "mapping", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "VMSA-2024-0002" + } + ], + "published": "2024-04-02T09:00:00+00:00", + "references": [ + { + "kind": "kb", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://kb.vmware.example/91234" + }, + "sourceTag": "kb", + "summary": null, + "url": "https://kb.vmware.example/91234" + }, + { + "kind": "advisory", + "provenance": { + "fieldMask": [], + "kind": "reference", + "recordedAt": "2024-04-05T00:00:00+00:00", + "source": "vmware", + "value": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + }, + "sourceTag": "advisory", + "summary": null, + "url": "https://www.vmware.com/security/advisories/VMSA-2024-0002.html" + } + ], + "severity": null, + "summary": "An update is available for VMware Cloud Foundation components to address a remote code execution vulnerability.", + "title": "VMware Cloud Foundation remote code execution vulnerability" + } ] \ No newline at end of file diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0001.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0002.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-detail-vmsa-2024-0003.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-initial.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/Fixtures/vmware-index-second.json diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs index a1f98a6c..c37cdd42 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareConnectorTests.cs @@ -15,21 +15,21 @@ using Microsoft.Extensions.Options; using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Source.Vndr.Vmware; -using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; -using StellaOps.Feedser.Source.Vndr.Vmware.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Connector.Vndr.Vmware; +using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; +using StellaOps.Concelier.Connector.Vndr.Vmware.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Testing; using Xunit.Abstractions; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Tests.Vmware; [Collection("mongo-fixture")] public sealed class VmwareConnectorTests : IAsyncLifetime diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs index 1f7c23b1..a2fd486d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware.Tests/Vmware/VmwareMapperTests.cs @@ -2,15 +2,15 @@ using System; using System.Collections.Generic; using System.Text.Json; using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Vndr.Vmware; -using StellaOps.Feedser.Source.Vndr.Vmware.Internal; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Vndr.Vmware; +using StellaOps.Concelier.Connector.Vndr.Vmware.Internal; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; using Xunit; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Tests; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Tests; public sealed class VmwareMapperTests { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md b/src/StellaOps.Concelier.Connector.Vndr.Vmware/AGENTS.md similarity index 80% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/AGENTS.md index 724c6e4e..e1cec064 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/AGENTS.md +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/AGENTS.md @@ -20,9 +20,9 @@ VMware/Broadcom PSIRT connector ingesting VMSA advisories; authoritative for VMw In: PSIRT precedence mapping, affected/fixedBy extraction, advisory references. Out: customer portal authentication flows beyond public advisories; downloading patches. ## Observability & security expectations -- Metrics: SourceDiagnostics emits shared `feedser.source.http.*` counters/histograms tagged `feedser.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names. +- Metrics: SourceDiagnostics emits shared `concelier.source.http.*` counters/histograms tagged `concelier.source=vmware`, allowing dashboards to measure fetch volume, parse failures, and map affected counts without bespoke metric names. - Logs: vmsa ids, product counts, extraction timings; handle portal rate limits politely. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Source.Vndr.Vmware.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Connector.Vndr.Vmware.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs index 362c07c8..9ed8f9b9 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Configuration/VmwareOptions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Configuration/VmwareOptions.cs @@ -1,6 +1,6 @@ using System.Diagnostics.CodeAnalysis; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; public sealed class VmwareOptions { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareCursor.cs similarity index 96% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareCursor.cs index 44b27ca8..52e83ca4 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareCursor.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareCursor.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; internal sealed record VmwareCursor( DateTimeOffset? LastModified, diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareDetailDto.cs similarity index 92% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareDetailDto.cs index d8d54799..f05d7964 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareDetailDto.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareDetailDto.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; internal sealed record VmwareDetailDto { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs index 89634a6d..2e2091fa 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareFetchCacheEntry.cs @@ -1,8 +1,8 @@ using System; using MongoDB.Bson; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; internal sealed record VmwareFetchCacheEntry(string? Sha256, string? ETag, DateTimeOffset? LastModified) { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareIndexItem.cs similarity index 81% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareIndexItem.cs index 2099cdb4..2b065fe5 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareIndexItem.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareIndexItem.cs @@ -1,7 +1,7 @@ using System; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; internal sealed record VmwareIndexItem { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareMapper.cs similarity index 93% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareMapper.cs index 9602719f..9e7db5e7 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Internal/VmwareMapper.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Internal/VmwareMapper.cs @@ -1,14 +1,14 @@ using System; using System.Collections.Generic; using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Packages; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; -namespace StellaOps.Feedser.Source.Vndr.Vmware.Internal; +namespace StellaOps.Concelier.Connector.Vndr.Vmware.Internal; internal static class VmwareMapper { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Jobs.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/Jobs.cs index 14ebcec8..9dc64a05 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Jobs.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Jobs.cs @@ -1,9 +1,9 @@ using System; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; internal static class VmwareJobKinds { diff --git a/src/StellaOps.Concelier.Connector.Vndr.Vmware/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..5d57512d --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Connector.Vndr.Vmware.Tests")] diff --git a/src/StellaOps.Concelier.Connector.Vndr.Vmware/StellaOps.Concelier.Connector.Vndr.Vmware.csproj b/src/StellaOps.Concelier.Connector.Vndr.Vmware/StellaOps.Concelier.Connector.Vndr.Vmware.csproj new file mode 100644 index 00000000..b2bd7ee1 --- /dev/null +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/StellaOps.Concelier.Connector.Vndr.Vmware.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + <_Parameter1>StellaOps.Concelier.Tests + + + + diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md b/src/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/TASKS.md rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/TASKS.md diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs similarity index 94% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs index f5fe1c3a..1c2095d8 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnector.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnector.cs @@ -9,19 +9,19 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Bson.IO; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; -using StellaOps.Feedser.Source.Vndr.Vmware.Internal; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; +using StellaOps.Concelier.Connector.Vndr.Vmware.Internal; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; public sealed class VmwareConnector : IFeedConnector { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnectorPlugin.cs similarity index 87% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnectorPlugin.cs index 98f53d16..60c5f21d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareConnectorPlugin.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareConnectorPlugin.cs @@ -2,7 +2,7 @@ using System; using Microsoft.Extensions.DependencyInjection; using StellaOps.Plugin; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; public sealed class VmwareConnectorPlugin : IConnectorPlugin { diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs index 914aceb7..c845660d 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDependencyInjectionRoutine.cs @@ -2,14 +2,14 @@ using System; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; public sealed class VmwareDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:sources:vmware"; + private const string ConfigurationSection = "concelier:sources:vmware"; private const string FetchCron = "10,40 * * * *"; private const string ParseCron = "15,45 * * * *"; private const string MapCron = "20,50 * * * *"; diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDiagnostics.cs similarity index 91% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDiagnostics.cs index 57a68e6b..950a5920 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareDiagnostics.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareDiagnostics.cs @@ -1,14 +1,14 @@ using System; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; /// /// VMware connector metrics (fetch, parse, map). /// public sealed class VmwareDiagnostics : IDisposable { - public const string MeterName = "StellaOps.Feedser.Source.Vndr.Vmware"; + public const string MeterName = "StellaOps.Concelier.Connector.Vndr.Vmware"; private const string MeterVersion = "1.0.0"; private readonly Meter _meter; diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs similarity index 83% rename from src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs index 7765876e..17d12617 100644 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/VmwareServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Connector.Vndr.Vmware/VmwareServiceCollectionExtensions.cs @@ -2,10 +2,10 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Http; -using StellaOps.Feedser.Source.Vndr.Vmware.Configuration; +using StellaOps.Concelier.Connector.Common.Http; +using StellaOps.Concelier.Connector.Vndr.Vmware.Configuration; -namespace StellaOps.Feedser.Source.Vndr.Vmware; +namespace StellaOps.Concelier.Connector.Vndr.Vmware; public static class VmwareServiceCollectionExtensions { @@ -23,7 +23,7 @@ public static class VmwareServiceCollectionExtensions var options = sp.GetRequiredService>().Value; clientOptions.BaseAddress = new Uri(options.IndexUri.GetLeftPart(UriPartial.Authority)); clientOptions.Timeout = options.HttpTimeout; - clientOptions.UserAgent = "StellaOps.Feedser.VMware/1.0"; + clientOptions.UserAgent = "StellaOps.Concelier.VMware/1.0"; clientOptions.AllowedHosts.Clear(); clientOptions.AllowedHosts.Add(options.IndexUri.Host); clientOptions.DefaultRequestHeaders["Accept"] = "application/json"; diff --git a/src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs b/src/StellaOps.Concelier.Core.Tests/CanonicalMergerTests.cs similarity index 97% rename from src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs rename to src/StellaOps.Concelier.Core.Tests/CanonicalMergerTests.cs index d7db35ae..774730fc 100644 --- a/src/StellaOps.Feedser.Core.Tests/CanonicalMergerTests.cs +++ b/src/StellaOps.Concelier.Core.Tests/CanonicalMergerTests.cs @@ -1,371 +1,371 @@ -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Core.Tests; - -public sealed class CanonicalMergerTests -{ - private static readonly DateTimeOffset BaseTimestamp = new(2025, 10, 10, 0, 0, 0, TimeSpan.Zero); - - [Fact] - public void Merge_PrefersGhsaTitleAndSummaryByPrecedence() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(6))); - - var ghsa = CreateAdvisory( - source: "ghsa", - advisoryKey: "GHSA-aaaa-bbbb-cccc", - title: "GHSA Title", - summary: "GHSA Summary", - modified: BaseTimestamp.AddHours(1)); - - var nvd = CreateAdvisory( - source: "nvd", - advisoryKey: "CVE-2025-0001", - title: "NVD Title", - summary: "NVD Summary", - modified: BaseTimestamp); - - var result = merger.Merge("CVE-2025-0001", ghsa, nvd, null); - - Assert.Equal("GHSA Title", result.Advisory.Title); - Assert.Equal("GHSA Summary", result.Advisory.Summary); - - Assert.Contains(result.Decisions, decision => - decision.Field == "summary" && - string.Equals(decision.SelectedSource, "ghsa", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); - - Assert.Contains(result.Advisory.Provenance, provenance => - string.Equals(provenance.Source, "ghsa", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - public void Merge_FreshnessOverrideUsesOsvSummaryWhenNewerByThreshold() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(10))); - - var ghsa = CreateAdvisory( - source: "ghsa", - advisoryKey: "GHSA-xxxx-yyyy-zzzz", - title: "Container Escape Vulnerability", - summary: "Initial GHSA summary.", - modified: BaseTimestamp); - - var osv = CreateAdvisory( - source: "osv", - advisoryKey: "GHSA-xxxx-yyyy-zzzz", - title: "Container Escape Vulnerability", - summary: "OSV summary with additional mitigation steps.", - modified: BaseTimestamp.AddHours(72)); - - var result = merger.Merge("CVE-2025-9000", ghsa, null, osv); - - Assert.Equal("OSV summary with additional mitigation steps.", result.Advisory.Summary); - - Assert.Contains(result.Decisions, decision => - decision.Field == "summary" && - string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); - - Assert.Contains(result.Advisory.Provenance, provenance => - string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - public void Merge_AffectedPackagesPreferOsvPrecedence() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(4))); - - var ghsaPackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/example@1", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - rangeKind: "semver", - introducedVersion: null, - fixedVersion: "1.2.3", - lastAffectedVersion: null, - rangeExpression: "<1.2.3", - provenance: CreateProvenance("ghsa", ProvenanceFieldMasks.VersionRanges), - primitives: null) - }, - statuses: new[] - { - new AffectedPackageStatus( - "affected", - CreateProvenance("ghsa", ProvenanceFieldMasks.PackageStatuses)) - }, - provenance: new[] { CreateProvenance("ghsa", ProvenanceFieldMasks.AffectedPackages) }, - normalizedVersions: Array.Empty()); - - var nvdPackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/example@1", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - rangeKind: "semver", - introducedVersion: null, - fixedVersion: "1.2.4", - lastAffectedVersion: null, - rangeExpression: "<1.2.4", - provenance: CreateProvenance("nvd", ProvenanceFieldMasks.VersionRanges), - primitives: null) - }, - statuses: Array.Empty(), - provenance: new[] { CreateProvenance("nvd", ProvenanceFieldMasks.AffectedPackages) }, - normalizedVersions: Array.Empty()); - - var osvPackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/example@1", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - rangeKind: "semver", - introducedVersion: "1.0.0", - fixedVersion: "1.2.5", - lastAffectedVersion: null, - rangeExpression: ">=1.0.0,<1.2.5", - provenance: CreateProvenance("osv", ProvenanceFieldMasks.VersionRanges), - primitives: null) - }, - statuses: Array.Empty(), - provenance: new[] { CreateProvenance("osv", ProvenanceFieldMasks.AffectedPackages) }, - normalizedVersions: Array.Empty()); - - var ghsa = CreateAdvisory("ghsa", "GHSA-1234", "GHSA Title", modified: BaseTimestamp.AddHours(1), packages: new[] { ghsaPackage }); - var nvd = CreateAdvisory("nvd", "CVE-2025-1111", "NVD Title", modified: BaseTimestamp.AddHours(2), packages: new[] { nvdPackage }); - var osv = CreateAdvisory("osv", "OSV-2025-xyz", "OSV Title", modified: BaseTimestamp.AddHours(3), packages: new[] { osvPackage }); - - var result = merger.Merge("CVE-2025-1111", ghsa, nvd, osv); - - var package = Assert.Single(result.Advisory.AffectedPackages); - Assert.Equal("pkg:npm/example@1", package.Identifier); - Assert.Contains(package.Provenance, provenance => - string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); - - Assert.Contains(result.Decisions, decision => - decision.Field.StartsWith("affectedPackages", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - public void Merge_CvssMetricsOrderedByPrecedence() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(5))); - - var nvdMetric = new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", 9.8, "critical", CreateProvenance("nvd", ProvenanceFieldMasks.CvssMetrics)); - var ghsaMetric = new CvssMetric("3.0", "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", 7.5, "high", CreateProvenance("ghsa", ProvenanceFieldMasks.CvssMetrics)); - - var nvd = CreateAdvisory("nvd", "CVE-2025-2000", "NVD Title", severity: null, modified: BaseTimestamp, metrics: new[] { nvdMetric }); - var ghsa = CreateAdvisory("ghsa", "GHSA-9999", "GHSA Title", severity: null, modified: BaseTimestamp.AddHours(1), metrics: new[] { ghsaMetric }); - - var result = merger.Merge("CVE-2025-2000", ghsa, nvd, null); - - Assert.Equal(2, result.Advisory.CvssMetrics.Length); - Assert.Equal("nvd", result.Decisions.Single(decision => decision.Field == "cvssMetrics").SelectedSource); - Assert.Equal("critical", result.Advisory.Severity); - Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H"); - Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H"); - Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", result.Advisory.CanonicalMetricId); - } - - [Fact] - public void Merge_ReferencesNormalizedAndFreshnessOverrides() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(80))); - - var ghsa = CreateAdvisory( - source: "ghsa", - advisoryKey: "GHSA-ref", - title: "GHSA Title", - references: new[] - { - new AdvisoryReference( - "http://Example.COM/path/resource?b=2&a=1#section", - kind: "advisory", - sourceTag: null, - summary: null, - CreateProvenance("ghsa", ProvenanceFieldMasks.References)) - }, - modified: BaseTimestamp); - - var osv = CreateAdvisory( - source: "osv", - advisoryKey: "OSV-ref", - title: "OSV Title", - references: new[] - { - new AdvisoryReference( - "https://example.com/path/resource?a=1&b=2", - kind: "advisory", - sourceTag: null, - summary: null, - CreateProvenance("osv", ProvenanceFieldMasks.References)) - }, - modified: BaseTimestamp.AddHours(80)); - - var result = merger.Merge("CVE-REF-2025-01", ghsa, null, osv); - - var reference = Assert.Single(result.Advisory.References); - Assert.Equal("https://example.com/path/resource?a=1&b=2", reference.Url); - - var unionDecision = Assert.Single(result.Decisions.Where(decision => decision.Field == "references")); - Assert.Null(unionDecision.SelectedSource); - Assert.Equal("union", unionDecision.DecisionReason); - - var itemDecision = Assert.Single(result.Decisions.Where(decision => decision.Field.StartsWith("references[", StringComparison.OrdinalIgnoreCase))); - Assert.Equal("osv", itemDecision.SelectedSource); - Assert.Equal("freshness_override", itemDecision.DecisionReason); - Assert.Contains("https://example.com/path/resource?a=1&b=2", itemDecision.Field, StringComparison.OrdinalIgnoreCase); - } - - [Fact] - public void Merge_DescriptionFreshnessOverride() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(12))); - - var ghsa = CreateAdvisory( - source: "ghsa", - advisoryKey: "GHSA-desc", - title: "GHSA Title", - summary: "Summary", - description: "Initial GHSA description", - modified: BaseTimestamp.AddHours(1)); - - var nvd = CreateAdvisory( - source: "nvd", - advisoryKey: "CVE-2025-5555", - title: "NVD Title", - summary: "Summary", - description: "NVD baseline description", - modified: BaseTimestamp.AddHours(2)); - - var osv = CreateAdvisory( - source: "osv", - advisoryKey: "OSV-2025-desc", - title: "OSV Title", - summary: "Summary", - description: "OSV fresher description", - modified: BaseTimestamp.AddHours(72)); - - var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv); - - Assert.Equal("OSV fresher description", result.Advisory.Description); - Assert.Contains(result.Decisions, decision => - decision.Field == "description" && - string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - public void Merge_CwesPreferNvdPrecedence() - { - var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(6))); - - var ghsaWeakness = CreateWeakness("ghsa", "CWE-79", "cross-site scripting", BaseTimestamp.AddHours(1)); - var nvdWeakness = CreateWeakness("nvd", "CWE-79", "Cross-Site Scripting", BaseTimestamp.AddHours(2)); - var osvWeakness = CreateWeakness("osv", "CWE-79", "XSS", BaseTimestamp.AddHours(3)); - - var ghsa = CreateAdvisory("ghsa", "GHSA-weakness", "GHSA Title", weaknesses: new[] { ghsaWeakness }, modified: BaseTimestamp.AddHours(1)); - var nvd = CreateAdvisory("nvd", "CVE-2025-7777", "NVD Title", weaknesses: new[] { nvdWeakness }, modified: BaseTimestamp.AddHours(2)); - var osv = CreateAdvisory("osv", "OSV-weakness", "OSV Title", weaknesses: new[] { osvWeakness }, modified: BaseTimestamp.AddHours(3)); - - var result = merger.Merge("CVE-2025-7777", ghsa, nvd, osv); - - var weakness = Assert.Single(result.Advisory.Cwes); - Assert.Equal("CWE-79", weakness.Identifier); - Assert.Equal("Cross-Site Scripting", weakness.Name); - Assert.Contains(result.Decisions, decision => - decision.Field == "cwes[cwe|CWE-79]" && - string.Equals(decision.SelectedSource, "nvd", StringComparison.OrdinalIgnoreCase) && - string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); - } - - private static Advisory CreateAdvisory( - string source, - string advisoryKey, - string title, - string? summary = null, - string? description = null, - DateTimeOffset? modified = null, - string? severity = null, - IEnumerable? packages = null, - IEnumerable? metrics = null, - IEnumerable? references = null, - IEnumerable? weaknesses = null, - string? canonicalMetricId = null) - { - var provenance = new AdvisoryProvenance( - source, - kind: "map", - value: advisoryKey, - recordedAt: modified ?? BaseTimestamp, - fieldMask: new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - advisoryKey, - title, - summary, - language: "en", - published: modified, - modified: modified, - severity: severity, - exploitKnown: false, - aliases: new[] { advisoryKey }, - credits: Array.Empty(), - references: references ?? Array.Empty(), - affectedPackages: packages ?? Array.Empty(), - cvssMetrics: metrics ?? Array.Empty(), - provenance: new[] { provenance }, - description: description, - cwes: weaknesses ?? Array.Empty(), - canonicalMetricId: canonicalMetricId); - } - - private static AdvisoryProvenance CreateProvenance(string source, string fieldMask) - => new( - source, - kind: "map", - value: source, - recordedAt: BaseTimestamp, - fieldMask: new[] { fieldMask }); - - private static AdvisoryWeakness CreateWeakness(string source, string identifier, string? name, DateTimeOffset recordedAt) - { - var provenance = new AdvisoryProvenance( - source, - kind: "map", - value: identifier, - recordedAt: recordedAt, - fieldMask: new[] { ProvenanceFieldMasks.Weaknesses }); - - return new AdvisoryWeakness("cwe", identifier, name, uri: null, provenance: new[] { provenance }); - } - - private sealed class FixedTimeProvider : TimeProvider - { - private readonly DateTimeOffset _utcNow; - - public FixedTimeProvider(DateTimeOffset utcNow) - { - _utcNow = utcNow; - } - - public override DateTimeOffset GetUtcNow() => _utcNow; - } -} +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core.Tests; + +public sealed class CanonicalMergerTests +{ + private static readonly DateTimeOffset BaseTimestamp = new(2025, 10, 10, 0, 0, 0, TimeSpan.Zero); + + [Fact] + public void Merge_PrefersGhsaTitleAndSummaryByPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(6))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-aaaa-bbbb-cccc", + title: "GHSA Title", + summary: "GHSA Summary", + modified: BaseTimestamp.AddHours(1)); + + var nvd = CreateAdvisory( + source: "nvd", + advisoryKey: "CVE-2025-0001", + title: "NVD Title", + summary: "NVD Summary", + modified: BaseTimestamp); + + var result = merger.Merge("CVE-2025-0001", ghsa, nvd, null); + + Assert.Equal("GHSA Title", result.Advisory.Title); + Assert.Equal("GHSA Summary", result.Advisory.Summary); + + Assert.Contains(result.Decisions, decision => + decision.Field == "summary" && + string.Equals(decision.SelectedSource, "ghsa", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Advisory.Provenance, provenance => + string.Equals(provenance.Source, "ghsa", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_FreshnessOverrideUsesOsvSummaryWhenNewerByThreshold() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(10))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-xxxx-yyyy-zzzz", + title: "Container Escape Vulnerability", + summary: "Initial GHSA summary.", + modified: BaseTimestamp); + + var osv = CreateAdvisory( + source: "osv", + advisoryKey: "GHSA-xxxx-yyyy-zzzz", + title: "Container Escape Vulnerability", + summary: "OSV summary with additional mitigation steps.", + modified: BaseTimestamp.AddHours(72)); + + var result = merger.Merge("CVE-2025-9000", ghsa, null, osv); + + Assert.Equal("OSV summary with additional mitigation steps.", result.Advisory.Summary); + + Assert.Contains(result.Decisions, decision => + decision.Field == "summary" && + string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Advisory.Provenance, provenance => + string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Value, "summary", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_AffectedPackagesPreferOsvPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(4))); + + var ghsaPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.2.3", + lastAffectedVersion: null, + rangeExpression: "<1.2.3", + provenance: CreateProvenance("ghsa", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: new[] + { + new AffectedPackageStatus( + "affected", + CreateProvenance("ghsa", ProvenanceFieldMasks.PackageStatuses)) + }, + provenance: new[] { CreateProvenance("ghsa", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var nvdPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: null, + fixedVersion: "1.2.4", + lastAffectedVersion: null, + rangeExpression: "<1.2.4", + provenance: CreateProvenance("nvd", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: Array.Empty(), + provenance: new[] { CreateProvenance("nvd", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var osvPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example@1", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.5", + lastAffectedVersion: null, + rangeExpression: ">=1.0.0,<1.2.5", + provenance: CreateProvenance("osv", ProvenanceFieldMasks.VersionRanges), + primitives: null) + }, + statuses: Array.Empty(), + provenance: new[] { CreateProvenance("osv", ProvenanceFieldMasks.AffectedPackages) }, + normalizedVersions: Array.Empty()); + + var ghsa = CreateAdvisory("ghsa", "GHSA-1234", "GHSA Title", modified: BaseTimestamp.AddHours(1), packages: new[] { ghsaPackage }); + var nvd = CreateAdvisory("nvd", "CVE-2025-1111", "NVD Title", modified: BaseTimestamp.AddHours(2), packages: new[] { nvdPackage }); + var osv = CreateAdvisory("osv", "OSV-2025-xyz", "OSV Title", modified: BaseTimestamp.AddHours(3), packages: new[] { osvPackage }); + + var result = merger.Merge("CVE-2025-1111", ghsa, nvd, osv); + + var package = Assert.Single(result.Advisory.AffectedPackages); + Assert.Equal("pkg:npm/example@1", package.Identifier); + Assert.Contains(package.Provenance, provenance => + string.Equals(provenance.Source, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(result.Decisions, decision => + decision.Field.StartsWith("affectedPackages", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_CvssMetricsOrderedByPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(5))); + + var nvdMetric = new CvssMetric("3.1", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", 9.8, "critical", CreateProvenance("nvd", ProvenanceFieldMasks.CvssMetrics)); + var ghsaMetric = new CvssMetric("3.0", "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H", 7.5, "high", CreateProvenance("ghsa", ProvenanceFieldMasks.CvssMetrics)); + + var nvd = CreateAdvisory("nvd", "CVE-2025-2000", "NVD Title", severity: null, modified: BaseTimestamp, metrics: new[] { nvdMetric }); + var ghsa = CreateAdvisory("ghsa", "GHSA-9999", "GHSA Title", severity: null, modified: BaseTimestamp.AddHours(1), metrics: new[] { ghsaMetric }); + + var result = merger.Merge("CVE-2025-2000", ghsa, nvd, null); + + Assert.Equal(2, result.Advisory.CvssMetrics.Length); + Assert.Equal("nvd", result.Decisions.Single(decision => decision.Field == "cvssMetrics").SelectedSource); + Assert.Equal("critical", result.Advisory.Severity); + Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H"); + Assert.Contains(result.Advisory.CvssMetrics, metric => metric.Vector == "CVSS:3.0/AV:L/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H"); + Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", result.Advisory.CanonicalMetricId); + } + + [Fact] + public void Merge_ReferencesNormalizedAndFreshnessOverrides() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(80))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-ref", + title: "GHSA Title", + references: new[] + { + new AdvisoryReference( + "http://Example.COM/path/resource?b=2&a=1#section", + kind: "advisory", + sourceTag: null, + summary: null, + CreateProvenance("ghsa", ProvenanceFieldMasks.References)) + }, + modified: BaseTimestamp); + + var osv = CreateAdvisory( + source: "osv", + advisoryKey: "OSV-ref", + title: "OSV Title", + references: new[] + { + new AdvisoryReference( + "https://example.com/path/resource?a=1&b=2", + kind: "advisory", + sourceTag: null, + summary: null, + CreateProvenance("osv", ProvenanceFieldMasks.References)) + }, + modified: BaseTimestamp.AddHours(80)); + + var result = merger.Merge("CVE-REF-2025-01", ghsa, null, osv); + + var reference = Assert.Single(result.Advisory.References); + Assert.Equal("https://example.com/path/resource?a=1&b=2", reference.Url); + + var unionDecision = Assert.Single(result.Decisions.Where(decision => decision.Field == "references")); + Assert.Null(unionDecision.SelectedSource); + Assert.Equal("union", unionDecision.DecisionReason); + + var itemDecision = Assert.Single(result.Decisions.Where(decision => decision.Field.StartsWith("references[", StringComparison.OrdinalIgnoreCase))); + Assert.Equal("osv", itemDecision.SelectedSource); + Assert.Equal("freshness_override", itemDecision.DecisionReason); + Assert.Contains("https://example.com/path/resource?a=1&b=2", itemDecision.Field, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void Merge_DescriptionFreshnessOverride() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(12))); + + var ghsa = CreateAdvisory( + source: "ghsa", + advisoryKey: "GHSA-desc", + title: "GHSA Title", + summary: "Summary", + description: "Initial GHSA description", + modified: BaseTimestamp.AddHours(1)); + + var nvd = CreateAdvisory( + source: "nvd", + advisoryKey: "CVE-2025-5555", + title: "NVD Title", + summary: "Summary", + description: "NVD baseline description", + modified: BaseTimestamp.AddHours(2)); + + var osv = CreateAdvisory( + source: "osv", + advisoryKey: "OSV-2025-desc", + title: "OSV Title", + summary: "Summary", + description: "OSV fresher description", + modified: BaseTimestamp.AddHours(72)); + + var result = merger.Merge("CVE-2025-5555", ghsa, nvd, osv); + + Assert.Equal("OSV fresher description", result.Advisory.Description); + Assert.Contains(result.Decisions, decision => + decision.Field == "description" && + string.Equals(decision.SelectedSource, "osv", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "freshness_override", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Merge_CwesPreferNvdPrecedence() + { + var merger = new CanonicalMerger(new FixedTimeProvider(BaseTimestamp.AddHours(6))); + + var ghsaWeakness = CreateWeakness("ghsa", "CWE-79", "cross-site scripting", BaseTimestamp.AddHours(1)); + var nvdWeakness = CreateWeakness("nvd", "CWE-79", "Cross-Site Scripting", BaseTimestamp.AddHours(2)); + var osvWeakness = CreateWeakness("osv", "CWE-79", "XSS", BaseTimestamp.AddHours(3)); + + var ghsa = CreateAdvisory("ghsa", "GHSA-weakness", "GHSA Title", weaknesses: new[] { ghsaWeakness }, modified: BaseTimestamp.AddHours(1)); + var nvd = CreateAdvisory("nvd", "CVE-2025-7777", "NVD Title", weaknesses: new[] { nvdWeakness }, modified: BaseTimestamp.AddHours(2)); + var osv = CreateAdvisory("osv", "OSV-weakness", "OSV Title", weaknesses: new[] { osvWeakness }, modified: BaseTimestamp.AddHours(3)); + + var result = merger.Merge("CVE-2025-7777", ghsa, nvd, osv); + + var weakness = Assert.Single(result.Advisory.Cwes); + Assert.Equal("CWE-79", weakness.Identifier); + Assert.Equal("Cross-Site Scripting", weakness.Name); + Assert.Contains(result.Decisions, decision => + decision.Field == "cwes[cwe|CWE-79]" && + string.Equals(decision.SelectedSource, "nvd", StringComparison.OrdinalIgnoreCase) && + string.Equals(decision.DecisionReason, "precedence", StringComparison.OrdinalIgnoreCase)); + } + + private static Advisory CreateAdvisory( + string source, + string advisoryKey, + string title, + string? summary = null, + string? description = null, + DateTimeOffset? modified = null, + string? severity = null, + IEnumerable? packages = null, + IEnumerable? metrics = null, + IEnumerable? references = null, + IEnumerable? weaknesses = null, + string? canonicalMetricId = null) + { + var provenance = new AdvisoryProvenance( + source, + kind: "map", + value: advisoryKey, + recordedAt: modified ?? BaseTimestamp, + fieldMask: new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey, + title, + summary, + language: "en", + published: modified, + modified: modified, + severity: severity, + exploitKnown: false, + aliases: new[] { advisoryKey }, + credits: Array.Empty(), + references: references ?? Array.Empty(), + affectedPackages: packages ?? Array.Empty(), + cvssMetrics: metrics ?? Array.Empty(), + provenance: new[] { provenance }, + description: description, + cwes: weaknesses ?? Array.Empty(), + canonicalMetricId: canonicalMetricId); + } + + private static AdvisoryProvenance CreateProvenance(string source, string fieldMask) + => new( + source, + kind: "map", + value: source, + recordedAt: BaseTimestamp, + fieldMask: new[] { fieldMask }); + + private static AdvisoryWeakness CreateWeakness(string source, string identifier, string? name, DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + source, + kind: "map", + value: identifier, + recordedAt: recordedAt, + fieldMask: new[] { ProvenanceFieldMasks.Weaknesses }); + + return new AdvisoryWeakness("cwe", identifier, name, uri: null, provenance: new[] { provenance }); + } + + private sealed class FixedTimeProvider : TimeProvider + { + private readonly DateTimeOffset _utcNow; + + public FixedTimeProvider(DateTimeOffset utcNow) + { + _utcNow = utcNow; + } + + public override DateTimeOffset GetUtcNow() => _utcNow; + } +} diff --git a/src/StellaOps.Concelier.Core.Tests/Events/AdvisoryEventLogTests.cs b/src/StellaOps.Concelier.Core.Tests/Events/AdvisoryEventLogTests.cs new file mode 100644 index 00000000..2d737f8f --- /dev/null +++ b/src/StellaOps.Concelier.Core.Tests/Events/AdvisoryEventLogTests.cs @@ -0,0 +1,198 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Core.Tests.Events; + +public sealed class AdvisoryEventLogTests +{ + [Fact] + public async Task AppendAsync_PersistsCanonicalStatementEntries() + { + var repository = new FakeRepository(); + var timeProvider = new FixedTimeProvider(DateTimeOffset.UtcNow); + var log = new AdvisoryEventLog(repository, timeProvider); + + var advisory = new Advisory( + "adv-1", + "Test Advisory", + summary: "Summary", + language: "en", + published: DateTimeOffset.Parse("2025-10-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2025-10-02T00:00:00Z"), + severity: "high", + exploitKnown: true, + aliases: new[] { "CVE-2025-0001" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var statementInput = new AdvisoryStatementInput( + VulnerabilityKey: "CVE-2025-0001", + Advisory: advisory, + AsOf: DateTimeOffset.Parse("2025-10-03T00:00:00Z"), + InputDocumentIds: new[] { Guid.Parse("11111111-1111-1111-1111-111111111111") }); + + await log.AppendAsync(new AdvisoryEventAppendRequest(new[] { statementInput }), CancellationToken.None); + + Assert.Single(repository.InsertedStatements); + var entry = repository.InsertedStatements.Single(); + Assert.Equal("cve-2025-0001", entry.VulnerabilityKey); + Assert.Equal("adv-1", entry.AdvisoryKey); + Assert.Equal(DateTimeOffset.Parse("2025-10-03T00:00:00Z"), entry.AsOf); + Assert.Contains("\"advisoryKey\":\"adv-1\"", entry.CanonicalJson); + Assert.NotEqual(ImmutableArray.Empty, entry.StatementHash); + } + + [Fact] + public async Task AppendAsync_PersistsConflictsWithCanonicalizedJson() + { + var repository = new FakeRepository(); + var timeProvider = new FixedTimeProvider(DateTimeOffset.Parse("2025-10-19T12:00:00Z")); + var log = new AdvisoryEventLog(repository, timeProvider); + + using var conflictJson = JsonDocument.Parse("{\"reason\":\"tie\",\"details\":{\"b\":2,\"a\":1}}"); + var conflictInput = new AdvisoryConflictInput( + VulnerabilityKey: "CVE-2025-0001", + Details: conflictJson, + AsOf: DateTimeOffset.Parse("2025-10-04T00:00:00Z"), + StatementIds: new[] { Guid.Parse("22222222-2222-2222-2222-222222222222") }); + + await log.AppendAsync(new AdvisoryEventAppendRequest(Array.Empty(), new[] { conflictInput }), CancellationToken.None); + + Assert.Single(repository.InsertedConflicts); + var entry = repository.InsertedConflicts.Single(); + Assert.Equal("cve-2025-0001", entry.VulnerabilityKey); + Assert.Equal("{\"details\":{\"a\":1,\"b\":2},\"reason\":\"tie\"}", entry.CanonicalJson); + Assert.NotEqual(ImmutableArray.Empty, entry.ConflictHash); + Assert.Equal(DateTimeOffset.Parse("2025-10-04T00:00:00Z"), entry.AsOf); + } + + [Fact] + public async Task ReplayAsync_ReturnsSortedSnapshots() + { + var repository = new FakeRepository(); + var timeProvider = new FixedTimeProvider(DateTimeOffset.Parse("2025-10-05T00:00:00Z")); + var log = new AdvisoryEventLog(repository, timeProvider); + + repository.StoredStatements.AddRange(new[] + { + new AdvisoryStatementEntry( + Guid.Parse("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"), + "cve-2025-0001", + "adv-2", + CanonicalJsonSerializer.Serialize(new Advisory( + "adv-2", + "B title", + null, + null, + null, + DateTimeOffset.Parse("2025-10-02T00:00:00Z"), + null, + false, + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty())), + ImmutableArray.Create(new byte[] { 0x01, 0x02 }), + DateTimeOffset.Parse("2025-10-04T00:00:00Z"), + DateTimeOffset.Parse("2025-10-04T01:00:00Z"), + ImmutableArray.Empty), + new AdvisoryStatementEntry( + Guid.Parse("bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"), + "cve-2025-0001", + "adv-1", + CanonicalJsonSerializer.Serialize(new Advisory( + "adv-1", + "A title", + null, + null, + null, + DateTimeOffset.Parse("2025-10-01T00:00:00Z"), + null, + false, + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty())), + ImmutableArray.Create(new byte[] { 0x03, 0x04 }), + DateTimeOffset.Parse("2025-10-03T00:00:00Z"), + DateTimeOffset.Parse("2025-10-04T02:00:00Z"), + ImmutableArray.Empty), + }); + + repository.StoredConflicts.Add(new AdvisoryConflictEntry( + Guid.Parse("cccccccc-cccc-cccc-cccc-cccccccccccc"), + "cve-2025-0001", + CanonicalJson: "{\"reason\":\"conflict\"}", + ConflictHash: ImmutableArray.Create(new byte[] { 0x10 }), + AsOf: DateTimeOffset.Parse("2025-10-04T00:00:00Z"), + RecordedAt: DateTimeOffset.Parse("2025-10-04T03:00:00Z"), + StatementIds: ImmutableArray.Empty)); + + var replay = await log.ReplayAsync("CVE-2025-0001", asOf: null, CancellationToken.None); + + Assert.Equal("cve-2025-0001", replay.VulnerabilityKey); + Assert.Collection( + replay.Statements, + first => Assert.Equal("adv-2", first.AdvisoryKey), + second => Assert.Equal("adv-1", second.AdvisoryKey)); + Assert.Single(replay.Conflicts); + Assert.Equal("{\"reason\":\"conflict\"}", replay.Conflicts[0].CanonicalJson); + } + + private sealed class FakeRepository : IAdvisoryEventRepository + { + public List InsertedStatements { get; } = new(); + + public List InsertedConflicts { get; } = new(); + + public List StoredStatements { get; } = new(); + + public List StoredConflicts { get; } = new(); + + public ValueTask InsertStatementsAsync(IReadOnlyCollection statements, CancellationToken cancellationToken) + { + InsertedStatements.AddRange(statements); + return ValueTask.CompletedTask; + } + + public ValueTask InsertConflictsAsync(IReadOnlyCollection conflicts, CancellationToken cancellationToken) + { + InsertedConflicts.AddRange(conflicts); + return ValueTask.CompletedTask; + } + + public ValueTask> GetStatementsAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + => ValueTask.FromResult>(StoredStatements.Where(entry => + string.Equals(entry.VulnerabilityKey, vulnerabilityKey, StringComparison.Ordinal) && + (!asOf.HasValue || entry.AsOf <= asOf.Value)).ToList()); + + public ValueTask> GetConflictsAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + => ValueTask.FromResult>(StoredConflicts.Where(entry => + string.Equals(entry.VulnerabilityKey, vulnerabilityKey, StringComparison.Ordinal) && + (!asOf.HasValue || entry.AsOf <= asOf.Value)).ToList()); + } + + private sealed class FixedTimeProvider : TimeProvider + { + private readonly DateTimeOffset _now; + + public FixedTimeProvider(DateTimeOffset now) + { + _now = now.ToUniversalTime(); + } + + public override DateTimeOffset GetUtcNow() => _now; + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs b/src/StellaOps.Concelier.Core.Tests/JobCoordinatorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs rename to src/StellaOps.Concelier.Core.Tests/JobCoordinatorTests.cs index 5dcdcfbc..be5977e9 100644 --- a/src/StellaOps.Feedser.Core.Tests/JobCoordinatorTests.cs +++ b/src/StellaOps.Concelier.Core.Tests/JobCoordinatorTests.cs @@ -3,10 +3,11 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Core.Tests; +namespace StellaOps.Concelier.Core.Tests; public sealed class JobCoordinatorTests { @@ -311,10 +312,11 @@ public sealed class JobCoordinatorTests public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public List CreatedRuns { get; } = new(); - public Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) - { - var run = new JobRunSnapshot( - Guid.NewGuid(), + public Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + var run = new JobRunSnapshot( + Guid.NewGuid(), request.Kind, JobRunStatus.Pending, request.CreatedAt, @@ -331,9 +333,10 @@ public sealed class JobCoordinatorTests return Task.FromResult(run); } - public Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) - { - if (_runs.TryGetValue(runId, out var run)) + public Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + if (_runs.TryGetValue(runId, out var run)) { var updated = run with { Status = JobRunStatus.Running, StartedAt = startedAt }; _runs[runId] = updated; @@ -343,9 +346,10 @@ public sealed class JobCoordinatorTests return Task.FromResult(null); } - public Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) - { - if (_runs.TryGetValue(runId, out var run)) + public Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + if (_runs.TryGetValue(runId, out var run)) { var updated = run with { Status = completion.Status, CompletedAt = completion.CompletedAt, Error = completion.Error }; _runs[runId] = updated; @@ -356,15 +360,17 @@ public sealed class JobCoordinatorTests return Task.FromResult(null); } - public Task FindAsync(Guid runId, CancellationToken cancellationToken) - { - _runs.TryGetValue(runId, out var run); - return Task.FromResult(run); - } + public Task FindAsync(Guid runId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + _runs.TryGetValue(runId, out var run); + return Task.FromResult(run); + } - public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) - { - var query = _runs.Values.AsEnumerable(); + public Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + var query = _runs.Values.AsEnumerable(); if (!string.IsNullOrWhiteSpace(kind)) { query = query.Where(r => r.Kind == kind); @@ -373,23 +379,26 @@ public sealed class JobCoordinatorTests return Task.FromResult>(query.OrderByDescending(r => r.CreatedAt).Take(limit).ToArray()); } - public Task> GetActiveRunsAsync(CancellationToken cancellationToken) - { - return Task.FromResult>(_runs.Values.Where(r => r.Status is JobRunStatus.Pending or JobRunStatus.Running).ToArray()); - } + public Task> GetActiveRunsAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult>(_runs.Values.Where(r => r.Status is JobRunStatus.Pending or JobRunStatus.Running).ToArray()); + } - public Task GetLastRunAsync(string kind, CancellationToken cancellationToken) - { - var run = _runs.Values - .Where(r => r.Kind == kind) - .OrderByDescending(r => r.CreatedAt) + public Task GetLastRunAsync(string kind, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + var run = _runs.Values + .Where(r => r.Kind == kind) + .OrderByDescending(r => r.CreatedAt) .FirstOrDefault(); return Task.FromResult(run); } - public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) - { - var results = new Dictionary(StringComparer.Ordinal); + public Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + var results = new Dictionary(StringComparer.Ordinal); foreach (var kind in kinds.Distinct(StringComparer.Ordinal)) { var run = _runs.Values diff --git a/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs b/src/StellaOps.Concelier.Core.Tests/JobPluginRegistrationExtensionsTests.cs similarity index 90% rename from src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs rename to src/StellaOps.Concelier.Core.Tests/JobPluginRegistrationExtensionsTests.cs index 3946b8e4..d235e0da 100644 --- a/src/StellaOps.Feedser.Core.Tests/JobPluginRegistrationExtensionsTests.cs +++ b/src/StellaOps.Concelier.Core.Tests/JobPluginRegistrationExtensionsTests.cs @@ -4,10 +4,10 @@ using System.IO; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; using StellaOps.Plugin.Hosting; -namespace StellaOps.Feedser.Core.Tests; +namespace StellaOps.Concelier.Core.Tests; public sealed class JobPluginRegistrationExtensionsTests { @@ -53,7 +53,7 @@ public sealed class JobPluginRegistrationExtensionsTests Assert.True(schedulerOptions.Definitions.TryGetValue(PluginJob.JobKind, out var definition)); Assert.NotNull(definition); Assert.Equal(PluginJob.JobKind, definition.Kind); - Assert.Equal("StellaOps.Feedser.Core.Tests.PluginJob", definition.JobType.FullName); + Assert.Equal("StellaOps.Concelier.Core.Tests.PluginJob", definition.JobType.FullName); Assert.Equal(TimeSpan.FromSeconds(45), definition.Timeout); Assert.Equal(TimeSpan.FromSeconds(5), definition.LeaseDuration); Assert.Equal("*/10 * * * *", definition.CronExpression); diff --git a/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs b/src/StellaOps.Concelier.Core.Tests/JobSchedulerBuilderTests.cs similarity index 94% rename from src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs rename to src/StellaOps.Concelier.Core.Tests/JobSchedulerBuilderTests.cs index 034546cf..6279f518 100644 --- a/src/StellaOps.Feedser.Core.Tests/JobSchedulerBuilderTests.cs +++ b/src/StellaOps.Concelier.Core.Tests/JobSchedulerBuilderTests.cs @@ -1,9 +1,9 @@ using System; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Core.Tests; +namespace StellaOps.Concelier.Core.Tests; public sealed class JobSchedulerBuilderTests { diff --git a/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs b/src/StellaOps.Concelier.Core.Tests/PluginRoutineFixtures.cs similarity index 90% rename from src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs rename to src/StellaOps.Concelier.Core.Tests/PluginRoutineFixtures.cs index 3ba6defa..2bb54b60 100644 --- a/src/StellaOps.Feedser.Core.Tests/PluginRoutineFixtures.cs +++ b/src/StellaOps.Concelier.Core.Tests/PluginRoutineFixtures.cs @@ -4,9 +4,9 @@ using System.Threading.Tasks; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Core.Tests; +namespace StellaOps.Concelier.Core.Tests; public sealed class TestPluginRoutine : IDependencyInjectionRoutine { diff --git a/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj b/src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj similarity index 66% rename from src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj rename to src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj index 43dcfcd7..ecb30f28 100644 --- a/src/StellaOps.Feedser.Source.Common.Tests/StellaOps.Feedser.Source.Common.Tests.csproj +++ b/src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj @@ -5,6 +5,6 @@ enable - + diff --git a/src/StellaOps.Feedser.Core/AGENTS.md b/src/StellaOps.Concelier.Core/AGENTS.md similarity index 93% rename from src/StellaOps.Feedser.Core/AGENTS.md rename to src/StellaOps.Concelier.Core/AGENTS.md index fc658bf7..e6e0b4a4 100644 --- a/src/StellaOps.Feedser.Core/AGENTS.md +++ b/src/StellaOps.Concelier.Core/AGENTS.md @@ -26,7 +26,7 @@ Out: business logic of connectors/exporters, HTTP handlers (owned by WebService) - Logs: kind, trigger, params hash, lease holder, outcome; redact params containing secrets. - Honor CancellationToken early and often. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Core.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Core.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Core/CanonicalMergeResult.cs b/src/StellaOps.Concelier.Core/CanonicalMergeResult.cs similarity index 87% rename from src/StellaOps.Feedser.Core/CanonicalMergeResult.cs rename to src/StellaOps.Concelier.Core/CanonicalMergeResult.cs index ad26a3fd..29c5844c 100644 --- a/src/StellaOps.Feedser.Core/CanonicalMergeResult.cs +++ b/src/StellaOps.Concelier.Core/CanonicalMergeResult.cs @@ -1,19 +1,19 @@ -using System.Collections.Immutable; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Core; - -/// -/// Result emitted by describing the merged advisory and analytics about key decisions. -/// -public sealed record CanonicalMergeResult(Advisory Advisory, ImmutableArray Decisions); - -/// -/// Describes how a particular canonical field was chosen during conflict resolution. -/// -public sealed record FieldDecision( - string Field, - string? SelectedSource, - string DecisionReason, - DateTimeOffset? SelectedModified, - ImmutableArray ConsideredSources); +using System.Collections.Immutable; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core; + +/// +/// Result emitted by describing the merged advisory and analytics about key decisions. +/// +public sealed record CanonicalMergeResult(Advisory Advisory, ImmutableArray Decisions); + +/// +/// Describes how a particular canonical field was chosen during conflict resolution. +/// +public sealed record FieldDecision( + string Field, + string? SelectedSource, + string DecisionReason, + DateTimeOffset? SelectedModified, + ImmutableArray ConsideredSources); diff --git a/src/StellaOps.Feedser.Core/CanonicalMerger.cs b/src/StellaOps.Concelier.Core/CanonicalMerger.cs similarity index 97% rename from src/StellaOps.Feedser.Core/CanonicalMerger.cs rename to src/StellaOps.Concelier.Core/CanonicalMerger.cs index 995708b5..8259f24f 100644 --- a/src/StellaOps.Feedser.Core/CanonicalMerger.cs +++ b/src/StellaOps.Concelier.Core/CanonicalMerger.cs @@ -1,898 +1,898 @@ -using System.Collections.Immutable; -using System.Security.Cryptography; -using System.Text; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Core; - -/// -/// Resolves conflicts between GHSA, NVD, and OSV advisories into a single canonical advisory following -/// DEDUP_CONFLICTS_RESOLUTION_ALGO.md. -/// -public sealed class CanonicalMerger -{ - private const string GhsaSource = "ghsa"; - private const string NvdSource = "nvd"; - private const string OsvSource = "osv"; - - private static readonly ImmutableDictionary FieldPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["title"] = new[] { GhsaSource, NvdSource, OsvSource }, - ["summary"] = new[] { GhsaSource, NvdSource, OsvSource }, - ["description"] = new[] { GhsaSource, NvdSource, OsvSource }, - ["language"] = new[] { GhsaSource, NvdSource, OsvSource }, - ["severity"] = new[] { NvdSource, GhsaSource, OsvSource }, - ["references"] = new[] { GhsaSource, NvdSource, OsvSource }, - ["credits"] = new[] { GhsaSource, OsvSource, NvdSource }, - ["affectedPackages"] = new[] { OsvSource, GhsaSource, NvdSource }, - ["cvssMetrics"] = new[] { NvdSource, GhsaSource, OsvSource }, - ["cwes"] = new[] { NvdSource, GhsaSource, OsvSource }, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private static readonly ImmutableHashSet FreshnessSensitiveFields = ImmutableHashSet.Create( - StringComparer.OrdinalIgnoreCase, - "title", - "summary", - "description", - "references", - "credits", - "affectedPackages"); - - private static readonly ImmutableDictionary SourceOrder = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - [GhsaSource] = 0, - [NvdSource] = 1, - [OsvSource] = 2, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private readonly TimeProvider _timeProvider; - private readonly TimeSpan _freshnessThreshold; - - public CanonicalMerger(TimeProvider? timeProvider = null, TimeSpan? freshnessThreshold = null) - { - _timeProvider = timeProvider ?? TimeProvider.System; - _freshnessThreshold = freshnessThreshold ?? TimeSpan.FromHours(48); - } - - public CanonicalMergeResult Merge(string advisoryKey, Advisory? ghsa, Advisory? nvd, Advisory? osv) - { - ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); - - var candidates = BuildCandidates(ghsa, nvd, osv); - if (candidates.Count == 0) - { - throw new ArgumentException("At least one advisory must be provided.", nameof(advisoryKey)); - } - - var now = _timeProvider.GetUtcNow(); - var decisions = new List(); - var provenanceSet = new HashSet(); - - foreach (var candidate in candidates) - { - foreach (var existingProvenance in candidate.Advisory.Provenance) - { - provenanceSet.Add(existingProvenance); - } - } - - var titleSelection = SelectStringField("title", candidates, advisory => advisory.Title, isFreshnessSensitive: true); - if (titleSelection.HasValue) - { - decisions.Add(titleSelection.Decision); - AddMergeProvenance(provenanceSet, titleSelection, now, ProvenanceFieldMasks.Advisory); - } - - var summarySelection = SelectStringField("summary", candidates, advisory => advisory.Summary, isFreshnessSensitive: true); - if (summarySelection.HasValue) - { - decisions.Add(summarySelection.Decision); - AddMergeProvenance(provenanceSet, summarySelection, now, ProvenanceFieldMasks.Advisory); - } - - var descriptionSelection = SelectStringField("description", candidates, advisory => advisory.Description, isFreshnessSensitive: true); - if (descriptionSelection.HasValue) - { - decisions.Add(descriptionSelection.Decision); - AddMergeProvenance(provenanceSet, descriptionSelection, now, ProvenanceFieldMasks.Advisory); - } - - var languageSelection = SelectStringField("language", candidates, advisory => advisory.Language, isFreshnessSensitive: false); - if (languageSelection.HasValue) - { - decisions.Add(languageSelection.Decision); - AddMergeProvenance(provenanceSet, languageSelection, now, ProvenanceFieldMasks.Advisory); - } - - var topLevelSeveritySelection = SelectStringField("severity", candidates, advisory => advisory.Severity, isFreshnessSensitive: false); - if (topLevelSeveritySelection.HasValue) - { - decisions.Add(topLevelSeveritySelection.Decision); - AddMergeProvenance(provenanceSet, topLevelSeveritySelection, now, ProvenanceFieldMasks.Advisory); - } - - var aliases = MergeAliases(candidates); - var creditsResult = MergeCredits(candidates); - if (creditsResult.UnionDecision is not null) - { - decisions.Add(creditsResult.UnionDecision); - } - decisions.AddRange(creditsResult.Decisions); - - var referencesResult = MergeReferences(candidates); - if (referencesResult.UnionDecision is not null) - { - decisions.Add(referencesResult.UnionDecision); - } - decisions.AddRange(referencesResult.Decisions); - - var weaknessesResult = MergeWeaknesses(candidates, now); - decisions.AddRange(weaknessesResult.Decisions); - foreach (var weaknessProvenance in weaknessesResult.AdditionalProvenance) - { - provenanceSet.Add(weaknessProvenance); - } - - var packagesResult = MergePackages(candidates, now); - decisions.AddRange(packagesResult.Decisions); - foreach (var packageProvenance in packagesResult.AdditionalProvenance) - { - provenanceSet.Add(packageProvenance); - } - - var metricsResult = MergeCvssMetrics(candidates); - if (metricsResult.Decision is not null) - { - decisions.Add(metricsResult.Decision); - } - - var exploitKnown = candidates.Any(candidate => candidate.Advisory.ExploitKnown); - var published = candidates - .Select(candidate => candidate.Advisory.Published) - .Where(static value => value.HasValue) - .Select(static value => value!.Value) - .DefaultIfEmpty() - .Min(); - var modified = candidates - .Select(candidate => candidate.Advisory.Modified) - .Where(static value => value.HasValue) - .Select(static value => value!.Value) - .DefaultIfEmpty() - .Max(); - - var title = titleSelection.Value ?? ghsa?.Title ?? nvd?.Title ?? osv?.Title ?? advisoryKey; - var summary = summarySelection.Value ?? ghsa?.Summary ?? nvd?.Summary ?? osv?.Summary; - var description = descriptionSelection.Value ?? ghsa?.Description ?? nvd?.Description ?? osv?.Description; - var language = languageSelection.Value ?? ghsa?.Language ?? nvd?.Language ?? osv?.Language; - var severity = topLevelSeveritySelection.Value ?? metricsResult.CanonicalSeverity ?? ghsa?.Severity ?? nvd?.Severity ?? osv?.Severity; - var canonicalMetricId = metricsResult.CanonicalMetricId ?? ghsa?.CanonicalMetricId ?? nvd?.CanonicalMetricId ?? osv?.CanonicalMetricId; - - if (string.IsNullOrWhiteSpace(title)) - { - title = advisoryKey; - } - - var provenance = provenanceSet - .OrderBy(static p => p.Source, StringComparer.Ordinal) - .ThenBy(static p => p.Kind, StringComparer.Ordinal) - .ThenBy(static p => p.RecordedAt) - .ToImmutableArray(); - - var advisory = new Advisory( - advisoryKey, - title, - summary, - language, - published == DateTimeOffset.MinValue ? null : published, - modified == DateTimeOffset.MinValue ? null : modified, - severity, - exploitKnown, - aliases, - creditsResult.Credits, - referencesResult.References, - packagesResult.Packages, - metricsResult.Metrics, - provenance, - description, - weaknessesResult.Weaknesses, - canonicalMetricId); - - return new CanonicalMergeResult( - advisory, - decisions - .OrderBy(static d => d.Field, StringComparer.Ordinal) - .ThenBy(static d => d.SelectedSource, StringComparer.Ordinal) - .ToImmutableArray()); - } - - private ImmutableArray MergeAliases(List candidates) - { - var set = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var candidate in candidates) - { - foreach (var alias in candidate.Advisory.Aliases) - { - if (!string.IsNullOrWhiteSpace(alias)) - { - set.Add(alias); - } - } - } - - return set.Count == 0 ? ImmutableArray.Empty : set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); - } - - private CreditsMergeResult MergeCredits(List candidates) - { - var precedence = GetPrecedence("credits"); - var isFreshnessSensitive = FreshnessSensitiveFields.Contains("credits"); - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - var considered = new HashSet(StringComparer.OrdinalIgnoreCase); - var decisions = new List(); - - foreach (var candidate in candidates) - { - foreach (var credit in candidate.Advisory.Credits) - { - var key = $"{credit.DisplayName}|{credit.Role}"; - considered.Add(candidate.Source); - - if (!map.TryGetValue(key, out var existing)) - { - map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); - continue; - } - - var candidateRank = GetRank(candidate.Source, precedence); - var existingRank = GetRank(existing.Source, precedence); - var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); - if (reason is null) - { - continue; - } - - var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) - { - existing.Source, - candidate.Source, - }; - - map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); - decisions.Add(new FieldDecision( - Field: $"credits[{key}]", - SelectedSource: candidate.Source, - DecisionReason: reason, - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray())); - } - } - - var credits = map.Values.Select(static s => s.Credit).ToImmutableArray(); - FieldDecision? decision = null; - - if (considered.Count > 0) - { - decision = new FieldDecision( - Field: "credits", - SelectedSource: null, - DecisionReason: "union", - SelectedModified: null, - ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); - } - - return new CreditsMergeResult(credits, decision, decisions); - } - - private ReferencesMergeResult MergeReferences(List candidates) - { - var precedence = GetPrecedence("references"); - var isFreshnessSensitive = FreshnessSensitiveFields.Contains("references"); - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - var considered = new HashSet(StringComparer.OrdinalIgnoreCase); - var decisions = new List(); - - foreach (var candidate in candidates) - { - foreach (var reference in candidate.Advisory.References) - { - if (string.IsNullOrWhiteSpace(reference.Url)) - { - continue; - } - - var key = NormalizeReferenceKey(reference.Url); - considered.Add(candidate.Source); - - if (!map.TryGetValue(key, out var existing)) - { - map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); - continue; - } - - var candidateRank = GetRank(candidate.Source, precedence); - var existingRank = GetRank(existing.Source, precedence); - var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); - if (reason is null) - { - continue; - } - - var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) - { - existing.Source, - candidate.Source, - }; - - map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); - decisions.Add(new FieldDecision( - Field: $"references[{key}]", - SelectedSource: candidate.Source, - DecisionReason: reason, - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray())); - } - } - - var references = map.Values.Select(static s => s.Reference).ToImmutableArray(); - FieldDecision? decision = null; - - if (considered.Count > 0) - { - decision = new FieldDecision( - Field: "references", - SelectedSource: null, - DecisionReason: "union", - SelectedModified: null, - ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); - } - - return new ReferencesMergeResult(references, decision, decisions); - } - - private PackagesMergeResult MergePackages(List candidates, DateTimeOffset now) - { - var precedence = GetPrecedence("affectedPackages"); - var isFreshnessSensitive = FreshnessSensitiveFields.Contains("affectedPackages"); - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - var decisions = new List(); - var additionalProvenance = new List(); - - foreach (var candidate in candidates) - { - foreach (var package in candidate.Advisory.AffectedPackages) - { - var key = CreatePackageKey(package); - var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) { candidate.Source }; - - if (!map.TryGetValue(key, out var existing)) - { - var enriched = AppendMergeProvenance(package, candidate.Source, "precedence", now); - additionalProvenance.Add(enriched.MergeProvenance); - map[key] = new PackageSelection(enriched.Package, candidate.Source, candidate.Modified); - - decisions.Add(new FieldDecision( - Field: $"affectedPackages[{key}]", - SelectedSource: candidate.Source, - DecisionReason: "precedence", - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.ToImmutableArray())); - continue; - } - - consideredSources.Add(existing.Source); - - var candidateRank = GetRank(candidate.Source, precedence); - var existingRank = GetRank(existing.Source, precedence); - var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); - if (reason is null) - { - continue; - } - - var enrichedPackage = AppendMergeProvenance(package, candidate.Source, reason, now); - additionalProvenance.Add(enrichedPackage.MergeProvenance); - map[key] = new PackageSelection(enrichedPackage.Package, candidate.Source, candidate.Modified); - - decisions.Add(new FieldDecision( - Field: $"affectedPackages[{key}]", - SelectedSource: candidate.Source, - DecisionReason: reason, - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.ToImmutableArray())); - } - } - - var packages = map.Values.Select(static s => s.Package).ToImmutableArray(); - return new PackagesMergeResult(packages, decisions, additionalProvenance); - } - - private WeaknessMergeResult MergeWeaknesses(List candidates, DateTimeOffset now) - { - var precedence = GetPrecedence("cwes"); - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - var decisions = new List(); - var additionalProvenance = new List(); - - foreach (var candidate in candidates) - { - var candidateWeaknesses = candidate.Advisory.Cwes.IsDefaultOrEmpty - ? ImmutableArray.Empty - : candidate.Advisory.Cwes; - - foreach (var weakness in candidateWeaknesses) - { - var key = $"{weakness.Taxonomy}|{weakness.Identifier}"; - var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) { candidate.Source }; - - if (!map.TryGetValue(key, out var existing)) - { - var enriched = AppendWeaknessProvenance(weakness, candidate.Source, "precedence", now); - map[key] = new WeaknessSelection(enriched.Weakness, candidate.Source, candidate.Modified); - additionalProvenance.Add(enriched.MergeProvenance); - - decisions.Add(new FieldDecision( - Field: $"cwes[{key}]", - SelectedSource: candidate.Source, - DecisionReason: "precedence", - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.ToImmutableArray())); - continue; - } - - consideredSources.Add(existing.Source); - - var candidateRank = GetRank(candidate.Source, precedence); - var existingRank = GetRank(existing.Source, precedence); - var decisionReason = string.Empty; - var shouldReplace = false; - - if (candidateRank < existingRank) - { - shouldReplace = true; - decisionReason = "precedence"; - } - else if (candidateRank == existingRank && candidate.Modified > existing.Modified) - { - shouldReplace = true; - decisionReason = "tie_breaker"; - } - - if (!shouldReplace) - { - continue; - } - - var enrichedWeakness = AppendWeaknessProvenance(weakness, candidate.Source, decisionReason, now); - map[key] = new WeaknessSelection(enrichedWeakness.Weakness, candidate.Source, candidate.Modified); - additionalProvenance.Add(enrichedWeakness.MergeProvenance); - - decisions.Add(new FieldDecision( - Field: $"cwes[{key}]", - SelectedSource: candidate.Source, - DecisionReason: decisionReason, - SelectedModified: candidate.Modified, - ConsideredSources: consideredSources.ToImmutableArray())); - } - } - - var mergedWeaknesses = map.Values - .Select(static value => value.Weakness) - .OrderBy(static value => value.Taxonomy, StringComparer.Ordinal) - .ThenBy(static value => value.Identifier, StringComparer.Ordinal) - .ThenBy(static value => value.Name, StringComparer.Ordinal) - .ToImmutableArray(); - - return new WeaknessMergeResult(mergedWeaknesses, decisions, additionalProvenance); - } - - private CvssMergeResult MergeCvssMetrics(List candidates) - { - var precedence = GetPrecedence("cvssMetrics"); - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - var considered = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var candidate in candidates) - { - foreach (var metric in candidate.Advisory.CvssMetrics) - { - var key = $"{metric.Version}|{metric.Vector}"; - considered.Add(candidate.Source); - - if (!map.TryGetValue(key, out var existing)) - { - map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); - continue; - } - - var candidateRank = GetRank(candidate.Source, precedence); - var existingRank = GetRank(existing.Source, precedence); - - if (candidateRank < existingRank || - (candidateRank == existingRank && candidate.Modified > existing.Modified)) - { - map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); - } - } - } - - var orderedMetrics = map - .Values - .OrderBy(selection => GetRank(selection.Source, precedence)) - .ThenByDescending(selection => selection.Modified) - .Select(static selection => selection.Metric) - .ToImmutableArray(); - - FieldDecision? decision = null; - string? canonicalMetricId = null; - string? canonicalSelectedSource = null; - DateTimeOffset? canonicalSelectedModified = null; - - var canonical = orderedMetrics.FirstOrDefault(); - if (canonical is not null) - { - canonicalMetricId = $"{canonical.Version}|{canonical.Vector}"; - if (map.TryGetValue(canonicalMetricId, out var selection)) - { - canonicalSelectedSource = selection.Source; - canonicalSelectedModified = selection.Modified; - } - } - - if (considered.Count > 0) - { - decision = new FieldDecision( - Field: "cvssMetrics", - SelectedSource: canonicalSelectedSource, - DecisionReason: "precedence", - SelectedModified: canonicalSelectedModified, - ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); - } - - var severity = canonical?.BaseSeverity; - return new CvssMergeResult(orderedMetrics, severity, canonicalMetricId, decision); - } - - private static string CreatePackageKey(AffectedPackage package) - => string.Join('|', package.Type ?? string.Empty, package.Identifier ?? string.Empty, package.Platform ?? string.Empty); - - private static (AffectedPackage Package, AdvisoryProvenance MergeProvenance) AppendMergeProvenance( - AffectedPackage package, - string source, - string decisionReason, - DateTimeOffset recordedAt) - { - var provenance = new AdvisoryProvenance( - source, - kind: "merge", - value: CreatePackageKey(package), - recordedAt: recordedAt, - fieldMask: new[] { ProvenanceFieldMasks.AffectedPackages }, - decisionReason: decisionReason); - - var provenanceList = package.Provenance.ToBuilder(); - provenanceList.Add(provenance); - - var packageWithProvenance = new AffectedPackage( - package.Type, - package.Identifier, - package.Platform, - package.VersionRanges, - package.Statuses, - provenanceList, - package.NormalizedVersions); - - return (packageWithProvenance, provenance); - } - - private static string NormalizeReferenceKey(string url) - { - var trimmed = url?.Trim(); - if (string.IsNullOrEmpty(trimmed)) - { - return string.Empty; - } - - if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)) - { - return trimmed; - } - - var builder = new StringBuilder(); - var scheme = uri.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) ? "https" : uri.Scheme.ToLowerInvariant(); - builder.Append(scheme).Append("://").Append(uri.Host.ToLowerInvariant()); - - if (!uri.IsDefaultPort) - { - builder.Append(':').Append(uri.Port); - } - - var path = uri.AbsolutePath; - if (!string.IsNullOrEmpty(path) && path != "/") - { - if (!path.StartsWith('/')) - { - builder.Append('/'); - } - - builder.Append(path.TrimEnd('/')); - } - - var query = uri.Query; - if (!string.IsNullOrEmpty(query)) - { - var parameters = query.TrimStart('?') - .Split('&', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - Array.Sort(parameters, StringComparer.Ordinal); - builder.Append('?').Append(string.Join('&', parameters)); - } - - return builder.ToString(); - } - - private string? EvaluateReplacementReason(int candidateRank, int existingRank, DateTimeOffset candidateModified, DateTimeOffset existingModified, bool isFreshnessSensitive) - { - if (candidateRank < existingRank) - { - return "precedence"; - } - - if (isFreshnessSensitive && candidateRank > existingRank && candidateModified - existingModified >= _freshnessThreshold) - { - return "freshness_override"; - } - - if (candidateRank == existingRank && candidateModified > existingModified) - { - return "tie_breaker"; - } - - return null; - } - - private static (AdvisoryWeakness Weakness, AdvisoryProvenance MergeProvenance) AppendWeaknessProvenance( - AdvisoryWeakness weakness, - string source, - string decisionReason, - DateTimeOffset recordedAt) - { - var provenance = new AdvisoryProvenance( - source, - kind: "merge", - value: $"{weakness.Taxonomy}:{weakness.Identifier}", - recordedAt: recordedAt, - fieldMask: new[] { ProvenanceFieldMasks.Weaknesses }, - decisionReason: decisionReason); - - var provenanceList = weakness.Provenance.IsDefaultOrEmpty - ? ImmutableArray.Create(provenance) - : weakness.Provenance.Add(provenance); - - var weaknessWithProvenance = new AdvisoryWeakness( - weakness.Taxonomy, - weakness.Identifier, - weakness.Name, - weakness.Uri, - provenanceList); - - return (weaknessWithProvenance, provenance); - } - - private FieldSelection SelectStringField( - string field, - List candidates, - Func selector, - bool isFreshnessSensitive) - { - var precedence = GetPrecedence(field); - var valueCandidates = new List(); - - foreach (var candidate in candidates) - { - var value = Validation.TrimToNull(selector(candidate.Advisory)); - if (!string.IsNullOrEmpty(value)) - { - valueCandidates.Add(new ValueCandidate(candidate, value)); - } - } - - if (valueCandidates.Count == 0) - { - return FieldSelection.Empty; - } - - var consideredSources = valueCandidates - .Select(vc => vc.Candidate.Source) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static source => source, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - var best = valueCandidates - .OrderBy(vc => GetRank(vc.Candidate.Source, precedence)) - .ThenByDescending(vc => vc.Candidate.Modified) - .First(); - - var decisionReason = "precedence"; - - if (isFreshnessSensitive) - { - var freshnessOverride = valueCandidates - .Where(vc => GetRank(vc.Candidate.Source, precedence) > GetRank(best.Candidate.Source, precedence)) - .Where(vc => vc.Candidate.Modified - best.Candidate.Modified >= _freshnessThreshold) - .OrderByDescending(vc => vc.Candidate.Modified) - .ThenBy(vc => GetRank(vc.Candidate.Source, precedence)) - .FirstOrDefault(); - - if (freshnessOverride is not null) - { - best = freshnessOverride; - decisionReason = "freshness_override"; - } - } - - var sameRankCandidates = valueCandidates - .Where(vc => GetRank(vc.Candidate.Source, precedence) == GetRank(best.Candidate.Source, precedence)) - .ToList(); - - if (sameRankCandidates.Count > 1) - { - var tied = sameRankCandidates - .OrderBy(vc => vc.Value.Length) - .ThenBy(vc => vc.Value, StringComparer.Ordinal) - .ThenBy(vc => ComputeStableHash(vc.Value)) - .First(); - - if (!ReferenceEquals(tied, best)) - { - best = tied; - decisionReason = "tie_breaker"; - } - } - - var decision = new FieldDecision( - field, - best.Candidate.Source, - decisionReason, - best.Candidate.Modified, - consideredSources); - - return new FieldSelection(field, best.Value, best.Candidate, decisionReason, decision); - } - - private static void AddMergeProvenance( - HashSet provenanceSet, - FieldSelection selection, - DateTimeOffset recordedAt, - string fieldMask) - { - if (!selection.HasValue || selection.Winner is null) - { - return; - } - - var provenance = new AdvisoryProvenance( - selection.Winner.Source, - kind: "merge", - value: selection.Field, - recordedAt: recordedAt, - fieldMask: new[] { fieldMask }, - decisionReason: selection.DecisionReason); - - provenanceSet.Add(provenance); - } - - private static List BuildCandidates(Advisory? ghsa, Advisory? nvd, Advisory? osv) - { - var list = new List(capacity: 3); - if (ghsa is not null) - { - list.Add(CreateSnapshot(GhsaSource, ghsa)); - } - - if (nvd is not null) - { - list.Add(CreateSnapshot(NvdSource, nvd)); - } - - if (osv is not null) - { - list.Add(CreateSnapshot(OsvSource, osv)); - } - - return list; - } - - private static AdvisorySnapshot CreateSnapshot(string source, Advisory advisory) - { - var modified = advisory.Modified - ?? advisory.Published - ?? DateTimeOffset.UnixEpoch; - return new AdvisorySnapshot(source, advisory, modified); - } - - private static ImmutableDictionary GetPrecedence(string field) - { - if (FieldPrecedence.TryGetValue(field, out var order)) - { - return order - .Select((source, index) => (source, index)) - .ToImmutableDictionary(item => item.source, item => item.index, StringComparer.OrdinalIgnoreCase); - } - - return SourceOrder; - } - - private static int GetRank(string source, ImmutableDictionary precedence) - => precedence.TryGetValue(source, out var rank) ? rank : int.MaxValue; - - private static string ComputeStableHash(string value) - { - var bytes = Encoding.UTF8.GetBytes(value); - var hash = SHA256.HashData(bytes); - return Convert.ToHexString(hash); - } - - private sealed class FieldSelection - { - public FieldSelection(string field, T? value, AdvisorySnapshot? winner, string decisionReason, FieldDecision decision) - { - Field = field; - Value = value; - Winner = winner; - DecisionReason = decisionReason; - Decision = decision; - } - - public string Field { get; } - - public T? Value { get; } - - public AdvisorySnapshot? Winner { get; } - - public string DecisionReason { get; } - - public FieldDecision Decision { get; } - - public bool HasValue => Winner is not null; - - public static FieldSelection Empty { get; } = new FieldSelection( - string.Empty, - default, - null, - string.Empty, - new FieldDecision(string.Empty, null, string.Empty, null, ImmutableArray.Empty)); - } - - private sealed record AdvisorySnapshot(string Source, Advisory Advisory, DateTimeOffset Modified); - - private sealed record ValueCandidate(AdvisorySnapshot Candidate, string Value); - - private readonly record struct PackageSelection(AffectedPackage Package, string Source, DateTimeOffset Modified); - - private readonly record struct ReferenceSelection(AdvisoryReference Reference, string Source, DateTimeOffset Modified); - - private readonly record struct CreditSelection(AdvisoryCredit Credit, string Source, DateTimeOffset Modified); - - private readonly record struct MetricSelection(CvssMetric Metric, string Source, DateTimeOffset Modified); - - private readonly record struct WeaknessSelection(AdvisoryWeakness Weakness, string Source, DateTimeOffset Modified); - - private readonly record struct CreditsMergeResult(ImmutableArray Credits, FieldDecision? UnionDecision, IReadOnlyList Decisions); - - private readonly record struct ReferencesMergeResult(ImmutableArray References, FieldDecision? UnionDecision, IReadOnlyList Decisions); - - private readonly record struct PackagesMergeResult( - ImmutableArray Packages, - IReadOnlyList Decisions, - IReadOnlyList AdditionalProvenance); - - private readonly record struct WeaknessMergeResult( - ImmutableArray Weaknesses, - IReadOnlyList Decisions, - IReadOnlyList AdditionalProvenance); - - private readonly record struct CvssMergeResult( - ImmutableArray Metrics, - string? CanonicalSeverity, - string? CanonicalMetricId, - FieldDecision? Decision); -} +using System.Collections.Immutable; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core; + +/// +/// Resolves conflicts between GHSA, NVD, and OSV advisories into a single canonical advisory following +/// DEDUP_CONFLICTS_RESOLUTION_ALGO.md. +/// +public sealed class CanonicalMerger +{ + private const string GhsaSource = "ghsa"; + private const string NvdSource = "nvd"; + private const string OsvSource = "osv"; + + private static readonly ImmutableDictionary FieldPrecedence = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["title"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["summary"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["description"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["language"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["severity"] = new[] { NvdSource, GhsaSource, OsvSource }, + ["references"] = new[] { GhsaSource, NvdSource, OsvSource }, + ["credits"] = new[] { GhsaSource, OsvSource, NvdSource }, + ["affectedPackages"] = new[] { OsvSource, GhsaSource, NvdSource }, + ["cvssMetrics"] = new[] { NvdSource, GhsaSource, OsvSource }, + ["cwes"] = new[] { NvdSource, GhsaSource, OsvSource }, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableHashSet FreshnessSensitiveFields = ImmutableHashSet.Create( + StringComparer.OrdinalIgnoreCase, + "title", + "summary", + "description", + "references", + "credits", + "affectedPackages"); + + private static readonly ImmutableDictionary SourceOrder = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [GhsaSource] = 0, + [NvdSource] = 1, + [OsvSource] = 2, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly TimeProvider _timeProvider; + private readonly TimeSpan _freshnessThreshold; + + public CanonicalMerger(TimeProvider? timeProvider = null, TimeSpan? freshnessThreshold = null) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _freshnessThreshold = freshnessThreshold ?? TimeSpan.FromHours(48); + } + + public CanonicalMergeResult Merge(string advisoryKey, Advisory? ghsa, Advisory? nvd, Advisory? osv) + { + ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey); + + var candidates = BuildCandidates(ghsa, nvd, osv); + if (candidates.Count == 0) + { + throw new ArgumentException("At least one advisory must be provided.", nameof(advisoryKey)); + } + + var now = _timeProvider.GetUtcNow(); + var decisions = new List(); + var provenanceSet = new HashSet(); + + foreach (var candidate in candidates) + { + foreach (var existingProvenance in candidate.Advisory.Provenance) + { + provenanceSet.Add(existingProvenance); + } + } + + var titleSelection = SelectStringField("title", candidates, advisory => advisory.Title, isFreshnessSensitive: true); + if (titleSelection.HasValue) + { + decisions.Add(titleSelection.Decision); + AddMergeProvenance(provenanceSet, titleSelection, now, ProvenanceFieldMasks.Advisory); + } + + var summarySelection = SelectStringField("summary", candidates, advisory => advisory.Summary, isFreshnessSensitive: true); + if (summarySelection.HasValue) + { + decisions.Add(summarySelection.Decision); + AddMergeProvenance(provenanceSet, summarySelection, now, ProvenanceFieldMasks.Advisory); + } + + var descriptionSelection = SelectStringField("description", candidates, advisory => advisory.Description, isFreshnessSensitive: true); + if (descriptionSelection.HasValue) + { + decisions.Add(descriptionSelection.Decision); + AddMergeProvenance(provenanceSet, descriptionSelection, now, ProvenanceFieldMasks.Advisory); + } + + var languageSelection = SelectStringField("language", candidates, advisory => advisory.Language, isFreshnessSensitive: false); + if (languageSelection.HasValue) + { + decisions.Add(languageSelection.Decision); + AddMergeProvenance(provenanceSet, languageSelection, now, ProvenanceFieldMasks.Advisory); + } + + var topLevelSeveritySelection = SelectStringField("severity", candidates, advisory => advisory.Severity, isFreshnessSensitive: false); + if (topLevelSeveritySelection.HasValue) + { + decisions.Add(topLevelSeveritySelection.Decision); + AddMergeProvenance(provenanceSet, topLevelSeveritySelection, now, ProvenanceFieldMasks.Advisory); + } + + var aliases = MergeAliases(candidates); + var creditsResult = MergeCredits(candidates); + if (creditsResult.UnionDecision is not null) + { + decisions.Add(creditsResult.UnionDecision); + } + decisions.AddRange(creditsResult.Decisions); + + var referencesResult = MergeReferences(candidates); + if (referencesResult.UnionDecision is not null) + { + decisions.Add(referencesResult.UnionDecision); + } + decisions.AddRange(referencesResult.Decisions); + + var weaknessesResult = MergeWeaknesses(candidates, now); + decisions.AddRange(weaknessesResult.Decisions); + foreach (var weaknessProvenance in weaknessesResult.AdditionalProvenance) + { + provenanceSet.Add(weaknessProvenance); + } + + var packagesResult = MergePackages(candidates, now); + decisions.AddRange(packagesResult.Decisions); + foreach (var packageProvenance in packagesResult.AdditionalProvenance) + { + provenanceSet.Add(packageProvenance); + } + + var metricsResult = MergeCvssMetrics(candidates); + if (metricsResult.Decision is not null) + { + decisions.Add(metricsResult.Decision); + } + + var exploitKnown = candidates.Any(candidate => candidate.Advisory.ExploitKnown); + var published = candidates + .Select(candidate => candidate.Advisory.Published) + .Where(static value => value.HasValue) + .Select(static value => value!.Value) + .DefaultIfEmpty() + .Min(); + var modified = candidates + .Select(candidate => candidate.Advisory.Modified) + .Where(static value => value.HasValue) + .Select(static value => value!.Value) + .DefaultIfEmpty() + .Max(); + + var title = titleSelection.Value ?? ghsa?.Title ?? nvd?.Title ?? osv?.Title ?? advisoryKey; + var summary = summarySelection.Value ?? ghsa?.Summary ?? nvd?.Summary ?? osv?.Summary; + var description = descriptionSelection.Value ?? ghsa?.Description ?? nvd?.Description ?? osv?.Description; + var language = languageSelection.Value ?? ghsa?.Language ?? nvd?.Language ?? osv?.Language; + var severity = topLevelSeveritySelection.Value ?? metricsResult.CanonicalSeverity ?? ghsa?.Severity ?? nvd?.Severity ?? osv?.Severity; + var canonicalMetricId = metricsResult.CanonicalMetricId ?? ghsa?.CanonicalMetricId ?? nvd?.CanonicalMetricId ?? osv?.CanonicalMetricId; + + if (string.IsNullOrWhiteSpace(title)) + { + title = advisoryKey; + } + + var provenance = provenanceSet + .OrderBy(static p => p.Source, StringComparer.Ordinal) + .ThenBy(static p => p.Kind, StringComparer.Ordinal) + .ThenBy(static p => p.RecordedAt) + .ToImmutableArray(); + + var advisory = new Advisory( + advisoryKey, + title, + summary, + language, + published == DateTimeOffset.MinValue ? null : published, + modified == DateTimeOffset.MinValue ? null : modified, + severity, + exploitKnown, + aliases, + creditsResult.Credits, + referencesResult.References, + packagesResult.Packages, + metricsResult.Metrics, + provenance, + description, + weaknessesResult.Weaknesses, + canonicalMetricId); + + return new CanonicalMergeResult( + advisory, + decisions + .OrderBy(static d => d.Field, StringComparer.Ordinal) + .ThenBy(static d => d.SelectedSource, StringComparer.Ordinal) + .ToImmutableArray()); + } + + private ImmutableArray MergeAliases(List candidates) + { + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var candidate in candidates) + { + foreach (var alias in candidate.Advisory.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + set.Add(alias); + } + } + } + + return set.Count == 0 ? ImmutableArray.Empty : set.OrderBy(static value => value, StringComparer.Ordinal).ToImmutableArray(); + } + + private CreditsMergeResult MergeCredits(List candidates) + { + var precedence = GetPrecedence("credits"); + var isFreshnessSensitive = FreshnessSensitiveFields.Contains("credits"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + var decisions = new List(); + + foreach (var candidate in candidates) + { + foreach (var credit in candidate.Advisory.Credits) + { + var key = $"{credit.DisplayName}|{credit.Role}"; + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); + if (reason is null) + { + continue; + } + + var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) + { + existing.Source, + candidate.Source, + }; + + map[key] = new CreditSelection(credit, candidate.Source, candidate.Modified); + decisions.Add(new FieldDecision( + Field: $"credits[{key}]", + SelectedSource: candidate.Source, + DecisionReason: reason, + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray())); + } + } + + var credits = map.Values.Select(static s => s.Credit).ToImmutableArray(); + FieldDecision? decision = null; + + if (considered.Count > 0) + { + decision = new FieldDecision( + Field: "credits", + SelectedSource: null, + DecisionReason: "union", + SelectedModified: null, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + return new CreditsMergeResult(credits, decision, decisions); + } + + private ReferencesMergeResult MergeReferences(List candidates) + { + var precedence = GetPrecedence("references"); + var isFreshnessSensitive = FreshnessSensitiveFields.Contains("references"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + var decisions = new List(); + + foreach (var candidate in candidates) + { + foreach (var reference in candidate.Advisory.References) + { + if (string.IsNullOrWhiteSpace(reference.Url)) + { + continue; + } + + var key = NormalizeReferenceKey(reference.Url); + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); + if (reason is null) + { + continue; + } + + var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) + { + existing.Source, + candidate.Source, + }; + + map[key] = new ReferenceSelection(reference, candidate.Source, candidate.Modified); + decisions.Add(new FieldDecision( + Field: $"references[{key}]", + SelectedSource: candidate.Source, + DecisionReason: reason, + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray())); + } + } + + var references = map.Values.Select(static s => s.Reference).ToImmutableArray(); + FieldDecision? decision = null; + + if (considered.Count > 0) + { + decision = new FieldDecision( + Field: "references", + SelectedSource: null, + DecisionReason: "union", + SelectedModified: null, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + return new ReferencesMergeResult(references, decision, decisions); + } + + private PackagesMergeResult MergePackages(List candidates, DateTimeOffset now) + { + var precedence = GetPrecedence("affectedPackages"); + var isFreshnessSensitive = FreshnessSensitiveFields.Contains("affectedPackages"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var decisions = new List(); + var additionalProvenance = new List(); + + foreach (var candidate in candidates) + { + foreach (var package in candidate.Advisory.AffectedPackages) + { + var key = CreatePackageKey(package); + var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) { candidate.Source }; + + if (!map.TryGetValue(key, out var existing)) + { + var enriched = AppendMergeProvenance(package, candidate.Source, "precedence", now); + additionalProvenance.Add(enriched.MergeProvenance); + map[key] = new PackageSelection(enriched.Package, candidate.Source, candidate.Modified); + + decisions.Add(new FieldDecision( + Field: $"affectedPackages[{key}]", + SelectedSource: candidate.Source, + DecisionReason: "precedence", + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + continue; + } + + consideredSources.Add(existing.Source); + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + var reason = EvaluateReplacementReason(candidateRank, existingRank, candidate.Modified, existing.Modified, isFreshnessSensitive); + if (reason is null) + { + continue; + } + + var enrichedPackage = AppendMergeProvenance(package, candidate.Source, reason, now); + additionalProvenance.Add(enrichedPackage.MergeProvenance); + map[key] = new PackageSelection(enrichedPackage.Package, candidate.Source, candidate.Modified); + + decisions.Add(new FieldDecision( + Field: $"affectedPackages[{key}]", + SelectedSource: candidate.Source, + DecisionReason: reason, + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + } + } + + var packages = map.Values.Select(static s => s.Package).ToImmutableArray(); + return new PackagesMergeResult(packages, decisions, additionalProvenance); + } + + private WeaknessMergeResult MergeWeaknesses(List candidates, DateTimeOffset now) + { + var precedence = GetPrecedence("cwes"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var decisions = new List(); + var additionalProvenance = new List(); + + foreach (var candidate in candidates) + { + var candidateWeaknesses = candidate.Advisory.Cwes.IsDefaultOrEmpty + ? ImmutableArray.Empty + : candidate.Advisory.Cwes; + + foreach (var weakness in candidateWeaknesses) + { + var key = $"{weakness.Taxonomy}|{weakness.Identifier}"; + var consideredSources = new HashSet(StringComparer.OrdinalIgnoreCase) { candidate.Source }; + + if (!map.TryGetValue(key, out var existing)) + { + var enriched = AppendWeaknessProvenance(weakness, candidate.Source, "precedence", now); + map[key] = new WeaknessSelection(enriched.Weakness, candidate.Source, candidate.Modified); + additionalProvenance.Add(enriched.MergeProvenance); + + decisions.Add(new FieldDecision( + Field: $"cwes[{key}]", + SelectedSource: candidate.Source, + DecisionReason: "precedence", + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + continue; + } + + consideredSources.Add(existing.Source); + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + var decisionReason = string.Empty; + var shouldReplace = false; + + if (candidateRank < existingRank) + { + shouldReplace = true; + decisionReason = "precedence"; + } + else if (candidateRank == existingRank && candidate.Modified > existing.Modified) + { + shouldReplace = true; + decisionReason = "tie_breaker"; + } + + if (!shouldReplace) + { + continue; + } + + var enrichedWeakness = AppendWeaknessProvenance(weakness, candidate.Source, decisionReason, now); + map[key] = new WeaknessSelection(enrichedWeakness.Weakness, candidate.Source, candidate.Modified); + additionalProvenance.Add(enrichedWeakness.MergeProvenance); + + decisions.Add(new FieldDecision( + Field: $"cwes[{key}]", + SelectedSource: candidate.Source, + DecisionReason: decisionReason, + SelectedModified: candidate.Modified, + ConsideredSources: consideredSources.ToImmutableArray())); + } + } + + var mergedWeaknesses = map.Values + .Select(static value => value.Weakness) + .OrderBy(static value => value.Taxonomy, StringComparer.Ordinal) + .ThenBy(static value => value.Identifier, StringComparer.Ordinal) + .ThenBy(static value => value.Name, StringComparer.Ordinal) + .ToImmutableArray(); + + return new WeaknessMergeResult(mergedWeaknesses, decisions, additionalProvenance); + } + + private CvssMergeResult MergeCvssMetrics(List candidates) + { + var precedence = GetPrecedence("cvssMetrics"); + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + var considered = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in candidates) + { + foreach (var metric in candidate.Advisory.CvssMetrics) + { + var key = $"{metric.Version}|{metric.Vector}"; + considered.Add(candidate.Source); + + if (!map.TryGetValue(key, out var existing)) + { + map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); + continue; + } + + var candidateRank = GetRank(candidate.Source, precedence); + var existingRank = GetRank(existing.Source, precedence); + + if (candidateRank < existingRank || + (candidateRank == existingRank && candidate.Modified > existing.Modified)) + { + map[key] = new MetricSelection(metric, candidate.Source, candidate.Modified); + } + } + } + + var orderedMetrics = map + .Values + .OrderBy(selection => GetRank(selection.Source, precedence)) + .ThenByDescending(selection => selection.Modified) + .Select(static selection => selection.Metric) + .ToImmutableArray(); + + FieldDecision? decision = null; + string? canonicalMetricId = null; + string? canonicalSelectedSource = null; + DateTimeOffset? canonicalSelectedModified = null; + + var canonical = orderedMetrics.FirstOrDefault(); + if (canonical is not null) + { + canonicalMetricId = $"{canonical.Version}|{canonical.Vector}"; + if (map.TryGetValue(canonicalMetricId, out var selection)) + { + canonicalSelectedSource = selection.Source; + canonicalSelectedModified = selection.Modified; + } + } + + if (considered.Count > 0) + { + decision = new FieldDecision( + Field: "cvssMetrics", + SelectedSource: canonicalSelectedSource, + DecisionReason: "precedence", + SelectedModified: canonicalSelectedModified, + ConsideredSources: considered.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + var severity = canonical?.BaseSeverity; + return new CvssMergeResult(orderedMetrics, severity, canonicalMetricId, decision); + } + + private static string CreatePackageKey(AffectedPackage package) + => string.Join('|', package.Type ?? string.Empty, package.Identifier ?? string.Empty, package.Platform ?? string.Empty); + + private static (AffectedPackage Package, AdvisoryProvenance MergeProvenance) AppendMergeProvenance( + AffectedPackage package, + string source, + string decisionReason, + DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + source, + kind: "merge", + value: CreatePackageKey(package), + recordedAt: recordedAt, + fieldMask: new[] { ProvenanceFieldMasks.AffectedPackages }, + decisionReason: decisionReason); + + var provenanceList = package.Provenance.ToBuilder(); + provenanceList.Add(provenance); + + var packageWithProvenance = new AffectedPackage( + package.Type, + package.Identifier, + package.Platform, + package.VersionRanges, + package.Statuses, + provenanceList, + package.NormalizedVersions); + + return (packageWithProvenance, provenance); + } + + private static string NormalizeReferenceKey(string url) + { + var trimmed = url?.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + return string.Empty; + } + + if (!Uri.TryCreate(trimmed, UriKind.Absolute, out var uri)) + { + return trimmed; + } + + var builder = new StringBuilder(); + var scheme = uri.Scheme.Equals("http", StringComparison.OrdinalIgnoreCase) ? "https" : uri.Scheme.ToLowerInvariant(); + builder.Append(scheme).Append("://").Append(uri.Host.ToLowerInvariant()); + + if (!uri.IsDefaultPort) + { + builder.Append(':').Append(uri.Port); + } + + var path = uri.AbsolutePath; + if (!string.IsNullOrEmpty(path) && path != "/") + { + if (!path.StartsWith('/')) + { + builder.Append('/'); + } + + builder.Append(path.TrimEnd('/')); + } + + var query = uri.Query; + if (!string.IsNullOrEmpty(query)) + { + var parameters = query.TrimStart('?') + .Split('&', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + Array.Sort(parameters, StringComparer.Ordinal); + builder.Append('?').Append(string.Join('&', parameters)); + } + + return builder.ToString(); + } + + private string? EvaluateReplacementReason(int candidateRank, int existingRank, DateTimeOffset candidateModified, DateTimeOffset existingModified, bool isFreshnessSensitive) + { + if (candidateRank < existingRank) + { + return "precedence"; + } + + if (isFreshnessSensitive && candidateRank > existingRank && candidateModified - existingModified >= _freshnessThreshold) + { + return "freshness_override"; + } + + if (candidateRank == existingRank && candidateModified > existingModified) + { + return "tie_breaker"; + } + + return null; + } + + private static (AdvisoryWeakness Weakness, AdvisoryProvenance MergeProvenance) AppendWeaknessProvenance( + AdvisoryWeakness weakness, + string source, + string decisionReason, + DateTimeOffset recordedAt) + { + var provenance = new AdvisoryProvenance( + source, + kind: "merge", + value: $"{weakness.Taxonomy}:{weakness.Identifier}", + recordedAt: recordedAt, + fieldMask: new[] { ProvenanceFieldMasks.Weaknesses }, + decisionReason: decisionReason); + + var provenanceList = weakness.Provenance.IsDefaultOrEmpty + ? ImmutableArray.Create(provenance) + : weakness.Provenance.Add(provenance); + + var weaknessWithProvenance = new AdvisoryWeakness( + weakness.Taxonomy, + weakness.Identifier, + weakness.Name, + weakness.Uri, + provenanceList); + + return (weaknessWithProvenance, provenance); + } + + private FieldSelection SelectStringField( + string field, + List candidates, + Func selector, + bool isFreshnessSensitive) + { + var precedence = GetPrecedence(field); + var valueCandidates = new List(); + + foreach (var candidate in candidates) + { + var value = Validation.TrimToNull(selector(candidate.Advisory)); + if (!string.IsNullOrEmpty(value)) + { + valueCandidates.Add(new ValueCandidate(candidate, value)); + } + } + + if (valueCandidates.Count == 0) + { + return FieldSelection.Empty; + } + + var consideredSources = valueCandidates + .Select(vc => vc.Candidate.Source) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static source => source, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + var best = valueCandidates + .OrderBy(vc => GetRank(vc.Candidate.Source, precedence)) + .ThenByDescending(vc => vc.Candidate.Modified) + .First(); + + var decisionReason = "precedence"; + + if (isFreshnessSensitive) + { + var freshnessOverride = valueCandidates + .Where(vc => GetRank(vc.Candidate.Source, precedence) > GetRank(best.Candidate.Source, precedence)) + .Where(vc => vc.Candidate.Modified - best.Candidate.Modified >= _freshnessThreshold) + .OrderByDescending(vc => vc.Candidate.Modified) + .ThenBy(vc => GetRank(vc.Candidate.Source, precedence)) + .FirstOrDefault(); + + if (freshnessOverride is not null) + { + best = freshnessOverride; + decisionReason = "freshness_override"; + } + } + + var sameRankCandidates = valueCandidates + .Where(vc => GetRank(vc.Candidate.Source, precedence) == GetRank(best.Candidate.Source, precedence)) + .ToList(); + + if (sameRankCandidates.Count > 1) + { + var tied = sameRankCandidates + .OrderBy(vc => vc.Value.Length) + .ThenBy(vc => vc.Value, StringComparer.Ordinal) + .ThenBy(vc => ComputeStableHash(vc.Value)) + .First(); + + if (!ReferenceEquals(tied, best)) + { + best = tied; + decisionReason = "tie_breaker"; + } + } + + var decision = new FieldDecision( + field, + best.Candidate.Source, + decisionReason, + best.Candidate.Modified, + consideredSources); + + return new FieldSelection(field, best.Value, best.Candidate, decisionReason, decision); + } + + private static void AddMergeProvenance( + HashSet provenanceSet, + FieldSelection selection, + DateTimeOffset recordedAt, + string fieldMask) + { + if (!selection.HasValue || selection.Winner is null) + { + return; + } + + var provenance = new AdvisoryProvenance( + selection.Winner.Source, + kind: "merge", + value: selection.Field, + recordedAt: recordedAt, + fieldMask: new[] { fieldMask }, + decisionReason: selection.DecisionReason); + + provenanceSet.Add(provenance); + } + + private static List BuildCandidates(Advisory? ghsa, Advisory? nvd, Advisory? osv) + { + var list = new List(capacity: 3); + if (ghsa is not null) + { + list.Add(CreateSnapshot(GhsaSource, ghsa)); + } + + if (nvd is not null) + { + list.Add(CreateSnapshot(NvdSource, nvd)); + } + + if (osv is not null) + { + list.Add(CreateSnapshot(OsvSource, osv)); + } + + return list; + } + + private static AdvisorySnapshot CreateSnapshot(string source, Advisory advisory) + { + var modified = advisory.Modified + ?? advisory.Published + ?? DateTimeOffset.UnixEpoch; + return new AdvisorySnapshot(source, advisory, modified); + } + + private static ImmutableDictionary GetPrecedence(string field) + { + if (FieldPrecedence.TryGetValue(field, out var order)) + { + return order + .Select((source, index) => (source, index)) + .ToImmutableDictionary(item => item.source, item => item.index, StringComparer.OrdinalIgnoreCase); + } + + return SourceOrder; + } + + private static int GetRank(string source, ImmutableDictionary precedence) + => precedence.TryGetValue(source, out var rank) ? rank : int.MaxValue; + + private static string ComputeStableHash(string value) + { + var bytes = Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash); + } + + private sealed class FieldSelection + { + public FieldSelection(string field, T? value, AdvisorySnapshot? winner, string decisionReason, FieldDecision decision) + { + Field = field; + Value = value; + Winner = winner; + DecisionReason = decisionReason; + Decision = decision; + } + + public string Field { get; } + + public T? Value { get; } + + public AdvisorySnapshot? Winner { get; } + + public string DecisionReason { get; } + + public FieldDecision Decision { get; } + + public bool HasValue => Winner is not null; + + public static FieldSelection Empty { get; } = new FieldSelection( + string.Empty, + default, + null, + string.Empty, + new FieldDecision(string.Empty, null, string.Empty, null, ImmutableArray.Empty)); + } + + private sealed record AdvisorySnapshot(string Source, Advisory Advisory, DateTimeOffset Modified); + + private sealed record ValueCandidate(AdvisorySnapshot Candidate, string Value); + + private readonly record struct PackageSelection(AffectedPackage Package, string Source, DateTimeOffset Modified); + + private readonly record struct ReferenceSelection(AdvisoryReference Reference, string Source, DateTimeOffset Modified); + + private readonly record struct CreditSelection(AdvisoryCredit Credit, string Source, DateTimeOffset Modified); + + private readonly record struct MetricSelection(CvssMetric Metric, string Source, DateTimeOffset Modified); + + private readonly record struct WeaknessSelection(AdvisoryWeakness Weakness, string Source, DateTimeOffset Modified); + + private readonly record struct CreditsMergeResult(ImmutableArray Credits, FieldDecision? UnionDecision, IReadOnlyList Decisions); + + private readonly record struct ReferencesMergeResult(ImmutableArray References, FieldDecision? UnionDecision, IReadOnlyList Decisions); + + private readonly record struct PackagesMergeResult( + ImmutableArray Packages, + IReadOnlyList Decisions, + IReadOnlyList AdditionalProvenance); + + private readonly record struct WeaknessMergeResult( + ImmutableArray Weaknesses, + IReadOnlyList Decisions, + IReadOnlyList AdditionalProvenance); + + private readonly record struct CvssMergeResult( + ImmutableArray Metrics, + string? CanonicalSeverity, + string? CanonicalMetricId, + FieldDecision? Decision); +} diff --git a/src/StellaOps.Concelier.Core/Events/AdvisoryEventContracts.cs b/src/StellaOps.Concelier.Core/Events/AdvisoryEventContracts.cs new file mode 100644 index 00000000..9f4d21ab --- /dev/null +++ b/src/StellaOps.Concelier.Core/Events/AdvisoryEventContracts.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Immutable; +using System.Text.Json; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core.Events; + +/// +/// Input payload for appending a canonical advisory statement to the event log. +/// +public sealed record AdvisoryStatementInput( + string VulnerabilityKey, + Advisory Advisory, + DateTimeOffset AsOf, + IReadOnlyCollection InputDocumentIds, + Guid? StatementId = null, + string? AdvisoryKey = null); + +/// +/// Input payload for appending an advisory conflict entry aligned with an advisory statement snapshot. +/// +public sealed record AdvisoryConflictInput( + string VulnerabilityKey, + JsonDocument Details, + DateTimeOffset AsOf, + IReadOnlyCollection StatementIds, + Guid? ConflictId = null); + +/// +/// Append request encapsulating statement and conflict batches sharing a single persistence window. +/// +public sealed record AdvisoryEventAppendRequest( + IReadOnlyCollection Statements, + IReadOnlyCollection? Conflicts = null); + +/// +/// Replay response describing immutable statement snapshots for a vulnerability key. +/// +public sealed record AdvisoryReplay( + string VulnerabilityKey, + DateTimeOffset? AsOf, + ImmutableArray Statements, + ImmutableArray Conflicts); + +/// +/// Immutable advisory statement snapshot captured at a specific asOf time. +/// +public sealed record AdvisoryStatementSnapshot( + Guid StatementId, + string VulnerabilityKey, + string AdvisoryKey, + Advisory Advisory, + ImmutableArray StatementHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + ImmutableArray InputDocumentIds); + +/// +/// Immutable advisory conflict snapshot describing divergence explanations for a vulnerability key. +/// +public sealed record AdvisoryConflictSnapshot( + Guid ConflictId, + string VulnerabilityKey, + ImmutableArray StatementIds, + ImmutableArray ConflictHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + string CanonicalJson); + +/// +/// Persistence-facing representation of an advisory statement used by repositories. +/// +public sealed record AdvisoryStatementEntry( + Guid StatementId, + string VulnerabilityKey, + string AdvisoryKey, + string CanonicalJson, + ImmutableArray StatementHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + ImmutableArray InputDocumentIds); + +/// +/// Persistence-facing representation of an advisory conflict used by repositories. +/// +public sealed record AdvisoryConflictEntry( + Guid ConflictId, + string VulnerabilityKey, + string CanonicalJson, + ImmutableArray ConflictHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + ImmutableArray StatementIds); diff --git a/src/StellaOps.Concelier.Core/Events/AdvisoryEventLog.cs b/src/StellaOps.Concelier.Core/Events/AdvisoryEventLog.cs new file mode 100644 index 00000000..b63ff7e7 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Events/AdvisoryEventLog.cs @@ -0,0 +1,297 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Core.Events; + +/// +/// Default implementation of that coordinates statement/conflict persistence. +/// +public sealed class AdvisoryEventLog : IAdvisoryEventLog +{ + private static readonly JsonWriterOptions CanonicalWriterOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Indented = false, + SkipValidation = false, + }; + + private readonly IAdvisoryEventRepository _repository; + private readonly TimeProvider _timeProvider; + + public AdvisoryEventLog(IAdvisoryEventRepository repository, TimeProvider? timeProvider = null) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var statements = request.Statements ?? Array.Empty(); + var conflicts = request.Conflicts ?? Array.Empty(); + + if (statements.Count == 0 && conflicts.Count == 0) + { + return; + } + + var recordedAt = _timeProvider.GetUtcNow(); + var statementEntries = BuildStatementEntries(statements, recordedAt); + var conflictEntries = BuildConflictEntries(conflicts, recordedAt); + + if (statementEntries.Count > 0) + { + await _repository.InsertStatementsAsync(statementEntries, cancellationToken).ConfigureAwait(false); + } + + if (conflictEntries.Count > 0) + { + await _repository.InsertConflictsAsync(conflictEntries, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(vulnerabilityKey)) + { + throw new ArgumentException("Vulnerability key must be provided.", nameof(vulnerabilityKey)); + } + + var normalizedKey = NormalizeKey(vulnerabilityKey, nameof(vulnerabilityKey)); + var statements = await _repository.GetStatementsAsync(normalizedKey, asOf, cancellationToken).ConfigureAwait(false); + var conflicts = await _repository.GetConflictsAsync(normalizedKey, asOf, cancellationToken).ConfigureAwait(false); + + var statementSnapshots = statements + .OrderByDescending(static entry => entry.AsOf) + .ThenByDescending(static entry => entry.RecordedAt) + .Select(ToStatementSnapshot) + .ToImmutableArray(); + + var conflictSnapshots = conflicts + .OrderByDescending(static entry => entry.AsOf) + .ThenByDescending(static entry => entry.RecordedAt) + .Select(ToConflictSnapshot) + .ToImmutableArray(); + + return new AdvisoryReplay(normalizedKey, asOf, statementSnapshots, conflictSnapshots); + } + + private static AdvisoryStatementSnapshot ToStatementSnapshot(AdvisoryStatementEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + + var advisory = CanonicalJsonSerializer.Deserialize(entry.CanonicalJson); + return new AdvisoryStatementSnapshot( + entry.StatementId, + entry.VulnerabilityKey, + entry.AdvisoryKey, + advisory, + entry.StatementHash, + entry.AsOf, + entry.RecordedAt, + entry.InputDocumentIds); + } + + private static AdvisoryConflictSnapshot ToConflictSnapshot(AdvisoryConflictEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + + return new AdvisoryConflictSnapshot( + entry.ConflictId, + entry.VulnerabilityKey, + entry.StatementIds, + entry.ConflictHash, + entry.AsOf, + entry.RecordedAt, + entry.CanonicalJson); + } + + private static IReadOnlyCollection BuildStatementEntries( + IReadOnlyCollection statements, + DateTimeOffset recordedAt) + { + if (statements.Count == 0) + { + return Array.Empty(); + } + + var entries = new List(statements.Count); + + foreach (var statement in statements) + { + ArgumentNullException.ThrowIfNull(statement); + ArgumentNullException.ThrowIfNull(statement.Advisory); + + var vulnerabilityKey = NormalizeKey(statement.VulnerabilityKey, nameof(statement.VulnerabilityKey)); + var advisory = CanonicalJsonSerializer.Normalize(statement.Advisory); + var advisoryKey = string.IsNullOrWhiteSpace(statement.AdvisoryKey) + ? advisory.AdvisoryKey + : statement.AdvisoryKey.Trim(); + + if (string.IsNullOrWhiteSpace(advisoryKey)) + { + throw new ArgumentException("Advisory key must be provided.", nameof(statement)); + } + + if (!string.Equals(advisory.AdvisoryKey, advisoryKey, StringComparison.Ordinal)) + { + throw new ArgumentException("Advisory key in payload must match provided advisory key.", nameof(statement)); + } + + var canonicalJson = CanonicalJsonSerializer.Serialize(advisory); + var hashBytes = ComputeHash(canonicalJson); + var asOf = statement.AsOf.ToUniversalTime(); + var inputDocuments = statement.InputDocumentIds?.Count > 0 + ? statement.InputDocumentIds + .Where(static id => id != Guid.Empty) + .Distinct() + .OrderBy(static id => id) + .ToImmutableArray() + : ImmutableArray.Empty; + + entries.Add(new AdvisoryStatementEntry( + statement.StatementId ?? Guid.NewGuid(), + vulnerabilityKey, + advisoryKey, + canonicalJson, + hashBytes, + asOf, + recordedAt, + inputDocuments)); + } + + return entries; + } + + private static IReadOnlyCollection BuildConflictEntries( + IReadOnlyCollection conflicts, + DateTimeOffset recordedAt) + { + if (conflicts.Count == 0) + { + return Array.Empty(); + } + + var entries = new List(conflicts.Count); + + foreach (var conflict in conflicts) + { + ArgumentNullException.ThrowIfNull(conflict); + ArgumentNullException.ThrowIfNull(conflict.Details); + + var vulnerabilityKey = NormalizeKey(conflict.VulnerabilityKey, nameof(conflict.VulnerabilityKey)); + var canonicalJson = Canonicalize(conflict.Details.RootElement); + var hashBytes = ComputeHash(canonicalJson); + var asOf = conflict.AsOf.ToUniversalTime(); + var statementIds = conflict.StatementIds?.Count > 0 + ? conflict.StatementIds + .Where(static id => id != Guid.Empty) + .Distinct() + .OrderBy(static id => id) + .ToImmutableArray() + : ImmutableArray.Empty; + + entries.Add(new AdvisoryConflictEntry( + conflict.ConflictId ?? Guid.NewGuid(), + vulnerabilityKey, + canonicalJson, + hashBytes, + asOf, + recordedAt, + statementIds)); + } + + return entries; + } + + private static string NormalizeKey(string value, string parameterName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Value must be provided.", parameterName); + } + + return value.Trim().ToLowerInvariant(); + } + + private static ImmutableArray ComputeHash(string canonicalJson) + { + var bytes = Encoding.UTF8.GetBytes(canonicalJson); + var hash = SHA256.HashData(bytes); + return ImmutableArray.Create(hash); + } + + private static string Canonicalize(JsonElement element) + { + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, CanonicalWriterOptions)) + { + WriteCanonical(element, writer); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static void WriteCanonical(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (var property in element.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal)) + { + writer.WritePropertyName(property.Name); + WriteCanonical(property.Value, writer); + } + + writer.WriteEndObject(); + break; + + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + { + WriteCanonical(item, writer); + } + + writer.WriteEndArray(); + break; + + case JsonValueKind.String: + writer.WriteStringValue(element.GetString()); + break; + + case JsonValueKind.Number: + writer.WriteRawValue(element.GetRawText()); + break; + + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + + case JsonValueKind.Undefined: + default: + writer.WriteNullValue(); + break; + } + } +} diff --git a/src/StellaOps.Concelier.Core/Events/IAdvisoryEventLog.cs b/src/StellaOps.Concelier.Core/Events/IAdvisoryEventLog.cs new file mode 100644 index 00000000..ea7c0fbf --- /dev/null +++ b/src/StellaOps.Concelier.Core/Events/IAdvisoryEventLog.cs @@ -0,0 +1,15 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Core.Events; + +/// +/// High-level API for recording and replaying advisory statements with deterministic as-of queries. +/// +public interface IAdvisoryEventLog +{ + ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken); + + ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Concelier.Core/Events/IAdvisoryEventRepository.cs b/src/StellaOps.Concelier.Core/Events/IAdvisoryEventRepository.cs new file mode 100644 index 00000000..7f99d18f --- /dev/null +++ b/src/StellaOps.Concelier.Core/Events/IAdvisoryEventRepository.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Core.Events; + +/// +/// Abstraction over the persistence layer for advisory statements and conflicts. +/// +public interface IAdvisoryEventRepository +{ + ValueTask InsertStatementsAsync( + IReadOnlyCollection statements, + CancellationToken cancellationToken); + + ValueTask InsertConflictsAsync( + IReadOnlyCollection conflicts, + CancellationToken cancellationToken); + + ValueTask> GetStatementsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken); + + ValueTask> GetConflictsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/IJob.cs b/src/StellaOps.Concelier.Core/Jobs/IJob.cs similarity index 71% rename from src/StellaOps.Feedser.Core/Jobs/IJob.cs rename to src/StellaOps.Concelier.Core/Jobs/IJob.cs index 8800c4ab..92a485f4 100644 --- a/src/StellaOps.Feedser.Core/Jobs/IJob.cs +++ b/src/StellaOps.Concelier.Core/Jobs/IJob.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public interface IJob { diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs b/src/StellaOps.Concelier.Core/Jobs/IJobCoordinator.cs similarity index 93% rename from src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs rename to src/StellaOps.Concelier.Core/Jobs/IJobCoordinator.cs index ab3588d8..961dd338 100644 --- a/src/StellaOps.Feedser.Core/Jobs/IJobCoordinator.cs +++ b/src/StellaOps.Concelier.Core/Jobs/IJobCoordinator.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public interface IJobCoordinator { diff --git a/src/StellaOps.Concelier.Core/Jobs/IJobStore.cs b/src/StellaOps.Concelier.Core/Jobs/IJobStore.cs new file mode 100644 index 00000000..c64c42c5 --- /dev/null +++ b/src/StellaOps.Concelier.Core/Jobs/IJobStore.cs @@ -0,0 +1,22 @@ +using MongoDB.Driver; + +namespace StellaOps.Concelier.Core.Jobs; + +public interface IJobStore +{ + Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task FindAsync(Guid runId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task> GetActiveRunsAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task GetLastRunAsync(string kind, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs b/src/StellaOps.Concelier.Core/Jobs/ILeaseStore.cs similarity index 89% rename from src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs rename to src/StellaOps.Concelier.Core/Jobs/ILeaseStore.cs index 802d4261..467f4c29 100644 --- a/src/StellaOps.Feedser.Core/Jobs/ILeaseStore.cs +++ b/src/StellaOps.Concelier.Core/Jobs/ILeaseStore.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public interface ILeaseStore { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs b/src/StellaOps.Concelier.Core/Jobs/JobCoordinator.cs similarity index 97% rename from src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs rename to src/StellaOps.Concelier.Core/Jobs/JobCoordinator.cs index a7e47b42..067648af 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobCoordinator.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobCoordinator.cs @@ -8,7 +8,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed class JobCoordinator : IJobCoordinator { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs b/src/StellaOps.Concelier.Core/Jobs/JobDefinition.cs similarity index 80% rename from src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs rename to src/StellaOps.Concelier.Core/Jobs/JobDefinition.cs index 1e56f674..de0ae8a2 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobDefinition.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobDefinition.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed record JobDefinition( string Kind, diff --git a/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs b/src/StellaOps.Concelier.Core/Jobs/JobDiagnostics.cs similarity index 85% rename from src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs rename to src/StellaOps.Concelier.Core/Jobs/JobDiagnostics.cs index d90ef781..a781e07d 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobDiagnostics.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobDiagnostics.cs @@ -1,15 +1,15 @@ using System.Diagnostics; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed class JobDiagnostics : IDisposable { - public const string ActivitySourceName = "StellaOps.Feedser.Jobs"; - public const string MeterName = "StellaOps.Feedser.Jobs"; - public const string TriggerActivityName = "feedser.job.trigger"; - public const string ExecuteActivityName = "feedser.job.execute"; - public const string SchedulerActivityName = "feedser.scheduler.evaluate"; + public const string ActivitySourceName = "StellaOps.Concelier.Jobs"; + public const string MeterName = "StellaOps.Concelier.Jobs"; + public const string TriggerActivityName = "concelier.job.trigger"; + public const string ExecuteActivityName = "concelier.job.execute"; + public const string SchedulerActivityName = "concelier.scheduler.evaluate"; private readonly Counter _triggersAccepted; private readonly Counter _triggersRejected; @@ -24,32 +24,32 @@ public sealed class JobDiagnostics : IDisposable Meter = new Meter(MeterName); _triggersAccepted = Meter.CreateCounter( - name: "feedser.jobs.triggers.accepted", + name: "concelier.jobs.triggers.accepted", unit: "count", description: "Number of job trigger requests accepted for execution."); _triggersRejected = Meter.CreateCounter( - name: "feedser.jobs.triggers.rejected", + name: "concelier.jobs.triggers.rejected", unit: "count", description: "Number of job trigger requests rejected or ignored by the coordinator."); _runsCompleted = Meter.CreateCounter( - name: "feedser.jobs.runs.completed", + name: "concelier.jobs.runs.completed", unit: "count", description: "Number of job executions that have finished grouped by outcome."); _runsActive = Meter.CreateUpDownCounter( - name: "feedser.jobs.runs.active", + name: "concelier.jobs.runs.active", unit: "count", description: "Current number of running job executions."); _runDurationSeconds = Meter.CreateHistogram( - name: "feedser.jobs.runs.duration", + name: "concelier.jobs.runs.duration", unit: "s", description: "Distribution of job execution durations in seconds."); _schedulerSkewMilliseconds = Meter.CreateHistogram( - name: "feedser.scheduler.skew", + name: "concelier.scheduler.skew", unit: "ms", description: "Difference between the intended and actual scheduler fire time in milliseconds."); } diff --git a/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs b/src/StellaOps.Concelier.Core/Jobs/JobExecutionContext.cs similarity index 93% rename from src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs rename to src/StellaOps.Concelier.Core/Jobs/JobExecutionContext.cs index 24141695..7302d183 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobExecutionContext.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobExecutionContext.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed class JobExecutionContext { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobLease.cs b/src/StellaOps.Concelier.Core/Jobs/JobLease.cs similarity index 78% rename from src/StellaOps.Feedser.Core/Jobs/JobLease.cs rename to src/StellaOps.Concelier.Core/Jobs/JobLease.cs index 2f01509e..d0cf12ae 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobLease.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobLease.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed record JobLease( string Key, diff --git a/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs b/src/StellaOps.Concelier.Core/Jobs/JobPluginRegistrationExtensions.cs similarity index 92% rename from src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs rename to src/StellaOps.Concelier.Core/Jobs/JobPluginRegistrationExtensions.cs index b60f2608..0197ecce 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobPluginRegistrationExtensions.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobPluginRegistrationExtensions.cs @@ -3,12 +3,13 @@ using System.Collections.Generic; using System.Linq; using System.Reflection; using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.DependencyInjection; -using StellaOps.Plugin.Hosting; - -namespace StellaOps.Feedser.Core.Jobs; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Concelier.Core.Jobs; public static class JobPluginRegistrationExtensions { @@ -32,12 +33,14 @@ public static class JobPluginRegistrationExtensions var currentServices = services; var seenRoutineTypes = new HashSet(StringComparer.Ordinal); - foreach (var plugin in loadResult.Plugins) - { - foreach (var routineType in GetRoutineTypes(plugin.Assembly)) - { - if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(routineType)) - { + foreach (var plugin in loadResult.Plugins) + { + PluginServiceRegistration.RegisterAssemblyMetadata(currentServices, plugin.Assembly, logger); + + foreach (var routineType in GetRoutineTypes(plugin.Assembly)) + { + if (!typeof(IDependencyInjectionRoutine).IsAssignableFrom(routineType)) + { continue; } diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs b/src/StellaOps.Concelier.Core/Jobs/JobRunCompletion.cs similarity index 71% rename from src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs rename to src/StellaOps.Concelier.Core/Jobs/JobRunCompletion.cs index 7018dba5..b6b5c5d5 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobRunCompletion.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobRunCompletion.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed record JobRunCompletion( JobRunStatus Status, diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs b/src/StellaOps.Concelier.Core/Jobs/JobRunCreateRequest.cs similarity index 82% rename from src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs rename to src/StellaOps.Concelier.Core/Jobs/JobRunCreateRequest.cs index bf576559..bcf91d6f 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobRunCreateRequest.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobRunCreateRequest.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed record JobRunCreateRequest( string Kind, diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs b/src/StellaOps.Concelier.Core/Jobs/JobRunSnapshot.cs similarity index 90% rename from src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs rename to src/StellaOps.Concelier.Core/Jobs/JobRunSnapshot.cs index 7ef3099b..fc4e4960 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobRunSnapshot.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobRunSnapshot.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; /// /// Immutable projection of a job run as stored in persistence. diff --git a/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs b/src/StellaOps.Concelier.Core/Jobs/JobRunStatus.cs similarity index 65% rename from src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs rename to src/StellaOps.Concelier.Core/Jobs/JobRunStatus.cs index a6871f22..a3812ca4 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobRunStatus.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobRunStatus.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public enum JobRunStatus { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerBuilder.cs similarity index 93% rename from src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs rename to src/StellaOps.Concelier.Core/Jobs/JobSchedulerBuilder.cs index 4c871ada..aa5970f7 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerBuilder.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerBuilder.cs @@ -1,7 +1,7 @@ using System; using Microsoft.Extensions.DependencyInjection; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed class JobSchedulerBuilder { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerHostedService.cs similarity index 96% rename from src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs rename to src/StellaOps.Concelier.Core/Jobs/JobSchedulerHostedService.cs index 7f74f378..15016f6f 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerHostedService.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerHostedService.cs @@ -4,7 +4,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; /// /// Background service that evaluates cron expressions for registered jobs and triggers them. diff --git a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerOptions.cs similarity index 88% rename from src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs rename to src/StellaOps.Concelier.Core/Jobs/JobSchedulerOptions.cs index f3f22186..50be8392 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobSchedulerOptions.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobSchedulerOptions.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public sealed class JobSchedulerOptions { diff --git a/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs b/src/StellaOps.Concelier.Core/Jobs/JobTriggerResult.cs similarity index 94% rename from src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs rename to src/StellaOps.Concelier.Core/Jobs/JobTriggerResult.cs index 49e1d60b..e2b2076c 100644 --- a/src/StellaOps.Feedser.Core/Jobs/JobTriggerResult.cs +++ b/src/StellaOps.Concelier.Core/Jobs/JobTriggerResult.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public enum JobTriggerOutcome { diff --git a/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Core/Jobs/ServiceCollectionExtensions.cs similarity index 93% rename from src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Core/Jobs/ServiceCollectionExtensions.cs index 99e98c96..c3762329 100644 --- a/src/StellaOps.Feedser.Core/Jobs/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Core/Jobs/ServiceCollectionExtensions.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; -namespace StellaOps.Feedser.Core.Jobs; +namespace StellaOps.Concelier.Core.Jobs; public static class JobServiceCollectionExtensions { diff --git a/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj b/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj similarity index 80% rename from src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj rename to src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj index c74e9e04..a85281ba 100644 --- a/src/StellaOps.Feedser.Core/StellaOps.Feedser.Core.csproj +++ b/src/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj @@ -6,14 +6,15 @@ enable true - - + + + - + diff --git a/src/StellaOps.Feedser.Core/TASKS.md b/src/StellaOps.Concelier.Core/TASKS.md similarity index 80% rename from src/StellaOps.Feedser.Core/TASKS.md rename to src/StellaOps.Concelier.Core/TASKS.md index 76724b04..ef2be75d 100644 --- a/src/StellaOps.Feedser.Core/TASKS.md +++ b/src/StellaOps.Concelier.Core/TASKS.md @@ -1,20 +1,21 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|JobCoordinator implementation (create/get/mark status)|BE-Core|Storage.Mongo|DONE – `JobCoordinator` drives Mongo-backed runs.| -|Cron scheduling loop with TimeProvider|BE-Core|Core|DONE – `JobSchedulerHostedService` evaluates cron expressions.| -|Single-flight/lease semantics|BE-Core|Storage.Mongo|DONE – lease acquisition backed by `MongoLeaseStore`.| -|Trigger API contract (Result mapping)|BE-Core|WebService|DONE – `JobTriggerResult` outcomes map to HTTP statuses.| -|Run telemetry enrichment|BE-Core|Observability|DONE – `JobDiagnostics` ties activities & counters into coordinator/scheduler paths.| -|Deterministic params hashing|BE-Core|Core|DONE – `JobParametersHasher` creates SHA256 hash.| -|Golden tests for timeout/cancel|QA|Core|DONE – JobCoordinatorTests cover cancellation timeout path.| -|JobSchedulerBuilder options registry coverage|BE-Core|Core|DONE – added scheduler tests confirming cron/timeout/lease metadata persists via JobSchedulerOptions.| -|Plugin discovery + DI glue with PluginHost|BE-Core|Plugin libs|DONE – JobPluginRegistrationExtensions now loads PluginHost routines and wires connector/exporter registrations.| -|Harden lease release error handling in JobCoordinator|BE-Core|Storage.Mongo|DONE – lease release failures now logged, wrapped, and drive run failure status; fire-and-forget execution guarded. Verified with `dotnet test --no-build --filter JobCoordinator`.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|JobCoordinator implementation (create/get/mark status)|BE-Core|Storage.Mongo|DONE – `JobCoordinator` drives Mongo-backed runs.| +|Cron scheduling loop with TimeProvider|BE-Core|Core|DONE – `JobSchedulerHostedService` evaluates cron expressions.| +|Single-flight/lease semantics|BE-Core|Storage.Mongo|DONE – lease acquisition backed by `MongoLeaseStore`.| +|Trigger API contract (Result mapping)|BE-Core|WebService|DONE – `JobTriggerResult` outcomes map to HTTP statuses.| +|Run telemetry enrichment|BE-Core|Observability|DONE – `JobDiagnostics` ties activities & counters into coordinator/scheduler paths.| +|Deterministic params hashing|BE-Core|Core|DONE – `JobParametersHasher` creates SHA256 hash.| +|Golden tests for timeout/cancel|QA|Core|DONE – JobCoordinatorTests cover cancellation timeout path.| +|JobSchedulerBuilder options registry coverage|BE-Core|Core|DONE – added scheduler tests confirming cron/timeout/lease metadata persists via JobSchedulerOptions.| +|Plugin discovery + DI glue with PluginHost|BE-Core|Plugin libs|DONE – JobPluginRegistrationExtensions now loads PluginHost routines and wires connector/exporter registrations.| +|Harden lease release error handling in JobCoordinator|BE-Core|Storage.Mongo|DONE – lease release failures now logged, wrapped, and drive run failure status; fire-and-forget execution guarded. Verified with `dotnet test --no-build --filter JobCoordinator`.| |Validate job trigger parameters for serialization|BE-Core|WebService|DONE – trigger parameters normalized/serialized with defensive checks returning InvalidParameters on failure. Full-suite `dotnet test --no-build` currently red from live connector fixture drift (Oracle/JVN/RedHat).| |FEEDCORE-ENGINE-03-001 Canonical merger implementation|BE-Core|Merge|DONE – `CanonicalMerger` applies GHSA/NVD/OSV conflict rules with deterministic provenance and comprehensive unit coverage. **Coordination:** Connector leads must align mapper outputs with the canonical field expectations before 2025-10-18 so Merge can activate the path globally.| |FEEDCORE-ENGINE-03-002 Field precedence and tie-breaker map|BE-Core|Merge|DONE – field precedence and freshness overrides enforced via `FieldPrecedence` map with tie-breakers and analytics capture. **Reminder:** Storage/Merge owners review precedence overrides when onboarding new feeds to ensure `decisionReason` tagging stays consistent.| |Canonical merger parity for description/CWE/canonical metric|BE-Core|Models|DONE (2025-10-15) – merger now populates description/CWEs/canonical metric id with provenance and regression tests cover the new decisions.| |Reference normalization & freshness instrumentation cleanup|BE-Core, QA|Models|DONE (2025-10-15) – reference keys normalized, freshness overrides applied to union fields, and new tests assert decision logging.| -|FEEDCORE-ENGINE-07-001 – Advisory event log & asOf queries|Team Core Engine & Storage Analytics|FEEDSTORAGE-DATA-07-001|TODO – Introduce immutable advisory statement events, expose `asOf` query surface for merge/export pipelines, and document determinism guarantees for replay.| -|FEEDCORE-ENGINE-07-002 – Noise prior computation service|Team Core Engine & Data Science|FEEDCORE-ENGINE-07-001|TODO – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Vexer/scan suppressors with reproducible statistics.| +|FEEDCORE-ENGINE-07-001 – Advisory event log & asOf queries|Team Core Engine & Storage Analytics|FEEDSTORAGE-DATA-07-001|**DONE (2025-10-19)** – Implemented `AdvisoryEventLog` service plus repository contracts, canonical hashing, and lower-cased key normalization with replay support; documented determinism guarantees. Tests: `dotnet test src/StellaOps.Concelier.Core.Tests/StellaOps.Concelier.Core.Tests.csproj`.| +|FEEDCORE-ENGINE-07-002 – Noise prior computation service|Team Core Engine & Data Science|FEEDCORE-ENGINE-07-001|TODO – Build rule-based learner capturing false-positive priors per package/env, persist summaries, and expose APIs for Excititor/scan suppressors with reproducible statistics.| +|FEEDCORE-ENGINE-07-003 – Unknown state ledger & confidence seeding|Team Core Engine & Storage Analytics|FEEDCORE-ENGINE-07-001|TODO – Persist `unknown_vuln_range/unknown_origin/ambiguous_fix` markers with initial confidence bands, expose query surface for Policy, and add fixtures validating canonical serialization.| diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs similarity index 89% rename from src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs rename to src/StellaOps.Concelier.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs index 4dada488..108a4809 100644 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExportSnapshotBuilderTests.cs @@ -7,10 +7,10 @@ using System.Security.Cryptography; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json.Tests; +namespace StellaOps.Concelier.Exporter.Json.Tests; public sealed class JsonExportSnapshotBuilderTests : IDisposable { @@ -18,7 +18,7 @@ public sealed class JsonExportSnapshotBuilderTests : IDisposable public JsonExportSnapshotBuilderTests() { - _root = Directory.CreateTempSubdirectory("feedser-json-export-tests").FullName; + _root = Directory.CreateTempSubdirectory("concelier-json-export-tests").FullName; } [Fact] @@ -42,10 +42,14 @@ public sealed class JsonExportSnapshotBuilderTests : IDisposable severity: "medium"), }; - var result = await builder.WriteAsync(advisories, exportedAt, cancellationToken: CancellationToken.None); - - Assert.Equal(advisories.Length, result.AdvisoryCount); - Assert.Equal(exportedAt, result.ExportedAt); + var result = await builder.WriteAsync(advisories, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + Assert.Equal(advisories.Length, result.Advisories.Length); + Assert.Equal( + advisories.Select(a => a.AdvisoryKey).OrderBy(key => key, StringComparer.Ordinal), + result.Advisories.Select(a => a.AdvisoryKey).OrderBy(key => key, StringComparer.Ordinal)); + Assert.Equal(exportedAt, result.ExportedAt); var expectedFiles = result.FilePaths.OrderBy(x => x, StringComparer.Ordinal).ToArray(); Assert.Contains("nvd/2024/CVE-2024-9999.json", expectedFiles); @@ -107,10 +111,11 @@ public sealed class JsonExportSnapshotBuilderTests : IDisposable }; var sequence = new SingleEnumerationAsyncSequence(advisories); - var result = await builder.WriteAsync(sequence, exportedAt, cancellationToken: CancellationToken.None); - - Assert.Equal(advisories.Length, result.AdvisoryCount); - } + var result = await builder.WriteAsync(sequence, exportedAt, cancellationToken: CancellationToken.None); + + Assert.Equal(advisories.Length, result.AdvisoryCount); + Assert.Equal(advisories.Length, result.Advisories.Length); + } private static Advisory CreateAdvisory(string advisoryKey, string[] aliases, string title, string severity) { @@ -141,7 +146,7 @@ public sealed class JsonExportSnapshotBuilderTests : IDisposable cvssMetrics: Array.Empty(), provenance: new[] { - new AdvisoryProvenance("feedser", "normalized", "canonical", DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture)), + new AdvisoryProvenance("concelier", "normalized", "canonical", DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture)), }); } diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs similarity index 58% rename from src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs rename to src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs index 9521557f..05a9175a 100644 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterDependencyInjectionRoutineTests.cs @@ -1,17 +1,20 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading.Tasks; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Exporting; -using StellaOps.Feedser.Models; +using System.Collections.Immutable; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Exporting; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json.Tests; +namespace StellaOps.Concelier.Exporter.Json.Tests; public sealed class JsonExporterDependencyInjectionRoutineTests { @@ -19,10 +22,11 @@ public sealed class JsonExporterDependencyInjectionRoutineTests public void Register_AddsJobDefinitionAndServices() { var services = new ServiceCollection(); - services.AddLogging(); - services.AddSingleton(); - services.AddSingleton(); - services.AddOptions(); + services.AddLogging(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddOptions(); var configuration = new ConfigurationBuilder() .AddInMemoryCollection(new Dictionary()) @@ -45,21 +49,31 @@ public sealed class JsonExporterDependencyInjectionRoutineTests private sealed class StubAdvisoryStore : IAdvisoryStore { - public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) - => Task.FromResult>(Array.Empty()); - - public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) - => Task.FromResult(null); - - public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) - => Task.CompletedTask; - - public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) - { - return Enumerate(cancellationToken); - - static async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken ct) - { + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult>(Array.Empty()); + } + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult(null); + } + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.CompletedTask; + } + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Enumerate(cancellationToken); + + static async IAsyncEnumerable Enumerate([EnumeratorCancellation] CancellationToken ct) + { ct.ThrowIfCancellationRequested(); await Task.Yield(); yield break; @@ -67,17 +81,34 @@ public sealed class JsonExporterDependencyInjectionRoutineTests } } - private sealed class StubExportStateStore : IExportStateStore - { - private ExportStateRecord? _record; + private sealed class StubExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; - public Task FindAsync(string id, CancellationToken cancellationToken) - => Task.FromResult(_record); - - public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) - { - _record = record; - return Task.FromResult(record); - } - } -} + public Task FindAsync(string id, CancellationToken cancellationToken) + { + return Task.FromResult(_record); + } + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class StubAdvisoryEventLog : IAdvisoryEventLog + { + public ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + { + return ValueTask.FromResult(new AdvisoryReplay( + vulnerabilityKey, + asOf, + ImmutableArray.Empty, + ImmutableArray.Empty)); + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs similarity index 93% rename from src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs rename to src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs index 49aca086..72d55d72 100644 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonExporterParitySmokeTests.cs @@ -5,10 +5,10 @@ using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json.Tests; +namespace StellaOps.Concelier.Exporter.Json.Tests; public sealed class JsonExporterParitySmokeTests : IDisposable { @@ -16,7 +16,7 @@ public sealed class JsonExporterParitySmokeTests : IDisposable public JsonExporterParitySmokeTests() { - _root = Directory.CreateTempSubdirectory("feedser-json-parity-tests").FullName; + _root = Directory.CreateTempSubdirectory("concelier-json-parity-tests").FullName; } [Fact] diff --git a/src/StellaOps.Concelier.Exporter.Json.Tests/JsonFeedExporterTests.cs b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonFeedExporterTests.cs new file mode 100644 index 00000000..c9dfc309 --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/JsonFeedExporterTests.cs @@ -0,0 +1,597 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Exporting; +using StellaOps.Cryptography; + +namespace StellaOps.Concelier.Exporter.Json.Tests; + +public sealed class JsonFeedExporterTests : IDisposable +{ + private readonly string _root; + + public JsonFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("concelier-json-exporter-tests").FullName; + } + + [Fact] + public async Task ExportAsync_SkipsWhenDigestUnchanged() + { + var advisory = new Advisory( + advisoryKey: "CVE-2024-1234", + title: "Test Advisory", + summary: null, + language: "en", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2024-1234" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var advisoryStore = new StubAdvisoryStore(advisory); + var options = Options.Create(new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }); + + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var eventLog = new StubAdvisoryEventLog(new[] { advisory }, timeProvider.GetUtcNow()); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + eventLog, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + var firstUpdated = record!.UpdatedAt; + Assert.Equal("20240715T120000Z", record.BaseExportId); + Assert.Equal(record.LastFullDigest, record.ExportCursor); + + var firstExportPath = Path.Combine(_root, "20240715T120000Z"); + Assert.True(Directory.Exists(firstExportPath)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal(firstUpdated, record!.UpdatedAt); + + var secondExportPath = Path.Combine(_root, "20240715T120500Z"); + Assert.False(Directory.Exists(secondExportPath)); + } + + [Fact] + public async Task ExportAsync_WritesManifestMetadata() + { + var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture); + var recordedAt = DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture); + var reference = new AdvisoryReference( + "http://Example.com/path/resource?b=2&a=1", + kind: "advisory", + sourceTag: "REF-001", + summary: "Primary vendor advisory", + provenance: new AdvisoryProvenance("ghsa", "map", "REF-001", recordedAt, new[] { ProvenanceFieldMasks.References })); + var weakness = new AdvisoryWeakness( + taxonomy: "cwe", + identifier: "CWE-79", + name: "Cross-site Scripting", + uri: "https://cwe.mitre.org/data/definitions/79.html", + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "CWE-79", recordedAt, new[] { ProvenanceFieldMasks.Weaknesses }) + }); + var cvssMetric = new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", + 9.8, + "critical", + new AdvisoryProvenance("nvd", "map", "CVE-2024-4321", recordedAt, new[] { ProvenanceFieldMasks.CvssMetrics })); + + var advisory = new Advisory( + advisoryKey: "CVE-2024-4321", + title: "Manifest Test", + summary: "Short summary", + language: "en", + published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: recordedAt, + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-4321", "GHSA-xxxx-yyyy-zzzz" }, + credits: Array.Empty(), + references: new[] { reference }, + affectedPackages: Array.Empty(), + cvssMetrics: new[] { cvssMetric }, + provenance: new[] + { + new AdvisoryProvenance("ghsa", "map", "GHSA-xxxx-yyyy-zzzz", recordedAt, new[] { ProvenanceFieldMasks.Advisory }), + new AdvisoryProvenance("nvd", "map", "CVE-2024-4321", recordedAt, new[] { ProvenanceFieldMasks.Advisory }) + }, + description: "Detailed description capturing remediation steps.", + cwes: new[] { weakness }, + canonicalMetricId: "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H"); + + var advisoryStore = new StubAdvisoryStore(advisory); + var optionsValue = new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + }; + + var options = Options.Create(optionsValue); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var eventLog = new StubAdvisoryEventLog(new[] { advisory }, exportedAt); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + eventLog, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportDirectory = Path.Combine(_root, exportId); + var manifestPath = Path.Combine(exportDirectory, "manifest.json"); + + Assert.True(File.Exists(manifestPath)); + + using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None)); + var root = document.RootElement; + + Assert.Equal(exportId, root.GetProperty("exportId").GetString()); + Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime()); + Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32()); + + var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories) + .Select(path => new + { + Absolute = path, + Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal), + }) + .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase)) + .OrderBy(file => file.Relative, StringComparer.Ordinal) + .ToArray(); + + var filesElement = root.GetProperty("files") + .EnumerateArray() + .Select(element => new + { + Path = element.GetProperty("path").GetString(), + Bytes = element.GetProperty("bytes").GetInt64(), + Digest = element.GetProperty("digest").GetString(), + }) + .OrderBy(file => file.Path, StringComparer.Ordinal) + .ToArray(); + + var dataFile = Assert.Single(exportedFiles); + using (var advisoryDocument = JsonDocument.Parse(await File.ReadAllBytesAsync(dataFile.Absolute, CancellationToken.None))) + { + var advisoryRoot = advisoryDocument.RootElement; + Assert.Equal("Detailed description capturing remediation steps.", advisoryRoot.GetProperty("description").GetString()); + Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", advisoryRoot.GetProperty("canonicalMetricId").GetString()); + + var referenceElement = advisoryRoot.GetProperty("references").EnumerateArray().Single(); + Assert.Equal(reference.Url, referenceElement.GetProperty("url").GetString(), StringComparer.OrdinalIgnoreCase); + + var weaknessElement = advisoryRoot.GetProperty("cwes").EnumerateArray().Single(); + Assert.Equal("cwe", weaknessElement.GetProperty("taxonomy").GetString()); + Assert.Equal("CWE-79", weaknessElement.GetProperty("identifier").GetString()); + } + + Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray()); + + long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum(); + Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64()); + Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32()); + + var digest = root.GetProperty("digest").GetString(); + var digestResult = new JsonExportResult( + exportDirectory, + exportedAt, + exportedFiles.Select(file => + { + var manifestEntry = filesElement.First(f => f.Path == file.Relative); + if (manifestEntry.Digest is null) + { + throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest."); + } + + return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest); + }), + exportedFiles.Length, + totalBytes); + var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult); + Assert.Equal(expectedDigest, digest); + + var exporterVersion = root.GetProperty("exporterVersion").GetString(); + Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion); + } + + [Fact] + public async Task ExportAsync_WritesMirrorBundlesWithSignatures() + { + var exportedAt = DateTimeOffset.Parse("2025-01-05T00:00:00Z", CultureInfo.InvariantCulture); + var advisoryOne = new Advisory( + advisoryKey: "CVE-2025-0001", + title: "Mirror Advisory One", + summary: null, + language: "en", + published: exportedAt.AddDays(-10), + modified: exportedAt.AddDays(-9), + severity: "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-0001", "GHSA-aaaa-bbbb-cccc" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("ghsa", "map", "GHSA-aaaa-bbbb-cccc", exportedAt.AddDays(-9)), + new AdvisoryProvenance("nvd", "map", "CVE-2025-0001", exportedAt.AddDays(-8)), + }); + + var advisoryTwo = new Advisory( + advisoryKey: "CVE-2025-0002", + title: "Mirror Advisory Two", + summary: null, + language: "en", + published: exportedAt.AddDays(-6), + modified: exportedAt.AddDays(-5), + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-0002" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "CVE-2025-0002", exportedAt.AddDays(-5)), + new AdvisoryProvenance("vendor", "map", "ADVISORY-0002", exportedAt.AddDays(-4)), + }); + + var advisoryStore = new StubAdvisoryStore(advisoryOne, advisoryTwo); + var optionsValue = new JsonExportOptions + { + OutputRoot = _root, + MaintainLatestSymlink = false, + TargetRepository = "s3://mirror/concelier" + }; + + optionsValue.Mirror.Enabled = true; + optionsValue.Mirror.DirectoryName = "mirror"; + optionsValue.Mirror.Domains.Add(new JsonExportOptions.JsonMirrorDomainOptions + { + Id = "primary", + DisplayName = "Primary" + }); + + optionsValue.Mirror.Signing.Enabled = true; + optionsValue.Mirror.Signing.KeyId = "mirror-signing-key"; + optionsValue.Mirror.Signing.Algorithm = SignatureAlgorithms.Es256; + optionsValue.Mirror.Signing.KeyPath = WriteSigningKey(_root); + + var options = Options.Create(optionsValue); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var eventLog = new StubAdvisoryEventLog(new[] { advisoryOne, advisoryTwo }, exportedAt); + var exporter = new JsonFeedExporter( + advisoryStore, + options, + new VulnListJsonExportPathResolver(), + stateManager, + eventLog, + NullLogger.Instance, + timeProvider); + + var services = new ServiceCollection(); + services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => + { + var provider = sp.GetRequiredService(); + return new CryptoProviderRegistry(new[] { provider }); + }); + + using var provider = services.BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); + var exportDirectory = Path.Combine(_root, exportId); + var mirrorDirectory = Path.Combine(exportDirectory, "mirror"); + var domainDirectory = Path.Combine(mirrorDirectory, "primary"); + + Assert.True(File.Exists(Path.Combine(mirrorDirectory, "index.json"))); + Assert.True(File.Exists(Path.Combine(domainDirectory, "bundle.json"))); + Assert.True(File.Exists(Path.Combine(domainDirectory, "bundle.json.jws"))); + Assert.True(File.Exists(Path.Combine(domainDirectory, "manifest.json"))); + + var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Contains(record!.Files, file => string.Equals(file.Path, "mirror/index.json", StringComparison.Ordinal)); + Assert.Contains(record.Files, file => string.Equals(file.Path, "mirror/primary/manifest.json", StringComparison.Ordinal)); + + var indexPath = Path.Combine(mirrorDirectory, "index.json"); + using (var indexDoc = JsonDocument.Parse(await File.ReadAllBytesAsync(indexPath, CancellationToken.None))) + { + var indexRoot = indexDoc.RootElement; + Assert.Equal("s3://mirror/concelier", indexRoot.GetProperty("targetRepository").GetString()); + + var domains = indexRoot.GetProperty("domains").EnumerateArray().ToArray(); + var domain = Assert.Single(domains); + Assert.Equal("primary", domain.GetProperty("domainId").GetString()); + Assert.Equal("Primary", domain.GetProperty("displayName").GetString()); + Assert.Equal(2, domain.GetProperty("advisoryCount").GetInt32()); + + var bundleDescriptor = domain.GetProperty("bundle"); + Assert.Equal("mirror/primary/bundle.json", bundleDescriptor.GetProperty("path").GetString()); + var signatureDescriptor = bundleDescriptor.GetProperty("signature"); + Assert.Equal("mirror/primary/bundle.json.jws", signatureDescriptor.GetProperty("path").GetString()); + + var manifestDescriptor = domain.GetProperty("manifest"); + Assert.Equal("mirror/primary/manifest.json", manifestDescriptor.GetProperty("path").GetString()); + } + + var bundlePathRel = "mirror/primary/bundle.json"; + var manifestPathRel = "mirror/primary/manifest.json"; + var signaturePathRel = "mirror/primary/bundle.json.jws"; + + var bundlePath = Path.Combine(exportDirectory, bundlePathRel.Replace('/', Path.DirectorySeparatorChar)); + var manifestPath = Path.Combine(exportDirectory, manifestPathRel.Replace('/', Path.DirectorySeparatorChar)); + var signaturePath = Path.Combine(exportDirectory, signaturePathRel.Replace('/', Path.DirectorySeparatorChar)); + + using (var bundleDoc = JsonDocument.Parse(await File.ReadAllBytesAsync(bundlePath, CancellationToken.None))) + { + var bundleRoot = bundleDoc.RootElement; + Assert.Equal("primary", bundleRoot.GetProperty("domainId").GetString()); + Assert.Equal(2, bundleRoot.GetProperty("advisoryCount").GetInt32()); + Assert.Equal("s3://mirror/concelier", bundleRoot.GetProperty("targetRepository").GetString()); + Assert.Equal(2, bundleRoot.GetProperty("advisories").GetArrayLength()); + + var sources = bundleRoot.GetProperty("sources").EnumerateArray().Select(element => element.GetProperty("source").GetString()).ToArray(); + Assert.Contains("ghsa", sources); + Assert.Contains("nvd", sources); + Assert.Contains("vendor", sources); + } + + using (var manifestDoc = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None))) + { + var manifestRoot = manifestDoc.RootElement; + Assert.Equal("primary", manifestRoot.GetProperty("domainId").GetString()); + Assert.Equal(2, manifestRoot.GetProperty("advisoryCount").GetInt32()); + Assert.Equal("mirror/primary/bundle.json", manifestRoot.GetProperty("bundle").GetProperty("path").GetString()); + } + + var bundleBytes = await File.ReadAllBytesAsync(bundlePath, CancellationToken.None); + var signatureValue = await File.ReadAllTextAsync(signaturePath, CancellationToken.None); + var signatureParts = signatureValue.Split("..", StringSplitOptions.None); + Assert.Equal(2, signatureParts.Length); + + var signingInput = BuildSigningInput(signatureParts[0], bundleBytes); + var signatureBytes = Base64UrlDecode(signatureParts[1]); + + var registry = provider.GetRequiredService(); + var verification = registry.ResolveSigner( + CryptoCapability.Signing, + optionsValue.Mirror.Signing.Algorithm, + new CryptoKeyReference(optionsValue.Mirror.Signing.KeyId, optionsValue.Mirror.Signing.Provider), + optionsValue.Mirror.Signing.Provider); + var verified = await verification.Signer.VerifyAsync(signingInput, signatureBytes, CancellationToken.None); + Assert.True(verified); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private static string WriteSigningKey(string directory) + { + using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var pkcs8 = ecdsa.ExportPkcs8PrivateKey(); + var pem = BuildPem("PRIVATE KEY", pkcs8); + var path = Path.Combine(directory, $"mirror-key-{Guid.NewGuid():N}.pem"); + File.WriteAllText(path, pem); + return path; + } + + private static string BuildPem(string label, byte[] data) + { + var base64 = Convert.ToBase64String(data, Base64FormattingOptions.InsertLineBreaks); + return $"-----BEGIN {label}-----\n{base64}\n-----END {label}-----\n"; + } + + private static byte[] BuildSigningInput(string protectedHeader, byte[] payload) + { + var headerBytes = Encoding.ASCII.GetBytes(protectedHeader); + var buffer = new byte[headerBytes.Length + 1 + payload.Length]; + Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length); + buffer[headerBytes.Length] = (byte)'.'; + Buffer.BlockCopy(payload, 0, buffer, headerBytes.Length + 1, payload.Length); + return buffer; + } + + private static byte[] Base64UrlDecode(string value) + { + var builder = new StringBuilder(value.Length + 3); + foreach (var ch in value) + { + builder.Append(ch switch + { + '-' => '+', + '_' => '/', + _ => ch + }); + } + + while (builder.Length % 4 != 0) + { + builder.Append('='); + } + + return Convert.FromBase64String(builder.ToString()); + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private readonly IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult(_advisories); + } + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + } + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.CompletedTask; + } + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class StubAdvisoryEventLog : IAdvisoryEventLog + { + private readonly Dictionary _advisories; + private readonly DateTimeOffset _recordedAt; + + public StubAdvisoryEventLog(IEnumerable advisories, DateTimeOffset recordedAt) + { + _advisories = advisories.ToDictionary(advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase); + _recordedAt = recordedAt; + } + + public ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken) + => throw new NotSupportedException(); + + public ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + { + if (_advisories.TryGetValue(vulnerabilityKey, out var advisory)) + { + var asOfTimestamp = advisory.Modified ?? advisory.Published ?? _recordedAt; + var snapshot = new AdvisoryStatementSnapshot( + Guid.NewGuid(), + vulnerabilityKey, + advisory.AdvisoryKey, + advisory, + ImmutableArray.Empty, + asOfTimestamp, + _recordedAt, + ImmutableArray.Empty); + + return ValueTask.FromResult(new AdvisoryReplay( + vulnerabilityKey, + asOf, + ImmutableArray.Create(snapshot), + ImmutableArray.Empty)); + } + + return ValueTask.FromResult(new AdvisoryReplay( + vulnerabilityKey, + asOf, + ImmutableArray.Empty, + ImmutableArray.Empty)); + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + { + return Task.FromResult(_record); + } + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } +} diff --git a/src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj b/src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj new file mode 100644 index 00000000..12cef41d --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs b/src/StellaOps.Concelier.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs similarity index 95% rename from src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs rename to src/StellaOps.Concelier.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs index 76a661e3..7ae8eb19 100644 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs +++ b/src/StellaOps.Concelier.Exporter.Json.Tests/VulnListJsonExportPathResolverTests.cs @@ -1,10 +1,10 @@ using System.Collections.Generic; using System.Globalization; using System.IO; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json.Tests; +namespace StellaOps.Concelier.Exporter.Json.Tests; public sealed class VulnListJsonExportPathResolverTests { diff --git a/src/StellaOps.Feedser.Exporter.Json/AGENTS.md b/src/StellaOps.Concelier.Exporter.Json/AGENTS.md similarity index 91% rename from src/StellaOps.Feedser.Exporter.Json/AGENTS.md rename to src/StellaOps.Concelier.Exporter.Json/AGENTS.md index 25046404..5ce605e3 100644 --- a/src/StellaOps.Feedser.Exporter.Json/AGENTS.md +++ b/src/StellaOps.Concelier.Exporter.Json/AGENTS.md @@ -22,7 +22,7 @@ Out: ORAS push and Trivy DB BoltDB writing (owned by Trivy exporter). - Metrics: export.json.records, bytes, duration, delta.changed. - Logs: target path, record counts, digest; no sensitive data. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Exporter.Json.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Exporter.Json.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs b/src/StellaOps.Concelier.Exporter.Json/ExportDigestCalculator.cs similarity index 94% rename from src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs rename to src/StellaOps.Concelier.Exporter.Json/ExportDigestCalculator.cs index 64079258..a29b9445 100644 --- a/src/StellaOps.Feedser.Exporter.Json/ExportDigestCalculator.cs +++ b/src/StellaOps.Concelier.Exporter.Json/ExportDigestCalculator.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Security.Cryptography; using System.Text; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public static class ExportDigestCalculator { diff --git a/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs b/src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs similarity index 91% rename from src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs rename to src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs index 351217d2..e5baeb46 100644 --- a/src/StellaOps.Feedser.Exporter.Json/ExporterVersion.cs +++ b/src/StellaOps.Concelier.Exporter.Json/ExporterVersion.cs @@ -1,7 +1,7 @@ using System; using System.Reflection; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public static class ExporterVersion { diff --git a/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs b/src/StellaOps.Concelier.Exporter.Json/IJsonExportPathResolver.cs similarity index 75% rename from src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs rename to src/StellaOps.Concelier.Exporter.Json/IJsonExportPathResolver.cs index e68af8e4..889e1b06 100644 --- a/src/StellaOps.Feedser.Exporter.Json/IJsonExportPathResolver.cs +++ b/src/StellaOps.Concelier.Exporter.Json/IJsonExportPathResolver.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public interface IJsonExportPathResolver { diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs similarity index 92% rename from src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs index c57c69a1..608833e5 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportFile.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportFile.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; /// /// Metadata describing a single file produced by the JSON exporter. diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs similarity index 90% rename from src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs index 904dc421..eaf032f3 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportJob.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportJob.cs @@ -2,9 +2,9 @@ using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonExportJob : IJob { diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportManifestWriter.cs similarity index 95% rename from src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExportManifestWriter.cs index 7a0c80d7..b8624e93 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportManifestWriter.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportManifestWriter.cs @@ -7,7 +7,7 @@ using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; internal static class JsonExportManifestWriter { diff --git a/src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs new file mode 100644 index 00000000..9d2a39b1 --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportOptions.cs @@ -0,0 +1,115 @@ +using System.Collections.Generic; +using System.IO; +using StellaOps.Cryptography; + +namespace StellaOps.Concelier.Exporter.Json; + +/// +/// Configuration for JSON exporter output paths and determinism controls. +/// +public sealed class JsonExportOptions +{ + /// + /// Root directory where exports are written. Default "exports/json". + /// + public string OutputRoot { get; set; } = Path.Combine("exports", "json"); + + /// + /// Format string applied to the export timestamp to produce the directory name. + /// + public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; + + /// + /// Optional static name for the symlink (or directory junction) pointing at the most recent export. + /// + public string LatestSymlinkName { get; set; } = "latest"; + + /// + /// When true, attempts to re-point after a successful export. + /// + public bool MaintainLatestSymlink { get; set; } = true; + + /// + /// Optional repository identifier recorded alongside export state metadata. + /// + public string? TargetRepository { get; set; } + + /// + /// Mirror distribution configuration producing aggregate bundles for downstream mirrors. + /// + public JsonMirrorOptions Mirror { get; set; } = new(); + + public sealed class JsonMirrorOptions + { + /// + /// Indicates whether mirror bundle generation is enabled. + /// + public bool Enabled { get; set; } + + /// + /// Directory name (relative to the export root) where mirror artefacts are written. + /// + public string DirectoryName { get; set; } = "mirror"; + + /// + /// Domains exposed to downstream mirrors. + /// + public IList Domains { get; } = new List(); + + /// + /// Signing configuration for mirror bundles. + /// + public JsonMirrorSigningOptions Signing { get; set; } = new(); + } + + public sealed class JsonMirrorDomainOptions + { + /// + /// Stable identifier for the mirror domain (used in URLs and directory names). + /// + public string Id { get; set; } = string.Empty; + + /// + /// Optional human-readable label for UI surfaces. + /// + public string? DisplayName { get; set; } + + /// + /// Optional advisory scheme filters (e.g. CVE, GHSA). Empty collection selects all schemes. + /// + public IList IncludeSchemes { get; } = new List(); + + /// + /// Optional provenance source filters (e.g. nvd, ghsa). Empty collection selects all sources. + /// + public IList IncludeSources { get; } = new List(); + } + + public sealed class JsonMirrorSigningOptions + { + /// + /// Indicates whether bundles should be signed. Defaults to disabled. + /// + public bool Enabled { get; set; } + + /// + /// Signing algorithm identifier (defaults to ES256). + /// + public string Algorithm { get; set; } = SignatureAlgorithms.Es256; + + /// + /// Active signing key identifier. + /// + public string KeyId { get; set; } = string.Empty; + + /// + /// Path to the private key (PEM) used for signing mirror bundles. + /// + public string KeyPath { get; set; } = string.Empty; + + /// + /// Optional crypto provider hint. When omitted the registry resolves an appropriate provider. + /// + public string? Provider { get; set; } + } +} diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs similarity index 67% rename from src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs index 0842d1a8..31f3d70b 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportResult.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportResult.cs @@ -2,8 +2,9 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonExportResult { @@ -12,24 +13,30 @@ public sealed class JsonExportResult DateTimeOffset exportedAt, IEnumerable files, int advisoryCount, - long totalBytes) + long totalBytes, + IEnumerable? advisories = null) { if (string.IsNullOrWhiteSpace(exportDirectory)) { throw new ArgumentException("Export directory must be provided.", nameof(exportDirectory)); } - ExportDirectory = exportDirectory; - ExportedAt = exportedAt; - AdvisoryCount = advisoryCount; - TotalBytes = totalBytes; - var list = (files ?? throw new ArgumentNullException(nameof(files))) .Where(static file => file is not null) .ToImmutableArray(); + var advisoryList = (advisories ?? Array.Empty()) + .Where(static advisory => advisory is not null) + .ToImmutableArray(); + + ExportDirectory = exportDirectory; + ExportedAt = exportedAt; + TotalBytes = totalBytes; + Files = list; FilePaths = list.Select(static file => file.RelativePath).ToImmutableArray(); + Advisories = advisoryList; + AdvisoryCount = advisoryList.IsDefaultOrEmpty ? advisoryCount : advisoryList.Length; } public string ExportDirectory { get; } @@ -40,6 +47,8 @@ public sealed class JsonExportResult public ImmutableArray FilePaths { get; } + public ImmutableArray Advisories { get; } + public int AdvisoryCount { get; } public long TotalBytes { get; } diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExportSnapshotBuilder.cs similarity index 91% rename from src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExportSnapshotBuilder.cs index c622486d..49c531cf 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportSnapshotBuilder.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExportSnapshotBuilder.cs @@ -5,9 +5,9 @@ using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; /// /// Writes canonical advisory snapshots into a vuln-list style directory tree with deterministic ordering. @@ -67,26 +67,27 @@ public sealed class JsonExportSnapshotBuilder Directory.CreateDirectory(exportDirectory); TrySetDirectoryTimestamp(exportDirectory, exportedAt); - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - var files = new List(); - long totalBytes = 0L; - var advisoryCount = 0; - - await foreach (var advisory in advisories.WithCancellation(cancellationToken)) - { - cancellationToken.ThrowIfCancellationRequested(); - - advisoryCount++; - var entry = Resolve(advisory); - if (!seen.Add(entry.RelativePath)) - { - throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'."); - } - - var destination = Combine(exportDirectory, entry.Segments); - var destinationDirectory = Path.GetDirectoryName(destination); - if (!string.IsNullOrEmpty(destinationDirectory)) - { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var files = new List(); + var advisoryList = new List(); + long totalBytes = 0L; + + await foreach (var advisory in advisories.WithCancellation(cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = Resolve(advisory); + if (!seen.Add(entry.RelativePath)) + { + throw new InvalidOperationException($"Multiple advisories resolved to the same path '{entry.RelativePath}'."); + } + + advisoryList.Add(entry.Advisory); + + var destination = Combine(exportDirectory, entry.Segments); + var destinationDirectory = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(destinationDirectory)) + { EnsureDirectoryExists(destinationDirectory); TrySetDirectoryTimestamp(destinationDirectory, exportedAt); } @@ -97,14 +98,14 @@ public sealed class JsonExportSnapshotBuilder File.SetLastWriteTimeUtc(destination, exportedAt.UtcDateTime); var digest = ComputeDigest(bytes); - files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest)); - totalBytes += bytes.LongLength; - } - - files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath)); - - return new JsonExportResult(exportDirectory, exportedAt, files, advisoryCount, totalBytes); - } + files.Add(new JsonExportFile(entry.RelativePath, bytes.LongLength, digest)); + totalBytes += bytes.LongLength; + } + + files.Sort(static (left, right) => string.CompareOrdinal(left.RelativePath, right.RelativePath)); + + return new JsonExportResult(exportDirectory, exportedAt, files, advisoryList.Count, totalBytes, advisoryList); + } private static async IAsyncEnumerable EnumerateAsync( IEnumerable advisories, @@ -168,10 +169,11 @@ public sealed class JsonExportSnapshotBuilder throw new ArgumentNullException(nameof(advisory)); } - var relativePath = _pathResolver.GetRelativePath(advisory); - var segments = NormalizeRelativePath(relativePath); - var normalized = string.Join('/', segments); - return new PathResolution(advisory, normalized, segments); + var normalized = CanonicalJsonSerializer.Normalize(advisory); + var relativePath = _pathResolver.GetRelativePath(normalized); + var segments = NormalizeRelativePath(relativePath); + var normalizedPath = string.Join('/', segments); + return new PathResolution(normalized, normalizedPath, segments); } private static string[] NormalizeRelativePath(string relativePath) diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs similarity index 81% rename from src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs index 67f5c655..e778eaf7 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExporterDependencyInjectionRoutine.cs @@ -5,14 +5,14 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Storage.Mongo.Exporting; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectionRoutine { - private const string ConfigurationSection = "feedser:exporters:json"; + private const string ConfigurationSection = "concelier:exporters:json"; public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) { @@ -31,14 +31,19 @@ public sealed class JsonExporterDependencyInjectionRoutine : IDependencyInjectio options.OutputRoot = Path.Combine("exports", "json"); } - if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat)) - { - options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'"; - } - }); - - services.AddSingleton(); - services.AddTransient(); + if (string.IsNullOrWhiteSpace(options.DirectoryNameFormat)) + { + options.DirectoryNameFormat = "yyyyMMdd'T'HHmmss'Z'"; + } + + if (string.IsNullOrWhiteSpace(options.Mirror.DirectoryName)) + { + options.Mirror.DirectoryName = "mirror"; + } + }); + + services.AddSingleton(); + services.AddTransient(); services.PostConfigure(options => { diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs b/src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs similarity index 83% rename from src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs index d03ab541..07e36e09 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExporterPlugin.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonExporterPlugin.cs @@ -1,9 +1,9 @@ using System; using Microsoft.Extensions.DependencyInjection; -using StellaOps.Feedser.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Advisories; using StellaOps.Plugin; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonExporterPlugin : IExporterPlugin { diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs b/src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs similarity index 75% rename from src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs rename to src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs index 84c03d48..f516e58c 100644 --- a/src/StellaOps.Feedser.Exporter.Json/JsonFeedExporter.cs +++ b/src/StellaOps.Concelier.Exporter.Json/JsonFeedExporter.cs @@ -1,14 +1,18 @@ using System; using System.Globalization; -using System.IO; -using System.Linq; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Exporting; -using StellaOps.Plugin; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Exporting; +using StellaOps.Plugin; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; public sealed class JsonFeedExporter : IFeedExporter { @@ -16,29 +20,32 @@ public sealed class JsonFeedExporter : IFeedExporter public const string ExporterId = "export:json"; private readonly IAdvisoryStore _advisoryStore; - private readonly JsonExportOptions _options; - private readonly IJsonExportPathResolver _pathResolver; - private readonly ExportStateManager _stateManager; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly string _exporterVersion; - - public JsonFeedExporter( - IAdvisoryStore advisoryStore, - IOptions options, - IJsonExportPathResolver pathResolver, - ExportStateManager stateManager, - ILogger logger, - TimeProvider? timeProvider = null) - { - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); - _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? TimeProvider.System; - _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter)); - } + private readonly JsonExportOptions _options; + private readonly IJsonExportPathResolver _pathResolver; + private readonly ExportStateManager _stateManager; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly string _exporterVersion; + private readonly IAdvisoryEventLog _eventLog; + + public JsonFeedExporter( + IAdvisoryStore advisoryStore, + IOptions options, + IJsonExportPathResolver pathResolver, + ExportStateManager stateManager, + IAdvisoryEventLog eventLog, + ILogger logger, + TimeProvider? timeProvider = null) + { + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _pathResolver = pathResolver ?? throw new ArgumentNullException(nameof(pathResolver)); + _stateManager = stateManager ?? throw new ArgumentNullException(nameof(stateManager)); + _eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _exporterVersion = ExporterVersion.GetVersion(typeof(JsonFeedExporter)); + } public string Name => ExporterName; @@ -52,11 +59,12 @@ public sealed class JsonFeedExporter : IFeedExporter var existingState = await _stateManager.GetAsync(ExporterId, cancellationToken).ConfigureAwait(false); - var builder = new JsonExportSnapshotBuilder(_options, _pathResolver); - var advisoryStream = _advisoryStore.StreamAsync(cancellationToken); - var result = await builder.WriteAsync(advisoryStream, exportedAt, exportId, cancellationToken).ConfigureAwait(false); - - var digest = ExportDigestCalculator.ComputeTreeDigest(result); + var builder = new JsonExportSnapshotBuilder(_options, _pathResolver); + var canonicalAdvisories = await MaterializeCanonicalAdvisoriesAsync(cancellationToken).ConfigureAwait(false); + var result = await builder.WriteAsync(canonicalAdvisories, exportedAt, exportId, cancellationToken).ConfigureAwait(false); + result = await JsonMirrorBundleWriter.WriteAsync(result, _options, services, _timeProvider, _logger, cancellationToken).ConfigureAwait(false); + + var digest = ExportDigestCalculator.ComputeTreeDigest(result); _logger.LogInformation( "JSON export {ExportId} wrote {FileCount} files ({Bytes} bytes) covering {AdvisoryCount} advisories with digest {Digest}", exportId, @@ -106,7 +114,34 @@ public sealed class JsonFeedExporter : IFeedExporter { TryUpdateLatestSymlink(exportRoot, result.ExportDirectory); } - } + } + + private async Task> MaterializeCanonicalAdvisoriesAsync(CancellationToken cancellationToken) + { + var keys = new SortedSet(StringComparer.OrdinalIgnoreCase); + + await foreach (var advisory in _advisoryStore.StreamAsync(cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + keys.Add(advisory.AdvisoryKey.Trim()); + } + } + + var advisories = new List(keys.Count); + foreach (var key in keys) + { + cancellationToken.ThrowIfCancellationRequested(); + var replay = await _eventLog.ReplayAsync(key, asOf: null, cancellationToken).ConfigureAwait(false); + if (!replay.Statements.IsDefaultOrEmpty) + { + advisories.Add(replay.Statements[0].Advisory); + } + } + + return advisories; + } private void TryUpdateLatestSymlink(string exportRoot, string exportDirectory) { diff --git a/src/StellaOps.Concelier.Exporter.Json/JsonMirrorBundleWriter.cs b/src/StellaOps.Concelier.Exporter.Json/JsonMirrorBundleWriter.cs new file mode 100644 index 00000000..4e976e26 --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.Json/JsonMirrorBundleWriter.cs @@ -0,0 +1,622 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Models; +using StellaOps.Cryptography; + +namespace StellaOps.Concelier.Exporter.Json; + +internal static class JsonMirrorBundleWriter +{ + private const int SchemaVersion = 1; + private const string BundleFileName = "bundle.json"; + private const string BundleSignatureFileName = "bundle.json.jws"; + private const string ManifestFileName = "manifest.json"; + private const string IndexFileName = "index.json"; + private const string SignatureMediaType = "application/vnd.stellaops.concelier.mirror-bundle+jws"; + private const string DefaultMirrorDirectoryName = "mirror"; + + private static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + + private static readonly JsonSerializerOptions HeaderSerializerOptions = new(JsonSerializerDefaults.General) + { + PropertyNamingPolicy = null, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + public static async Task WriteAsync( + JsonExportResult result, + JsonExportOptions options, + IServiceProvider services, + TimeProvider timeProvider, + ILogger logger, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(result); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(logger); + + var mirrorOptions = options.Mirror ?? new JsonExportOptions.JsonMirrorOptions(); + if (!mirrorOptions.Enabled || mirrorOptions.Domains.Count == 0) + { + return result; + } + + cancellationToken.ThrowIfCancellationRequested(); + + var exportedAtUtc = result.ExportedAt.UtcDateTime; + var mirrorDirectoryName = string.IsNullOrWhiteSpace(mirrorOptions.DirectoryName) + ? DefaultMirrorDirectoryName + : mirrorOptions.DirectoryName.Trim(); + + var mirrorRoot = Path.Combine(result.ExportDirectory, mirrorDirectoryName); + Directory.CreateDirectory(mirrorRoot); + TrySetDirectoryTimestamp(mirrorRoot, exportedAtUtc); + + var advisories = result.Advisories.IsDefaultOrEmpty + ? Array.Empty() + : result.Advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + + var signingContext = PrepareSigningContext(mirrorOptions.Signing, services, timeProvider, logger); + var additionalFiles = new List(); + var domainEntries = new List(); + + foreach (var domainOption in mirrorOptions.Domains) + { + cancellationToken.ThrowIfCancellationRequested(); + if (domainOption is null) + { + logger.LogWarning("Encountered null mirror domain configuration; skipping."); + continue; + } + + var domainId = (domainOption.Id ?? string.Empty).Trim(); + if (domainId.Length == 0) + { + logger.LogWarning("Skipping mirror domain with empty id."); + continue; + } + + var schemeFilter = CreateFilterSet(domainOption.IncludeSchemes); + var sourceFilter = CreateFilterSet(domainOption.IncludeSources); + var domainAdvisories = advisories + .Where(advisory => MatchesFilters(advisory, schemeFilter, sourceFilter)) + .ToArray(); + + var sources = BuildSourceSummaries(domainAdvisories); + var domainDisplayName = string.IsNullOrWhiteSpace(domainOption.DisplayName) + ? domainId + : domainOption.DisplayName!.Trim(); + + var domainDirectory = Path.Combine(mirrorRoot, domainId); + Directory.CreateDirectory(domainDirectory); + TrySetDirectoryTimestamp(domainDirectory, exportedAtUtc); + + var bundleDocument = new MirrorDomainBundleDocument( + SchemaVersion, + result.ExportedAt, + options.TargetRepository, + domainId, + domainDisplayName, + domainAdvisories.Length, + domainAdvisories, + sources); + + var bundleBytes = Serialize(bundleDocument); + var bundlePath = Path.Combine(domainDirectory, BundleFileName); + await WriteFileAsync(bundlePath, bundleBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false); + + var bundleRelativePath = ToRelativePath(result.ExportDirectory, bundlePath); + var bundleDigest = ComputeDigest(bundleBytes); + var bundleLength = (long)bundleBytes.LongLength; + additionalFiles.Add(new JsonExportFile(bundleRelativePath, bundleLength, bundleDigest)); + + MirrorSignatureDescriptor? signatureDescriptor = null; + if (signingContext is not null) + { + var (signatureValue, signedAt) = await CreateSignatureAsync( + signingContext, + bundleBytes, + timeProvider, + cancellationToken) + .ConfigureAwait(false); + + var signatureBytes = Utf8NoBom.GetBytes(signatureValue); + var signaturePath = Path.Combine(domainDirectory, BundleSignatureFileName); + await WriteFileAsync(signaturePath, signatureBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false); + + var signatureRelativePath = ToRelativePath(result.ExportDirectory, signaturePath); + var signatureDigest = ComputeDigest(signatureBytes); + var signatureLength = (long)signatureBytes.LongLength; + additionalFiles.Add(new JsonExportFile(signatureRelativePath, signatureLength, signatureDigest)); + + signatureDescriptor = new MirrorSignatureDescriptor( + signatureRelativePath, + signingContext.Algorithm, + signingContext.KeyId, + signingContext.Provider, + signedAt); + } + + var bundleDescriptor = new MirrorFileDescriptor(bundleRelativePath, bundleLength, bundleDigest, signatureDescriptor); + + var manifestDocument = new MirrorDomainManifestDocument( + SchemaVersion, + result.ExportedAt, + domainId, + domainDisplayName, + domainAdvisories.Length, + sources, + bundleDescriptor); + + var manifestBytes = Serialize(manifestDocument); + var manifestPath = Path.Combine(domainDirectory, ManifestFileName); + await WriteFileAsync(manifestPath, manifestBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false); + + var manifestRelativePath = ToRelativePath(result.ExportDirectory, manifestPath); + var manifestDigest = ComputeDigest(manifestBytes); + var manifestLength = (long)manifestBytes.LongLength; + additionalFiles.Add(new JsonExportFile(manifestRelativePath, manifestLength, manifestDigest)); + + var manifestDescriptor = new MirrorFileDescriptor(manifestRelativePath, manifestLength, manifestDigest, null); + + domainEntries.Add(new MirrorIndexDomainEntry( + domainId, + domainDisplayName, + domainAdvisories.Length, + manifestDescriptor, + bundleDescriptor, + sources)); + } + + domainEntries.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId)); + + var indexDocument = new MirrorIndexDocument( + SchemaVersion, + result.ExportedAt, + options.TargetRepository, + domainEntries); + + var indexBytes = Serialize(indexDocument); + var indexPath = Path.Combine(mirrorRoot, IndexFileName); + await WriteFileAsync(indexPath, indexBytes, exportedAtUtc, cancellationToken).ConfigureAwait(false); + + var indexRelativePath = ToRelativePath(result.ExportDirectory, indexPath); + var indexDigest = ComputeDigest(indexBytes); + var indexLength = (long)indexBytes.LongLength; + additionalFiles.Add(new JsonExportFile(indexRelativePath, indexLength, indexDigest)); + + logger.LogInformation( + "Generated {DomainCount} Concelier mirror domain bundle(s) under {MirrorRoot}.", + domainEntries.Count, + mirrorDirectoryName); + + var combinedFiles = new List(result.Files.Length + additionalFiles.Count); + combinedFiles.AddRange(result.Files); + combinedFiles.AddRange(additionalFiles); + + var combinedTotalBytes = checked(result.TotalBytes + additionalFiles.Sum(static file => file.Length)); + + return new JsonExportResult( + result.ExportDirectory, + result.ExportedAt, + combinedFiles, + result.AdvisoryCount, + combinedTotalBytes, + result.Advisories); + } + + private static JsonMirrorSigningContext? PrepareSigningContext( + JsonExportOptions.JsonMirrorSigningOptions signingOptions, + IServiceProvider services, + TimeProvider timeProvider, + ILogger logger) + { + if (signingOptions is null || !signingOptions.Enabled) + { + return null; + } + + var algorithm = string.IsNullOrWhiteSpace(signingOptions.Algorithm) + ? SignatureAlgorithms.Es256 + : signingOptions.Algorithm.Trim(); + var keyId = (signingOptions.KeyId ?? string.Empty).Trim(); + if (keyId.Length == 0) + { + throw new InvalidOperationException("Mirror signing requires mirror.signing.keyId to be configured."); + } + + var registry = services.GetService() + ?? throw new InvalidOperationException("Mirror signing requires ICryptoProviderRegistry to be registered."); + + var providerHint = signingOptions.Provider?.Trim(); + var keyReference = new CryptoKeyReference(keyId, providerHint); + + CryptoSignerResolution resolved; + try + { + resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, providerHint); + } + catch (KeyNotFoundException) + { + var provider = ResolveProvider(registry, algorithm, providerHint); + var signingKey = LoadSigningKey(signingOptions, provider, services, timeProvider, algorithm); + provider.UpsertSigningKey(signingKey); + resolved = registry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, provider.Name); + } + + logger.LogDebug( + "Mirror signing configured with key {KeyId} via provider {Provider} using {Algorithm}.", + resolved.Signer.KeyId, + resolved.ProviderName, + algorithm); + + return new JsonMirrorSigningContext(resolved.Signer, algorithm, resolved.Signer.KeyId, resolved.ProviderName); + } + + private static ICryptoProvider ResolveProvider(ICryptoProviderRegistry registry, string algorithm, string? providerHint) + { + if (!string.IsNullOrWhiteSpace(providerHint) && registry.TryResolve(providerHint, out var hinted)) + { + if (!hinted.Supports(CryptoCapability.Signing, algorithm)) + { + throw new InvalidOperationException( + $"Crypto provider '{providerHint}' does not support signing algorithm '{algorithm}'."); + } + + return hinted; + } + + return registry.ResolveOrThrow(CryptoCapability.Signing, algorithm); + } + + private static CryptoSigningKey LoadSigningKey( + JsonExportOptions.JsonMirrorSigningOptions signingOptions, + ICryptoProvider provider, + IServiceProvider services, + TimeProvider timeProvider, + string algorithm) + { + var keyPath = (signingOptions.KeyPath ?? string.Empty).Trim(); + if (keyPath.Length == 0) + { + throw new InvalidOperationException("Mirror signing requires mirror.signing.keyPath to be configured."); + } + + var environment = services.GetService(); + var basePath = environment?.ContentRootPath ?? AppContext.BaseDirectory; + var resolvedPath = Path.IsPathRooted(keyPath) + ? keyPath + : Path.GetFullPath(Path.Combine(basePath, keyPath)); + + if (!File.Exists(resolvedPath)) + { + throw new FileNotFoundException($"Mirror signing key '{signingOptions.KeyId}' not found.", resolvedPath); + } + + var pem = File.ReadAllText(resolvedPath); + using var ecdsa = ECDsa.Create(); + try + { + ecdsa.ImportFromPem(pem); + } + catch (CryptographicException ex) + { + throw new InvalidOperationException("Failed to import mirror signing key. Ensure the PEM contains an EC private key.", ex); + } + + var parameters = ecdsa.ExportParameters(includePrivateParameters: true); + return new CryptoSigningKey( + new CryptoKeyReference(signingOptions.KeyId, provider.Name), + algorithm, + in parameters, + timeProvider.GetUtcNow()); + } + + private static async Task<(string Value, DateTimeOffset SignedAt)> CreateSignatureAsync( + JsonMirrorSigningContext context, + ReadOnlyMemory payload, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var header = new Dictionary + { + ["alg"] = context.Algorithm, + ["kid"] = context.KeyId, + ["typ"] = SignatureMediaType, + ["b64"] = false, + ["crit"] = new[] { "b64" } + }; + + if (!string.IsNullOrWhiteSpace(context.Provider)) + { + header["provider"] = context.Provider; + } + + var headerJson = JsonSerializer.Serialize(header, HeaderSerializerOptions); + var protectedHeader = Base64UrlEncode(Utf8NoBom.GetBytes(headerJson)); + var signingInputLength = protectedHeader.Length + 1 + payload.Length; + var buffer = ArrayPool.Shared.Rent(signingInputLength); + + try + { + var headerBytes = Encoding.ASCII.GetBytes(protectedHeader); + Buffer.BlockCopy(headerBytes, 0, buffer, 0, headerBytes.Length); + buffer[headerBytes.Length] = (byte)'.'; + var payloadArray = payload.ToArray(); + Buffer.BlockCopy(payloadArray, 0, buffer, headerBytes.Length + 1, payloadArray.Length); + + var signingInput = new ReadOnlyMemory(buffer, 0, signingInputLength); + var signatureBytes = await context.Signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false); + var encodedSignature = Base64UrlEncode(signatureBytes); + var signedAt = timeProvider.GetUtcNow(); + return (string.Concat(protectedHeader, "..", encodedSignature), signedAt); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static IReadOnlyList BuildSourceSummaries(IReadOnlyList advisories) + { + var builders = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var advisory in advisories) + { + var counted = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var provenance in advisory.Provenance) + { + if (string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + var source = provenance.Source.Trim(); + if (!builders.TryGetValue(source, out var accumulator)) + { + accumulator = new SourceAccumulator(); + builders[source] = accumulator; + } + + accumulator.Record(provenance.RecordedAt); + if (counted.Add(source)) + { + accumulator.IncrementAdvisoryCount(); + } + } + } + + return builders + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .Select(pair => new JsonMirrorSourceSummary( + pair.Key, + pair.Value.FirstRecordedAt, + pair.Value.LastRecordedAt, + pair.Value.AdvisoryCount)) + .ToArray(); + } + + private static HashSet? CreateFilterSet(IList? values) + { + if (values is null || values.Count == 0) + { + return null; + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + set.Add(value.Trim()); + } + + return set.Count == 0 ? null : set; + } + + private static bool MatchesFilters(Advisory advisory, HashSet? schemeFilter, HashSet? sourceFilter) + { + if (schemeFilter is not null) + { + var scheme = ExtractScheme(advisory.AdvisoryKey); + if (!schemeFilter.Contains(scheme)) + { + return false; + } + } + + if (sourceFilter is not null) + { + var hasSource = advisory.Provenance.Any(provenance => + !string.IsNullOrWhiteSpace(provenance.Source) && + sourceFilter.Contains(provenance.Source.Trim())); + + if (!hasSource) + { + return false; + } + } + + return true; + } + + private static string ExtractScheme(string advisoryKey) + { + if (string.IsNullOrWhiteSpace(advisoryKey)) + { + return string.Empty; + } + + var trimmed = advisoryKey.Trim(); + var separatorIndex = trimmed.IndexOf(':'); + return separatorIndex <= 0 ? trimmed : trimmed[..separatorIndex]; + } + + private static byte[] Serialize(T value) + { + var json = CanonicalJsonSerializer.SerializeIndented(value); + return Utf8NoBom.GetBytes(json); + } + + private static async Task WriteFileAsync(string path, byte[] content, DateTime exportedAtUtc, CancellationToken cancellationToken) + { + await File.WriteAllBytesAsync(path, content, cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, exportedAtUtc); + } + + private static string ToRelativePath(string root, string fullPath) + { + var relative = Path.GetRelativePath(root, fullPath); + return relative.Replace(Path.DirectorySeparatorChar, '/'); + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static void TrySetDirectoryTimestamp(string directory, DateTime exportedAtUtc) + { + try + { + Directory.SetLastWriteTimeUtc(directory, exportedAtUtc); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + catch (PlatformNotSupportedException) + { + } + } + + private static string Base64UrlEncode(ReadOnlySpan value) + { + var encoded = Convert.ToBase64String(value); + var builder = new StringBuilder(encoded.Length); + foreach (var ch in encoded) + { + switch (ch) + { + case '+': + builder.Append('-'); + break; + case '/': + builder.Append('_'); + break; + case '=': + break; + default: + builder.Append(ch); + break; + } + } + + return builder.ToString(); + } + + private sealed record JsonMirrorSigningContext(ICryptoSigner Signer, string Algorithm, string KeyId, string Provider); + + private sealed record MirrorIndexDocument( + int SchemaVersion, + DateTimeOffset GeneratedAt, + string? TargetRepository, + IReadOnlyList Domains); + + private sealed record MirrorIndexDomainEntry( + string DomainId, + string DisplayName, + int AdvisoryCount, + MirrorFileDescriptor Manifest, + MirrorFileDescriptor Bundle, + IReadOnlyList Sources); + + private sealed record MirrorDomainManifestDocument( + int SchemaVersion, + DateTimeOffset GeneratedAt, + string DomainId, + string DisplayName, + int AdvisoryCount, + IReadOnlyList Sources, + MirrorFileDescriptor Bundle); + + private sealed record MirrorDomainBundleDocument( + int SchemaVersion, + DateTimeOffset GeneratedAt, + string? TargetRepository, + string DomainId, + string DisplayName, + int AdvisoryCount, + IReadOnlyList Advisories, + IReadOnlyList Sources); + + private sealed record MirrorFileDescriptor( + string Path, + long SizeBytes, + string Digest, + MirrorSignatureDescriptor? Signature); + + private sealed record MirrorSignatureDescriptor( + string Path, + string Algorithm, + string KeyId, + string Provider, + DateTimeOffset SignedAt); + + private sealed record JsonMirrorSourceSummary( + string Source, + DateTimeOffset? FirstRecordedAt, + DateTimeOffset? LastRecordedAt, + int AdvisoryCount); + + private sealed class SourceAccumulator + { + public DateTimeOffset? FirstRecordedAt { get; private set; } + + public DateTimeOffset? LastRecordedAt { get; private set; } + + public int AdvisoryCount { get; private set; } + + public void Record(DateTimeOffset recordedAt) + { + var normalized = recordedAt.ToUniversalTime(); + if (FirstRecordedAt is null || normalized < FirstRecordedAt.Value) + { + FirstRecordedAt = normalized; + } + + if (LastRecordedAt is null || normalized > LastRecordedAt.Value) + { + LastRecordedAt = normalized; + } + } + + public void IncrementAdvisoryCount() + { + AdvisoryCount++; + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj b/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj similarity index 58% rename from src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj rename to src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj index fe28b621..967a1a9e 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/StellaOps.Feedser.Exporter.TrivyDb.csproj +++ b/src/StellaOps.Concelier.Exporter.Json/StellaOps.Concelier.Exporter.Json.csproj @@ -1,22 +1,24 @@ - - - - net10.0 - preview - enable - enable - true - - - - - - + + + + net10.0 + preview + enable + enable + true + + + + + + + - + + - \ No newline at end of file + diff --git a/src/StellaOps.Feedser.Exporter.Json/TASKS.md b/src/StellaOps.Concelier.Exporter.Json/TASKS.md similarity index 73% rename from src/StellaOps.Feedser.Exporter.Json/TASKS.md rename to src/StellaOps.Concelier.Exporter.Json/TASKS.md index 04f14621..e52740fa 100644 --- a/src/StellaOps.Feedser.Exporter.Json/TASKS.md +++ b/src/StellaOps.Concelier.Exporter.Json/TASKS.md @@ -9,4 +9,5 @@ |Parity smoke vs upstream vuln-list|QA|Exporters|DONE – `JsonExporterParitySmokeTests` covers common ecosystems against vuln-list layout.| |Stream advisories during export|BE-Export|Storage.Mongo|DONE – exporter + streaming-only test ensures single enumeration and per-file digest capture.| |Emit export manifest with digest metadata|BE-Export|Exporters|DONE – manifest now includes per-file digests/sizes alongside tree digest.| -|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Feedser.Exporter.Json.Tests` run 2025-10-15 for regression coverage.| +|Surface new advisory fields (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – JSON exporter validated with new fixtures ensuring description/CWEs/canonical metric are preserved in outputs; `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests` run 2025-10-15 for regression coverage.| +|CONCELIER-EXPORT-08-201 – Mirror bundle + domain manifest|Team Concelier Export|FEEDCORE-ENGINE-07-001|DONE (2025-10-19) – Mirror bundle writer emits domain aggregates + manifests with cosign-compatible JWS signatures; index/tests updated via `dotnet test src/StellaOps.Concelier.Exporter.Json.Tests/StellaOps.Concelier.Exporter.Json.Tests.csproj` (2025-10-19).| diff --git a/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs b/src/StellaOps.Concelier.Exporter.Json/VulnListJsonExportPathResolver.cs similarity index 96% rename from src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs rename to src/StellaOps.Concelier.Exporter.Json/VulnListJsonExportPathResolver.cs index 31747aa3..cc4aeb7e 100644 --- a/src/StellaOps.Feedser.Exporter.Json/VulnListJsonExportPathResolver.cs +++ b/src/StellaOps.Concelier.Exporter.Json/VulnListJsonExportPathResolver.cs @@ -3,10 +3,10 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Identifiers; -namespace StellaOps.Feedser.Exporter.Json; +namespace StellaOps.Concelier.Exporter.Json; /// /// Path resolver approximating the directory layout used by aquasecurity/vuln-list. diff --git a/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj new file mode 100644 index 00000000..a138b6ef --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs similarity index 90% rename from src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs index 5b58d2ea..9dc78247 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbExportPlannerTests.cs @@ -1,86 +1,86 @@ -using System; -using StellaOps.Feedser.Exporter.TrivyDb; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; - -public sealed class TrivyDbExportPlannerTests -{ - [Fact] - public void CreatePlan_ReturnsFullWhenStateMissing() - { - var planner = new TrivyDbExportPlanner(); - var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; - var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd", manifest); - - Assert.Equal(TrivyDbExportMode.Full, plan.Mode); - Assert.Equal("sha256:abcd", plan.TreeDigest); - Assert.Null(plan.BaseExportId); - Assert.Null(plan.BaseManifestDigest); - Assert.True(plan.ResetBaseline); - Assert.Equal(manifest, plan.Manifest); - } - - [Fact] - public void CreatePlan_ReturnsSkipWhenCursorMatches() - { - var planner = new TrivyDbExportPlanner(); - var existingManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; - var state = new ExportStateRecord( - Id: TrivyDbFeedExporter.ExporterId, - BaseExportId: "20240810T000000Z", - BaseDigest: "sha256:base", - LastFullDigest: "sha256:base", - LastDeltaDigest: null, - ExportCursor: "sha256:unchanged", - TargetRepository: "feedser/trivy", - ExporterVersion: "1.0", - UpdatedAt: DateTimeOffset.UtcNow, - Files: existingManifest); - - var plan = planner.CreatePlan(state, "sha256:unchanged", existingManifest); - - Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); - Assert.Equal("sha256:unchanged", plan.TreeDigest); - Assert.Equal("20240810T000000Z", plan.BaseExportId); - Assert.Equal("sha256:base", plan.BaseManifestDigest); - Assert.False(plan.ResetBaseline); - Assert.Empty(plan.ChangedFiles); - Assert.Empty(plan.RemovedPaths); - } - - [Fact] - public void CreatePlan_ReturnsFullWhenCursorDiffers() - { - var planner = new TrivyDbExportPlanner(); - var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; - var state = new ExportStateRecord( - Id: TrivyDbFeedExporter.ExporterId, - BaseExportId: "20240810T000000Z", - BaseDigest: "sha256:base", - LastFullDigest: "sha256:base", - LastDeltaDigest: null, - ExportCursor: "sha256:old", - TargetRepository: "feedser/trivy", - ExporterVersion: "1.0", - UpdatedAt: DateTimeOffset.UtcNow, - Files: manifest); - - var newManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:b") }; - var plan = planner.CreatePlan(state, "sha256:new", newManifest); - - Assert.Equal(TrivyDbExportMode.Delta, plan.Mode); - Assert.Equal("sha256:new", plan.TreeDigest); - Assert.Equal("20240810T000000Z", plan.BaseExportId); - Assert.Equal("sha256:base", plan.BaseManifestDigest); - Assert.False(plan.ResetBaseline); - Assert.Single(plan.ChangedFiles); - - var deltaState = state with { LastDeltaDigest = "sha256:delta" }; - var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer", newManifest); - - Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); - Assert.True(deltaPlan.ResetBaseline); - Assert.Equal(deltaPlan.Manifest, deltaPlan.ChangedFiles); - } -} +using System; +using StellaOps.Concelier.Exporter.TrivyDb; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +namespace StellaOps.Concelier.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbExportPlannerTests +{ + [Fact] + public void CreatePlan_ReturnsFullWhenStateMissing() + { + var planner = new TrivyDbExportPlanner(); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var plan = planner.CreatePlan(existingState: null, treeDigest: "sha256:abcd", manifest); + + Assert.Equal(TrivyDbExportMode.Full, plan.Mode); + Assert.Equal("sha256:abcd", plan.TreeDigest); + Assert.Null(plan.BaseExportId); + Assert.Null(plan.BaseManifestDigest); + Assert.True(plan.ResetBaseline); + Assert.Equal(manifest, plan.Manifest); + } + + [Fact] + public void CreatePlan_ReturnsSkipWhenCursorMatches() + { + var planner = new TrivyDbExportPlanner(); + var existingManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:unchanged", + TargetRepository: "concelier/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow, + Files: existingManifest); + + var plan = planner.CreatePlan(state, "sha256:unchanged", existingManifest); + + Assert.Equal(TrivyDbExportMode.Skip, plan.Mode); + Assert.Equal("sha256:unchanged", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); + Assert.Empty(plan.ChangedFiles); + Assert.Empty(plan.RemovedPaths); + } + + [Fact] + public void CreatePlan_ReturnsFullWhenCursorDiffers() + { + var planner = new TrivyDbExportPlanner(); + var manifest = new[] { new ExportFileRecord("path.json", 10, "sha256:a") }; + var state = new ExportStateRecord( + Id: TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240810T000000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: null, + ExportCursor: "sha256:old", + TargetRepository: "concelier/trivy", + ExporterVersion: "1.0", + UpdatedAt: DateTimeOffset.UtcNow, + Files: manifest); + + var newManifest = new[] { new ExportFileRecord("path.json", 10, "sha256:b") }; + var plan = planner.CreatePlan(state, "sha256:new", newManifest); + + Assert.Equal(TrivyDbExportMode.Delta, plan.Mode); + Assert.Equal("sha256:new", plan.TreeDigest); + Assert.Equal("20240810T000000Z", plan.BaseExportId); + Assert.Equal("sha256:base", plan.BaseManifestDigest); + Assert.False(plan.ResetBaseline); + Assert.Single(plan.ChangedFiles); + + var deltaState = state with { LastDeltaDigest = "sha256:delta" }; + var deltaPlan = planner.CreatePlan(deltaState, "sha256:newer", newManifest); + + Assert.Equal(TrivyDbExportMode.Full, deltaPlan.Mode); + Assert.True(deltaPlan.ResetBaseline); + Assert.Equal(deltaPlan.Manifest, deltaPlan.ChangedFiles); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs similarity index 80% rename from src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs index 0b97a963..60fc0c9e 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbFeedExporterTests.cs @@ -1,1052 +1,1214 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Exporter.TrivyDb; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; - -public sealed class TrivyDbFeedExporterTests : IDisposable -{ - private readonly string _root; - private readonly string _jsonRoot; - - public TrivyDbFeedExporterTests() - { - _root = Directory.CreateTempSubdirectory("feedser-trivy-exporter-tests").FullName; - _jsonRoot = Path.Combine(_root, "tree"); - } - - [Fact] - public async Task ExportAsync_SortsAdvisoriesByKeyDeterministically() - { - var advisoryB = CreateSampleAdvisory("CVE-2024-1002", "Second advisory"); - var advisoryA = CreateSampleAdvisory("CVE-2024-1001", "First advisory"); - - var advisoryStore = new StubAdvisoryStore(advisoryB, advisoryA); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - KeepWorkingTree = false, - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-20T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-09-21T00:00:00Z", - UpdatedAt = "2024-09-20T00:00:00Z", - }); - - var recordingBuilder = new RecordingTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - recordingBuilder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var paths = recordingBuilder.LastRelativePaths; - Assert.NotNull(paths); - - var sorted = paths!.OrderBy(static p => p, StringComparer.Ordinal).ToArray(); - Assert.Equal(sorted, paths); - - advisoryStore.SetAdvisories(advisoryA, advisoryB); - timeProvider.Advance(TimeSpan.FromMinutes(7)); - await exporter.ExportAsync(provider, CancellationToken.None); - - var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(record); - Assert.Equal("20240920T000000Z", record!.BaseExportId); - Assert.Single(recordingBuilder.ManifestDigests); - } - - [Fact] - public async Task ExportAsync_SmallDatasetProducesDeterministicOciLayout() - { - var advisories = new[] - { - CreateSampleAdvisory("CVE-2024-3000", "Demo advisory 1"), - CreateSampleAdvisory("CVE-2024-3001", "Demo advisory 2"), - }; - - var run1 = await RunDeterministicExportAsync(advisories); - var run2 = await RunDeterministicExportAsync(advisories); - - Assert.Equal(run1.ManifestDigest, run2.ManifestDigest); - Assert.Equal(run1.IndexJson, run2.IndexJson); - Assert.Equal(run1.MetadataJson, run2.MetadataJson); - Assert.Equal(run1.ManifestJson, run2.ManifestJson); - - var digests1 = run1.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); - var digests2 = run2.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); - Assert.Equal(digests1, digests2); - - foreach (var digest in digests1) - { - Assert.True(run2.Blobs.TryGetValue(digest, out var other), $"Missing digest {digest} in second run"); - Assert.True(run1.Blobs[digest].SequenceEqual(other), $"Blob {digest} differs between runs"); - } - - using var metadataDoc = JsonDocument.Parse(run1.MetadataJson); - Assert.Equal(2, metadataDoc.RootElement.GetProperty("advisoryCount").GetInt32()); - - using var manifestDoc = JsonDocument.Parse(run1.ManifestJson); - Assert.Equal(TrivyDbMediaTypes.TrivyConfig, manifestDoc.RootElement.GetProperty("config").GetProperty("mediaType").GetString()); - var layer = manifestDoc.RootElement.GetProperty("layers")[0]; - Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.GetProperty("mediaType").GetString()); - } - - [Fact] - public void ExportOptions_GetExportRoot_NormalizesRelativeRoot() - { - var options = new TrivyDbExportOptions - { - OutputRoot = Path.Combine("..", "exports", "trivy-test"), - }; - - var exportId = "20240901T000000Z"; - var path = options.GetExportRoot(exportId); - - Assert.True(Path.IsPathRooted(path)); - Assert.EndsWith(Path.Combine("exports", "trivy-test", exportId), path, StringComparison.Ordinal); - } - - [Fact] - public async Task ExportAsync_PersistsStateAndSkipsWhenDigestUnchanged() - { - var advisory = CreateSampleAdvisory(); - var advisoryStore = new StubAdvisoryStore(advisory); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - KeepWorkingTree = false, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-01T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-09-02T00:00:00Z", - UpdatedAt = "2024-09-01T00:00:00Z", - }); - var builder = new StubTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(record); - Assert.Equal("20240901T000000Z", record!.BaseExportId); - Assert.False(string.IsNullOrEmpty(record.ExportCursor)); - - var baseExportId = record.BaseExportId ?? string.Empty; - Assert.False(string.IsNullOrEmpty(baseExportId)); - var firstExportDirectory = Path.Combine(_root, baseExportId); - Assert.True(Directory.Exists(firstExportDirectory)); - - timeProvider.Advance(TimeSpan.FromMinutes(5)); - await exporter.ExportAsync(provider, CancellationToken.None); - - var updatedRecord = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(updatedRecord); - Assert.Equal(record.UpdatedAt, updatedRecord!.UpdatedAt); - Assert.Equal(record.LastFullDigest, updatedRecord.LastFullDigest); - - var skippedExportDirectory = Path.Combine(_root, "20240901T000500Z"); - Assert.False(Directory.Exists(skippedExportDirectory)); - - Assert.Empty(orasPusher.Pushes); - } - - [Fact] - public async Task ExportAsync_CreatesOfflineBundle() - { - var advisory = CreateSampleAdvisory(); - var advisoryStore = new StubAdvisoryStore(advisory); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - KeepWorkingTree = false, - OfflineBundle = new TrivyDbOfflineBundleOptions - { - Enabled = true, - FileName = "{exportId}.bundle.tar.gz", - }, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-15T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-09-16T00:00:00Z", - UpdatedAt = "2024-09-15T00:00:00Z", - }); - var builder = new StubTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var exportId = "20240915T000000Z"; - var bundlePath = Path.Combine(_root, $"{exportId}.bundle.tar.gz"); - Assert.True(File.Exists(bundlePath)); - Assert.Empty(orasPusher.Pushes); - } - - [Fact] - public async Task ExportAsync_SkipsOrasPushWhenDeltaPublishingDisabled() - { - var initial = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles"); - var updated = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles delta"); - var advisoryStore = new StubAdvisoryStore(initial); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - KeepWorkingTree = true, - }; - - optionsValue.Oras.Enabled = true; - optionsValue.Oras.PublishFull = false; - optionsValue.Oras.PublishDelta = false; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-20T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-10-21T00:00:00Z", - UpdatedAt = "2024-10-20T00:00:00Z", - }); - var builder = new StubTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - advisoryStore.SetAdvisories(updated); - timeProvider.Advance(TimeSpan.FromMinutes(15)); - await exporter.ExportAsync(provider, CancellationToken.None); - - Assert.Empty(orasPusher.Pushes); - } - - [Fact] - public async Task ExportAsync_SkipsOfflineBundleForDeltaWhenDisabled() - { - var initial = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles"); - var updated = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles updated"); - var advisoryStore = new StubAdvisoryStore(initial); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - KeepWorkingTree = true, - OfflineBundle = new TrivyDbOfflineBundleOptions - { - Enabled = true, - IncludeFull = true, - IncludeDelta = false, - FileName = "{exportId}.bundle.tar.gz", - }, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-21T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-10-22T00:00:00Z", - UpdatedAt = "2024-10-21T00:00:00Z", - }); - var builder = new StubTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var fullExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - var fullBundlePath = Path.Combine(_root, $"{fullExportId}.bundle.tar.gz"); - Assert.True(File.Exists(fullBundlePath)); - - advisoryStore.SetAdvisories(updated); - timeProvider.Advance(TimeSpan.FromMinutes(10)); - await exporter.ExportAsync(provider, CancellationToken.None); - - var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - var deltaBundlePath = Path.Combine(_root, $"{deltaExportId}.bundle.tar.gz"); - Assert.False(File.Exists(deltaBundlePath)); - } - - [Fact] - public async Task ExportAsync_ResetsBaselineWhenDeltaChainExists() - { - var advisory = CreateSampleAdvisory("CVE-2024-5000", "Baseline reset"); - var advisoryStore = new StubAdvisoryStore(advisory); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - KeepWorkingTree = false, - TargetRepository = "registry.example/trivy", - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-22T00:00:00Z", CultureInfo.InvariantCulture)); - var existingRecord = new ExportStateRecord( - TrivyDbFeedExporter.ExporterId, - BaseExportId: "20240919T120000Z", - BaseDigest: "sha256:base", - LastFullDigest: "sha256:base", - LastDeltaDigest: "sha256:delta", - ExportCursor: "sha256:old", - TargetRepository: "registry.example/trivy", - ExporterVersion: "0.9.0", - UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30), - Files: Array.Empty()); - await stateStore.UpsertAsync(existingRecord, CancellationToken.None); - - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-09-23T00:00:00Z", - UpdatedAt = "2024-09-22T00:00:00Z", - }); - var builder = new StubTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var updated = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(updated); - Assert.Equal("20240922T000000Z", updated!.BaseExportId); - Assert.Equal(updated.BaseDigest, updated.LastFullDigest); - Assert.Null(updated.LastDeltaDigest); - Assert.NotEqual("sha256:old", updated.ExportCursor); - Assert.Equal("registry.example/trivy", updated.TargetRepository); - Assert.NotEmpty(updated.Files); - } - - [Fact] - public async Task ExportAsync_DeltaSequencePromotesBaselineReset() - { - var baseline = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory"); - var firstDelta = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory updated"); - var secondDelta = CreateSampleAdvisory("CVE-2024-8200", "New advisory triggers full rebuild"); - - var advisoryStore = new StubAdvisoryStore(baseline); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - KeepWorkingTree = true, - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-01T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-11-02T00:00:00Z", - UpdatedAt = "2024-11-01T00:00:00Z", - }); - var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - - var initialExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - await exporter.ExportAsync(provider, CancellationToken.None); - - var initialLayout = Path.Combine(optionsValue.OutputRoot, initialExportId); - var initialMetadata = ReadMetadata(Path.Combine(initialLayout, "metadata.json")); - Assert.Equal("full", initialMetadata.Mode); - var initialManifestDigest = ReadManifestDigest(initialLayout); - - advisoryStore.SetAdvisories(firstDelta); - timeProvider.Advance(TimeSpan.FromMinutes(15)); - var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - await exporter.ExportAsync(provider, CancellationToken.None); - - var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); - var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); - Assert.Equal("delta", deltaMetadata.Mode); - Assert.Equal(initialExportId, deltaMetadata.BaseExportId); - Assert.Equal(initialManifestDigest, deltaMetadata.BaseManifestDigest); - Assert.True(deltaMetadata.DeltaChangedCount > 0); - - var reusedManifestPath = Path.Combine(deltaLayout, "blobs", "sha256", initialManifestDigest[7..]); - Assert.True(File.Exists(reusedManifestPath)); - - advisoryStore.SetAdvisories(secondDelta); - timeProvider.Advance(TimeSpan.FromMinutes(15)); - var finalExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - await exporter.ExportAsync(provider, CancellationToken.None); - - var finalLayout = Path.Combine(optionsValue.OutputRoot, finalExportId); - var finalMetadata = ReadMetadata(Path.Combine(finalLayout, "metadata.json")); - Assert.Equal("full", finalMetadata.Mode); - Assert.True(finalMetadata.ResetBaseline); - - var state = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(state); - Assert.Null(state!.LastDeltaDigest); - Assert.Equal(finalExportId, state.BaseExportId); - } - - [Fact] - public async Task ExportAsync_DeltaReusesBaseLayerOnDisk() - { - var baseline = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse baseline"); - var delta = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse delta"); - - var advisoryStore = new StubAdvisoryStore(baseline); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = _root, - ReferencePrefix = "example/trivy", - KeepWorkingTree = true, - Json = new JsonExportOptions - { - OutputRoot = _jsonRoot, - MaintainLatestSymlink = false, - }, - }; - - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-05T00:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-11-06T00:00:00Z", - UpdatedAt = "2024-11-05T00:00:00Z", - }); - var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - - var baselineExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - await exporter.ExportAsync(provider, CancellationToken.None); - - var baselineLayout = Path.Combine(optionsValue.OutputRoot, baselineExportId); - var baselineManifestDigest = ReadManifestDigest(baselineLayout); - var baselineLayerDigests = ReadManifestLayerDigests(baselineLayout, baselineManifestDigest); - var baselineLayerDigest = Assert.Single(baselineLayerDigests); - var baselineLayerPath = Path.Combine(baselineLayout, "blobs", "sha256", baselineLayerDigest[7..]); - var baselineLayerBytes = File.ReadAllBytes(baselineLayerPath); - - advisoryStore.SetAdvisories(delta); - timeProvider.Advance(TimeSpan.FromMinutes(30)); - var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - await exporter.ExportAsync(provider, CancellationToken.None); - - var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); - var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); - Assert.Equal("delta", deltaMetadata.Mode); - Assert.Equal(baselineExportId, deltaMetadata.BaseExportId); - Assert.Equal(baselineManifestDigest, deltaMetadata.BaseManifestDigest); - Assert.True(deltaMetadata.DeltaChangedCount > 0); - - var deltaManifestDigest = ReadManifestDigest(deltaLayout); - Assert.NotEqual(baselineManifestDigest, deltaManifestDigest); - var deltaLayerDigests = ReadManifestLayerDigests(deltaLayout, deltaManifestDigest); - Assert.Contains(baselineLayerDigest, deltaLayerDigests); - - var deltaLayerPath = Path.Combine(deltaLayout, "blobs", "sha256", baselineLayerDigest[7..]); - Assert.True(File.Exists(deltaLayerPath)); - var deltaLayerBytes = File.ReadAllBytes(deltaLayerPath); - Assert.Equal(baselineLayerBytes, deltaLayerBytes); - } - - private static Advisory CreateSampleAdvisory( - string advisoryKey = "CVE-2024-9999", - string title = "Trivy Export Test") - { - var published = DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture); - var modified = DateTimeOffset.Parse("2024-08-02T00:00:00Z", CultureInfo.InvariantCulture); - var reference = new AdvisoryReference( - "https://example.org/advisories/CVE-2024-9999", - kind: "advisory", - sourceTag: "EXAMPLE", - summary: null, - provenance: new AdvisoryProvenance("ghsa", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.References })); - var weakness = new AdvisoryWeakness( - taxonomy: "cwe", - identifier: "CWE-89", - name: "SQL Injection", - uri: "https://cwe.mitre.org/data/definitions/89.html", - provenance: new[] { new AdvisoryProvenance("nvd", "map", "CWE-89", modified, new[] { ProvenanceFieldMasks.Weaknesses }) }); - var cvssMetric = new CvssMetric( - "3.1", - "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - 9.8, - "critical", - new AdvisoryProvenance("nvd", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.CvssMetrics })); - - return new Advisory( - advisoryKey: advisoryKey, - title: title, - summary: "Trivy export fixture", - language: "en", - published: published, - modified: modified, - severity: "medium", - exploitKnown: false, - aliases: new[] { "CVE-2024-9999" }, - credits: Array.Empty(), - references: new[] { reference }, - affectedPackages: Array.Empty(), - cvssMetrics: new[] { cvssMetric }, - provenance: new[] { new AdvisoryProvenance("nvd", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.Advisory }) }, - description: "Detailed description for Trivy exporter testing.", - cwes: new[] { weakness }, - canonicalMetricId: "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"); - } - - public void Dispose() - { - try - { - if (Directory.Exists(_root)) - { - Directory.Delete(_root, recursive: true); - } - } - catch - { - // best effort cleanup - } - } - - private sealed class StubAdvisoryStore : IAdvisoryStore - { - private IReadOnlyList _advisories; - - public StubAdvisoryStore(params Advisory[] advisories) - { - _advisories = advisories; - } - - public void SetAdvisories(params Advisory[] advisories) - { - _advisories = advisories; - } - - public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) - => Task.FromResult(_advisories); - - public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) - => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); - - public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) - => Task.CompletedTask; - - public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) - { - return EnumerateAsync(cancellationToken); - - async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) - { - foreach (var advisory in _advisories) - { - ct.ThrowIfCancellationRequested(); - yield return advisory; - await Task.Yield(); - } - } - } - } - - private sealed class InMemoryExportStateStore : IExportStateStore - { - private ExportStateRecord? _record; - - public Task FindAsync(string id, CancellationToken cancellationToken) - => Task.FromResult(_record); - - public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) - { - _record = record; - return Task.FromResult(record); - } - } - - private sealed class TestTimeProvider : TimeProvider - { - private DateTimeOffset _now; - - public TestTimeProvider(DateTimeOffset start) => _now = start; - - public override DateTimeOffset GetUtcNow() => _now; - - public void Advance(TimeSpan delta) => _now = _now.Add(delta); - } - - private sealed class StubTrivyDbBuilder : ITrivyDbBuilder - { - private readonly string _root; - private readonly byte[] _metadata; - - public StubTrivyDbBuilder(string root, byte[] metadata) - { - _root = root; - _metadata = metadata; - } - - public Task BuildAsync( - JsonExportResult jsonTree, - DateTimeOffset exportedAt, - string exportId, - CancellationToken cancellationToken) - { - var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; - var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); - var payload = new byte[] { 0x1, 0x2, 0x3, 0x4 }; - File.WriteAllBytes(archivePath, payload); - using var sha256 = SHA256.Create(); - var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); - var length = payload.Length; - - return Task.FromResult(new TrivyDbBuilderResult( - archivePath, - digest, - length, - _metadata, - workingDirectory)); - } - } - - private sealed class RecordingTrivyDbBuilder : ITrivyDbBuilder - { - private readonly string _root; - private readonly byte[] _metadata; - private readonly List _manifestDigests = new(); - - public RecordingTrivyDbBuilder(string root, byte[] metadata) - { - _root = root; - _metadata = metadata; - } - - public IReadOnlyList ManifestDigests => _manifestDigests; - public string[]? LastRelativePaths { get; private set; } - - public Task BuildAsync( - JsonExportResult jsonTree, - DateTimeOffset exportedAt, - string exportId, - CancellationToken cancellationToken) - { - LastRelativePaths = jsonTree.Files.Select(static file => file.RelativePath).ToArray(); - - var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; - var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); - var payload = new byte[] { 0x5, 0x6, 0x7, 0x8 }; - File.WriteAllBytes(archivePath, payload); - using var sha256 = SHA256.Create(); - var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); - _manifestDigests.Add(digest); - - return Task.FromResult(new TrivyDbBuilderResult( - archivePath, - digest, - payload.Length, - _metadata, - workingDirectory)); - } - } - - private sealed record MetadataView(string Mode, bool ResetBaseline, string? BaseExportId, string? BaseManifestDigest, int DeltaChangedCount); - - private static MetadataView ReadMetadata(string path) - { - using var document = JsonDocument.Parse(File.ReadAllText(path)); - var root = document.RootElement; - var mode = root.TryGetProperty("mode", out var modeNode) ? modeNode.GetString() ?? string.Empty : string.Empty; - var resetBaseline = root.TryGetProperty("resetBaseline", out var resetNode) && resetNode.ValueKind == JsonValueKind.True; - string? baseExportId = null; - if (root.TryGetProperty("baseExportId", out var baseExportNode) && baseExportNode.ValueKind == JsonValueKind.String) - { - baseExportId = baseExportNode.GetString(); - } - - string? baseManifestDigest = null; - if (root.TryGetProperty("baseManifestDigest", out var baseManifestNode) && baseManifestNode.ValueKind == JsonValueKind.String) - { - baseManifestDigest = baseManifestNode.GetString(); - } - - var deltaChangedCount = 0; - if (root.TryGetProperty("delta", out var deltaNode) && deltaNode.ValueKind == JsonValueKind.Object) - { - if (deltaNode.TryGetProperty("changedFiles", out var changedFilesNode) && changedFilesNode.ValueKind == JsonValueKind.Array) - { - deltaChangedCount = changedFilesNode.GetArrayLength(); - } - } - - return new MetadataView(mode, resetBaseline, baseExportId, baseManifestDigest, deltaChangedCount); - } - - private static string ReadManifestDigest(string layoutPath) - { - var indexPath = Path.Combine(layoutPath, "index.json"); - using var document = JsonDocument.Parse(File.ReadAllText(indexPath)); - var manifests = document.RootElement.GetProperty("manifests"); - if (manifests.GetArrayLength() == 0) - { - throw new InvalidOperationException("No manifests present in OCI index."); - } - - return manifests[0].GetProperty("digest").GetString() ?? string.Empty; - } - - private static string[] ReadManifestLayerDigests(string layoutPath, string manifestDigest) - { - var manifestPath = Path.Combine(layoutPath, "blobs", "sha256", manifestDigest[7..]); - using var document = JsonDocument.Parse(File.ReadAllText(manifestPath)); - var layers = document.RootElement.GetProperty("layers"); - var digests = new string[layers.GetArrayLength()]; - var index = 0; - foreach (var layer in layers.EnumerateArray()) - { - digests[index++] = layer.GetProperty("digest").GetString() ?? string.Empty; - } - - return digests; - } - - private sealed record RunArtifacts( - string ExportId, - string ManifestDigest, - string IndexJson, - string MetadataJson, - string ManifestJson, - IReadOnlyDictionary Blobs); - - private async Task RunDeterministicExportAsync(IReadOnlyList advisories) - { - var workspace = Path.Combine(_root, $"deterministic-{Guid.NewGuid():N}"); - var jsonRoot = Path.Combine(workspace, "tree"); - Directory.CreateDirectory(workspace); - - var advisoryStore = new StubAdvisoryStore(advisories.ToArray()); - - var optionsValue = new TrivyDbExportOptions - { - OutputRoot = workspace, - ReferencePrefix = "example/trivy", - KeepWorkingTree = true, - Json = new JsonExportOptions - { - OutputRoot = jsonRoot, - MaintainLatestSymlink = false, - }, - }; - - var exportedAt = DateTimeOffset.Parse("2024-10-01T00:00:00Z", CultureInfo.InvariantCulture); - var options = Options.Create(optionsValue); - var packageBuilder = new TrivyDbPackageBuilder(); - var ociWriter = new TrivyDbOciWriter(); - var planner = new TrivyDbExportPlanner(); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(exportedAt); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new - { - Version = 2, - NextUpdate = "2024-10-02T00:00:00Z", - UpdatedAt = "2024-10-01T00:00:00Z", - }); - - var builder = new DeterministicTrivyDbBuilder(workspace, builderMetadata); - var orasPusher = new StubTrivyDbOrasPusher(); - var exporter = new TrivyDbFeedExporter( - advisoryStore, - new VulnListJsonExportPathResolver(), - options, - packageBuilder, - ociWriter, - stateManager, - planner, - builder, - orasPusher, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var exportId = exportedAt.ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); - var layoutPath = Path.Combine(workspace, exportId); - - var indexJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "index.json"), Encoding.UTF8); - var metadataJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "metadata.json"), Encoding.UTF8); - - using var indexDoc = JsonDocument.Parse(indexJson); - var manifestNode = indexDoc.RootElement.GetProperty("manifests")[0]; - var manifestDigest = manifestNode.GetProperty("digest").GetString()!; - - var manifestHex = manifestDigest[7..]; - var manifestJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "blobs", "sha256", manifestHex), Encoding.UTF8); - - var blobs = new Dictionary(StringComparer.Ordinal); - var blobsRoot = Path.Combine(layoutPath, "blobs", "sha256"); - foreach (var file in Directory.GetFiles(blobsRoot)) - { - var name = Path.GetFileName(file); - var content = await File.ReadAllBytesAsync(file); - blobs[name] = content; - } - - Directory.Delete(workspace, recursive: true); - - return new RunArtifacts(exportId, manifestDigest, indexJson, metadataJson, manifestJson, blobs); - } - - private sealed class DeterministicTrivyDbBuilder : ITrivyDbBuilder - { - private readonly string _root; - private readonly byte[] _metadata; - private readonly byte[] _payload; - - public DeterministicTrivyDbBuilder(string root, byte[] metadata) - { - _root = root; - _metadata = metadata; - _payload = new byte[] { 0x21, 0x22, 0x23, 0x24, 0x25 }; - } - - public Task BuildAsync( - JsonExportResult jsonTree, - DateTimeOffset exportedAt, - string exportId, - CancellationToken cancellationToken) - { - var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; - var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); - File.WriteAllBytes(archivePath, _payload); - using var sha256 = SHA256.Create(); - var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(_payload)).ToLowerInvariant(); - - return Task.FromResult(new TrivyDbBuilderResult( - archivePath, - digest, - _payload.Length, - _metadata, - workingDirectory)); - } - } - - private sealed class StubTrivyDbOrasPusher : ITrivyDbOrasPusher - { - public List<(string Layout, string Reference, string ExportId)> Pushes { get; } = new(); - - public Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) - { - Pushes.Add((layoutPath, reference, exportId)); - return Task.CompletedTask; - } - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Exporter.TrivyDb; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +namespace StellaOps.Concelier.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbFeedExporterTests : IDisposable +{ + private readonly string _root; + private readonly string _jsonRoot; + + public TrivyDbFeedExporterTests() + { + _root = Directory.CreateTempSubdirectory("concelier-trivy-exporter-tests").FullName; + _jsonRoot = Path.Combine(_root, "tree"); + } + + [Fact] + public async Task ExportAsync_SortsAdvisoriesByKeyDeterministically() + { + var advisoryB = CreateSampleAdvisory("CVE-2024-1002", "Second advisory"); + var advisoryA = CreateSampleAdvisory("CVE-2024-1001", "First advisory"); + + var advisoryStore = new StubAdvisoryStore(advisoryB, advisoryA); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = false, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-20T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-21T00:00:00Z", + UpdatedAt = "2024-09-20T00:00:00Z", + }); + + var recordingBuilder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + recordingBuilder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var paths = recordingBuilder.LastRelativePaths; + Assert.NotNull(paths); + + var sorted = paths!.OrderBy(static p => p, StringComparer.Ordinal).ToArray(); + Assert.Equal(sorted, paths); + + advisoryStore.SetAdvisories(advisoryA, advisoryB); + timeProvider.Advance(TimeSpan.FromMinutes(7)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240920T000000Z", record!.BaseExportId); + Assert.Single(recordingBuilder.ManifestDigests); + } + + [Fact] + public async Task ExportAsync_SmallDatasetProducesDeterministicOciLayout() + { + var advisories = new[] + { + CreateSampleAdvisory("CVE-2024-3000", "Demo advisory 1"), + CreateSampleAdvisory("CVE-2024-3001", "Demo advisory 2"), + }; + + var run1 = await RunDeterministicExportAsync(advisories); + var run2 = await RunDeterministicExportAsync(advisories); + + Assert.Equal(run1.ManifestDigest, run2.ManifestDigest); + Assert.Equal(run1.IndexJson, run2.IndexJson); + Assert.Equal(run1.MetadataJson, run2.MetadataJson); + Assert.Equal(run1.ManifestJson, run2.ManifestJson); + + var digests1 = run1.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + var digests2 = run2.Blobs.Keys.OrderBy(static d => d, StringComparer.Ordinal).ToArray(); + Assert.Equal(digests1, digests2); + + foreach (var digest in digests1) + { + Assert.True(run2.Blobs.TryGetValue(digest, out var other), $"Missing digest {digest} in second run"); + Assert.True(run1.Blobs[digest].SequenceEqual(other), $"Blob {digest} differs between runs"); + } + + using var metadataDoc = JsonDocument.Parse(run1.MetadataJson); + Assert.Equal(2, metadataDoc.RootElement.GetProperty("advisoryCount").GetInt32()); + + using var manifestDoc = JsonDocument.Parse(run1.ManifestJson); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, manifestDoc.RootElement.GetProperty("config").GetProperty("mediaType").GetString()); + var layer = manifestDoc.RootElement.GetProperty("layers")[0]; + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.GetProperty("mediaType").GetString()); + } + + [Fact] + public void ExportOptions_GetExportRoot_NormalizesRelativeRoot() + { + var options = new TrivyDbExportOptions + { + OutputRoot = Path.Combine("..", "exports", "trivy-test"), + }; + + var exportId = "20240901T000000Z"; + var path = options.GetExportRoot(exportId); + + Assert.True(Path.IsPathRooted(path)); + Assert.EndsWith(Path.Combine("exports", "trivy-test", exportId), path, StringComparison.Ordinal); + } + + [Fact] + public async Task ExportAsync_PersistsStateAndSkipsWhenDigestUnchanged() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-01T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-02T00:00:00Z", + UpdatedAt = "2024-09-01T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var record = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(record); + Assert.Equal("20240901T000000Z", record!.BaseExportId); + Assert.False(string.IsNullOrEmpty(record.ExportCursor)); + + var baseExportId = record.BaseExportId ?? string.Empty; + Assert.False(string.IsNullOrEmpty(baseExportId)); + var firstExportDirectory = Path.Combine(_root, baseExportId); + Assert.True(Directory.Exists(firstExportDirectory)); + + timeProvider.Advance(TimeSpan.FromMinutes(5)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updatedRecord = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updatedRecord); + Assert.Equal(record.UpdatedAt, updatedRecord!.UpdatedAt); + Assert.Equal(record.LastFullDigest, updatedRecord.LastFullDigest); + + var skippedExportDirectory = Path.Combine(_root, "20240901T000500Z"); + Assert.False(Directory.Exists(skippedExportDirectory)); + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_CreatesOfflineBundle() + { + var advisory = CreateSampleAdvisory(); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + OfflineBundle = new TrivyDbOfflineBundleOptions + { + Enabled = true, + FileName = "{exportId}.bundle.tar.gz", + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-15T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-16T00:00:00Z", + UpdatedAt = "2024-09-15T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = "20240915T000000Z"; + var bundlePath = Path.Combine(_root, $"{exportId}.bundle.tar.gz"); + Assert.True(File.Exists(bundlePath)); + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_WritesMirrorBundlesWhenConfigured() + { + var advisoryOne = CreateSampleAdvisory("CVE-2025-1001", "Mirror Advisory One"); + var advisoryTwo = CreateSampleAdvisory("CVE-2025-1002", "Mirror Advisory Two"); + var advisoryStore = new StubAdvisoryStore(advisoryOne, advisoryTwo); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + TargetRepository = "s3://mirror/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + }; + + optionsValue.Mirror.Enabled = true; + optionsValue.Mirror.DirectoryName = "mirror"; + optionsValue.Mirror.Domains.Add(new TrivyDbMirrorDomainOptions + { + Id = "primary", + DisplayName = "Primary Mirror", + }); + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var exportedAt = DateTimeOffset.Parse("2024-09-18T12:00:00Z", CultureInfo.InvariantCulture); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-19T12:00:00Z", + UpdatedAt = "2024-09-18T12:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var layoutPath = optionsValue.GetExportRoot(exportId); + var mirrorRoot = Path.Combine(layoutPath, "mirror"); + var domainRoot = Path.Combine(mirrorRoot, "primary"); + + Assert.True(File.Exists(Path.Combine(mirrorRoot, "index.json"))); + Assert.True(File.Exists(Path.Combine(domainRoot, "manifest.json"))); + Assert.True(File.Exists(Path.Combine(domainRoot, "metadata.json"))); + Assert.True(File.Exists(Path.Combine(domainRoot, "db.tar.gz"))); + + var reference = $"{optionsValue.ReferencePrefix}:{exportId}"; + var manifestDigest = ReadManifestDigest(layoutPath); + var indexPath = Path.Combine(mirrorRoot, "index.json"); + string? indexManifestDescriptorDigest = null; + string? indexMetadataDigest = null; + string? indexDatabaseDigest = null; + + using (var indexDoc = JsonDocument.Parse(File.ReadAllBytes(indexPath))) + { + var root = indexDoc.RootElement; + Assert.Equal(1, root.GetProperty("schemaVersion").GetInt32()); + Assert.Equal(reference, root.GetProperty("reference").GetString()); + Assert.Equal(manifestDigest, root.GetProperty("manifestDigest").GetString()); + Assert.Equal("full", root.GetProperty("mode").GetString()); + Assert.Equal("s3://mirror/trivy", root.GetProperty("targetRepository").GetString()); + Assert.False(root.TryGetProperty("delta", out _)); + + var domains = root.GetProperty("domains").EnumerateArray().ToArray(); + var domain = Assert.Single(domains); + Assert.Equal("primary", domain.GetProperty("domainId").GetString()); + Assert.Equal("Primary Mirror", domain.GetProperty("displayName").GetString()); + Assert.Equal(2, domain.GetProperty("advisoryCount").GetInt32()); + + var manifestDescriptor = domain.GetProperty("manifest"); + Assert.Equal("mirror/primary/manifest.json", manifestDescriptor.GetProperty("path").GetString()); + indexManifestDescriptorDigest = manifestDescriptor.GetProperty("digest").GetString(); + + var metadataDescriptor = domain.GetProperty("metadata"); + Assert.Equal("mirror/primary/metadata.json", metadataDescriptor.GetProperty("path").GetString()); + indexMetadataDigest = metadataDescriptor.GetProperty("digest").GetString(); + + var databaseDescriptor = domain.GetProperty("database"); + Assert.Equal("mirror/primary/db.tar.gz", databaseDescriptor.GetProperty("path").GetString()); + indexDatabaseDigest = databaseDescriptor.GetProperty("digest").GetString(); + } + + var domainManifestPath = Path.Combine(domainRoot, "manifest.json"); + var rootMetadataPath = Path.Combine(layoutPath, "metadata.json"); + var domainMetadataPath = Path.Combine(domainRoot, "metadata.json"); + var domainDbPath = Path.Combine(domainRoot, "db.tar.gz"); + + var domainManifestBytes = File.ReadAllBytes(domainManifestPath); + var domainManifestDigest = "sha256:" + Convert.ToHexString(SHA256.HashData(domainManifestBytes)).ToLowerInvariant(); + var rootMetadataBytes = File.ReadAllBytes(rootMetadataPath); + var domainMetadataBytes = File.ReadAllBytes(domainMetadataPath); + Assert.Equal(rootMetadataBytes, domainMetadataBytes); + + var metadataDigest = "sha256:" + Convert.ToHexString(SHA256.HashData(domainMetadataBytes)).ToLowerInvariant(); + var databaseDigest = "sha256:" + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(domainDbPath))).ToLowerInvariant(); + Assert.Equal(domainManifestDigest, indexManifestDescriptorDigest); + Assert.Equal(metadataDigest, indexMetadataDigest); + Assert.Equal(databaseDigest, indexDatabaseDigest); + + using (var manifestDoc = JsonDocument.Parse(File.ReadAllBytes(domainManifestPath))) + { + var manifestRoot = manifestDoc.RootElement; + Assert.Equal("primary", manifestRoot.GetProperty("domainId").GetString()); + Assert.Equal("Primary Mirror", manifestRoot.GetProperty("displayName").GetString()); + Assert.Equal(reference, manifestRoot.GetProperty("reference").GetString()); + Assert.Equal(manifestDigest, manifestRoot.GetProperty("manifestDigest").GetString()); + Assert.Equal("full", manifestRoot.GetProperty("mode").GetString()); + Assert.Equal("s3://mirror/trivy", manifestRoot.GetProperty("targetRepository").GetString()); + + var metadataDescriptor = manifestRoot.GetProperty("metadata"); + Assert.Equal("mirror/primary/metadata.json", metadataDescriptor.GetProperty("path").GetString()); + Assert.Equal(metadataDigest, metadataDescriptor.GetProperty("digest").GetString()); + + var databaseDescriptor = manifestRoot.GetProperty("database"); + Assert.Equal("mirror/primary/db.tar.gz", databaseDescriptor.GetProperty("path").GetString()); + Assert.Equal(databaseDigest, databaseDescriptor.GetProperty("digest").GetString()); + + var sources = manifestRoot.GetProperty("sources").EnumerateArray().ToArray(); + Assert.NotEmpty(sources); + Assert.Contains(sources, element => string.Equals(element.GetProperty("source").GetString(), "nvd", StringComparison.OrdinalIgnoreCase)); + } + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_SkipsOrasPushWhenDeltaPublishingDisabled() + { + var initial = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles"); + var updated = CreateSampleAdvisory("CVE-2024-7100", "Publish toggles delta"); + var advisoryStore = new StubAdvisoryStore(initial); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = true, + }; + + optionsValue.Oras.Enabled = true; + optionsValue.Oras.PublishFull = false; + optionsValue.Oras.PublishDelta = false; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-20T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-21T00:00:00Z", + UpdatedAt = "2024-10-20T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + advisoryStore.SetAdvisories(updated); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + await exporter.ExportAsync(provider, CancellationToken.None); + + Assert.Empty(orasPusher.Pushes); + } + + [Fact] + public async Task ExportAsync_SkipsOfflineBundleForDeltaWhenDisabled() + { + var initial = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles"); + var updated = CreateSampleAdvisory("CVE-2024-7200", "Offline delta toggles updated"); + var advisoryStore = new StubAdvisoryStore(initial); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = true, + OfflineBundle = new TrivyDbOfflineBundleOptions + { + Enabled = true, + IncludeFull = true, + IncludeDelta = false, + FileName = "{exportId}.bundle.tar.gz", + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-10-21T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-22T00:00:00Z", + UpdatedAt = "2024-10-21T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var fullExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var fullBundlePath = Path.Combine(_root, $"{fullExportId}.bundle.tar.gz"); + Assert.True(File.Exists(fullBundlePath)); + + advisoryStore.SetAdvisories(updated); + timeProvider.Advance(TimeSpan.FromMinutes(10)); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var deltaBundlePath = Path.Combine(_root, $"{deltaExportId}.bundle.tar.gz"); + Assert.False(File.Exists(deltaBundlePath)); + } + + [Fact] + public async Task ExportAsync_ResetsBaselineWhenDeltaChainExists() + { + var advisory = CreateSampleAdvisory("CVE-2024-5000", "Baseline reset"); + var advisoryStore = new StubAdvisoryStore(advisory); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + KeepWorkingTree = false, + TargetRepository = "registry.example/trivy", + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-09-22T00:00:00Z", CultureInfo.InvariantCulture)); + var existingRecord = new ExportStateRecord( + TrivyDbFeedExporter.ExporterId, + BaseExportId: "20240919T120000Z", + BaseDigest: "sha256:base", + LastFullDigest: "sha256:base", + LastDeltaDigest: "sha256:delta", + ExportCursor: "sha256:old", + TargetRepository: "registry.example/trivy", + ExporterVersion: "0.9.0", + UpdatedAt: timeProvider.GetUtcNow().AddMinutes(-30), + Files: Array.Empty()); + await stateStore.UpsertAsync(existingRecord, CancellationToken.None); + + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-09-23T00:00:00Z", + UpdatedAt = "2024-09-22T00:00:00Z", + }); + var builder = new StubTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var updated = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(updated); + Assert.Equal("20240922T000000Z", updated!.BaseExportId); + Assert.Equal(updated.BaseDigest, updated.LastFullDigest); + Assert.Null(updated.LastDeltaDigest); + Assert.NotEqual("sha256:old", updated.ExportCursor); + Assert.Equal("registry.example/trivy", updated.TargetRepository); + Assert.NotEmpty(updated.Files); + } + + [Fact] + public async Task ExportAsync_DeltaSequencePromotesBaselineReset() + { + var baseline = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory"); + var firstDelta = CreateSampleAdvisory("CVE-2024-8100", "Baseline advisory updated"); + var secondDelta = CreateSampleAdvisory("CVE-2024-8200", "New advisory triggers full rebuild"); + + var advisoryStore = new StubAdvisoryStore(baseline); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-01T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-11-02T00:00:00Z", + UpdatedAt = "2024-11-01T00:00:00Z", + }); + var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + + var initialExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var initialLayout = Path.Combine(optionsValue.OutputRoot, initialExportId); + var initialMetadata = ReadMetadata(Path.Combine(initialLayout, "metadata.json")); + Assert.Equal("full", initialMetadata.Mode); + var initialManifestDigest = ReadManifestDigest(initialLayout); + + advisoryStore.SetAdvisories(firstDelta); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); + var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); + Assert.Equal("delta", deltaMetadata.Mode); + Assert.Equal(initialExportId, deltaMetadata.BaseExportId); + Assert.Equal(initialManifestDigest, deltaMetadata.BaseManifestDigest); + Assert.True(deltaMetadata.DeltaChangedCount > 0); + + var reusedManifestPath = Path.Combine(deltaLayout, "blobs", "sha256", initialManifestDigest[7..]); + Assert.True(File.Exists(reusedManifestPath)); + + advisoryStore.SetAdvisories(secondDelta); + timeProvider.Advance(TimeSpan.FromMinutes(15)); + var finalExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var finalLayout = Path.Combine(optionsValue.OutputRoot, finalExportId); + var finalMetadata = ReadMetadata(Path.Combine(finalLayout, "metadata.json")); + Assert.Equal("full", finalMetadata.Mode); + Assert.True(finalMetadata.ResetBaseline); + + var state = await stateStore.FindAsync(TrivyDbFeedExporter.ExporterId, CancellationToken.None); + Assert.NotNull(state); + Assert.Null(state!.LastDeltaDigest); + Assert.Equal(finalExportId, state.BaseExportId); + } + + [Fact] + public async Task ExportAsync_DeltaReusesBaseLayerOnDisk() + { + var baseline = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse baseline"); + var delta = CreateSampleAdvisory("CVE-2024-8300", "Layer reuse delta"); + + var advisoryStore = new StubAdvisoryStore(baseline); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = _root, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = _jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-11-05T00:00:00Z", CultureInfo.InvariantCulture)); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-11-06T00:00:00Z", + UpdatedAt = "2024-11-05T00:00:00Z", + }); + var builder = new RecordingTrivyDbBuilder(_root, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + + var baselineExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var baselineLayout = Path.Combine(optionsValue.OutputRoot, baselineExportId); + var baselineManifestDigest = ReadManifestDigest(baselineLayout); + var baselineLayerDigests = ReadManifestLayerDigests(baselineLayout, baselineManifestDigest); + var baselineLayerDigest = Assert.Single(baselineLayerDigests); + var baselineLayerPath = Path.Combine(baselineLayout, "blobs", "sha256", baselineLayerDigest[7..]); + var baselineLayerBytes = File.ReadAllBytes(baselineLayerPath); + + advisoryStore.SetAdvisories(delta); + timeProvider.Advance(TimeSpan.FromMinutes(30)); + var deltaExportId = timeProvider.GetUtcNow().ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + await exporter.ExportAsync(provider, CancellationToken.None); + + var deltaLayout = Path.Combine(optionsValue.OutputRoot, deltaExportId); + var deltaMetadata = ReadMetadata(Path.Combine(deltaLayout, "metadata.json")); + Assert.Equal("delta", deltaMetadata.Mode); + Assert.Equal(baselineExportId, deltaMetadata.BaseExportId); + Assert.Equal(baselineManifestDigest, deltaMetadata.BaseManifestDigest); + Assert.True(deltaMetadata.DeltaChangedCount > 0); + + var deltaManifestDigest = ReadManifestDigest(deltaLayout); + Assert.NotEqual(baselineManifestDigest, deltaManifestDigest); + var deltaLayerDigests = ReadManifestLayerDigests(deltaLayout, deltaManifestDigest); + Assert.Contains(baselineLayerDigest, deltaLayerDigests); + + var deltaLayerPath = Path.Combine(deltaLayout, "blobs", "sha256", baselineLayerDigest[7..]); + Assert.True(File.Exists(deltaLayerPath)); + var deltaLayerBytes = File.ReadAllBytes(deltaLayerPath); + Assert.Equal(baselineLayerBytes, deltaLayerBytes); + } + + private static Advisory CreateSampleAdvisory( + string advisoryKey = "CVE-2024-9999", + string title = "Trivy Export Test") + { + var published = DateTimeOffset.Parse("2024-08-01T00:00:00Z", CultureInfo.InvariantCulture); + var modified = DateTimeOffset.Parse("2024-08-02T00:00:00Z", CultureInfo.InvariantCulture); + var reference = new AdvisoryReference( + "https://example.org/advisories/CVE-2024-9999", + kind: "advisory", + sourceTag: "EXAMPLE", + summary: null, + provenance: new AdvisoryProvenance("ghsa", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.References })); + var weakness = new AdvisoryWeakness( + taxonomy: "cwe", + identifier: "CWE-89", + name: "SQL Injection", + uri: "https://cwe.mitre.org/data/definitions/89.html", + provenance: new[] { new AdvisoryProvenance("nvd", "map", "CWE-89", modified, new[] { ProvenanceFieldMasks.Weaknesses }) }); + var cvssMetric = new CvssMetric( + "3.1", + "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + 9.8, + "critical", + new AdvisoryProvenance("nvd", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.CvssMetrics })); + + return new Advisory( + advisoryKey: advisoryKey, + title: title, + summary: "Trivy export fixture", + language: "en", + published: published, + modified: modified, + severity: "medium", + exploitKnown: false, + aliases: new[] { "CVE-2024-9999" }, + credits: Array.Empty(), + references: new[] { reference }, + affectedPackages: Array.Empty(), + cvssMetrics: new[] { cvssMetric }, + provenance: new[] { new AdvisoryProvenance("nvd", "map", "CVE-2024-9999", modified, new[] { ProvenanceFieldMasks.Advisory }) }, + description: "Detailed description for Trivy exporter testing.", + cwes: new[] { weakness }, + canonicalMetricId: "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + + private sealed class StubAdvisoryStore : IAdvisoryStore + { + private IReadOnlyList _advisories; + + public StubAdvisoryStore(params Advisory[] advisories) + { + _advisories = advisories; + } + + public void SetAdvisories(params Advisory[] advisories) + { + _advisories = advisories; + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult(_advisories); + } + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); + } + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.CompletedTask; + } + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return EnumerateAsync(cancellationToken); + + async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) + { + foreach (var advisory in _advisories) + { + ct.ThrowIfCancellationRequested(); + yield return advisory; + await Task.Yield(); + } + } + } + } + + private sealed class InMemoryExportStateStore : IExportStateStore + { + private ExportStateRecord? _record; + + public Task FindAsync(string id, CancellationToken cancellationToken) + { + return Task.FromResult(_record); + } + + public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) + { + _record = record; + return Task.FromResult(record); + } + } + + private sealed class TestTimeProvider : TimeProvider + { + private DateTimeOffset _now; + + public TestTimeProvider(DateTimeOffset start) => _now = start; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan delta) => _now = _now.Add(delta); + } + + private sealed class StubTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + + public StubTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x1, 0x2, 0x3, 0x4 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + var length = payload.Length; + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + length, + _metadata, + workingDirectory)); + } + } + + private sealed class RecordingTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly List _manifestDigests = new(); + + public RecordingTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + } + + public IReadOnlyList ManifestDigests => _manifestDigests; + public string[]? LastRelativePaths { get; private set; } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + LastRelativePaths = jsonTree.Files.Select(static file => file.RelativePath).ToArray(); + + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + var payload = new byte[] { 0x5, 0x6, 0x7, 0x8 }; + File.WriteAllBytes(archivePath, payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(payload)).ToLowerInvariant(); + _manifestDigests.Add(digest); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed record MetadataView(string Mode, bool ResetBaseline, string? BaseExportId, string? BaseManifestDigest, int DeltaChangedCount); + + private static MetadataView ReadMetadata(string path) + { + using var document = JsonDocument.Parse(File.ReadAllText(path)); + var root = document.RootElement; + var mode = root.TryGetProperty("mode", out var modeNode) ? modeNode.GetString() ?? string.Empty : string.Empty; + var resetBaseline = root.TryGetProperty("resetBaseline", out var resetNode) && resetNode.ValueKind == JsonValueKind.True; + string? baseExportId = null; + if (root.TryGetProperty("baseExportId", out var baseExportNode) && baseExportNode.ValueKind == JsonValueKind.String) + { + baseExportId = baseExportNode.GetString(); + } + + string? baseManifestDigest = null; + if (root.TryGetProperty("baseManifestDigest", out var baseManifestNode) && baseManifestNode.ValueKind == JsonValueKind.String) + { + baseManifestDigest = baseManifestNode.GetString(); + } + + var deltaChangedCount = 0; + if (root.TryGetProperty("delta", out var deltaNode) && deltaNode.ValueKind == JsonValueKind.Object) + { + if (deltaNode.TryGetProperty("changedFiles", out var changedFilesNode) && changedFilesNode.ValueKind == JsonValueKind.Array) + { + deltaChangedCount = changedFilesNode.GetArrayLength(); + } + } + + return new MetadataView(mode, resetBaseline, baseExportId, baseManifestDigest, deltaChangedCount); + } + + private static string ReadManifestDigest(string layoutPath) + { + var indexPath = Path.Combine(layoutPath, "index.json"); + using var document = JsonDocument.Parse(File.ReadAllText(indexPath)); + var manifests = document.RootElement.GetProperty("manifests"); + if (manifests.GetArrayLength() == 0) + { + throw new InvalidOperationException("No manifests present in OCI index."); + } + + return manifests[0].GetProperty("digest").GetString() ?? string.Empty; + } + + private static string[] ReadManifestLayerDigests(string layoutPath, string manifestDigest) + { + var manifestPath = Path.Combine(layoutPath, "blobs", "sha256", manifestDigest[7..]); + using var document = JsonDocument.Parse(File.ReadAllText(manifestPath)); + var layers = document.RootElement.GetProperty("layers"); + var digests = new string[layers.GetArrayLength()]; + var index = 0; + foreach (var layer in layers.EnumerateArray()) + { + digests[index++] = layer.GetProperty("digest").GetString() ?? string.Empty; + } + + return digests; + } + + private sealed record RunArtifacts( + string ExportId, + string ManifestDigest, + string IndexJson, + string MetadataJson, + string ManifestJson, + IReadOnlyDictionary Blobs); + + private async Task RunDeterministicExportAsync(IReadOnlyList advisories) + { + var workspace = Path.Combine(_root, $"deterministic-{Guid.NewGuid():N}"); + var jsonRoot = Path.Combine(workspace, "tree"); + Directory.CreateDirectory(workspace); + + var advisoryStore = new StubAdvisoryStore(advisories.ToArray()); + + var optionsValue = new TrivyDbExportOptions + { + OutputRoot = workspace, + ReferencePrefix = "example/trivy", + KeepWorkingTree = true, + Json = new JsonExportOptions + { + OutputRoot = jsonRoot, + MaintainLatestSymlink = false, + }, + }; + + var exportedAt = DateTimeOffset.Parse("2024-10-01T00:00:00Z", CultureInfo.InvariantCulture); + var options = Options.Create(optionsValue); + var packageBuilder = new TrivyDbPackageBuilder(); + var ociWriter = new TrivyDbOciWriter(); + var planner = new TrivyDbExportPlanner(); + var stateStore = new InMemoryExportStateStore(); + var timeProvider = new TestTimeProvider(exportedAt); + var stateManager = new ExportStateManager(stateStore, timeProvider); + var builderMetadata = JsonSerializer.SerializeToUtf8Bytes(new + { + Version = 2, + NextUpdate = "2024-10-02T00:00:00Z", + UpdatedAt = "2024-10-01T00:00:00Z", + }); + + var builder = new DeterministicTrivyDbBuilder(workspace, builderMetadata); + var orasPusher = new StubTrivyDbOrasPusher(); + var exporter = new TrivyDbFeedExporter( + advisoryStore, + new VulnListJsonExportPathResolver(), + options, + packageBuilder, + ociWriter, + stateManager, + planner, + builder, + orasPusher, + NullLogger.Instance, + timeProvider); + + using var provider = new ServiceCollection().BuildServiceProvider(); + await exporter.ExportAsync(provider, CancellationToken.None); + + var exportId = exportedAt.ToString(optionsValue.TagFormat, CultureInfo.InvariantCulture); + var layoutPath = Path.Combine(workspace, exportId); + + var indexJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "index.json"), Encoding.UTF8); + var metadataJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "metadata.json"), Encoding.UTF8); + + using var indexDoc = JsonDocument.Parse(indexJson); + var manifestNode = indexDoc.RootElement.GetProperty("manifests")[0]; + var manifestDigest = manifestNode.GetProperty("digest").GetString()!; + + var manifestHex = manifestDigest[7..]; + var manifestJson = await File.ReadAllTextAsync(Path.Combine(layoutPath, "blobs", "sha256", manifestHex), Encoding.UTF8); + + var blobs = new Dictionary(StringComparer.Ordinal); + var blobsRoot = Path.Combine(layoutPath, "blobs", "sha256"); + foreach (var file in Directory.GetFiles(blobsRoot)) + { + var name = Path.GetFileName(file); + var content = await File.ReadAllBytesAsync(file); + blobs[name] = content; + } + + Directory.Delete(workspace, recursive: true); + + return new RunArtifacts(exportId, manifestDigest, indexJson, metadataJson, manifestJson, blobs); + } + + private sealed class DeterministicTrivyDbBuilder : ITrivyDbBuilder + { + private readonly string _root; + private readonly byte[] _metadata; + private readonly byte[] _payload; + + public DeterministicTrivyDbBuilder(string root, byte[] metadata) + { + _root = root; + _metadata = metadata; + _payload = new byte[] { 0x21, 0x22, 0x23, 0x24, 0x25 }; + } + + public Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + var workingDirectory = Directory.CreateDirectory(Path.Combine(_root, $"builder-{exportId}")).FullName; + var archivePath = Path.Combine(workingDirectory, "db.tar.gz"); + File.WriteAllBytes(archivePath, _payload); + using var sha256 = SHA256.Create(); + var digest = "sha256:" + Convert.ToHexString(sha256.ComputeHash(_payload)).ToLowerInvariant(); + + return Task.FromResult(new TrivyDbBuilderResult( + archivePath, + digest, + _payload.Length, + _metadata, + workingDirectory)); + } + } + + private sealed class StubTrivyDbOrasPusher : ITrivyDbOrasPusher + { + public List<(string Layout, string Reference, string ExportId)> Pushes { get; } = new(); + + public Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + Pushes.Add((layoutPath, reference, exportId)); + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs similarity index 95% rename from src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs index 64ae7f0b..0d7eb254 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbOciWriterTests.cs @@ -1,149 +1,149 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Storage.Mongo.Exporting; -using Xunit; - -namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; - -public sealed class TrivyDbOciWriterTests : IDisposable -{ - private readonly string _root; - - public TrivyDbOciWriterTests() - { - _root = Directory.CreateTempSubdirectory("trivy-writer-tests").FullName; - } - - [Fact] - public async Task WriteAsync_ReusesBlobsFromBaseLayout_WhenDigestMatches() - { - var baseLayout = Path.Combine(_root, "base"); - Directory.CreateDirectory(Path.Combine(baseLayout, "blobs", "sha256")); - - var configBytes = Encoding.UTF8.GetBytes("base-config"); - var configDigest = ComputeDigest(configBytes); - WriteBlob(baseLayout, configDigest, configBytes); - - var layerBytes = Encoding.UTF8.GetBytes("base-layer"); - var layerDigest = ComputeDigest(layerBytes); - WriteBlob(baseLayout, layerDigest, layerBytes); - - var manifest = CreateManifest(configDigest, layerDigest); - var manifestBytes = SerializeManifest(manifest); - var manifestDigest = ComputeDigest(manifestBytes); - WriteBlob(baseLayout, manifestDigest, manifestBytes); - - var plan = new TrivyDbExportPlan( - TrivyDbExportMode.Delta, - TreeDigest: "sha256:tree", - BaseExportId: "20241101T000000Z", - BaseManifestDigest: manifestDigest, - ResetBaseline: false, - Manifest: Array.Empty(), - ChangedFiles: new[] { new ExportFileRecord("data.json", 1, "sha256:data") }, - RemovedPaths: Array.Empty()); - - var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, configBytes.Length); - var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, layerBytes.Length); - var package = new TrivyDbPackage( - manifest, - new TrivyConfigDocument( - TrivyDbMediaTypes.TrivyConfig, - DateTimeOffset.Parse("2024-11-01T00:00:00Z"), - "20241101T000000Z", - layerDigest, - layerBytes.Length), - new Dictionary(StringComparer.Ordinal) - { - [configDigest] = CreateThrowingBlob(), - [layerDigest] = CreateThrowingBlob(), - }, - JsonSerializer.SerializeToUtf8Bytes(new { mode = "delta" })); - - var writer = new TrivyDbOciWriter(); - var destination = Path.Combine(_root, "delta"); - await writer.WriteAsync(package, destination, reference: "example/trivy:delta", plan, baseLayout, CancellationToken.None); - - var reusedConfig = File.ReadAllBytes(GetBlobPath(destination, configDigest)); - Assert.Equal(configBytes, reusedConfig); - - var reusedLayer = File.ReadAllBytes(GetBlobPath(destination, layerDigest)); - Assert.Equal(layerBytes, reusedLayer); - } - - private static TrivyDbBlob CreateThrowingBlob() - { - var ctor = typeof(TrivyDbBlob).GetConstructor( - BindingFlags.NonPublic | BindingFlags.Instance, - binder: null, - new[] { typeof(Func>), typeof(long) }, - modifiers: null) - ?? throw new InvalidOperationException("Unable to access TrivyDbBlob constructor."); - - Func> factory = _ => throw new InvalidOperationException("Blob should have been reused from base layout."); - return (TrivyDbBlob)ctor.Invoke(new object[] { factory, 0L }); - } - - private static OciManifest CreateManifest(string configDigest, string layerDigest) - { - var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, 0); - var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, 0); - return new OciManifest( - SchemaVersion: 2, - MediaType: TrivyDbMediaTypes.OciManifest, - Config: configDescriptor, - Layers: new[] { layerDescriptor }); - } - - private static byte[] SerializeManifest(OciManifest manifest) - { - var options = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, - WriteIndented = false, - }; - return JsonSerializer.SerializeToUtf8Bytes(manifest, options); - } - - private static void WriteBlob(string layoutRoot, string digest, byte[] payload) - { - var path = GetBlobPath(layoutRoot, digest); - Directory.CreateDirectory(Path.GetDirectoryName(path)!); - File.WriteAllBytes(path, payload); - } - - private static string GetBlobPath(string layoutRoot, string digest) - { - var fileName = digest[7..]; - return Path.Combine(layoutRoot, "blobs", "sha256", fileName); - } - - private static string ComputeDigest(byte[] payload) - { - var hash = SHA256.HashData(payload); - return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); - } - - public void Dispose() - { - try - { - if (Directory.Exists(_root)) - { - Directory.Delete(_root, recursive: true); - } - } - catch - { - // best effort cleanup - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Storage.Mongo.Exporting; +using Xunit; + +namespace StellaOps.Concelier.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbOciWriterTests : IDisposable +{ + private readonly string _root; + + public TrivyDbOciWriterTests() + { + _root = Directory.CreateTempSubdirectory("trivy-writer-tests").FullName; + } + + [Fact] + public async Task WriteAsync_ReusesBlobsFromBaseLayout_WhenDigestMatches() + { + var baseLayout = Path.Combine(_root, "base"); + Directory.CreateDirectory(Path.Combine(baseLayout, "blobs", "sha256")); + + var configBytes = Encoding.UTF8.GetBytes("base-config"); + var configDigest = ComputeDigest(configBytes); + WriteBlob(baseLayout, configDigest, configBytes); + + var layerBytes = Encoding.UTF8.GetBytes("base-layer"); + var layerDigest = ComputeDigest(layerBytes); + WriteBlob(baseLayout, layerDigest, layerBytes); + + var manifest = CreateManifest(configDigest, layerDigest); + var manifestBytes = SerializeManifest(manifest); + var manifestDigest = ComputeDigest(manifestBytes); + WriteBlob(baseLayout, manifestDigest, manifestBytes); + + var plan = new TrivyDbExportPlan( + TrivyDbExportMode.Delta, + TreeDigest: "sha256:tree", + BaseExportId: "20241101T000000Z", + BaseManifestDigest: manifestDigest, + ResetBaseline: false, + Manifest: Array.Empty(), + ChangedFiles: new[] { new ExportFileRecord("data.json", 1, "sha256:data") }, + RemovedPaths: Array.Empty()); + + var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, configBytes.Length); + var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, layerBytes.Length); + var package = new TrivyDbPackage( + manifest, + new TrivyConfigDocument( + TrivyDbMediaTypes.TrivyConfig, + DateTimeOffset.Parse("2024-11-01T00:00:00Z"), + "20241101T000000Z", + layerDigest, + layerBytes.Length), + new Dictionary(StringComparer.Ordinal) + { + [configDigest] = CreateThrowingBlob(), + [layerDigest] = CreateThrowingBlob(), + }, + JsonSerializer.SerializeToUtf8Bytes(new { mode = "delta" })); + + var writer = new TrivyDbOciWriter(); + var destination = Path.Combine(_root, "delta"); + await writer.WriteAsync(package, destination, reference: "example/trivy:delta", plan, baseLayout, CancellationToken.None); + + var reusedConfig = File.ReadAllBytes(GetBlobPath(destination, configDigest)); + Assert.Equal(configBytes, reusedConfig); + + var reusedLayer = File.ReadAllBytes(GetBlobPath(destination, layerDigest)); + Assert.Equal(layerBytes, reusedLayer); + } + + private static TrivyDbBlob CreateThrowingBlob() + { + var ctor = typeof(TrivyDbBlob).GetConstructor( + BindingFlags.NonPublic | BindingFlags.Instance, + binder: null, + new[] { typeof(Func>), typeof(long) }, + modifiers: null) + ?? throw new InvalidOperationException("Unable to access TrivyDbBlob constructor."); + + Func> factory = _ => throw new InvalidOperationException("Blob should have been reused from base layout."); + return (TrivyDbBlob)ctor.Invoke(new object[] { factory, 0L }); + } + + private static OciManifest CreateManifest(string configDigest, string layerDigest) + { + var configDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyConfig, configDigest, 0); + var layerDescriptor = new OciDescriptor(TrivyDbMediaTypes.TrivyLayer, layerDigest, 0); + return new OciManifest( + SchemaVersion: 2, + MediaType: TrivyDbMediaTypes.OciManifest, + Config: configDescriptor, + Layers: new[] { layerDescriptor }); + } + + private static byte[] SerializeManifest(OciManifest manifest) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + return JsonSerializer.SerializeToUtf8Bytes(manifest, options); + } + + private static void WriteBlob(string layoutRoot, string digest, byte[] payload) + { + var path = GetBlobPath(layoutRoot, digest); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllBytes(path, payload); + } + + private static string GetBlobPath(string layoutRoot, string digest) + { + var fileName = digest[7..]; + return Path.Combine(layoutRoot, "blobs", "sha256", fileName); + } + + private static string ComputeDigest(byte[] payload) + { + var hash = SHA256.HashData(payload); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + public void Dispose() + { + try + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + catch + { + // best effort cleanup + } + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs similarity index 94% rename from src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs index dcd4a6ab..a8c42149 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb.Tests/TrivyDbPackageBuilderTests.cs @@ -1,93 +1,93 @@ -using System; -using System.IO; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using StellaOps.Feedser.Exporter.TrivyDb; - -namespace StellaOps.Feedser.Exporter.TrivyDb.Tests; - -public sealed class TrivyDbPackageBuilderTests -{ - [Fact] - public void BuildsOciManifestWithExpectedMediaTypes() - { - var metadata = Encoding.UTF8.GetBytes("{\"generatedAt\":\"2024-07-15T12:00:00Z\"}"); - var archive = Enumerable.Range(0, 256).Select(static b => (byte)b).ToArray(); - var archivePath = Path.GetTempFileName(); - File.WriteAllBytes(archivePath, archive); - var archiveDigest = ComputeDigest(archive); - - try - { - var request = new TrivyDbPackageRequest( - metadata, - archivePath, - archiveDigest, - archive.LongLength, - DateTimeOffset.Parse("2024-07-15T12:00:00Z"), - "2024.07.15"); - - var builder = new TrivyDbPackageBuilder(); - var package = builder.BuildPackage(request); - - Assert.Equal(TrivyDbMediaTypes.OciManifest, package.Manifest.MediaType); - Assert.Equal(TrivyDbMediaTypes.TrivyConfig, package.Manifest.Config.MediaType); - var layer = Assert.Single(package.Manifest.Layers); - Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.MediaType); - - var configBytes = JsonSerializer.SerializeToUtf8Bytes(package.Config, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); - var expectedConfigDigest = ComputeDigest(configBytes); - Assert.Equal(expectedConfigDigest, package.Manifest.Config.Digest); - - Assert.Equal(archiveDigest, layer.Digest); - Assert.True(package.Blobs.ContainsKey(archiveDigest)); - Assert.Equal(archive.LongLength, package.Blobs[archiveDigest].Length); - Assert.True(package.Blobs.ContainsKey(expectedConfigDigest)); - Assert.Equal(metadata, package.MetadataJson.ToArray()); - } - finally - { - if (File.Exists(archivePath)) - { - File.Delete(archivePath); - } - } - } - - [Fact] - public void ThrowsWhenMetadataMissing() - { - var builder = new TrivyDbPackageBuilder(); - var archivePath = Path.GetTempFileName(); - var archiveBytes = new byte[] { 1, 2, 3 }; - File.WriteAllBytes(archivePath, archiveBytes); - var digest = ComputeDigest(archiveBytes); - - try - { - Assert.Throws(() => builder.BuildPackage(new TrivyDbPackageRequest( - ReadOnlyMemory.Empty, - archivePath, - digest, - archiveBytes.LongLength, - DateTimeOffset.UtcNow, - "1"))); - } - finally - { - if (File.Exists(archivePath)) - { - File.Delete(archivePath); - } - } - } - - private static string ComputeDigest(ReadOnlySpan payload) - { - var hash = SHA256.HashData(payload); - var hex = Convert.ToHexString(hash); - return "sha256:" + hex.ToLowerInvariant(); - } -} +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using StellaOps.Concelier.Exporter.TrivyDb; + +namespace StellaOps.Concelier.Exporter.TrivyDb.Tests; + +public sealed class TrivyDbPackageBuilderTests +{ + [Fact] + public void BuildsOciManifestWithExpectedMediaTypes() + { + var metadata = Encoding.UTF8.GetBytes("{\"generatedAt\":\"2024-07-15T12:00:00Z\"}"); + var archive = Enumerable.Range(0, 256).Select(static b => (byte)b).ToArray(); + var archivePath = Path.GetTempFileName(); + File.WriteAllBytes(archivePath, archive); + var archiveDigest = ComputeDigest(archive); + + try + { + var request = new TrivyDbPackageRequest( + metadata, + archivePath, + archiveDigest, + archive.LongLength, + DateTimeOffset.Parse("2024-07-15T12:00:00Z"), + "2024.07.15"); + + var builder = new TrivyDbPackageBuilder(); + var package = builder.BuildPackage(request); + + Assert.Equal(TrivyDbMediaTypes.OciManifest, package.Manifest.MediaType); + Assert.Equal(TrivyDbMediaTypes.TrivyConfig, package.Manifest.Config.MediaType); + var layer = Assert.Single(package.Manifest.Layers); + Assert.Equal(TrivyDbMediaTypes.TrivyLayer, layer.MediaType); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(package.Config, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var expectedConfigDigest = ComputeDigest(configBytes); + Assert.Equal(expectedConfigDigest, package.Manifest.Config.Digest); + + Assert.Equal(archiveDigest, layer.Digest); + Assert.True(package.Blobs.ContainsKey(archiveDigest)); + Assert.Equal(archive.LongLength, package.Blobs[archiveDigest].Length); + Assert.True(package.Blobs.ContainsKey(expectedConfigDigest)); + Assert.Equal(metadata, package.MetadataJson.ToArray()); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + [Fact] + public void ThrowsWhenMetadataMissing() + { + var builder = new TrivyDbPackageBuilder(); + var archivePath = Path.GetTempFileName(); + var archiveBytes = new byte[] { 1, 2, 3 }; + File.WriteAllBytes(archivePath, archiveBytes); + var digest = ComputeDigest(archiveBytes); + + try + { + Assert.Throws(() => builder.BuildPackage(new TrivyDbPackageRequest( + ReadOnlyMemory.Empty, + archivePath, + digest, + archiveBytes.LongLength, + DateTimeOffset.UtcNow, + "1"))); + } + finally + { + if (File.Exists(archivePath)) + { + File.Delete(archivePath); + } + } + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + return "sha256:" + hex.ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md b/src/StellaOps.Concelier.Exporter.TrivyDb/AGENTS.md similarity index 88% rename from src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md rename to src/StellaOps.Concelier.Exporter.TrivyDb/AGENTS.md index 2897733c..cdd6ffdb 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/AGENTS.md +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/AGENTS.md @@ -1,29 +1,29 @@ -# AGENTS -## Role -Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS. -## Scope -- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts). -- Output root: exports/trivy/; deterministic path components. -- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2). -- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json). -- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine. -- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging. -## Participants -- Storage.Mongo.AdvisoryStore as input. -- Core scheduler runs export job; WebService/Plugins trigger it. -- JSON exporter (optional precursor) if choosing the builder path. -## Interfaces & contracts -- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken). -- FeedserOptions.packaging.trivy governs repo/tag/publish/offline_bundle. -- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs). -## In/Out of scope -In: assembling builder inputs, packing tar.gz, pushing to registry when configured. -Out: signing (external pipeline), scanner behavior. -## Observability & security expectations -- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail. -- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors. -## Tests -- Author and review coverage in `../StellaOps.Feedser.Exporter.TrivyDb.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. -- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. - +# AGENTS +## Role +Exporter producing a Trivy-compatible database artifact for self-hosting or offline use. v0: JSON list + metadata; v1: integrate official trivy-db builder or write BoltDB directly; pack and optionally push via ORAS. +## Scope +- Read canonical advisories; serialize payload for builder or intermediate; write metadata.json (generatedAt, counts). +- Output root: exports/trivy/; deterministic path components. +- OCI/Trivy expectations: layer media type application/vnd.aquasec.trivy.db.layer.v1.tar+gzip; config media type application/vnd.aquasec.trivy.config.v1+json; tag (e.g., 2). +- Optional ORAS push; optional offline bundle (db.tar.gz + metadata.json). +- DI: TrivyExporter + Jobs.TrivyExportJob registered by TrivyExporterDependencyInjectionRoutine. +- Export_state recording: capture digests, counts, start/end timestamps for idempotent reruns and incremental packaging. +## Participants +- Storage.Mongo.AdvisoryStore as input. +- Core scheduler runs export job; WebService/Plugins trigger it. +- JSON exporter (optional precursor) if choosing the builder path. +## Interfaces & contracts +- IFeedExporter.Name = "trivy-db"; ExportAsync(IServiceProvider, CancellationToken). +- ConcelierOptions.packaging.trivy governs repo/tag/publish/offline_bundle. +- Deterministic sorting and timestamp discipline (UTC; consider build reproducibility knobs). +## In/Out of scope +In: assembling builder inputs, packing tar.gz, pushing to registry when configured. +Out: signing (external pipeline), scanner behavior. +## Observability & security expectations +- Metrics: export.trivy.records, size_bytes, duration, oras.push.success/fail. +- Logs: export path, repo/tag, digest; redact credentials; backoff on push errors. +## Tests +- Author and review coverage in `../StellaOps.Concelier.Exporter.TrivyDb.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. +- Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbBuilder.cs similarity index 73% rename from src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbBuilder.cs index 0f9854ba..4d31e1ea 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbBuilder.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbBuilder.cs @@ -1,15 +1,15 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Feedser.Exporter.Json; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public interface ITrivyDbBuilder -{ - Task BuildAsync( - JsonExportResult jsonTree, - DateTimeOffset exportedAt, - string exportId, - CancellationToken cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Concelier.Exporter.Json; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public interface ITrivyDbBuilder +{ + Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbOrasPusher.cs similarity index 78% rename from src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbOrasPusher.cs index d8a44075..82aa9ecd 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/ITrivyDbOrasPusher.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/ITrivyDbOrasPusher.cs @@ -1,9 +1,9 @@ -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public interface ITrivyDbOrasPusher -{ - Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken); -} +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public interface ITrivyDbOrasPusher +{ + Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/OciDescriptor.cs similarity index 87% rename from src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/OciDescriptor.cs index 58aeaec8..d0a67b76 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/OciDescriptor.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/OciDescriptor.cs @@ -1,10 +1,10 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record OciDescriptor( - [property: JsonPropertyName("mediaType")] string MediaType, - [property: JsonPropertyName("digest")] string Digest, - [property: JsonPropertyName("size")] long Size, - [property: JsonPropertyName("annotations")] IReadOnlyDictionary? Annotations = null); +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record OciDescriptor( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("annotations")] IReadOnlyDictionary? Annotations = null); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/OciIndex.cs similarity index 82% rename from src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/OciIndex.cs index eb00c179..aa033fb2 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/OciIndex.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/OciIndex.cs @@ -1,8 +1,8 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record OciIndex( - [property: JsonPropertyName("schemaVersion")] int SchemaVersion, - [property: JsonPropertyName("manifests")] IReadOnlyList Manifests); +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record OciIndex( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("manifests")] IReadOnlyList Manifests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/OciManifest.cs similarity index 86% rename from src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/OciManifest.cs index ee99d638..83b0a63a 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/OciManifest.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/OciManifest.cs @@ -1,10 +1,10 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record OciManifest( - [property: JsonPropertyName("schemaVersion")] int SchemaVersion, - [property: JsonPropertyName("mediaType")] string MediaType, - [property: JsonPropertyName("config")] OciDescriptor Config, - [property: JsonPropertyName("layers")] IReadOnlyList Layers); +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record OciManifest( + [property: JsonPropertyName("schemaVersion")] int SchemaVersion, + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("config")] OciDescriptor Config, + [property: JsonPropertyName("layers")] IReadOnlyList Layers); diff --git a/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj b/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj similarity index 68% rename from src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj rename to src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj index 72b8b2a8..eb183fc4 100644 --- a/src/StellaOps.Feedser.Exporter.Json/StellaOps.Feedser.Exporter.Json.csproj +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/StellaOps.Concelier.Exporter.TrivyDb.csproj @@ -3,19 +3,19 @@ net10.0 preview - enable enable + enable true - - - - + + + + - + diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md b/src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md similarity index 77% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md rename to src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md index 83ca9b8c..e7ee2ed0 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TASKS.md +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TASKS.md @@ -11,4 +11,5 @@ |ExportState persistence & idempotence|BE-Export|Storage.Mongo|DONE – baseline resets wired into `ExportStateManager`, planner signals resets after delta runs, and exporters update state w/ repository-aware baseline rotation + tests.| |Streamed package building to avoid large copies|BE-Export|Exporters|DONE – metadata/config now reuse backing arrays and OCI writer streams directly without double buffering.| |Plan incremental/delta exports|BE-Export|Exporters|DONE – state captures per-file manifests, planner schedules delta vs full resets, layer reuse smoke test verifies OCI reuse, and operator guide documents the validation flow.| -|Advisory schema parity export (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – exporter/test fixtures updated to handle description/CWEs/canonical metric fields during Trivy DB packaging; `dotnet test src/StellaOps.Feedser.Exporter.TrivyDb.Tests` re-run 2025-10-15 to confirm coverage.| +|Advisory schema parity export (description/CWEs/canonical metric)|BE-Export|Models, Core|DONE (2025-10-15) – exporter/test fixtures updated to handle description/CWEs/canonical metric fields during Trivy DB packaging; `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests` re-run 2025-10-15 to confirm coverage.| +|CONCELIER-EXPORT-08-202 – Mirror-ready Trivy DB bundles|Team Concelier Export|CONCELIER-EXPORT-08-201|**DONE (2025-10-19)** – Added mirror export options and writer emitting `mirror/index.json` plus per-domain `manifest.json`/`metadata.json`/`db.tar.gz` with deterministic SHA-256 digests; regression covered via `dotnet test src/StellaOps.Concelier.Exporter.TrivyDb.Tests/StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj`.| diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyConfigDocument.cs similarity index 88% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyConfigDocument.cs index 10e947c8..e4f1b671 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyConfigDocument.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyConfigDocument.cs @@ -1,11 +1,11 @@ -using System; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record TrivyConfigDocument( - [property: JsonPropertyName("mediaType")] string MediaType, - [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, - [property: JsonPropertyName("databaseVersion")] string DatabaseVersion, - [property: JsonPropertyName("databaseDigest")] string DatabaseDigest, - [property: JsonPropertyName("databaseSize")] long DatabaseSize); +using System; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record TrivyConfigDocument( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("databaseVersion")] string DatabaseVersion, + [property: JsonPropertyName("databaseDigest")] string DatabaseDigest, + [property: JsonPropertyName("databaseSize")] long DatabaseSize); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBlob.cs similarity index 94% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBlob.cs index 2ff58e60..3015a8b9 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBlob.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBlob.cs @@ -1,78 +1,78 @@ -using System; -using System.IO; -using System.Runtime.InteropServices; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbBlob -{ - private readonly Func> _openReadAsync; - - private TrivyDbBlob(Func> openReadAsync, long length) - { - _openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync)); - if (length < 0) - { - throw new ArgumentOutOfRangeException(nameof(length)); - } - - Length = length; - } - - public long Length { get; } - - public ValueTask OpenReadAsync(CancellationToken cancellationToken) - => _openReadAsync(cancellationToken); - - public static TrivyDbBlob FromBytes(ReadOnlyMemory payload) - { - if (payload.IsEmpty) - { - return new TrivyDbBlob(static _ => ValueTask.FromResult(Stream.Null), 0); - } - - if (MemoryMarshal.TryGetArray(payload, out ArraySegment segment) && segment.Array is not null && segment.Offset == 0) - { - return FromArray(segment.Array); - } - - return FromArray(payload.ToArray()); - } - - public static TrivyDbBlob FromFile(string path, long length) - { - if (string.IsNullOrWhiteSpace(path)) - { - throw new ArgumentException("File path must be provided.", nameof(path)); - } - - if (length < 0) - { - throw new ArgumentOutOfRangeException(nameof(length)); - } - - return new TrivyDbBlob( - cancellationToken => ValueTask.FromResult(new FileStream( - path, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan)), - length); - } - - public static TrivyDbBlob FromArray(byte[] buffer) - { - if (buffer is null) - { - throw new ArgumentNullException(nameof(buffer)); - } - - return new TrivyDbBlob( - _ => ValueTask.FromResult(new MemoryStream(buffer, writable: false)), - buffer.LongLength); - } -} +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbBlob +{ + private readonly Func> _openReadAsync; + + private TrivyDbBlob(Func> openReadAsync, long length) + { + _openReadAsync = openReadAsync ?? throw new ArgumentNullException(nameof(openReadAsync)); + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + Length = length; + } + + public long Length { get; } + + public ValueTask OpenReadAsync(CancellationToken cancellationToken) + => _openReadAsync(cancellationToken); + + public static TrivyDbBlob FromBytes(ReadOnlyMemory payload) + { + if (payload.IsEmpty) + { + return new TrivyDbBlob(static _ => ValueTask.FromResult(Stream.Null), 0); + } + + if (MemoryMarshal.TryGetArray(payload, out ArraySegment segment) && segment.Array is not null && segment.Offset == 0) + { + return FromArray(segment.Array); + } + + return FromArray(payload.ToArray()); + } + + public static TrivyDbBlob FromFile(string path, long length) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("File path must be provided.", nameof(path)); + } + + if (length < 0) + { + throw new ArgumentOutOfRangeException(nameof(length)); + } + + return new TrivyDbBlob( + cancellationToken => ValueTask.FromResult(new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan)), + length); + } + + public static TrivyDbBlob FromArray(byte[] buffer) + { + if (buffer is null) + { + throw new ArgumentNullException(nameof(buffer)); + } + + return new TrivyDbBlob( + _ => ValueTask.FromResult(new MemoryStream(buffer, writable: false)), + buffer.LongLength); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBoltBuilder.cs similarity index 96% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBoltBuilder.cs index e7723005..0a0e7306 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBoltBuilder.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBoltBuilder.cs @@ -1,376 +1,376 @@ -using System; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Formats.Tar; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Exporter.Json; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder -{ - private readonly TrivyDbExportOptions _options; - private readonly ILogger _logger; - - public TrivyDbBoltBuilder(IOptions options, ILogger logger) - { - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task BuildAsync( - JsonExportResult jsonTree, - DateTimeOffset exportedAt, - string exportId, - CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(jsonTree); - ArgumentException.ThrowIfNullOrEmpty(exportId); - - var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId); - var outputDir = Path.Combine(builderRoot, "out"); - Directory.CreateDirectory(outputDir); - - try - { - await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false); - } - catch - { - TryDeleteDirectory(builderRoot); - throw; - } - - var metadataPath = Path.Combine(outputDir, "metadata.json"); - var dbPath = Path.Combine(outputDir, "trivy.db"); - - if (!File.Exists(metadataPath)) - { - TryDeleteDirectory(builderRoot); - throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'."); - } - - if (!File.Exists(dbPath)) - { - TryDeleteDirectory(builderRoot); - throw new InvalidOperationException($"trivy.db not found at '{dbPath}'."); - } - - var archivePath = Path.Combine(builderRoot, "db.tar.gz"); - await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false); - - var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false); - var length = new FileInfo(archivePath).Length; - var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false); - - return new TrivyDbBuilderResult( - archivePath, - digest, - length, - builderMetadata, - builderRoot); - } - - private string PrepareBuilderRoot(string exportDirectory, string exportId) - { - var root = Path.Combine(exportDirectory, $".builder-{exportId}"); - if (Directory.Exists(root)) - { - Directory.Delete(root, recursive: true); - } - - Directory.CreateDirectory(root); - return root; - } - - private static void TryDeleteDirectory(string directory) - { - try - { - if (Directory.Exists(directory)) - { - Directory.Delete(directory, recursive: true); - } - } - catch - { - // ignore cleanup failures - } - } - - private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken) - { - var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions(); - var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath) - ? "trivy-db" - : builderOptions.ExecutablePath; - - var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List(); - var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); - - var startInfo = new ProcessStartInfo - { - FileName = executable, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - }; - - startInfo.ArgumentList.Add("build"); - startInfo.ArgumentList.Add("--cache-dir"); - startInfo.ArgumentList.Add(cacheDir); - startInfo.ArgumentList.Add("--output-dir"); - startInfo.ArgumentList.Add(outputDir); - - if (builderOptions.UpdateInterval != default) - { - startInfo.ArgumentList.Add("--update-interval"); - startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval)); - } - - if (targets.Count > 0) - { - foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t))) - { - startInfo.ArgumentList.Add("--only-update"); - startInfo.ArgumentList.Add(target); - } - } - - if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory)) - { - startInfo.WorkingDirectory = builderOptions.WorkingDirectory; - } - - if (!builderOptions.InheritEnvironment) - { - startInfo.Environment.Clear(); - } - - foreach (var kvp in environment) - { - startInfo.Environment[kvp.Key] = kvp.Value; - } - - using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false }; - - var stdOut = new StringBuilder(); - var stdErr = new StringBuilder(); - - var stdoutCompletion = new TaskCompletionSource(); - var stderrCompletion = new TaskCompletionSource(); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is null) - { - stdoutCompletion.TrySetResult(null); - } - else - { - stdOut.AppendLine(e.Data); - } - }; - - process.ErrorDataReceived += (_, e) => - { - if (e.Data is null) - { - stderrCompletion.TrySetResult(null); - } - else - { - stdErr.AppendLine(e.Data); - } - }; - - _logger.LogInformation("Running {Executable} to build Trivy DB", executable); - - try - { - if (!process.Start()) - { - throw new InvalidOperationException($"Failed to start '{executable}'."); - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to start '{executable}'.", ex); - } - - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - using var registration = cancellationToken.Register(() => - { - try - { - if (!process.HasExited) - { - process.Kill(entireProcessTree: true); - } - } - catch - { - // Ignore kill failures. - } - }); - -#if NET8_0_OR_GREATER - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); -#else - await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); -#endif - - await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - _logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString()); - throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); - } - - if (stdOut.Length > 0) - { - _logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString()); - } - - if (stdErr.Length > 0) - { - _logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString()); - } - } - - private static async Task CreateArchiveAsync( - string archivePath, - DateTimeOffset exportedAt, - string metadataPath, - string dbPath, - CancellationToken cancellationToken) - { - await using var archiveStream = new FileStream( - archivePath, - FileMode.Create, - FileAccess.Write, - FileShare.None, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true); - await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false); - - var timestamp = exportedAt.UtcDateTime; - foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath)) - { - cancellationToken.ThrowIfCancellationRequested(); - - var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name) - { - ModificationTime = timestamp, - Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, - }; - - await using var source = new FileStream( - file.Path, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - entry.DataStream = source; - writer.WriteEntry(entry); - } - - await writer.DisposeAsync().ConfigureAwait(false); - await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false); - } - - private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath) - { - yield return ("metadata.json", metadataPath); - yield return ("trivy.db", dbPath); - } - - private static async Task ComputeDigestAsync(string archivePath, CancellationToken cancellationToken) - { - await using var stream = new FileStream( - archivePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); - return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; - } - - private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) - { - await using var stream = new FileStream( - archivePath, - FileMode.Open, - FileAccess.ReadWrite, - FileShare.None, - bufferSize: 8, - options: FileOptions.Asynchronous); - - if (stream.Length < 10) - { - return; - } - - stream.Position = 4; - var zeros = new byte[4]; - await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); - await stream.FlushAsync(cancellationToken).ConfigureAwait(false); - } - - private static string ToGoDuration(TimeSpan span) - { - if (span <= TimeSpan.Zero) - { - return "0s"; - } - - span = span.Duration(); - var builder = new StringBuilder(); - - var totalHours = (int)span.TotalHours; - if (totalHours > 0) - { - builder.Append(totalHours); - builder.Append('h'); - } - - var minutes = span.Minutes; - if (minutes > 0) - { - builder.Append(minutes); - builder.Append('m'); - } - - var seconds = span.Seconds + span.Milliseconds / 1000.0; - if (seconds > 0 || builder.Length == 0) - { - if (span.Milliseconds == 0) - { - builder.Append(span.Seconds); - } - else - { - builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture)); - } - builder.Append('s'); - } - - return builder.ToString(); - } - -} +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Formats.Tar; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Exporter.Json; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbBoltBuilder : ITrivyDbBuilder +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbBoltBuilder(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task BuildAsync( + JsonExportResult jsonTree, + DateTimeOffset exportedAt, + string exportId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(jsonTree); + ArgumentException.ThrowIfNullOrEmpty(exportId); + + var builderRoot = PrepareBuilderRoot(jsonTree.ExportDirectory, exportId); + var outputDir = Path.Combine(builderRoot, "out"); + Directory.CreateDirectory(outputDir); + + try + { + await RunCliAsync(jsonTree.ExportDirectory, outputDir, cancellationToken).ConfigureAwait(false); + } + catch + { + TryDeleteDirectory(builderRoot); + throw; + } + + var metadataPath = Path.Combine(outputDir, "metadata.json"); + var dbPath = Path.Combine(outputDir, "trivy.db"); + + if (!File.Exists(metadataPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy-db metadata not found at '{metadataPath}'."); + } + + if (!File.Exists(dbPath)) + { + TryDeleteDirectory(builderRoot); + throw new InvalidOperationException($"trivy.db not found at '{dbPath}'."); + } + + var archivePath = Path.Combine(builderRoot, "db.tar.gz"); + await CreateArchiveAsync(archivePath, exportedAt, metadataPath, dbPath, cancellationToken).ConfigureAwait(false); + + var digest = await ComputeDigestAsync(archivePath, cancellationToken).ConfigureAwait(false); + var length = new FileInfo(archivePath).Length; + var builderMetadata = await File.ReadAllBytesAsync(metadataPath, cancellationToken).ConfigureAwait(false); + + return new TrivyDbBuilderResult( + archivePath, + digest, + length, + builderMetadata, + builderRoot); + } + + private string PrepareBuilderRoot(string exportDirectory, string exportId) + { + var root = Path.Combine(exportDirectory, $".builder-{exportId}"); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + return root; + } + + private static void TryDeleteDirectory(string directory) + { + try + { + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + catch + { + // ignore cleanup failures + } + } + + private async Task RunCliAsync(string cacheDir, string outputDir, CancellationToken cancellationToken) + { + var builderOptions = _options.Builder ?? new TrivyDbBuilderOptions(); + var executable = string.IsNullOrWhiteSpace(builderOptions.ExecutablePath) + ? "trivy-db" + : builderOptions.ExecutablePath; + + var targets = builderOptions.OnlyUpdateTargets ?? new System.Collections.Generic.List(); + var environment = builderOptions.Environment ?? new System.Collections.Generic.Dictionary(StringComparer.OrdinalIgnoreCase); + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("build"); + startInfo.ArgumentList.Add("--cache-dir"); + startInfo.ArgumentList.Add(cacheDir); + startInfo.ArgumentList.Add("--output-dir"); + startInfo.ArgumentList.Add(outputDir); + + if (builderOptions.UpdateInterval != default) + { + startInfo.ArgumentList.Add("--update-interval"); + startInfo.ArgumentList.Add(ToGoDuration(builderOptions.UpdateInterval)); + } + + if (targets.Count > 0) + { + foreach (var target in targets.Where(static t => !string.IsNullOrWhiteSpace(t))) + { + startInfo.ArgumentList.Add("--only-update"); + startInfo.ArgumentList.Add(target); + } + } + + if (!string.IsNullOrWhiteSpace(builderOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = builderOptions.WorkingDirectory; + } + + if (!builderOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + foreach (var kvp in environment) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + + using var process = new Process { StartInfo = startInfo, EnableRaisingEvents = false }; + + var stdOut = new StringBuilder(); + var stdErr = new StringBuilder(); + + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdOut.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stdErr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Running {Executable} to build Trivy DB", executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // Ignore kill failures. + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("trivy-db exited with code {ExitCode}. stderr: {Stderr}", process.ExitCode, stdErr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdOut.Length > 0) + { + _logger.LogDebug("trivy-db output: {StdOut}", stdOut.ToString()); + } + + if (stdErr.Length > 0) + { + _logger.LogWarning("trivy-db warnings: {StdErr}", stdErr.ToString()); + } + } + + private static async Task CreateArchiveAsync( + string archivePath, + DateTimeOffset exportedAt, + string metadataPath, + string dbPath, + CancellationToken cancellationToken) + { + await using var archiveStream = new FileStream( + archivePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var gzip = new GZipStream(archiveStream, CompressionLevel.SmallestSize, leaveOpen: true); + await using var writer = new TarWriter(gzip, TarEntryFormat.Pax, leaveOpen: false); + + var timestamp = exportedAt.UtcDateTime; + foreach (var file in EnumerateArchiveEntries(metadataPath, dbPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var entry = new PaxTarEntry(TarEntryType.RegularFile, file.Name) + { + ModificationTime = timestamp, + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.GroupRead | UnixFileMode.OtherRead, + }; + + await using var source = new FileStream( + file.Path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + entry.DataStream = source; + writer.WriteEntry(entry); + } + + await writer.DisposeAsync().ConfigureAwait(false); + await ZeroGzipMtimeAsync(archivePath, cancellationToken).ConfigureAwait(false); + } + + private static IEnumerable<(string Name, string Path)> EnumerateArchiveEntries(string metadataPath, string dbPath) + { + yield return ("metadata.json", metadataPath); + yield return ("trivy.db", dbPath); + } + + private static async Task ComputeDigestAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static async Task ZeroGzipMtimeAsync(string archivePath, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + archivePath, + FileMode.Open, + FileAccess.ReadWrite, + FileShare.None, + bufferSize: 8, + options: FileOptions.Asynchronous); + + if (stream.Length < 10) + { + return; + } + + stream.Position = 4; + var zeros = new byte[4]; + await stream.WriteAsync(zeros, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + private static string ToGoDuration(TimeSpan span) + { + if (span <= TimeSpan.Zero) + { + return "0s"; + } + + span = span.Duration(); + var builder = new StringBuilder(); + + var totalHours = (int)span.TotalHours; + if (totalHours > 0) + { + builder.Append(totalHours); + builder.Append('h'); + } + + var minutes = span.Minutes; + if (minutes > 0) + { + builder.Append(minutes); + builder.Append('m'); + } + + var seconds = span.Seconds + span.Milliseconds / 1000.0; + if (seconds > 0 || builder.Length == 0) + { + if (span.Milliseconds == 0) + { + builder.Append(span.Seconds); + } + else + { + builder.Append(seconds.ToString("0.###", CultureInfo.InvariantCulture)); + } + builder.Append('s'); + } + + return builder.ToString(); + } + +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBuilderResult.cs similarity index 77% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBuilderResult.cs index cba55dbd..951f8969 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbBuilderResult.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbBuilderResult.cs @@ -1,10 +1,10 @@ -using System; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record TrivyDbBuilderResult( - string ArchivePath, - string ArchiveDigest, - long ArchiveLength, - ReadOnlyMemory BuilderMetadata, - string WorkingDirectory); +using System; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record TrivyDbBuilderResult( + string ArchivePath, + string ArchiveDigest, + long ArchiveLength, + ReadOnlyMemory BuilderMetadata, + string WorkingDirectory); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs similarity index 94% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs index 3d0acb02..9158b1de 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportJob.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs @@ -1,94 +1,94 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Core.Jobs; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbExportJob : IJob -{ - public const string JobKind = "export:trivy-db"; - public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20); - public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10); - - private readonly TrivyDbFeedExporter _exporter; - private readonly ILogger _logger; - - public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger logger) - { - _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) - { - _logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId); - var overrides = CreateOverrides(context.Parameters); - if (overrides?.HasOverrides == true) - { - using var scope = TrivyDbExportOverrideScope.Begin(overrides); - await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); - } - else - { - await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); - } - - _logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId); - } - - private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary parameters) - { - if (parameters is null || parameters.Count == 0) - { - return null; - } - - var publishFull = TryReadBoolean(parameters, "publishFull"); - var publishDelta = TryReadBoolean(parameters, "publishDelta"); - var includeFull = TryReadBoolean(parameters, "includeFull"); - var includeDelta = TryReadBoolean(parameters, "includeDelta"); - - var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta); - return overrides.HasOverrides ? overrides : null; - } - - private static bool? TryReadBoolean(IReadOnlyDictionary parameters, string key) - { - if (!parameters.TryGetValue(key, out var value) || value is null) - { - return null; - } - - switch (value) - { - case bool b: - return b; - case string s when bool.TryParse(s, out var result): - return result; - case JsonElement element: - return element.ValueKind switch - { - JsonValueKind.True => true, - JsonValueKind.False => false, - JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed, - _ => null, - }; - case IConvertible convertible: - try - { - return convertible.ToBoolean(CultureInfo.InvariantCulture); - } - catch - { - return null; - } - } - - return null; - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Core.Jobs; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbExportJob : IJob +{ + public const string JobKind = "export:trivy-db"; + public static readonly TimeSpan DefaultTimeout = TimeSpan.FromMinutes(20); + public static readonly TimeSpan DefaultLeaseDuration = TimeSpan.FromMinutes(10); + + private readonly TrivyDbFeedExporter _exporter; + private readonly ILogger _logger; + + public TrivyDbExportJob(TrivyDbFeedExporter exporter, ILogger logger) + { + _exporter = exporter ?? throw new ArgumentNullException(nameof(exporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken) + { + _logger.LogInformation("Executing Trivy DB export job {RunId}", context.RunId); + var overrides = CreateOverrides(context.Parameters); + if (overrides?.HasOverrides == true) + { + using var scope = TrivyDbExportOverrideScope.Begin(overrides); + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + } + else + { + await _exporter.ExportAsync(context.Services, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation("Completed Trivy DB export job {RunId}", context.RunId); + } + + private static TrivyDbExportOverrides? CreateOverrides(IReadOnlyDictionary parameters) + { + if (parameters is null || parameters.Count == 0) + { + return null; + } + + var publishFull = TryReadBoolean(parameters, "publishFull"); + var publishDelta = TryReadBoolean(parameters, "publishDelta"); + var includeFull = TryReadBoolean(parameters, "includeFull"); + var includeDelta = TryReadBoolean(parameters, "includeDelta"); + + var overrides = new TrivyDbExportOverrides(publishFull, publishDelta, includeFull, includeDelta); + return overrides.HasOverrides ? overrides : null; + } + + private static bool? TryReadBoolean(IReadOnlyDictionary parameters, string key) + { + if (!parameters.TryGetValue(key, out var value) || value is null) + { + return null; + } + + switch (value) + { + case bool b: + return b; + case string s when bool.TryParse(s, out var result): + return result; + case JsonElement element: + return element.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(element.GetString(), out var parsed) => parsed, + _ => null, + }; + case IConvertible convertible: + try + { + return convertible.ToBoolean(CultureInfo.InvariantCulture); + } + catch + { + return null; + } + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportMode.cs similarity index 54% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportMode.cs index 25dc5a8c..862ebbec 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportMode.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportMode.cs @@ -1,8 +1,8 @@ -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public enum TrivyDbExportMode -{ - Full, - Delta, - Skip, -} +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public enum TrivyDbExportMode +{ + Full, + Delta, + Skip, +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOptions.cs similarity index 77% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOptions.cs index 8adec7b2..bdbdbddc 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOptions.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOptions.cs @@ -1,15 +1,15 @@ using System; using System.IO; using System.Collections.Generic; -using StellaOps.Feedser.Exporter.Json; +using StellaOps.Concelier.Exporter.Json; -namespace StellaOps.Feedser.Exporter.TrivyDb; +namespace StellaOps.Concelier.Exporter.TrivyDb; public sealed class TrivyDbExportOptions { public string OutputRoot { get; set; } = Path.Combine("exports", "trivy"); - public string ReferencePrefix { get; set; } = "feedser/trivy"; + public string ReferencePrefix { get; set; } = "concelier/trivy"; public string TagFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; @@ -24,25 +24,43 @@ public sealed class TrivyDbExportOptions OutputRoot = Path.Combine("exports", "trivy", "tree") }; - public TrivyDbBuilderOptions Builder { get; set; } = new(); - - public TrivyDbOrasOptions Oras { get; set; } = new(); - - public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new(); - - public string GetExportRoot(string exportId) - { - ArgumentException.ThrowIfNullOrEmpty(exportId); - var root = Path.GetFullPath(OutputRoot); - return Path.Combine(root, exportId); - } -} - -public sealed class TrivyDbBuilderOptions -{ - public string ExecutablePath { get; set; } = "trivy-db"; - - public string? WorkingDirectory { get; set; } + public TrivyDbBuilderOptions Builder { get; set; } = new(); + + public TrivyDbOrasOptions Oras { get; set; } = new(); + + public TrivyDbOfflineBundleOptions OfflineBundle { get; set; } = new(); + + public TrivyDbMirrorOptions Mirror { get; set; } = new(); + + public string GetExportRoot(string exportId) + { + ArgumentException.ThrowIfNullOrEmpty(exportId); + var root = Path.GetFullPath(OutputRoot); + return Path.Combine(root, exportId); + } +} + +public sealed class TrivyDbMirrorOptions +{ + public bool Enabled { get; set; } + + public string DirectoryName { get; set; } = "mirror"; + + public IList Domains { get; } = new List(); +} + +public sealed class TrivyDbMirrorDomainOptions +{ + public string Id { get; set; } = string.Empty; + + public string? DisplayName { get; set; } +} + +public sealed class TrivyDbBuilderOptions +{ + public string ExecutablePath { get; set; } = "trivy-db"; + + public string? WorkingDirectory { get; set; } public TimeSpan UpdateInterval { get; set; } = TimeSpan.FromHours(24); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOverrides.cs similarity index 92% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOverrides.cs index 6da26b0a..40fa6164 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportOverrides.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOverrides.cs @@ -1,50 +1,50 @@ -using System; -using System.Threading; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -internal sealed record TrivyDbExportOverrides( - bool? PublishFull, - bool? PublishDelta, - bool? IncludeFull, - bool? IncludeDelta) -{ - public bool HasOverrides => - PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue; -} - -internal static class TrivyDbExportOverrideScope -{ - private sealed class Scope : IDisposable - { - private readonly TrivyDbExportOverrides? _previous; - private bool _disposed; - - public Scope(TrivyDbExportOverrides? previous) - { - _previous = previous; - } - - public void Dispose() - { - if (_disposed) - { - return; - } - - _disposed = true; - CurrentOverrides.Value = _previous; - } - } - - private static readonly AsyncLocal CurrentOverrides = new(); - - public static TrivyDbExportOverrides? Current => CurrentOverrides.Value; - - public static IDisposable Begin(TrivyDbExportOverrides overrides) - { - var previous = CurrentOverrides.Value; - CurrentOverrides.Value = overrides; - return new Scope(previous); - } -} +using System; +using System.Threading; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +internal sealed record TrivyDbExportOverrides( + bool? PublishFull, + bool? PublishDelta, + bool? IncludeFull, + bool? IncludeDelta) +{ + public bool HasOverrides => + PublishFull.HasValue || PublishDelta.HasValue || IncludeFull.HasValue || IncludeDelta.HasValue; +} + +internal static class TrivyDbExportOverrideScope +{ + private sealed class Scope : IDisposable + { + private readonly TrivyDbExportOverrides? _previous; + private bool _disposed; + + public Scope(TrivyDbExportOverrides? previous) + { + _previous = previous; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + CurrentOverrides.Value = _previous; + } + } + + private static readonly AsyncLocal CurrentOverrides = new(); + + public static TrivyDbExportOverrides? Current => CurrentOverrides.Value; + + public static IDisposable Begin(TrivyDbExportOverrides overrides) + { + var previous = CurrentOverrides.Value; + CurrentOverrides.Value = overrides; + return new Scope(previous); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlan.cs similarity index 75% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlan.cs index f75a2644..39411209 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlan.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlan.cs @@ -1,14 +1,14 @@ -namespace StellaOps.Feedser.Exporter.TrivyDb; - -using System.Collections.Generic; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -public sealed record TrivyDbExportPlan( - TrivyDbExportMode Mode, - string TreeDigest, - string? BaseExportId, - string? BaseManifestDigest, - bool ResetBaseline, - IReadOnlyList Manifest, - IReadOnlyList ChangedFiles, - IReadOnlyList RemovedPaths); +namespace StellaOps.Concelier.Exporter.TrivyDb; + +using System.Collections.Generic; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +public sealed record TrivyDbExportPlan( + TrivyDbExportMode Mode, + string TreeDigest, + string? BaseExportId, + string? BaseManifestDigest, + bool ResetBaseline, + IReadOnlyList Manifest, + IReadOnlyList ChangedFiles, + IReadOnlyList RemovedPaths); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlanner.cs similarity index 93% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlanner.cs index b82b73c7..d465abf7 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExportPlanner.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportPlanner.cs @@ -1,115 +1,115 @@ -using System; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -public sealed class TrivyDbExportPlanner -{ - public TrivyDbExportPlan CreatePlan( - ExportStateRecord? existingState, - string treeDigest, - IReadOnlyList manifest) - { - ArgumentException.ThrowIfNullOrEmpty(treeDigest); - manifest ??= Array.Empty(); - - if (existingState is null || (existingState.Files?.Count ?? 0) == 0) - { - return new TrivyDbExportPlan( - TrivyDbExportMode.Full, - treeDigest, - BaseExportId: existingState?.BaseExportId, - BaseManifestDigest: existingState?.LastFullDigest, - ResetBaseline: true, - Manifest: manifest, - ChangedFiles: manifest, - RemovedPaths: Array.Empty()); - } - - var existingFiles = existingState.Files ?? Array.Empty(); - var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal); - if (cursorMatches) - { - return new TrivyDbExportPlan( - TrivyDbExportMode.Skip, - treeDigest, - existingState.BaseExportId, - existingState.LastFullDigest, - ResetBaseline: false, - Manifest: existingFiles, - ChangedFiles: Array.Empty(), - RemovedPaths: Array.Empty()); - } - - var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); - var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); - - var removed = existingMap.Keys - .Where(path => !newMap.ContainsKey(path)) - .ToArray(); - - if (removed.Length > 0) - { - return new TrivyDbExportPlan( - TrivyDbExportMode.Full, - treeDigest, - existingState.BaseExportId, - existingState.LastFullDigest, - ResetBaseline: true, - Manifest: manifest, - ChangedFiles: manifest, - RemovedPaths: removed); - } - - var changed = new List(); - foreach (var file in manifest) - { - if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal)) - { - changed.Add(file); - } - } - - if (changed.Count == 0) - { - return new TrivyDbExportPlan( - TrivyDbExportMode.Skip, - treeDigest, - existingState.BaseExportId, - existingState.LastFullDigest, - ResetBaseline: false, - Manifest: existingFiles, - ChangedFiles: Array.Empty(), - RemovedPaths: Array.Empty()); - } - - var hasOutstandingDelta = existingState.LastDeltaDigest is not null; - if (hasOutstandingDelta) - { - return new TrivyDbExportPlan( - TrivyDbExportMode.Full, - treeDigest, - existingState.BaseExportId, - existingState.LastFullDigest, - ResetBaseline: true, - Manifest: manifest, - ChangedFiles: manifest, - RemovedPaths: Array.Empty()); - } - - return new TrivyDbExportPlan( - TrivyDbExportMode.Delta, - treeDigest, - existingState.BaseExportId, - existingState.LastFullDigest, - ResetBaseline: false, - Manifest: manifest, - ChangedFiles: changed, - RemovedPaths: Array.Empty()); - } -} +using System; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +public sealed class TrivyDbExportPlanner +{ + public TrivyDbExportPlan CreatePlan( + ExportStateRecord? existingState, + string treeDigest, + IReadOnlyList manifest) + { + ArgumentException.ThrowIfNullOrEmpty(treeDigest); + manifest ??= Array.Empty(); + + if (existingState is null || (existingState.Files?.Count ?? 0) == 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + BaseExportId: existingState?.BaseExportId, + BaseManifestDigest: existingState?.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: Array.Empty()); + } + + var existingFiles = existingState.Files ?? Array.Empty(); + var cursorMatches = string.Equals(existingState.ExportCursor, treeDigest, StringComparison.Ordinal); + if (cursorMatches) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); + } + + var existingMap = existingFiles.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + var newMap = manifest.ToDictionary(static file => file.Path, StringComparer.OrdinalIgnoreCase); + + var removed = existingMap.Keys + .Where(path => !newMap.ContainsKey(path)) + .ToArray(); + + if (removed.Length > 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: removed); + } + + var changed = new List(); + foreach (var file in manifest) + { + if (!existingMap.TryGetValue(file.Path, out var previous) || !string.Equals(previous.Digest, file.Digest, StringComparison.Ordinal)) + { + changed.Add(file); + } + } + + if (changed.Count == 0) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Skip, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: existingFiles, + ChangedFiles: Array.Empty(), + RemovedPaths: Array.Empty()); + } + + var hasOutstandingDelta = existingState.LastDeltaDigest is not null; + if (hasOutstandingDelta) + { + return new TrivyDbExportPlan( + TrivyDbExportMode.Full, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: true, + Manifest: manifest, + ChangedFiles: manifest, + RemovedPaths: Array.Empty()); + } + + return new TrivyDbExportPlan( + TrivyDbExportMode.Delta, + treeDigest, + existingState.BaseExportId, + existingState.LastFullDigest, + ResetBaseline: false, + Manifest: manifest, + ChangedFiles: changed, + RemovedPaths: Array.Empty()); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs similarity index 86% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs index c55cb223..18e45c7d 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterDependencyInjectionRoutine.cs @@ -1,64 +1,64 @@ -using System; -using System.IO; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.DependencyInjection; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine -{ - private const string ConfigurationSection = "feedser:exporters:trivyDb"; - - public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) - { - ArgumentNullException.ThrowIfNull(services); - ArgumentNullException.ThrowIfNull(configuration); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddOptions() - .Bind(configuration.GetSection(ConfigurationSection)) - .PostConfigure(static options => - { - options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy")); - options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree")); - options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat; - options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat; - options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "feedser/trivy" : options.ReferencePrefix; - }); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddTransient(); - - services.PostConfigure(options => - { - if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind)) - { - options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition( - TrivyDbExportJob.JobKind, - typeof(TrivyDbExportJob), - TrivyDbExportJob.DefaultTimeout, - TrivyDbExportJob.DefaultLeaseDuration, - null, - true); - } - }); - - return services; - } - - private static string Normalize(string? value, string fallback) - => string.IsNullOrWhiteSpace(value) ? fallback : value; -} +using System; +using System.IO; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.DependencyInjection; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Storage.Mongo.Exporting; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbExporterDependencyInjectionRoutine : IDependencyInjectionRoutine +{ + private const string ConfigurationSection = "concelier:exporters:trivyDb"; + + public IServiceCollection Register(IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Bind(configuration.GetSection(ConfigurationSection)) + .PostConfigure(static options => + { + options.OutputRoot = Normalize(options.OutputRoot, Path.Combine("exports", "trivy")); + options.Json.OutputRoot = Normalize(options.Json.OutputRoot, Path.Combine("exports", "trivy", "tree")); + options.TagFormat = string.IsNullOrWhiteSpace(options.TagFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.TagFormat; + options.DatabaseVersionFormat = string.IsNullOrWhiteSpace(options.DatabaseVersionFormat) ? "yyyyMMdd'T'HHmmss'Z'" : options.DatabaseVersionFormat; + options.ReferencePrefix = string.IsNullOrWhiteSpace(options.ReferencePrefix) ? "concelier/trivy" : options.ReferencePrefix; + }); + + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddTransient(); + + services.PostConfigure(options => + { + if (!options.Definitions.ContainsKey(TrivyDbExportJob.JobKind)) + { + options.Definitions[TrivyDbExportJob.JobKind] = new JobDefinition( + TrivyDbExportJob.JobKind, + typeof(TrivyDbExportJob), + TrivyDbExportJob.DefaultTimeout, + TrivyDbExportJob.DefaultLeaseDuration, + null, + true); + } + }); + + return services; + } + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value; +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterPlugin.cs similarity index 83% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterPlugin.cs index aab67973..2aa230e1 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbExporterPlugin.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExporterPlugin.cs @@ -1,23 +1,23 @@ -using System; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Plugin; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbExporterPlugin : IExporterPlugin -{ - public string Name => TrivyDbFeedExporter.ExporterName; - - public bool IsAvailable(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return services.GetService() is not null; - } - - public IFeedExporter Create(IServiceProvider services) - { - ArgumentNullException.ThrowIfNull(services); - return ActivatorUtilities.CreateInstance(services); - } -} +using System; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Plugin; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbExporterPlugin : IExporterPlugin +{ + public string Name => TrivyDbFeedExporter.ExporterName; + + public bool IsAvailable(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return services.GetService() is not null; + } + + public IFeedExporter Create(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + return ActivatorUtilities.CreateInstance(services); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs similarity index 92% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs index 11a1aa88..04598dbc 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbFeedExporter.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbFeedExporter.cs @@ -12,13 +12,13 @@ using System.Threading.Tasks; using System.Formats.Tar; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Exporting; using StellaOps.Plugin; -namespace StellaOps.Feedser.Exporter.TrivyDb; +namespace StellaOps.Concelier.Exporter.TrivyDb; public sealed class TrivyDbFeedExporter : IFeedExporter { @@ -118,6 +118,8 @@ public sealed class TrivyDbFeedExporter : IFeedExporter var builderResult = await _builder.BuildAsync(jsonResult, exportedAt, exportId, cancellationToken).ConfigureAwait(false); var metadataBytes = CreateMetadataJson(plan, builderResult.BuilderMetadata, treeDigest, jsonResult, exportedAt); + var metadataDigest = ComputeDigest(metadataBytes); + var metadataLength = metadataBytes.LongLength; try { @@ -137,6 +139,22 @@ public sealed class TrivyDbFeedExporter : IFeedExporter } var ociResult = await _ociWriter.WriteAsync(package, destination, reference, plan, baseLayout, cancellationToken).ConfigureAwait(false); + + await TrivyDbMirrorBundleWriter.WriteAsync( + destination, + jsonResult, + _options, + plan, + builderResult, + reference, + ociResult.ManifestDigest, + metadataBytes, + metadataDigest, + metadataLength, + _exporterVersion, + exportedAt, + _logger, + cancellationToken).ConfigureAwait(false); if (_options.Oras.Enabled && ShouldPublishToOras(plan.Mode)) { @@ -421,6 +439,13 @@ public sealed class TrivyDbFeedExporter : IFeedExporter return string.IsNullOrEmpty(normalized) ? "." : normalized; } + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } + private bool ShouldPublishToOras(TrivyDbExportMode mode) { var overrides = TrivyDbExportOverrideScope.Current; diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMediaTypes.cs similarity index 87% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMediaTypes.cs index cf667440..be225698 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbMediaTypes.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMediaTypes.cs @@ -1,9 +1,9 @@ -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public static class TrivyDbMediaTypes -{ - public const string OciManifest = "application/vnd.oci.image.manifest.v1+json"; - public const string OciImageIndex = "application/vnd.oci.image.index.v1+json"; - public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json"; - public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"; -} +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public static class TrivyDbMediaTypes +{ + public const string OciManifest = "application/vnd.oci.image.manifest.v1+json"; + public const string OciImageIndex = "application/vnd.oci.image.index.v1+json"; + public const string TrivyConfig = "application/vnd.aquasec.trivy.config.v1+json"; + public const string TrivyLayer = "application/vnd.aquasec.trivy.db.layer.v1.tar+gzip"; +} diff --git a/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMirrorBundleWriter.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMirrorBundleWriter.cs new file mode 100644 index 00000000..594d0b60 --- /dev/null +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbMirrorBundleWriter.cs @@ -0,0 +1,392 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Exporter.Json; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +internal static class TrivyDbMirrorBundleWriter +{ + private const int SchemaVersion = 1; + private const string DefaultDirectoryName = "mirror"; + private const string MetadataFileName = "metadata.json"; + private const string DatabaseFileName = "db.tar.gz"; + private const string ManifestFileName = "manifest.json"; + + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + public static async Task WriteAsync( + string layoutRoot, + JsonExportResult jsonResult, + TrivyDbExportOptions options, + TrivyDbExportPlan plan, + TrivyDbBuilderResult builderResult, + string reference, + string manifestDigest, + ReadOnlyMemory metadataBytes, + string metadataDigest, + long metadataLength, + string exporterVersion, + DateTimeOffset exportedAt, + ILogger logger, + CancellationToken cancellationToken) + { + if (options?.Mirror is null || !options.Mirror.Enabled || options.Mirror.Domains.Count == 0) + { + return; + } + + if (string.IsNullOrWhiteSpace(layoutRoot)) + { + throw new ArgumentException("Layout root must be provided.", nameof(layoutRoot)); + } + + if (builderResult is null) + { + throw new ArgumentNullException(nameof(builderResult)); + } + + if (jsonResult is null) + { + throw new ArgumentNullException(nameof(jsonResult)); + } + + var directoryName = string.IsNullOrWhiteSpace(options.Mirror.DirectoryName) + ? DefaultDirectoryName + : options.Mirror.DirectoryName.Trim(); + + if (directoryName.Length == 0) + { + directoryName = DefaultDirectoryName; + } + + var root = Path.Combine(layoutRoot, directoryName); + Directory.CreateDirectory(root); + + var timestamp = exportedAt.UtcDateTime; + TrySetDirectoryTimestamp(root, timestamp); + + var advisories = jsonResult.Advisories.IsDefaultOrEmpty + ? Array.Empty() + : jsonResult.Advisories + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.Ordinal) + .ToArray(); + + var domains = new List(); + + foreach (var domainOption in options.Mirror.Domains) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (domainOption is null) + { + logger.LogWarning("Encountered null Trivy mirror domain configuration; skipping."); + continue; + } + + var domainId = (domainOption.Id ?? string.Empty).Trim(); + if (domainId.Length == 0) + { + logger.LogWarning("Skipping Trivy mirror domain with empty id."); + continue; + } + + var displayName = string.IsNullOrWhiteSpace(domainOption.DisplayName) + ? domainId + : domainOption.DisplayName!.Trim(); + + var domainDirectory = Path.Combine(root, domainId); + Directory.CreateDirectory(domainDirectory); + TrySetDirectoryTimestamp(domainDirectory, timestamp); + + var metadataPath = Path.Combine(domainDirectory, MetadataFileName); + await WriteFileAsync(metadataPath, metadataBytes, timestamp, cancellationToken).ConfigureAwait(false); + var metadataRelativePath = ToRelativePath(layoutRoot, metadataPath); + + var databasePath = Path.Combine(domainDirectory, DatabaseFileName); + await CopyDatabaseAsync(builderResult.ArchivePath, databasePath, timestamp, cancellationToken).ConfigureAwait(false); + var databaseRelativePath = ToRelativePath(layoutRoot, databasePath); + + var sources = BuildSourceSummaries(advisories); + + var manifestDocument = new MirrorDomainManifestDocument( + SchemaVersion, + exportedAt, + exporterVersion, + reference, + manifestDigest, + options.TargetRepository, + domainId, + displayName, + plan.Mode.ToString().ToLowerInvariant(), + plan.BaseExportId, + plan.BaseManifestDigest, + plan.ResetBaseline, + new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest), + new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest), + sources); + + var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(manifestDocument, SerializerOptions); + var manifestPath = Path.Combine(domainDirectory, ManifestFileName); + await WriteFileAsync(manifestPath, manifestBytes, timestamp, cancellationToken).ConfigureAwait(false); + var manifestRelativePath = ToRelativePath(layoutRoot, manifestPath); + var manifestDigestValue = ComputeDigest(manifestBytes); + + domains.Add(new MirrorIndexDomainEntry( + domainId, + displayName, + advisories.Length, + new MirrorFileDescriptor(manifestRelativePath, manifestBytes.LongLength, manifestDigestValue), + new MirrorFileDescriptor(metadataRelativePath, metadataLength, metadataDigest), + new MirrorFileDescriptor(databaseRelativePath, builderResult.ArchiveLength, builderResult.ArchiveDigest), + sources)); + } + + if (domains.Count == 0) + { + Directory.Delete(root, recursive: true); + return; + } + + domains.Sort(static (left, right) => string.CompareOrdinal(left.DomainId, right.DomainId)); + + var delta = plan.Mode == TrivyDbExportMode.Delta + ? new MirrorDeltaMetadata( + plan.ChangedFiles.Select(static file => new MirrorDeltaFile(file.Path, file.Digest)).ToArray(), + plan.RemovedPaths.ToArray()) + : null; + + var indexDocument = new MirrorIndexDocument( + SchemaVersion, + exportedAt, + exporterVersion, + options.TargetRepository, + reference, + manifestDigest, + plan.Mode.ToString().ToLowerInvariant(), + plan.BaseExportId, + plan.BaseManifestDigest, + plan.ResetBaseline, + delta, + domains); + + var indexBytes = JsonSerializer.SerializeToUtf8Bytes(indexDocument, SerializerOptions); + var indexPath = Path.Combine(root, "index.json"); + await WriteFileAsync(indexPath, indexBytes, timestamp, cancellationToken).ConfigureAwait(false); + + logger.LogInformation( + "Generated {DomainCount} Trivy DB mirror bundle(s) under {Directory}.", + domains.Count, + directoryName); + } + + private static IReadOnlyList BuildSourceSummaries(IReadOnlyList advisories) + { + if (advisories.Count == 0) + { + return Array.Empty(); + } + + var builders = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var advisory in advisories) + { + var counted = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var provenance in advisory.Provenance) + { + if (string.IsNullOrWhiteSpace(provenance.Source)) + { + continue; + } + + var source = provenance.Source.Trim(); + if (!builders.TryGetValue(source, out var accumulator)) + { + accumulator = new SourceAccumulator(); + builders[source] = accumulator; + } + + accumulator.Record(provenance.RecordedAt); + if (counted.Add(source)) + { + accumulator.Increment(); + } + } + } + + var entries = builders + .Select(static pair => new TrivyMirrorSourceSummary( + pair.Key, + pair.Value.FirstRecordedAt, + pair.Value.LastRecordedAt, + pair.Value.Count)) + .OrderBy(static summary => summary.Source, StringComparer.Ordinal) + .ToArray(); + + return entries; + } + + private static async Task CopyDatabaseAsync( + string sourcePath, + string destinationPath, + DateTime timestamp, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + await using var source = new FileStream( + sourcePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var destination = new FileStream( + destinationPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destinationPath, timestamp); + } + + private static async Task WriteFileAsync( + string path, + ReadOnlyMemory bytes, + DateTime timestamp, + CancellationToken cancellationToken) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await using var stream = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await stream.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, timestamp); + } + + private static string ToRelativePath(string root, string fullPath) + { + var relative = Path.GetRelativePath(root, fullPath); + var normalized = relative.Replace(Path.DirectorySeparatorChar, '/'); + return string.IsNullOrEmpty(normalized) ? "." : normalized; + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + return $"sha256:{hex}"; + } + + private static void TrySetDirectoryTimestamp(string directory, DateTime timestamp) + { + try + { + Directory.SetLastWriteTimeUtc(directory, timestamp); + } + catch + { + // Best effort – ignore failures. + } + } + + private sealed record MirrorIndexDocument( + int SchemaVersion, + DateTimeOffset GeneratedAt, + string ExporterVersion, + string? TargetRepository, + string Reference, + string ManifestDigest, + string Mode, + string? BaseExportId, + string? BaseManifestDigest, + bool ResetBaseline, + MirrorDeltaMetadata? Delta, + IReadOnlyList Domains); + + private sealed record MirrorDeltaMetadata( + IReadOnlyList ChangedFiles, + IReadOnlyList RemovedPaths); + + private sealed record MirrorDeltaFile(string Path, string Digest); + + private sealed record MirrorIndexDomainEntry( + string DomainId, + string DisplayName, + int AdvisoryCount, + MirrorFileDescriptor Manifest, + MirrorFileDescriptor Metadata, + MirrorFileDescriptor Database, + IReadOnlyList Sources); + + private sealed record MirrorDomainManifestDocument( + int SchemaVersion, + DateTimeOffset GeneratedAt, + string ExporterVersion, + string Reference, + string ManifestDigest, + string? TargetRepository, + string DomainId, + string DisplayName, + string Mode, + string? BaseExportId, + string? BaseManifestDigest, + bool ResetBaseline, + MirrorFileDescriptor Metadata, + MirrorFileDescriptor Database, + IReadOnlyList Sources); + + private sealed record MirrorFileDescriptor(string Path, long SizeBytes, string Digest); + + private sealed record TrivyMirrorSourceSummary( + string Source, + DateTimeOffset? FirstRecordedAt, + DateTimeOffset? LastRecordedAt, + int AdvisoryCount); + + private sealed class SourceAccumulator + { + public DateTimeOffset? FirstRecordedAt { get; private set; } + + public DateTimeOffset? LastRecordedAt { get; private set; } + + public int Count { get; private set; } + + public void Record(DateTimeOffset recordedAt) + { + var utc = recordedAt.ToUniversalTime(); + if (FirstRecordedAt is null || utc < FirstRecordedAt.Value) + { + FirstRecordedAt = utc; + } + + if (LastRecordedAt is null || utc > LastRecordedAt.Value) + { + LastRecordedAt = utc; + } + } + + public void Increment() => Count++; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriteResult.cs similarity index 76% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriteResult.cs index 3a22a7ab..09be0056 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriteResult.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriteResult.cs @@ -1,8 +1,8 @@ -using System.Collections.Generic; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record TrivyDbOciWriteResult( - string RootDirectory, - string ManifestDigest, - IReadOnlyCollection BlobDigests); +using System.Collections.Generic; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record TrivyDbOciWriteResult( + string RootDirectory, + string ManifestDigest, + IReadOnlyCollection BlobDigests); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriter.cs similarity index 96% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriter.cs index eaf2a74d..f13ea911 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOciWriter.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOciWriter.cs @@ -1,375 +1,375 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -/// -/// Writes a Trivy DB package to an OCI image layout directory with deterministic content. -/// -public sealed class TrivyDbOciWriter -{ - private static readonly JsonSerializerOptions SerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, - WriteIndented = false, - }; - - private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}"); - - public async Task WriteAsync( - TrivyDbPackage package, - string destination, - string reference, - TrivyDbExportPlan plan, - string? baseLayoutPath, - CancellationToken cancellationToken) - { - if (package is null) - { - throw new ArgumentNullException(nameof(package)); - } - - if (string.IsNullOrWhiteSpace(destination)) - { - throw new ArgumentException("Destination directory must be provided.", nameof(destination)); - } - - if (string.IsNullOrWhiteSpace(reference)) - { - throw new ArgumentException("Reference tag must be provided.", nameof(reference)); - } - - if (plan is null) - { - throw new ArgumentNullException(nameof(plan)); - } - - var root = Path.GetFullPath(destination); - if (Directory.Exists(root)) - { - Directory.Delete(root, recursive: true); - } - - Directory.CreateDirectory(root); - var timestamp = package.Config.GeneratedAt.UtcDateTime; - - await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false); - await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false); - - var blobsRoot = Path.Combine(root, "blobs", "sha256"); - Directory.CreateDirectory(blobsRoot); - Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp); - Directory.SetLastWriteTimeUtc(blobsRoot, timestamp); - - var writtenDigests = new HashSet(StringComparer.Ordinal); - foreach (var pair in package.Blobs) - { - if (writtenDigests.Contains(pair.Key)) - { - continue; - } - - var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); - if (reused) - { - writtenDigests.Add(pair.Key); - continue; - } - - if (writtenDigests.Add(pair.Key)) - { - await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false); - } - } - - var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions); - var manifestDigest = ComputeDigest(manifestBytes); - if (!writtenDigests.Contains(manifestDigest)) - { - var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); - if (!reused) - { - await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false); - } - - writtenDigests.Add(manifestDigest); - } - - var manifestDescriptor = new OciDescriptor( - TrivyDbMediaTypes.OciManifest, - manifestDigest, - manifestBytes.LongLength, - new Dictionary - { - ["org.opencontainers.image.ref.name"] = reference, - }); - var index = new OciIndex(2, new[] { manifestDescriptor }); - var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions); - await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false); - - if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath)) - { - var reuseDigests = await TryReuseBaseBlobsAsync( - blobsRoot, - timestamp, - writtenDigests, - plan, - baseLayoutPath, - cancellationToken).ConfigureAwait(false); - foreach (var digest in reuseDigests) - { - writtenDigests.Add(digest); - } - } - - Directory.SetLastWriteTimeUtc(root, timestamp); - - var blobDigests = writtenDigests.ToArray(); - Array.Sort(blobDigests, StringComparer.Ordinal); - return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests); - } - - private static async Task WriteFileAsync(string path, ReadOnlyMemory bytes, DateTime utcTimestamp, CancellationToken cancellationToken) - { - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - Directory.SetLastWriteTimeUtc(directory, utcTimestamp); - } - - await using var destination = new FileStream( - path, - FileMode.Create, - FileAccess.Write, - FileShare.None, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); - await destination.FlushAsync(cancellationToken).ConfigureAwait(false); - File.SetLastWriteTimeUtc(path, utcTimestamp); - } - - private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken) - { - var fileName = ResolveDigestFileName(digest); - var path = Path.Combine(blobsRoot, fileName); - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - Directory.SetLastWriteTimeUtc(directory, utcTimestamp); - } - - await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false); - await using var destination = new FileStream( - path, - FileMode.Create, - FileAccess.Write, - FileShare.None, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - - await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); - await destination.FlushAsync(cancellationToken).ConfigureAwait(false); - File.SetLastWriteTimeUtc(path, utcTimestamp); - } - - private static string ResolveDigestFileName(string digest) - { - if (!digest.StartsWith("sha256:", StringComparison.Ordinal)) - { - throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'."); - } - - var hex = digest[7..]; - if (hex.Length == 0) - { - throw new InvalidOperationException("Digest hex component cannot be empty."); - } - - return hex; - } - - private static string ComputeDigest(ReadOnlySpan payload) - { - var hash = System.Security.Cryptography.SHA256.HashData(payload); - var hex = Convert.ToHexString(hash); - Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex - buffer[0] = 's'; - buffer[1] = 'h'; - buffer[2] = 'a'; - buffer[3] = '2'; - buffer[4] = '5'; - buffer[5] = '6'; - buffer[6] = ':'; - for (var i = 0; i < hex.Length; i++) - { - buffer[7 + i] = char.ToLowerInvariant(hex[i]); - } - - return new string(buffer); - } - - private static async Task> TryReuseBaseBlobsAsync( - string destinationBlobsRoot, - DateTime timestamp, - HashSet written, - TrivyDbExportPlan plan, - string baseLayoutPath, - CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest)) - { - return Array.Empty(); - } - - var baseRoot = Path.GetFullPath(baseLayoutPath); - if (!Directory.Exists(baseRoot)) - { - return Array.Empty(); - } - - var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest); - if (!File.Exists(manifestPath)) - { - return Array.Empty(); - } - - await using var stream = new FileStream( - manifestPath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); - var root = document.RootElement; - - var digests = new SortedSet(StringComparer.Ordinal) - { - plan.BaseManifestDigest, - }; - - if (root.TryGetProperty("config", out var configNode)) - { - var digest = configNode.GetProperty("digest").GetString(); - if (!string.IsNullOrWhiteSpace(digest)) - { - digests.Add(digest); - } - } - - if (root.TryGetProperty("layers", out var layersNode)) - { - foreach (var layer in layersNode.EnumerateArray()) - { - var digest = layer.GetProperty("digest").GetString(); - if (!string.IsNullOrWhiteSpace(digest)) - { - digests.Add(digest); - } - } - } - - var copied = new List(); - foreach (var digest in digests) - { - if (written.Contains(digest)) - { - continue; - } - - var sourcePath = ResolveBlobPath(baseRoot, digest); - if (!File.Exists(sourcePath)) - { - continue; - } - - var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); - Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); - await using var source = new FileStream( - sourcePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await using var destination = new FileStream( - destinationPath, - FileMode.Create, - FileAccess.Write, - FileShare.None, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); - await destination.FlushAsync(cancellationToken).ConfigureAwait(false); - File.SetLastWriteTimeUtc(destinationPath, timestamp); - copied.Add(digest); - } - - if (copied.Count > 0) - { - Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); - Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); - } - - return copied; - } - - private static string ResolveBlobPath(string layoutRoot, string digest) - { - var fileName = ResolveDigestFileName(digest); - return Path.Combine(layoutRoot, "blobs", "sha256", fileName); - } - - private static async Task TryReuseExistingBlobAsync( - string? baseLayoutPath, - string digest, - string destinationBlobsRoot, - DateTime timestamp, - CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(baseLayoutPath)) - { - return false; - } - - var baseRoot = Path.GetFullPath(baseLayoutPath); - var sourcePath = ResolveBlobPath(baseRoot, digest); - if (!File.Exists(sourcePath)) - { - return false; - } - - var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); - Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); - await using var source = new FileStream( - sourcePath, - FileMode.Open, - FileAccess.Read, - FileShare.Read, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await using var destination = new FileStream( - destinationPath, - FileMode.Create, - FileAccess.Write, - FileShare.None, - bufferSize: 81920, - options: FileOptions.Asynchronous | FileOptions.SequentialScan); - await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); - await destination.FlushAsync(cancellationToken).ConfigureAwait(false); - File.SetLastWriteTimeUtc(destinationPath, timestamp); - Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); - Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); - return true; - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +/// +/// Writes a Trivy DB package to an OCI image layout directory with deterministic content. +/// +public sealed class TrivyDbOciWriter +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + private static readonly byte[] OciLayoutBytes = Encoding.UTF8.GetBytes("{\"imageLayoutVersion\":\"1.0.0\"}"); + + public async Task WriteAsync( + TrivyDbPackage package, + string destination, + string reference, + TrivyDbExportPlan plan, + string? baseLayoutPath, + CancellationToken cancellationToken) + { + if (package is null) + { + throw new ArgumentNullException(nameof(package)); + } + + if (string.IsNullOrWhiteSpace(destination)) + { + throw new ArgumentException("Destination directory must be provided.", nameof(destination)); + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new ArgumentException("Reference tag must be provided.", nameof(reference)); + } + + if (plan is null) + { + throw new ArgumentNullException(nameof(plan)); + } + + var root = Path.GetFullPath(destination); + if (Directory.Exists(root)) + { + Directory.Delete(root, recursive: true); + } + + Directory.CreateDirectory(root); + var timestamp = package.Config.GeneratedAt.UtcDateTime; + + await WriteFileAsync(Path.Combine(root, "metadata.json"), package.MetadataJson, timestamp, cancellationToken).ConfigureAwait(false); + await WriteFileAsync(Path.Combine(root, "oci-layout"), OciLayoutBytes, timestamp, cancellationToken).ConfigureAwait(false); + + var blobsRoot = Path.Combine(root, "blobs", "sha256"); + Directory.CreateDirectory(blobsRoot); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(blobsRoot)!, timestamp); + Directory.SetLastWriteTimeUtc(blobsRoot, timestamp); + + var writtenDigests = new HashSet(StringComparer.Ordinal); + foreach (var pair in package.Blobs) + { + if (writtenDigests.Contains(pair.Key)) + { + continue; + } + + var reused = await TryReuseExistingBlobAsync(baseLayoutPath, pair.Key, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); + if (reused) + { + writtenDigests.Add(pair.Key); + continue; + } + + if (writtenDigests.Add(pair.Key)) + { + await WriteBlobAsync(blobsRoot, pair.Key, pair.Value, timestamp, cancellationToken).ConfigureAwait(false); + } + } + + var manifestBytes = JsonSerializer.SerializeToUtf8Bytes(package.Manifest, SerializerOptions); + var manifestDigest = ComputeDigest(manifestBytes); + if (!writtenDigests.Contains(manifestDigest)) + { + var reused = await TryReuseExistingBlobAsync(baseLayoutPath, manifestDigest, blobsRoot, timestamp, cancellationToken).ConfigureAwait(false); + if (!reused) + { + await WriteBlobAsync(blobsRoot, manifestDigest, TrivyDbBlob.FromBytes(manifestBytes), timestamp, cancellationToken).ConfigureAwait(false); + } + + writtenDigests.Add(manifestDigest); + } + + var manifestDescriptor = new OciDescriptor( + TrivyDbMediaTypes.OciManifest, + manifestDigest, + manifestBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.ref.name"] = reference, + }); + var index = new OciIndex(2, new[] { manifestDescriptor }); + var indexBytes = JsonSerializer.SerializeToUtf8Bytes(index, SerializerOptions); + await WriteFileAsync(Path.Combine(root, "index.json"), indexBytes, timestamp, cancellationToken).ConfigureAwait(false); + + if (plan.Mode == TrivyDbExportMode.Delta && !string.IsNullOrWhiteSpace(baseLayoutPath)) + { + var reuseDigests = await TryReuseBaseBlobsAsync( + blobsRoot, + timestamp, + writtenDigests, + plan, + baseLayoutPath, + cancellationToken).ConfigureAwait(false); + foreach (var digest in reuseDigests) + { + writtenDigests.Add(digest); + } + } + + Directory.SetLastWriteTimeUtc(root, timestamp); + + var blobDigests = writtenDigests.ToArray(); + Array.Sort(blobDigests, StringComparer.Ordinal); + return new TrivyDbOciWriteResult(root, manifestDigest, blobDigests); + } + + private static async Task WriteFileAsync(string path, ReadOnlyMemory bytes, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await destination.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static async Task WriteBlobAsync(string blobsRoot, string digest, TrivyDbBlob blob, DateTime utcTimestamp, CancellationToken cancellationToken) + { + var fileName = ResolveDigestFileName(digest); + var path = Path.Combine(blobsRoot, fileName); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + Directory.SetLastWriteTimeUtc(directory, utcTimestamp); + } + + await using var source = await blob.OpenReadAsync(cancellationToken).ConfigureAwait(false); + await using var destination = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(path, utcTimestamp); + } + + private static string ResolveDigestFileName(string digest) + { + if (!digest.StartsWith("sha256:", StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Only sha256 digests are supported. Received '{digest}'."); + } + + var hex = digest[7..]; + if (hex.Length == 0) + { + throw new InvalidOperationException("Digest hex component cannot be empty."); + } + + return hex; + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = System.Security.Cryptography.SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } + + private static async Task> TryReuseBaseBlobsAsync( + string destinationBlobsRoot, + DateTime timestamp, + HashSet written, + TrivyDbExportPlan plan, + string baseLayoutPath, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(plan.BaseManifestDigest)) + { + return Array.Empty(); + } + + var baseRoot = Path.GetFullPath(baseLayoutPath); + if (!Directory.Exists(baseRoot)) + { + return Array.Empty(); + } + + var manifestPath = ResolveBlobPath(baseRoot, plan.BaseManifestDigest); + if (!File.Exists(manifestPath)) + { + return Array.Empty(); + } + + await using var stream = new FileStream( + manifestPath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + var digests = new SortedSet(StringComparer.Ordinal) + { + plan.BaseManifestDigest, + }; + + if (root.TryGetProperty("config", out var configNode)) + { + var digest = configNode.GetProperty("digest").GetString(); + if (!string.IsNullOrWhiteSpace(digest)) + { + digests.Add(digest); + } + } + + if (root.TryGetProperty("layers", out var layersNode)) + { + foreach (var layer in layersNode.EnumerateArray()) + { + var digest = layer.GetProperty("digest").GetString(); + if (!string.IsNullOrWhiteSpace(digest)) + { + digests.Add(digest); + } + } + } + + var copied = new List(); + foreach (var digest in digests) + { + if (written.Contains(digest)) + { + continue; + } + + var sourcePath = ResolveBlobPath(baseRoot, digest); + if (!File.Exists(sourcePath)) + { + continue; + } + + var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + await using var source = new FileStream( + sourcePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var destination = new FileStream( + destinationPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destinationPath, timestamp); + copied.Add(digest); + } + + if (copied.Count > 0) + { + Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); + } + + return copied; + } + + private static string ResolveBlobPath(string layoutRoot, string digest) + { + var fileName = ResolveDigestFileName(digest); + return Path.Combine(layoutRoot, "blobs", "sha256", fileName); + } + + private static async Task TryReuseExistingBlobAsync( + string? baseLayoutPath, + string digest, + string destinationBlobsRoot, + DateTime timestamp, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(baseLayoutPath)) + { + return false; + } + + var baseRoot = Path.GetFullPath(baseLayoutPath); + var sourcePath = ResolveBlobPath(baseRoot, digest); + if (!File.Exists(sourcePath)) + { + return false; + } + + var destinationPath = Path.Combine(destinationBlobsRoot, ResolveDigestFileName(digest)); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + await using var source = new FileStream( + sourcePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var destination = new FileStream( + destinationPath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 81920, + options: FileOptions.Asynchronous | FileOptions.SequentialScan); + await source.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + File.SetLastWriteTimeUtc(destinationPath, timestamp); + Directory.SetLastWriteTimeUtc(destinationBlobsRoot, timestamp); + Directory.SetLastWriteTimeUtc(Path.GetDirectoryName(destinationBlobsRoot)!, timestamp); + return true; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOrasPusher.cs similarity index 96% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOrasPusher.cs index 5a723a01..341553d1 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbOrasPusher.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbOrasPusher.cs @@ -1,209 +1,209 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher -{ - private readonly TrivyDbExportOptions _options; - private readonly ILogger _logger; - - public TrivyDbOrasPusher(IOptions options, ILogger logger) - { - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) - { - var orasOptions = _options.Oras; - if (!orasOptions.Enabled) - { - return; - } - - if (string.IsNullOrWhiteSpace(reference)) - { - throw new InvalidOperationException("ORAS push requested but reference is empty."); - } - - if (!Directory.Exists(layoutPath)) - { - throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist."); - } - - var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath; - var tag = ResolveTag(reference, exportId); - var layoutReference = $"{layoutPath}:{tag}"; - - var startInfo = new ProcessStartInfo - { - FileName = executable, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - }; - - startInfo.ArgumentList.Add("cp"); - startInfo.ArgumentList.Add("--from-oci-layout"); - startInfo.ArgumentList.Add(layoutReference); - if (orasOptions.SkipTlsVerify) - { - startInfo.ArgumentList.Add("--insecure"); - } - if (orasOptions.UseHttp) - { - startInfo.ArgumentList.Add("--plain-http"); - } - - if (orasOptions.AdditionalArguments is { Count: > 0 }) - { - foreach (var arg in orasOptions.AdditionalArguments) - { - if (!string.IsNullOrWhiteSpace(arg)) - { - startInfo.ArgumentList.Add(arg); - } - } - } - - startInfo.ArgumentList.Add(reference); - - if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory)) - { - startInfo.WorkingDirectory = orasOptions.WorkingDirectory; - } - - if (!orasOptions.InheritEnvironment) - { - startInfo.Environment.Clear(); - } - - if (orasOptions.Environment is { Count: > 0 }) - { - foreach (var kvp in orasOptions.Environment) - { - if (!string.IsNullOrEmpty(kvp.Key)) - { - startInfo.Environment[kvp.Key] = kvp.Value; - } - } - } - - using var process = new Process { StartInfo = startInfo }; - var stdout = new StringBuilder(); - var stderr = new StringBuilder(); - var stdoutCompletion = new TaskCompletionSource(); - var stderrCompletion = new TaskCompletionSource(); - - process.OutputDataReceived += (_, e) => - { - if (e.Data is null) - { - stdoutCompletion.TrySetResult(null); - } - else - { - stdout.AppendLine(e.Data); - } - }; - - process.ErrorDataReceived += (_, e) => - { - if (e.Data is null) - { - stderrCompletion.TrySetResult(null); - } - else - { - stderr.AppendLine(e.Data); - } - }; - - _logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable); - - try - { - if (!process.Start()) - { - throw new InvalidOperationException($"Failed to start '{executable}'."); - } - } - catch (Exception ex) - { - throw new InvalidOperationException($"Failed to start '{executable}'.", ex); - } - - process.BeginOutputReadLine(); - process.BeginErrorReadLine(); - - using var registration = cancellationToken.Register(() => - { - try - { - if (!process.HasExited) - { - process.Kill(entireProcessTree: true); - } - } - catch - { - // ignore - } - }); - -#if NET8_0_OR_GREATER - await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); -#else - await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); -#endif - - await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); - - if (process.ExitCode != 0) - { - _logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString()); - throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); - } - - if (stdout.Length > 0) - { - _logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString()); - } - - if (stderr.Length > 0) - { - _logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString()); - } - } - - private static string ResolveTag(string reference, string fallback) - { - if (string.IsNullOrWhiteSpace(reference)) - { - return fallback; - } - - var atIndex = reference.IndexOf('@'); - if (atIndex >= 0) - { - reference = reference[..atIndex]; - } - - var slashIndex = reference.LastIndexOf('/'); - var colonIndex = reference.LastIndexOf(':'); - if (colonIndex > slashIndex && colonIndex >= 0) - { - return reference[(colonIndex + 1)..]; - } - - return fallback; - } -} +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbOrasPusher : ITrivyDbOrasPusher +{ + private readonly TrivyDbExportOptions _options; + private readonly ILogger _logger; + + public TrivyDbOrasPusher(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PushAsync(string layoutPath, string reference, string exportId, CancellationToken cancellationToken) + { + var orasOptions = _options.Oras; + if (!orasOptions.Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(reference)) + { + throw new InvalidOperationException("ORAS push requested but reference is empty."); + } + + if (!Directory.Exists(layoutPath)) + { + throw new DirectoryNotFoundException($"OCI layout directory '{layoutPath}' does not exist."); + } + + var executable = string.IsNullOrWhiteSpace(orasOptions.ExecutablePath) ? "oras" : orasOptions.ExecutablePath; + var tag = ResolveTag(reference, exportId); + var layoutReference = $"{layoutPath}:{tag}"; + + var startInfo = new ProcessStartInfo + { + FileName = executable, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + }; + + startInfo.ArgumentList.Add("cp"); + startInfo.ArgumentList.Add("--from-oci-layout"); + startInfo.ArgumentList.Add(layoutReference); + if (orasOptions.SkipTlsVerify) + { + startInfo.ArgumentList.Add("--insecure"); + } + if (orasOptions.UseHttp) + { + startInfo.ArgumentList.Add("--plain-http"); + } + + if (orasOptions.AdditionalArguments is { Count: > 0 }) + { + foreach (var arg in orasOptions.AdditionalArguments) + { + if (!string.IsNullOrWhiteSpace(arg)) + { + startInfo.ArgumentList.Add(arg); + } + } + } + + startInfo.ArgumentList.Add(reference); + + if (!string.IsNullOrWhiteSpace(orasOptions.WorkingDirectory)) + { + startInfo.WorkingDirectory = orasOptions.WorkingDirectory; + } + + if (!orasOptions.InheritEnvironment) + { + startInfo.Environment.Clear(); + } + + if (orasOptions.Environment is { Count: > 0 }) + { + foreach (var kvp in orasOptions.Environment) + { + if (!string.IsNullOrEmpty(kvp.Key)) + { + startInfo.Environment[kvp.Key] = kvp.Value; + } + } + } + + using var process = new Process { StartInfo = startInfo }; + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + var stdoutCompletion = new TaskCompletionSource(); + var stderrCompletion = new TaskCompletionSource(); + + process.OutputDataReceived += (_, e) => + { + if (e.Data is null) + { + stdoutCompletion.TrySetResult(null); + } + else + { + stdout.AppendLine(e.Data); + } + }; + + process.ErrorDataReceived += (_, e) => + { + if (e.Data is null) + { + stderrCompletion.TrySetResult(null); + } + else + { + stderr.AppendLine(e.Data); + } + }; + + _logger.LogInformation("Pushing Trivy DB export {ExportId} to {Reference} using {Executable}", exportId, reference, executable); + + try + { + if (!process.Start()) + { + throw new InvalidOperationException($"Failed to start '{executable}'."); + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to start '{executable}'.", ex); + } + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var registration = cancellationToken.Register(() => + { + try + { + if (!process.HasExited) + { + process.Kill(entireProcessTree: true); + } + } + catch + { + // ignore + } + }); + +#if NET8_0_OR_GREATER + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); +#else + await Task.Run(() => process.WaitForExit(), cancellationToken).ConfigureAwait(false); +#endif + + await Task.WhenAll(stdoutCompletion.Task, stderrCompletion.Task).ConfigureAwait(false); + + if (process.ExitCode != 0) + { + _logger.LogError("ORAS push for {Reference} failed with code {Code}. stderr: {Stderr}", reference, process.ExitCode, stderr.ToString()); + throw new InvalidOperationException($"'{executable}' exited with code {process.ExitCode}."); + } + + if (stdout.Length > 0) + { + _logger.LogDebug("ORAS push output: {Stdout}", stdout.ToString()); + } + + if (stderr.Length > 0) + { + _logger.LogWarning("ORAS push warnings: {Stderr}", stderr.ToString()); + } + } + + private static string ResolveTag(string reference, string fallback) + { + if (string.IsNullOrWhiteSpace(reference)) + { + return fallback; + } + + var atIndex = reference.IndexOf('@'); + if (atIndex >= 0) + { + reference = reference[..atIndex]; + } + + var slashIndex = reference.LastIndexOf('/'); + var colonIndex = reference.LastIndexOf(':'); + if (colonIndex > slashIndex && colonIndex >= 0) + { + return reference[(colonIndex + 1)..]; + } + + return fallback; + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackage.cs similarity index 79% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackage.cs index c2d842da..ddd02021 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackage.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackage.cs @@ -1,9 +1,9 @@ -using System.Collections.Generic; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record TrivyDbPackage( - OciManifest Manifest, - TrivyConfigDocument Config, - IReadOnlyDictionary Blobs, - ReadOnlyMemory MetadataJson); +using System.Collections.Generic; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record TrivyDbPackage( + OciManifest Manifest, + TrivyConfigDocument Config, + IReadOnlyDictionary Blobs, + ReadOnlyMemory MetadataJson); diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageBuilder.cs similarity index 95% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageBuilder.cs index 2f5c8a45..4863da9e 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageBuilder.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageBuilder.cs @@ -1,116 +1,116 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO; -using System.Security.Cryptography; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed class TrivyDbPackageBuilder -{ - private static readonly JsonSerializerOptions SerializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = false, - }; - - public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request) - { - if (request is null) - { - throw new ArgumentNullException(nameof(request)); - } - - if (request.MetadataJson.IsEmpty) - { - throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request)); - } - - if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath)) - { - throw new ArgumentException("Database archive path must be provided.", nameof(request)); - } - - if (!File.Exists(request.DatabaseArchivePath)) - { - throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath); - } - - if (string.IsNullOrWhiteSpace(request.DatabaseDigest)) - { - throw new ArgumentException("Database archive digest must be provided.", nameof(request)); - } - - if (request.DatabaseLength < 0) - { - throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength)); - } - - var metadataBytes = request.MetadataJson; - var generatedAt = request.GeneratedAt.ToUniversalTime(); - var configDocument = new TrivyConfigDocument( - TrivyDbMediaTypes.TrivyConfig, - generatedAt, - request.DatabaseVersion, - request.DatabaseDigest, - request.DatabaseLength); - - var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions); - var configDigest = ComputeDigest(configBytes); - - var configDescriptor = new OciDescriptor( - TrivyDbMediaTypes.TrivyConfig, - configDigest, - configBytes.LongLength, - new Dictionary - { - ["org.opencontainers.image.title"] = "config.json", - }); - - var layerDescriptor = new OciDescriptor( - TrivyDbMediaTypes.TrivyLayer, - request.DatabaseDigest, - request.DatabaseLength, - new Dictionary - { - ["org.opencontainers.image.title"] = "db.tar.gz", - }); - - var manifest = new OciManifest( - 2, - TrivyDbMediaTypes.OciManifest, - configDescriptor, - ImmutableArray.Create(layerDescriptor)); - - var blobs = new SortedDictionary(StringComparer.Ordinal) - { - [configDigest] = TrivyDbBlob.FromBytes(configBytes), - [request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength), - }; - - return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes); - } - - private static string ComputeDigest(ReadOnlySpan payload) - { - var hash = SHA256.HashData(payload); - var hex = Convert.ToHexString(hash); - Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex - buffer[0] = 's'; - buffer[1] = 'h'; - buffer[2] = 'a'; - buffer[3] = '2'; - buffer[4] = '5'; - buffer[5] = '6'; - buffer[6] = ':'; - for (var i = 0; i < hex.Length; i++) - { - buffer[7 + i] = char.ToLowerInvariant(hex[i]); - } - - return new string(buffer); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed class TrivyDbPackageBuilder +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false, + }; + + public TrivyDbPackage BuildPackage(TrivyDbPackageRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (request.MetadataJson.IsEmpty) + { + throw new ArgumentException("Metadata JSON payload must be provided.", nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseArchivePath)) + { + throw new ArgumentException("Database archive path must be provided.", nameof(request)); + } + + if (!File.Exists(request.DatabaseArchivePath)) + { + throw new FileNotFoundException("Database archive path not found.", request.DatabaseArchivePath); + } + + if (string.IsNullOrWhiteSpace(request.DatabaseDigest)) + { + throw new ArgumentException("Database archive digest must be provided.", nameof(request)); + } + + if (request.DatabaseLength < 0) + { + throw new ArgumentOutOfRangeException(nameof(request.DatabaseLength)); + } + + var metadataBytes = request.MetadataJson; + var generatedAt = request.GeneratedAt.ToUniversalTime(); + var configDocument = new TrivyConfigDocument( + TrivyDbMediaTypes.TrivyConfig, + generatedAt, + request.DatabaseVersion, + request.DatabaseDigest, + request.DatabaseLength); + + var configBytes = JsonSerializer.SerializeToUtf8Bytes(configDocument, SerializerOptions); + var configDigest = ComputeDigest(configBytes); + + var configDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyConfig, + configDigest, + configBytes.LongLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "config.json", + }); + + var layerDescriptor = new OciDescriptor( + TrivyDbMediaTypes.TrivyLayer, + request.DatabaseDigest, + request.DatabaseLength, + new Dictionary + { + ["org.opencontainers.image.title"] = "db.tar.gz", + }); + + var manifest = new OciManifest( + 2, + TrivyDbMediaTypes.OciManifest, + configDescriptor, + ImmutableArray.Create(layerDescriptor)); + + var blobs = new SortedDictionary(StringComparer.Ordinal) + { + [configDigest] = TrivyDbBlob.FromBytes(configBytes), + [request.DatabaseDigest] = TrivyDbBlob.FromFile(request.DatabaseArchivePath, request.DatabaseLength), + }; + + return new TrivyDbPackage(manifest, configDocument, blobs, metadataBytes); + } + + private static string ComputeDigest(ReadOnlySpan payload) + { + var hash = SHA256.HashData(payload); + var hex = Convert.ToHexString(hash); + Span buffer = stackalloc char[7 + hex.Length]; // "sha256:" + hex + buffer[0] = 's'; + buffer[1] = 'h'; + buffer[2] = 'a'; + buffer[3] = '2'; + buffer[4] = '5'; + buffer[5] = '6'; + buffer[6] = ':'; + for (var i = 0; i < hex.Length; i++) + { + buffer[7 + i] = char.ToLowerInvariant(hex[i]); + } + + return new string(buffer); + } +} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageRequest.cs similarity index 80% rename from src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs rename to src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageRequest.cs index e39618a2..6f5e194c 100644 --- a/src/StellaOps.Feedser.Exporter.TrivyDb/TrivyDbPackageRequest.cs +++ b/src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbPackageRequest.cs @@ -1,11 +1,11 @@ -using System; - -namespace StellaOps.Feedser.Exporter.TrivyDb; - -public sealed record TrivyDbPackageRequest( - ReadOnlyMemory MetadataJson, - string DatabaseArchivePath, - string DatabaseDigest, - long DatabaseLength, - DateTimeOffset GeneratedAt, - string DatabaseVersion); +using System; + +namespace StellaOps.Concelier.Exporter.TrivyDb; + +public sealed record TrivyDbPackageRequest( + ReadOnlyMemory MetadataJson, + string DatabaseArchivePath, + string DatabaseDigest, + long DatabaseLength, + DateTimeOffset GeneratedAt, + string DatabaseVersion); diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs b/src/StellaOps.Concelier.Merge.Tests/AdvisoryIdentityResolverTests.cs similarity index 94% rename from src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs rename to src/StellaOps.Concelier.Merge.Tests/AdvisoryIdentityResolverTests.cs index 8924a18a..1d2771a5 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AdvisoryIdentityResolverTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/AdvisoryIdentityResolverTests.cs @@ -1,92 +1,92 @@ -using System; -using System.Linq; -using StellaOps.Feedser.Merge.Identity; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Merge.Tests; - -public sealed class AdvisoryIdentityResolverTests -{ - private readonly AdvisoryIdentityResolver _resolver = new(); - - [Fact] - public void Resolve_GroupsBySharedCveAlias() - { - var nvd = CreateAdvisory("CVE-2025-1234", aliases: new[] { "CVE-2025-1234" }, source: "nvd"); - var vendor = CreateAdvisory("VSA-2025-01", aliases: new[] { "CVE-2025-1234", "VSA-2025-01" }, source: "vendor"); - - var clusters = _resolver.Resolve(new[] { nvd, vendor }); - - var cluster = Assert.Single(clusters); - Assert.Equal("CVE-2025-1234", cluster.AdvisoryKey); - Assert.Equal(2, cluster.Advisories.Length); - Assert.True(cluster.Aliases.Any(alias => alias.Value == "CVE-2025-1234")); - } - - [Fact] - public void Resolve_PrefersPsirtAliasWhenNoCve() - { - var vendor = CreateAdvisory("VMSA-2025-0001", aliases: new[] { "VMSA-2025-0001" }, source: "vmware"); - var osv = CreateAdvisory("OSV-2025-1", aliases: new[] { "OSV-2025-1", "GHSA-xxxx-yyyy-zzzz", "VMSA-2025-0001" }, source: "osv"); - - var clusters = _resolver.Resolve(new[] { vendor, osv }); - - var cluster = Assert.Single(clusters); - Assert.Equal("VMSA-2025-0001", cluster.AdvisoryKey); - Assert.Equal(2, cluster.Advisories.Length); - Assert.True(cluster.Aliases.Any(alias => alias.Value == "VMSA-2025-0001")); - } - - [Fact] - public void Resolve_FallsBackToGhsaWhenOnlyGhsaPresent() - { - var ghsa = CreateAdvisory("GHSA-aaaa-bbbb-cccc", aliases: new[] { "GHSA-aaaa-bbbb-cccc" }, source: "ghsa"); - var osv = CreateAdvisory("OSV-2025-99", aliases: new[] { "OSV-2025-99", "GHSA-aaaa-bbbb-cccc" }, source: "osv"); - - var clusters = _resolver.Resolve(new[] { ghsa, osv }); - - var cluster = Assert.Single(clusters); - Assert.Equal("GHSA-aaaa-bbbb-cccc", cluster.AdvisoryKey); - Assert.Equal(2, cluster.Advisories.Length); - Assert.True(cluster.Aliases.Any(alias => alias.Value == "GHSA-aaaa-bbbb-cccc")); - } - - [Fact] - public void Resolve_GroupsByKeyWhenNoAliases() - { - var first = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-a"); - var second = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-b"); - - var clusters = _resolver.Resolve(new[] { first, second }); - - var cluster = Assert.Single(clusters); - Assert.Equal("custom-1", cluster.AdvisoryKey); - Assert.Equal(2, cluster.Advisories.Length); - Assert.Contains(cluster.Aliases, alias => alias.Value == "custom-1"); - } - - private static Advisory CreateAdvisory(string key, string[] aliases, string source) - { - var provenance = new[] - { - new AdvisoryProvenance(source, "mapping", key, DateTimeOffset.UtcNow), - }; - - return new Advisory( - key, - $"{key} title", - $"{key} summary", - "en", - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow, - null, - exploitKnown: false, - aliases, - Array.Empty(), - Array.Empty(), - Array.Empty(), - Array.Empty(), - provenance); - } -} +using System; +using System.Linq; +using StellaOps.Concelier.Merge.Identity; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Merge.Tests; + +public sealed class AdvisoryIdentityResolverTests +{ + private readonly AdvisoryIdentityResolver _resolver = new(); + + [Fact] + public void Resolve_GroupsBySharedCveAlias() + { + var nvd = CreateAdvisory("CVE-2025-1234", aliases: new[] { "CVE-2025-1234" }, source: "nvd"); + var vendor = CreateAdvisory("VSA-2025-01", aliases: new[] { "CVE-2025-1234", "VSA-2025-01" }, source: "vendor"); + + var clusters = _resolver.Resolve(new[] { nvd, vendor }); + + var cluster = Assert.Single(clusters); + Assert.Equal("CVE-2025-1234", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "CVE-2025-1234")); + } + + [Fact] + public void Resolve_PrefersPsirtAliasWhenNoCve() + { + var vendor = CreateAdvisory("VMSA-2025-0001", aliases: new[] { "VMSA-2025-0001" }, source: "vmware"); + var osv = CreateAdvisory("OSV-2025-1", aliases: new[] { "OSV-2025-1", "GHSA-xxxx-yyyy-zzzz", "VMSA-2025-0001" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { vendor, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("VMSA-2025-0001", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "VMSA-2025-0001")); + } + + [Fact] + public void Resolve_FallsBackToGhsaWhenOnlyGhsaPresent() + { + var ghsa = CreateAdvisory("GHSA-aaaa-bbbb-cccc", aliases: new[] { "GHSA-aaaa-bbbb-cccc" }, source: "ghsa"); + var osv = CreateAdvisory("OSV-2025-99", aliases: new[] { "OSV-2025-99", "GHSA-aaaa-bbbb-cccc" }, source: "osv"); + + var clusters = _resolver.Resolve(new[] { ghsa, osv }); + + var cluster = Assert.Single(clusters); + Assert.Equal("GHSA-aaaa-bbbb-cccc", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.True(cluster.Aliases.Any(alias => alias.Value == "GHSA-aaaa-bbbb-cccc")); + } + + [Fact] + public void Resolve_GroupsByKeyWhenNoAliases() + { + var first = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-a"); + var second = CreateAdvisory("custom-1", aliases: Array.Empty(), source: "source-b"); + + var clusters = _resolver.Resolve(new[] { first, second }); + + var cluster = Assert.Single(clusters); + Assert.Equal("custom-1", cluster.AdvisoryKey); + Assert.Equal(2, cluster.Advisories.Length); + Assert.Contains(cluster.Aliases, alias => alias.Value == "custom-1"); + } + + private static Advisory CreateAdvisory(string key, string[] aliases, string source) + { + var provenance = new[] + { + new AdvisoryProvenance(source, "mapping", key, DateTimeOffset.UtcNow), + }; + + return new Advisory( + key, + $"{key} title", + $"{key} summary", + "en", + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow, + null, + exploitKnown: false, + aliases, + Array.Empty(), + Array.Empty(), + Array.Empty(), + Array.Empty(), + provenance); + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs b/src/StellaOps.Concelier.Merge.Tests/AdvisoryMergeServiceTests.cs similarity index 75% rename from src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs rename to src/StellaOps.Concelier.Merge.Tests/AdvisoryMergeServiceTests.cs index b35c31c5..f5a09608 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AdvisoryMergeServiceTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/AdvisoryMergeServiceTests.cs @@ -1,201 +1,241 @@ -using System.Collections.Concurrent; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Core; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Aliases; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; - -namespace StellaOps.Feedser.Merge.Tests; - -public sealed class AdvisoryMergeServiceTests -{ - [Fact] - public async Task MergeAsync_AppliesCanonicalRulesAndPersistsDecisions() - { - var aliasStore = new FakeAliasStore(); - aliasStore.Register("GHSA-aaaa-bbbb-cccc", - (AliasSchemes.Ghsa, "GHSA-aaaa-bbbb-cccc"), - (AliasSchemes.Cve, "CVE-2025-4242")); - aliasStore.Register("CVE-2025-4242", - (AliasSchemes.Cve, "CVE-2025-4242")); - aliasStore.Register("OSV-2025-xyz", - (AliasSchemes.OsV, "OSV-2025-xyz"), - (AliasSchemes.Cve, "CVE-2025-4242")); - - var advisoryStore = new FakeAdvisoryStore(); - advisoryStore.Seed(CreateGhsaAdvisory(), CreateNvdAdvisory(), CreateOsvAdvisory()); - - var mergeEventStore = new InMemoryMergeEventStore(); - var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 4, 1, 0, 0, 0, TimeSpan.Zero)); - var writer = new MergeEventWriter(mergeEventStore, new CanonicalHashCalculator(), timeProvider, NullLogger.Instance); - var precedenceMerger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); - var aliasResolver = new AliasGraphResolver(aliasStore); - var canonicalMerger = new CanonicalMerger(timeProvider); - var service = new AdvisoryMergeService(aliasResolver, advisoryStore, precedenceMerger, writer, canonicalMerger, NullLogger.Instance); - - var result = await service.MergeAsync("GHSA-aaaa-bbbb-cccc", CancellationToken.None); - - Assert.NotNull(result.Merged); - Assert.Equal("OSV summary overrides", result.Merged!.Summary); - - var upserted = advisoryStore.LastUpserted; - Assert.NotNull(upserted); - Assert.Equal("CVE-2025-4242", upserted!.AdvisoryKey); - Assert.Equal("OSV summary overrides", upserted.Summary); - - var mergeRecord = mergeEventStore.LastRecord; - Assert.NotNull(mergeRecord); - var summaryDecision = Assert.Single(mergeRecord!.FieldDecisions, decision => decision.Field == "summary"); - Assert.Equal("osv", summaryDecision.SelectedSource); - Assert.Equal("freshness_override", summaryDecision.DecisionReason); - } - - private static Advisory CreateGhsaAdvisory() - { - var recorded = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); - var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-aaaa-bbbb-cccc", recorded, new[] { ProvenanceFieldMasks.Advisory }); - return new Advisory( - "GHSA-aaaa-bbbb-cccc", - "Container escape", - "Initial GHSA summary.", - "en", - recorded, - recorded, - "medium", - exploitKnown: false, - aliases: new[] { "CVE-2025-4242", "GHSA-aaaa-bbbb-cccc" }, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] { provenance }); - } - - private static Advisory CreateNvdAdvisory() - { - var recorded = DateTimeOffset.Parse("2025-03-02T00:00:00Z"); - var provenance = new AdvisoryProvenance("nvd", "map", "CVE-2025-4242", recorded, new[] { ProvenanceFieldMasks.Advisory }); - return new Advisory( - "CVE-2025-4242", - "CVE-2025-4242", - "Baseline NVD summary.", - "en", - recorded, - recorded, - "high", - exploitKnown: false, - aliases: new[] { "CVE-2025-4242" }, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] { provenance }); - } - - private static Advisory CreateOsvAdvisory() - { - var recorded = DateTimeOffset.Parse("2025-03-05T12:00:00Z"); - var provenance = new AdvisoryProvenance("osv", "map", "OSV-2025-xyz", recorded, new[] { ProvenanceFieldMasks.Advisory }); - return new Advisory( - "OSV-2025-xyz", - "Container escape", - "OSV summary overrides", - "en", - recorded, - recorded, - "critical", - exploitKnown: false, - aliases: new[] { "OSV-2025-xyz", "CVE-2025-4242" }, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] { provenance }); - } - - private sealed class FakeAliasStore : IAliasStore - { - private readonly ConcurrentDictionary> _records = new(StringComparer.OrdinalIgnoreCase); - - public void Register(string advisoryKey, params (string Scheme, string Value)[] aliases) - { - var list = new List(); - foreach (var (scheme, value) in aliases) - { - list.Add(new AliasRecord(advisoryKey, scheme, value, DateTimeOffset.UtcNow)); - } - - _records[advisoryKey] = list; - } - - public Task ReplaceAsync(string advisoryKey, IEnumerable aliases, DateTimeOffset updatedAt, CancellationToken cancellationToken) - => Task.FromResult(new AliasUpsertResult(advisoryKey, Array.Empty())); - - public Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken) - { - var matches = _records.Values - .SelectMany(static records => records) - .Where(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase) && string.Equals(record.Value, value, StringComparison.OrdinalIgnoreCase)) - .ToList(); - - return Task.FromResult>(matches); - } - - public Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken) - { - if (_records.TryGetValue(advisoryKey, out var records)) - { - return Task.FromResult>(records); - } - - return Task.FromResult>(Array.Empty()); - } - } - - private sealed class FakeAdvisoryStore : IAdvisoryStore - { - private readonly ConcurrentDictionary _advisories = new(StringComparer.OrdinalIgnoreCase); - - public Advisory? LastUpserted { get; private set; } - - public void Seed(params Advisory[] advisories) - { - foreach (var advisory in advisories) - { - _advisories[advisory.AdvisoryKey] = advisory; - } - } - - public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) - { - _advisories.TryGetValue(advisoryKey, out var advisory); - return Task.FromResult(advisory); - } - - public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) - => Task.FromResult>(Array.Empty()); - - public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) - { - _advisories[advisory.AdvisoryKey] = advisory; - LastUpserted = advisory; - return Task.CompletedTask; - } - - public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) => AsyncEnumerable.Empty(); - } - - private sealed class InMemoryMergeEventStore : IMergeEventStore - { - public MergeEventRecord? LastRecord { get; private set; } - - public Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) - { - LastRecord = record; - return Task.CompletedTask; - } - - public Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) - => Task.FromResult>(Array.Empty()); - } -} +using System.Collections.Concurrent; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using MongoDB.Driver; +using StellaOps.Concelier.Core; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; + +namespace StellaOps.Concelier.Merge.Tests; + +public sealed class AdvisoryMergeServiceTests +{ + [Fact] + public async Task MergeAsync_AppliesCanonicalRulesAndPersistsDecisions() + { + var aliasStore = new FakeAliasStore(); + aliasStore.Register("GHSA-aaaa-bbbb-cccc", + (AliasSchemes.Ghsa, "GHSA-aaaa-bbbb-cccc"), + (AliasSchemes.Cve, "CVE-2025-4242")); + aliasStore.Register("CVE-2025-4242", + (AliasSchemes.Cve, "CVE-2025-4242")); + aliasStore.Register("OSV-2025-xyz", + (AliasSchemes.OsV, "OSV-2025-xyz"), + (AliasSchemes.Cve, "CVE-2025-4242")); + + var advisoryStore = new FakeAdvisoryStore(); + advisoryStore.Seed(CreateGhsaAdvisory(), CreateNvdAdvisory(), CreateOsvAdvisory()); + + var mergeEventStore = new InMemoryMergeEventStore(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 4, 1, 0, 0, 0, TimeSpan.Zero)); + var writer = new MergeEventWriter(mergeEventStore, new CanonicalHashCalculator(), timeProvider, NullLogger.Instance); + var precedenceMerger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + var aliasResolver = new AliasGraphResolver(aliasStore); + var canonicalMerger = new CanonicalMerger(timeProvider); + var eventLog = new RecordingAdvisoryEventLog(); + var service = new AdvisoryMergeService(aliasResolver, advisoryStore, precedenceMerger, writer, canonicalMerger, eventLog, timeProvider, NullLogger.Instance); + + var result = await service.MergeAsync("GHSA-aaaa-bbbb-cccc", CancellationToken.None); + + Assert.NotNull(result.Merged); + Assert.Equal("OSV summary overrides", result.Merged!.Summary); + + var upserted = advisoryStore.LastUpserted; + Assert.NotNull(upserted); + Assert.Equal("CVE-2025-4242", upserted!.AdvisoryKey); + Assert.Equal("OSV summary overrides", upserted.Summary); + + var mergeRecord = mergeEventStore.LastRecord; + Assert.NotNull(mergeRecord); + var summaryDecision = Assert.Single(mergeRecord!.FieldDecisions, decision => decision.Field == "summary"); + Assert.Equal("osv", summaryDecision.SelectedSource); + Assert.Equal("freshness_override", summaryDecision.DecisionReason); + + var appendRequest = eventLog.LastRequest; + Assert.NotNull(appendRequest); + Assert.Contains(appendRequest!.Statements, statement => string.Equals(statement.Advisory.AdvisoryKey, "CVE-2025-4242", StringComparison.OrdinalIgnoreCase)); + Assert.True(appendRequest.Conflicts is null || appendRequest.Conflicts.Count == 0); + } + + private static Advisory CreateGhsaAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); + var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-aaaa-bbbb-cccc", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "GHSA-aaaa-bbbb-cccc", + "Container escape", + "Initial GHSA summary.", + "en", + recorded, + recorded, + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-4242", "GHSA-aaaa-bbbb-cccc" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static Advisory CreateNvdAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-02T00:00:00Z"); + var provenance = new AdvisoryProvenance("nvd", "map", "CVE-2025-4242", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "CVE-2025-4242", + "CVE-2025-4242", + "Baseline NVD summary.", + "en", + recorded, + recorded, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-4242" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private static Advisory CreateOsvAdvisory() + { + var recorded = DateTimeOffset.Parse("2025-03-05T12:00:00Z"); + var provenance = new AdvisoryProvenance("osv", "map", "OSV-2025-xyz", recorded, new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + "OSV-2025-xyz", + "Container escape", + "OSV summary overrides", + "en", + recorded, + recorded, + "critical", + exploitKnown: false, + aliases: new[] { "OSV-2025-xyz", "CVE-2025-4242" }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + + private sealed class RecordingAdvisoryEventLog : IAdvisoryEventLog + { + public AdvisoryEventAppendRequest? LastRequest { get; private set; } + + public ValueTask AppendAsync(AdvisoryEventAppendRequest request, CancellationToken cancellationToken) + { + LastRequest = request; + return ValueTask.CompletedTask; + } + + public ValueTask ReplayAsync(string vulnerabilityKey, DateTimeOffset? asOf, CancellationToken cancellationToken) + { + throw new NotSupportedException(); + } + } + + private sealed class FakeAliasStore : IAliasStore + { + private readonly ConcurrentDictionary> _records = new(StringComparer.OrdinalIgnoreCase); + + public void Register(string advisoryKey, params (string Scheme, string Value)[] aliases) + { + var list = new List(); + foreach (var (scheme, value) in aliases) + { + list.Add(new AliasRecord(advisoryKey, scheme, value, DateTimeOffset.UtcNow)); + } + + _records[advisoryKey] = list; + } + + public Task ReplaceAsync(string advisoryKey, IEnumerable aliases, DateTimeOffset updatedAt, CancellationToken cancellationToken) + { + return Task.FromResult(new AliasUpsertResult(advisoryKey, Array.Empty())); + } + + public Task> GetByAliasAsync(string scheme, string value, CancellationToken cancellationToken) + { + var matches = _records.Values + .SelectMany(static records => records) + .Where(record => string.Equals(record.Scheme, scheme, StringComparison.OrdinalIgnoreCase) && string.Equals(record.Value, value, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + return Task.FromResult>(matches); + } + + public Task> GetByAdvisoryAsync(string advisoryKey, CancellationToken cancellationToken) + { + if (_records.TryGetValue(advisoryKey, out var records)) + { + return Task.FromResult>(records); + } + + return Task.FromResult>(Array.Empty()); + } + } + + private sealed class FakeAdvisoryStore : IAdvisoryStore + { + private readonly ConcurrentDictionary _advisories = new(StringComparer.OrdinalIgnoreCase); + + public Advisory? LastUpserted { get; private set; } + + public void Seed(params Advisory[] advisories) + { + foreach (var advisory in advisories) + { + _advisories[advisory.AdvisoryKey] = advisory; + } + } + + public Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + _advisories.TryGetValue(advisoryKey, out var advisory); + return Task.FromResult(advisory); + } + + public Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return Task.FromResult>(Array.Empty()); + } + + public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + _advisories[advisory.AdvisoryKey] = advisory; + LastUpserted = advisory; + return Task.CompletedTask; + } + + public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _ = session; + return AsyncEnumerable.Empty(); + } + } + + private sealed class InMemoryMergeEventStore : IMergeEventStore + { + public MergeEventRecord? LastRecord { get; private set; } + + public Task AppendAsync(MergeEventRecord record, CancellationToken cancellationToken) + { + LastRecord = record; + return Task.CompletedTask; + } + + public Task> GetRecentAsync(string advisoryKey, int limit, CancellationToken cancellationToken) + { + return Task.FromResult>(Array.Empty()); + } + } +} diff --git a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs b/src/StellaOps.Concelier.Merge.Tests/AdvisoryPrecedenceMergerTests.cs similarity index 94% rename from src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs rename to src/StellaOps.Concelier.Merge.Tests/AdvisoryPrecedenceMergerTests.cs index 547dda1f..778a4eec 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AdvisoryPrecedenceMergerTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/AdvisoryPrecedenceMergerTests.cs @@ -3,11 +3,11 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Merge.Options; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Merge.Options; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class AdvisoryPrecedenceMergerTests { @@ -16,12 +16,12 @@ public sealed class AdvisoryPrecedenceMergerTests { var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); - using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); + using var metrics = new MetricCollector("StellaOps.Concelier.Merge"); var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); var expectedMergeTimestamp = timeProvider.GetUtcNow(); - var merged = merger.Merge(new[] { nvd, redHat }); + var merged = merger.Merge(new[] { nvd, redHat }).Advisory; Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); Assert.Equal("Red Hat Security Advisory", merged.Title); @@ -48,11 +48,11 @@ public sealed class AdvisoryPrecedenceMergerTests Assert.Contains("redhat", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); Assert.Contains("nvd", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); - var rangeMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + var rangeMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "concelier.merge.range_overrides"); Assert.Equal(1, rangeMeasurement.Value); Assert.Contains(rangeMeasurement.Tags, tag => string.Equals(tag.Key, "suppressed_source", StringComparison.Ordinal) && tag.Value?.ToString()?.Contains("nvd", StringComparison.OrdinalIgnoreCase) == true); - var severityConflict = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + var severityConflict = Assert.Single(metrics.Measurements, measurement => measurement.Name == "concelier.merge.conflicts"); Assert.Equal(1, severityConflict.Value); Assert.Contains(severityConflict.Tags, tag => string.Equals(tag.Key, "type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); } @@ -65,17 +65,17 @@ public sealed class AdvisoryPrecedenceMergerTests var nvdProvenance = new AdvisoryProvenance("nvd", "document", "https://nvd", timeProvider.GetUtcNow()); var baseAdvisory = new Advisory( - "CVE-2025-2000", - "CVE-2025-2000", - "Base registry summary", - "en", - new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), - new DateTimeOffset(2025, 1, 6, 0, 0, 0, TimeSpan.Zero), - "medium", - exploitKnown: false, - aliases: new[] { "CVE-2025-2000" }, - credits: Array.Empty(), - references: Array.Empty(), + "CVE-2025-2000", + "CVE-2025-2000", + "Base registry summary", + "en", + new DateTimeOffset(2025, 1, 5, 0, 0, 0, TimeSpan.Zero), + new DateTimeOffset(2025, 1, 6, 0, 0, 0, TimeSpan.Zero), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-2000" }, + credits: Array.Empty(), + references: Array.Empty(), affectedPackages: new[] { new AffectedPackage( @@ -106,374 +106,374 @@ public sealed class AdvisoryPrecedenceMergerTests language: null, published: null, modified: null, - severity: null, - exploitKnown: true, - aliases: new[] { "KEV-CVE-2025-2000" }, - credits: Array.Empty(), - references: Array.Empty(), + severity: null, + exploitKnown: true, + aliases: new[] { "KEV-CVE-2025-2000" }, + credits: Array.Empty(), + references: Array.Empty(), affectedPackages: Array.Empty(), cvssMetrics: Array.Empty(), provenance: new[] { kevProvenance }); - var merged = merger.Merge(new[] { baseAdvisory, kevAdvisory }); + var merged = merger.Merge(new[] { baseAdvisory, kevAdvisory }).Advisory; Assert.True(merged.ExploitKnown); Assert.Equal("medium", merged.Severity); // KEV must not override severity Assert.Equal("Base registry summary", merged.Summary); Assert.Contains("CVE-2025-2000", merged.Aliases); Assert.Contains("KEV-CVE-2025-2000", merged.Aliases); - Assert.Contains(merged.Provenance, provenance => provenance.Source == "kev"); - Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge"); - } - - [Fact] - public void Merge_UnionsCreditsFromSources() - { - var timeProvider = new FakeTimeProvider(); - var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); - - var ghsaCredits = new[] - { - new AdvisoryCredit( - displayName: "maintainer-team", - role: "remediation_developer", - contacts: new[] { "https://github.com/maintainer-team" }, - provenance: new AdvisoryProvenance( - "ghsa", - "credit", - "mantainer-team", - timeProvider.GetUtcNow(), - new[] { ProvenanceFieldMasks.Credits })), - new AdvisoryCredit( - displayName: "security-reporter", - role: "reporter", - contacts: new[] { "https://github.com/security-reporter" }, - provenance: new AdvisoryProvenance( - "ghsa", - "credit", - "security-reporter", - timeProvider.GetUtcNow(), - new[] { ProvenanceFieldMasks.Credits })), - }; - - var ghsa = new Advisory( - "CVE-2025-9000", - "GHSA advisory", - "Reported in GHSA", - "en", - timeProvider.GetUtcNow(), - timeProvider.GetUtcNow(), - "high", - exploitKnown: false, - aliases: new[] { "GHSA-aaaa-bbbb-cccc", "CVE-2025-9000" }, - credits: ghsaCredits, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] { new AdvisoryProvenance("ghsa", "document", "https://github.com/advisories/GHSA-aaaa-bbbb-cccc", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); - - var osvCredits = new[] - { - new AdvisoryCredit( - displayName: "osv-researcher", - role: "reporter", - contacts: new[] { "mailto:osv-researcher@example.com" }, - provenance: new AdvisoryProvenance( - "osv", - "credit", - "osv-researcher", - timeProvider.GetUtcNow(), - new[] { ProvenanceFieldMasks.Credits })), - new AdvisoryCredit( - displayName: "maintainer-team", - role: "remediation_developer", - contacts: new[] { "https://github.com/maintainer-team" }, - provenance: new AdvisoryProvenance( - "osv", - "credit", - "maintainer-team", - timeProvider.GetUtcNow(), - new[] { ProvenanceFieldMasks.Credits })), - }; - - var osv = new Advisory( - "CVE-2025-9000", - "OSV advisory", - "Reported in OSV.dev", - "en", - timeProvider.GetUtcNow().AddDays(-1), - timeProvider.GetUtcNow().AddHours(-1), - "medium", - exploitKnown: false, - aliases: new[] { "CVE-2025-9000" }, - credits: osvCredits, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] { new AdvisoryProvenance("osv", "document", "https://osv.dev/vulnerability/CVE-2025-9000", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); - - var merged = merger.Merge(new[] { ghsa, osv }); - - Assert.Equal("CVE-2025-9000", merged.AdvisoryKey); - Assert.Contains(merged.Credits, credit => - string.Equals(credit.DisplayName, "maintainer-team", StringComparison.OrdinalIgnoreCase) && - string.Equals(credit.Role, "remediation_developer", StringComparison.OrdinalIgnoreCase)); - Assert.Contains(merged.Credits, credit => - string.Equals(credit.DisplayName, "osv-researcher", StringComparison.OrdinalIgnoreCase) && - string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); - Assert.Contains(merged.Credits, credit => - string.Equals(credit.DisplayName, "security-reporter", StringComparison.OrdinalIgnoreCase) && - string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); - - Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "ghsa"); - Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "osv"); - } - - [Fact] - public void Merge_AcscActsAsEnrichmentSource() - { - var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); - var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); - - var vendorDocumentProvenance = new AdvisoryProvenance( - source: "vndr-cisco", - kind: "document", - value: "https://vendor.example/advisories/router-critical", - recordedAt: timeProvider.GetUtcNow(), - fieldMask: new[] { ProvenanceFieldMasks.Advisory }); - - var vendorReference = new AdvisoryReference( - "https://vendor.example/advisories/router-critical", - kind: "advisory", - sourceTag: "vendor", - summary: "Vendor advisory", - provenance: new AdvisoryProvenance("vndr-cisco", "reference", "https://vendor.example/advisories/router-critical", timeProvider.GetUtcNow())); - - var vendorPackage = new AffectedPackage( - AffectedPackageTypes.Vendor, - "ExampleCo Router X", - platform: null, - versionRanges: Array.Empty(), - statuses: Array.Empty(), - normalizedVersions: Array.Empty(), - provenance: new[] { vendorDocumentProvenance }); - - var vendor = new Advisory( - advisoryKey: "acsc-2025-010", - title: "Vendor Critical Router Advisory", - summary: "Vendor-confirmed exploit.", - language: "en", - published: new DateTimeOffset(2025, 10, 11, 23, 0, 0, TimeSpan.Zero), - modified: new DateTimeOffset(2025, 10, 11, 23, 30, 0, TimeSpan.Zero), - severity: "critical", - exploitKnown: false, - aliases: new[] { "VENDOR-2025-010" }, - references: new[] { vendorReference }, - affectedPackages: new[] { vendorPackage }, - cvssMetrics: Array.Empty(), - provenance: new[] { vendorDocumentProvenance }); - - var acscDocumentProvenance = new AdvisoryProvenance( - source: "acsc", - kind: "document", - value: "https://origin.example/feeds/alerts/rss", - recordedAt: timeProvider.GetUtcNow(), - fieldMask: new[] { ProvenanceFieldMasks.Advisory }); - - var acscReference = new AdvisoryReference( - "https://origin.example/advisories/router-critical", - kind: "advisory", - sourceTag: "acsc", - summary: "ACSC alert", - provenance: new AdvisoryProvenance("acsc", "reference", "https://origin.example/advisories/router-critical", timeProvider.GetUtcNow())); - - var acscPackage = new AffectedPackage( - AffectedPackageTypes.Vendor, - "ExampleCo Router X", - platform: null, - versionRanges: Array.Empty(), - statuses: Array.Empty(), - normalizedVersions: Array.Empty(), - provenance: new[] { acscDocumentProvenance }); - - var acsc = new Advisory( - advisoryKey: "acsc-2025-010", - title: "ACSC Router Alert", - summary: "ACSC recommends installing vendor update.", - language: "en", - published: new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero), - modified: null, - severity: "medium", - exploitKnown: false, - aliases: new[] { "ACSC-2025-010" }, - references: new[] { acscReference }, - affectedPackages: new[] { acscPackage }, - cvssMetrics: Array.Empty(), - provenance: new[] { acscDocumentProvenance }); - - var merged = merger.Merge(new[] { acsc, vendor }); - - Assert.Equal("critical", merged.Severity); // ACSC must not override vendor severity - Assert.Equal("Vendor-confirmed exploit.", merged.Summary); - - Assert.Contains("ACSC-2025-010", merged.Aliases); - Assert.Contains("VENDOR-2025-010", merged.Aliases); - - Assert.Contains(merged.References, reference => reference.SourceTag == "vendor" && reference.Url == vendorReference.Url); - Assert.Contains(merged.References, reference => reference.SourceTag == "acsc" && reference.Url == acscReference.Url); - - var enrichedPackage = Assert.Single(merged.AffectedPackages, package => package.Identifier == "ExampleCo Router X"); - Assert.Contains(enrichedPackage.Provenance, provenance => provenance.Source == "vndr-cisco"); - Assert.Contains(enrichedPackage.Provenance, provenance => provenance.Source == "acsc"); - - Assert.Contains(merged.Provenance, provenance => provenance.Source == "acsc"); - Assert.Contains(merged.Provenance, provenance => provenance.Source == "vndr-cisco"); - Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge" && (provenance.Value?.Contains("acsc", StringComparison.OrdinalIgnoreCase) ?? false)); - } - - [Fact] - public void Merge_RecordsNormalizedRuleMetrics() - { - var now = new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero); - var timeProvider = new FakeTimeProvider(now); - var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); - using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); - - var normalizedRule = new NormalizedVersionRule( - NormalizedVersionSchemes.SemVer, - NormalizedVersionRuleTypes.Range, - min: "1.0.0", - minInclusive: true, - max: "2.0.0", - maxInclusive: false, - notes: "ghsa:GHSA-xxxx-yyyy"); - - var ghsaProvenance = new AdvisoryProvenance("ghsa", "package", "pkg:npm/example", now); - var ghsaPackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/example", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - NormalizedVersionSchemes.SemVer, - "1.0.0", - "2.0.0", - null, - ">= 1.0.0 < 2.0.0", - ghsaProvenance) - }, - statuses: Array.Empty(), - provenance: new[] - { - ghsaProvenance, - }, - normalizedVersions: new[] { normalizedRule }); - - var nvdPackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/example", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - NormalizedVersionSchemes.SemVer, - "1.0.0", - "2.0.0", - null, - ">= 1.0.0 < 2.0.0", - new AdvisoryProvenance("nvd", "cpe_match", "pkg:npm/example", now)) - }, - statuses: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), - }, - normalizedVersions: Array.Empty()); - - var nvdExclusivePackage = new AffectedPackage( - AffectedPackageTypes.SemVer, - "pkg:npm/another", - platform: null, - versionRanges: new[] - { - new AffectedVersionRange( - NormalizedVersionSchemes.SemVer, - "3.0.0", - null, - null, - ">= 3.0.0", - new AdvisoryProvenance("nvd", "cpe_match", "pkg:npm/another", now)) - }, - statuses: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), - }, - normalizedVersions: Array.Empty()); - - var ghsaAdvisory = new Advisory( - "CVE-2025-7000", - "GHSA advisory", - "GHSA summary", - "en", - now, - now, - "high", - exploitKnown: false, - aliases: new[] { "CVE-2025-7000", "GHSA-xxxx-yyyy" }, - credits: Array.Empty(), - references: Array.Empty(), - affectedPackages: new[] { ghsaPackage }, - cvssMetrics: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("ghsa", "document", "https://github.com/advisories/GHSA-xxxx-yyyy", now), - }); - - var nvdAdvisory = new Advisory( - "CVE-2025-7000", - "NVD entry", - "NVD summary", - "en", - now, - now, - "high", - exploitKnown: false, - aliases: new[] { "CVE-2025-7000" }, - credits: Array.Empty(), - references: Array.Empty(), - affectedPackages: new[] { nvdPackage, nvdExclusivePackage }, - cvssMetrics: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), - }); - - var merged = merger.Merge(new[] { nvdAdvisory, ghsaAdvisory }); - Assert.Equal(2, merged.AffectedPackages.Length); - - var normalizedPackage = Assert.Single(merged.AffectedPackages, pkg => pkg.Identifier == "pkg:npm/example"); - Assert.Single(normalizedPackage.NormalizedVersions); - - var missingPackage = Assert.Single(merged.AffectedPackages, pkg => pkg.Identifier == "pkg:npm/another"); - Assert.Empty(missingPackage.NormalizedVersions); - Assert.NotEmpty(missingPackage.VersionRanges); - - var normalizedMeasurements = metrics.Measurements.Where(m => m.Name == "feedser.merge.normalized_rules").ToList(); - Assert.Contains(normalizedMeasurements, measurement => - measurement.Value == 1 - && measurement.Tags.Any(tag => string.Equals(tag.Key, "scheme", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal)) - && measurement.Tags.Any(tag => string.Equals(tag.Key, "package_type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal))); - - var missingMeasurements = metrics.Measurements.Where(m => m.Name == "feedser.merge.normalized_rules_missing").ToList(); - var missingMeasurement = Assert.Single(missingMeasurements); - Assert.Equal(1, missingMeasurement.Value); - Assert.Contains(missingMeasurement.Tags, tag => string.Equals(tag.Key, "package_type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal)); - } - - [Fact] - public void Merge_RespectsConfiguredPrecedenceOverrides() + Assert.Contains(merged.Provenance, provenance => provenance.Source == "kev"); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge"); + } + + [Fact] + public void Merge_UnionsCreditsFromSources() + { + var timeProvider = new FakeTimeProvider(); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var ghsaCredits = new[] + { + new AdvisoryCredit( + displayName: "maintainer-team", + role: "remediation_developer", + contacts: new[] { "https://github.com/maintainer-team" }, + provenance: new AdvisoryProvenance( + "ghsa", + "credit", + "mantainer-team", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + new AdvisoryCredit( + displayName: "security-reporter", + role: "reporter", + contacts: new[] { "https://github.com/security-reporter" }, + provenance: new AdvisoryProvenance( + "ghsa", + "credit", + "security-reporter", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + }; + + var ghsa = new Advisory( + "CVE-2025-9000", + "GHSA advisory", + "Reported in GHSA", + "en", + timeProvider.GetUtcNow(), + timeProvider.GetUtcNow(), + "high", + exploitKnown: false, + aliases: new[] { "GHSA-aaaa-bbbb-cccc", "CVE-2025-9000" }, + credits: ghsaCredits, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("ghsa", "document", "https://github.com/advisories/GHSA-aaaa-bbbb-cccc", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); + + var osvCredits = new[] + { + new AdvisoryCredit( + displayName: "osv-researcher", + role: "reporter", + contacts: new[] { "mailto:osv-researcher@example.com" }, + provenance: new AdvisoryProvenance( + "osv", + "credit", + "osv-researcher", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + new AdvisoryCredit( + displayName: "maintainer-team", + role: "remediation_developer", + contacts: new[] { "https://github.com/maintainer-team" }, + provenance: new AdvisoryProvenance( + "osv", + "credit", + "maintainer-team", + timeProvider.GetUtcNow(), + new[] { ProvenanceFieldMasks.Credits })), + }; + + var osv = new Advisory( + "CVE-2025-9000", + "OSV advisory", + "Reported in OSV.dev", + "en", + timeProvider.GetUtcNow().AddDays(-1), + timeProvider.GetUtcNow().AddHours(-1), + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-9000" }, + credits: osvCredits, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("osv", "document", "https://osv.dev/vulnerability/CVE-2025-9000", timeProvider.GetUtcNow(), new[] { ProvenanceFieldMasks.Advisory }) }); + + var merged = merger.Merge(new[] { ghsa, osv }).Advisory; + + Assert.Equal("CVE-2025-9000", merged.AdvisoryKey); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "maintainer-team", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "remediation_developer", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "osv-researcher", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(merged.Credits, credit => + string.Equals(credit.DisplayName, "security-reporter", StringComparison.OrdinalIgnoreCase) && + string.Equals(credit.Role, "reporter", StringComparison.OrdinalIgnoreCase)); + + Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "ghsa"); + Assert.Contains(merged.Credits, credit => credit.Provenance.Source == "osv"); + } + + [Fact] + public void Merge_AcscActsAsEnrichmentSource() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero)); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + + var vendorDocumentProvenance = new AdvisoryProvenance( + source: "vndr-cisco", + kind: "document", + value: "https://vendor.example/advisories/router-critical", + recordedAt: timeProvider.GetUtcNow(), + fieldMask: new[] { ProvenanceFieldMasks.Advisory }); + + var vendorReference = new AdvisoryReference( + "https://vendor.example/advisories/router-critical", + kind: "advisory", + sourceTag: "vendor", + summary: "Vendor advisory", + provenance: new AdvisoryProvenance("vndr-cisco", "reference", "https://vendor.example/advisories/router-critical", timeProvider.GetUtcNow())); + + var vendorPackage = new AffectedPackage( + AffectedPackageTypes.Vendor, + "ExampleCo Router X", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + normalizedVersions: Array.Empty(), + provenance: new[] { vendorDocumentProvenance }); + + var vendor = new Advisory( + advisoryKey: "acsc-2025-010", + title: "Vendor Critical Router Advisory", + summary: "Vendor-confirmed exploit.", + language: "en", + published: new DateTimeOffset(2025, 10, 11, 23, 0, 0, TimeSpan.Zero), + modified: new DateTimeOffset(2025, 10, 11, 23, 30, 0, TimeSpan.Zero), + severity: "critical", + exploitKnown: false, + aliases: new[] { "VENDOR-2025-010" }, + references: new[] { vendorReference }, + affectedPackages: new[] { vendorPackage }, + cvssMetrics: Array.Empty(), + provenance: new[] { vendorDocumentProvenance }); + + var acscDocumentProvenance = new AdvisoryProvenance( + source: "acsc", + kind: "document", + value: "https://origin.example/feeds/alerts/rss", + recordedAt: timeProvider.GetUtcNow(), + fieldMask: new[] { ProvenanceFieldMasks.Advisory }); + + var acscReference = new AdvisoryReference( + "https://origin.example/advisories/router-critical", + kind: "advisory", + sourceTag: "acsc", + summary: "ACSC alert", + provenance: new AdvisoryProvenance("acsc", "reference", "https://origin.example/advisories/router-critical", timeProvider.GetUtcNow())); + + var acscPackage = new AffectedPackage( + AffectedPackageTypes.Vendor, + "ExampleCo Router X", + platform: null, + versionRanges: Array.Empty(), + statuses: Array.Empty(), + normalizedVersions: Array.Empty(), + provenance: new[] { acscDocumentProvenance }); + + var acsc = new Advisory( + advisoryKey: "acsc-2025-010", + title: "ACSC Router Alert", + summary: "ACSC recommends installing vendor update.", + language: "en", + published: new DateTimeOffset(2025, 10, 12, 0, 0, 0, TimeSpan.Zero), + modified: null, + severity: "medium", + exploitKnown: false, + aliases: new[] { "ACSC-2025-010" }, + references: new[] { acscReference }, + affectedPackages: new[] { acscPackage }, + cvssMetrics: Array.Empty(), + provenance: new[] { acscDocumentProvenance }); + + var merged = merger.Merge(new[] { acsc, vendor }).Advisory; + + Assert.Equal("critical", merged.Severity); // ACSC must not override vendor severity + Assert.Equal("Vendor-confirmed exploit.", merged.Summary); + + Assert.Contains("ACSC-2025-010", merged.Aliases); + Assert.Contains("VENDOR-2025-010", merged.Aliases); + + Assert.Contains(merged.References, reference => reference.SourceTag == "vendor" && reference.Url == vendorReference.Url); + Assert.Contains(merged.References, reference => reference.SourceTag == "acsc" && reference.Url == acscReference.Url); + + var enrichedPackage = Assert.Single(merged.AffectedPackages, package => package.Identifier == "ExampleCo Router X"); + Assert.Contains(enrichedPackage.Provenance, provenance => provenance.Source == "vndr-cisco"); + Assert.Contains(enrichedPackage.Provenance, provenance => provenance.Source == "acsc"); + + Assert.Contains(merged.Provenance, provenance => provenance.Source == "acsc"); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "vndr-cisco"); + Assert.Contains(merged.Provenance, provenance => provenance.Source == "merge" && (provenance.Value?.Contains("acsc", StringComparison.OrdinalIgnoreCase) ?? false)); + } + + [Fact] + public void Merge_RecordsNormalizedRuleMetrics() + { + var now = new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero); + var timeProvider = new FakeTimeProvider(now); + var merger = new AdvisoryPrecedenceMerger(new AffectedPackagePrecedenceResolver(), timeProvider); + using var metrics = new MetricCollector("StellaOps.Concelier.Merge"); + + var normalizedRule = new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.Range, + min: "1.0.0", + minInclusive: true, + max: "2.0.0", + maxInclusive: false, + notes: "ghsa:GHSA-xxxx-yyyy"); + + var ghsaProvenance = new AdvisoryProvenance("ghsa", "package", "pkg:npm/example", now); + var ghsaPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + NormalizedVersionSchemes.SemVer, + "1.0.0", + "2.0.0", + null, + ">= 1.0.0 < 2.0.0", + ghsaProvenance) + }, + statuses: Array.Empty(), + provenance: new[] + { + ghsaProvenance, + }, + normalizedVersions: new[] { normalizedRule }); + + var nvdPackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/example", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + NormalizedVersionSchemes.SemVer, + "1.0.0", + "2.0.0", + null, + ">= 1.0.0 < 2.0.0", + new AdvisoryProvenance("nvd", "cpe_match", "pkg:npm/example", now)) + }, + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), + }, + normalizedVersions: Array.Empty()); + + var nvdExclusivePackage = new AffectedPackage( + AffectedPackageTypes.SemVer, + "pkg:npm/another", + platform: null, + versionRanges: new[] + { + new AffectedVersionRange( + NormalizedVersionSchemes.SemVer, + "3.0.0", + null, + null, + ">= 3.0.0", + new AdvisoryProvenance("nvd", "cpe_match", "pkg:npm/another", now)) + }, + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), + }, + normalizedVersions: Array.Empty()); + + var ghsaAdvisory = new Advisory( + "CVE-2025-7000", + "GHSA advisory", + "GHSA summary", + "en", + now, + now, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-7000", "GHSA-xxxx-yyyy" }, + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: new[] { ghsaPackage }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("ghsa", "document", "https://github.com/advisories/GHSA-xxxx-yyyy", now), + }); + + var nvdAdvisory = new Advisory( + "CVE-2025-7000", + "NVD entry", + "NVD summary", + "en", + now, + now, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-7000" }, + credits: Array.Empty(), + references: Array.Empty(), + affectedPackages: new[] { nvdPackage, nvdExclusivePackage }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "document", "https://nvd.nist.gov/vuln/detail/CVE-2025-7000", now), + }); + + var merged = merger.Merge(new[] { nvdAdvisory, ghsaAdvisory }).Advisory; + Assert.Equal(2, merged.AffectedPackages.Length); + + var normalizedPackage = Assert.Single(merged.AffectedPackages, pkg => pkg.Identifier == "pkg:npm/example"); + Assert.Single(normalizedPackage.NormalizedVersions); + + var missingPackage = Assert.Single(merged.AffectedPackages, pkg => pkg.Identifier == "pkg:npm/another"); + Assert.Empty(missingPackage.NormalizedVersions); + Assert.NotEmpty(missingPackage.VersionRanges); + + var normalizedMeasurements = metrics.Measurements.Where(m => m.Name == "concelier.merge.normalized_rules").ToList(); + Assert.Contains(normalizedMeasurements, measurement => + measurement.Value == 1 + && measurement.Tags.Any(tag => string.Equals(tag.Key, "scheme", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal)) + && measurement.Tags.Any(tag => string.Equals(tag.Key, "package_type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal))); + + var missingMeasurements = metrics.Measurements.Where(m => m.Name == "concelier.merge.normalized_rules_missing").ToList(); + var missingMeasurement = Assert.Single(missingMeasurements); + Assert.Equal(1, missingMeasurement.Value); + Assert.Contains(missingMeasurement.Tags, tag => string.Equals(tag.Key, "package_type", StringComparison.Ordinal) && string.Equals(tag.Value?.ToString(), "semver", StringComparison.Ordinal)); + } + + [Fact] + public void Merge_RespectsConfiguredPrecedenceOverrides() { var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero)); var options = new AdvisoryPrecedenceOptions @@ -486,7 +486,7 @@ public sealed class AdvisoryPrecedenceMergerTests }; var logger = new TestLogger(); - using var metrics = new MetricCollector("StellaOps.Feedser.Merge"); + using var metrics = new MetricCollector("StellaOps.Concelier.Merge"); var merger = new AdvisoryPrecedenceMerger( new AffectedPackagePrecedenceResolver(), @@ -495,7 +495,7 @@ public sealed class AdvisoryPrecedenceMergerTests logger); var (redHat, nvd) = CreateVendorAndRegistryAdvisories(); - var merged = merger.Merge(new[] { redHat, nvd }); + var merged = merger.Merge(new[] { redHat, nvd }).Advisory; Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); Assert.Equal("CVE-2025-1000", merged.Title); // NVD preferred @@ -507,14 +507,14 @@ public sealed class AdvisoryPrecedenceMergerTests Assert.Contains(package.Provenance, provenance => provenance.Source == "nvd"); Assert.Contains(package.Provenance, provenance => provenance.Source == "redhat"); - var overrideMeasurement = Assert.Single(metrics.Measurements, m => m.Name == "feedser.merge.overrides"); + var overrideMeasurement = Assert.Single(metrics.Measurements, m => m.Name == "concelier.merge.overrides"); Assert.Equal(1, overrideMeasurement.Value); Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "primary_source" && string.Equals(tag.Value?.ToString(), "nvd", StringComparison.OrdinalIgnoreCase)); Assert.Contains(overrideMeasurement.Tags, tag => tag.Key == "suppressed_source" && tag.Value?.ToString()?.Contains("redhat", StringComparison.OrdinalIgnoreCase) == true); - Assert.DoesNotContain(metrics.Measurements, measurement => measurement.Name == "feedser.merge.range_overrides"); + Assert.DoesNotContain(metrics.Measurements, measurement => measurement.Name == "concelier.merge.range_overrides"); - var conflictMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "feedser.merge.conflicts"); + var conflictMeasurement = Assert.Single(metrics.Measurements, measurement => measurement.Name == "concelier.merge.conflicts"); Assert.Equal(1, conflictMeasurement.Value); Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "type" && string.Equals(tag.Value?.ToString(), "severity", StringComparison.OrdinalIgnoreCase)); Assert.Contains(conflictMeasurement.Tags, tag => tag.Key == "reason" && string.Equals(tag.Value?.ToString(), "mismatch", StringComparison.OrdinalIgnoreCase)); @@ -540,20 +540,20 @@ public sealed class AdvisoryPrecedenceMergerTests Array.Empty(), new[] { new AffectedPackageStatus("known_affected", redHatProvenance) }, new[] { redHatProvenance }); - var redHat = new Advisory( - "CVE-2025-1000", - "Red Hat Security Advisory", - "Vendor-confirmed impact on RHEL 9.", - "en", - redHatPublished, - redHatModified, - "high", - exploitKnown: false, - aliases: new[] { "CVE-2025-1000", "RHSA-2025:0001" }, - credits: Array.Empty(), - references: new[] - { - new AdvisoryReference( + var redHat = new Advisory( + "CVE-2025-1000", + "Red Hat Security Advisory", + "Vendor-confirmed impact on RHEL 9.", + "en", + redHatPublished, + redHatModified, + "high", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000", "RHSA-2025:0001" }, + credits: Array.Empty(), + references: new[] + { + new AdvisoryReference( "https://access.redhat.com/errata/RHSA-2025:0001", "advisory", "redhat", @@ -591,20 +591,20 @@ public sealed class AdvisoryPrecedenceMergerTests }, Array.Empty(), new[] { nvdProvenance }); - var nvd = new Advisory( - "CVE-2025-1000", - "CVE-2025-1000", - "NVD summary", - "en", - nvdPublished, - nvdModified, - "medium", - exploitKnown: false, - aliases: new[] { "CVE-2025-1000" }, - credits: Array.Empty(), - references: new[] - { - new AdvisoryReference( + var nvd = new Advisory( + "CVE-2025-1000", + "CVE-2025-1000", + "NVD summary", + "en", + nvdPublished, + nvdModified, + "medium", + exploitKnown: false, + aliases: new[] { "CVE-2025-1000" }, + credits: Array.Empty(), + references: new[] + { + new AdvisoryReference( "https://nvd.nist.gov/vuln/detail/CVE-2025-1000", "advisory", "nvd", diff --git a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs b/src/StellaOps.Concelier.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs similarity index 94% rename from src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs rename to src/StellaOps.Concelier.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs index aa7233eb..48e1c018 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/AffectedPackagePrecedenceResolverTests.cs @@ -1,8 +1,8 @@ using System; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class AffectedPackagePrecedenceResolverTests { diff --git a/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs b/src/StellaOps.Concelier.Merge.Tests/AliasGraphResolverTests.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs rename to src/StellaOps.Concelier.Merge.Tests/AliasGraphResolverTests.cs index 6f2542ab..1b3b6a7c 100644 --- a/src/StellaOps.Feedser.Merge.Tests/AliasGraphResolverTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/AliasGraphResolverTests.cs @@ -3,12 +3,12 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Driver; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Aliases; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Aliases; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; [Collection("mongo-fixture")] public sealed class AliasGraphResolverTests : IClassFixture diff --git a/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs b/src/StellaOps.Concelier.Merge.Tests/CanonicalHashCalculatorTests.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs rename to src/StellaOps.Concelier.Merge.Tests/CanonicalHashCalculatorTests.cs index 93c38494..d85f9947 100644 --- a/src/StellaOps.Feedser.Merge.Tests/CanonicalHashCalculatorTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/CanonicalHashCalculatorTests.cs @@ -1,8 +1,8 @@ using System.Linq; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class CanonicalHashCalculatorTests { diff --git a/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs b/src/StellaOps.Concelier.Merge.Tests/DebianEvrComparerTests.cs similarity index 90% rename from src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs rename to src/StellaOps.Concelier.Merge.Tests/DebianEvrComparerTests.cs index 91925b68..42e8405a 100644 --- a/src/StellaOps.Feedser.Merge.Tests/DebianEvrComparerTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/DebianEvrComparerTests.cs @@ -1,7 +1,7 @@ -using StellaOps.Feedser.Merge.Comparers; -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Merge.Comparers; +using StellaOps.Concelier.Normalization.Distro; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class DebianEvrComparerTests { diff --git a/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs b/src/StellaOps.Concelier.Merge.Tests/MergeEventWriterTests.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs rename to src/StellaOps.Concelier.Merge.Tests/MergeEventWriterTests.cs index 9b6f6f5b..4577a262 100644 --- a/src/StellaOps.Feedser.Merge.Tests/MergeEventWriterTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/MergeEventWriterTests.cs @@ -1,10 +1,10 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class MergeEventWriterTests { diff --git a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs b/src/StellaOps.Concelier.Merge.Tests/MergePrecedenceIntegrationTests.cs similarity index 91% rename from src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs rename to src/StellaOps.Concelier.Merge.Tests/MergePrecedenceIntegrationTests.cs index 1d2fca32..61919920 100644 --- a/src/StellaOps.Feedser.Merge.Tests/MergePrecedenceIntegrationTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/MergePrecedenceIntegrationTests.cs @@ -5,13 +5,13 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using MongoDB.Driver; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; [Collection("mongo-fixture")] public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime @@ -43,7 +43,7 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime var vendor = CreateVendorOverride(); var kev = CreateKevSignal(); - var merged = merger.Merge(new[] { nvd, vendor, kev }); + var merged = merger.Merge(new[] { nvd, vendor, kev }).Advisory; Assert.Equal("CVE-2025-1000", merged.AdvisoryKey); Assert.Equal("Vendor Security Advisory", merged.Title); @@ -62,7 +62,7 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime Assert.Contains("kev", mergeProvenance.Value, StringComparison.OrdinalIgnoreCase); var inputDocumentIds = new[] { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() }; - var record = await writer.AppendAsync(merged.AdvisoryKey, nvd, merged, inputDocumentIds, Array.Empty(), CancellationToken.None); + var record = await writer.AppendAsync(merged.AdvisoryKey, nvd, merged, inputDocumentIds, Array.Empty(), CancellationToken.None); Assert.Equal(expectedTimestamp, record.MergedAt); Assert.Equal(inputDocumentIds, record.InputDocumentIds); @@ -84,8 +84,11 @@ public sealed class MergePrecedenceIntegrationTests : IAsyncLifetime var merger = _merger!; var calculator = new CanonicalHashCalculator(); - var first = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); - var second = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + var firstResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + var secondResult = merger.Merge(new[] { CreateNvdBaseline(), CreateVendorOverride() }); + + var first = firstResult.Advisory; + var second = secondResult.Advisory; var firstHash = calculator.ComputeHash(first); var secondHash = calculator.ComputeHash(second); diff --git a/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs b/src/StellaOps.Concelier.Merge.Tests/MetricCollector.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs rename to src/StellaOps.Concelier.Merge.Tests/MetricCollector.cs index 36e3bde2..fd8c478a 100644 --- a/src/StellaOps.Feedser.Merge.Tests/MetricCollector.cs +++ b/src/StellaOps.Concelier.Merge.Tests/MetricCollector.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; internal sealed class MetricCollector : IDisposable { diff --git a/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs b/src/StellaOps.Concelier.Merge.Tests/NevraComparerTests.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs rename to src/StellaOps.Concelier.Merge.Tests/NevraComparerTests.cs index 2dbbe53e..65fef9d6 100644 --- a/src/StellaOps.Feedser.Merge.Tests/NevraComparerTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/NevraComparerTests.cs @@ -1,7 +1,7 @@ -using StellaOps.Feedser.Merge.Comparers; -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Merge.Comparers; +using StellaOps.Concelier.Normalization.Distro; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class NevraComparerTests { diff --git a/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs b/src/StellaOps.Concelier.Merge.Tests/SemanticVersionRangeResolverTests.cs similarity index 93% rename from src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs rename to src/StellaOps.Concelier.Merge.Tests/SemanticVersionRangeResolverTests.cs index 3937575a..7695e2b7 100644 --- a/src/StellaOps.Feedser.Merge.Tests/SemanticVersionRangeResolverTests.cs +++ b/src/StellaOps.Concelier.Merge.Tests/SemanticVersionRangeResolverTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Merge.Comparers; +using StellaOps.Concelier.Merge.Comparers; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; public sealed class SemanticVersionRangeResolverTests { diff --git a/src/StellaOps.Concelier.Merge.Tests/StellaOps.Concelier.Merge.Tests.csproj b/src/StellaOps.Concelier.Merge.Tests/StellaOps.Concelier.Merge.Tests.csproj new file mode 100644 index 00000000..fb330b60 --- /dev/null +++ b/src/StellaOps.Concelier.Merge.Tests/StellaOps.Concelier.Merge.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs b/src/StellaOps.Concelier.Merge.Tests/TestLogger.cs similarity index 94% rename from src/StellaOps.Feedser.Merge.Tests/TestLogger.cs rename to src/StellaOps.Concelier.Merge.Tests/TestLogger.cs index 5f42c8f3..e41561cb 100644 --- a/src/StellaOps.Feedser.Merge.Tests/TestLogger.cs +++ b/src/StellaOps.Concelier.Merge.Tests/TestLogger.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.Logging; -namespace StellaOps.Feedser.Merge.Tests; +namespace StellaOps.Concelier.Merge.Tests; internal sealed class TestLogger : ILogger { diff --git a/src/StellaOps.Feedser.Merge/AGENTS.md b/src/StellaOps.Concelier.Merge/AGENTS.md similarity index 82% rename from src/StellaOps.Feedser.Merge/AGENTS.md rename to src/StellaOps.Concelier.Merge/AGENTS.md index ba7327f7..9900155f 100644 --- a/src/StellaOps.Feedser.Merge/AGENTS.md +++ b/src/StellaOps.Concelier.Merge/AGENTS.md @@ -19,8 +19,8 @@ Deterministic merge and reconciliation engine; builds identity graph via aliases - Provenance propagation merges unique entries; references deduped by (url, type). ## Configuration -- Precedence overrides bind via `feedser:merge:precedence:ranks` (dictionary of `source` → `rank`, lower wins). Absent entries fall back to defaults. -- Operator workflow: update `etc/feedser.yaml` or environment variables, restart merge job; overrides surface in metrics/logs as `AdvisoryOverride` entries. +- Precedence overrides bind via `concelier:merge:precedence:ranks` (dictionary of `source` → `rank`, lower wins). Absent entries fall back to defaults. +- Operator workflow: update `etc/concelier.yaml` or environment variables, restart merge job; overrides surface in metrics/logs as `AdvisoryOverride` entries. ## In/Out of scope In: merge logic, precedence policy, hashing, event records, comparers. Out: fetching/parsing, exporter packaging, signing. @@ -28,6 +28,6 @@ Out: fetching/parsing, exporter packaging, signing. - Metrics: merge.delta.count, merge.identity.conflicts, merge.range.overrides, merge.duration_ms. - Logs: decisions (why replaced), keys involved, hashes; avoid dumping large blobs; redact secrets (none expected). ## Tests -- Author and review coverage in `../StellaOps.Feedser.Merge.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Merge.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Merge/Class1.cs b/src/StellaOps.Concelier.Merge/Class1.cs similarity index 100% rename from src/StellaOps.Feedser.Merge/Class1.cs rename to src/StellaOps.Concelier.Merge/Class1.cs diff --git a/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs b/src/StellaOps.Concelier.Merge/Comparers/DebianEvr.cs similarity index 94% rename from src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs rename to src/StellaOps.Concelier.Merge/Comparers/DebianEvr.cs index f478cbc0..bf3ebeab 100644 --- a/src/StellaOps.Feedser.Merge/Comparers/DebianEvr.cs +++ b/src/StellaOps.Concelier.Merge/Comparers/DebianEvr.cs @@ -1,7 +1,7 @@ -namespace StellaOps.Feedser.Merge.Comparers; +namespace StellaOps.Concelier.Merge.Comparers; using System; -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Normalization.Distro; public sealed class DebianEvrComparer : IComparer, IComparer { diff --git a/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs b/src/StellaOps.Concelier.Merge/Comparers/Nevra.cs similarity index 94% rename from src/StellaOps.Feedser.Merge/Comparers/Nevra.cs rename to src/StellaOps.Concelier.Merge/Comparers/Nevra.cs index 4914a6ac..5b03c6e1 100644 --- a/src/StellaOps.Feedser.Merge/Comparers/Nevra.cs +++ b/src/StellaOps.Concelier.Merge/Comparers/Nevra.cs @@ -1,7 +1,7 @@ -namespace StellaOps.Feedser.Merge.Comparers; +namespace StellaOps.Concelier.Merge.Comparers; using System; -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Normalization.Distro; public sealed class NevraComparer : IComparer, IComparer { diff --git a/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs b/src/StellaOps.Concelier.Merge/Comparers/SemanticVersionRangeResolver.cs similarity index 95% rename from src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs rename to src/StellaOps.Concelier.Merge/Comparers/SemanticVersionRangeResolver.cs index 5b8c2fe7..e41e6ee5 100644 --- a/src/StellaOps.Feedser.Merge/Comparers/SemanticVersionRangeResolver.cs +++ b/src/StellaOps.Concelier.Merge/Comparers/SemanticVersionRangeResolver.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Merge.Comparers; +namespace StellaOps.Concelier.Merge.Comparers; using System.Diagnostics.CodeAnalysis; using Semver; diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs b/src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityCluster.cs similarity index 94% rename from src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs rename to src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityCluster.cs index 97dd7dee..27dd0fc4 100644 --- a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityCluster.cs +++ b/src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityCluster.cs @@ -1,56 +1,56 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Merge.Identity; - -/// -/// Represents a connected component of advisories that refer to the same vulnerability. -/// -public sealed class AdvisoryIdentityCluster -{ - public AdvisoryIdentityCluster(string advisoryKey, IEnumerable advisories, IEnumerable aliases) - { - AdvisoryKey = !string.IsNullOrWhiteSpace(advisoryKey) - ? advisoryKey.Trim() - : throw new ArgumentException("Canonical advisory key must be provided.", nameof(advisoryKey)); - - var advisoriesArray = (advisories ?? throw new ArgumentNullException(nameof(advisories))) - .Where(static advisory => advisory is not null) - .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .ThenBy(static advisory => advisory.Provenance.Length) - .ThenBy(static advisory => advisory.Title, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - if (advisoriesArray.IsDefaultOrEmpty) - { - throw new ArgumentException("At least one advisory is required for a cluster.", nameof(advisories)); - } - - var aliasArray = (aliases ?? throw new ArgumentNullException(nameof(aliases))) - .Where(static alias => alias is not null && !string.IsNullOrWhiteSpace(alias.Value)) - .GroupBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) - .Select(static group => - { - var representative = group - .OrderBy(static entry => entry.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) - .ThenBy(static entry => entry.Value, StringComparer.OrdinalIgnoreCase) - .First(); - return representative; - }) - .OrderBy(static alias => alias.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) - .ThenBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - - Advisories = advisoriesArray; - Aliases = aliasArray; - } - - public string AdvisoryKey { get; } - - public ImmutableArray Advisories { get; } - - public ImmutableArray Aliases { get; } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Merge.Identity; + +/// +/// Represents a connected component of advisories that refer to the same vulnerability. +/// +public sealed class AdvisoryIdentityCluster +{ + public AdvisoryIdentityCluster(string advisoryKey, IEnumerable advisories, IEnumerable aliases) + { + AdvisoryKey = !string.IsNullOrWhiteSpace(advisoryKey) + ? advisoryKey.Trim() + : throw new ArgumentException("Canonical advisory key must be provided.", nameof(advisoryKey)); + + var advisoriesArray = (advisories ?? throw new ArgumentNullException(nameof(advisories))) + .Where(static advisory => advisory is not null) + .OrderBy(static advisory => advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ThenBy(static advisory => advisory.Provenance.Length) + .ThenBy(static advisory => advisory.Title, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + if (advisoriesArray.IsDefaultOrEmpty) + { + throw new ArgumentException("At least one advisory is required for a cluster.", nameof(advisories)); + } + + var aliasArray = (aliases ?? throw new ArgumentNullException(nameof(aliases))) + .Where(static alias => alias is not null && !string.IsNullOrWhiteSpace(alias.Value)) + .GroupBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .Select(static group => + { + var representative = group + .OrderBy(static entry => entry.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static entry => entry.Value, StringComparer.OrdinalIgnoreCase) + .First(); + return representative; + }) + .OrderBy(static alias => alias.Scheme ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static alias => alias.Value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + Advisories = advisoriesArray; + Aliases = aliasArray; + } + + public string AdvisoryKey { get; } + + public ImmutableArray Advisories { get; } + + public ImmutableArray Aliases { get; } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs b/src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityResolver.cs similarity index 96% rename from src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs rename to src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityResolver.cs index 52c94599..3256a170 100644 --- a/src/StellaOps.Feedser.Merge/Identity/AdvisoryIdentityResolver.cs +++ b/src/StellaOps.Concelier.Merge/Identity/AdvisoryIdentityResolver.cs @@ -1,303 +1,303 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Runtime.CompilerServices; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Merge.Identity; - -/// -/// Builds an alias-driven identity graph that groups advisories referring to the same vulnerability. -/// -public sealed class AdvisoryIdentityResolver -{ - private static readonly string[] CanonicalAliasPriority = - { - AliasSchemes.Cve, - AliasSchemes.Rhsa, - AliasSchemes.Usn, - AliasSchemes.Dsa, - AliasSchemes.SuseSu, - AliasSchemes.Msrc, - AliasSchemes.CiscoSa, - AliasSchemes.OracleCpu, - AliasSchemes.Vmsa, - AliasSchemes.Apsb, - AliasSchemes.Apa, - AliasSchemes.AppleHt, - AliasSchemes.ChromiumPost, - AliasSchemes.Icsa, - AliasSchemes.Jvndb, - AliasSchemes.Jvn, - AliasSchemes.Bdu, - AliasSchemes.Vu, - AliasSchemes.Ghsa, - AliasSchemes.OsV, - }; - - /// - /// Groups the provided advisories into identity clusters using normalized aliases. - /// - public IReadOnlyList Resolve(IEnumerable advisories) - { - ArgumentNullException.ThrowIfNull(advisories); - - var materialized = advisories - .Where(static advisory => advisory is not null) - .Distinct() - .ToArray(); - - if (materialized.Length == 0) - { - return Array.Empty(); - } - - var aliasIndex = BuildAliasIndex(materialized); - var visited = new HashSet(); - var clusters = new List(); - - foreach (var advisory in materialized) - { - if (!visited.Add(advisory)) - { - continue; - } - - var component = TraverseComponent(advisory, visited, aliasIndex); - var key = DetermineCanonicalKey(component); - var aliases = component - .SelectMany(static entry => entry.Aliases) - .Select(static alias => new AliasIdentity(alias.Normalized, alias.Scheme)); - clusters.Add(new AdvisoryIdentityCluster(key, component.Select(static entry => entry.Advisory), aliases)); - } - - return clusters - .OrderBy(static cluster => cluster.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - } - - private static Dictionary> BuildAliasIndex(IEnumerable advisories) - { - var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); - - foreach (var advisory in advisories) - { - foreach (var alias in ExtractAliases(advisory)) - { - if (!index.TryGetValue(alias.Normalized, out var list)) - { - list = new List(); - index[alias.Normalized] = list; - } - - list.Add(new AdvisoryAliasEntry(advisory, alias.Normalized, alias.Scheme)); - } - } - - return index; - } - - private static IReadOnlyList TraverseComponent( - Advisory root, - HashSet visited, - Dictionary> aliasIndex) - { - var stack = new Stack(); - stack.Push(root); - - var bindings = new Dictionary(ReferenceEqualityComparer.Instance); - - while (stack.Count > 0) - { - var advisory = stack.Pop(); - - if (!bindings.TryGetValue(advisory, out var binding)) - { - binding = new AliasBinding(advisory); - bindings[advisory] = binding; - } - - foreach (var alias in ExtractAliases(advisory)) - { - binding.AddAlias(alias.Normalized, alias.Scheme); - - if (!aliasIndex.TryGetValue(alias.Normalized, out var neighbors)) - { - continue; - } - - foreach (var neighbor in neighbors.Select(static entry => entry.Advisory)) - { - if (visited.Add(neighbor)) - { - stack.Push(neighbor); - } - - if (!bindings.TryGetValue(neighbor, out var neighborBinding)) - { - neighborBinding = new AliasBinding(neighbor); - bindings[neighbor] = neighborBinding; - } - - neighborBinding.AddAlias(alias.Normalized, alias.Scheme); - } - } - } - - return bindings.Values - .OrderBy(static binding => binding.Advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) - .ToImmutableArray(); - } - - private static string DetermineCanonicalKey(IReadOnlyList component) - { - var aliases = component - .SelectMany(static binding => binding.Aliases) - .Where(static alias => !string.IsNullOrWhiteSpace(alias.Normalized)) - .ToArray(); - - foreach (var scheme in CanonicalAliasPriority) - { - var candidate = aliases - .Where(alias => string.Equals(alias.Scheme, scheme, StringComparison.OrdinalIgnoreCase)) - .Select(static alias => alias.Normalized) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .FirstOrDefault(); - - if (candidate is not null) - { - return candidate; - } - } - - var fallbackAlias = aliases - .Select(static alias => alias.Normalized) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .FirstOrDefault(); - - if (!string.IsNullOrWhiteSpace(fallbackAlias)) - { - return fallbackAlias; - } - - var advisoryKey = component - .Select(static binding => binding.Advisory.AdvisoryKey) - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) - .FirstOrDefault(); - - if (!string.IsNullOrWhiteSpace(advisoryKey)) - { - return advisoryKey.Trim(); - } - - throw new InvalidOperationException("Unable to determine canonical advisory key for cluster."); - } - - private static IEnumerable ExtractAliases(Advisory advisory) - { - var seen = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var candidate in EnumerateAliasCandidates(advisory)) - { - if (string.IsNullOrWhiteSpace(candidate)) - { - continue; - } - - var trimmed = candidate.Trim(); - if (!seen.Add(trimmed)) - { - continue; - } - - if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme) && - !string.IsNullOrWhiteSpace(normalized)) - { - yield return new AliasProjection(normalized.Trim(), string.IsNullOrWhiteSpace(scheme) ? null : scheme); - } - else if (!string.IsNullOrWhiteSpace(normalized)) - { - yield return new AliasProjection(normalized.Trim(), null); - } - } - } - - private static IEnumerable EnumerateAliasCandidates(Advisory advisory) - { - if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) - { - yield return advisory.AdvisoryKey; - } - - if (!advisory.Aliases.IsDefaultOrEmpty) - { - foreach (var alias in advisory.Aliases) - { - if (!string.IsNullOrWhiteSpace(alias)) - { - yield return alias; - } - } - } - } - - private readonly record struct AdvisoryAliasEntry(Advisory Advisory, string Normalized, string? Scheme); - - private readonly record struct AliasProjection(string Normalized, string? Scheme); - - private sealed class AliasBinding - { - private readonly HashSet _aliases = new(HashSetAliasComparer.Instance); - - public AliasBinding(Advisory advisory) - { - Advisory = advisory ?? throw new ArgumentNullException(nameof(advisory)); - } - - public Advisory Advisory { get; } - - public IReadOnlyCollection Aliases => _aliases; - - public void AddAlias(string normalized, string? scheme) - { - if (string.IsNullOrWhiteSpace(normalized)) - { - return; - } - - _aliases.Add(new AliasProjection(normalized.Trim(), scheme is null ? null : scheme.Trim())); - } - } - - private sealed class HashSetAliasComparer : IEqualityComparer - { - public static readonly HashSetAliasComparer Instance = new(); - - public bool Equals(AliasProjection x, AliasProjection y) - => string.Equals(x.Normalized, y.Normalized, StringComparison.OrdinalIgnoreCase) - && string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase); - - public int GetHashCode(AliasProjection obj) - { - var hash = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Normalized); - if (!string.IsNullOrWhiteSpace(obj.Scheme)) - { - hash = HashCode.Combine(hash, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Scheme)); - } - - return hash; - } - } - - private sealed class ReferenceEqualityComparer : IEqualityComparer - where T : class - { - public static readonly ReferenceEqualityComparer Instance = new(); - - public bool Equals(T? x, T? y) => ReferenceEquals(x, y); - - public int GetHashCode(T obj) => RuntimeHelpers.GetHashCode(obj); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Merge.Identity; + +/// +/// Builds an alias-driven identity graph that groups advisories referring to the same vulnerability. +/// +public sealed class AdvisoryIdentityResolver +{ + private static readonly string[] CanonicalAliasPriority = + { + AliasSchemes.Cve, + AliasSchemes.Rhsa, + AliasSchemes.Usn, + AliasSchemes.Dsa, + AliasSchemes.SuseSu, + AliasSchemes.Msrc, + AliasSchemes.CiscoSa, + AliasSchemes.OracleCpu, + AliasSchemes.Vmsa, + AliasSchemes.Apsb, + AliasSchemes.Apa, + AliasSchemes.AppleHt, + AliasSchemes.ChromiumPost, + AliasSchemes.Icsa, + AliasSchemes.Jvndb, + AliasSchemes.Jvn, + AliasSchemes.Bdu, + AliasSchemes.Vu, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + }; + + /// + /// Groups the provided advisories into identity clusters using normalized aliases. + /// + public IReadOnlyList Resolve(IEnumerable advisories) + { + ArgumentNullException.ThrowIfNull(advisories); + + var materialized = advisories + .Where(static advisory => advisory is not null) + .Distinct() + .ToArray(); + + if (materialized.Length == 0) + { + return Array.Empty(); + } + + var aliasIndex = BuildAliasIndex(materialized); + var visited = new HashSet(); + var clusters = new List(); + + foreach (var advisory in materialized) + { + if (!visited.Add(advisory)) + { + continue; + } + + var component = TraverseComponent(advisory, visited, aliasIndex); + var key = DetermineCanonicalKey(component); + var aliases = component + .SelectMany(static entry => entry.Aliases) + .Select(static alias => new AliasIdentity(alias.Normalized, alias.Scheme)); + clusters.Add(new AdvisoryIdentityCluster(key, component.Select(static entry => entry.Advisory), aliases)); + } + + return clusters + .OrderBy(static cluster => cluster.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static Dictionary> BuildAliasIndex(IEnumerable advisories) + { + var index = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var advisory in advisories) + { + foreach (var alias in ExtractAliases(advisory)) + { + if (!index.TryGetValue(alias.Normalized, out var list)) + { + list = new List(); + index[alias.Normalized] = list; + } + + list.Add(new AdvisoryAliasEntry(advisory, alias.Normalized, alias.Scheme)); + } + } + + return index; + } + + private static IReadOnlyList TraverseComponent( + Advisory root, + HashSet visited, + Dictionary> aliasIndex) + { + var stack = new Stack(); + stack.Push(root); + + var bindings = new Dictionary(ReferenceEqualityComparer.Instance); + + while (stack.Count > 0) + { + var advisory = stack.Pop(); + + if (!bindings.TryGetValue(advisory, out var binding)) + { + binding = new AliasBinding(advisory); + bindings[advisory] = binding; + } + + foreach (var alias in ExtractAliases(advisory)) + { + binding.AddAlias(alias.Normalized, alias.Scheme); + + if (!aliasIndex.TryGetValue(alias.Normalized, out var neighbors)) + { + continue; + } + + foreach (var neighbor in neighbors.Select(static entry => entry.Advisory)) + { + if (visited.Add(neighbor)) + { + stack.Push(neighbor); + } + + if (!bindings.TryGetValue(neighbor, out var neighborBinding)) + { + neighborBinding = new AliasBinding(neighbor); + bindings[neighbor] = neighborBinding; + } + + neighborBinding.AddAlias(alias.Normalized, alias.Scheme); + } + } + } + + return bindings.Values + .OrderBy(static binding => binding.Advisory.AdvisoryKey, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static string DetermineCanonicalKey(IReadOnlyList component) + { + var aliases = component + .SelectMany(static binding => binding.Aliases) + .Where(static alias => !string.IsNullOrWhiteSpace(alias.Normalized)) + .ToArray(); + + foreach (var scheme in CanonicalAliasPriority) + { + var candidate = aliases + .Where(alias => string.Equals(alias.Scheme, scheme, StringComparison.OrdinalIgnoreCase)) + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (candidate is not null) + { + return candidate; + } + } + + var fallbackAlias = aliases + .Select(static alias => alias.Normalized) + .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(fallbackAlias)) + { + return fallbackAlias; + } + + var advisoryKey = component + .Select(static binding => binding.Advisory.AdvisoryKey) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .FirstOrDefault(); + + if (!string.IsNullOrWhiteSpace(advisoryKey)) + { + return advisoryKey.Trim(); + } + + throw new InvalidOperationException("Unable to determine canonical advisory key for cluster."); + } + + private static IEnumerable ExtractAliases(Advisory advisory) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var candidate in EnumerateAliasCandidates(advisory)) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + continue; + } + + var trimmed = candidate.Trim(); + if (!seen.Add(trimmed)) + { + continue; + } + + if (AliasSchemeRegistry.TryNormalize(trimmed, out var normalized, out var scheme) && + !string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), string.IsNullOrWhiteSpace(scheme) ? null : scheme); + } + else if (!string.IsNullOrWhiteSpace(normalized)) + { + yield return new AliasProjection(normalized.Trim(), null); + } + } + } + + private static IEnumerable EnumerateAliasCandidates(Advisory advisory) + { + if (!string.IsNullOrWhiteSpace(advisory.AdvisoryKey)) + { + yield return advisory.AdvisoryKey; + } + + if (!advisory.Aliases.IsDefaultOrEmpty) + { + foreach (var alias in advisory.Aliases) + { + if (!string.IsNullOrWhiteSpace(alias)) + { + yield return alias; + } + } + } + } + + private readonly record struct AdvisoryAliasEntry(Advisory Advisory, string Normalized, string? Scheme); + + private readonly record struct AliasProjection(string Normalized, string? Scheme); + + private sealed class AliasBinding + { + private readonly HashSet _aliases = new(HashSetAliasComparer.Instance); + + public AliasBinding(Advisory advisory) + { + Advisory = advisory ?? throw new ArgumentNullException(nameof(advisory)); + } + + public Advisory Advisory { get; } + + public IReadOnlyCollection Aliases => _aliases; + + public void AddAlias(string normalized, string? scheme) + { + if (string.IsNullOrWhiteSpace(normalized)) + { + return; + } + + _aliases.Add(new AliasProjection(normalized.Trim(), scheme is null ? null : scheme.Trim())); + } + } + + private sealed class HashSetAliasComparer : IEqualityComparer + { + public static readonly HashSetAliasComparer Instance = new(); + + public bool Equals(AliasProjection x, AliasProjection y) + => string.Equals(x.Normalized, y.Normalized, StringComparison.OrdinalIgnoreCase) + && string.Equals(x.Scheme, y.Scheme, StringComparison.OrdinalIgnoreCase); + + public int GetHashCode(AliasProjection obj) + { + var hash = StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Normalized); + if (!string.IsNullOrWhiteSpace(obj.Scheme)) + { + hash = HashCode.Combine(hash, StringComparer.OrdinalIgnoreCase.GetHashCode(obj.Scheme)); + } + + return hash; + } + } + + private sealed class ReferenceEqualityComparer : IEqualityComparer + where T : class + { + public static readonly ReferenceEqualityComparer Instance = new(); + + public bool Equals(T? x, T? y) => ReferenceEquals(x, y); + + public int GetHashCode(T obj) => RuntimeHelpers.GetHashCode(obj); + } +} diff --git a/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs b/src/StellaOps.Concelier.Merge/Identity/AliasIdentity.cs similarity index 88% rename from src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs rename to src/StellaOps.Concelier.Merge/Identity/AliasIdentity.cs index 193f7cc2..991fb729 100644 --- a/src/StellaOps.Feedser.Merge/Identity/AliasIdentity.cs +++ b/src/StellaOps.Concelier.Merge/Identity/AliasIdentity.cs @@ -1,24 +1,24 @@ -using System; - -namespace StellaOps.Feedser.Merge.Identity; - -/// -/// Normalized alias representation used within identity clusters. -/// -public sealed class AliasIdentity -{ - public AliasIdentity(string value, string? scheme) - { - if (string.IsNullOrWhiteSpace(value)) - { - throw new ArgumentException("Alias value must be provided.", nameof(value)); - } - - Value = value.Trim(); - Scheme = string.IsNullOrWhiteSpace(scheme) ? null : scheme.Trim(); - } - - public string Value { get; } - - public string? Scheme { get; } -} +using System; + +namespace StellaOps.Concelier.Merge.Identity; + +/// +/// Normalized alias representation used within identity clusters. +/// +public sealed class AliasIdentity +{ + public AliasIdentity(string value, string? scheme) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Alias value must be provided.", nameof(value)); + } + + Value = value.Trim(); + Scheme = string.IsNullOrWhiteSpace(scheme) ? null : scheme.Trim(); + } + + public string Value { get; } + + public string? Scheme { get; } +} diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs b/src/StellaOps.Concelier.Merge/Jobs/MergeJobKinds.cs similarity index 66% rename from src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs rename to src/StellaOps.Concelier.Merge/Jobs/MergeJobKinds.cs index 95cd7b3f..0f06e9cb 100644 --- a/src/StellaOps.Feedser.Merge/Jobs/MergeJobKinds.cs +++ b/src/StellaOps.Concelier.Merge/Jobs/MergeJobKinds.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Merge.Jobs; +namespace StellaOps.Concelier.Merge.Jobs; internal static class MergeJobKinds { diff --git a/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs b/src/StellaOps.Concelier.Merge/Jobs/MergeReconcileJob.cs similarity index 90% rename from src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs rename to src/StellaOps.Concelier.Merge/Jobs/MergeReconcileJob.cs index 5c98f117..fa7479c6 100644 --- a/src/StellaOps.Feedser.Merge/Jobs/MergeReconcileJob.cs +++ b/src/StellaOps.Concelier.Merge/Jobs/MergeReconcileJob.cs @@ -2,10 +2,10 @@ using System; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Merge.Services; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Merge.Services; -namespace StellaOps.Feedser.Merge.Jobs; +namespace StellaOps.Concelier.Merge.Jobs; public sealed class MergeReconcileJob : IJob { diff --git a/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Merge/MergeServiceCollectionExtensions.cs similarity index 76% rename from src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Merge/MergeServiceCollectionExtensions.cs index 1896e6dc..eb120c5c 100644 --- a/src/StellaOps.Feedser.Merge/MergeServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Merge/MergeServiceCollectionExtensions.cs @@ -2,12 +2,12 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Core; -using StellaOps.Feedser.Merge.Jobs; -using StellaOps.Feedser.Merge.Options; -using StellaOps.Feedser.Merge.Services; +using StellaOps.Concelier.Core; +using StellaOps.Concelier.Merge.Jobs; +using StellaOps.Concelier.Merge.Options; +using StellaOps.Concelier.Merge.Services; -namespace StellaOps.Feedser.Merge; +namespace StellaOps.Concelier.Merge; public static class MergeServiceCollectionExtensions { @@ -21,14 +21,14 @@ public static class MergeServiceCollectionExtensions services.TryAddSingleton(); services.TryAddSingleton(sp => { - var options = configuration.GetSection("feedser:merge:precedence").Get(); + var options = configuration.GetSection("concelier:merge:precedence").Get(); return options is null ? new AffectedPackagePrecedenceResolver() : new AffectedPackagePrecedenceResolver(options); }); services.TryAddSingleton(sp => { var resolver = sp.GetRequiredService(); - var options = configuration.GetSection("feedser:merge:precedence").Get(); + var options = configuration.GetSection("concelier:merge:precedence").Get(); var timeProvider = sp.GetRequiredService(); var logger = sp.GetRequiredService>(); return new AdvisoryPrecedenceMerger(resolver, options, timeProvider, logger); diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceDefaults.cs similarity index 94% rename from src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs rename to src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceDefaults.cs index 5c22f4e3..ec432812 100644 --- a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceDefaults.cs +++ b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceDefaults.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Merge.Options; +namespace StellaOps.Concelier.Merge.Options; /// /// Provides the built-in precedence table used by the merge engine when no overrides are supplied. diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceOptions.cs similarity index 88% rename from src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs rename to src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceOptions.cs index bda12f2b..c8f5b40a 100644 --- a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceOptions.cs +++ b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceOptions.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Merge.Options; +namespace StellaOps.Concelier.Merge.Options; /// /// Configurable precedence overrides for advisory sources. diff --git a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceTable.cs similarity index 91% rename from src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs rename to src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceTable.cs index 12bd9903..a7a43da7 100644 --- a/src/StellaOps.Feedser.Merge/Options/AdvisoryPrecedenceTable.cs +++ b/src/StellaOps.Concelier.Merge/Options/AdvisoryPrecedenceTable.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Merge.Options; +namespace StellaOps.Concelier.Merge.Options; internal static class AdvisoryPrecedenceTable { diff --git a/src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md b/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md similarity index 95% rename from src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md rename to src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md index 25aad08f..86612721 100644 --- a/src/StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md +++ b/src/StellaOps.Concelier.Merge/RANGE_PRIMITIVES_COORDINATION.md @@ -32,7 +32,7 @@ Until these blocks land, connectors should stage changes behind a feature flag o | Ru.Bdu | BE-Conn-BDU | All tasks TODO | Map product releases into normalized rules; add provenance notes referencing BDU advisory identifiers. | Verify we have UTF-8 safe handling in builder; share sample sanitized inputs. | | Ru.Nkcki | BE-Conn-Nkcki | All tasks TODO | Similar to BDU; capture vendor firmware/build numbers and map into normalized rules. | Coordinate with Localization WG for Cyrillic transliteration strategy. | | Vndr.Apple | BE-Conn-Apple | Mapper/tests/telemetry marked DOING | Continue extending vendor range primitives (`apple.version`, `apple.build`) and adopt normalized rule arrays for OS build spans. | Request builder integration review on 2025-10-16; ensure fixtures cover multi-range tables and include provenance notes. | -| Vndr.Cisco | BE-Conn-Cisco | ✅ Emits SemVer primitives with vendor notes | Parser maps versions into SemVer primitives with `cisco.productId` vendor extensions; sample fixtures landing in `StellaOps.Feedser.Source.Vndr.Cisco.Tests`. | No custom comparer required; SemVer + vendor metadata suffices. | +| Vndr.Cisco | BE-Conn-Cisco | ✅ Emits SemVer primitives with vendor notes | Parser maps versions into SemVer primitives with `cisco.productId` vendor extensions; sample fixtures landing in `StellaOps.Concelier.Connector.Vndr.Cisco.Tests`. | No custom comparer required; SemVer + vendor metadata suffices. | | Vndr.Msrc | BE-Conn-MSRC | All tasks TODO | Canonical mapper must output product/build coverage as normalized rules (likely `msrc.patch` scheme) with provenance referencing KB IDs. | Sync with Models on adding scheme identifiers for MSRC packages; plan fixture coverage for monthly rollups. | ## Storage alignment quick reference (2025-10-11) @@ -86,10 +86,10 @@ Until these blocks land, connectors should stage changes behind a feature flag o - Normalization team to share draft `SemVerRangeRuleBuilder` API by **2025-10-13** for review; Merge will circulate feedback within 24 hours. - Connector owners to prepare fixture pull requests demonstrating sample normalized rule arrays (even if feature-flagged) by **2025-10-17**. - Merge team will run a cross-connector review on **2025-10-18** to confirm consistent field usage and provenance tagging before enabling merge union logic. -- Schedule held for **2025-10-14 14:00 UTC** to review the CERT/CC staging VINCE advisory sample once `enableDetailMapping` is flipped; capture findings in `#feedser-merge` with snapshot diffs. +- Schedule held for **2025-10-14 14:00 UTC** to review the CERT/CC staging VINCE advisory sample once `enableDetailMapping` is flipped; capture findings in `#concelier-merge` with snapshot diffs. ## Tracking & follow-up - Capture connector progress updates in stand-ups twice per week; link PRs/issues back to this document and the rollout dashboard (`docs/dev/normalized_versions_rollout.md`). -- Monitor merge counters `feedser.merge.normalized_rules` and `feedser.merge.normalized_rules_missing` to spot advisories that still lack normalized arrays after precedence merge. -- When a connector is ready to emit normalized rules, update its module `TASKS.md` status and ping Merge in `#feedser-merge` with fixture diff screenshots. +- Monitor merge counters `concelier.merge.normalized_rules` and `concelier.merge.normalized_rules_missing` to spot advisories that still lack normalized arrays after precedence merge. +- When a connector is ready to emit normalized rules, update its module `TASKS.md` status and ping Merge in `#concelier-merge` with fixture diff screenshots. - If new schemes or comparer logic is required (e.g., Cisco IOS), open a Models issue referencing `FEEDMODELS-SCHEMA-02-900` before implementing. diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs b/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs similarity index 61% rename from src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs rename to src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs index af520e5e..2ac49674 100644 --- a/src/StellaOps.Feedser.Merge/Services/AdvisoryMergeService.cs +++ b/src/StellaOps.Concelier.Merge/Services/AdvisoryMergeService.cs @@ -1,57 +1,65 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics.Metrics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Core; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Aliases; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Concelier.Core; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; +using System.Text.Json; -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; public sealed class AdvisoryMergeService { - private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Meter MergeMeter = new("StellaOps.Concelier.Merge"); private static readonly Counter AliasCollisionCounter = MergeMeter.CreateCounter( - "feedser.merge.identity_conflicts", + "concelier.merge.identity_conflicts", unit: "count", description: "Number of alias collisions detected during merge."); - private static readonly string[] PreferredAliasSchemes = - { - AliasSchemes.Cve, - AliasSchemes.Ghsa, - AliasSchemes.OsV, - AliasSchemes.Msrc, - }; - - private readonly AliasGraphResolver _aliasResolver; - private readonly IAdvisoryStore _advisoryStore; - private readonly AdvisoryPrecedenceMerger _precedenceMerger; - private readonly MergeEventWriter _mergeEventWriter; - private readonly CanonicalMerger _canonicalMerger; - private readonly ILogger _logger; - - public AdvisoryMergeService( - AliasGraphResolver aliasResolver, - IAdvisoryStore advisoryStore, - AdvisoryPrecedenceMerger precedenceMerger, - MergeEventWriter mergeEventWriter, - CanonicalMerger canonicalMerger, - ILogger logger) - { - _aliasResolver = aliasResolver ?? throw new ArgumentNullException(nameof(aliasResolver)); - _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); - _precedenceMerger = precedenceMerger ?? throw new ArgumentNullException(nameof(precedenceMerger)); - _mergeEventWriter = mergeEventWriter ?? throw new ArgumentNullException(nameof(mergeEventWriter)); - _canonicalMerger = canonicalMerger ?? throw new ArgumentNullException(nameof(canonicalMerger)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } + private static readonly string[] PreferredAliasSchemes = + { + AliasSchemes.Cve, + AliasSchemes.Ghsa, + AliasSchemes.OsV, + AliasSchemes.Msrc, + }; + + private readonly AliasGraphResolver _aliasResolver; + private readonly IAdvisoryStore _advisoryStore; + private readonly AdvisoryPrecedenceMerger _precedenceMerger; + private readonly MergeEventWriter _mergeEventWriter; + private readonly IAdvisoryEventLog _eventLog; + private readonly TimeProvider _timeProvider; + private readonly CanonicalMerger _canonicalMerger; + private readonly ILogger _logger; + + public AdvisoryMergeService( + AliasGraphResolver aliasResolver, + IAdvisoryStore advisoryStore, + AdvisoryPrecedenceMerger precedenceMerger, + MergeEventWriter mergeEventWriter, + CanonicalMerger canonicalMerger, + IAdvisoryEventLog eventLog, + TimeProvider timeProvider, + ILogger logger) + { + _aliasResolver = aliasResolver ?? throw new ArgumentNullException(nameof(aliasResolver)); + _advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore)); + _precedenceMerger = precedenceMerger ?? throw new ArgumentNullException(nameof(precedenceMerger)); + _mergeEventWriter = mergeEventWriter ?? throw new ArgumentNullException(nameof(mergeEventWriter)); + _canonicalMerger = canonicalMerger ?? throw new ArgumentNullException(nameof(canonicalMerger)); + _eventLog = eventLog ?? throw new ArgumentNullException(nameof(eventLog)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } public async Task MergeAsync(string seedAdvisoryKey, CancellationToken cancellationToken) { @@ -76,15 +84,15 @@ public sealed class AdvisoryMergeService return AdvisoryMergeResult.Empty(seedAdvisoryKey, component); } - var canonicalKey = SelectCanonicalKey(component) ?? seedAdvisoryKey; - var canonicalMerge = ApplyCanonicalMergeIfNeeded(canonicalKey, inputs); - var before = await _advisoryStore.FindAsync(canonicalKey, cancellationToken).ConfigureAwait(false); - var normalizedInputs = NormalizeInputs(inputs, canonicalKey).ToList(); - - Advisory? merged; - try - { - merged = _precedenceMerger.Merge(normalizedInputs); + var canonicalKey = SelectCanonicalKey(component) ?? seedAdvisoryKey; + var canonicalMerge = ApplyCanonicalMergeIfNeeded(canonicalKey, inputs); + var before = await _advisoryStore.FindAsync(canonicalKey, cancellationToken).ConfigureAwait(false); + var normalizedInputs = NormalizeInputs(inputs, canonicalKey).ToList(); + + PrecedenceMergeResult precedenceResult; + try + { + precedenceResult = _precedenceMerger.Merge(normalizedInputs); } catch (Exception ex) { @@ -92,6 +100,9 @@ public sealed class AdvisoryMergeService throw; } + var merged = precedenceResult.Advisory; + var conflictDetails = precedenceResult.Conflicts; + if (component.Collisions.Count > 0) { foreach (var collision in component.Collisions) @@ -113,21 +124,146 @@ public sealed class AdvisoryMergeService } } - if (merged is not null) - { - await _advisoryStore.UpsertAsync(merged, cancellationToken).ConfigureAwait(false); - await _mergeEventWriter.AppendAsync( - canonicalKey, - before, - merged, - Array.Empty(), - ConvertFieldDecisions(canonicalMerge?.Decisions), - cancellationToken).ConfigureAwait(false); - } - - return new AdvisoryMergeResult(seedAdvisoryKey, canonicalKey, component, inputs, before, merged); + await _advisoryStore.UpsertAsync(merged, cancellationToken).ConfigureAwait(false); + await _mergeEventWriter.AppendAsync( + canonicalKey, + before, + merged, + Array.Empty(), + ConvertFieldDecisions(canonicalMerge?.Decisions), + cancellationToken).ConfigureAwait(false); + + await AppendEventLogAsync(canonicalKey, normalizedInputs, merged, conflictDetails, cancellationToken).ConfigureAwait(false); + + return new AdvisoryMergeResult(seedAdvisoryKey, canonicalKey, component, inputs, before, merged); } + private async Task AppendEventLogAsync( + string vulnerabilityKey, + IReadOnlyList inputs, + Advisory merged, + IReadOnlyList conflicts, + CancellationToken cancellationToken) + { + var recordedAt = _timeProvider.GetUtcNow(); + var statements = new List(inputs.Count + 1); + var statementIds = new Dictionary(ReferenceEqualityComparer.Instance); + + foreach (var advisory in inputs) + { + var statementId = Guid.NewGuid(); + statementIds[advisory] = statementId; + statements.Add(new AdvisoryStatementInput( + vulnerabilityKey, + advisory, + DetermineAsOf(advisory, recordedAt), + InputDocumentIds: Array.Empty(), + StatementId: statementId, + AdvisoryKey: advisory.AdvisoryKey)); + } + + var canonicalStatementId = Guid.NewGuid(); + statementIds[merged] = canonicalStatementId; + statements.Add(new AdvisoryStatementInput( + vulnerabilityKey, + merged, + recordedAt, + InputDocumentIds: Array.Empty(), + StatementId: canonicalStatementId, + AdvisoryKey: merged.AdvisoryKey)); + + var conflictInputs = BuildConflictInputs(conflicts, vulnerabilityKey, statementIds, canonicalStatementId, recordedAt); + + if (statements.Count == 0 && conflictInputs.Count == 0) + { + return; + } + + var request = new AdvisoryEventAppendRequest(statements, conflictInputs.Count > 0 ? conflictInputs : null); + + try + { + await _eventLog.AppendAsync(request, cancellationToken).ConfigureAwait(false); + } + finally + { + foreach (var conflict in conflictInputs) + { + conflict.Details.Dispose(); + } + } + } + + private static DateTimeOffset DetermineAsOf(Advisory advisory, DateTimeOffset fallback) + { + return (advisory.Modified ?? advisory.Published ?? fallback).ToUniversalTime(); + } + + private static List BuildConflictInputs( + IReadOnlyList conflicts, + string vulnerabilityKey, + IReadOnlyDictionary statementIds, + Guid canonicalStatementId, + DateTimeOffset recordedAt) + { + if (conflicts.Count == 0) + { + return new List(0); + } + + var inputs = new List(conflicts.Count); + + foreach (var detail in conflicts) + { + if (!statementIds.TryGetValue(detail.Suppressed, out var suppressedId)) + { + continue; + } + + var related = new List { canonicalStatementId, suppressedId }; + if (statementIds.TryGetValue(detail.Primary, out var primaryId)) + { + if (!related.Contains(primaryId)) + { + related.Add(primaryId); + } + } + + var payload = new ConflictDetailPayload( + detail.ConflictType, + detail.Reason, + detail.PrimarySources, + detail.PrimaryRank, + detail.SuppressedSources, + detail.SuppressedRank, + detail.PrimaryValue, + detail.SuppressedValue); + + var json = CanonicalJsonSerializer.Serialize(payload); + var document = JsonDocument.Parse(json); + var asOf = (detail.Primary.Modified ?? detail.Suppressed.Modified ?? recordedAt).ToUniversalTime(); + + inputs.Add(new AdvisoryConflictInput( + vulnerabilityKey, + document, + asOf, + related, + ConflictId: null)); + } + + return inputs; + } + + private sealed record ConflictDetailPayload( + string Type, + string Reason, + IReadOnlyList PrimarySources, + int PrimaryRank, + IReadOnlyList SuppressedSources, + int SuppressedRank, + string? PrimaryValue, + string? SuppressedValue); + private static IEnumerable NormalizeInputs(IEnumerable advisories, string canonicalKey) { foreach (var advisory in advisories) @@ -136,118 +272,118 @@ public sealed class AdvisoryMergeService } } - private static Advisory CloneWithKey(Advisory source, string advisoryKey) - => new( - advisoryKey, - source.Title, - source.Summary, - source.Language, - source.Published, - source.Modified, - source.Severity, - source.ExploitKnown, - source.Aliases, - source.Credits, - source.References, - source.AffectedPackages, - source.CvssMetrics, - source.Provenance, - source.Description, - source.Cwes, - source.CanonicalMetricId); - - private CanonicalMergeResult? ApplyCanonicalMergeIfNeeded(string canonicalKey, List inputs) - { - if (inputs.Count == 0) - { - return null; - } - - var ghsa = FindBySource(inputs, CanonicalSources.Ghsa); - var nvd = FindBySource(inputs, CanonicalSources.Nvd); - var osv = FindBySource(inputs, CanonicalSources.Osv); - - var participatingSources = 0; - if (ghsa is not null) - { - participatingSources++; - } - - if (nvd is not null) - { - participatingSources++; - } - - if (osv is not null) - { - participatingSources++; - } - - if (participatingSources < 2) - { - return null; - } - - var result = _canonicalMerger.Merge(canonicalKey, ghsa, nvd, osv); - - inputs.RemoveAll(advisory => MatchesCanonicalSource(advisory)); - inputs.Add(result.Advisory); - - return result; - } - - private static Advisory? FindBySource(IEnumerable advisories, string source) - => advisories.FirstOrDefault(advisory => advisory.Provenance.Any(provenance => - !string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && - string.Equals(provenance.Source, source, StringComparison.OrdinalIgnoreCase))); - - private static bool MatchesCanonicalSource(Advisory advisory) - { - foreach (var provenance in advisory.Provenance) - { - if (string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (string.Equals(provenance.Source, CanonicalSources.Ghsa, StringComparison.OrdinalIgnoreCase) || - string.Equals(provenance.Source, CanonicalSources.Nvd, StringComparison.OrdinalIgnoreCase) || - string.Equals(provenance.Source, CanonicalSources.Osv, StringComparison.OrdinalIgnoreCase)) - { - return true; - } - } - - return false; - } - - private static IReadOnlyList ConvertFieldDecisions(ImmutableArray? decisions) - { - if (decisions is null || decisions.Value.IsDefaultOrEmpty) - { - return Array.Empty(); - } - - var builder = ImmutableArray.CreateBuilder(decisions.Value.Length); - foreach (var decision in decisions.Value) - { - builder.Add(new MergeFieldDecision( - decision.Field, - decision.SelectedSource, - decision.DecisionReason, - decision.SelectedModified, - decision.ConsideredSources.ToArray())); - } - - return builder.ToImmutable(); - } - - private static class CanonicalSources - { - public const string Ghsa = "ghsa"; - public const string Nvd = "nvd"; - public const string Osv = "osv"; - } + private static Advisory CloneWithKey(Advisory source, string advisoryKey) + => new( + advisoryKey, + source.Title, + source.Summary, + source.Language, + source.Published, + source.Modified, + source.Severity, + source.ExploitKnown, + source.Aliases, + source.Credits, + source.References, + source.AffectedPackages, + source.CvssMetrics, + source.Provenance, + source.Description, + source.Cwes, + source.CanonicalMetricId); + + private CanonicalMergeResult? ApplyCanonicalMergeIfNeeded(string canonicalKey, List inputs) + { + if (inputs.Count == 0) + { + return null; + } + + var ghsa = FindBySource(inputs, CanonicalSources.Ghsa); + var nvd = FindBySource(inputs, CanonicalSources.Nvd); + var osv = FindBySource(inputs, CanonicalSources.Osv); + + var participatingSources = 0; + if (ghsa is not null) + { + participatingSources++; + } + + if (nvd is not null) + { + participatingSources++; + } + + if (osv is not null) + { + participatingSources++; + } + + if (participatingSources < 2) + { + return null; + } + + var result = _canonicalMerger.Merge(canonicalKey, ghsa, nvd, osv); + + inputs.RemoveAll(advisory => MatchesCanonicalSource(advisory)); + inputs.Add(result.Advisory); + + return result; + } + + private static Advisory? FindBySource(IEnumerable advisories, string source) + => advisories.FirstOrDefault(advisory => advisory.Provenance.Any(provenance => + !string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase) && + string.Equals(provenance.Source, source, StringComparison.OrdinalIgnoreCase))); + + private static bool MatchesCanonicalSource(Advisory advisory) + { + foreach (var provenance in advisory.Provenance) + { + if (string.Equals(provenance.Kind, "merge", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (string.Equals(provenance.Source, CanonicalSources.Ghsa, StringComparison.OrdinalIgnoreCase) || + string.Equals(provenance.Source, CanonicalSources.Nvd, StringComparison.OrdinalIgnoreCase) || + string.Equals(provenance.Source, CanonicalSources.Osv, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static IReadOnlyList ConvertFieldDecisions(ImmutableArray? decisions) + { + if (decisions is null || decisions.Value.IsDefaultOrEmpty) + { + return Array.Empty(); + } + + var builder = ImmutableArray.CreateBuilder(decisions.Value.Length); + foreach (var decision in decisions.Value) + { + builder.Add(new MergeFieldDecision( + decision.Field, + decision.SelectedSource, + decision.DecisionReason, + decision.SelectedModified, + decision.ConsideredSources.ToArray())); + } + + return builder.ToImmutable(); + } + + private static class CanonicalSources + { + public const string Ghsa = "ghsa"; + public const string Nvd = "nvd"; + public const string Osv = "osv"; + } private static string? SelectCanonicalKey(AliasComponent component) { diff --git a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs b/src/StellaOps.Concelier.Merge/Services/AdvisoryPrecedenceMerger.cs similarity index 91% rename from src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs rename to src/StellaOps.Concelier.Merge/Services/AdvisoryPrecedenceMerger.cs index 621ce27d..22c72979 100644 --- a/src/StellaOps.Feedser.Merge/Services/AdvisoryPrecedenceMerger.cs +++ b/src/StellaOps.Concelier.Merge/Services/AdvisoryPrecedenceMerger.cs @@ -5,44 +5,44 @@ using System.Globalization; using System.Linq; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Merge.Options; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Merge.Options; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; /// /// Merges canonical advisories emitted by different sources into a single precedence-resolved advisory. /// public sealed class AdvisoryPrecedenceMerger { - private static readonly Meter MergeMeter = new("StellaOps.Feedser.Merge"); + private static readonly Meter MergeMeter = new("StellaOps.Concelier.Merge"); private static readonly Counter MergeCounter = MergeMeter.CreateCounter( - "feedser.merge.operations", + "concelier.merge.operations", unit: "count", description: "Number of merge invocations executed by the precedence engine."); private static readonly Counter OverridesCounter = MergeMeter.CreateCounter( - "feedser.merge.overrides", + "concelier.merge.overrides", unit: "count", description: "Number of times lower-precedence advisories were overridden by higher-precedence sources."); private static readonly Counter RangeOverrideCounter = MergeMeter.CreateCounter( - "feedser.merge.range_overrides", + "concelier.merge.range_overrides", unit: "count", description: "Number of affected-package range overrides performed during precedence merge."); private static readonly Counter ConflictCounter = MergeMeter.CreateCounter( - "feedser.merge.conflicts", + "concelier.merge.conflicts", unit: "count", description: "Number of precedence conflicts detected (severity, rank ties, etc.)."); private static readonly Counter NormalizedRuleCounter = MergeMeter.CreateCounter( - "feedser.merge.normalized_rules", + "concelier.merge.normalized_rules", unit: "rule", description: "Number of normalized version rules retained after precedence merge."); private static readonly Counter MissingNormalizedRuleCounter = MergeMeter.CreateCounter( - "feedser.merge.normalized_rules_missing", + "concelier.merge.normalized_rules_missing", unit: "package", description: "Number of affected packages with version ranges but no normalized rules."); @@ -111,7 +111,7 @@ public sealed class AdvisoryPrecedenceMerger _logger = logger ?? NullLogger.Instance; } - public Advisory Merge(IEnumerable advisories) + public PrecedenceMergeResult Merge(IEnumerable advisories) { if (advisories is null) { @@ -193,11 +193,12 @@ public sealed class AdvisoryPrecedenceMerger var exploitKnown = ordered.Any(entry => entry.Advisory.ExploitKnown); - LogOverrides(advisoryKey, ordered); - LogPackageOverrides(advisoryKey, packageResult.Overrides); - RecordFieldConflicts(advisoryKey, ordered); - - return new Advisory( + LogOverrides(advisoryKey, ordered); + LogPackageOverrides(advisoryKey, packageResult.Overrides); + var conflicts = new List(); + RecordFieldConflicts(advisoryKey, ordered, conflicts); + + var merged = new Advisory( advisoryKey, title, summary, @@ -212,6 +213,8 @@ public sealed class AdvisoryPrecedenceMerger affectedPackages, cvssMetrics, provenance); + + return new PrecedenceMergeResult(merged, conflicts); } private static void RecordNormalizedRuleMetrics(IReadOnlyList packages) @@ -379,7 +382,7 @@ public sealed class AdvisoryPrecedenceMerger } } - private void RecordFieldConflicts(string advisoryKey, IReadOnlyList ordered) + private void RecordFieldConflicts(string advisoryKey, IReadOnlyList ordered, List conflicts) { if (ordered.Count <= 1) { @@ -396,42 +399,45 @@ public sealed class AdvisoryPrecedenceMerger if (!string.IsNullOrEmpty(candidateSeverity)) { - var reason = string.IsNullOrEmpty(primarySeverity) ? "primary_missing" : "mismatch"; - if (string.IsNullOrEmpty(primarySeverity) || !string.Equals(primarySeverity, candidateSeverity, StringComparison.OrdinalIgnoreCase)) - { - RecordConflict( - advisoryKey, - "severity", - reason, - primary, - candidate, - primarySeverity ?? "(none)", - candidateSeverity); - } - } - - if (candidate.Rank == primary.Rank) - { - RecordConflict( - advisoryKey, - "precedence_tie", - "equal_rank", - primary, - candidate, - primary.Rank.ToString(CultureInfo.InvariantCulture), - candidate.Rank.ToString(CultureInfo.InvariantCulture)); - } - } - } - - private void RecordConflict( - string advisoryKey, - string conflictType, - string reason, - AdvisoryEntry primary, - AdvisoryEntry suppressed, - string? primaryValue, - string? suppressedValue) + var reason = string.IsNullOrEmpty(primarySeverity) ? "primary_missing" : "mismatch"; + if (string.IsNullOrEmpty(primarySeverity) || !string.Equals(primarySeverity, candidateSeverity, StringComparison.OrdinalIgnoreCase)) + { + RecordConflict( + advisoryKey, + "severity", + reason, + primary, + candidate, + primarySeverity ?? "(none)", + candidateSeverity, + conflicts); + } + } + + if (candidate.Rank == primary.Rank) + { + RecordConflict( + advisoryKey, + "precedence_tie", + "equal_rank", + primary, + candidate, + primary.Rank.ToString(CultureInfo.InvariantCulture), + candidate.Rank.ToString(CultureInfo.InvariantCulture), + conflicts); + } + } + } + + private void RecordConflict( + string advisoryKey, + string conflictType, + string reason, + AdvisoryEntry primary, + AdvisoryEntry suppressed, + string? primaryValue, + string? suppressedValue, + List conflicts) { var tags = new KeyValuePair[] { @@ -445,18 +451,30 @@ public sealed class AdvisoryPrecedenceMerger ConflictCounter.Add(1, tags); - var audit = new MergeFieldConflictAudit( - advisoryKey, - conflictType, - reason, - primary.Sources, - primary.Rank, - suppressed.Sources, - suppressed.Rank, - primaryValue, - suppressedValue); - - ConflictLogged(_logger, audit, null); + var audit = new MergeFieldConflictAudit( + advisoryKey, + conflictType, + reason, + primary.Sources, + primary.Rank, + suppressed.Sources, + suppressed.Rank, + primaryValue, + suppressedValue); + + ConflictLogged(_logger, audit, null); + + conflicts.Add(new MergeConflictDetail( + primary.Advisory, + suppressed.Advisory, + conflictType, + reason, + primary.Sources.ToArray(), + primary.Rank, + suppressed.Sources.ToArray(), + suppressed.Rank, + primaryValue, + suppressedValue)); } private readonly record struct AdvisoryEntry(Advisory Advisory, int Rank) diff --git a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs b/src/StellaOps.Concelier.Merge/Services/AffectedPackagePrecedenceResolver.cs similarity index 95% rename from src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs rename to src/StellaOps.Concelier.Merge/Services/AffectedPackagePrecedenceResolver.cs index ff4bf6f5..8f67c173 100644 --- a/src/StellaOps.Feedser.Merge/Services/AffectedPackagePrecedenceResolver.cs +++ b/src/StellaOps.Concelier.Merge/Services/AffectedPackagePrecedenceResolver.cs @@ -2,10 +2,10 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; -using StellaOps.Feedser.Merge.Options; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Merge.Options; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; /// /// Applies source precedence rules to affected package sets so authoritative distro ranges override generic registry data. diff --git a/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs b/src/StellaOps.Concelier.Merge/Services/AliasGraphResolver.cs similarity index 95% rename from src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs rename to src/StellaOps.Concelier.Merge/Services/AliasGraphResolver.cs index 5ff393f2..43125973 100644 --- a/src/StellaOps.Feedser.Merge/Services/AliasGraphResolver.cs +++ b/src/StellaOps.Concelier.Merge/Services/AliasGraphResolver.cs @@ -3,9 +3,9 @@ using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo.Aliases; -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; public sealed class AliasGraphResolver { diff --git a/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs b/src/StellaOps.Concelier.Merge/Services/CanonicalHashCalculator.cs similarity index 85% rename from src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs rename to src/StellaOps.Concelier.Merge/Services/CanonicalHashCalculator.cs index 7d6e9c4e..7235103c 100644 --- a/src/StellaOps.Feedser.Merge/Services/CanonicalHashCalculator.cs +++ b/src/StellaOps.Concelier.Merge/Services/CanonicalHashCalculator.cs @@ -1,8 +1,8 @@ -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; using System.Security.Cryptography; using System.Text; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; /// /// Computes deterministic hashes over canonical advisory JSON payloads. diff --git a/src/StellaOps.Concelier.Merge/Services/MergeConflictDetail.cs b/src/StellaOps.Concelier.Merge/Services/MergeConflictDetail.cs new file mode 100644 index 00000000..afc43494 --- /dev/null +++ b/src/StellaOps.Concelier.Merge/Services/MergeConflictDetail.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Merge.Services; + +public sealed record MergeConflictDetail( + Advisory Primary, + Advisory Suppressed, + string ConflictType, + string Reason, + IReadOnlyList PrimarySources, + int PrimaryRank, + IReadOnlyList SuppressedSources, + int SuppressedRank, + string? PrimaryValue, + string? SuppressedValue); diff --git a/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs b/src/StellaOps.Concelier.Merge/Services/MergeEventWriter.cs similarity index 93% rename from src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs rename to src/StellaOps.Concelier.Merge/Services/MergeEventWriter.cs index 6945e545..1d2e3c02 100644 --- a/src/StellaOps.Feedser.Merge/Services/MergeEventWriter.cs +++ b/src/StellaOps.Concelier.Merge/Services/MergeEventWriter.cs @@ -1,10 +1,10 @@ -namespace StellaOps.Feedser.Merge.Services; +namespace StellaOps.Concelier.Merge.Services; using System.Security.Cryptography; using System.Linq; using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; /// /// Persists merge events with canonical before/after hashes for auditability. diff --git a/src/StellaOps.Concelier.Merge/Services/PrecedenceMergeResult.cs b/src/StellaOps.Concelier.Merge/Services/PrecedenceMergeResult.cs new file mode 100644 index 00000000..88312df7 --- /dev/null +++ b/src/StellaOps.Concelier.Merge/Services/PrecedenceMergeResult.cs @@ -0,0 +1,8 @@ +using System.Collections.Generic; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Merge.Services; + +public sealed record PrecedenceMergeResult( + Advisory Advisory, + IReadOnlyList Conflicts); diff --git a/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj b/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj new file mode 100644 index 00000000..94468bc2 --- /dev/null +++ b/src/StellaOps.Concelier.Merge/StellaOps.Concelier.Merge.csproj @@ -0,0 +1,18 @@ + + + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Merge/TASKS.md b/src/StellaOps.Concelier.Merge/TASKS.md similarity index 79% rename from src/StellaOps.Feedser.Merge/TASKS.md rename to src/StellaOps.Concelier.Merge/TASKS.md index bf3b3190..1a43620c 100644 --- a/src/StellaOps.Feedser.Merge/TASKS.md +++ b/src/StellaOps.Concelier.Merge/TASKS.md @@ -1,12 +1,12 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| -|Precedence policy engine|BE-Merge|Architecture|**DONE** – precedence defaults enforced by `AdvisoryPrecedenceMerger`/`AdvisoryPrecedenceDefaults` with distro/PSIRT overriding registry feeds and CERT/KEV enrichers.| -|NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| -|Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| -|SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| -|Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|Identity graph and alias resolver|BE-Merge|Models, Storage.Mongo|DONE – `AdvisoryIdentityResolver` builds alias-driven clusters with canonical key selection + unit coverage.| +|Precedence policy engine|BE-Merge|Architecture|**DONE** – precedence defaults enforced by `AdvisoryPrecedenceMerger`/`AdvisoryPrecedenceDefaults` with distro/PSIRT overriding registry feeds and CERT/KEV enrichers.| +|NEVRA comparer plus tests|BE-Merge (Distro WG)|Source.Distro fixtures|DONE – Added Nevra parser/comparer with tilde-aware rpm ordering and unit coverage.| +|Debian EVR comparer plus tests|BE-Merge (Distro WG)|Debian fixtures|DONE – DebianEvr comparer mirrors dpkg ordering with tilde/epoch handling and unit coverage.| +|SemVer range resolver plus tests|BE-Merge (OSS WG)|OSV/GHSA fixtures|DONE – SemanticVersionRangeResolver covers introduced/fixed/lastAffected semantics with SemVer ordering tests.| +|Canonical hash and merge_event writer|BE-Merge|Models, Storage.Mongo|DONE – Hash calculator + MergeEventWriter compute canonical SHA-256 digests and persist merge events.| |Conflict detection and metrics|BE-Merge|Core|**DONE** – merge meters emit override/conflict counters and structured audits (`AdvisoryPrecedenceMerger`).| |FEEDMERGE-ENGINE-04-001 GHSA/NVD/OSV conflict rules|BE-Merge|Core, Storage.Mongo|DONE – `AdvisoryMergeService` applies `CanonicalMerger` output before precedence merge, replacing source advisories with the canonical transcript. **Coordination:** connector fixture owners should surface canonical deltas to Merge QA before regression sign-off.| |FEEDMERGE-ENGINE-04-002 Override metrics instrumentation|BE-Merge|Observability|DONE – merge events persist `MergeFieldDecision` records enabling analytics on precedence/freshness decisions. **Next:** hand off metrics schema to Ops for dashboard wiring.| @@ -14,8 +14,8 @@ |End-to-end determinism test|QA|Merge, key connectors|**DONE** – `MergePrecedenceIntegrationTests.MergePipeline_IsDeterministicAcrossRuns` guards determinism.| |FEEDMERGE-QA-04-001 End-to-end conflict regression suite|QA|Merge|DONE – `AdvisoryMergeServiceTests.MergeAsync_AppliesCanonicalRulesAndPersistsDecisions` exercises GHSA/NVD/OSV conflict path and merge-event analytics. **Reminder:** QA to sync with connector teams once new fixture triples land.| |Override audit logging|BE-Merge|Observability|DONE – override audits now emit structured logs plus bounded-tag metrics suitable for prod telemetry.| -|Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via feedser:merge:precedence:ranks with docs/tests covering operator workflow.| -|Range primitives backlog|BE-Merge|Connector WGs|**DOING** – Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.
          2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).
          2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.
          2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.
          2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.
          2025-10-11 21:55Z: Merge now emits `feedser.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.
          2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.| +|Configurable precedence table|BE-Merge|Architecture|DONE – precedence options bind via concelier:merge:precedence:ranks with docs/tests covering operator workflow.| +|Range primitives backlog|BE-Merge|Connector WGs|**DOING** – Coordinate remaining connectors (`Acsc`, `Cccs`, `CertBund`, `CertCc`, `Cve`, `Ghsa`, `Ics.Cisa`, `Kisa`, `Ru.Bdu`, `Ru.Nkcki`, `Vndr.Apple`, `Vndr.Cisco`, `Vndr.Msrc`) to emit canonical RangePrimitives with provenance tags; track progress/fixtures here.
          2025-10-11: Storage alignment notes + sample normalized rule JSON now captured in `RANGE_PRIMITIVES_COORDINATION.md` (see “Storage alignment quick reference”).
          2025-10-11 18:45Z: GHSA normalized rules landed; OSV connector picked up next for rollout.
          2025-10-11 21:10Z: `docs/dev/merge_semver_playbook.md` Section 8 now documents the persisted Mongo projection (SemVer + NEVRA) for connector reviewers.
          2025-10-11 21:30Z: Added `docs/dev/normalized_versions_rollout.md` dashboard to centralize connector status and upcoming milestones.
          2025-10-11 21:55Z: Merge now emits `concelier.merge.normalized_rules*` counters and unions connector-provided normalized arrays; see new test coverage in `AdvisoryPrecedenceMergerTests.Merge_RecordsNormalizedRuleMetrics`.
          2025-10-12 17:05Z: CVE + KEV normalized rule verification complete; OSV parity fixtures revalidated—downstream parity/monitoring tasks may proceed.
          2025-10-19 14:35Z: Prerequisites reviewed (none outstanding); FEEDMERGE-COORD-02-900 remains in DOING with connector follow-ups unchanged.
          2025-10-19 15:25Z: Refreshed `RANGE_PRIMITIVES_COORDINATION.md` matrix + added targeted follow-ups (Cccs, CertBund, ICS-CISA, Kisa, Vndr.Cisco) with delivery dates 2025-10-21 → 2025-10-25; monitoring merge counters for regression.| |Merge pipeline parity for new advisory fields|BE-Merge|Models, Core|DONE (2025-10-15) – merge service now surfaces description/CWE/canonical metric decisions with updated metrics/tests.| |Connector coordination for new advisory fields|Connector Leads, BE-Merge|Models, Core|**DONE (2025-10-15)** – GHSA, NVD, and OSV connectors now emit advisory descriptions, CWE weaknesses, and canonical metric ids. Fixtures refreshed (GHSA connector regression suite, `conflict-nvd.canonical.json`, OSV parity snapshots) and completion recorded in coordination log.| -|FEEDMERGE-ENGINE-07-001 Conflict sets & explainers|BE-Merge|FEEDSTORAGE-DATA-07-001|TODO – Persist conflict sets referencing advisory statements, output rule/explainer payloads with replay hashes, and add integration tests covering deterministic `asOf` evaluations.| +|FEEDMERGE-ENGINE-07-001 Conflict sets & explainers|BE-Merge|FEEDSTORAGE-DATA-07-001|**DOING (2025-10-19)** – Merge now captures canonical advisory statements + prepares conflict payload scaffolding (statement hashes, deterministic JSON, tests). Next: surface conflict explainers and replay APIs for Core/WebService before marking DONE.| diff --git a/src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs b/src/StellaOps.Concelier.Models.Tests/AdvisoryProvenanceTests.cs similarity index 92% rename from src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs rename to src/StellaOps.Concelier.Models.Tests/AdvisoryProvenanceTests.cs index b2f4e70a..10386746 100644 --- a/src/StellaOps.Feedser.Models.Tests/AdvisoryProvenanceTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/AdvisoryProvenanceTests.cs @@ -1,49 +1,49 @@ -using System; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class AdvisoryProvenanceTests -{ - [Fact] - public void FieldMask_NormalizesAndDeduplicates() - { - var timestamp = DateTimeOffset.Parse("2025-01-01T00:00:00Z"); - var provenance = new AdvisoryProvenance( - source: "nvd", - kind: "map", - value: "CVE-2025-0001", - recordedAt: timestamp, - fieldMask: new[] { " AffectedPackages[] ", "affectedpackages[]", "references[]" }); - - Assert.Equal(timestamp, provenance.RecordedAt); - Assert.Collection( - provenance.FieldMask, - mask => Assert.Equal("affectedpackages[]", mask), - mask => Assert.Equal("references[]", mask)); - Assert.Null(provenance.DecisionReason); - } - - [Fact] - public void EmptyProvenance_ExposesEmptyFieldMask() - { - Assert.True(AdvisoryProvenance.Empty.FieldMask.IsEmpty); - Assert.Null(AdvisoryProvenance.Empty.DecisionReason); - } - - [Fact] - public void DecisionReason_IsTrimmed() - { - var timestamp = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); - var provenance = new AdvisoryProvenance( - source: "merge", - kind: "precedence", - value: "summary", - recordedAt: timestamp, - fieldMask: new[] { ProvenanceFieldMasks.Advisory }, - decisionReason: " freshness_override "); - - Assert.Equal("freshness_override", provenance.DecisionReason); - } -} +using System; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class AdvisoryProvenanceTests +{ + [Fact] + public void FieldMask_NormalizesAndDeduplicates() + { + var timestamp = DateTimeOffset.Parse("2025-01-01T00:00:00Z"); + var provenance = new AdvisoryProvenance( + source: "nvd", + kind: "map", + value: "CVE-2025-0001", + recordedAt: timestamp, + fieldMask: new[] { " AffectedPackages[] ", "affectedpackages[]", "references[]" }); + + Assert.Equal(timestamp, provenance.RecordedAt); + Assert.Collection( + provenance.FieldMask, + mask => Assert.Equal("affectedpackages[]", mask), + mask => Assert.Equal("references[]", mask)); + Assert.Null(provenance.DecisionReason); + } + + [Fact] + public void EmptyProvenance_ExposesEmptyFieldMask() + { + Assert.True(AdvisoryProvenance.Empty.FieldMask.IsEmpty); + Assert.Null(AdvisoryProvenance.Empty.DecisionReason); + } + + [Fact] + public void DecisionReason_IsTrimmed() + { + var timestamp = DateTimeOffset.Parse("2025-03-01T00:00:00Z"); + var provenance = new AdvisoryProvenance( + source: "merge", + kind: "precedence", + value: "summary", + recordedAt: timestamp, + fieldMask: new[] { ProvenanceFieldMasks.Advisory }, + decisionReason: " freshness_override "); + + Assert.Equal("freshness_override", provenance.DecisionReason); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs b/src/StellaOps.Concelier.Models.Tests/AdvisoryTests.cs similarity index 95% rename from src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs rename to src/StellaOps.Concelier.Models.Tests/AdvisoryTests.cs index e8997e0b..fdd9f355 100644 --- a/src/StellaOps.Feedser.Models.Tests/AdvisoryTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/AdvisoryTests.cs @@ -1,62 +1,62 @@ -using System.Linq; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class AdvisoryTests -{ - [Fact] - public void CanonicalizesAliasesAndReferences() - { - var advisory = new Advisory( - advisoryKey: "TEST-123", - title: "Sample Advisory", - summary: " summary with spaces ", - language: "EN", - published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), - modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), - severity: "CRITICAL", - exploitKnown: true, - aliases: new[] { " CVE-2024-0001", "GHSA-aaaa", "cve-2024-0001" }, - references: new[] - { - new AdvisoryReference("https://example.com/b", "patch", null, null, AdvisoryProvenance.Empty), - new AdvisoryReference("https://example.com/a", null, null, null, AdvisoryProvenance.Empty), - }, - affectedPackages: new[] - { - new AffectedPackage( - type: AffectedPackageTypes.SemVer, - identifier: "pkg:npm/sample", - platform: "node", - versionRanges: new[] - { - new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), - new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), - new AffectedVersionRange("semver", "0.9.0", null, "0.9.9", null, AdvisoryProvenance.Empty), - }, - statuses: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), - new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), - }) - }, - cvssMetrics: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), - new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), - }); - - Assert.Equal(new[] { "CVE-2024-0001", "GHSA-aaaa" }, advisory.Aliases); - Assert.Equal(new[] { "https://example.com/a", "https://example.com/b" }, advisory.References.Select(r => r.Url)); - Assert.Equal( - new[] - { - "semver|0.9.0||0.9.9|", - "semver|1.0.0|1.0.1||", - }, - advisory.AffectedPackages.Single().VersionRanges.Select(r => r.CreateDeterministicKey())); - } -} +using System.Linq; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class AdvisoryTests +{ + [Fact] + public void CanonicalizesAliasesAndReferences() + { + var advisory = new Advisory( + advisoryKey: "TEST-123", + title: "Sample Advisory", + summary: " summary with spaces ", + language: "EN", + published: DateTimeOffset.Parse("2024-01-01T00:00:00Z"), + modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z"), + severity: "CRITICAL", + exploitKnown: true, + aliases: new[] { " CVE-2024-0001", "GHSA-aaaa", "cve-2024-0001" }, + references: new[] + { + new AdvisoryReference("https://example.com/b", "patch", null, null, AdvisoryProvenance.Empty), + new AdvisoryReference("https://example.com/a", null, null, null, AdvisoryProvenance.Empty), + }, + affectedPackages: new[] + { + new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/sample", + platform: "node", + versionRanges: new[] + { + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "1.0.0", "1.0.1", null, null, AdvisoryProvenance.Empty), + new AffectedVersionRange("semver", "0.9.0", null, "0.9.9", null, AdvisoryProvenance.Empty), + }, + statuses: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }) + }, + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("nvd", "map", "", DateTimeOffset.Parse("2024-01-01T00:00:00Z")), + new AdvisoryProvenance("vendor", "map", "", DateTimeOffset.Parse("2024-01-02T00:00:00Z")), + }); + + Assert.Equal(new[] { "CVE-2024-0001", "GHSA-aaaa" }, advisory.Aliases); + Assert.Equal(new[] { "https://example.com/a", "https://example.com/b" }, advisory.References.Select(r => r.Url)); + Assert.Equal( + new[] + { + "semver|0.9.0||0.9.9|", + "semver|1.0.0|1.0.1||", + }, + advisory.AffectedPackages.Single().VersionRanges.Select(r => r.CreateDeterministicKey())); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs b/src/StellaOps.Concelier.Models.Tests/AffectedPackageStatusTests.cs similarity index 97% rename from src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs rename to src/StellaOps.Concelier.Models.Tests/AffectedPackageStatusTests.cs index 6bac3d85..b56b6226 100644 --- a/src/StellaOps.Feedser.Models.Tests/AffectedPackageStatusTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/AffectedPackageStatusTests.cs @@ -1,7 +1,7 @@ using System; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class AffectedPackageStatusTests { diff --git a/src/StellaOps.Feedser.Models.Tests/AffectedVersionRangeExtensionsTests.cs b/src/StellaOps.Concelier.Models.Tests/AffectedVersionRangeExtensionsTests.cs similarity index 94% rename from src/StellaOps.Feedser.Models.Tests/AffectedVersionRangeExtensionsTests.cs rename to src/StellaOps.Concelier.Models.Tests/AffectedVersionRangeExtensionsTests.cs index 534bbb0a..de5e3517 100644 --- a/src/StellaOps.Feedser.Models.Tests/AffectedVersionRangeExtensionsTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/AffectedVersionRangeExtensionsTests.cs @@ -1,74 +1,74 @@ -using System; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class AffectedVersionRangeExtensionsTests -{ - [Fact] - public void ToNormalizedVersionRule_UsesNevraPrimitivesWhenAvailable() - { - var range = new AffectedVersionRange( - rangeKind: "nevra", - introducedVersion: "raw-introduced", - fixedVersion: "raw-fixed", - lastAffectedVersion: null, - rangeExpression: null, - provenance: AdvisoryProvenance.Empty, - primitives: new RangePrimitives( - SemVer: null, - Nevra: new NevraPrimitive( - new NevraComponent("pkg", 0, "1.0.0", "1", "x86_64"), - new NevraComponent("pkg", 0, "1.2.0", "2", "x86_64"), - null), - Evr: null, - VendorExtensions: null)); - - var rule = range.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); - Assert.Equal("pkg-1.0.0-1.x86_64", rule.Min); - Assert.Equal("pkg-1.2.0-2.x86_64", rule.Max); - } - - [Fact] - public void ToNormalizedVersionRule_FallsBackForEvrWhenPrimitivesMissing() - { - var range = new AffectedVersionRange( - rangeKind: "EVR", - introducedVersion: "1:1.0.0-1", - fixedVersion: "1:1.2.0-1ubuntu2", - lastAffectedVersion: null, - rangeExpression: null, - provenance: new AdvisoryProvenance("debian", "range", "pkg", DateTimeOffset.UtcNow), - primitives: null); - - var rule = range.ToNormalizedVersionRule("fallback"); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.Evr, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); - Assert.Equal("1:1.0.0-1", rule.Min); - Assert.Equal("1:1.2.0-1ubuntu2", rule.Max); - Assert.Equal("fallback", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_ReturnsNullForUnknownKind() - { - var range = new AffectedVersionRange( - rangeKind: "vendor", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: null, - provenance: AdvisoryProvenance.Empty, - primitives: null); - - var rule = range.ToNormalizedVersionRule(); - - Assert.Null(rule); - } -} +using System; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class AffectedVersionRangeExtensionsTests +{ + [Fact] + public void ToNormalizedVersionRule_UsesNevraPrimitivesWhenAvailable() + { + var range = new AffectedVersionRange( + rangeKind: "nevra", + introducedVersion: "raw-introduced", + fixedVersion: "raw-fixed", + lastAffectedVersion: null, + rangeExpression: null, + provenance: AdvisoryProvenance.Empty, + primitives: new RangePrimitives( + SemVer: null, + Nevra: new NevraPrimitive( + new NevraComponent("pkg", 0, "1.0.0", "1", "x86_64"), + new NevraComponent("pkg", 0, "1.2.0", "2", "x86_64"), + null), + Evr: null, + VendorExtensions: null)); + + var rule = range.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); + Assert.Equal("pkg-1.0.0-1.x86_64", rule.Min); + Assert.Equal("pkg-1.2.0-2.x86_64", rule.Max); + } + + [Fact] + public void ToNormalizedVersionRule_FallsBackForEvrWhenPrimitivesMissing() + { + var range = new AffectedVersionRange( + rangeKind: "EVR", + introducedVersion: "1:1.0.0-1", + fixedVersion: "1:1.2.0-1ubuntu2", + lastAffectedVersion: null, + rangeExpression: null, + provenance: new AdvisoryProvenance("debian", "range", "pkg", DateTimeOffset.UtcNow), + primitives: null); + + var rule = range.ToNormalizedVersionRule("fallback"); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.Evr, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("1:1.0.0-1", rule.Min); + Assert.Equal("1:1.2.0-1ubuntu2", rule.Max); + Assert.Equal("fallback", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_ReturnsNullForUnknownKind() + { + var range = new AffectedVersionRange( + rangeKind: "vendor", + introducedVersion: null, + fixedVersion: null, + lastAffectedVersion: null, + rangeExpression: null, + provenance: AdvisoryProvenance.Empty, + primitives: null); + + var rule = range.ToNormalizedVersionRule(); + + Assert.Null(rule); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs b/src/StellaOps.Concelier.Models.Tests/AliasSchemeRegistryTests.cs similarity index 93% rename from src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs rename to src/StellaOps.Concelier.Models.Tests/AliasSchemeRegistryTests.cs index d197b70c..30cf3cd7 100644 --- a/src/StellaOps.Feedser.Models.Tests/AliasSchemeRegistryTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/AliasSchemeRegistryTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class AliasSchemeRegistryTests { diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs b/src/StellaOps.Concelier.Models.Tests/CanonicalExampleFactory.cs similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs rename to src/StellaOps.Concelier.Models.Tests/CanonicalExampleFactory.cs index e308635a..82da571a 100644 --- a/src/StellaOps.Feedser.Models.Tests/CanonicalExampleFactory.cs +++ b/src/StellaOps.Concelier.Models.Tests/CanonicalExampleFactory.cs @@ -1,8 +1,8 @@ using System.Collections.Generic; using System.Globalization; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; internal static class CanonicalExampleFactory { diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs b/src/StellaOps.Concelier.Models.Tests/CanonicalExamplesTests.cs similarity index 93% rename from src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs rename to src/StellaOps.Concelier.Models.Tests/CanonicalExamplesTests.cs index 0b2864da..e70e64bc 100644 --- a/src/StellaOps.Feedser.Models.Tests/CanonicalExamplesTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/CanonicalExamplesTests.cs @@ -1,7 +1,7 @@ using System.Text; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class CanonicalExamplesTests { diff --git a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs b/src/StellaOps.Concelier.Models.Tests/CanonicalJsonSerializerTests.cs similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs rename to src/StellaOps.Concelier.Models.Tests/CanonicalJsonSerializerTests.cs index 7fb361e2..cd64f279 100644 --- a/src/StellaOps.Feedser.Models.Tests/CanonicalJsonSerializerTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/CanonicalJsonSerializerTests.cs @@ -2,9 +2,9 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class CanonicalJsonSerializerTests { diff --git a/src/StellaOps.Feedser.Models.Tests/EvrPrimitiveExtensionsTests.cs b/src/StellaOps.Concelier.Models.Tests/EvrPrimitiveExtensionsTests.cs similarity index 91% rename from src/StellaOps.Feedser.Models.Tests/EvrPrimitiveExtensionsTests.cs rename to src/StellaOps.Concelier.Models.Tests/EvrPrimitiveExtensionsTests.cs index 2c30aac2..1bf64855 100644 --- a/src/StellaOps.Feedser.Models.Tests/EvrPrimitiveExtensionsTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/EvrPrimitiveExtensionsTests.cs @@ -1,43 +1,43 @@ -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class EvrPrimitiveExtensionsTests -{ - [Fact] - public void ToNormalizedVersionRule_ProducesRangeForIntroducedAndFixed() - { - var primitive = new EvrPrimitive( - Introduced: new EvrComponent(1, "1.2.3", "4"), - Fixed: new EvrComponent(1, "1.2.9", "0ubuntu1"), - LastAffected: null); - - var rule = primitive.ToNormalizedVersionRule("ubuntu:focal"); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.Evr, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); - Assert.Equal("1:1.2.3-4", rule.Min); - Assert.True(rule.MinInclusive); - Assert.Equal("1:1.2.9-0ubuntu1", rule.Max); - Assert.False(rule.MaxInclusive); - Assert.Equal("ubuntu:focal", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_GreaterThanOrEqualWhenOnlyIntroduced() - { - var primitive = new EvrPrimitive( - Introduced: new EvrComponent(0, "2.0.0", null), - Fixed: null, - LastAffected: null); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionRuleTypes.GreaterThanOrEqual, rule!.Type); - Assert.Equal("2.0.0", rule.Min); - Assert.True(rule.MinInclusive); - } -} +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class EvrPrimitiveExtensionsTests +{ + [Fact] + public void ToNormalizedVersionRule_ProducesRangeForIntroducedAndFixed() + { + var primitive = new EvrPrimitive( + Introduced: new EvrComponent(1, "1.2.3", "4"), + Fixed: new EvrComponent(1, "1.2.9", "0ubuntu1"), + LastAffected: null); + + var rule = primitive.ToNormalizedVersionRule("ubuntu:focal"); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.Evr, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("1:1.2.3-4", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("1:1.2.9-0ubuntu1", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Equal("ubuntu:focal", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_GreaterThanOrEqualWhenOnlyIntroduced() + { + var primitive = new EvrPrimitive( + Introduced: new EvrComponent(0, "2.0.0", null), + Fixed: null, + LastAffected: null); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionRuleTypes.GreaterThanOrEqual, rule!.Type); + Assert.Equal("2.0.0", rule.Min); + Assert.True(rule.MinInclusive); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json b/src/StellaOps.Concelier.Models.Tests/Fixtures/ghsa-semver.json similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json rename to src/StellaOps.Concelier.Models.Tests/Fixtures/ghsa-semver.json index 54c5c32a..b344948a 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/ghsa-semver.json +++ b/src/StellaOps.Concelier.Models.Tests/Fixtures/ghsa-semver.json @@ -1,124 +1,124 @@ -{ - "advisoryKey": "GHSA-aaaa-bbbb-cccc", - "affectedPackages": [ - { - "type": "semver", - "identifier": "pkg:npm/example-widget", - "platform": null, - "versionRanges": [ - { - "fixedVersion": "2.5.1", - "introducedVersion": null, - "lastAffectedVersion": null, - "primitives": null, - "provenance": { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": ">=0.0.0 <2.5.1", - "rangeKind": "semver" - }, - { - "fixedVersion": "3.2.4", - "introducedVersion": "3.0.0", - "lastAffectedVersion": null, - "primitives": null, - "provenance": { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": null, - "rangeKind": "semver" - } - ], - "normalizedVersions": [], - "statuses": [], - "provenance": [ - { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "aliases": [ - "CVE-2024-2222", - "GHSA-aaaa-bbbb-cccc" - ], - "credits": [], - "cvssMetrics": [ - { - "baseScore": 8.8, - "baseSeverity": "high", - "provenance": { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2024-03-04T12:00:00+00:00", - "provenance": [ - { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - } - ], - "published": "2024-03-04T00:00:00+00:00", - "references": [ - { - "kind": "patch", - "provenance": { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "ghsa", - "summary": "Patch commit", - "url": "https://github.com/example/widget/commit/abcd1234" - }, - { - "kind": "advisory", - "provenance": { - "source": "ghsa", - "kind": "map", - "value": "ghsa-aaaa-bbbb-cccc", - "decisionReason": null, - "recordedAt": "2024-03-05T10:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "ghsa", - "summary": "GitHub Security Advisory", - "url": "https://github.com/example/widget/security/advisories/GHSA-aaaa-bbbb-cccc" - } - ], - "severity": "high", - "summary": "A crafted payload can pollute Object.prototype leading to RCE.", - "title": "Prototype pollution in widget.js" +{ + "advisoryKey": "GHSA-aaaa-bbbb-cccc", + "affectedPackages": [ + { + "type": "semver", + "identifier": "pkg:npm/example-widget", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "2.5.1", + "introducedVersion": null, + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": ">=0.0.0 <2.5.1", + "rangeKind": "semver" + }, + { + "fixedVersion": "3.2.4", + "introducedVersion": "3.0.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "semver" + } + ], + "normalizedVersions": [], + "statuses": [], + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-2222", + "GHSA-aaaa-bbbb-cccc" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 8.8, + "baseSeverity": "high", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-03-04T12:00:00+00:00", + "provenance": [ + { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-03-04T00:00:00+00:00", + "references": [ + { + "kind": "patch", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "ghsa", + "summary": "Patch commit", + "url": "https://github.com/example/widget/commit/abcd1234" + }, + { + "kind": "advisory", + "provenance": { + "source": "ghsa", + "kind": "map", + "value": "ghsa-aaaa-bbbb-cccc", + "decisionReason": null, + "recordedAt": "2024-03-05T10:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "ghsa", + "summary": "GitHub Security Advisory", + "url": "https://github.com/example/widget/security/advisories/GHSA-aaaa-bbbb-cccc" + } + ], + "severity": "high", + "summary": "A crafted payload can pollute Object.prototype leading to RCE.", + "title": "Prototype pollution in widget.js" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json b/src/StellaOps.Concelier.Models.Tests/Fixtures/kev-flag.json similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json rename to src/StellaOps.Concelier.Models.Tests/Fixtures/kev-flag.json index f0636285..8c0c1964 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/kev-flag.json +++ b/src/StellaOps.Concelier.Models.Tests/Fixtures/kev-flag.json @@ -1,42 +1,42 @@ -{ - "advisoryKey": "CVE-2023-9999", - "affectedPackages": [], - "aliases": [ - "CVE-2023-9999" - ], - "credits": [], - "cvssMetrics": [], - "exploitKnown": true, - "language": "en", - "modified": "2024-02-09T16:22:00+00:00", - "provenance": [ - { - "source": "cisa-kev", - "kind": "annotate", - "value": "kev", - "decisionReason": null, - "recordedAt": "2024-02-10T09:30:00+00:00", - "fieldMask": [] - } - ], - "published": "2023-11-20T00:00:00+00:00", - "references": [ - { - "kind": "kev", - "provenance": { - "source": "cisa-kev", - "kind": "annotate", - "value": "kev", - "decisionReason": null, - "recordedAt": "2024-02-10T09:30:00+00:00", - "fieldMask": [] - }, - "sourceTag": "cisa", - "summary": "CISA KEV entry", - "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog" - } - ], - "severity": "critical", - "summary": "Unauthenticated RCE due to unsafe deserialization.", - "title": "Remote code execution in LegacyServer" +{ + "advisoryKey": "CVE-2023-9999", + "affectedPackages": [], + "aliases": [ + "CVE-2023-9999" + ], + "credits": [], + "cvssMetrics": [], + "exploitKnown": true, + "language": "en", + "modified": "2024-02-09T16:22:00+00:00", + "provenance": [ + { + "source": "cisa-kev", + "kind": "annotate", + "value": "kev", + "decisionReason": null, + "recordedAt": "2024-02-10T09:30:00+00:00", + "fieldMask": [] + } + ], + "published": "2023-11-20T00:00:00+00:00", + "references": [ + { + "kind": "kev", + "provenance": { + "source": "cisa-kev", + "kind": "annotate", + "value": "kev", + "decisionReason": null, + "recordedAt": "2024-02-10T09:30:00+00:00", + "fieldMask": [] + }, + "sourceTag": "cisa", + "summary": "CISA KEV entry", + "url": "https://www.cisa.gov/known-exploited-vulnerabilities-catalog" + } + ], + "severity": "critical", + "summary": "Unauthenticated RCE due to unsafe deserialization.", + "title": "Remote code execution in LegacyServer" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json b/src/StellaOps.Concelier.Models.Tests/Fixtures/nvd-basic.json similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json rename to src/StellaOps.Concelier.Models.Tests/Fixtures/nvd-basic.json index bc8dc524..e486abb8 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/nvd-basic.json +++ b/src/StellaOps.Concelier.Models.Tests/Fixtures/nvd-basic.json @@ -1,119 +1,119 @@ -{ - "advisoryKey": "CVE-2024-1234", - "affectedPackages": [ - { - "type": "cpe", - "identifier": "cpe:/a:examplecms:examplecms:1.0", - "platform": null, - "versionRanges": [ - { - "fixedVersion": "1.0.5", - "introducedVersion": "1.0", - "lastAffectedVersion": null, - "primitives": null, - "provenance": { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": null, - "rangeKind": "version" - } - ], - "normalizedVersions": [], - "statuses": [ - { - "provenance": { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - }, - "status": "affected" - } - ], - "provenance": [ - { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "aliases": [ - "CVE-2024-1234" - ], - "credits": [], - "cvssMetrics": [ - { - "baseScore": 9.8, - "baseSeverity": "critical", - "provenance": { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2024-07-16T10:35:00+00:00", - "provenance": [ - { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - } - ], - "published": "2024-07-15T00:00:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "example", - "kind": "fetch", - "value": "bulletin", - "decisionReason": null, - "recordedAt": "2024-07-14T15:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "vendor", - "summary": "Vendor bulletin", - "url": "https://example.org/security/CVE-2024-1234" - }, - { - "kind": "advisory", - "provenance": { - "source": "nvd", - "kind": "map", - "value": "cve-2024-1234", - "decisionReason": null, - "recordedAt": "2024-08-01T12:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "nvd", - "summary": "NVD entry", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" - } - ], - "severity": "high", - "summary": "An integer overflow in ExampleCMS allows remote attackers to escalate privileges.", - "title": "Integer overflow in ExampleCMS" +{ + "advisoryKey": "CVE-2024-1234", + "affectedPackages": [ + { + "type": "cpe", + "identifier": "cpe:/a:examplecms:examplecms:1.0", + "platform": null, + "versionRanges": [ + { + "fixedVersion": "1.0.5", + "introducedVersion": "1.0", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "version" + } + ], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "status": "affected" + } + ], + "provenance": [ + { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-1234" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 9.8, + "baseSeverity": "critical", + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-07-16T10:35:00+00:00", + "provenance": [ + { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-07-15T00:00:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "example", + "kind": "fetch", + "value": "bulletin", + "decisionReason": null, + "recordedAt": "2024-07-14T15:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "vendor", + "summary": "Vendor bulletin", + "url": "https://example.org/security/CVE-2024-1234" + }, + { + "kind": "advisory", + "provenance": { + "source": "nvd", + "kind": "map", + "value": "cve-2024-1234", + "decisionReason": null, + "recordedAt": "2024-08-01T12:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "nvd", + "summary": "NVD entry", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-1234" + } + ], + "severity": "high", + "summary": "An integer overflow in ExampleCMS allows remote attackers to escalate privileges.", + "title": "Integer overflow in ExampleCMS" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json b/src/StellaOps.Concelier.Models.Tests/Fixtures/psirt-overlay.json similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json rename to src/StellaOps.Concelier.Models.Tests/Fixtures/psirt-overlay.json index cc488094..9b13ee3b 100644 --- a/src/StellaOps.Feedser.Models.Tests/Fixtures/psirt-overlay.json +++ b/src/StellaOps.Concelier.Models.Tests/Fixtures/psirt-overlay.json @@ -1,122 +1,122 @@ -{ - "advisoryKey": "RHSA-2024:0252", - "affectedPackages": [ - { - "type": "rpm", - "identifier": "kernel-0:4.18.0-553.el8.x86_64", - "platform": "rhel-8", - "versionRanges": [ - { - "fixedVersion": null, - "introducedVersion": "0:4.18.0-553.el8", - "lastAffectedVersion": null, - "primitives": null, - "provenance": { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - }, - "rangeExpression": null, - "rangeKind": "nevra" - } - ], - "normalizedVersions": [], - "statuses": [ - { - "provenance": { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - }, - "status": "fixed" - } - ], - "provenance": [ - { - "source": "redhat", - "kind": "enrich", - "value": "cve-2024-5678", - "decisionReason": null, - "recordedAt": "2024-05-11T09:05:00+00:00", - "fieldMask": [] - }, - { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - } - ] - } - ], - "aliases": [ - "CVE-2024-5678", - "RHSA-2024:0252" - ], - "credits": [], - "cvssMetrics": [ - { - "baseScore": 6.7, - "baseSeverity": "medium", - "provenance": { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - }, - "vector": "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1" - } - ], - "exploitKnown": false, - "language": "en", - "modified": "2024-05-11T08:15:00+00:00", - "provenance": [ - { - "source": "redhat", - "kind": "enrich", - "value": "cve-2024-5678", - "decisionReason": null, - "recordedAt": "2024-05-11T09:05:00+00:00", - "fieldMask": [] - }, - { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - } - ], - "published": "2024-05-10T19:28:00+00:00", - "references": [ - { - "kind": "advisory", - "provenance": { - "source": "redhat", - "kind": "map", - "value": "rhsa-2024:0252", - "decisionReason": null, - "recordedAt": "2024-05-11T09:00:00+00:00", - "fieldMask": [] - }, - "sourceTag": "redhat", - "summary": "Red Hat security advisory", - "url": "https://access.redhat.com/errata/RHSA-2024:0252" - } - ], - "severity": "critical", - "summary": "Updates the Red Hat Enterprise Linux kernel to address CVE-2024-5678.", - "title": "Important: kernel security update" +{ + "advisoryKey": "RHSA-2024:0252", + "affectedPackages": [ + { + "type": "rpm", + "identifier": "kernel-0:4.18.0-553.el8.x86_64", + "platform": "rhel-8", + "versionRanges": [ + { + "fixedVersion": null, + "introducedVersion": "0:4.18.0-553.el8", + "lastAffectedVersion": null, + "primitives": null, + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "rangeExpression": null, + "rangeKind": "nevra" + } + ], + "normalizedVersions": [], + "statuses": [ + { + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "status": "fixed" + } + ], + "provenance": [ + { + "source": "redhat", + "kind": "enrich", + "value": "cve-2024-5678", + "decisionReason": null, + "recordedAt": "2024-05-11T09:05:00+00:00", + "fieldMask": [] + }, + { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + } + ] + } + ], + "aliases": [ + "CVE-2024-5678", + "RHSA-2024:0252" + ], + "credits": [], + "cvssMetrics": [ + { + "baseScore": 6.7, + "baseSeverity": "medium", + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "vector": "CVSS:3.1/AV:L/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + } + ], + "exploitKnown": false, + "language": "en", + "modified": "2024-05-11T08:15:00+00:00", + "provenance": [ + { + "source": "redhat", + "kind": "enrich", + "value": "cve-2024-5678", + "decisionReason": null, + "recordedAt": "2024-05-11T09:05:00+00:00", + "fieldMask": [] + }, + { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + } + ], + "published": "2024-05-10T19:28:00+00:00", + "references": [ + { + "kind": "advisory", + "provenance": { + "source": "redhat", + "kind": "map", + "value": "rhsa-2024:0252", + "decisionReason": null, + "recordedAt": "2024-05-11T09:00:00+00:00", + "fieldMask": [] + }, + "sourceTag": "redhat", + "summary": "Red Hat security advisory", + "url": "https://access.redhat.com/errata/RHSA-2024:0252" + } + ], + "severity": "critical", + "summary": "Updates the Red Hat Enterprise Linux kernel to address CVE-2024-5678.", + "title": "Important: kernel security update" } \ No newline at end of file diff --git a/src/StellaOps.Feedser.Models.Tests/NevraPrimitiveExtensionsTests.cs b/src/StellaOps.Concelier.Models.Tests/NevraPrimitiveExtensionsTests.cs similarity index 92% rename from src/StellaOps.Feedser.Models.Tests/NevraPrimitiveExtensionsTests.cs rename to src/StellaOps.Concelier.Models.Tests/NevraPrimitiveExtensionsTests.cs index 548b2f3a..d5c22268 100644 --- a/src/StellaOps.Feedser.Models.Tests/NevraPrimitiveExtensionsTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/NevraPrimitiveExtensionsTests.cs @@ -1,44 +1,44 @@ -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class NevraPrimitiveExtensionsTests -{ - [Fact] - public void ToNormalizedVersionRule_ProducesRangeWhenBoundsAvailable() - { - var primitive = new NevraPrimitive( - Introduced: new NevraComponent("openssl", 1, "1.1.1k", "4", "x86_64"), - Fixed: new NevraComponent("openssl", 1, "1.1.1n", "5", "x86_64"), - LastAffected: null); - - var rule = primitive.ToNormalizedVersionRule("rhel-8"); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); - Assert.Equal("openssl-1:1.1.1k-4.x86_64", rule.Min); - Assert.True(rule.MinInclusive); - Assert.Equal("openssl-1:1.1.1n-5.x86_64", rule.Max); - Assert.False(rule.MaxInclusive); - Assert.Equal("rhel-8", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_UsesLastAffectedAsInclusiveUpperBound() - { - var primitive = new NevraPrimitive( - Introduced: null, - Fixed: null, - LastAffected: new NevraComponent("kernel", 0, "5.15.0", "1024.18", "x86_64")); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, rule.Type); - Assert.Equal("kernel-5.15.0-1024.18.x86_64", rule.Max); - Assert.True(rule.MaxInclusive); - } -} +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class NevraPrimitiveExtensionsTests +{ + [Fact] + public void ToNormalizedVersionRule_ProducesRangeWhenBoundsAvailable() + { + var primitive = new NevraPrimitive( + Introduced: new NevraComponent("openssl", 1, "1.1.1k", "4", "x86_64"), + Fixed: new NevraComponent("openssl", 1, "1.1.1n", "5", "x86_64"), + LastAffected: null); + + var rule = primitive.ToNormalizedVersionRule("rhel-8"); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("openssl-1:1.1.1k-4.x86_64", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("openssl-1:1.1.1n-5.x86_64", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Equal("rhel-8", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_UsesLastAffectedAsInclusiveUpperBound() + { + var primitive = new NevraPrimitive( + Introduced: null, + Fixed: null, + LastAffected: new NevraComponent("kernel", 0, "5.15.0", "1024.18", "x86_64")); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.Nevra, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.LessThanOrEqual, rule.Type); + Assert.Equal("kernel-5.15.0-1024.18.x86_64", rule.Max); + Assert.True(rule.MaxInclusive); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs b/src/StellaOps.Concelier.Models.Tests/NormalizedVersionRuleTests.cs similarity index 94% rename from src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs rename to src/StellaOps.Concelier.Models.Tests/NormalizedVersionRuleTests.cs index 7414eed6..4034a572 100644 --- a/src/StellaOps.Feedser.Models.Tests/NormalizedVersionRuleTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/NormalizedVersionRuleTests.cs @@ -1,68 +1,68 @@ -using System; -using System.Linq; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class NormalizedVersionRuleTests -{ - [Fact] - public void NormalizedVersions_AreDeduplicatedAndOrdered() - { - var recordedAt = DateTimeOffset.Parse("2025-01-05T00:00:00Z"); - var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-abc", recordedAt); - var package = new AffectedPackage( - type: AffectedPackageTypes.SemVer, - identifier: "pkg:npm/example", - versionRanges: Array.Empty(), - normalizedVersions: new[] - { - new NormalizedVersionRule("SemVer", "Exact", value: "1.0.0 "), - new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), - new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), - new NormalizedVersionRule("semver", "gt", min: "0.9.0", minInclusive: false, notes: " originating nvd "), - }, - statuses: Array.Empty(), - provenance: new[] { provenance }); - - var normalized = package.NormalizedVersions.ToArray(); - Assert.Equal(3, normalized.Length); - - Assert.Collection( - normalized, - rule => - { - Assert.Equal("semver", rule.Scheme); - Assert.Equal("exact", rule.Type); - Assert.Equal("1.0.0", rule.Value); - Assert.Null(rule.Min); - Assert.Null(rule.Max); - }, - rule => - { - Assert.Equal("semver", rule.Scheme); - Assert.Equal("gt", rule.Type); - Assert.Equal("0.9.0", rule.Min); - Assert.False(rule.MinInclusive); - Assert.Equal("originating nvd", rule.Notes); - }, - rule => - { - Assert.Equal("semver", rule.Scheme); - Assert.Equal("range", rule.Type); - Assert.Equal("1.2.0", rule.Min); - Assert.True(rule.MinInclusive); - Assert.Equal("2.0.0", rule.Max); - Assert.False(rule.MaxInclusive); - Assert.Equal("GHSA-abc", rule.Notes); - }); - } - - [Fact] - public void NormalizedVersionRule_NormalizesTypeSeparators() - { - var rule = new NormalizedVersionRule("semver", "tie_breaker", value: "1.2.3"); - Assert.Equal("tie-breaker", rule.Type); - } -} +using System; +using System.Linq; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class NormalizedVersionRuleTests +{ + [Fact] + public void NormalizedVersions_AreDeduplicatedAndOrdered() + { + var recordedAt = DateTimeOffset.Parse("2025-01-05T00:00:00Z"); + var provenance = new AdvisoryProvenance("ghsa", "map", "GHSA-abc", recordedAt); + var package = new AffectedPackage( + type: AffectedPackageTypes.SemVer, + identifier: "pkg:npm/example", + versionRanges: Array.Empty(), + normalizedVersions: new[] + { + new NormalizedVersionRule("SemVer", "Exact", value: "1.0.0 "), + new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), + new NormalizedVersionRule("semver", "range", min: "1.2.0", minInclusive: true, max: "2.0.0", maxInclusive: false, notes: "GHSA-abc"), + new NormalizedVersionRule("semver", "gt", min: "0.9.0", minInclusive: false, notes: " originating nvd "), + }, + statuses: Array.Empty(), + provenance: new[] { provenance }); + + var normalized = package.NormalizedVersions.ToArray(); + Assert.Equal(3, normalized.Length); + + Assert.Collection( + normalized, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("exact", rule.Type); + Assert.Equal("1.0.0", rule.Value); + Assert.Null(rule.Min); + Assert.Null(rule.Max); + }, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("gt", rule.Type); + Assert.Equal("0.9.0", rule.Min); + Assert.False(rule.MinInclusive); + Assert.Equal("originating nvd", rule.Notes); + }, + rule => + { + Assert.Equal("semver", rule.Scheme); + Assert.Equal("range", rule.Type); + Assert.Equal("1.2.0", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("2.0.0", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Equal("GHSA-abc", rule.Notes); + }); + } + + [Fact] + public void NormalizedVersionRule_NormalizesTypeSeparators() + { + var rule = new NormalizedVersionRule("semver", "tie_breaker", value: "1.2.3"); + Assert.Equal("tie-breaker", rule.Type); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs b/src/StellaOps.Concelier.Models.Tests/OsvGhsaParityDiagnosticsTests.cs similarity index 92% rename from src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs rename to src/StellaOps.Concelier.Models.Tests/OsvGhsaParityDiagnosticsTests.cs index 38899e04..b75aab16 100644 --- a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityDiagnosticsTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/OsvGhsaParityDiagnosticsTests.cs @@ -1,88 +1,88 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics.Metrics; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class OsvGhsaParityDiagnosticsTests -{ - [Fact] - public void RecordReport_EmitsTotalAndIssues() - { - var issues = ImmutableArray.Create( - new OsvGhsaParityIssue( - GhsaId: "GHSA-AAA", - IssueKind: "missing_osv", - Detail: "", - FieldMask: ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages)), - new OsvGhsaParityIssue( - GhsaId: "GHSA-BBB", - IssueKind: "severity_mismatch", - Detail: "", - FieldMask: ImmutableArray.Empty)); - var report = new OsvGhsaParityReport(2, issues); - - var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); - using var listener = CreateListener(measurements); - - OsvGhsaParityDiagnostics.RecordReport(report, "QA"); - - listener.Dispose(); - - Assert.Equal(3, measurements.Count); - - var total = Assert.Single(measurements, m => m.Instrument == "feedser.osv_ghsa.total"); - Assert.Equal(2, total.Value); - Assert.Equal("qa", total.Tags["dataset"]); - - var missing = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "missing_osv", StringComparison.Ordinal)); - Assert.Equal("affectedpackages[]", missing.Tags["fieldMask"]); - - var severity = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "severity_mismatch", StringComparison.Ordinal)); - Assert.Equal("none", severity.Tags["fieldMask"]); - } - - [Fact] - public void RecordReport_NoIssues_OnlyEmitsTotal() - { - var report = new OsvGhsaParityReport(0, ImmutableArray.Empty); - var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); - using var listener = CreateListener(measurements); - - OsvGhsaParityDiagnostics.RecordReport(report, ""); - - listener.Dispose(); - Assert.Empty(measurements); - } - - private static MeterListener CreateListener(List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements) - { - var listener = new MeterListener - { - InstrumentPublished = (instrument, l) => - { - if (instrument.Meter.Name.StartsWith("StellaOps.Feedser.Models.OsvGhsaParity", StringComparison.Ordinal)) - { - l.EnableMeasurementEvents(instrument); - } - } - }; - - listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => - { - var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var tag in tags) - { - dict[tag.Key] = tag.Value; - } - - measurements.Add((instrument.Name, measurement, dict)); - }); - - listener.Start(); - return listener; - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics.Metrics; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class OsvGhsaParityDiagnosticsTests +{ + [Fact] + public void RecordReport_EmitsTotalAndIssues() + { + var issues = ImmutableArray.Create( + new OsvGhsaParityIssue( + GhsaId: "GHSA-AAA", + IssueKind: "missing_osv", + Detail: "", + FieldMask: ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages)), + new OsvGhsaParityIssue( + GhsaId: "GHSA-BBB", + IssueKind: "severity_mismatch", + Detail: "", + FieldMask: ImmutableArray.Empty)); + var report = new OsvGhsaParityReport(2, issues); + + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + OsvGhsaParityDiagnostics.RecordReport(report, "QA"); + + listener.Dispose(); + + Assert.Equal(3, measurements.Count); + + var total = Assert.Single(measurements, m => m.Instrument == "concelier.osv_ghsa.total"); + Assert.Equal(2, total.Value); + Assert.Equal("qa", total.Tags["dataset"]); + + var missing = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "missing_osv", StringComparison.Ordinal)); + Assert.Equal("affectedpackages[]", missing.Tags["fieldMask"]); + + var severity = Assert.Single(measurements, m => m.Tags.TryGetValue("issueKind", out var kind) && string.Equals((string)kind!, "severity_mismatch", StringComparison.Ordinal)); + Assert.Equal("none", severity.Tags["fieldMask"]); + } + + [Fact] + public void RecordReport_NoIssues_OnlyEmitsTotal() + { + var report = new OsvGhsaParityReport(0, ImmutableArray.Empty); + var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); + using var listener = CreateListener(measurements); + + OsvGhsaParityDiagnostics.RecordReport(report, ""); + + listener.Dispose(); + Assert.Empty(measurements); + } + + private static MeterListener CreateListener(List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements) + { + var listener = new MeterListener + { + InstrumentPublished = (instrument, l) => + { + if (instrument.Meter.Name.StartsWith("StellaOps.Concelier.Models.OsvGhsaParity", StringComparison.Ordinal)) + { + l.EnableMeasurementEvents(instrument); + } + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var tag in tags) + { + dict[tag.Key] = tag.Value; + } + + measurements.Add((instrument.Name, measurement, dict)); + }); + + listener.Start(); + return listener; + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs b/src/StellaOps.Concelier.Models.Tests/OsvGhsaParityInspectorTests.cs similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs rename to src/StellaOps.Concelier.Models.Tests/OsvGhsaParityInspectorTests.cs index b348d803..532fd336 100644 --- a/src/StellaOps.Feedser.Models.Tests/OsvGhsaParityInspectorTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/OsvGhsaParityInspectorTests.cs @@ -1,148 +1,148 @@ -using System; -using System.Collections.Generic; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class OsvGhsaParityInspectorTests -{ - [Fact] - public void Compare_ReturnsNoIssues_WhenDatasetsMatch() - { - var ghsaId = "GHSA-1111"; - var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: true); - var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); - - var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); - - Assert.False(report.HasIssues); - Assert.Equal(1, report.TotalGhsaIds); - Assert.Empty(report.Issues); - } - - [Fact] - public void Compare_FlagsMissingOsvEntry() - { - var ghsaId = "GHSA-2222"; - var ghsa = CreateGhsaAdvisory(ghsaId, severity: "medium", includeRanges: true); - - var report = OsvGhsaParityInspector.Compare(Array.Empty(), new[] { ghsa }); - - var issue = Assert.Single(report.Issues); - Assert.Equal("missing_osv", issue.IssueKind); - Assert.Equal(ghsaId, issue.GhsaId); - Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); - } - - [Fact] - public void Compare_FlagsMissingGhsaEntry() - { - var ghsaId = "GHSA-2424"; - var osv = CreateOsvAdvisory(ghsaId, severity: "medium", includeRanges: true); - - var report = OsvGhsaParityInspector.Compare(new[] { osv }, Array.Empty()); - - var issue = Assert.Single(report.Issues); - Assert.Equal("missing_ghsa", issue.IssueKind); - Assert.Equal(ghsaId, issue.GhsaId); - Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); - } - - [Fact] - public void Compare_FlagsSeverityMismatch() - { - var ghsaId = "GHSA-3333"; - var osv = CreateOsvAdvisory(ghsaId, severity: "low", includeRanges: true); - var ghsa = CreateGhsaAdvisory(ghsaId, severity: "critical", includeRanges: true); - - var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); - - var issue = Assert.Single(report.Issues, i => i.IssueKind == "severity_mismatch"); - Assert.Equal(ghsaId, issue.GhsaId); - Assert.Contains(ProvenanceFieldMasks.Advisory, issue.FieldMask); - } - - [Fact] - public void Compare_FlagsRangeMismatch() - { - var ghsaId = "GHSA-4444"; - var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: false); - var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); - - var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); - - var issue = Assert.Single(report.Issues, i => i.IssueKind == "range_mismatch"); - Assert.Equal(ghsaId, issue.GhsaId); - Assert.Contains(ProvenanceFieldMasks.VersionRanges, issue.FieldMask); - } - - private static Advisory CreateOsvAdvisory(string ghsaId, string? severity, bool includeRanges) - { - var timestamp = DateTimeOffset.UtcNow; - return new Advisory( - advisoryKey: $"osv-{ghsaId.ToLowerInvariant()}", - title: $"OSV {ghsaId}", - summary: null, - language: null, - published: timestamp, - modified: timestamp, - severity: severity, - exploitKnown: false, - aliases: new[] { ghsaId }, - references: Array.Empty(), - affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("osv", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) - }); - } - - private static Advisory CreateGhsaAdvisory(string ghsaId, string? severity, bool includeRanges) - { - var timestamp = DateTimeOffset.UtcNow; - return new Advisory( - advisoryKey: ghsaId.ToLowerInvariant(), - title: $"GHSA {ghsaId}", - summary: null, - language: null, - published: timestamp, - modified: timestamp, - severity: severity, - exploitKnown: false, - aliases: new[] { ghsaId }, - references: Array.Empty(), - affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: new[] - { - new AdvisoryProvenance("ghsa", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) - }); - } - - private static AffectedPackage CreatePackage(DateTimeOffset recordedAt, bool includeRanges) - { - var ranges = includeRanges - ? new[] - { - new AffectedVersionRange( - rangeKind: "semver", - introducedVersion: "1.0.0", - fixedVersion: "1.2.0", - lastAffectedVersion: null, - rangeExpression: null, - provenance: new AdvisoryProvenance("mapper", "range", "package@1", recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }), - primitives: null) - } - : Array.Empty(); - - return new AffectedPackage( - type: "semver", - identifier: "pkg@1", - platform: null, - versionRanges: ranges, - statuses: Array.Empty(), - provenance: new[] { new AdvisoryProvenance("mapper", "package", "pkg@1", recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }) }); - } -} +using System; +using System.Collections.Generic; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class OsvGhsaParityInspectorTests +{ + [Fact] + public void Compare_ReturnsNoIssues_WhenDatasetsMatch() + { + var ghsaId = "GHSA-1111"; + var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: true); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + Assert.False(report.HasIssues); + Assert.Equal(1, report.TotalGhsaIds); + Assert.Empty(report.Issues); + } + + [Fact] + public void Compare_FlagsMissingOsvEntry() + { + var ghsaId = "GHSA-2222"; + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "medium", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(Array.Empty(), new[] { ghsa }); + + var issue = Assert.Single(report.Issues); + Assert.Equal("missing_osv", issue.IssueKind); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsMissingGhsaEntry() + { + var ghsaId = "GHSA-2424"; + var osv = CreateOsvAdvisory(ghsaId, severity: "medium", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, Array.Empty()); + + var issue = Assert.Single(report.Issues); + Assert.Equal("missing_ghsa", issue.IssueKind); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.AffectedPackages, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsSeverityMismatch() + { + var ghsaId = "GHSA-3333"; + var osv = CreateOsvAdvisory(ghsaId, severity: "low", includeRanges: true); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "critical", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + var issue = Assert.Single(report.Issues, i => i.IssueKind == "severity_mismatch"); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.Advisory, issue.FieldMask); + } + + [Fact] + public void Compare_FlagsRangeMismatch() + { + var ghsaId = "GHSA-4444"; + var osv = CreateOsvAdvisory(ghsaId, severity: "high", includeRanges: false); + var ghsa = CreateGhsaAdvisory(ghsaId, severity: "high", includeRanges: true); + + var report = OsvGhsaParityInspector.Compare(new[] { osv }, new[] { ghsa }); + + var issue = Assert.Single(report.Issues, i => i.IssueKind == "range_mismatch"); + Assert.Equal(ghsaId, issue.GhsaId); + Assert.Contains(ProvenanceFieldMasks.VersionRanges, issue.FieldMask); + } + + private static Advisory CreateOsvAdvisory(string ghsaId, string? severity, bool includeRanges) + { + var timestamp = DateTimeOffset.UtcNow; + return new Advisory( + advisoryKey: $"osv-{ghsaId.ToLowerInvariant()}", + title: $"OSV {ghsaId}", + summary: null, + language: null, + published: timestamp, + modified: timestamp, + severity: severity, + exploitKnown: false, + aliases: new[] { ghsaId }, + references: Array.Empty(), + affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("osv", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) + }); + } + + private static Advisory CreateGhsaAdvisory(string ghsaId, string? severity, bool includeRanges) + { + var timestamp = DateTimeOffset.UtcNow; + return new Advisory( + advisoryKey: ghsaId.ToLowerInvariant(), + title: $"GHSA {ghsaId}", + summary: null, + language: null, + published: timestamp, + modified: timestamp, + severity: severity, + exploitKnown: false, + aliases: new[] { ghsaId }, + references: Array.Empty(), + affectedPackages: includeRanges ? new[] { CreatePackage(timestamp, includeRanges) } : Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] + { + new AdvisoryProvenance("ghsa", "map", ghsaId, timestamp, new[] { ProvenanceFieldMasks.Advisory }) + }); + } + + private static AffectedPackage CreatePackage(DateTimeOffset recordedAt, bool includeRanges) + { + var ranges = includeRanges + ? new[] + { + new AffectedVersionRange( + rangeKind: "semver", + introducedVersion: "1.0.0", + fixedVersion: "1.2.0", + lastAffectedVersion: null, + rangeExpression: null, + provenance: new AdvisoryProvenance("mapper", "range", "package@1", recordedAt, new[] { ProvenanceFieldMasks.VersionRanges }), + primitives: null) + } + : Array.Empty(); + + return new AffectedPackage( + type: "semver", + identifier: "pkg@1", + platform: null, + versionRanges: ranges, + statuses: Array.Empty(), + provenance: new[] { new AdvisoryProvenance("mapper", "package", "pkg@1", recordedAt, new[] { ProvenanceFieldMasks.AffectedPackages }) }); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs b/src/StellaOps.Concelier.Models.Tests/ProvenanceDiagnosticsTests.cs similarity index 90% rename from src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs rename to src/StellaOps.Concelier.Models.Tests/ProvenanceDiagnosticsTests.cs index b410834b..37f68602 100644 --- a/src/StellaOps.Feedser.Models.Tests/ProvenanceDiagnosticsTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/ProvenanceDiagnosticsTests.cs @@ -4,10 +4,10 @@ using System.Diagnostics.Metrics; using System.Linq; using System.Reflection; using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; using Xunit; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class ProvenanceDiagnosticsTests { @@ -30,7 +30,7 @@ public sealed class ProvenanceDiagnosticsTests var first = measurements[0]; Assert.Equal(1, first.Value); - Assert.Equal("feedser.provenance.missing", first.Instrument); + Assert.Equal("concelier.provenance.missing", first.Instrument); Assert.Equal("source-A", first.Tags["source"]); Assert.Equal("range:pkg", first.Tags["component"]); Assert.Equal("range", first.Tags["category"]); @@ -38,7 +38,7 @@ public sealed class ProvenanceDiagnosticsTests Assert.Equal(ProvenanceFieldMasks.VersionRanges, first.Tags["fieldMask"]); var second = measurements[1]; - Assert.Equal("feedser.provenance.missing", second.Instrument); + Assert.Equal("concelier.provenance.missing", second.Instrument); Assert.Equal("reference", second.Tags["category"]); Assert.Equal("low", second.Tags["severity"]); Assert.Equal(ProvenanceFieldMasks.References, second.Tags["fieldMask"]); @@ -106,14 +106,14 @@ public sealed class ProvenanceDiagnosticsTests VendorExtensions: new Dictionary { ["debian.release"] = "bullseye" })); var measurements = new List<(string Instrument, long Value, IReadOnlyDictionary Tags)>(); - using var listener = CreateListener(measurements, "feedser.range.primitives"); + using var listener = CreateListener(measurements, "concelier.range.primitives"); ProvenanceDiagnostics.RecordRangePrimitive("source-D", range); listener.Dispose(); var record = Assert.Single(measurements); - Assert.Equal("feedser.range.primitives", record.Instrument); + Assert.Equal("concelier.range.primitives", record.Instrument); Assert.Equal(1, record.Value); Assert.Equal("source-D", record.Tags["source"]); Assert.Equal("evr", record.Tags["rangeKind"]); @@ -125,14 +125,14 @@ public sealed class ProvenanceDiagnosticsTests List<(string Instrument, long Value, IReadOnlyDictionary Tags)> measurements, params string[] instrumentNames) { - var allowed = instrumentNames is { Length: > 0 } ? instrumentNames : new[] { "feedser.provenance.missing" }; + var allowed = instrumentNames is { Length: > 0 } ? instrumentNames : new[] { "concelier.provenance.missing" }; var allowedSet = new HashSet(allowed, StringComparer.OrdinalIgnoreCase); var listener = new MeterListener { InstrumentPublished = (instrument, l) => { - if (instrument.Meter.Name == "StellaOps.Feedser.Models.Provenance" && allowedSet.Contains(instrument.Name)) + if (instrument.Meter.Name == "StellaOps.Concelier.Models.Provenance" && allowedSet.Contains(instrument.Name)) { l.EnableMeasurementEvents(instrument); } diff --git a/src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs b/src/StellaOps.Concelier.Models.Tests/RangePrimitivesTests.cs similarity index 90% rename from src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs rename to src/StellaOps.Concelier.Models.Tests/RangePrimitivesTests.cs index 82e55ea2..2c75eab1 100644 --- a/src/StellaOps.Feedser.Models.Tests/RangePrimitivesTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/RangePrimitivesTests.cs @@ -1,41 +1,41 @@ -using System.Collections.Generic; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class RangePrimitivesTests -{ - [Fact] - public void GetCoverageTag_ReturnsSpecificKinds() - { - var primitives = new RangePrimitives( - new SemVerPrimitive("1.0.0", true, "1.2.0", false, null, false, null), - new NevraPrimitive(null, null, null), - null, - null); - - Assert.Equal("nevra+semver", primitives.GetCoverageTag()); - } - - [Fact] - public void GetCoverageTag_ReturnsVendorWhenOnlyExtensions() - { - var primitives = new RangePrimitives( - null, - null, - null, - new Dictionary { ["vendor.status"] = "beta" }); - - Assert.True(primitives.HasVendorExtensions); - Assert.Equal("vendor", primitives.GetCoverageTag()); - } - - [Fact] - public void GetCoverageTag_ReturnsNoneWhenEmpty() - { - var primitives = new RangePrimitives(null, null, null, null); - Assert.False(primitives.HasVendorExtensions); - Assert.Equal("none", primitives.GetCoverageTag()); - } -} +using System.Collections.Generic; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class RangePrimitivesTests +{ + [Fact] + public void GetCoverageTag_ReturnsSpecificKinds() + { + var primitives = new RangePrimitives( + new SemVerPrimitive("1.0.0", true, "1.2.0", false, null, false, null), + new NevraPrimitive(null, null, null), + null, + null); + + Assert.Equal("nevra+semver", primitives.GetCoverageTag()); + } + + [Fact] + public void GetCoverageTag_ReturnsVendorWhenOnlyExtensions() + { + var primitives = new RangePrimitives( + null, + null, + null, + new Dictionary { ["vendor.status"] = "beta" }); + + Assert.True(primitives.HasVendorExtensions); + Assert.Equal("vendor", primitives.GetCoverageTag()); + } + + [Fact] + public void GetCoverageTag_ReturnsNoneWhenEmpty() + { + var primitives = new RangePrimitives(null, null, null, null); + Assert.False(primitives.HasVendorExtensions); + Assert.Equal("none", primitives.GetCoverageTag()); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs b/src/StellaOps.Concelier.Models.Tests/SemVerPrimitiveTests.cs similarity index 96% rename from src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs rename to src/StellaOps.Concelier.Models.Tests/SemVerPrimitiveTests.cs index 76990678..9452a717 100644 --- a/src/StellaOps.Feedser.Models.Tests/SemVerPrimitiveTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/SemVerPrimitiveTests.cs @@ -1,189 +1,189 @@ -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class SemVerPrimitiveTests -{ - [Theory] - [InlineData("1.0.0", true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.Range)] - [InlineData("1.0.0", true, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThanOrEqual)] - [InlineData("1.0.0", false, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThan)] - [InlineData(null, true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.LessThan)] - [InlineData(null, true, "2.0.0", true, null, false, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] - [InlineData(null, true, null, false, "2.0.0", true, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] - [InlineData(null, true, null, false, "2.0.0", false, null, null, SemVerPrimitiveStyles.LessThan)] - [InlineData(null, true, null, false, null, false, null, "1.5.0", SemVerPrimitiveStyles.Exact)] - public void StyleReflectsSemantics( - string? introduced, - bool introducedInclusive, - string? fixedVersion, - bool fixedInclusive, - string? lastAffected, - bool lastAffectedInclusive, - string? constraintExpression, - string? exactValue, - string expectedStyle) - { - var primitive = new SemVerPrimitive( - introduced, - introducedInclusive, - fixedVersion, - fixedInclusive, - lastAffected, - lastAffectedInclusive, - constraintExpression, - exactValue); - - Assert.Equal(expectedStyle, primitive.Style); - } - - [Fact] - public void EqualityIncludesExactValue() - { - var baseline = new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: true, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: null); - - var variant = baseline with { ExactValue = "1.2.3" }; - - Assert.NotEqual(baseline, variant); - Assert.Equal(SemVerPrimitiveStyles.Exact, variant.Style); - Assert.Equal(SemVerPrimitiveStyles.Range, baseline.Style); - } - - [Fact] - public void ToNormalizedVersionRule_MapsRangeBounds() - { - var primitive = new SemVerPrimitive( - Introduced: "1.0.0", - IntroducedInclusive: true, - Fixed: "2.0.0", - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: ">=1.0.0 <2.0.0"); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); - Assert.Equal("1.0.0", rule.Min); - Assert.True(rule.MinInclusive); - Assert.Equal("2.0.0", rule.Max); - Assert.False(rule.MaxInclusive); - Assert.Null(rule.Value); - Assert.Equal(">=1.0.0 <2.0.0", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_ExactUsesExactValue() - { - var primitive = new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: true, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: null, - ExactValue: "3.1.4"); - - var rule = primitive.ToNormalizedVersionRule("from-ghsa"); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); - Assert.Null(rule.Min); - Assert.Null(rule.Max); - Assert.Equal("3.1.4", rule.Value); - Assert.Equal("from-ghsa", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_GreaterThanMapsMinimum() - { - var primitive = new SemVerPrimitive( - Introduced: "1.5.0", - IntroducedInclusive: false, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: null); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.GreaterThan, rule.Type); - Assert.Equal("1.5.0", rule.Min); - Assert.False(rule.MinInclusive); - Assert.Null(rule.Max); - Assert.Null(rule.Value); - Assert.Null(rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_UsesConstraintExpressionAsFallbackNotes() - { - var primitive = new SemVerPrimitive( - Introduced: "1.4.0", - IntroducedInclusive: false, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: "> 1.4.0"); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal("> 1.4.0", rule!.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_ExactCarriesConstraintExpressionWhenNotesMissing() - { - var primitive = new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: true, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: "= 3.2.1", - ExactValue: "3.2.1"); - - var rule = primitive.ToNormalizedVersionRule(); - - Assert.NotNull(rule); - Assert.Equal(NormalizedVersionRuleTypes.Exact, rule!.Type); - Assert.Equal("3.2.1", rule.Value); - Assert.Equal("= 3.2.1", rule.Notes); - } - - [Fact] - public void ToNormalizedVersionRule_ExplicitNotesOverrideConstraintExpression() - { - var primitive = new SemVerPrimitive( - Introduced: "1.0.0", - IntroducedInclusive: true, - Fixed: "1.1.0", - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: ">=1.0.0 <1.1.0"); - - var rule = primitive.ToNormalizedVersionRule("ghsa:range"); - - Assert.NotNull(rule); - Assert.Equal("ghsa:range", rule!.Notes); - } -} +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class SemVerPrimitiveTests +{ + [Theory] + [InlineData("1.0.0", true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.Range)] + [InlineData("1.0.0", true, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThanOrEqual)] + [InlineData("1.0.0", false, null, false, null, false, null, null, SemVerPrimitiveStyles.GreaterThan)] + [InlineData(null, true, "2.0.0", false, null, false, null, null, SemVerPrimitiveStyles.LessThan)] + [InlineData(null, true, "2.0.0", true, null, false, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] + [InlineData(null, true, null, false, "2.0.0", true, null, null, SemVerPrimitiveStyles.LessThanOrEqual)] + [InlineData(null, true, null, false, "2.0.0", false, null, null, SemVerPrimitiveStyles.LessThan)] + [InlineData(null, true, null, false, null, false, null, "1.5.0", SemVerPrimitiveStyles.Exact)] + public void StyleReflectsSemantics( + string? introduced, + bool introducedInclusive, + string? fixedVersion, + bool fixedInclusive, + string? lastAffected, + bool lastAffectedInclusive, + string? constraintExpression, + string? exactValue, + string expectedStyle) + { + var primitive = new SemVerPrimitive( + introduced, + introducedInclusive, + fixedVersion, + fixedInclusive, + lastAffected, + lastAffectedInclusive, + constraintExpression, + exactValue); + + Assert.Equal(expectedStyle, primitive.Style); + } + + [Fact] + public void EqualityIncludesExactValue() + { + var baseline = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + + var variant = baseline with { ExactValue = "1.2.3" }; + + Assert.NotEqual(baseline, variant); + Assert.Equal(SemVerPrimitiveStyles.Exact, variant.Style); + Assert.Equal(SemVerPrimitiveStyles.Range, baseline.Style); + } + + [Fact] + public void ToNormalizedVersionRule_MapsRangeBounds() + { + var primitive = new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "2.0.0", + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: ">=1.0.0 <2.0.0"); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("1.0.0", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("2.0.0", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Null(rule.Value); + Assert.Equal(">=1.0.0 <2.0.0", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_ExactUsesExactValue() + { + var primitive = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null, + ExactValue: "3.1.4"); + + var rule = primitive.ToNormalizedVersionRule("from-ghsa"); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Exact, rule.Type); + Assert.Null(rule.Min); + Assert.Null(rule.Max); + Assert.Equal("3.1.4", rule.Value); + Assert.Equal("from-ghsa", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_GreaterThanMapsMinimum() + { + var primitive = new SemVerPrimitive( + Introduced: "1.5.0", + IntroducedInclusive: false, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: null); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionSchemes.SemVer, rule!.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.GreaterThan, rule.Type); + Assert.Equal("1.5.0", rule.Min); + Assert.False(rule.MinInclusive); + Assert.Null(rule.Max); + Assert.Null(rule.Value); + Assert.Null(rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_UsesConstraintExpressionAsFallbackNotes() + { + var primitive = new SemVerPrimitive( + Introduced: "1.4.0", + IntroducedInclusive: false, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: "> 1.4.0"); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal("> 1.4.0", rule!.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_ExactCarriesConstraintExpressionWhenNotesMissing() + { + var primitive = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: "= 3.2.1", + ExactValue: "3.2.1"); + + var rule = primitive.ToNormalizedVersionRule(); + + Assert.NotNull(rule); + Assert.Equal(NormalizedVersionRuleTypes.Exact, rule!.Type); + Assert.Equal("3.2.1", rule.Value); + Assert.Equal("= 3.2.1", rule.Notes); + } + + [Fact] + public void ToNormalizedVersionRule_ExplicitNotesOverrideConstraintExpression() + { + var primitive = new SemVerPrimitive( + Introduced: "1.0.0", + IntroducedInclusive: true, + Fixed: "1.1.0", + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: ">=1.0.0 <1.1.0"); + + var rule = primitive.ToNormalizedVersionRule("ghsa:range"); + + Assert.NotNull(rule); + Assert.Equal("ghsa:range", rule!.Notes); + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs b/src/StellaOps.Concelier.Models.Tests/SerializationDeterminismTests.cs similarity index 93% rename from src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs rename to src/StellaOps.Concelier.Models.Tests/SerializationDeterminismTests.cs index 0f04755f..13c4baad 100644 --- a/src/StellaOps.Feedser.Models.Tests/SerializationDeterminismTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/SerializationDeterminismTests.cs @@ -1,68 +1,68 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using StellaOps.Feedser.Models; -using Xunit; - -namespace StellaOps.Feedser.Models.Tests; - -public sealed class SerializationDeterminismTests -{ - private static readonly string[] Cultures = - { - "en-US", - "fr-FR", - "tr-TR", - "ja-JP", - "ar-SA" - }; - - [Fact] - public void CanonicalSerializer_ProducesStableJsonAcrossCultures() - { - var examples = CanonicalExampleFactory.GetExamples().ToArray(); - var baseline = SerializeUnderCulture(CultureInfo.InvariantCulture, examples); - - foreach (var cultureName in Cultures) - { - var culture = CultureInfo.GetCultureInfo(cultureName); - var serialized = SerializeUnderCulture(culture, examples); - - Assert.Equal(baseline.Count, serialized.Count); - for (var i = 0; i < baseline.Count; i++) - { - Assert.Equal(baseline[i].Compact, serialized[i].Compact); - Assert.Equal(baseline[i].Indented, serialized[i].Indented); - } - } - } - - private static List<(string Name, string Compact, string Indented)> SerializeUnderCulture( - CultureInfo culture, - IReadOnlyList<(string Name, Advisory Advisory)> examples) - { - var originalCulture = CultureInfo.CurrentCulture; - var originalUiCulture = CultureInfo.CurrentUICulture; - try - { - CultureInfo.CurrentCulture = culture; - CultureInfo.CurrentUICulture = culture; - - var results = new List<(string Name, string Compact, string Indented)>(examples.Count); - foreach (var (name, advisory) in examples) - { - var compact = CanonicalJsonSerializer.Serialize(advisory); - var indented = CanonicalJsonSerializer.SerializeIndented(advisory); - results.Add((name, compact, indented)); - } - - return results; - } - finally - { - CultureInfo.CurrentCulture = originalCulture; - CultureInfo.CurrentUICulture = originalUiCulture; - } - } -} +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using StellaOps.Concelier.Models; +using Xunit; + +namespace StellaOps.Concelier.Models.Tests; + +public sealed class SerializationDeterminismTests +{ + private static readonly string[] Cultures = + { + "en-US", + "fr-FR", + "tr-TR", + "ja-JP", + "ar-SA" + }; + + [Fact] + public void CanonicalSerializer_ProducesStableJsonAcrossCultures() + { + var examples = CanonicalExampleFactory.GetExamples().ToArray(); + var baseline = SerializeUnderCulture(CultureInfo.InvariantCulture, examples); + + foreach (var cultureName in Cultures) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var serialized = SerializeUnderCulture(culture, examples); + + Assert.Equal(baseline.Count, serialized.Count); + for (var i = 0; i < baseline.Count; i++) + { + Assert.Equal(baseline[i].Compact, serialized[i].Compact); + Assert.Equal(baseline[i].Indented, serialized[i].Indented); + } + } + } + + private static List<(string Name, string Compact, string Indented)> SerializeUnderCulture( + CultureInfo culture, + IReadOnlyList<(string Name, Advisory Advisory)> examples) + { + var originalCulture = CultureInfo.CurrentCulture; + var originalUiCulture = CultureInfo.CurrentUICulture; + try + { + CultureInfo.CurrentCulture = culture; + CultureInfo.CurrentUICulture = culture; + + var results = new List<(string Name, string Compact, string Indented)>(examples.Count); + foreach (var (name, advisory) in examples) + { + var compact = CanonicalJsonSerializer.Serialize(advisory); + var indented = CanonicalJsonSerializer.SerializeIndented(advisory); + results.Add((name, compact, indented)); + } + + return results; + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + CultureInfo.CurrentUICulture = originalUiCulture; + } + } +} diff --git a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs b/src/StellaOps.Concelier.Models.Tests/SeverityNormalizationTests.cs similarity index 92% rename from src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs rename to src/StellaOps.Concelier.Models.Tests/SeverityNormalizationTests.cs index 04bc0f91..d8007744 100644 --- a/src/StellaOps.Feedser.Models.Tests/SeverityNormalizationTests.cs +++ b/src/StellaOps.Concelier.Models.Tests/SeverityNormalizationTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Models.Tests; +namespace StellaOps.Concelier.Models.Tests; public sealed class SeverityNormalizationTests { diff --git a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj b/src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj similarity index 79% rename from src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj rename to src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj index 8a7e97a1..5320b137 100644 --- a/src/StellaOps.Feedser.Models.Tests/StellaOps.Feedser.Models.Tests.csproj +++ b/src/StellaOps.Concelier.Models.Tests/StellaOps.Concelier.Models.Tests.csproj @@ -5,7 +5,7 @@ enable - + diff --git a/src/StellaOps.Feedser.Models/AGENTS.md b/src/StellaOps.Concelier.Models/AGENTS.md similarity index 92% rename from src/StellaOps.Feedser.Models/AGENTS.md rename to src/StellaOps.Concelier.Models/AGENTS.md index 481ffaeb..23beb90a 100644 --- a/src/StellaOps.Feedser.Models/AGENTS.md +++ b/src/StellaOps.Concelier.Models/AGENTS.md @@ -24,7 +24,7 @@ Out: fetching/parsing external schemas, storage, HTTP. - Provide debug renders for test snapshots (canonical JSON). - Emit model version identifiers in logs when canonical structures change; keep adapters for older readers until deprecated. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Models.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Models.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Models/Advisory.cs b/src/StellaOps.Concelier.Models/Advisory.cs similarity index 99% rename from src/StellaOps.Feedser.Models/Advisory.cs rename to src/StellaOps.Concelier.Models/Advisory.cs index 75a8ac19..d522d329 100644 --- a/src/StellaOps.Feedser.Models/Advisory.cs +++ b/src/StellaOps.Concelier.Models/Advisory.cs @@ -2,7 +2,7 @@ using System.Collections.Immutable; using System.Linq; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Canonical advisory document produced after merge. Collections are pre-sorted for deterministic serialization. diff --git a/src/StellaOps.Feedser.Models/AdvisoryCredit.cs b/src/StellaOps.Concelier.Models/AdvisoryCredit.cs similarity index 95% rename from src/StellaOps.Feedser.Models/AdvisoryCredit.cs rename to src/StellaOps.Concelier.Models/AdvisoryCredit.cs index e4252410..56f8b7c6 100644 --- a/src/StellaOps.Feedser.Models/AdvisoryCredit.cs +++ b/src/StellaOps.Concelier.Models/AdvisoryCredit.cs @@ -1,101 +1,101 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Models; - -/// -/// Canonical acknowledgement/credit metadata associated with an advisory. -/// -public sealed record AdvisoryCredit -{ - public static AdvisoryCredit Empty { get; } = new("unknown", role: null, contacts: Array.Empty(), AdvisoryProvenance.Empty); - - [JsonConstructor] - public AdvisoryCredit(string displayName, string? role, ImmutableArray contacts, AdvisoryProvenance provenance) - : this(displayName, role, contacts.IsDefault ? null : contacts.AsEnumerable(), provenance) - { - } - - public AdvisoryCredit(string displayName, string? role, IEnumerable? contacts, AdvisoryProvenance provenance) - { - DisplayName = Validation.EnsureNotNullOrWhiteSpace(displayName, nameof(displayName)); - Role = NormalizeRole(role); - Contacts = NormalizeContacts(contacts); - Provenance = provenance ?? AdvisoryProvenance.Empty; - } - - public string DisplayName { get; } - - public string? Role { get; } - - public ImmutableArray Contacts { get; } - - public AdvisoryProvenance Provenance { get; } - - private static string? NormalizeRole(string? role) - { - if (string.IsNullOrWhiteSpace(role)) - { - return null; - } - - var span = role.AsSpan(); - var buffer = new StringBuilder(span.Length); - - foreach (var ch in span) - { - if (char.IsLetterOrDigit(ch)) - { - buffer.Append(char.ToLowerInvariant(ch)); - continue; - } - - if (ch is '-' or '_' or ' ') - { - if (buffer.Length > 0 && buffer[^1] != '_') - { - buffer.Append('_'); - } - - continue; - } - } - - while (buffer.Length > 0 && buffer[^1] == '_') - { - buffer.Length--; - } - - return buffer.Length == 0 ? null : buffer.ToString(); - } - - private static ImmutableArray NormalizeContacts(IEnumerable? contacts) - { - if (contacts is null) - { - return ImmutableArray.Empty; - } - - var set = new SortedSet(StringComparer.Ordinal); - foreach (var contact in contacts) - { - if (string.IsNullOrWhiteSpace(contact)) - { - continue; - } - - var trimmed = contact.Trim(); - if (trimmed.Length == 0) - { - continue; - } - - set.Add(trimmed); - } - - return set.Count == 0 ? ImmutableArray.Empty : set.ToImmutableArray(); - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Models; + +/// +/// Canonical acknowledgement/credit metadata associated with an advisory. +/// +public sealed record AdvisoryCredit +{ + public static AdvisoryCredit Empty { get; } = new("unknown", role: null, contacts: Array.Empty(), AdvisoryProvenance.Empty); + + [JsonConstructor] + public AdvisoryCredit(string displayName, string? role, ImmutableArray contacts, AdvisoryProvenance provenance) + : this(displayName, role, contacts.IsDefault ? null : contacts.AsEnumerable(), provenance) + { + } + + public AdvisoryCredit(string displayName, string? role, IEnumerable? contacts, AdvisoryProvenance provenance) + { + DisplayName = Validation.EnsureNotNullOrWhiteSpace(displayName, nameof(displayName)); + Role = NormalizeRole(role); + Contacts = NormalizeContacts(contacts); + Provenance = provenance ?? AdvisoryProvenance.Empty; + } + + public string DisplayName { get; } + + public string? Role { get; } + + public ImmutableArray Contacts { get; } + + public AdvisoryProvenance Provenance { get; } + + private static string? NormalizeRole(string? role) + { + if (string.IsNullOrWhiteSpace(role)) + { + return null; + } + + var span = role.AsSpan(); + var buffer = new StringBuilder(span.Length); + + foreach (var ch in span) + { + if (char.IsLetterOrDigit(ch)) + { + buffer.Append(char.ToLowerInvariant(ch)); + continue; + } + + if (ch is '-' or '_' or ' ') + { + if (buffer.Length > 0 && buffer[^1] != '_') + { + buffer.Append('_'); + } + + continue; + } + } + + while (buffer.Length > 0 && buffer[^1] == '_') + { + buffer.Length--; + } + + return buffer.Length == 0 ? null : buffer.ToString(); + } + + private static ImmutableArray NormalizeContacts(IEnumerable? contacts) + { + if (contacts is null) + { + return ImmutableArray.Empty; + } + + var set = new SortedSet(StringComparer.Ordinal); + foreach (var contact in contacts) + { + if (string.IsNullOrWhiteSpace(contact)) + { + continue; + } + + var trimmed = contact.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + set.Add(trimmed); + } + + return set.Count == 0 ? ImmutableArray.Empty : set.ToImmutableArray(); + } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs b/src/StellaOps.Concelier.Models/AdvisoryProvenance.cs similarity index 95% rename from src/StellaOps.Feedser.Models/AdvisoryProvenance.cs rename to src/StellaOps.Concelier.Models/AdvisoryProvenance.cs index d70d4dce..77499f72 100644 --- a/src/StellaOps.Feedser.Models/AdvisoryProvenance.cs +++ b/src/StellaOps.Concelier.Models/AdvisoryProvenance.cs @@ -1,70 +1,70 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Models; - -/// -/// Describes the origin of a canonical field and how/when it was captured. -/// -public sealed record AdvisoryProvenance -{ - public static AdvisoryProvenance Empty { get; } = new("unknown", "unspecified", string.Empty, DateTimeOffset.UnixEpoch); - - [JsonConstructor] - public AdvisoryProvenance( - string source, - string kind, - string value, - string? decisionReason, - DateTimeOffset recordedAt, - ImmutableArray fieldMask) - : this(source, kind, value, recordedAt, fieldMask.IsDefault ? null : fieldMask.AsEnumerable(), decisionReason) - { - } - - public AdvisoryProvenance( - string source, - string kind, - string value, - DateTimeOffset recordedAt, - IEnumerable? fieldMask = null, - string? decisionReason = null) - { - Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)); - Kind = Validation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)); - Value = Validation.TrimToNull(value); - DecisionReason = Validation.TrimToNull(decisionReason); - RecordedAt = recordedAt.ToUniversalTime(); - FieldMask = NormalizeFieldMask(fieldMask); - } - - public string Source { get; } - - public string Kind { get; } - - public string? Value { get; } - - public string? DecisionReason { get; } - - public DateTimeOffset RecordedAt { get; } - - public ImmutableArray FieldMask { get; } - - private static ImmutableArray NormalizeFieldMask(IEnumerable? fieldMask) - { - if (fieldMask is null) - { - return ImmutableArray.Empty; - } - - var buffer = fieldMask - .Where(static value => !string.IsNullOrWhiteSpace(value)) - .Select(static value => value.Trim().ToLowerInvariant()) - .Distinct(StringComparer.Ordinal) - .OrderBy(static value => value, StringComparer.Ordinal) - .ToImmutableArray(); - - return buffer.IsDefault ? ImmutableArray.Empty : buffer; - } -} +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Concelier.Models; + +/// +/// Describes the origin of a canonical field and how/when it was captured. +/// +public sealed record AdvisoryProvenance +{ + public static AdvisoryProvenance Empty { get; } = new("unknown", "unspecified", string.Empty, DateTimeOffset.UnixEpoch); + + [JsonConstructor] + public AdvisoryProvenance( + string source, + string kind, + string value, + string? decisionReason, + DateTimeOffset recordedAt, + ImmutableArray fieldMask) + : this(source, kind, value, recordedAt, fieldMask.IsDefault ? null : fieldMask.AsEnumerable(), decisionReason) + { + } + + public AdvisoryProvenance( + string source, + string kind, + string value, + DateTimeOffset recordedAt, + IEnumerable? fieldMask = null, + string? decisionReason = null) + { + Source = Validation.EnsureNotNullOrWhiteSpace(source, nameof(source)); + Kind = Validation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)); + Value = Validation.TrimToNull(value); + DecisionReason = Validation.TrimToNull(decisionReason); + RecordedAt = recordedAt.ToUniversalTime(); + FieldMask = NormalizeFieldMask(fieldMask); + } + + public string Source { get; } + + public string Kind { get; } + + public string? Value { get; } + + public string? DecisionReason { get; } + + public DateTimeOffset RecordedAt { get; } + + public ImmutableArray FieldMask { get; } + + private static ImmutableArray NormalizeFieldMask(IEnumerable? fieldMask) + { + if (fieldMask is null) + { + return ImmutableArray.Empty; + } + + var buffer = fieldMask + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + + return buffer.IsDefault ? ImmutableArray.Empty : buffer; + } +} diff --git a/src/StellaOps.Feedser.Models/AdvisoryReference.cs b/src/StellaOps.Concelier.Models/AdvisoryReference.cs similarity index 93% rename from src/StellaOps.Feedser.Models/AdvisoryReference.cs rename to src/StellaOps.Concelier.Models/AdvisoryReference.cs index e173745c..0c414050 100644 --- a/src/StellaOps.Feedser.Models/AdvisoryReference.cs +++ b/src/StellaOps.Concelier.Models/AdvisoryReference.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Canonical external reference associated with an advisory. diff --git a/src/StellaOps.Feedser.Models/AdvisoryWeakness.cs b/src/StellaOps.Concelier.Models/AdvisoryWeakness.cs similarity index 98% rename from src/StellaOps.Feedser.Models/AdvisoryWeakness.cs rename to src/StellaOps.Concelier.Models/AdvisoryWeakness.cs index d498e7ba..c68a6330 100644 --- a/src/StellaOps.Feedser.Models/AdvisoryWeakness.cs +++ b/src/StellaOps.Concelier.Models/AdvisoryWeakness.cs @@ -3,7 +3,7 @@ using System.Collections.Immutable; using System.Linq; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Canonical weakness (e.g., CWE entry) associated with an advisory. diff --git a/src/StellaOps.Feedser.Models/AffectedPackage.cs b/src/StellaOps.Concelier.Models/AffectedPackage.cs similarity index 97% rename from src/StellaOps.Feedser.Models/AffectedPackage.cs rename to src/StellaOps.Concelier.Models/AffectedPackage.cs index a305ae02..922bcceb 100644 --- a/src/StellaOps.Feedser.Models/AffectedPackage.cs +++ b/src/StellaOps.Concelier.Models/AffectedPackage.cs @@ -4,7 +4,7 @@ using System.Collections.Immutable; using System.Linq; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Canonical affected package descriptor with deterministic ordering of ranges and provenance. diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs b/src/StellaOps.Concelier.Models/AffectedPackageStatus.cs similarity index 94% rename from src/StellaOps.Feedser.Models/AffectedPackageStatus.cs rename to src/StellaOps.Concelier.Models/AffectedPackageStatus.cs index 5ca773b7..3e875bb1 100644 --- a/src/StellaOps.Feedser.Models/AffectedPackageStatus.cs +++ b/src/StellaOps.Concelier.Models/AffectedPackageStatus.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Represents a vendor-supplied status tag for an affected package when a concrete version range is unavailable or supplementary. diff --git a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs b/src/StellaOps.Concelier.Models/AffectedPackageStatusCatalog.cs similarity index 96% rename from src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs rename to src/StellaOps.Concelier.Models/AffectedPackageStatusCatalog.cs index 905fbee1..76946b0c 100644 --- a/src/StellaOps.Feedser.Models/AffectedPackageStatusCatalog.cs +++ b/src/StellaOps.Concelier.Models/AffectedPackageStatusCatalog.cs @@ -1,157 +1,157 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; - -namespace StellaOps.Feedser.Models; - -/// -/// Central registry of allowed affected-package status labels to keep connectors consistent. -/// -public static class AffectedPackageStatusCatalog -{ - public const string KnownAffected = "known_affected"; - public const string KnownNotAffected = "known_not_affected"; - public const string UnderInvestigation = "under_investigation"; - public const string Fixed = "fixed"; - public const string FirstFixed = "first_fixed"; - public const string Mitigated = "mitigated"; - public const string NotApplicable = "not_applicable"; - public const string Affected = "affected"; - public const string NotAffected = "not_affected"; - public const string Pending = "pending"; - public const string Unknown = "unknown"; - - private static readonly string[] CanonicalStatuses = - { - KnownAffected, - KnownNotAffected, - UnderInvestigation, - Fixed, - FirstFixed, - Mitigated, - NotApplicable, - Affected, - NotAffected, - Pending, - Unknown, - }; - - private static readonly IReadOnlyList AllowedStatuses = Array.AsReadOnly(CanonicalStatuses); - - private static readonly IReadOnlyDictionary StatusMap = BuildStatusMap(); - - public static IReadOnlyList Allowed => AllowedStatuses; - - public static string Normalize(string status) - { - if (!TryNormalize(status, out var normalized)) - { - throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary."); - } - - return normalized; - } - - public static bool TryNormalize(string? status, [NotNullWhen(true)] out string? normalized) - { - normalized = null; - - if (string.IsNullOrWhiteSpace(status)) - { - return false; - } - - var token = Sanitize(status); - if (token.Length == 0) - { - return false; - } - - if (!StatusMap.TryGetValue(token, out normalized)) - { - return false; - } - - return true; - } - - public static bool IsAllowed(string? status) - => TryNormalize(status, out _); - - private static IReadOnlyDictionary BuildStatusMap() - { - var map = new Dictionary(StringComparer.Ordinal); - foreach (var status in CanonicalStatuses) - { - map[Sanitize(status)] = status; - } - - Add(map, "known not vulnerable", KnownNotAffected); - Add(map, "known unaffected", KnownNotAffected); - Add(map, "known not impacted", KnownNotAffected); - Add(map, "vulnerable", Affected); - Add(map, "impacted", Affected); - Add(map, "impacting", Affected); - Add(map, "not vulnerable", NotAffected); - Add(map, "unaffected", NotAffected); - Add(map, "not impacted", NotAffected); - Add(map, "no impact", NotAffected); - Add(map, "impact free", NotAffected); - Add(map, "investigating", UnderInvestigation); - Add(map, "analysis in progress", UnderInvestigation); - Add(map, "analysis pending", UnderInvestigation); - Add(map, "open", UnderInvestigation); - Add(map, "patch available", Fixed); - Add(map, "fix available", Fixed); - Add(map, "patched", Fixed); - Add(map, "resolved", Fixed); - Add(map, "remediated", Fixed); - Add(map, "workaround available", Mitigated); - Add(map, "mitigation available", Mitigated); - Add(map, "mitigation provided", Mitigated); - Add(map, "not applicable", NotApplicable); - Add(map, "n/a", NotApplicable); - Add(map, "na", NotApplicable); - Add(map, "does not apply", NotApplicable); - Add(map, "out of scope", NotApplicable); - Add(map, "pending fix", Pending); - Add(map, "awaiting fix", Pending); - Add(map, "awaiting patch", Pending); - Add(map, "scheduled", Pending); - Add(map, "planned", Pending); - Add(map, "tbd", Unknown); - Add(map, "to be determined", Unknown); - Add(map, "undetermined", Unknown); - Add(map, "not yet known", Unknown); - - return map; - } - - private static void Add(IDictionary map, string alias, string canonical) - { - var key = Sanitize(alias); - if (key.Length == 0) - { - return; - } - - map[key] = canonical; - } - - private static string Sanitize(string value) - { - var span = value.AsSpan(); - var buffer = new char[span.Length]; - var index = 0; - - foreach (var ch in span) - { - if (char.IsLetterOrDigit(ch)) - { - buffer[index++] = char.ToLowerInvariant(ch); - } - } - - return index == 0 ? string.Empty : new string(buffer, 0, index); - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace StellaOps.Concelier.Models; + +/// +/// Central registry of allowed affected-package status labels to keep connectors consistent. +/// +public static class AffectedPackageStatusCatalog +{ + public const string KnownAffected = "known_affected"; + public const string KnownNotAffected = "known_not_affected"; + public const string UnderInvestigation = "under_investigation"; + public const string Fixed = "fixed"; + public const string FirstFixed = "first_fixed"; + public const string Mitigated = "mitigated"; + public const string NotApplicable = "not_applicable"; + public const string Affected = "affected"; + public const string NotAffected = "not_affected"; + public const string Pending = "pending"; + public const string Unknown = "unknown"; + + private static readonly string[] CanonicalStatuses = + { + KnownAffected, + KnownNotAffected, + UnderInvestigation, + Fixed, + FirstFixed, + Mitigated, + NotApplicable, + Affected, + NotAffected, + Pending, + Unknown, + }; + + private static readonly IReadOnlyList AllowedStatuses = Array.AsReadOnly(CanonicalStatuses); + + private static readonly IReadOnlyDictionary StatusMap = BuildStatusMap(); + + public static IReadOnlyList Allowed => AllowedStatuses; + + public static string Normalize(string status) + { + if (!TryNormalize(status, out var normalized)) + { + throw new ArgumentOutOfRangeException(nameof(status), status, "Status is not part of the allowed affected-package status glossary."); + } + + return normalized; + } + + public static bool TryNormalize(string? status, [NotNullWhen(true)] out string? normalized) + { + normalized = null; + + if (string.IsNullOrWhiteSpace(status)) + { + return false; + } + + var token = Sanitize(status); + if (token.Length == 0) + { + return false; + } + + if (!StatusMap.TryGetValue(token, out normalized)) + { + return false; + } + + return true; + } + + public static bool IsAllowed(string? status) + => TryNormalize(status, out _); + + private static IReadOnlyDictionary BuildStatusMap() + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var status in CanonicalStatuses) + { + map[Sanitize(status)] = status; + } + + Add(map, "known not vulnerable", KnownNotAffected); + Add(map, "known unaffected", KnownNotAffected); + Add(map, "known not impacted", KnownNotAffected); + Add(map, "vulnerable", Affected); + Add(map, "impacted", Affected); + Add(map, "impacting", Affected); + Add(map, "not vulnerable", NotAffected); + Add(map, "unaffected", NotAffected); + Add(map, "not impacted", NotAffected); + Add(map, "no impact", NotAffected); + Add(map, "impact free", NotAffected); + Add(map, "investigating", UnderInvestigation); + Add(map, "analysis in progress", UnderInvestigation); + Add(map, "analysis pending", UnderInvestigation); + Add(map, "open", UnderInvestigation); + Add(map, "patch available", Fixed); + Add(map, "fix available", Fixed); + Add(map, "patched", Fixed); + Add(map, "resolved", Fixed); + Add(map, "remediated", Fixed); + Add(map, "workaround available", Mitigated); + Add(map, "mitigation available", Mitigated); + Add(map, "mitigation provided", Mitigated); + Add(map, "not applicable", NotApplicable); + Add(map, "n/a", NotApplicable); + Add(map, "na", NotApplicable); + Add(map, "does not apply", NotApplicable); + Add(map, "out of scope", NotApplicable); + Add(map, "pending fix", Pending); + Add(map, "awaiting fix", Pending); + Add(map, "awaiting patch", Pending); + Add(map, "scheduled", Pending); + Add(map, "planned", Pending); + Add(map, "tbd", Unknown); + Add(map, "to be determined", Unknown); + Add(map, "undetermined", Unknown); + Add(map, "not yet known", Unknown); + + return map; + } + + private static void Add(IDictionary map, string alias, string canonical) + { + var key = Sanitize(alias); + if (key.Length == 0) + { + return; + } + + map[key] = canonical; + } + + private static string Sanitize(string value) + { + var span = value.AsSpan(); + var buffer = new char[span.Length]; + var index = 0; + + foreach (var ch in span) + { + if (char.IsLetterOrDigit(ch)) + { + buffer[index++] = char.ToLowerInvariant(ch); + } + } + + return index == 0 ? string.Empty : new string(buffer, 0, index); + } +} diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs b/src/StellaOps.Concelier.Models/AffectedVersionRange.cs similarity index 96% rename from src/StellaOps.Feedser.Models/AffectedVersionRange.cs rename to src/StellaOps.Concelier.Models/AffectedVersionRange.cs index a71e1224..e0e5d236 100644 --- a/src/StellaOps.Feedser.Models/AffectedVersionRange.cs +++ b/src/StellaOps.Concelier.Models/AffectedVersionRange.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Describes a contiguous range of versions impacted by an advisory. diff --git a/src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs b/src/StellaOps.Concelier.Models/AffectedVersionRangeExtensions.cs similarity index 96% rename from src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs rename to src/StellaOps.Concelier.Models/AffectedVersionRangeExtensions.cs index 6cc951cd..f6e99864 100644 --- a/src/StellaOps.Feedser.Models/AffectedVersionRangeExtensions.cs +++ b/src/StellaOps.Concelier.Models/AffectedVersionRangeExtensions.cs @@ -1,221 +1,221 @@ -using System; - -namespace StellaOps.Feedser.Models; - -/// -/// Helpers for deriving normalized version rules from affected version ranges. -/// -public static class AffectedVersionRangeExtensions -{ - public static NormalizedVersionRule? ToNormalizedVersionRule(this AffectedVersionRange? range, string? notes = null) - { - if (range is null) - { - return null; - } - - var primitives = range.Primitives; - - var semVerRule = primitives?.SemVer?.ToNormalizedVersionRule(notes); - if (semVerRule is not null) - { - return semVerRule; - } - - var nevraRule = primitives?.Nevra?.ToNormalizedVersionRule(notes); - if (nevraRule is not null) - { - return nevraRule; - } - - var evrRule = primitives?.Evr?.ToNormalizedVersionRule(notes); - if (evrRule is not null) - { - return evrRule; - } - - var scheme = Validation.TrimToNull(range.RangeKind)?.ToLowerInvariant(); - return scheme switch - { - NormalizedVersionSchemes.SemVer => BuildSemVerFallback(range, notes), - NormalizedVersionSchemes.Nevra => BuildNevraFallback(range, notes), - NormalizedVersionSchemes.Evr => BuildEvrFallback(range, notes), - _ => null, - }; - } - - private static NormalizedVersionRule? BuildSemVerFallback(AffectedVersionRange range, string? notes) - { - var min = Validation.TrimToNull(range.IntroducedVersion); - var max = Validation.TrimToNull(range.FixedVersion); - var last = Validation.TrimToNull(range.LastAffectedVersion); - var resolvedNotes = Validation.TrimToNull(notes); - - if (string.IsNullOrEmpty(min) && string.IsNullOrEmpty(max) && string.IsNullOrEmpty(last)) - { - return null; - } - - if (!string.IsNullOrEmpty(max)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.SemVer, - NormalizedVersionRuleTypes.Range, - min: min, - minInclusive: min is null ? null : true, - max: max, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(last)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.SemVer, - NormalizedVersionRuleTypes.LessThanOrEqual, - max: last, - maxInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(min)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.SemVer, - NormalizedVersionRuleTypes.GreaterThanOrEqual, - min: min, - minInclusive: true, - notes: resolvedNotes); - } - - return null; - } - - private static NormalizedVersionRule? BuildNevraFallback(AffectedVersionRange range, string? notes) - { - var resolvedNotes = Validation.TrimToNull(notes); - var introduced = Validation.TrimToNull(range.IntroducedVersion); - var fixedVersion = Validation.TrimToNull(range.FixedVersion); - var lastAffected = Validation.TrimToNull(range.LastAffectedVersion); - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Nevra, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Nevra, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Nevra, - NormalizedVersionRuleTypes.GreaterThanOrEqual, - min: introduced, - minInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Nevra, - NormalizedVersionRuleTypes.LessThan, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Nevra, - NormalizedVersionRuleTypes.LessThanOrEqual, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - return null; - } - - private static NormalizedVersionRule? BuildEvrFallback(AffectedVersionRange range, string? notes) - { - var resolvedNotes = Validation.TrimToNull(notes); - var introduced = Validation.TrimToNull(range.IntroducedVersion); - var fixedVersion = Validation.TrimToNull(range.FixedVersion); - var lastAffected = Validation.TrimToNull(range.LastAffectedVersion); - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Evr, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Evr, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Evr, - NormalizedVersionRuleTypes.GreaterThanOrEqual, - min: introduced, - minInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Evr, - NormalizedVersionRuleTypes.LessThan, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - NormalizedVersionSchemes.Evr, - NormalizedVersionRuleTypes.LessThanOrEqual, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - return null; - } -} +using System; + +namespace StellaOps.Concelier.Models; + +/// +/// Helpers for deriving normalized version rules from affected version ranges. +/// +public static class AffectedVersionRangeExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this AffectedVersionRange? range, string? notes = null) + { + if (range is null) + { + return null; + } + + var primitives = range.Primitives; + + var semVerRule = primitives?.SemVer?.ToNormalizedVersionRule(notes); + if (semVerRule is not null) + { + return semVerRule; + } + + var nevraRule = primitives?.Nevra?.ToNormalizedVersionRule(notes); + if (nevraRule is not null) + { + return nevraRule; + } + + var evrRule = primitives?.Evr?.ToNormalizedVersionRule(notes); + if (evrRule is not null) + { + return evrRule; + } + + var scheme = Validation.TrimToNull(range.RangeKind)?.ToLowerInvariant(); + return scheme switch + { + NormalizedVersionSchemes.SemVer => BuildSemVerFallback(range, notes), + NormalizedVersionSchemes.Nevra => BuildNevraFallback(range, notes), + NormalizedVersionSchemes.Evr => BuildEvrFallback(range, notes), + _ => null, + }; + } + + private static NormalizedVersionRule? BuildSemVerFallback(AffectedVersionRange range, string? notes) + { + var min = Validation.TrimToNull(range.IntroducedVersion); + var max = Validation.TrimToNull(range.FixedVersion); + var last = Validation.TrimToNull(range.LastAffectedVersion); + var resolvedNotes = Validation.TrimToNull(notes); + + if (string.IsNullOrEmpty(min) && string.IsNullOrEmpty(max) && string.IsNullOrEmpty(last)) + { + return null; + } + + if (!string.IsNullOrEmpty(max)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.Range, + min: min, + minInclusive: min is null ? null : true, + max: max, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(last)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: last, + maxInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(min)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.SemVer, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: min, + minInclusive: true, + notes: resolvedNotes); + } + + return null; + } + + private static NormalizedVersionRule? BuildNevraFallback(AffectedVersionRange range, string? notes) + { + var resolvedNotes = Validation.TrimToNull(notes); + var introduced = Validation.TrimToNull(range.IntroducedVersion); + var fixedVersion = Validation.TrimToNull(range.FixedVersion); + var lastAffected = Validation.TrimToNull(range.LastAffectedVersion); + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Nevra, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Nevra, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Nevra, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: introduced, + minInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Nevra, + NormalizedVersionRuleTypes.LessThan, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Nevra, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + return null; + } + + private static NormalizedVersionRule? BuildEvrFallback(AffectedVersionRange range, string? notes) + { + var resolvedNotes = Validation.TrimToNull(notes); + var introduced = Validation.TrimToNull(range.IntroducedVersion); + var fixedVersion = Validation.TrimToNull(range.FixedVersion); + var lastAffected = Validation.TrimToNull(range.LastAffectedVersion); + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Evr, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Evr, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Evr, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: introduced, + minInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Evr, + NormalizedVersionRuleTypes.LessThan, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + NormalizedVersionSchemes.Evr, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs b/src/StellaOps.Concelier.Models/AliasSchemeRegistry.cs similarity index 97% rename from src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs rename to src/StellaOps.Concelier.Models/AliasSchemeRegistry.cs index 5e5c01e5..7efd509b 100644 --- a/src/StellaOps.Feedser.Models/AliasSchemeRegistry.cs +++ b/src/StellaOps.Concelier.Models/AliasSchemeRegistry.cs @@ -4,7 +4,7 @@ using System.Collections.Immutable; using System.Linq; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; public static class AliasSchemeRegistry { diff --git a/src/StellaOps.Feedser.Models/AliasSchemes.cs b/src/StellaOps.Concelier.Models/AliasSchemes.cs similarity index 93% rename from src/StellaOps.Feedser.Models/AliasSchemes.cs rename to src/StellaOps.Concelier.Models/AliasSchemes.cs index 8227212a..ca60793d 100644 --- a/src/StellaOps.Feedser.Models/AliasSchemes.cs +++ b/src/StellaOps.Concelier.Models/AliasSchemes.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Well-known alias scheme identifiers referenced throughout the pipeline. diff --git a/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md b/src/StellaOps.Concelier.Models/BACKWARD_COMPATIBILITY.md similarity index 92% rename from src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md rename to src/StellaOps.Concelier.Models/BACKWARD_COMPATIBILITY.md index 5cee4f67..09e97ed9 100644 --- a/src/StellaOps.Feedser.Models/BACKWARD_COMPATIBILITY.md +++ b/src/StellaOps.Concelier.Models/BACKWARD_COMPATIBILITY.md @@ -1,7 +1,7 @@ # Canonical Model Backward-Compatibility Playbook This playbook captures the policies and workflow required when evolving the canonical -`StellaOps.Feedser.Models` surface. +`StellaOps.Concelier.Models` surface. ## Principles @@ -33,7 +33,7 @@ This playbook captures the policies and workflow required when evolving the cano ## Testing Checklist -- `StellaOps.Feedser.Models.Tests` – update unit tests and golden examples. +- `StellaOps.Concelier.Models.Tests` – update unit tests and golden examples. - `Serialization determinism` – ensure the hash regression tests cover the new fields. - Exporter integration (`Json`, `TrivyDb`) – confirm manifests include provenance + tree metadata for the new shape. diff --git a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md b/src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md similarity index 97% rename from src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md rename to src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md index 0eaf32f4..4da20b58 100644 --- a/src/StellaOps.Feedser.Models/CANONICAL_RECORDS.md +++ b/src/StellaOps.Concelier.Models/CANONICAL_RECORDS.md @@ -1,7 +1,7 @@ # Canonical Record Definitions -> Source of truth for the normalized advisory schema emitted by `StellaOps.Feedser.Models`. -> Keep this document in sync with the public record types under `StellaOps.Feedser.Models` and +> Source of truth for the normalized advisory schema emitted by `StellaOps.Concelier.Models`. +> Keep this document in sync with the public record types under `StellaOps.Concelier.Models` and > update it whenever a new field is introduced or semantics change. ## Advisory diff --git a/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs b/src/StellaOps.Concelier.Models/CanonicalJsonSerializer.cs similarity index 96% rename from src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs rename to src/StellaOps.Concelier.Models/CanonicalJsonSerializer.cs index 337fe129..dd0a2325 100644 --- a/src/StellaOps.Feedser.Models/CanonicalJsonSerializer.cs +++ b/src/StellaOps.Concelier.Models/CanonicalJsonSerializer.cs @@ -1,175 +1,175 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Metadata; - -namespace StellaOps.Feedser.Models; - -/// -/// Deterministic JSON serializer tuned for canonical advisory output. -/// -public static class CanonicalJsonSerializer -{ - private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); - private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); - - private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary - { - { - typeof(AdvisoryProvenance), - new[] - { - "source", - "kind", - "value", - "decisionReason", - "recordedAt", - "fieldMask", - } - }, - { - typeof(AffectedPackage), - new[] - { - "type", - "identifier", - "platform", - "versionRanges", - "normalizedVersions", - "statuses", - "provenance", - } - }, - { - typeof(AdvisoryCredit), - new[] - { - "displayName", - "role", - "contacts", - "provenance", - } - }, - { - typeof(NormalizedVersionRule), - new[] - { - "scheme", - "type", - "min", - "minInclusive", - "max", - "maxInclusive", - "value", - "notes", - } - }, - { - typeof(AdvisoryWeakness), - new[] - { - "taxonomy", - "identifier", - "name", - "uri", - "provenance", - } - }, - }; - - public static string Serialize(T value) - => JsonSerializer.Serialize(value, CompactOptions); - - public static string SerializeIndented(T value) - => JsonSerializer.Serialize(value, PrettyOptions); - - public static Advisory Normalize(Advisory advisory) - => new( - advisory.AdvisoryKey, - advisory.Title, - advisory.Summary, - advisory.Language, - advisory.Published, - advisory.Modified, - advisory.Severity, - advisory.ExploitKnown, - advisory.Aliases, - advisory.Credits, - advisory.References, - advisory.AffectedPackages, - advisory.CvssMetrics, - advisory.Provenance, - advisory.Description, - advisory.Cwes, - advisory.CanonicalMetricId); - - public static T Deserialize(string json) - => JsonSerializer.Deserialize(json, PrettyOptions)! - ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); - - private static JsonSerializerOptions CreateOptions(bool writeIndented) - { - var options = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.Never, - WriteIndented = writeIndented, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - - var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); - options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); - options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); - return options; - } - - private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver - { - private readonly IJsonTypeInfoResolver _inner; - - public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) - { - _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - } - - public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) - { - var info = _inner.GetTypeInfo(type, options); - if (info is null) - { - throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); - } - - if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) - { - var ordered = info.Properties - .OrderBy(property => GetPropertyOrder(type, property.Name)) - .ThenBy(property => property.Name, StringComparer.Ordinal) - .ToArray(); - - info.Properties.Clear(); - foreach (var property in ordered) - { - info.Properties.Add(property); - } - } - - return info; - } - - private static int GetPropertyOrder(Type type, string propertyName) - { - if (PropertyOrderOverrides.TryGetValue(type, out var order) && - Array.IndexOf(order, propertyName) is var index && - index >= 0) - { - return index; - } - - return int.MaxValue; - } - } -} +using System.Collections.Generic; +using System.Linq; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Concelier.Models; + +/// +/// Deterministic JSON serializer tuned for canonical advisory output. +/// +public static class CanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary + { + { + typeof(AdvisoryProvenance), + new[] + { + "source", + "kind", + "value", + "decisionReason", + "recordedAt", + "fieldMask", + } + }, + { + typeof(AffectedPackage), + new[] + { + "type", + "identifier", + "platform", + "versionRanges", + "normalizedVersions", + "statuses", + "provenance", + } + }, + { + typeof(AdvisoryCredit), + new[] + { + "displayName", + "role", + "contacts", + "provenance", + } + }, + { + typeof(NormalizedVersionRule), + new[] + { + "scheme", + "type", + "min", + "minInclusive", + "max", + "maxInclusive", + "value", + "notes", + } + }, + { + typeof(AdvisoryWeakness), + new[] + { + "taxonomy", + "identifier", + "name", + "uri", + "provenance", + } + }, + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static Advisory Normalize(Advisory advisory) + => new( + advisory.AdvisoryKey, + advisory.Title, + advisory.Summary, + advisory.Language, + advisory.Published, + advisory.Modified, + advisory.Severity, + advisory.ExploitKnown, + advisory.Aliases, + advisory.Credits, + advisory.References, + advisory.AffectedPackages, + advisory.CvssMetrics, + advisory.Provenance, + advisory.Description, + advisory.Cwes, + advisory.CanonicalMetricId); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, PrettyOptions)! + ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options); + if (info is null) + { + throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + } + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetPropertyOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetPropertyOrder(Type type, string propertyName) + { + if (PropertyOrderOverrides.TryGetValue(type, out var order) && + Array.IndexOf(order, propertyName) is var index && + index >= 0) + { + return index; + } + + return int.MaxValue; + } + } +} diff --git a/src/StellaOps.Feedser.Models/CvssMetric.cs b/src/StellaOps.Concelier.Models/CvssMetric.cs similarity index 94% rename from src/StellaOps.Feedser.Models/CvssMetric.cs rename to src/StellaOps.Concelier.Models/CvssMetric.cs index 90765754..c7dacf06 100644 --- a/src/StellaOps.Feedser.Models/CvssMetric.cs +++ b/src/StellaOps.Concelier.Models/CvssMetric.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Canonicalized CVSS metric details supporting deterministic serialization. diff --git a/src/StellaOps.Feedser.Models/EvrPrimitiveExtensions.cs b/src/StellaOps.Concelier.Models/EvrPrimitiveExtensions.cs similarity index 95% rename from src/StellaOps.Feedser.Models/EvrPrimitiveExtensions.cs rename to src/StellaOps.Concelier.Models/EvrPrimitiveExtensions.cs index f979ccd6..b010a98c 100644 --- a/src/StellaOps.Feedser.Models/EvrPrimitiveExtensions.cs +++ b/src/StellaOps.Concelier.Models/EvrPrimitiveExtensions.cs @@ -1,87 +1,87 @@ -namespace StellaOps.Feedser.Models; - -/// -/// Helper extensions for converting instances into normalized rules. -/// -public static class EvrPrimitiveExtensions -{ - public static NormalizedVersionRule? ToNormalizedVersionRule(this EvrPrimitive? primitive, string? notes = null) - { - if (primitive is null) - { - return null; - } - - var resolvedNotes = Validation.TrimToNull(notes); - var introduced = Normalize(primitive.Introduced); - var fixedVersion = Normalize(primitive.Fixed); - var lastAffected = Normalize(primitive.LastAffected); - var scheme = NormalizedVersionSchemes.Evr; - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.GreaterThanOrEqual, - min: introduced, - minInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.LessThan, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.LessThanOrEqual, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - return null; - } - - private static string? Normalize(EvrComponent? component) - { - if (component is null) - { - return null; - } - - return Validation.TrimToNull(component.ToCanonicalString()); - } -} +namespace StellaOps.Concelier.Models; + +/// +/// Helper extensions for converting instances into normalized rules. +/// +public static class EvrPrimitiveExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this EvrPrimitive? primitive, string? notes = null) + { + if (primitive is null) + { + return null; + } + + var resolvedNotes = Validation.TrimToNull(notes); + var introduced = Normalize(primitive.Introduced); + var fixedVersion = Normalize(primitive.Fixed); + var lastAffected = Normalize(primitive.LastAffected); + var scheme = NormalizedVersionSchemes.Evr; + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: introduced, + minInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.LessThan, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + return null; + } + + private static string? Normalize(EvrComponent? component) + { + if (component is null) + { + return null; + } + + return Validation.TrimToNull(component.ToCanonicalString()); + } +} diff --git a/src/StellaOps.Feedser.Models/NevraPrimitiveExtensions.cs b/src/StellaOps.Concelier.Models/NevraPrimitiveExtensions.cs similarity index 95% rename from src/StellaOps.Feedser.Models/NevraPrimitiveExtensions.cs rename to src/StellaOps.Concelier.Models/NevraPrimitiveExtensions.cs index f5b5550a..860a3ad2 100644 --- a/src/StellaOps.Feedser.Models/NevraPrimitiveExtensions.cs +++ b/src/StellaOps.Concelier.Models/NevraPrimitiveExtensions.cs @@ -1,87 +1,87 @@ -namespace StellaOps.Feedser.Models; - -/// -/// Helper extensions for converting instances into normalized rules. -/// -public static class NevraPrimitiveExtensions -{ - public static NormalizedVersionRule? ToNormalizedVersionRule(this NevraPrimitive? primitive, string? notes = null) - { - if (primitive is null) - { - return null; - } - - var resolvedNotes = Validation.TrimToNull(notes); - var introduced = Normalize(primitive.Introduced); - var fixedVersion = Normalize(primitive.Fixed); - var lastAffected = Normalize(primitive.LastAffected); - var scheme = NormalizedVersionSchemes.Nevra; - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: true, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.GreaterThanOrEqual, - min: introduced, - minInclusive: true, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.LessThan, - max: fixedVersion, - maxInclusive: false, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.LessThanOrEqual, - max: lastAffected, - maxInclusive: true, - notes: resolvedNotes); - } - - return null; - } - - private static string? Normalize(NevraComponent? component) - { - if (component is null) - { - return null; - } - - return Validation.TrimToNull(component.ToCanonicalString()); - } -} +namespace StellaOps.Concelier.Models; + +/// +/// Helper extensions for converting instances into normalized rules. +/// +public static class NevraPrimitiveExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this NevraPrimitive? primitive, string? notes = null) + { + if (primitive is null) + { + return null; + } + + var resolvedNotes = Validation.TrimToNull(notes); + var introduced = Normalize(primitive.Introduced); + var fixedVersion = Normalize(primitive.Fixed); + var lastAffected = Normalize(primitive.LastAffected); + var scheme = NormalizedVersionSchemes.Nevra; + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: true, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.GreaterThanOrEqual, + min: introduced, + minInclusive: true, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.LessThan, + max: fixedVersion, + maxInclusive: false, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.LessThanOrEqual, + max: lastAffected, + maxInclusive: true, + notes: resolvedNotes); + } + + return null; + } + + private static string? Normalize(NevraComponent? component) + { + if (component is null) + { + return null; + } + + return Validation.TrimToNull(component.ToCanonicalString()); + } +} diff --git a/src/StellaOps.Feedser.Models/NormalizedVersionRule.cs b/src/StellaOps.Concelier.Models/NormalizedVersionRule.cs similarity index 95% rename from src/StellaOps.Feedser.Models/NormalizedVersionRule.cs rename to src/StellaOps.Concelier.Models/NormalizedVersionRule.cs index 86406b34..9ca35465 100644 --- a/src/StellaOps.Feedser.Models/NormalizedVersionRule.cs +++ b/src/StellaOps.Concelier.Models/NormalizedVersionRule.cs @@ -1,185 +1,185 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Models; - -/// -/// Canonical normalized version rule emitted by range builders for analytical queries. -/// -public sealed record NormalizedVersionRule -{ - public NormalizedVersionRule( - string scheme, - string type, - string? min = null, - bool? minInclusive = null, - string? max = null, - bool? maxInclusive = null, - string? value = null, - string? notes = null) - { - Scheme = Validation.EnsureNotNullOrWhiteSpace(scheme, nameof(scheme)).ToLowerInvariant(); - Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).Replace('_', '-').ToLowerInvariant(); - Min = Validation.TrimToNull(min); - MinInclusive = minInclusive; - Max = Validation.TrimToNull(max); - MaxInclusive = maxInclusive; - Value = Validation.TrimToNull(value); - Notes = Validation.TrimToNull(notes); - } - - public string Scheme { get; } - - public string Type { get; } - - public string? Min { get; } - - public bool? MinInclusive { get; } - - public string? Max { get; } - - public bool? MaxInclusive { get; } - - public string? Value { get; } - - public string? Notes { get; } -} - -public sealed class NormalizedVersionRuleEqualityComparer : IEqualityComparer -{ - public static NormalizedVersionRuleEqualityComparer Instance { get; } = new(); - - public bool Equals(NormalizedVersionRule? x, NormalizedVersionRule? y) - { - if (ReferenceEquals(x, y)) - { - return true; - } - - if (x is null || y is null) - { - return false; - } - - return string.Equals(x.Scheme, y.Scheme, StringComparison.Ordinal) - && string.Equals(x.Type, y.Type, StringComparison.Ordinal) - && string.Equals(x.Min, y.Min, StringComparison.Ordinal) - && x.MinInclusive == y.MinInclusive - && string.Equals(x.Max, y.Max, StringComparison.Ordinal) - && x.MaxInclusive == y.MaxInclusive - && string.Equals(x.Value, y.Value, StringComparison.Ordinal) - && string.Equals(x.Notes, y.Notes, StringComparison.Ordinal); - } - - public int GetHashCode(NormalizedVersionRule obj) - => HashCode.Combine( - obj.Scheme, - obj.Type, - obj.Min, - obj.MinInclusive, - obj.Max, - obj.MaxInclusive, - obj.Value, - obj.Notes); -} - -public sealed class NormalizedVersionRuleComparer : IComparer -{ - public static NormalizedVersionRuleComparer Instance { get; } = new(); - - public int Compare(NormalizedVersionRule? x, NormalizedVersionRule? y) - { - if (ReferenceEquals(x, y)) - { - return 0; - } - - if (x is null) - { - return -1; - } - - if (y is null) - { - return 1; - } - - var schemeComparison = string.Compare(x.Scheme, y.Scheme, StringComparison.Ordinal); - if (schemeComparison != 0) - { - return schemeComparison; - } - - var typeComparison = string.Compare(x.Type, y.Type, StringComparison.Ordinal); - if (typeComparison != 0) - { - return typeComparison; - } - - var minComparison = string.Compare(x.Min, y.Min, StringComparison.Ordinal); - if (minComparison != 0) - { - return minComparison; - } - - var minInclusiveComparison = NullableBoolCompare(x.MinInclusive, y.MinInclusive); - if (minInclusiveComparison != 0) - { - return minInclusiveComparison; - } - - var maxComparison = string.Compare(x.Max, y.Max, StringComparison.Ordinal); - if (maxComparison != 0) - { - return maxComparison; - } - - var maxInclusiveComparison = NullableBoolCompare(x.MaxInclusive, y.MaxInclusive); - if (maxInclusiveComparison != 0) - { - return maxInclusiveComparison; - } - - var valueComparison = string.Compare(x.Value, y.Value, StringComparison.Ordinal); - if (valueComparison != 0) - { - return valueComparison; - } - - return string.Compare(x.Notes, y.Notes, StringComparison.Ordinal); - } - - private static int NullableBoolCompare(bool? x, bool? y) - { - if (x == y) - { - return 0; - } - - return (x, y) switch - { - (null, not null) => -1, - (not null, null) => 1, - (false, true) => -1, - (true, false) => 1, - _ => 0, - }; - } -} - -public static class NormalizedVersionSchemes -{ - public const string SemVer = "semver"; - public const string Nevra = "nevra"; - public const string Evr = "evr"; -} - -public static class NormalizedVersionRuleTypes -{ - public const string Range = "range"; - public const string Exact = "exact"; - public const string LessThan = "lt"; - public const string LessThanOrEqual = "lte"; - public const string GreaterThan = "gt"; - public const string GreaterThanOrEqual = "gte"; -} +using System; +using System.Collections.Generic; + +namespace StellaOps.Concelier.Models; + +/// +/// Canonical normalized version rule emitted by range builders for analytical queries. +/// +public sealed record NormalizedVersionRule +{ + public NormalizedVersionRule( + string scheme, + string type, + string? min = null, + bool? minInclusive = null, + string? max = null, + bool? maxInclusive = null, + string? value = null, + string? notes = null) + { + Scheme = Validation.EnsureNotNullOrWhiteSpace(scheme, nameof(scheme)).ToLowerInvariant(); + Type = Validation.EnsureNotNullOrWhiteSpace(type, nameof(type)).Replace('_', '-').ToLowerInvariant(); + Min = Validation.TrimToNull(min); + MinInclusive = minInclusive; + Max = Validation.TrimToNull(max); + MaxInclusive = maxInclusive; + Value = Validation.TrimToNull(value); + Notes = Validation.TrimToNull(notes); + } + + public string Scheme { get; } + + public string Type { get; } + + public string? Min { get; } + + public bool? MinInclusive { get; } + + public string? Max { get; } + + public bool? MaxInclusive { get; } + + public string? Value { get; } + + public string? Notes { get; } +} + +public sealed class NormalizedVersionRuleEqualityComparer : IEqualityComparer +{ + public static NormalizedVersionRuleEqualityComparer Instance { get; } = new(); + + public bool Equals(NormalizedVersionRule? x, NormalizedVersionRule? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Scheme, y.Scheme, StringComparison.Ordinal) + && string.Equals(x.Type, y.Type, StringComparison.Ordinal) + && string.Equals(x.Min, y.Min, StringComparison.Ordinal) + && x.MinInclusive == y.MinInclusive + && string.Equals(x.Max, y.Max, StringComparison.Ordinal) + && x.MaxInclusive == y.MaxInclusive + && string.Equals(x.Value, y.Value, StringComparison.Ordinal) + && string.Equals(x.Notes, y.Notes, StringComparison.Ordinal); + } + + public int GetHashCode(NormalizedVersionRule obj) + => HashCode.Combine( + obj.Scheme, + obj.Type, + obj.Min, + obj.MinInclusive, + obj.Max, + obj.MaxInclusive, + obj.Value, + obj.Notes); +} + +public sealed class NormalizedVersionRuleComparer : IComparer +{ + public static NormalizedVersionRuleComparer Instance { get; } = new(); + + public int Compare(NormalizedVersionRule? x, NormalizedVersionRule? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var schemeComparison = string.Compare(x.Scheme, y.Scheme, StringComparison.Ordinal); + if (schemeComparison != 0) + { + return schemeComparison; + } + + var typeComparison = string.Compare(x.Type, y.Type, StringComparison.Ordinal); + if (typeComparison != 0) + { + return typeComparison; + } + + var minComparison = string.Compare(x.Min, y.Min, StringComparison.Ordinal); + if (minComparison != 0) + { + return minComparison; + } + + var minInclusiveComparison = NullableBoolCompare(x.MinInclusive, y.MinInclusive); + if (minInclusiveComparison != 0) + { + return minInclusiveComparison; + } + + var maxComparison = string.Compare(x.Max, y.Max, StringComparison.Ordinal); + if (maxComparison != 0) + { + return maxComparison; + } + + var maxInclusiveComparison = NullableBoolCompare(x.MaxInclusive, y.MaxInclusive); + if (maxInclusiveComparison != 0) + { + return maxInclusiveComparison; + } + + var valueComparison = string.Compare(x.Value, y.Value, StringComparison.Ordinal); + if (valueComparison != 0) + { + return valueComparison; + } + + return string.Compare(x.Notes, y.Notes, StringComparison.Ordinal); + } + + private static int NullableBoolCompare(bool? x, bool? y) + { + if (x == y) + { + return 0; + } + + return (x, y) switch + { + (null, not null) => -1, + (not null, null) => 1, + (false, true) => -1, + (true, false) => 1, + _ => 0, + }; + } +} + +public static class NormalizedVersionSchemes +{ + public const string SemVer = "semver"; + public const string Nevra = "nevra"; + public const string Evr = "evr"; +} + +public static class NormalizedVersionRuleTypes +{ + public const string Range = "range"; + public const string Exact = "exact"; + public const string LessThan = "lt"; + public const string LessThanOrEqual = "lte"; + public const string GreaterThan = "gt"; + public const string GreaterThanOrEqual = "gte"; +} diff --git a/src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs b/src/StellaOps.Concelier.Models/OsvGhsaParityDiagnostics.cs similarity index 88% rename from src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs rename to src/StellaOps.Concelier.Models/OsvGhsaParityDiagnostics.cs index fa2757b0..870e0fb5 100644 --- a/src/StellaOps.Feedser.Models/OsvGhsaParityDiagnostics.cs +++ b/src/StellaOps.Concelier.Models/OsvGhsaParityDiagnostics.cs @@ -1,72 +1,72 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Models; - -/// -/// Emits telemetry for OSV vs GHSA parity reports so QA dashboards can track regression trends. -/// -public static class OsvGhsaParityDiagnostics -{ - private static readonly Meter Meter = new("StellaOps.Feedser.Models.OsvGhsaParity"); - private static readonly Counter TotalCounter = Meter.CreateCounter( - "feedser.osv_ghsa.total", - unit: "count", - description: "Total GHSA identifiers evaluated for OSV parity."); - private static readonly Counter IssueCounter = Meter.CreateCounter( - "feedser.osv_ghsa.issues", - unit: "count", - description: "Parity issues grouped by dataset, issue kind, and field mask."); - - public static void RecordReport(OsvGhsaParityReport report, string dataset) - { - ArgumentNullException.ThrowIfNull(report); - dataset = NormalizeDataset(dataset); - - if (report.TotalGhsaIds > 0) - { - TotalCounter.Add(report.TotalGhsaIds, CreateTotalTags(dataset)); - } - - if (!report.HasIssues) - { - return; - } - - foreach (var issue in report.Issues) - { - IssueCounter.Add(1, CreateIssueTags(dataset, issue)); - } - } - - private static KeyValuePair[] CreateTotalTags(string dataset) - => new[] - { - new KeyValuePair("dataset", dataset), - }; - - private static KeyValuePair[] CreateIssueTags(string dataset, OsvGhsaParityIssue issue) - { - var mask = issue.FieldMask.IsDefaultOrEmpty - ? "none" - : string.Join('|', issue.FieldMask); - - return new[] - { - new KeyValuePair("dataset", dataset), - new KeyValuePair("issueKind", issue.IssueKind), - new KeyValuePair("fieldMask", mask), - }; - } - - private static string NormalizeDataset(string dataset) - { - if (string.IsNullOrWhiteSpace(dataset)) - { - return "default"; - } - - return dataset.Trim().ToLowerInvariant(); - } -} +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Concelier.Models; + +/// +/// Emits telemetry for OSV vs GHSA parity reports so QA dashboards can track regression trends. +/// +public static class OsvGhsaParityDiagnostics +{ + private static readonly Meter Meter = new("StellaOps.Concelier.Models.OsvGhsaParity"); + private static readonly Counter TotalCounter = Meter.CreateCounter( + "concelier.osv_ghsa.total", + unit: "count", + description: "Total GHSA identifiers evaluated for OSV parity."); + private static readonly Counter IssueCounter = Meter.CreateCounter( + "concelier.osv_ghsa.issues", + unit: "count", + description: "Parity issues grouped by dataset, issue kind, and field mask."); + + public static void RecordReport(OsvGhsaParityReport report, string dataset) + { + ArgumentNullException.ThrowIfNull(report); + dataset = NormalizeDataset(dataset); + + if (report.TotalGhsaIds > 0) + { + TotalCounter.Add(report.TotalGhsaIds, CreateTotalTags(dataset)); + } + + if (!report.HasIssues) + { + return; + } + + foreach (var issue in report.Issues) + { + IssueCounter.Add(1, CreateIssueTags(dataset, issue)); + } + } + + private static KeyValuePair[] CreateTotalTags(string dataset) + => new[] + { + new KeyValuePair("dataset", dataset), + }; + + private static KeyValuePair[] CreateIssueTags(string dataset, OsvGhsaParityIssue issue) + { + var mask = issue.FieldMask.IsDefaultOrEmpty + ? "none" + : string.Join('|', issue.FieldMask); + + return new[] + { + new KeyValuePair("dataset", dataset), + new KeyValuePair("issueKind", issue.IssueKind), + new KeyValuePair("fieldMask", mask), + }; + } + + private static string NormalizeDataset(string dataset) + { + if (string.IsNullOrWhiteSpace(dataset)) + { + return "default"; + } + + return dataset.Trim().ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs b/src/StellaOps.Concelier.Models/OsvGhsaParityInspector.cs similarity index 96% rename from src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs rename to src/StellaOps.Concelier.Models/OsvGhsaParityInspector.cs index 0ab7e862..83aeb6f8 100644 --- a/src/StellaOps.Feedser.Models/OsvGhsaParityInspector.cs +++ b/src/StellaOps.Concelier.Models/OsvGhsaParityInspector.cs @@ -1,183 +1,183 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; - -namespace StellaOps.Feedser.Models; - -/// -/// Compares OSV and GHSA advisory datasets to surface mismatches in coverage, severity, or presence. -/// -public static class OsvGhsaParityInspector -{ - public static OsvGhsaParityReport Compare(IEnumerable osvAdvisories, IEnumerable ghsaAdvisories) - { - ArgumentNullException.ThrowIfNull(osvAdvisories); - ArgumentNullException.ThrowIfNull(ghsaAdvisories); - - var osvByGhsa = BuildOsvMap(osvAdvisories); - var ghsaById = BuildGhsaMap(ghsaAdvisories); - - var union = osvByGhsa.Keys - .Union(ghsaById.Keys, StringComparer.OrdinalIgnoreCase) - .OrderBy(static key => key, StringComparer.OrdinalIgnoreCase) - .ToArray(); - - var issues = ImmutableArray.CreateBuilder(); - - foreach (var ghsaId in union) - { - osvByGhsa.TryGetValue(ghsaId, out var osv); - ghsaById.TryGetValue(ghsaId, out var ghsa); - var normalizedId = ghsaId.ToUpperInvariant(); - - if (osv is null) - { - issues.Add(new OsvGhsaParityIssue( - normalizedId, - "missing_osv", - "GHSA advisory missing from OSV dataset.", - ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); - continue; - } - - if (ghsa is null) - { - issues.Add(new OsvGhsaParityIssue( - normalizedId, - "missing_ghsa", - "OSV mapped GHSA alias without a matching GHSA advisory.", - ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); - continue; - } - - if (!SeverityMatches(osv, ghsa)) - { - var detail = $"Severity mismatch: OSV={osv.Severity ?? "(null)"}, GHSA={ghsa.Severity ?? "(null)"}."; - issues.Add(new OsvGhsaParityIssue( - normalizedId, - "severity_mismatch", - detail, - ImmutableArray.Create(ProvenanceFieldMasks.Advisory))); - } - - if (!RangeCoverageMatches(osv, ghsa)) - { - var detail = $"Range coverage mismatch: OSV ranges={CountRanges(osv)}, GHSA ranges={CountRanges(ghsa)}."; - issues.Add(new OsvGhsaParityIssue( - normalizedId, - "range_mismatch", - detail, - ImmutableArray.Create(ProvenanceFieldMasks.VersionRanges))); - } - } - - return new OsvGhsaParityReport(union.Length, issues.ToImmutable()); - } - - private static IReadOnlyDictionary BuildOsvMap(IEnumerable advisories) - { - var comparer = StringComparer.OrdinalIgnoreCase; - var map = new Dictionary(comparer); - - foreach (var advisory in advisories) - { - if (advisory is null) - { - continue; - } - - foreach (var alias in advisory.Aliases) - { - if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) - { - map.TryAdd(alias, advisory); - } - } - } - - return map; - } - - private static IReadOnlyDictionary BuildGhsaMap(IEnumerable advisories) - { - var comparer = StringComparer.OrdinalIgnoreCase; - var map = new Dictionary(comparer); - - foreach (var advisory in advisories) - { - if (advisory is null) - { - continue; - } - - if (advisory.AdvisoryKey.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) - { - map.TryAdd(advisory.AdvisoryKey, advisory); - continue; - } - - foreach (var alias in advisory.Aliases) - { - if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) - { - map.TryAdd(alias, advisory); - } - } - } - - return map; - } - - private static bool SeverityMatches(Advisory osv, Advisory ghsa) - => string.Equals(osv.Severity, ghsa.Severity, StringComparison.OrdinalIgnoreCase); - - private static bool RangeCoverageMatches(Advisory osv, Advisory ghsa) - { - var osvRanges = CountRanges(osv); - var ghsaRanges = CountRanges(ghsa); - if (osvRanges == ghsaRanges) - { - return true; - } - - // Consider zero-vs-nonzero mismatches as actionable even if raw counts differ. - return osvRanges == 0 && ghsaRanges == 0; - } - - private static int CountRanges(Advisory advisory) - { - if (advisory.AffectedPackages.IsDefaultOrEmpty) - { - return 0; - } - - var count = 0; - foreach (var package in advisory.AffectedPackages) - { - if (package.VersionRanges.IsDefaultOrEmpty) - { - continue; - } - - count += package.VersionRanges.Length; - } - - return count; - } -} - -public sealed record OsvGhsaParityIssue( - string GhsaId, - string IssueKind, - string Detail, - ImmutableArray FieldMask); - -public sealed record OsvGhsaParityReport(int TotalGhsaIds, ImmutableArray Issues) -{ - public bool HasIssues => !Issues.IsDefaultOrEmpty && Issues.Length > 0; - - public int MissingFromOsv => Issues.Count(issue => issue.IssueKind.Equals("missing_osv", StringComparison.OrdinalIgnoreCase)); - - public int MissingFromGhsa => Issues.Count(issue => issue.IssueKind.Equals("missing_ghsa", StringComparison.OrdinalIgnoreCase)); -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Concelier.Models; + +/// +/// Compares OSV and GHSA advisory datasets to surface mismatches in coverage, severity, or presence. +/// +public static class OsvGhsaParityInspector +{ + public static OsvGhsaParityReport Compare(IEnumerable osvAdvisories, IEnumerable ghsaAdvisories) + { + ArgumentNullException.ThrowIfNull(osvAdvisories); + ArgumentNullException.ThrowIfNull(ghsaAdvisories); + + var osvByGhsa = BuildOsvMap(osvAdvisories); + var ghsaById = BuildGhsaMap(ghsaAdvisories); + + var union = osvByGhsa.Keys + .Union(ghsaById.Keys, StringComparer.OrdinalIgnoreCase) + .OrderBy(static key => key, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + var issues = ImmutableArray.CreateBuilder(); + + foreach (var ghsaId in union) + { + osvByGhsa.TryGetValue(ghsaId, out var osv); + ghsaById.TryGetValue(ghsaId, out var ghsa); + var normalizedId = ghsaId.ToUpperInvariant(); + + if (osv is null) + { + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "missing_osv", + "GHSA advisory missing from OSV dataset.", + ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); + continue; + } + + if (ghsa is null) + { + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "missing_ghsa", + "OSV mapped GHSA alias without a matching GHSA advisory.", + ImmutableArray.Create(ProvenanceFieldMasks.AffectedPackages))); + continue; + } + + if (!SeverityMatches(osv, ghsa)) + { + var detail = $"Severity mismatch: OSV={osv.Severity ?? "(null)"}, GHSA={ghsa.Severity ?? "(null)"}."; + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "severity_mismatch", + detail, + ImmutableArray.Create(ProvenanceFieldMasks.Advisory))); + } + + if (!RangeCoverageMatches(osv, ghsa)) + { + var detail = $"Range coverage mismatch: OSV ranges={CountRanges(osv)}, GHSA ranges={CountRanges(ghsa)}."; + issues.Add(new OsvGhsaParityIssue( + normalizedId, + "range_mismatch", + detail, + ImmutableArray.Create(ProvenanceFieldMasks.VersionRanges))); + } + } + + return new OsvGhsaParityReport(union.Length, issues.ToImmutable()); + } + + private static IReadOnlyDictionary BuildOsvMap(IEnumerable advisories) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var map = new Dictionary(comparer); + + foreach (var advisory in advisories) + { + if (advisory is null) + { + continue; + } + + foreach (var alias in advisory.Aliases) + { + if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(alias, advisory); + } + } + } + + return map; + } + + private static IReadOnlyDictionary BuildGhsaMap(IEnumerable advisories) + { + var comparer = StringComparer.OrdinalIgnoreCase; + var map = new Dictionary(comparer); + + foreach (var advisory in advisories) + { + if (advisory is null) + { + continue; + } + + if (advisory.AdvisoryKey.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(advisory.AdvisoryKey, advisory); + continue; + } + + foreach (var alias in advisory.Aliases) + { + if (alias.StartsWith("ghsa-", StringComparison.OrdinalIgnoreCase)) + { + map.TryAdd(alias, advisory); + } + } + } + + return map; + } + + private static bool SeverityMatches(Advisory osv, Advisory ghsa) + => string.Equals(osv.Severity, ghsa.Severity, StringComparison.OrdinalIgnoreCase); + + private static bool RangeCoverageMatches(Advisory osv, Advisory ghsa) + { + var osvRanges = CountRanges(osv); + var ghsaRanges = CountRanges(ghsa); + if (osvRanges == ghsaRanges) + { + return true; + } + + // Consider zero-vs-nonzero mismatches as actionable even if raw counts differ. + return osvRanges == 0 && ghsaRanges == 0; + } + + private static int CountRanges(Advisory advisory) + { + if (advisory.AffectedPackages.IsDefaultOrEmpty) + { + return 0; + } + + var count = 0; + foreach (var package in advisory.AffectedPackages) + { + if (package.VersionRanges.IsDefaultOrEmpty) + { + continue; + } + + count += package.VersionRanges.Length; + } + + return count; + } +} + +public sealed record OsvGhsaParityIssue( + string GhsaId, + string IssueKind, + string Detail, + ImmutableArray FieldMask); + +public sealed record OsvGhsaParityReport(int TotalGhsaIds, ImmutableArray Issues) +{ + public bool HasIssues => !Issues.IsDefaultOrEmpty && Issues.Length > 0; + + public int MissingFromOsv => Issues.Count(issue => issue.IssueKind.Equals("missing_osv", StringComparison.OrdinalIgnoreCase)); + + public int MissingFromGhsa => Issues.Count(issue => issue.IssueKind.Equals("missing_ghsa", StringComparison.OrdinalIgnoreCase)); +} diff --git a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md b/src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md similarity index 84% rename from src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md rename to src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md index 4c6f9166..aa4bbc1b 100644 --- a/src/StellaOps.Feedser.Models/PROVENANCE_GUIDELINES.md +++ b/src/StellaOps.Concelier.Models/PROVENANCE_GUIDELINES.md @@ -1,6 +1,6 @@ # Canonical Field Provenance Guidelines -- **Always attach provenance** when mapping any field into `StellaOps.Feedser.Models`. Use `AdvisoryProvenance` to capture `source` (feed identifier), `kind` (fetch|parse|map|merge), `value` (cursor or extractor hint), and the UTC timestamp when it was recorded. +- **Always attach provenance** when mapping any field into `StellaOps.Concelier.Models`. Use `AdvisoryProvenance` to capture `source` (feed identifier), `kind` (fetch|parse|map|merge), `value` (cursor or extractor hint), and the UTC timestamp when it was recorded. - **Per-field strategy** - `Advisory` metadata (title, summary, severity) should record the connector responsible for the value. When merge overrides occur, add an additional provenance record rather than mutating the original. - `References` must record whether the link originated from the primary advisory (`kind=advisory`), a vendor patch (`kind=patch`), or an enrichment feed (`kind=enrichment`). @@ -11,5 +11,5 @@ - **Determinism**: provenance collections are sorted by source → kind → recordedAt before serialization; avoid generating random identifiers inside provenance. - **Field masks**: populate `fieldMask` on each provenance entry using lowercase canonical masks (see `ProvenanceFieldMasks`). This powers metrics, parity checks, and resume diagnostics. Recent additions include `affectedpackages[].normalizedversions[]`, `affectedpackages[].versionranges[].primitives.semver`, and `credits[]`. - **Redaction**: keep provenance values free of secrets; prefer tokens or normalized descriptors when referencing authenticated fetches. -- **Range telemetry**: each `AffectedVersionRange` is observed by the `feedser.range.primitives` metric. Emit the richest `RangePrimitives` possible (SemVer/NEVRA/EVR plus vendor extensions); the telemetry tags make it easy to spot connectors missing structured range data. +- **Range telemetry**: each `AffectedVersionRange` is observed by the `concelier.range.primitives` metric. Emit the richest `RangePrimitives` possible (SemVer/NEVRA/EVR plus vendor extensions); the telemetry tags make it easy to spot connectors missing structured range data. - **Vendor extensions**: when vendor feeds surface bespoke status flags, capture them in `RangePrimitives.VendorExtensions`. SUSE advisories publish `suse.status` (open/resolved/investigating) and Ubuntu notices expose `ubuntu.pocket`/`ubuntu.release` to distinguish security vs ESM pockets; Adobe APSB bulletins emit `adobe.track`, `adobe.platform`, `adobe.priority`, `adobe.availability`, plus `adobe.affected.raw`/`adobe.updated.raw` to preserve PSIRT metadata while keeping the status catalog canonical. These values are exported for dashboards and alerting. diff --git a/src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs b/src/StellaOps.Concelier.Models/ProvenanceFieldMasks.cs similarity index 92% rename from src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs rename to src/StellaOps.Concelier.Models/ProvenanceFieldMasks.cs index 8d0695cd..54f3050a 100644 --- a/src/StellaOps.Feedser.Models/ProvenanceFieldMasks.cs +++ b/src/StellaOps.Concelier.Models/ProvenanceFieldMasks.cs @@ -1,17 +1,17 @@ -namespace StellaOps.Feedser.Models; - -/// -/// Canonical field-mask identifiers for provenance coverage. -/// -public static class ProvenanceFieldMasks -{ - public const string Advisory = "advisory"; - public const string References = "references[]"; - public const string Credits = "credits[]"; - public const string AffectedPackages = "affectedpackages[]"; - public const string VersionRanges = "affectedpackages[].versionranges[]"; - public const string NormalizedVersions = "affectedpackages[].normalizedversions[]"; - public const string PackageStatuses = "affectedpackages[].statuses[]"; - public const string CvssMetrics = "cvssmetrics[]"; - public const string Weaknesses = "cwes[]"; -} +namespace StellaOps.Concelier.Models; + +/// +/// Canonical field-mask identifiers for provenance coverage. +/// +public static class ProvenanceFieldMasks +{ + public const string Advisory = "advisory"; + public const string References = "references[]"; + public const string Credits = "credits[]"; + public const string AffectedPackages = "affectedpackages[]"; + public const string VersionRanges = "affectedpackages[].versionranges[]"; + public const string NormalizedVersions = "affectedpackages[].normalizedversions[]"; + public const string PackageStatuses = "affectedpackages[].statuses[]"; + public const string CvssMetrics = "cvssmetrics[]"; + public const string Weaknesses = "cwes[]"; +} diff --git a/src/StellaOps.Feedser.Models/ProvenanceInspector.cs b/src/StellaOps.Concelier.Models/ProvenanceInspector.cs similarity index 96% rename from src/StellaOps.Feedser.Models/ProvenanceInspector.cs rename to src/StellaOps.Concelier.Models/ProvenanceInspector.cs index d867e14b..acf97aa4 100644 --- a/src/StellaOps.Feedser.Models/ProvenanceInspector.cs +++ b/src/StellaOps.Concelier.Models/ProvenanceInspector.cs @@ -5,7 +5,7 @@ using System.Diagnostics.Metrics; using System.Linq; using Microsoft.Extensions.Logging; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; public static class ProvenanceInspector { @@ -111,13 +111,13 @@ public sealed record MissingProvenance( public static class ProvenanceDiagnostics { - private static readonly Meter Meter = new("StellaOps.Feedser.Models.Provenance"); + private static readonly Meter Meter = new("StellaOps.Concelier.Models.Provenance"); private static readonly Counter MissingCounter = Meter.CreateCounter( - "feedser.provenance.missing", + "concelier.provenance.missing", unit: "count", description: "Number of canonical objects missing provenance metadata."); private static readonly Counter RangePrimitiveCounter = Meter.CreateCounter( - "feedser.range.primitives", + "concelier.range.primitives", unit: "count", description: "Range coverage by kind, primitive availability, and vendor extensions."); diff --git a/src/StellaOps.Feedser.Models/RangePrimitives.cs b/src/StellaOps.Concelier.Models/RangePrimitives.cs similarity index 98% rename from src/StellaOps.Feedser.Models/RangePrimitives.cs rename to src/StellaOps.Concelier.Models/RangePrimitives.cs index 635163a5..2f645bba 100644 --- a/src/StellaOps.Feedser.Models/RangePrimitives.cs +++ b/src/StellaOps.Concelier.Models/RangePrimitives.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Optional structured representations of range semantics attached to . diff --git a/src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs b/src/StellaOps.Concelier.Models/SemVerPrimitiveExtensions.cs similarity index 96% rename from src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs rename to src/StellaOps.Concelier.Models/SemVerPrimitiveExtensions.cs index e99f6fab..8f4c60c6 100644 --- a/src/StellaOps.Feedser.Models/SemVerPrimitiveExtensions.cs +++ b/src/StellaOps.Concelier.Models/SemVerPrimitiveExtensions.cs @@ -1,102 +1,102 @@ -using System; - -namespace StellaOps.Feedser.Models; - -/// -/// Helper extensions for converting values into normalized rules. -/// -public static class SemVerPrimitiveExtensions -{ - public static NormalizedVersionRule? ToNormalizedVersionRule(this SemVerPrimitive? primitive, string? notes = null) - { - if (primitive is null) - { - return null; - } - - var trimmedNotes = Validation.TrimToNull(notes); - var constraintNotes = Validation.TrimToNull(primitive.ConstraintExpression); - var resolvedNotes = trimmedNotes ?? constraintNotes; - var scheme = NormalizedVersionSchemes.SemVer; - - if (!string.IsNullOrWhiteSpace(primitive.ExactValue)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Exact, - value: primitive.ExactValue, - notes: resolvedNotes); - } - - var introduced = Validation.TrimToNull(primitive.Introduced); - var fixedVersion = Validation.TrimToNull(primitive.Fixed); - var lastAffected = Validation.TrimToNull(primitive.LastAffected); - - if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: primitive.IntroducedInclusive, - max: fixedVersion, - maxInclusive: primitive.FixedInclusive, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && string.IsNullOrEmpty(fixedVersion) && !string.IsNullOrEmpty(lastAffected)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - min: introduced, - minInclusive: primitive.IntroducedInclusive, - max: lastAffected, - maxInclusive: primitive.LastAffectedInclusive, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(introduced) && string.IsNullOrEmpty(fixedVersion) && string.IsNullOrEmpty(lastAffected)) - { - var type = primitive.IntroducedInclusive ? NormalizedVersionRuleTypes.GreaterThanOrEqual : NormalizedVersionRuleTypes.GreaterThan; - return new NormalizedVersionRule( - scheme, - type, - min: introduced, - minInclusive: primitive.IntroducedInclusive, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(fixedVersion)) - { - var type = primitive.FixedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; - return new NormalizedVersionRule( - scheme, - type, - max: fixedVersion, - maxInclusive: primitive.FixedInclusive, - notes: resolvedNotes); - } - - if (!string.IsNullOrEmpty(lastAffected)) - { - var type = primitive.LastAffectedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; - return new NormalizedVersionRule( - scheme, - type, - max: lastAffected, - maxInclusive: primitive.LastAffectedInclusive, - notes: resolvedNotes); - } - - if (!string.IsNullOrWhiteSpace(primitive.ConstraintExpression)) - { - return new NormalizedVersionRule( - scheme, - NormalizedVersionRuleTypes.Range, - notes: resolvedNotes); - } - - return null; - } -} +using System; + +namespace StellaOps.Concelier.Models; + +/// +/// Helper extensions for converting values into normalized rules. +/// +public static class SemVerPrimitiveExtensions +{ + public static NormalizedVersionRule? ToNormalizedVersionRule(this SemVerPrimitive? primitive, string? notes = null) + { + if (primitive is null) + { + return null; + } + + var trimmedNotes = Validation.TrimToNull(notes); + var constraintNotes = Validation.TrimToNull(primitive.ConstraintExpression); + var resolvedNotes = trimmedNotes ?? constraintNotes; + var scheme = NormalizedVersionSchemes.SemVer; + + if (!string.IsNullOrWhiteSpace(primitive.ExactValue)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Exact, + value: primitive.ExactValue, + notes: resolvedNotes); + } + + var introduced = Validation.TrimToNull(primitive.Introduced); + var fixedVersion = Validation.TrimToNull(primitive.Fixed); + var lastAffected = Validation.TrimToNull(primitive.LastAffected); + + if (!string.IsNullOrEmpty(introduced) && !string.IsNullOrEmpty(fixedVersion)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: primitive.IntroducedInclusive, + max: fixedVersion, + maxInclusive: primitive.FixedInclusive, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && string.IsNullOrEmpty(fixedVersion) && !string.IsNullOrEmpty(lastAffected)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + min: introduced, + minInclusive: primitive.IntroducedInclusive, + max: lastAffected, + maxInclusive: primitive.LastAffectedInclusive, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(introduced) && string.IsNullOrEmpty(fixedVersion) && string.IsNullOrEmpty(lastAffected)) + { + var type = primitive.IntroducedInclusive ? NormalizedVersionRuleTypes.GreaterThanOrEqual : NormalizedVersionRuleTypes.GreaterThan; + return new NormalizedVersionRule( + scheme, + type, + min: introduced, + minInclusive: primitive.IntroducedInclusive, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(fixedVersion)) + { + var type = primitive.FixedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; + return new NormalizedVersionRule( + scheme, + type, + max: fixedVersion, + maxInclusive: primitive.FixedInclusive, + notes: resolvedNotes); + } + + if (!string.IsNullOrEmpty(lastAffected)) + { + var type = primitive.LastAffectedInclusive ? NormalizedVersionRuleTypes.LessThanOrEqual : NormalizedVersionRuleTypes.LessThan; + return new NormalizedVersionRule( + scheme, + type, + max: lastAffected, + maxInclusive: primitive.LastAffectedInclusive, + notes: resolvedNotes); + } + + if (!string.IsNullOrWhiteSpace(primitive.ConstraintExpression)) + { + return new NormalizedVersionRule( + scheme, + NormalizedVersionRuleTypes.Range, + notes: resolvedNotes); + } + + return null; + } +} diff --git a/src/StellaOps.Feedser.Models/SeverityNormalization.cs b/src/StellaOps.Concelier.Models/SeverityNormalization.cs similarity index 98% rename from src/StellaOps.Feedser.Models/SeverityNormalization.cs rename to src/StellaOps.Concelier.Models/SeverityNormalization.cs index 280d528c..c33bb0f9 100644 --- a/src/StellaOps.Feedser.Models/SeverityNormalization.cs +++ b/src/StellaOps.Concelier.Models/SeverityNormalization.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using System.Text; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Provides helpers to normalize vendor-provided severity labels into canonical values. diff --git a/src/StellaOps.Feedser.Models/SnapshotSerializer.cs b/src/StellaOps.Concelier.Models/SnapshotSerializer.cs similarity index 93% rename from src/StellaOps.Feedser.Models/SnapshotSerializer.cs rename to src/StellaOps.Concelier.Models/SnapshotSerializer.cs index 101c044f..3b621d69 100644 --- a/src/StellaOps.Feedser.Models/SnapshotSerializer.cs +++ b/src/StellaOps.Concelier.Models/SnapshotSerializer.cs @@ -1,7 +1,7 @@ using System.Text; using System.Text.Json; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Helper for tests/fixtures that need deterministic JSON snapshots. diff --git a/src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj b/src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj similarity index 100% rename from src/StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj rename to src/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj diff --git a/src/StellaOps.Feedser.Models/TASKS.md b/src/StellaOps.Concelier.Models/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Models/TASKS.md rename to src/StellaOps.Concelier.Models/TASKS.md diff --git a/src/StellaOps.Feedser.Models/Validation.cs b/src/StellaOps.Concelier.Models/Validation.cs similarity index 94% rename from src/StellaOps.Feedser.Models/Validation.cs rename to src/StellaOps.Concelier.Models/Validation.cs index 6f1b0bdd..bcc5bbd2 100644 --- a/src/StellaOps.Feedser.Models/Validation.cs +++ b/src/StellaOps.Concelier.Models/Validation.cs @@ -1,7 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Models; +namespace StellaOps.Concelier.Models; /// /// Lightweight validation helpers shared across canonical model constructors. diff --git a/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs b/src/StellaOps.Concelier.Normalization.Tests/CpeNormalizerTests.cs similarity index 92% rename from src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/CpeNormalizerTests.cs index 4fdf8a45..8842660f 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/CpeNormalizerTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/CpeNormalizerTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Identifiers; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class CpeNormalizerTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs b/src/StellaOps.Concelier.Normalization.Tests/CvssMetricNormalizerTests.cs similarity index 90% rename from src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/CvssMetricNormalizerTests.cs index 1635e078..4eff58a2 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/CvssMetricNormalizerTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/CvssMetricNormalizerTests.cs @@ -1,7 +1,7 @@ -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.Cvss; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.Cvss; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class CvssMetricNormalizerTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs b/src/StellaOps.Concelier.Normalization.Tests/DebianEvrParserTests.cs similarity index 84% rename from src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/DebianEvrParserTests.cs index dbc4f4a9..f6447318 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/DebianEvrParserTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/DebianEvrParserTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Normalization.Distro; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class DebianEvrParserTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs b/src/StellaOps.Concelier.Normalization.Tests/DescriptionNormalizerTests.cs similarity index 88% rename from src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/DescriptionNormalizerTests.cs index 79b7f25b..b6a289ed 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/DescriptionNormalizerTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/DescriptionNormalizerTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Normalization.Text; +using StellaOps.Concelier.Normalization.Text; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class DescriptionNormalizerTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs b/src/StellaOps.Concelier.Normalization.Tests/NevraParserTests.cs similarity index 91% rename from src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/NevraParserTests.cs index 6a16797d..ab44f846 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/NevraParserTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/NevraParserTests.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Normalization.Distro; +using StellaOps.Concelier.Normalization.Distro; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class NevraParserTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs b/src/StellaOps.Concelier.Normalization.Tests/PackageUrlNormalizerTests.cs similarity index 89% rename from src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/PackageUrlNormalizerTests.cs index 99599d30..22b19c31 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/PackageUrlNormalizerTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/PackageUrlNormalizerTests.cs @@ -1,7 +1,7 @@ using System.Linq; -using StellaOps.Feedser.Normalization.Identifiers; +using StellaOps.Concelier.Normalization.Identifiers; -namespace StellaOps.Feedser.Normalization.Tests; +namespace StellaOps.Concelier.Normalization.Tests; public sealed class PackageUrlNormalizerTests { diff --git a/src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs b/src/StellaOps.Concelier.Normalization.Tests/SemVerRangeRuleBuilderTests.cs similarity index 95% rename from src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs rename to src/StellaOps.Concelier.Normalization.Tests/SemVerRangeRuleBuilderTests.cs index f90efa5f..dc67f325 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/SemVerRangeRuleBuilderTests.cs +++ b/src/StellaOps.Concelier.Normalization.Tests/SemVerRangeRuleBuilderTests.cs @@ -1,183 +1,183 @@ -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Normalization.SemVer; -using Xunit; - -namespace StellaOps.Feedser.Normalization.Tests; - -public sealed class SemVerRangeRuleBuilderTests -{ - private const string Note = "spec:test"; - - [Theory] - [InlineData("< 1.5.0", null, NormalizedVersionRuleTypes.LessThan, null, true, "1.5.0", false, null, false)] - [InlineData(">= 1.0.0, < 2.0.0", null, NormalizedVersionRuleTypes.Range, "1.0.0", true, "2.0.0", false, null, false)] - [InlineData(">1.2.3, <=1.3.0", null, NormalizedVersionRuleTypes.Range, "1.2.3", false, null, false, "1.3.0", true)] - public void Build_ParsesCommonRanges( - string range, - string? patched, - string expectedNormalizedType, - string? expectedIntroduced, - bool expectedIntroducedInclusive, - string? expectedFixed, - bool expectedFixedInclusive, - string? expectedLastAffected, - bool expectedLastInclusive) - { - var results = SemVerRangeRuleBuilder.Build(range, patched, Note); - var result = Assert.Single(results); - - var primitive = result.Primitive; - Assert.Equal(expectedIntroduced, primitive.Introduced); - Assert.Equal(expectedIntroducedInclusive, primitive.IntroducedInclusive); - Assert.Equal(expectedFixed, primitive.Fixed); - Assert.Equal(expectedFixedInclusive, primitive.FixedInclusive); - Assert.Equal(expectedLastAffected, primitive.LastAffected); - Assert.Equal(expectedLastInclusive, primitive.LastAffectedInclusive); - - var normalized = result.NormalizedRule; - Assert.Equal(NormalizedVersionSchemes.SemVer, normalized.Scheme); - Assert.Equal(expectedNormalizedType, normalized.Type); - Assert.Equal(expectedIntroduced, normalized.Min); - Assert.Equal(expectedIntroduced is null ? (bool?)null : expectedIntroducedInclusive, normalized.MinInclusive); - Assert.Equal(expectedFixed ?? expectedLastAffected, normalized.Max); - Assert.Equal( - expectedFixed is not null ? expectedFixedInclusive : expectedLastInclusive, - normalized.MaxInclusive); - Assert.Equal(patched is null && expectedIntroduced is null && expectedFixed is null && expectedLastAffected is null ? null : Note, normalized.Notes); - } - - [Fact] - public void Build_UsesPatchedVersionWhenUpperBoundMissing() - { - var results = SemVerRangeRuleBuilder.Build(">= 4.0.0", "4.3.6", Note); - var result = Assert.Single(results); - - Assert.Equal("4.0.0", result.Primitive.Introduced); - Assert.Equal("4.3.6", result.Primitive.Fixed); - Assert.False(result.Primitive.FixedInclusive); - - var normalized = result.NormalizedRule; - Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); - Assert.Equal("4.0.0", normalized.Min); - Assert.True(normalized.MinInclusive); - Assert.Equal("4.3.6", normalized.Max); - Assert.False(normalized.MaxInclusive); - Assert.Equal(Note, normalized.Notes); - } - - [Theory] - [InlineData("^1.2.3", "1.2.3", "2.0.0")] - [InlineData("~1.2.3", "1.2.3", "1.3.0")] - [InlineData("~> 1.2", "1.2.0", "1.3.0")] - public void Build_HandlesCaretAndTilde(string range, string expectedMin, string expectedMax) - { - var results = SemVerRangeRuleBuilder.Build(range, null, Note); - var result = Assert.Single(results); - var normalized = result.NormalizedRule; - Assert.Equal(expectedMin, normalized.Min); - Assert.True(normalized.MinInclusive); - Assert.Equal(expectedMax, normalized.Max); - Assert.False(normalized.MaxInclusive); - Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); - } - - [Theory] - [InlineData("1.2.x", "1.2.0", "1.3.0")] - [InlineData("1.x", "1.0.0", "2.0.0")] - public void Build_HandlesWildcardNotation(string range, string expectedMin, string expectedMax) - { - var results = SemVerRangeRuleBuilder.Build(range, null, Note); - var result = Assert.Single(results); - Assert.Equal(expectedMin, result.Primitive.Introduced); - Assert.Equal(expectedMax, result.Primitive.Fixed); - - var normalized = result.NormalizedRule; - Assert.Equal(expectedMin, normalized.Min); - Assert.Equal(expectedMax, normalized.Max); - Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); - } - - [Fact] - public void Build_PreservesPreReleaseAndMetadataInExactRule() - { - var results = SemVerRangeRuleBuilder.Build("= 2.5.1-alpha.1+build.7", null, Note); - var result = Assert.Single(results); - - Assert.Equal("2.5.1-alpha.1+build.7", result.Primitive.ExactValue); - - var normalized = result.NormalizedRule; - Assert.Equal(NormalizedVersionRuleTypes.Exact, normalized.Type); - Assert.Equal("2.5.1-alpha.1+build.7", normalized.Value); - Assert.Equal(Note, normalized.Notes); - } - - [Fact] - public void Build_ParsesComparatorWithoutCommaSeparators() - { - var results = SemVerRangeRuleBuilder.Build(">=1.0.0 <1.2.0", null, Note); - var result = Assert.Single(results); - - var primitive = result.Primitive; - Assert.Equal("1.0.0", primitive.Introduced); - Assert.True(primitive.IntroducedInclusive); - Assert.Equal("1.2.0", primitive.Fixed); - Assert.False(primitive.FixedInclusive); - Assert.Equal(">= 1.0.0, < 1.2.0", primitive.ConstraintExpression); - - var normalized = result.NormalizedRule; - Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); - Assert.Equal("1.0.0", normalized.Min); - Assert.True(normalized.MinInclusive); - Assert.Equal("1.2.0", normalized.Max); - Assert.False(normalized.MaxInclusive); - Assert.Equal(Note, normalized.Notes); - } - - [Fact] - public void Build_HandlesMultipleSegmentsSeparatedByOr() - { - var results = SemVerRangeRuleBuilder.Build(">=1.0.0 <1.2.0 || >=2.0.0 <2.2.0", null, Note); - Assert.Equal(2, results.Count); - - var first = results[0]; - Assert.Equal("1.0.0", first.Primitive.Introduced); - Assert.Equal("1.2.0", first.Primitive.Fixed); - Assert.Equal(NormalizedVersionRuleTypes.Range, first.NormalizedRule.Type); - Assert.Equal("1.0.0", first.NormalizedRule.Min); - Assert.Equal("1.2.0", first.NormalizedRule.Max); - - var second = results[1]; - Assert.Equal("2.0.0", second.Primitive.Introduced); - Assert.Equal("2.2.0", second.Primitive.Fixed); - Assert.Equal(NormalizedVersionRuleTypes.Range, second.NormalizedRule.Type); - Assert.Equal("2.0.0", second.NormalizedRule.Min); - Assert.Equal("2.2.0", second.NormalizedRule.Max); - - foreach (var result in results) - { - Assert.Equal(Note, result.NormalizedRule.Notes); - } - } - - [Fact] - public void BuildNormalizedRules_ProjectsNormalizedRules() - { - var rules = SemVerRangeRuleBuilder.BuildNormalizedRules(">=1.0.0 <1.2.0", null, Note); - var rule = Assert.Single(rules); - - Assert.Equal(NormalizedVersionSchemes.SemVer, rule.Scheme); - Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); - Assert.Equal("1.0.0", rule.Min); - Assert.True(rule.MinInclusive); - Assert.Equal("1.2.0", rule.Max); - Assert.False(rule.MaxInclusive); - Assert.Equal(Note, rule.Notes); - } - - [Fact] - public void BuildNormalizedRules_ReturnsEmptyWhenNoRules() - { - var rules = SemVerRangeRuleBuilder.BuildNormalizedRules(" ", null, Note); - Assert.Empty(rules); - } -} +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Normalization.SemVer; +using Xunit; + +namespace StellaOps.Concelier.Normalization.Tests; + +public sealed class SemVerRangeRuleBuilderTests +{ + private const string Note = "spec:test"; + + [Theory] + [InlineData("< 1.5.0", null, NormalizedVersionRuleTypes.LessThan, null, true, "1.5.0", false, null, false)] + [InlineData(">= 1.0.0, < 2.0.0", null, NormalizedVersionRuleTypes.Range, "1.0.0", true, "2.0.0", false, null, false)] + [InlineData(">1.2.3, <=1.3.0", null, NormalizedVersionRuleTypes.Range, "1.2.3", false, null, false, "1.3.0", true)] + public void Build_ParsesCommonRanges( + string range, + string? patched, + string expectedNormalizedType, + string? expectedIntroduced, + bool expectedIntroducedInclusive, + string? expectedFixed, + bool expectedFixedInclusive, + string? expectedLastAffected, + bool expectedLastInclusive) + { + var results = SemVerRangeRuleBuilder.Build(range, patched, Note); + var result = Assert.Single(results); + + var primitive = result.Primitive; + Assert.Equal(expectedIntroduced, primitive.Introduced); + Assert.Equal(expectedIntroducedInclusive, primitive.IntroducedInclusive); + Assert.Equal(expectedFixed, primitive.Fixed); + Assert.Equal(expectedFixedInclusive, primitive.FixedInclusive); + Assert.Equal(expectedLastAffected, primitive.LastAffected); + Assert.Equal(expectedLastInclusive, primitive.LastAffectedInclusive); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionSchemes.SemVer, normalized.Scheme); + Assert.Equal(expectedNormalizedType, normalized.Type); + Assert.Equal(expectedIntroduced, normalized.Min); + Assert.Equal(expectedIntroduced is null ? (bool?)null : expectedIntroducedInclusive, normalized.MinInclusive); + Assert.Equal(expectedFixed ?? expectedLastAffected, normalized.Max); + Assert.Equal( + expectedFixed is not null ? expectedFixedInclusive : expectedLastInclusive, + normalized.MaxInclusive); + Assert.Equal(patched is null && expectedIntroduced is null && expectedFixed is null && expectedLastAffected is null ? null : Note, normalized.Notes); + } + + [Fact] + public void Build_UsesPatchedVersionWhenUpperBoundMissing() + { + var results = SemVerRangeRuleBuilder.Build(">= 4.0.0", "4.3.6", Note); + var result = Assert.Single(results); + + Assert.Equal("4.0.0", result.Primitive.Introduced); + Assert.Equal("4.3.6", result.Primitive.Fixed); + Assert.False(result.Primitive.FixedInclusive); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + Assert.Equal("4.0.0", normalized.Min); + Assert.True(normalized.MinInclusive); + Assert.Equal("4.3.6", normalized.Max); + Assert.False(normalized.MaxInclusive); + Assert.Equal(Note, normalized.Notes); + } + + [Theory] + [InlineData("^1.2.3", "1.2.3", "2.0.0")] + [InlineData("~1.2.3", "1.2.3", "1.3.0")] + [InlineData("~> 1.2", "1.2.0", "1.3.0")] + public void Build_HandlesCaretAndTilde(string range, string expectedMin, string expectedMax) + { + var results = SemVerRangeRuleBuilder.Build(range, null, Note); + var result = Assert.Single(results); + var normalized = result.NormalizedRule; + Assert.Equal(expectedMin, normalized.Min); + Assert.True(normalized.MinInclusive); + Assert.Equal(expectedMax, normalized.Max); + Assert.False(normalized.MaxInclusive); + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + } + + [Theory] + [InlineData("1.2.x", "1.2.0", "1.3.0")] + [InlineData("1.x", "1.0.0", "2.0.0")] + public void Build_HandlesWildcardNotation(string range, string expectedMin, string expectedMax) + { + var results = SemVerRangeRuleBuilder.Build(range, null, Note); + var result = Assert.Single(results); + Assert.Equal(expectedMin, result.Primitive.Introduced); + Assert.Equal(expectedMax, result.Primitive.Fixed); + + var normalized = result.NormalizedRule; + Assert.Equal(expectedMin, normalized.Min); + Assert.Equal(expectedMax, normalized.Max); + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + } + + [Fact] + public void Build_PreservesPreReleaseAndMetadataInExactRule() + { + var results = SemVerRangeRuleBuilder.Build("= 2.5.1-alpha.1+build.7", null, Note); + var result = Assert.Single(results); + + Assert.Equal("2.5.1-alpha.1+build.7", result.Primitive.ExactValue); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionRuleTypes.Exact, normalized.Type); + Assert.Equal("2.5.1-alpha.1+build.7", normalized.Value); + Assert.Equal(Note, normalized.Notes); + } + + [Fact] + public void Build_ParsesComparatorWithoutCommaSeparators() + { + var results = SemVerRangeRuleBuilder.Build(">=1.0.0 <1.2.0", null, Note); + var result = Assert.Single(results); + + var primitive = result.Primitive; + Assert.Equal("1.0.0", primitive.Introduced); + Assert.True(primitive.IntroducedInclusive); + Assert.Equal("1.2.0", primitive.Fixed); + Assert.False(primitive.FixedInclusive); + Assert.Equal(">= 1.0.0, < 1.2.0", primitive.ConstraintExpression); + + var normalized = result.NormalizedRule; + Assert.Equal(NormalizedVersionRuleTypes.Range, normalized.Type); + Assert.Equal("1.0.0", normalized.Min); + Assert.True(normalized.MinInclusive); + Assert.Equal("1.2.0", normalized.Max); + Assert.False(normalized.MaxInclusive); + Assert.Equal(Note, normalized.Notes); + } + + [Fact] + public void Build_HandlesMultipleSegmentsSeparatedByOr() + { + var results = SemVerRangeRuleBuilder.Build(">=1.0.0 <1.2.0 || >=2.0.0 <2.2.0", null, Note); + Assert.Equal(2, results.Count); + + var first = results[0]; + Assert.Equal("1.0.0", first.Primitive.Introduced); + Assert.Equal("1.2.0", first.Primitive.Fixed); + Assert.Equal(NormalizedVersionRuleTypes.Range, first.NormalizedRule.Type); + Assert.Equal("1.0.0", first.NormalizedRule.Min); + Assert.Equal("1.2.0", first.NormalizedRule.Max); + + var second = results[1]; + Assert.Equal("2.0.0", second.Primitive.Introduced); + Assert.Equal("2.2.0", second.Primitive.Fixed); + Assert.Equal(NormalizedVersionRuleTypes.Range, second.NormalizedRule.Type); + Assert.Equal("2.0.0", second.NormalizedRule.Min); + Assert.Equal("2.2.0", second.NormalizedRule.Max); + + foreach (var result in results) + { + Assert.Equal(Note, result.NormalizedRule.Notes); + } + } + + [Fact] + public void BuildNormalizedRules_ProjectsNormalizedRules() + { + var rules = SemVerRangeRuleBuilder.BuildNormalizedRules(">=1.0.0 <1.2.0", null, Note); + var rule = Assert.Single(rules); + + Assert.Equal(NormalizedVersionSchemes.SemVer, rule.Scheme); + Assert.Equal(NormalizedVersionRuleTypes.Range, rule.Type); + Assert.Equal("1.0.0", rule.Min); + Assert.True(rule.MinInclusive); + Assert.Equal("1.2.0", rule.Max); + Assert.False(rule.MaxInclusive); + Assert.Equal(Note, rule.Notes); + } + + [Fact] + public void BuildNormalizedRules_ReturnsEmptyWhenNoRules() + { + var rules = SemVerRangeRuleBuilder.BuildNormalizedRules(" ", null, Note); + Assert.Empty(rules); + } +} diff --git a/src/StellaOps.Concelier.Normalization.Tests/StellaOps.Concelier.Normalization.Tests.csproj b/src/StellaOps.Concelier.Normalization.Tests/StellaOps.Concelier.Normalization.Tests.csproj new file mode 100644 index 00000000..3b6886da --- /dev/null +++ b/src/StellaOps.Concelier.Normalization.Tests/StellaOps.Concelier.Normalization.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs b/src/StellaOps.Concelier.Normalization/AssemblyInfo.cs similarity index 59% rename from src/StellaOps.Feedser.Normalization/AssemblyInfo.cs rename to src/StellaOps.Concelier.Normalization/AssemblyInfo.cs index c367451d..425bb5d0 100644 --- a/src/StellaOps.Feedser.Normalization/AssemblyInfo.cs +++ b/src/StellaOps.Concelier.Normalization/AssemblyInfo.cs @@ -1,8 +1,8 @@ using System.Reflection; [assembly: AssemblyCompany("StellaOps")] -[assembly: AssemblyProduct("StellaOps.Feedser.Normalization")] -[assembly: AssemblyTitle("StellaOps.Feedser.Normalization")] +[assembly: AssemblyProduct("StellaOps.Concelier.Normalization")] +[assembly: AssemblyTitle("StellaOps.Concelier.Normalization")] [assembly: AssemblyVersion("1.0.0.0")] [assembly: AssemblyFileVersion("1.0.0.0")] [assembly: AssemblyInformationalVersion("1.0.0")] diff --git a/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs b/src/StellaOps.Concelier.Normalization/Cvss/CvssMetricNormalizer.cs similarity index 96% rename from src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs rename to src/StellaOps.Concelier.Normalization/Cvss/CvssMetricNormalizer.cs index b6aecb23..698a27fe 100644 --- a/src/StellaOps.Feedser.Normalization/Cvss/CvssMetricNormalizer.cs +++ b/src/StellaOps.Concelier.Normalization/Cvss/CvssMetricNormalizer.cs @@ -1,8 +1,8 @@ using System.Collections.Immutable; using System.Linq; -using StellaOps.Feedser.Models; +using StellaOps.Concelier.Models; -namespace StellaOps.Feedser.Normalization.Cvss; +namespace StellaOps.Concelier.Normalization.Cvss; /// /// Provides helpers to canonicalize CVSS vectors and fill in derived score/severity information. diff --git a/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs b/src/StellaOps.Concelier.Normalization/Distro/DebianEvr.cs similarity index 95% rename from src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs rename to src/StellaOps.Concelier.Normalization/Distro/DebianEvr.cs index f563d869..afe5a132 100644 --- a/src/StellaOps.Feedser.Normalization/Distro/DebianEvr.cs +++ b/src/StellaOps.Concelier.Normalization/Distro/DebianEvr.cs @@ -1,6 +1,6 @@ using System.Globalization; -namespace StellaOps.Feedser.Normalization.Distro; +namespace StellaOps.Concelier.Normalization.Distro; /// /// Represents a Debian epoch:version-revision tuple and exposes parsing/formatting helpers. diff --git a/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs b/src/StellaOps.Concelier.Normalization/Distro/Nevra.cs similarity index 95% rename from src/StellaOps.Feedser.Normalization/Distro/Nevra.cs rename to src/StellaOps.Concelier.Normalization/Distro/Nevra.cs index daf61944..c34ae453 100644 --- a/src/StellaOps.Feedser.Normalization/Distro/Nevra.cs +++ b/src/StellaOps.Concelier.Normalization/Distro/Nevra.cs @@ -1,6 +1,6 @@ using System.Globalization; -namespace StellaOps.Feedser.Normalization.Distro; +namespace StellaOps.Concelier.Normalization.Distro; /// /// Represents a parsed NEVRA (Name-Epoch:Version-Release.Architecture) identifier and exposes helpers for canonical formatting. diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs b/src/StellaOps.Concelier.Normalization/Identifiers/Cpe23.cs similarity index 96% rename from src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs rename to src/StellaOps.Concelier.Normalization/Identifiers/Cpe23.cs index bd92d20d..4ae8ecff 100644 --- a/src/StellaOps.Feedser.Normalization/Identifiers/Cpe23.cs +++ b/src/StellaOps.Concelier.Normalization/Identifiers/Cpe23.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.Text; -namespace StellaOps.Feedser.Normalization.Identifiers; +namespace StellaOps.Concelier.Normalization.Identifiers; /// /// Implements canonical normalization for CPE 2.3 identifiers (and URI binding conversion). diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs b/src/StellaOps.Concelier.Normalization/Identifiers/IdentifierNormalizer.cs similarity index 90% rename from src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs rename to src/StellaOps.Concelier.Normalization/Identifiers/IdentifierNormalizer.cs index 6b36081f..6c3da4c2 100644 --- a/src/StellaOps.Feedser.Normalization/Identifiers/IdentifierNormalizer.cs +++ b/src/StellaOps.Concelier.Normalization/Identifiers/IdentifierNormalizer.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Normalization.Identifiers; +namespace StellaOps.Concelier.Normalization.Identifiers; /// /// Provides canonical normalization helpers for package identifiers. diff --git a/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs b/src/StellaOps.Concelier.Normalization/Identifiers/PackageUrl.cs similarity index 95% rename from src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs rename to src/StellaOps.Concelier.Normalization/Identifiers/PackageUrl.cs index fbfda254..d9f1b106 100644 --- a/src/StellaOps.Feedser.Normalization/Identifiers/PackageUrl.cs +++ b/src/StellaOps.Concelier.Normalization/Identifiers/PackageUrl.cs @@ -2,7 +2,7 @@ using System.Collections.Immutable; using System.Linq; using System.Text; -namespace StellaOps.Feedser.Normalization.Identifiers; +namespace StellaOps.Concelier.Normalization.Identifiers; /// /// Represents a parsed Package URL (purl) identifier with canonical string rendering. diff --git a/src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs b/src/StellaOps.Concelier.Normalization/SemVer/SemVerRangeRuleBuilder.cs similarity index 96% rename from src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs rename to src/StellaOps.Concelier.Normalization/SemVer/SemVerRangeRuleBuilder.cs index 392324b2..a2425b89 100644 --- a/src/StellaOps.Feedser.Normalization/SemVer/SemVerRangeRuleBuilder.cs +++ b/src/StellaOps.Concelier.Normalization/SemVer/SemVerRangeRuleBuilder.cs @@ -1,645 +1,645 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using NuGet.Versioning; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Normalization.SemVer; - -/// -/// Parses SemVer-style range expressions into deterministic primitives and normalized rules. -/// Supports caret, tilde, wildcard, hyphen, and comparator syntaxes with optional provenance notes. -/// -public static class SemVerRangeRuleBuilder -{ - private static readonly Regex ComparatorRegex = new(@"^(?>=|<=|>|<|==|=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex ComparatorTokenRegex = new(@"(?(?>=|<=|>|<|==|=)\s*(?[^,\s\|]+))", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly Regex HyphenRegex = new(@"^\s*(?.+?)\s+-\s+(?.+)\s*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); - private static readonly char[] SegmentTrimCharacters = { '(', ')', '[', ']', '{', '}', ';' }; - private static readonly char[] FragmentSplitCharacters = { ',', ' ' }; - - public static IReadOnlyList Build(string? rawRange, string? patchedVersion = null, string? provenanceNote = null) - { - var results = new List(); - - if (!string.IsNullOrWhiteSpace(rawRange)) - { - foreach (var segment in SplitSegments(rawRange)) - { - if (TryBuildFromSegment(segment, patchedVersion, provenanceNote, out var result)) - { - results.Add(result); - } - } - } - - if (results.Count == 0 && TryNormalizeVersion(patchedVersion, out var normalizedPatched)) - { - var constraint = $"< {normalizedPatched}"; - var fallbackPrimitive = new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: false, - Fixed: normalizedPatched, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: constraint); - - var fallbackRule = fallbackPrimitive.ToNormalizedVersionRule(provenanceNote); - if (fallbackRule is not null) - { - results.Add(new SemVerRangeBuildResult(fallbackPrimitive, fallbackRule, constraint)); - } - } - - return results.Count == 0 ? Array.Empty() : results; - } - - public static IReadOnlyList BuildNormalizedRules(string? rawRange, string? patchedVersion = null, string? provenanceNote = null) - { - var results = Build(rawRange, patchedVersion, provenanceNote); - if (results.Count == 0) - { - return Array.Empty(); - } - - var rules = new NormalizedVersionRule[results.Count]; - for (var i = 0; i < results.Count; i++) - { - rules[i] = results[i].NormalizedRule; - } - - return rules; - } - - private static IEnumerable SplitSegments(string rawRange) - => rawRange.Split("||", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - - private static bool TryBuildFromSegment(string segment, string? patchedVersion, string? provenanceNote, out SemVerRangeBuildResult result) - { - result = default!; - - if (!TryParsePrimitive(segment, patchedVersion, out var primitive)) - { - return false; - } - - var normalized = primitive.ToNormalizedVersionRule(provenanceNote); - if (normalized is null) - { - return false; - } - - result = new SemVerRangeBuildResult(primitive, normalized, primitive.ConstraintExpression ?? segment.Trim()); - return true; - } - - private static bool TryParsePrimitive(string segment, string? patchedVersion, out SemVerPrimitive primitive) - { - primitive = default!; - - var trimmed = NormalizeWhitespace(segment); - if (string.IsNullOrEmpty(trimmed) || IsWildcardToken(trimmed)) - { - return false; - } - - if (TryParseCaret(trimmed, out primitive)) - { - return true; - } - - if (TryParseTilde(trimmed, out primitive)) - { - return true; - } - - if (TryParseHyphen(trimmed, out primitive)) - { - return true; - } - - if (TryParseWildcard(trimmed, out primitive)) - { - return true; - } - - if (TryParseComparators(trimmed, patchedVersion, out primitive)) - { - return true; - } - - if (TryParseExact(trimmed, out primitive)) - { - return true; - } - - return false; - } - - private static bool TryParseCaret(string expression, out SemVerPrimitive primitive) - { - primitive = default!; - - if (!expression.StartsWith("^", StringComparison.Ordinal)) - { - return false; - } - - var value = expression[1..].Trim(); - if (!TryParseSemanticVersion(value, out var version, out var normalizedBase)) - { - return false; - } - - var upper = CalculateCaretUpperBound(version); - var upperNormalized = FormatVersion(upper); - var constraint = $">= {normalizedBase} < {upperNormalized}"; - - primitive = new SemVerPrimitive( - Introduced: normalizedBase, - IntroducedInclusive: true, - Fixed: upperNormalized, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: constraint); - return true; - } - - private static bool TryParseTilde(string expression, out SemVerPrimitive primitive) - { - primitive = default!; - - if (!expression.StartsWith("~", StringComparison.Ordinal)) - { - return false; - } - - var remainder = expression.TrimStart('~'); - if (remainder.StartsWith(">", StringComparison.Ordinal)) - { - remainder = remainder[1..]; - } - - remainder = remainder.Trim(); - if (!TryParseSemanticVersion(remainder, out var version, out var normalizedBase)) - { - return false; - } - - var componentCount = CountExplicitComponents(remainder); - var upper = CalculateTildeUpperBound(version, componentCount); - var upperNormalized = FormatVersion(upper); - var constraint = $">= {normalizedBase} < {upperNormalized}"; - - primitive = new SemVerPrimitive( - Introduced: normalizedBase, - IntroducedInclusive: true, - Fixed: upperNormalized, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: constraint); - return true; - } - - private static bool TryParseHyphen(string expression, out SemVerPrimitive primitive) - { - primitive = default!; - - var match = HyphenRegex.Match(expression); - if (!match.Success) - { - return false; - } - - var startRaw = match.Groups["start"].Value.Trim(); - var endRaw = match.Groups["end"].Value.Trim(); - - if (!TryNormalizeVersion(startRaw, out var start) || !TryNormalizeVersion(endRaw, out var end)) - { - return false; - } - - var constraint = $"{start} - {end}"; - primitive = new SemVerPrimitive( - Introduced: start, - IntroducedInclusive: true, - Fixed: null, - FixedInclusive: false, - LastAffected: end, - LastAffectedInclusive: true, - ConstraintExpression: constraint); - return true; - } - - private static bool TryParseWildcard(string expression, out SemVerPrimitive primitive) - { - primitive = default!; - - var sanitized = expression.Trim(SegmentTrimCharacters).Trim(); - if (string.IsNullOrEmpty(sanitized)) - { - return false; - } - - var parts = sanitized.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (parts.Length == 0 || parts.All(static part => !IsWildcardToken(part))) - { - return false; - } - - if (!int.TryParse(RemoveLeadingV(parts[0]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var major)) - { - return false; - } - - var hasMinor = parts.Length > 1 && !IsWildcardToken(parts[1]); - var hasPatch = parts.Length > 2 && !IsWildcardToken(parts[2]); - var minor = hasMinor && int.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedMinor) ? parsedMinor : 0; - var patch = hasPatch && int.TryParse(parts[2], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedPatch) ? parsedPatch : 0; - - var minVersion = new SemanticVersion(major, hasMinor ? minor : 0, hasPatch ? patch : 0); - var minNormalized = FormatVersion(minVersion); - - SemanticVersion upperVersion; - if (!hasMinor) - { - upperVersion = new SemanticVersion(major + 1, 0, 0); - } - else if (!hasPatch) - { - upperVersion = new SemanticVersion(major, minor + 1, 0); - } - else - { - upperVersion = new SemanticVersion(major, minor + 1, 0); - } - - var upperNormalized = FormatVersion(upperVersion); - var constraint = $">= {minNormalized} < {upperNormalized}"; - - primitive = new SemVerPrimitive( - Introduced: minNormalized, - IntroducedInclusive: true, - Fixed: upperNormalized, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: constraint); - return true; - } - - private static bool TryParseComparators(string expression, string? patchedVersion, out SemVerPrimitive primitive) - { - primitive = default!; - - var fragments = expression.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - if (fragments.Length == 0) - { - fragments = new[] { expression }; - } - - var constraintTokens = new List(); - - string? introduced = null; - bool introducedInclusive = true; - var hasIntroduced = false; - - string? fixedVersion = null; - bool fixedInclusive = false; - var hasFixed = false; - - string? lastAffected = null; - bool lastInclusive = true; - var hasLast = false; - - string? exactValue = null; - - foreach (var fragment in fragments) - { - var handled = false; - - foreach (Match match in ComparatorTokenRegex.Matches(fragment)) - { - if (match.Groups["token"].Success - && TryParseComparatorFragment(match.Groups["token"].Value, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) - { - handled = true; - } - } - - if (!handled) - { - var parts = fragment.Split(FragmentSplitCharacters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - foreach (var part in parts) - { - if (TryParseComparatorFragment(part, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) - { - handled = true; - } - } - } - - if (handled) - { - continue; - } - - if (TryNormalizeVersion(fragment, out var normalizedExact)) - { - exactValue = normalizedExact; - constraintTokens.Add(normalizedExact); - } - } - - if (exactValue is not null) - { - primitive = new SemVerPrimitive( - Introduced: exactValue, - IntroducedInclusive: true, - Fixed: exactValue, - FixedInclusive: true, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim(), - ExactValue: exactValue); - return true; - } - - if (hasIntroduced && !hasFixed && !hasLast && TryNormalizeVersion(patchedVersion, out var patchedNormalized)) - { - fixedVersion = patchedNormalized; - fixedInclusive = false; - hasFixed = true; - } - - if (!hasIntroduced && !hasFixed && !hasLast) - { - primitive = default!; - return false; - } - - var constraint = constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim(); - primitive = new SemVerPrimitive( - Introduced: hasIntroduced ? introduced : null, - IntroducedInclusive: hasIntroduced ? introducedInclusive : true, - Fixed: hasFixed ? fixedVersion : null, - FixedInclusive: hasFixed ? fixedInclusive : false, - LastAffected: hasLast ? lastAffected : null, - LastAffectedInclusive: hasLast ? lastInclusive : false, - ConstraintExpression: constraint); - return true; - } - - private static bool TryParseExact(string expression, out SemVerPrimitive primitive) - { - primitive = default!; - - if (!TryNormalizeVersion(expression, out var normalized)) - { - return false; - } - - primitive = new SemVerPrimitive( - Introduced: normalized, - IntroducedInclusive: true, - Fixed: normalized, - FixedInclusive: true, - LastAffected: null, - LastAffectedInclusive: false, - ConstraintExpression: normalized); - return true; - } - - private static bool TryParseComparatorFragment( - string fragment, - ICollection constraintTokens, - ref string? introduced, - ref bool introducedInclusive, - ref bool hasIntroduced, - ref string? fixedVersion, - ref bool fixedInclusive, - ref bool hasFixed, - ref string? lastAffected, - ref bool lastInclusive, - ref bool hasLast, - ref string? exactValue) - { - var trimmed = fragment.Trim(); - trimmed = trimmed.Trim(SegmentTrimCharacters); - - if (string.IsNullOrEmpty(trimmed)) - { - return false; - } - - var match = ComparatorRegex.Match(trimmed); - if (!match.Success) - { - return false; - } - - var op = match.Groups["op"].Value; - var rawValue = match.Groups["value"].Value; - if (!TryNormalizeVersion(rawValue, out var value)) - { - return true; - } - - switch (op) - { - case ">=": - introduced = value!; - introducedInclusive = true; - hasIntroduced = true; - constraintTokens.Add($">= {value}"); - break; - case ">": - introduced = value!; - introducedInclusive = false; - hasIntroduced = true; - constraintTokens.Add($"> {value}"); - break; - case "<=": - lastAffected = value!; - lastInclusive = true; - hasLast = true; - constraintTokens.Add($"<= {value}"); - break; - case "<": - fixedVersion = value!; - fixedInclusive = false; - hasFixed = true; - constraintTokens.Add($"< {value}"); - break; - case "=": - case "==": - exactValue = value!; - constraintTokens.Add($"= {value}"); - break; - } - - return true; - } - - private static bool TryNormalizeVersion(string? value, [NotNullWhen(true)] out string normalized) - { - normalized = string.Empty; - if (string.IsNullOrWhiteSpace(value)) - { - return false; - } - - var trimmed = value.Trim(); - trimmed = trimmed.Trim(SegmentTrimCharacters); - trimmed = trimmed.Trim('\'', '"', '`'); - - if (string.IsNullOrWhiteSpace(trimmed) || IsWildcardToken(trimmed)) - { - return false; - } - - var candidate = RemoveLeadingV(trimmed); - if (SemanticVersion.TryParse(candidate, out var semanticVersion)) - { - normalized = FormatVersion(semanticVersion); - return true; - } - - if (trimmed.IndexOfAny(new[] { '*', 'x', 'X' }) >= 0) - { - return false; - } - - normalized = candidate; - return true; - } - - private static bool TryParseSemanticVersion(string value, [NotNullWhen(true)] out SemanticVersion version, out string normalized) - { - normalized = string.Empty; - - var candidate = RemoveLeadingV(value); - if (!SemanticVersion.TryParse(candidate, out var parsed)) - { - candidate = ExpandSemanticVersion(candidate); - if (!SemanticVersion.TryParse(candidate, out parsed)) - { - version = null!; - return false; - } - } - - version = parsed!; - normalized = FormatVersion(parsed); - return true; - } - - private static string ExpandSemanticVersion(string value) - { - var partCount = value.Count(static ch => ch == '.'); - return partCount switch - { - 0 => value + ".0.0", - 1 => value + ".0", - _ => value, - }; - } - - private static SemanticVersion CalculateCaretUpperBound(SemanticVersion baseVersion) - { - if (baseVersion.Major > 0) - { - return new SemanticVersion(baseVersion.Major + 1, 0, 0); - } - - if (baseVersion.Minor > 0) - { - return new SemanticVersion(0, baseVersion.Minor + 1, 0); - } - - return new SemanticVersion(0, 0, baseVersion.Patch + 1); - } - - private static SemanticVersion CalculateTildeUpperBound(SemanticVersion baseVersion, int componentCount) - { - if (componentCount <= 1) - { - return new SemanticVersion(baseVersion.Major + 1, 0, 0); - } - - return new SemanticVersion(baseVersion.Major, baseVersion.Minor + 1, 0); - } - - private static int CountExplicitComponents(string value) - { - var head = value.Split(new[] { '-', '+' }, 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - return head.Length == 0 - ? 0 - : head[0].Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length; - } - - private static string NormalizeWhitespace(string value) - { - var builder = new StringBuilder(value.Length); - var previousWhitespace = false; - - foreach (var ch in value) - { - if (char.IsWhiteSpace(ch)) - { - if (!previousWhitespace) - { - builder.Append(' '); - previousWhitespace = true; - } - } - else - { - builder.Append(ch); - previousWhitespace = false; - } - } - - return builder.ToString().Trim(); - } - - private static string RemoveLeadingV(string value) - { - if (value.Length > 1 && (value[0] == 'v' || value[0] == 'V') && char.IsDigit(value[1])) - { - return value[1..]; - } - - return value; - } - - private static string FormatVersion(SemanticVersion version) - { - var normalized = version.ToNormalizedString(); - if (!string.IsNullOrEmpty(version.Metadata)) - { - normalized += "+" + version.Metadata; - } - - return normalized; - } - - private static bool IsWildcardToken(string value) - => string.Equals(value, "*", StringComparison.OrdinalIgnoreCase) - || string.Equals(value, "x", StringComparison.OrdinalIgnoreCase); -} - -public sealed record SemVerRangeBuildResult( - SemVerPrimitive Primitive, - NormalizedVersionRule NormalizedRule, - string Expression) -{ - public string? ConstraintExpression => Primitive.ConstraintExpression; -} +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using NuGet.Versioning; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Normalization.SemVer; + +/// +/// Parses SemVer-style range expressions into deterministic primitives and normalized rules. +/// Supports caret, tilde, wildcard, hyphen, and comparator syntaxes with optional provenance notes. +/// +public static class SemVerRangeRuleBuilder +{ + private static readonly Regex ComparatorRegex = new(@"^(?>=|<=|>|<|==|=)\s*(?.+)$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex ComparatorTokenRegex = new(@"(?(?>=|<=|>|<|==|=)\s*(?[^,\s\|]+))", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex HyphenRegex = new(@"^\s*(?.+?)\s+-\s+(?.+)\s*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly char[] SegmentTrimCharacters = { '(', ')', '[', ']', '{', '}', ';' }; + private static readonly char[] FragmentSplitCharacters = { ',', ' ' }; + + public static IReadOnlyList Build(string? rawRange, string? patchedVersion = null, string? provenanceNote = null) + { + var results = new List(); + + if (!string.IsNullOrWhiteSpace(rawRange)) + { + foreach (var segment in SplitSegments(rawRange)) + { + if (TryBuildFromSegment(segment, patchedVersion, provenanceNote, out var result)) + { + results.Add(result); + } + } + } + + if (results.Count == 0 && TryNormalizeVersion(patchedVersion, out var normalizedPatched)) + { + var constraint = $"< {normalizedPatched}"; + var fallbackPrimitive = new SemVerPrimitive( + Introduced: null, + IntroducedInclusive: false, + Fixed: normalizedPatched, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + + var fallbackRule = fallbackPrimitive.ToNormalizedVersionRule(provenanceNote); + if (fallbackRule is not null) + { + results.Add(new SemVerRangeBuildResult(fallbackPrimitive, fallbackRule, constraint)); + } + } + + return results.Count == 0 ? Array.Empty() : results; + } + + public static IReadOnlyList BuildNormalizedRules(string? rawRange, string? patchedVersion = null, string? provenanceNote = null) + { + var results = Build(rawRange, patchedVersion, provenanceNote); + if (results.Count == 0) + { + return Array.Empty(); + } + + var rules = new NormalizedVersionRule[results.Count]; + for (var i = 0; i < results.Count; i++) + { + rules[i] = results[i].NormalizedRule; + } + + return rules; + } + + private static IEnumerable SplitSegments(string rawRange) + => rawRange.Split("||", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + private static bool TryBuildFromSegment(string segment, string? patchedVersion, string? provenanceNote, out SemVerRangeBuildResult result) + { + result = default!; + + if (!TryParsePrimitive(segment, patchedVersion, out var primitive)) + { + return false; + } + + var normalized = primitive.ToNormalizedVersionRule(provenanceNote); + if (normalized is null) + { + return false; + } + + result = new SemVerRangeBuildResult(primitive, normalized, primitive.ConstraintExpression ?? segment.Trim()); + return true; + } + + private static bool TryParsePrimitive(string segment, string? patchedVersion, out SemVerPrimitive primitive) + { + primitive = default!; + + var trimmed = NormalizeWhitespace(segment); + if (string.IsNullOrEmpty(trimmed) || IsWildcardToken(trimmed)) + { + return false; + } + + if (TryParseCaret(trimmed, out primitive)) + { + return true; + } + + if (TryParseTilde(trimmed, out primitive)) + { + return true; + } + + if (TryParseHyphen(trimmed, out primitive)) + { + return true; + } + + if (TryParseWildcard(trimmed, out primitive)) + { + return true; + } + + if (TryParseComparators(trimmed, patchedVersion, out primitive)) + { + return true; + } + + if (TryParseExact(trimmed, out primitive)) + { + return true; + } + + return false; + } + + private static bool TryParseCaret(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!expression.StartsWith("^", StringComparison.Ordinal)) + { + return false; + } + + var value = expression[1..].Trim(); + if (!TryParseSemanticVersion(value, out var version, out var normalizedBase)) + { + return false; + } + + var upper = CalculateCaretUpperBound(version); + var upperNormalized = FormatVersion(upper); + var constraint = $">= {normalizedBase} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: normalizedBase, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseTilde(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!expression.StartsWith("~", StringComparison.Ordinal)) + { + return false; + } + + var remainder = expression.TrimStart('~'); + if (remainder.StartsWith(">", StringComparison.Ordinal)) + { + remainder = remainder[1..]; + } + + remainder = remainder.Trim(); + if (!TryParseSemanticVersion(remainder, out var version, out var normalizedBase)) + { + return false; + } + + var componentCount = CountExplicitComponents(remainder); + var upper = CalculateTildeUpperBound(version, componentCount); + var upperNormalized = FormatVersion(upper); + var constraint = $">= {normalizedBase} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: normalizedBase, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseHyphen(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + var match = HyphenRegex.Match(expression); + if (!match.Success) + { + return false; + } + + var startRaw = match.Groups["start"].Value.Trim(); + var endRaw = match.Groups["end"].Value.Trim(); + + if (!TryNormalizeVersion(startRaw, out var start) || !TryNormalizeVersion(endRaw, out var end)) + { + return false; + } + + var constraint = $"{start} - {end}"; + primitive = new SemVerPrimitive( + Introduced: start, + IntroducedInclusive: true, + Fixed: null, + FixedInclusive: false, + LastAffected: end, + LastAffectedInclusive: true, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseWildcard(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + var sanitized = expression.Trim(SegmentTrimCharacters).Trim(); + if (string.IsNullOrEmpty(sanitized)) + { + return false; + } + + var parts = sanitized.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0 || parts.All(static part => !IsWildcardToken(part))) + { + return false; + } + + if (!int.TryParse(RemoveLeadingV(parts[0]), NumberStyles.Integer, CultureInfo.InvariantCulture, out var major)) + { + return false; + } + + var hasMinor = parts.Length > 1 && !IsWildcardToken(parts[1]); + var hasPatch = parts.Length > 2 && !IsWildcardToken(parts[2]); + var minor = hasMinor && int.TryParse(parts[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedMinor) ? parsedMinor : 0; + var patch = hasPatch && int.TryParse(parts[2], NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedPatch) ? parsedPatch : 0; + + var minVersion = new SemanticVersion(major, hasMinor ? minor : 0, hasPatch ? patch : 0); + var minNormalized = FormatVersion(minVersion); + + SemanticVersion upperVersion; + if (!hasMinor) + { + upperVersion = new SemanticVersion(major + 1, 0, 0); + } + else if (!hasPatch) + { + upperVersion = new SemanticVersion(major, minor + 1, 0); + } + else + { + upperVersion = new SemanticVersion(major, minor + 1, 0); + } + + var upperNormalized = FormatVersion(upperVersion); + var constraint = $">= {minNormalized} < {upperNormalized}"; + + primitive = new SemVerPrimitive( + Introduced: minNormalized, + IntroducedInclusive: true, + Fixed: upperNormalized, + FixedInclusive: false, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseComparators(string expression, string? patchedVersion, out SemVerPrimitive primitive) + { + primitive = default!; + + var fragments = expression.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (fragments.Length == 0) + { + fragments = new[] { expression }; + } + + var constraintTokens = new List(); + + string? introduced = null; + bool introducedInclusive = true; + var hasIntroduced = false; + + string? fixedVersion = null; + bool fixedInclusive = false; + var hasFixed = false; + + string? lastAffected = null; + bool lastInclusive = true; + var hasLast = false; + + string? exactValue = null; + + foreach (var fragment in fragments) + { + var handled = false; + + foreach (Match match in ComparatorTokenRegex.Matches(fragment)) + { + if (match.Groups["token"].Success + && TryParseComparatorFragment(match.Groups["token"].Value, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) + { + handled = true; + } + } + + if (!handled) + { + var parts = fragment.Split(FragmentSplitCharacters, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var part in parts) + { + if (TryParseComparatorFragment(part, constraintTokens, ref introduced, ref introducedInclusive, ref hasIntroduced, ref fixedVersion, ref fixedInclusive, ref hasFixed, ref lastAffected, ref lastInclusive, ref hasLast, ref exactValue)) + { + handled = true; + } + } + } + + if (handled) + { + continue; + } + + if (TryNormalizeVersion(fragment, out var normalizedExact)) + { + exactValue = normalizedExact; + constraintTokens.Add(normalizedExact); + } + } + + if (exactValue is not null) + { + primitive = new SemVerPrimitive( + Introduced: exactValue, + IntroducedInclusive: true, + Fixed: exactValue, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim(), + ExactValue: exactValue); + return true; + } + + if (hasIntroduced && !hasFixed && !hasLast && TryNormalizeVersion(patchedVersion, out var patchedNormalized)) + { + fixedVersion = patchedNormalized; + fixedInclusive = false; + hasFixed = true; + } + + if (!hasIntroduced && !hasFixed && !hasLast) + { + primitive = default!; + return false; + } + + var constraint = constraintTokens.Count > 0 ? string.Join(", ", constraintTokens) : expression.Trim(); + primitive = new SemVerPrimitive( + Introduced: hasIntroduced ? introduced : null, + IntroducedInclusive: hasIntroduced ? introducedInclusive : true, + Fixed: hasFixed ? fixedVersion : null, + FixedInclusive: hasFixed ? fixedInclusive : false, + LastAffected: hasLast ? lastAffected : null, + LastAffectedInclusive: hasLast ? lastInclusive : false, + ConstraintExpression: constraint); + return true; + } + + private static bool TryParseExact(string expression, out SemVerPrimitive primitive) + { + primitive = default!; + + if (!TryNormalizeVersion(expression, out var normalized)) + { + return false; + } + + primitive = new SemVerPrimitive( + Introduced: normalized, + IntroducedInclusive: true, + Fixed: normalized, + FixedInclusive: true, + LastAffected: null, + LastAffectedInclusive: false, + ConstraintExpression: normalized); + return true; + } + + private static bool TryParseComparatorFragment( + string fragment, + ICollection constraintTokens, + ref string? introduced, + ref bool introducedInclusive, + ref bool hasIntroduced, + ref string? fixedVersion, + ref bool fixedInclusive, + ref bool hasFixed, + ref string? lastAffected, + ref bool lastInclusive, + ref bool hasLast, + ref string? exactValue) + { + var trimmed = fragment.Trim(); + trimmed = trimmed.Trim(SegmentTrimCharacters); + + if (string.IsNullOrEmpty(trimmed)) + { + return false; + } + + var match = ComparatorRegex.Match(trimmed); + if (!match.Success) + { + return false; + } + + var op = match.Groups["op"].Value; + var rawValue = match.Groups["value"].Value; + if (!TryNormalizeVersion(rawValue, out var value)) + { + return true; + } + + switch (op) + { + case ">=": + introduced = value!; + introducedInclusive = true; + hasIntroduced = true; + constraintTokens.Add($">= {value}"); + break; + case ">": + introduced = value!; + introducedInclusive = false; + hasIntroduced = true; + constraintTokens.Add($"> {value}"); + break; + case "<=": + lastAffected = value!; + lastInclusive = true; + hasLast = true; + constraintTokens.Add($"<= {value}"); + break; + case "<": + fixedVersion = value!; + fixedInclusive = false; + hasFixed = true; + constraintTokens.Add($"< {value}"); + break; + case "=": + case "==": + exactValue = value!; + constraintTokens.Add($"= {value}"); + break; + } + + return true; + } + + private static bool TryNormalizeVersion(string? value, [NotNullWhen(true)] out string normalized) + { + normalized = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + trimmed = trimmed.Trim(SegmentTrimCharacters); + trimmed = trimmed.Trim('\'', '"', '`'); + + if (string.IsNullOrWhiteSpace(trimmed) || IsWildcardToken(trimmed)) + { + return false; + } + + var candidate = RemoveLeadingV(trimmed); + if (SemanticVersion.TryParse(candidate, out var semanticVersion)) + { + normalized = FormatVersion(semanticVersion); + return true; + } + + if (trimmed.IndexOfAny(new[] { '*', 'x', 'X' }) >= 0) + { + return false; + } + + normalized = candidate; + return true; + } + + private static bool TryParseSemanticVersion(string value, [NotNullWhen(true)] out SemanticVersion version, out string normalized) + { + normalized = string.Empty; + + var candidate = RemoveLeadingV(value); + if (!SemanticVersion.TryParse(candidate, out var parsed)) + { + candidate = ExpandSemanticVersion(candidate); + if (!SemanticVersion.TryParse(candidate, out parsed)) + { + version = null!; + return false; + } + } + + version = parsed!; + normalized = FormatVersion(parsed); + return true; + } + + private static string ExpandSemanticVersion(string value) + { + var partCount = value.Count(static ch => ch == '.'); + return partCount switch + { + 0 => value + ".0.0", + 1 => value + ".0", + _ => value, + }; + } + + private static SemanticVersion CalculateCaretUpperBound(SemanticVersion baseVersion) + { + if (baseVersion.Major > 0) + { + return new SemanticVersion(baseVersion.Major + 1, 0, 0); + } + + if (baseVersion.Minor > 0) + { + return new SemanticVersion(0, baseVersion.Minor + 1, 0); + } + + return new SemanticVersion(0, 0, baseVersion.Patch + 1); + } + + private static SemanticVersion CalculateTildeUpperBound(SemanticVersion baseVersion, int componentCount) + { + if (componentCount <= 1) + { + return new SemanticVersion(baseVersion.Major + 1, 0, 0); + } + + return new SemanticVersion(baseVersion.Major, baseVersion.Minor + 1, 0); + } + + private static int CountExplicitComponents(string value) + { + var head = value.Split(new[] { '-', '+' }, 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return head.Length == 0 + ? 0 + : head[0].Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Length; + } + + private static string NormalizeWhitespace(string value) + { + var builder = new StringBuilder(value.Length); + var previousWhitespace = false; + + foreach (var ch in value) + { + if (char.IsWhiteSpace(ch)) + { + if (!previousWhitespace) + { + builder.Append(' '); + previousWhitespace = true; + } + } + else + { + builder.Append(ch); + previousWhitespace = false; + } + } + + return builder.ToString().Trim(); + } + + private static string RemoveLeadingV(string value) + { + if (value.Length > 1 && (value[0] == 'v' || value[0] == 'V') && char.IsDigit(value[1])) + { + return value[1..]; + } + + return value; + } + + private static string FormatVersion(SemanticVersion version) + { + var normalized = version.ToNormalizedString(); + if (!string.IsNullOrEmpty(version.Metadata)) + { + normalized += "+" + version.Metadata; + } + + return normalized; + } + + private static bool IsWildcardToken(string value) + => string.Equals(value, "*", StringComparison.OrdinalIgnoreCase) + || string.Equals(value, "x", StringComparison.OrdinalIgnoreCase); +} + +public sealed record SemVerRangeBuildResult( + SemVerPrimitive Primitive, + NormalizedVersionRule NormalizedRule, + string Expression) +{ + public string? ConstraintExpression => Primitive.ConstraintExpression; +} diff --git a/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj b/src/StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj similarity index 81% rename from src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj rename to src/StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj index 203716ed..e3e2f671 100644 --- a/src/StellaOps.Feedser.Normalization/StellaOps.Feedser.Normalization.csproj +++ b/src/StellaOps.Concelier.Normalization/StellaOps.Concelier.Normalization.csproj @@ -13,7 +13,7 @@ - + diff --git a/src/StellaOps.Feedser.Normalization/TASKS.md b/src/StellaOps.Concelier.Normalization/TASKS.md similarity index 100% rename from src/StellaOps.Feedser.Normalization/TASKS.md rename to src/StellaOps.Concelier.Normalization/TASKS.md diff --git a/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs b/src/StellaOps.Concelier.Normalization/Text/DescriptionNormalizer.cs similarity index 95% rename from src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs rename to src/StellaOps.Concelier.Normalization/Text/DescriptionNormalizer.cs index d43d25a5..047f2511 100644 --- a/src/StellaOps.Feedser.Normalization/Text/DescriptionNormalizer.cs +++ b/src/StellaOps.Concelier.Normalization/Text/DescriptionNormalizer.cs @@ -3,7 +3,7 @@ using System.Linq; using System.Net; using System.Text.RegularExpressions; -namespace StellaOps.Feedser.Normalization.Text; +namespace StellaOps.Concelier.Normalization.Text; /// /// Normalizes advisory descriptions by stripping markup, collapsing whitespace, and selecting the best locale fallback. diff --git a/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryConflictStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryConflictStoreTests.cs new file mode 100644 index 00000000..4da4feed --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryConflictStoreTests.cs @@ -0,0 +1,82 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Conflicts; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryConflictStoreTests +{ + private readonly IMongoDatabase _database; + + public AdvisoryConflictStoreTests(MongoIntegrationFixture fixture) + { + _database = fixture.Database ?? throw new ArgumentNullException(nameof(fixture.Database)); + } + + [Fact] + public async Task InsertAndRetrieve_PersistsConflicts() + { + var store = new AdvisoryConflictStore(_database); + var vulnerabilityKey = $"CVE-{Guid.NewGuid():N}"; + var baseTime = DateTimeOffset.UtcNow; + var statementIds = new[] { Guid.NewGuid(), Guid.NewGuid() }; + + var conflict = new AdvisoryConflictRecord( + Guid.NewGuid(), + vulnerabilityKey, + new byte[] { 0x10, 0x20 }, + baseTime, + baseTime.AddSeconds(30), + statementIds, + new BsonDocument("explanation", "first-pass")); + + await store.InsertAsync(new[] { conflict }, CancellationToken.None); + + var results = await store.GetConflictsAsync(vulnerabilityKey, null, CancellationToken.None); + + Assert.Single(results); + Assert.Equal(conflict.Id, results[0].Id); + Assert.Equal(statementIds, results[0].StatementIds); + } + + [Fact] + public async Task GetConflicts_AsOfFilters() + { + var store = new AdvisoryConflictStore(_database); + var vulnerabilityKey = $"CVE-{Guid.NewGuid():N}"; + var baseTime = DateTimeOffset.UtcNow; + + var earlyConflict = new AdvisoryConflictRecord( + Guid.NewGuid(), + vulnerabilityKey, + new byte[] { 0x01 }, + baseTime, + baseTime.AddSeconds(10), + new[] { Guid.NewGuid() }, + new BsonDocument("stage", "early")); + + var lateConflict = new AdvisoryConflictRecord( + Guid.NewGuid(), + vulnerabilityKey, + new byte[] { 0x02 }, + baseTime.AddMinutes(10), + baseTime.AddMinutes(10).AddSeconds(15), + new[] { Guid.NewGuid() }, + new BsonDocument("stage", "late")); + + await store.InsertAsync(new[] { earlyConflict, lateConflict }, CancellationToken.None); + + var results = await store.GetConflictsAsync(vulnerabilityKey, baseTime.AddMinutes(1), CancellationToken.None); + + Assert.Single(results); + Assert.Equal("early", results[0].Details["stage"].AsString); + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStatementStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStatementStoreTests.cs new file mode 100644 index 00000000..e96394d7 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStatementStoreTests.cs @@ -0,0 +1,96 @@ +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Statements; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class AdvisoryStatementStoreTests +{ + private readonly IMongoDatabase _database; + + public AdvisoryStatementStoreTests(MongoIntegrationFixture fixture) + { + _database = fixture.Database ?? throw new ArgumentNullException(nameof(fixture.Database)); + } + + [Fact] + public async Task InsertAndRetrieve_WritesImmutableStatements() + { + var store = new AdvisoryStatementStore(_database); + var vulnerabilityKey = $"CVE-{Guid.NewGuid():N}"; + var baseTime = DateTimeOffset.UtcNow; + + var statements = new[] + { + new AdvisoryStatementRecord( + Guid.NewGuid(), + vulnerabilityKey, + vulnerabilityKey, + new byte[] { 0x01 }, + baseTime, + baseTime.AddSeconds(5), + new BsonDocument("version", "A"), + new[] { Guid.NewGuid() }), + new AdvisoryStatementRecord( + Guid.NewGuid(), + vulnerabilityKey, + vulnerabilityKey, + new byte[] { 0x02 }, + baseTime.AddMinutes(1), + baseTime.AddMinutes(1).AddSeconds(5), + new BsonDocument("version", "B"), + Array.Empty()), + }; + + await store.InsertAsync(statements, CancellationToken.None); + + var results = await store.GetStatementsAsync(vulnerabilityKey, null, CancellationToken.None); + + Assert.Equal(2, results.Count); + Assert.Equal(statements[1].Id, results[0].Id); // sorted by AsOf desc + Assert.True(results.All(record => record.Payload.Contains("version"))); + } + + [Fact] + public async Task GetStatements_AsOfFiltersResults() + { + var store = new AdvisoryStatementStore(_database); + var vulnerabilityKey = $"CVE-{Guid.NewGuid():N}"; + var baseTime = DateTimeOffset.UtcNow; + + var early = new AdvisoryStatementRecord( + Guid.NewGuid(), + vulnerabilityKey, + vulnerabilityKey, + new byte[] { 0xAA }, + baseTime, + baseTime.AddSeconds(10), + new BsonDocument("state", "early"), + Array.Empty()); + + var late = new AdvisoryStatementRecord( + Guid.NewGuid(), + vulnerabilityKey, + vulnerabilityKey, + new byte[] { 0xBB }, + baseTime.AddMinutes(5), + baseTime.AddMinutes(5).AddSeconds(10), + new BsonDocument("state", "late"), + Array.Empty()); + + await store.InsertAsync(new[] { early, late }, CancellationToken.None); + + var results = await store.GetStatementsAsync(vulnerabilityKey, baseTime.AddMinutes(1), CancellationToken.None); + + Assert.Single(results); + Assert.Equal("early", results[0].Payload["state"].AsString); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs index 07d79762..4e0c1d16 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/AdvisoryStorePerformanceTests.cs @@ -3,15 +3,15 @@ using System.Linq; using System.Threading; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Aliases; -using StellaOps.Feedser.Storage.Mongo.Migrations; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo.Migrations; using Xunit; using Xunit.Abstractions; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class AdvisoryStorePerformanceTests : IClassFixture @@ -43,7 +43,7 @@ public sealed class AdvisoryStorePerformanceTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/AliasStoreTests.cs similarity index 91% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/AliasStoreTests.cs index 122b29b3..7ab62387 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/AliasStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/AliasStoreTests.cs @@ -3,10 +3,10 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Driver; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Aliases; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class AliasStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/DocumentStoreTests.cs similarity index 91% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/DocumentStoreTests.cs index f4f9b2b5..66f41f09 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/DocumentStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/DocumentStoreTests.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Documents; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class DocumentStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/DtoStoreTests.cs similarity index 89% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/DtoStoreTests.cs index 4bdf309e..c9046dfd 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/DtoStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/DtoStoreTests.cs @@ -1,8 +1,8 @@ using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Bson; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.Dtos; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class DtoStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateManagerTests.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateManagerTests.cs index 7e4dc9dd..f7b1a720 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateManagerTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateManagerTests.cs @@ -2,9 +2,9 @@ using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Concelier.Storage.Mongo.Exporting; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; public sealed class ExportStateManagerTests { diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateStoreTests.cs similarity index 90% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateStoreTests.cs index 8eed8ef3..67f9ba63 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/ExportStateStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/ExportStateStoreTests.cs @@ -1,8 +1,8 @@ using System; using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Storage.Mongo.Exporting; +using StellaOps.Concelier.Storage.Mongo.Exporting; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class ExportStateStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/MergeEventStoreTests.cs similarity index 88% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/MergeEventStoreTests.cs index 3758cf73..496a5ed6 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/MergeEventStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/MergeEventStoreTests.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class MergeEventStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs similarity index 75% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs index 9db94f41..4ad33e26 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/Migrations/MongoMigrationRunnerTests.cs @@ -6,11 +6,11 @@ using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Migrations; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Migrations; using Xunit; -namespace StellaOps.Feedser.Storage.Mongo.Tests.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Tests.Migrations; [Collection("mongo-fixture")] public sealed class MongoMigrationRunnerTests @@ -25,7 +25,7 @@ public sealed class MongoMigrationRunnerTests [Fact] public async Task RunAsync_AppliesPendingMigrationsOnce() { - var databaseName = $"feedser-migrations-{Guid.NewGuid():N}"; + var databaseName = $"concelier-migrations-{Guid.NewGuid():N}"; var database = _fixture.Client.GetDatabase(databaseName); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); @@ -57,7 +57,7 @@ public sealed class MongoMigrationRunnerTests [Fact] public async Task EnsureDocumentExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() { - var databaseName = $"feedser-doc-ttl-{Guid.NewGuid():N}"; + var databaseName = $"concelier-doc-ttl-{Guid.NewGuid():N}"; var database = _fixture.Client.GetDatabase(databaseName); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); @@ -97,7 +97,7 @@ public sealed class MongoMigrationRunnerTests [Fact] public async Task EnsureDocumentExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() { - var databaseName = $"feedser-doc-notl-{Guid.NewGuid():N}"; + var databaseName = $"concelier-doc-notl-{Guid.NewGuid():N}"; var database = _fixture.Client.GetDatabase(databaseName); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Document); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); @@ -144,7 +144,7 @@ public sealed class MongoMigrationRunnerTests [Fact] public async Task EnsureGridFsExpiryIndexesMigration_CreatesTtlIndexWhenRetentionEnabled() { - var databaseName = $"feedser-gridfs-ttl-{Guid.NewGuid():N}"; + var databaseName = $"concelier-gridfs-ttl-{Guid.NewGuid():N}"; var database = _fixture.Client.GetDatabase(databaseName); await database.CreateCollectionAsync("documents.files"); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); @@ -179,7 +179,7 @@ public sealed class MongoMigrationRunnerTests [Fact] public async Task EnsureGridFsExpiryIndexesMigration_DropsTtlIndexWhenRetentionDisabled() { - var databaseName = $"feedser-gridfs-notl-{Guid.NewGuid():N}"; + var databaseName = $"concelier-gridfs-notl-{Guid.NewGuid():N}"; var database = _fixture.Client.GetDatabase(databaseName); await database.CreateCollectionAsync("documents.files"); await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); @@ -215,15 +215,59 @@ public sealed class MongoMigrationRunnerTests Assert.DoesNotContain(indexList, x => x["name"].AsString == "gridfs_files_expiresAt_ttl"); } - finally - { - await _fixture.Client.DropDatabaseAsync(databaseName); - } - } - - private sealed class TestMigration : IMongoMigration - { - public int ApplyCount { get; private set; } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task EnsureAdvisoryEventCollectionsMigration_CreatesIndexes() + { + var databaseName = $"concelier-advisory-events-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.AdvisoryStatements); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.AdvisoryConflicts); + await database.CreateCollectionAsync(MongoStorageDefaults.Collections.Migrations); + + try + { + var migration = new EnsureAdvisoryEventCollectionsMigration(); + var runner = new MongoMigrationRunner( + database, + new IMongoMigration[] { migration }, + NullLogger.Instance, + TimeProvider.System); + + await runner.RunAsync(CancellationToken.None); + + var statementIndexes = await database + .GetCollection(MongoStorageDefaults.Collections.AdvisoryStatements) + .Indexes + .ListAsync(); + var statementIndexNames = (await statementIndexes.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.Contains("advisory_statements_vulnerability_asof_desc", statementIndexNames); + Assert.Contains("advisory_statements_statementHash_unique", statementIndexNames); + + var conflictIndexes = await database + .GetCollection(MongoStorageDefaults.Collections.AdvisoryConflicts) + .Indexes + .ListAsync(); + var conflictIndexNames = (await conflictIndexes.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.Contains("advisory_conflicts_vulnerability_asof_desc", conflictIndexNames); + Assert.Contains("advisory_conflicts_conflictHash_unique", conflictIndexNames); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + private sealed class TestMigration : IMongoMigration + { + public int ApplyCount { get; private set; } public string Id => "999_test"; diff --git a/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoAdvisoryEventRepositoryTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoAdvisoryEventRepositoryTests.cs new file mode 100644 index 00000000..1b59cc28 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoAdvisoryEventRepositoryTests.cs @@ -0,0 +1,107 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Conflicts; +using StellaOps.Concelier.Storage.Mongo.Events; +using StellaOps.Concelier.Storage.Mongo.Statements; +using StellaOps.Concelier.Testing; +using Xunit; + +namespace StellaOps.Concelier.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoAdvisoryEventRepositoryTests +{ + private readonly IMongoDatabase _database; + private readonly MongoAdvisoryEventRepository _repository; + + public MongoAdvisoryEventRepositoryTests(MongoIntegrationFixture fixture) + { + _database = fixture.Database ?? throw new ArgumentNullException(nameof(fixture.Database)); + var statementStore = new AdvisoryStatementStore(_database); + var conflictStore = new AdvisoryConflictStore(_database); + _repository = new MongoAdvisoryEventRepository(statementStore, conflictStore); + } + + [Fact] + public async Task InsertAndFetchStatements_RoundTripsCanonicalPayload() + { + var advisory = CreateSampleAdvisory("CVE-2025-7777", "Sample advisory"); + var canonicalJson = CanonicalJsonSerializer.Serialize(advisory); + var hash = ImmutableArray.Create(SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson))); + + var entry = new AdvisoryStatementEntry( + Guid.NewGuid(), + "CVE-2025-7777", + "CVE-2025-7777", + canonicalJson, + hash, + DateTimeOffset.Parse("2025-10-19T14:00:00Z"), + DateTimeOffset.Parse("2025-10-19T14:05:00Z"), + ImmutableArray.Empty); + + await _repository.InsertStatementsAsync(new[] { entry }, CancellationToken.None); + + var results = await _repository.GetStatementsAsync("CVE-2025-7777", null, CancellationToken.None); + + var snapshot = Assert.Single(results); + Assert.Equal(entry.StatementId, snapshot.StatementId); + Assert.Equal(entry.CanonicalJson, snapshot.CanonicalJson); + Assert.True(entry.StatementHash.SequenceEqual(snapshot.StatementHash)); + } + + [Fact] + public async Task InsertAndFetchConflicts_PreservesDetails() + { + var detailJson = CanonicalJsonSerializer.Serialize(new ConflictPayload("severity", "mismatch")); + var hash = ImmutableArray.Create(SHA256.HashData(Encoding.UTF8.GetBytes(detailJson))); + var statementIds = ImmutableArray.Create(Guid.NewGuid(), Guid.NewGuid()); + + var entry = new AdvisoryConflictEntry( + Guid.NewGuid(), + "CVE-2025-4242", + detailJson, + hash, + DateTimeOffset.Parse("2025-10-19T15:00:00Z"), + DateTimeOffset.Parse("2025-10-19T15:05:00Z"), + statementIds); + + await _repository.InsertConflictsAsync(new[] { entry }, CancellationToken.None); + + var results = await _repository.GetConflictsAsync("CVE-2025-4242", null, CancellationToken.None); + + var conflict = Assert.Single(results); + Assert.Equal(entry.CanonicalJson, conflict.CanonicalJson); + Assert.True(entry.StatementIds.SequenceEqual(conflict.StatementIds)); + Assert.True(entry.ConflictHash.SequenceEqual(conflict.ConflictHash)); + } + + private static Advisory CreateSampleAdvisory(string key, string summary) + { + var provenance = new AdvisoryProvenance("nvd", "document", key, DateTimeOffset.Parse("2025-10-18T00:00:00Z"), new[] { ProvenanceFieldMasks.Advisory }); + return new Advisory( + key, + key, + summary, + "en", + DateTimeOffset.Parse("2025-10-17T00:00:00Z"), + DateTimeOffset.Parse("2025-10-18T00:00:00Z"), + "medium", + exploitKnown: false, + aliases: new[] { key }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: new[] { provenance }); + } + + private sealed record ConflictPayload(string Type, string Reason); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoBootstrapperTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoBootstrapperTests.cs similarity index 57% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/MongoBootstrapperTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/MongoBootstrapperTests.cs index 80b5fefd..d3a87da9 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoBootstrapperTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoBootstrapperTests.cs @@ -1,97 +1,143 @@ -using System; -using System.Linq; -using System.Threading; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Migrations; -using Xunit; - -namespace StellaOps.Feedser.Storage.Mongo.Tests; - -[Collection("mongo-fixture")] -public sealed class MongoBootstrapperTests : IClassFixture -{ - private readonly MongoIntegrationFixture _fixture; - - public MongoBootstrapperTests(MongoIntegrationFixture fixture) - { - _fixture = fixture; - } - - [Fact] - public async Task InitializeAsync_CreatesNormalizedIndexesWhenSemVerStyleEnabled() - { - var databaseName = $"feedser-bootstrap-semver-{Guid.NewGuid():N}"; - var database = _fixture.Client.GetDatabase(databaseName); - - try - { - var runner = new MongoMigrationRunner( - database, - Array.Empty(), - NullLogger.Instance, - TimeProvider.System); - - var bootstrapper = new MongoBootstrapper( - database, - Options.Create(new MongoStorageOptions { EnableSemVerStyle = true }), - NullLogger.Instance, - runner); - - await bootstrapper.InitializeAsync(CancellationToken.None); - - var indexCursor = await database - .GetCollection(MongoStorageDefaults.Collections.Advisory) - .Indexes - .ListAsync(); - var indexNames = (await indexCursor.ToListAsync()).Select(x => x["name"].AsString).ToArray(); - - Assert.Contains("advisory_normalizedVersions_pkg_scheme_type", indexNames); - Assert.Contains("advisory_normalizedVersions_value", indexNames); - } - finally - { - await _fixture.Client.DropDatabaseAsync(databaseName); - } - } - - [Fact] - public async Task InitializeAsync_DoesNotCreateNormalizedIndexesWhenFeatureDisabled() - { - var databaseName = $"feedser-bootstrap-no-semver-{Guid.NewGuid():N}"; - var database = _fixture.Client.GetDatabase(databaseName); - - try - { - var runner = new MongoMigrationRunner( - database, - Array.Empty(), - NullLogger.Instance, - TimeProvider.System); - - var bootstrapper = new MongoBootstrapper( - database, - Options.Create(new MongoStorageOptions { EnableSemVerStyle = false }), - NullLogger.Instance, - runner); - - await bootstrapper.InitializeAsync(CancellationToken.None); - - var indexCursor = await database - .GetCollection(MongoStorageDefaults.Collections.Advisory) - .Indexes - .ListAsync(); - var indexNames = (await indexCursor.ToListAsync()).Select(x => x["name"].AsString).ToArray(); - - Assert.DoesNotContain("advisory_normalizedVersions_pkg_scheme_type", indexNames); - Assert.DoesNotContain("advisory_normalizedVersions_value", indexNames); - } - finally - { - await _fixture.Client.DropDatabaseAsync(databaseName); - } - } -} +using System; +using System.Linq; +using System.Threading; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Migrations; +using Xunit; + +namespace StellaOps.Concelier.Storage.Mongo.Tests; + +[Collection("mongo-fixture")] +public sealed class MongoBootstrapperTests : IClassFixture +{ + private readonly MongoIntegrationFixture _fixture; + + public MongoBootstrapperTests(MongoIntegrationFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task InitializeAsync_CreatesNormalizedIndexesWhenSemVerStyleEnabled() + { + var databaseName = $"concelier-bootstrap-semver-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + + try + { + var runner = new MongoMigrationRunner( + database, + Array.Empty(), + NullLogger.Instance, + TimeProvider.System); + + var bootstrapper = new MongoBootstrapper( + database, + Options.Create(new MongoStorageOptions { EnableSemVerStyle = true }), + NullLogger.Instance, + runner); + + await bootstrapper.InitializeAsync(CancellationToken.None); + + var indexCursor = await database + .GetCollection(MongoStorageDefaults.Collections.Advisory) + .Indexes + .ListAsync(); + var indexNames = (await indexCursor.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.Contains("advisory_normalizedVersions_pkg_scheme_type", indexNames); + Assert.Contains("advisory_normalizedVersions_value", indexNames); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task InitializeAsync_DoesNotCreateNormalizedIndexesWhenFeatureDisabled() + { + var databaseName = $"concelier-bootstrap-no-semver-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + + try + { + var runner = new MongoMigrationRunner( + database, + Array.Empty(), + NullLogger.Instance, + TimeProvider.System); + + var bootstrapper = new MongoBootstrapper( + database, + Options.Create(new MongoStorageOptions { EnableSemVerStyle = false }), + NullLogger.Instance, + runner); + + await bootstrapper.InitializeAsync(CancellationToken.None); + + var indexCursor = await database + .GetCollection(MongoStorageDefaults.Collections.Advisory) + .Indexes + .ListAsync(); + var indexNames = (await indexCursor.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.DoesNotContain("advisory_normalizedVersions_pkg_scheme_type", indexNames); + Assert.DoesNotContain("advisory_normalizedVersions_value", indexNames); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } + + [Fact] + public async Task InitializeAsync_CreatesAdvisoryEventIndexes() + { + var databaseName = $"concelier-bootstrap-events-{Guid.NewGuid():N}"; + var database = _fixture.Client.GetDatabase(databaseName); + + try + { + var runner = new MongoMigrationRunner( + database, + Array.Empty(), + NullLogger.Instance, + TimeProvider.System); + + var bootstrapper = new MongoBootstrapper( + database, + Options.Create(new MongoStorageOptions()), + NullLogger.Instance, + runner); + + await bootstrapper.InitializeAsync(CancellationToken.None); + + var statementIndexes = await database + .GetCollection(MongoStorageDefaults.Collections.AdvisoryStatements) + .Indexes + .ListAsync(); + var statementIndexNames = (await statementIndexes.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.Contains("advisory_statements_vulnerability_asof_desc", statementIndexNames); + Assert.Contains("advisory_statements_statementHash_unique", statementIndexNames); + + var conflictIndexes = await database + .GetCollection(MongoStorageDefaults.Collections.AdvisoryConflicts) + .Indexes + .ListAsync(); + var conflictIndexNames = (await conflictIndexes.ToListAsync()).Select(x => x["name"].AsString).ToArray(); + + Assert.Contains("advisory_conflicts_vulnerability_asof_desc", conflictIndexNames); + Assert.Contains("advisory_conflicts_conflictHash_unique", conflictIndexNames); + } + finally + { + await _fixture.Client.DropDatabaseAsync(databaseName); + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoJobStoreTests.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/MongoJobStoreTests.cs index 1aea3b9c..c7bde49d 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoJobStoreTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoJobStoreTests.cs @@ -1,9 +1,9 @@ using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Driver; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Storage.Mongo; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class MongoJobStoreTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs index af24393a..3ef2e1c3 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/MongoSourceStateRepositoryTests.cs @@ -1,8 +1,8 @@ using Microsoft.Extensions.Logging.Abstractions; using MongoDB.Bson; -using StellaOps.Feedser.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class MongoSourceStateRepositoryTests : IClassFixture diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs b/src/StellaOps.Concelier.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs similarity index 92% rename from src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs rename to src/StellaOps.Concelier.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs index 8112b16b..7e062c19 100644 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/RawDocumentRetentionServiceTests.cs @@ -4,11 +4,11 @@ using Microsoft.Extensions.Time.Testing; using MongoDB.Bson; using MongoDB.Driver; using MongoDB.Driver.GridFS; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; -namespace StellaOps.Feedser.Storage.Mongo.Tests; +namespace StellaOps.Concelier.Storage.Mongo.Tests; [Collection("mongo-fixture")] public sealed class RawDocumentRetentionServiceTests : IClassFixture diff --git a/src/StellaOps.Concelier.Storage.Mongo.Tests/StellaOps.Concelier.Storage.Mongo.Tests.csproj b/src/StellaOps.Concelier.Storage.Mongo.Tests/StellaOps.Concelier.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..6b70273e --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo.Tests/StellaOps.Concelier.Storage.Mongo.Tests.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md b/src/StellaOps.Concelier.Storage.Mongo/AGENTS.md similarity index 90% rename from src/StellaOps.Feedser.Storage.Mongo/AGENTS.md rename to src/StellaOps.Concelier.Storage.Mongo/AGENTS.md index a1f2b543..b60532c6 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/AGENTS.md +++ b/src/StellaOps.Concelier.Storage.Mongo/AGENTS.md @@ -12,7 +12,7 @@ Canonical persistence for raw documents, DTOs, canonical advisories, jobs, and s - Source connectors store raw docs, DTOs, and mapped canonical advisories with provenance; Update SourceState cursor/backoff. - Exporters read advisories and write export_state. ## Interfaces & contracts -- IMongoDatabase injected; MongoUrl from options; database name from options or MongoUrl or default "feedser". +- IMongoDatabase injected; MongoUrl from options; database name from options or MongoUrl or default "concelier". - Repositories expose async methods with CancellationToken; deterministic sorting. - All date/time values stored as UTC; identifiers normalized. ## In/Out of scope @@ -23,7 +23,7 @@ Out: business mapping logic, HTTP, packaging. - Timeouts and retry policies; avoid unbounded scans; page reads. - Do not log DSNs with credentials; redact in diagnostics. ## Tests -- Author and review coverage in `../StellaOps.Feedser.Storage.Mongo.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.Storage.Mongo.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryDocument.cs similarity index 90% rename from src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryDocument.cs index 8077974e..9891e6b5 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryDocument.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Advisories; +namespace StellaOps.Concelier.Storage.Mongo.Advisories; [BsonIgnoreExtraElements] public sealed class AdvisoryDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryStore.cs similarity index 92% rename from src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryStore.cs index 0fa8f135..903dd6b8 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/AdvisoryStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Advisories/AdvisoryStore.cs @@ -8,10 +8,10 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Aliases; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Aliases; -namespace StellaOps.Feedser.Storage.Mongo.Advisories; +namespace StellaOps.Concelier.Storage.Mongo.Advisories; public sealed class AdvisoryStore : IAdvisoryStore { @@ -37,7 +37,7 @@ public sealed class AdvisoryStore : IAdvisoryStore } - public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) + public async Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null) { ArgumentNullException.ThrowIfNull(advisory); @@ -67,24 +67,35 @@ public sealed class AdvisoryStore : IAdvisoryStore NormalizedVersions = normalizedVersions, }; - var options = new ReplaceOptions { IsUpsert = true }; - await _collection.ReplaceOneAsync(x => x.AdvisoryKey == advisory.AdvisoryKey, document, options, cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey); - - var aliasEntries = BuildAliasEntries(advisory); - var updatedAt = _timeProvider.GetUtcNow(); - await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false); - } - - public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrEmpty(advisoryKey); - var document = await _collection.Find(x => x.AdvisoryKey == advisoryKey) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - - return document is null ? null : Deserialize(document.Payload); - } + var options = new ReplaceOptions { IsUpsert = true }; + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisory.AdvisoryKey); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + _logger.LogDebug("Upserted advisory {AdvisoryKey}", advisory.AdvisoryKey); + + var aliasEntries = BuildAliasEntries(advisory); + var updatedAt = _timeProvider.GetUtcNow(); + await _aliasStore.ReplaceAsync(advisory.AdvisoryKey, aliasEntries, updatedAt, cancellationToken).ConfigureAwait(false); + } + + public async Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrEmpty(advisoryKey); + var filter = Builders.Filter.Eq(x => x.AdvisoryKey, advisoryKey); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + return document is null ? null : Deserialize(document.Payload); + } private static IEnumerable BuildAliasEntries(Advisory advisory) { @@ -103,29 +114,31 @@ public sealed class AdvisoryStore : IAdvisoryStore yield return new AliasEntry(AliasStoreConstants.PrimaryScheme, advisory.AdvisoryKey); } - public async Task> GetRecentAsync(int limit, CancellationToken cancellationToken) - { - var cursor = await _collection.Find(FilterDefinition.Empty) - .SortByDescending(x => x.Modified) - .Limit(limit) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - return cursor.Select(static doc => Deserialize(doc.Payload)).ToArray(); - } - - public async IAsyncEnumerable StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken) - { - var options = new FindOptions - { - Sort = Builders.Sort.Ascending(static doc => doc.AdvisoryKey), - }; - - using var cursor = await _collection.FindAsync( - FilterDefinition.Empty, - options, - cancellationToken) - .ConfigureAwait(false); + public async Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = FilterDefinition.Empty; + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + var cursor = await query + .SortByDescending(x => x.Modified) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => Deserialize(doc.Payload)).ToArray(); + } + + public async IAsyncEnumerable StreamAsync([EnumeratorCancellation] CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var options = new FindOptions + { + Sort = Builders.Sort.Ascending(static doc => doc.AdvisoryKey), + }; + + using var cursor = session is null + ? await _collection.FindAsync(FilterDefinition.Empty, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindAsync(session, FilterDefinition.Empty, options, cancellationToken).ConfigureAwait(false); while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) { diff --git a/src/StellaOps.Concelier.Storage.Mongo/Advisories/IAdvisoryStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Advisories/IAdvisoryStore.cs new file mode 100644 index 00000000..3fb6dafd --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Advisories/IAdvisoryStore.cs @@ -0,0 +1,15 @@ +using MongoDB.Driver; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Storage.Mongo.Advisories; + +public interface IAdvisoryStore +{ + Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task FindAsync(string advisoryKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task> GetRecentAsync(int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + IAsyncEnumerable StreamAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocument.cs similarity index 92% rename from src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocument.cs index 2d89e021..6abc7cf1 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocument.cs @@ -1,64 +1,64 @@ -using System; -using MongoDB.Bson.Serialization.Attributes; - -namespace StellaOps.Feedser.Storage.Mongo.Advisories; - -[BsonIgnoreExtraElements] -public sealed class NormalizedVersionDocument -{ - [BsonElement("packageId")] - public string PackageId { get; set; } = string.Empty; - - [BsonElement("packageType")] - public string PackageType { get; set; } = string.Empty; - - [BsonElement("scheme")] - public string Scheme { get; set; } = string.Empty; - - [BsonElement("type")] - public string Type { get; set; } = string.Empty; - - [BsonElement("style")] - [BsonIgnoreIfNull] - public string? Style { get; set; } - - [BsonElement("min")] - [BsonIgnoreIfNull] - public string? Min { get; set; } - - [BsonElement("minInclusive")] - [BsonIgnoreIfNull] - public bool? MinInclusive { get; set; } - - [BsonElement("max")] - [BsonIgnoreIfNull] - public string? Max { get; set; } - - [BsonElement("maxInclusive")] - [BsonIgnoreIfNull] - public bool? MaxInclusive { get; set; } - - [BsonElement("value")] - [BsonIgnoreIfNull] - public string? Value { get; set; } - - [BsonElement("notes")] - [BsonIgnoreIfNull] - public string? Notes { get; set; } - - [BsonElement("decisionReason")] - [BsonIgnoreIfNull] - public string? DecisionReason { get; set; } - - [BsonElement("constraint")] - [BsonIgnoreIfNull] - public string? Constraint { get; set; } - - [BsonElement("source")] - [BsonIgnoreIfNull] - public string? Source { get; set; } - - [BsonElement("recordedAt")] - [BsonIgnoreIfNull] - public DateTime? RecordedAtUtc { get; set; } -} +using System; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Concelier.Storage.Mongo.Advisories; + +[BsonIgnoreExtraElements] +public sealed class NormalizedVersionDocument +{ + [BsonElement("packageId")] + public string PackageId { get; set; } = string.Empty; + + [BsonElement("packageType")] + public string PackageType { get; set; } = string.Empty; + + [BsonElement("scheme")] + public string Scheme { get; set; } = string.Empty; + + [BsonElement("type")] + public string Type { get; set; } = string.Empty; + + [BsonElement("style")] + [BsonIgnoreIfNull] + public string? Style { get; set; } + + [BsonElement("min")] + [BsonIgnoreIfNull] + public string? Min { get; set; } + + [BsonElement("minInclusive")] + [BsonIgnoreIfNull] + public bool? MinInclusive { get; set; } + + [BsonElement("max")] + [BsonIgnoreIfNull] + public string? Max { get; set; } + + [BsonElement("maxInclusive")] + [BsonIgnoreIfNull] + public bool? MaxInclusive { get; set; } + + [BsonElement("value")] + [BsonIgnoreIfNull] + public string? Value { get; set; } + + [BsonElement("notes")] + [BsonIgnoreIfNull] + public string? Notes { get; set; } + + [BsonElement("decisionReason")] + [BsonIgnoreIfNull] + public string? DecisionReason { get; set; } + + [BsonElement("constraint")] + [BsonIgnoreIfNull] + public string? Constraint { get; set; } + + [BsonElement("source")] + [BsonIgnoreIfNull] + public string? Source { get; set; } + + [BsonElement("recordedAt")] + [BsonIgnoreIfNull] + public DateTime? RecordedAtUtc { get; set; } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs b/src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs rename to src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs index bf9238a2..4addfa86 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Advisories/NormalizedVersionDocumentFactory.cs @@ -1,100 +1,100 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Storage.Mongo.Advisories; - -internal static class NormalizedVersionDocumentFactory -{ - public static List? Create(Advisory advisory) - { - if (advisory.AffectedPackages.IsDefaultOrEmpty || advisory.AffectedPackages.Length == 0) - { - return null; - } - - var documents = new List(); - var advisoryFallbackReason = advisory.Provenance.FirstOrDefault()?.DecisionReason; - var advisoryFallbackSource = advisory.Provenance.FirstOrDefault()?.Source; - var advisoryFallbackRecordedAt = advisory.Provenance.FirstOrDefault()?.RecordedAt; - - foreach (var package in advisory.AffectedPackages) - { - if (package.NormalizedVersions.IsDefaultOrEmpty || package.NormalizedVersions.Length == 0) - { - continue; - } - - foreach (var rule in package.NormalizedVersions) - { - var matchingRange = FindMatchingRange(package, rule); - var decisionReason = matchingRange?.Provenance.DecisionReason - ?? package.Provenance.FirstOrDefault()?.DecisionReason - ?? advisoryFallbackReason; - - var source = matchingRange?.Provenance.Source - ?? package.Provenance.FirstOrDefault()?.Source - ?? advisoryFallbackSource; - - var recordedAt = matchingRange?.Provenance.RecordedAt - ?? package.Provenance.FirstOrDefault()?.RecordedAt - ?? advisoryFallbackRecordedAt; - - var constraint = matchingRange?.Primitives?.SemVer?.ConstraintExpression - ?? matchingRange?.RangeExpression; - - var style = matchingRange?.Primitives?.SemVer?.Style ?? rule.Type; - - documents.Add(new NormalizedVersionDocument - { - PackageId = package.Identifier ?? string.Empty, - PackageType = package.Type ?? string.Empty, - Scheme = rule.Scheme, - Type = rule.Type, - Style = style, - Min = rule.Min, - MinInclusive = rule.MinInclusive, - Max = rule.Max, - MaxInclusive = rule.MaxInclusive, - Value = rule.Value, - Notes = rule.Notes, - DecisionReason = decisionReason, - Constraint = constraint, - Source = source, - RecordedAtUtc = recordedAt?.UtcDateTime, - }); - } - } - - return documents.Count == 0 ? null : documents; - } - - private static AffectedVersionRange? FindMatchingRange(AffectedPackage package, NormalizedVersionRule rule) - { - foreach (var range in package.VersionRanges) - { - var candidate = range.ToNormalizedVersionRule(rule.Notes); - if (candidate is null) - { - continue; - } - - if (NormalizedRulesEquivalent(candidate, rule)) - { - return range; - } - } - - return null; - } - - private static bool NormalizedRulesEquivalent(NormalizedVersionRule left, NormalizedVersionRule right) - => string.Equals(left.Scheme, right.Scheme, StringComparison.Ordinal) - && string.Equals(left.Type, right.Type, StringComparison.Ordinal) - && string.Equals(left.Min, right.Min, StringComparison.Ordinal) - && left.MinInclusive == right.MinInclusive - && string.Equals(left.Max, right.Max, StringComparison.Ordinal) - && left.MaxInclusive == right.MaxInclusive - && string.Equals(left.Value, right.Value, StringComparison.Ordinal); -} +using System; +using System.Collections.Generic; +using System.Linq; +using StellaOps.Concelier.Models; + +namespace StellaOps.Concelier.Storage.Mongo.Advisories; + +internal static class NormalizedVersionDocumentFactory +{ + public static List? Create(Advisory advisory) + { + if (advisory.AffectedPackages.IsDefaultOrEmpty || advisory.AffectedPackages.Length == 0) + { + return null; + } + + var documents = new List(); + var advisoryFallbackReason = advisory.Provenance.FirstOrDefault()?.DecisionReason; + var advisoryFallbackSource = advisory.Provenance.FirstOrDefault()?.Source; + var advisoryFallbackRecordedAt = advisory.Provenance.FirstOrDefault()?.RecordedAt; + + foreach (var package in advisory.AffectedPackages) + { + if (package.NormalizedVersions.IsDefaultOrEmpty || package.NormalizedVersions.Length == 0) + { + continue; + } + + foreach (var rule in package.NormalizedVersions) + { + var matchingRange = FindMatchingRange(package, rule); + var decisionReason = matchingRange?.Provenance.DecisionReason + ?? package.Provenance.FirstOrDefault()?.DecisionReason + ?? advisoryFallbackReason; + + var source = matchingRange?.Provenance.Source + ?? package.Provenance.FirstOrDefault()?.Source + ?? advisoryFallbackSource; + + var recordedAt = matchingRange?.Provenance.RecordedAt + ?? package.Provenance.FirstOrDefault()?.RecordedAt + ?? advisoryFallbackRecordedAt; + + var constraint = matchingRange?.Primitives?.SemVer?.ConstraintExpression + ?? matchingRange?.RangeExpression; + + var style = matchingRange?.Primitives?.SemVer?.Style ?? rule.Type; + + documents.Add(new NormalizedVersionDocument + { + PackageId = package.Identifier ?? string.Empty, + PackageType = package.Type ?? string.Empty, + Scheme = rule.Scheme, + Type = rule.Type, + Style = style, + Min = rule.Min, + MinInclusive = rule.MinInclusive, + Max = rule.Max, + MaxInclusive = rule.MaxInclusive, + Value = rule.Value, + Notes = rule.Notes, + DecisionReason = decisionReason, + Constraint = constraint, + Source = source, + RecordedAtUtc = recordedAt?.UtcDateTime, + }); + } + } + + return documents.Count == 0 ? null : documents; + } + + private static AffectedVersionRange? FindMatchingRange(AffectedPackage package, NormalizedVersionRule rule) + { + foreach (var range in package.VersionRanges) + { + var candidate = range.ToNormalizedVersionRule(rule.Notes); + if (candidate is null) + { + continue; + } + + if (NormalizedRulesEquivalent(candidate, rule)) + { + return range; + } + } + + return null; + } + + private static bool NormalizedRulesEquivalent(NormalizedVersionRule left, NormalizedVersionRule right) + => string.Equals(left.Scheme, right.Scheme, StringComparison.Ordinal) + && string.Equals(left.Type, right.Type, StringComparison.Ordinal) + && string.Equals(left.Min, right.Min, StringComparison.Ordinal) + && left.MinInclusive == right.MinInclusive + && string.Equals(left.Max, right.Max, StringComparison.Ordinal) + && left.MaxInclusive == right.MaxInclusive + && string.Equals(left.Value, right.Value, StringComparison.Ordinal); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasDocument.cs similarity index 91% rename from src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasDocument.cs index b093292e..fefa080f 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasDocument.cs @@ -2,7 +2,7 @@ using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Aliases; +namespace StellaOps.Concelier.Storage.Mongo.Aliases; [BsonIgnoreExtraElements] internal sealed class AliasDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStore.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStore.cs index 7b0d71cd..3137e3ff 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStore.cs @@ -4,7 +4,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Aliases; +namespace StellaOps.Concelier.Storage.Mongo.Aliases; public sealed class AliasStore : IAliasStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreConstants.cs similarity index 71% rename from src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs rename to src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreConstants.cs index d847acf5..f3f4af4f 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreConstants.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreConstants.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.Aliases; +namespace StellaOps.Concelier.Storage.Mongo.Aliases; public static class AliasStoreConstants { diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreMetrics.cs similarity index 76% rename from src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs rename to src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreMetrics.cs index 7117434d..c90113b9 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Aliases/AliasStoreMetrics.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Aliases/AliasStoreMetrics.cs @@ -1,14 +1,14 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.Storage.Mongo.Aliases; +namespace StellaOps.Concelier.Storage.Mongo.Aliases; internal static class AliasStoreMetrics { - private static readonly Meter Meter = new("StellaOps.Feedser.Merge"); + private static readonly Meter Meter = new("StellaOps.Concelier.Merge"); internal static readonly Counter AliasCollisionCounter = Meter.CreateCounter( - "feedser.merge.alias_conflict", + "concelier.merge.alias_conflict", unit: "count", description: "Number of alias collisions detected when the same alias maps to multiple advisories."); diff --git a/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Aliases/IAliasStore.cs similarity index 92% rename from src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Aliases/IAliasStore.cs index 200346a5..3a7f8a6a 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Aliases/IAliasStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Aliases/IAliasStore.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo.Aliases; +namespace StellaOps.Concelier.Storage.Mongo.Aliases; public interface IAliasStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs similarity index 91% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs index d9b5e746..44aeca47 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocument.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; [BsonIgnoreExtraElements] public sealed class ChangeHistoryDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs index a5f7f9ea..8cd5e71c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryDocumentExtensions.cs @@ -2,7 +2,7 @@ using System; using System.Collections.Generic; using MongoDB.Bson; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; internal static class ChangeHistoryDocumentExtensions { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs similarity index 87% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs index c8c80a88..6b635c20 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryFieldChange.cs @@ -1,6 +1,6 @@ using System; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; public sealed record ChangeHistoryFieldChange { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs index 3b18af98..6a3cf580 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/ChangeHistoryRecord.cs @@ -1,7 +1,7 @@ using System; using System.Collections.Generic; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; public sealed class ChangeHistoryRecord { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs similarity index 83% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs index b409f452..6f2d0ce3 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/IChangeHistoryStore.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; public interface IChangeHistoryStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs index 8f7616ce..6e3de766 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ChangeHistory/MongoChangeHistoryStore.cs @@ -5,7 +5,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.ChangeHistory; +namespace StellaOps.Concelier.Storage.Mongo.ChangeHistory; public sealed class MongoChangeHistoryStore : IChangeHistoryStore { diff --git a/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictDocument.cs new file mode 100644 index 00000000..406da6c2 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictDocument.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Concelier.Storage.Mongo.Conflicts; + +[BsonIgnoreExtraElements] +public sealed class AdvisoryConflictDocument +{ + [BsonId] + public string Id { get; set; } = Guid.Empty.ToString("N"); + + [BsonElement("vulnerabilityKey")] + public string VulnerabilityKey { get; set; } = string.Empty; + + [BsonElement("conflictHash")] + public byte[] ConflictHash { get; set; } = Array.Empty(); + + [BsonElement("asOf")] + public DateTime AsOf { get; set; } + + [BsonElement("recordedAt")] + public DateTime RecordedAt { get; set; } + + [BsonElement("statementIds")] + public List StatementIds { get; set; } = new(); + + [BsonElement("details")] + public BsonDocument Details { get; set; } = new(); +} + +internal static class AdvisoryConflictDocumentExtensions +{ + public static AdvisoryConflictDocument FromRecord(AdvisoryConflictRecord record) + => new() + { + Id = record.Id.ToString(), + VulnerabilityKey = record.VulnerabilityKey, + ConflictHash = record.ConflictHash, + AsOf = record.AsOf.UtcDateTime, + RecordedAt = record.RecordedAt.UtcDateTime, + StatementIds = record.StatementIds.Select(static id => id.ToString()).ToList(), + Details = (BsonDocument)record.Details.DeepClone(), + }; + + public static AdvisoryConflictRecord ToRecord(this AdvisoryConflictDocument document) + => new( + Guid.Parse(document.Id), + document.VulnerabilityKey, + document.ConflictHash, + DateTime.SpecifyKind(document.AsOf, DateTimeKind.Utc), + DateTime.SpecifyKind(document.RecordedAt, DateTimeKind.Utc), + document.StatementIds.Select(static value => Guid.Parse(value)).ToList(), + (BsonDocument)document.Details.DeepClone()); +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictRecord.cs new file mode 100644 index 00000000..62b294bf --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictRecord.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Storage.Mongo.Conflicts; + +public sealed record AdvisoryConflictRecord( + Guid Id, + string VulnerabilityKey, + byte[] ConflictHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + IReadOnlyList StatementIds, + BsonDocument Details); diff --git a/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictStore.cs new file mode 100644 index 00000000..ac878cce --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Conflicts/AdvisoryConflictStore.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo.Conflicts; + +public interface IAdvisoryConflictStore +{ + ValueTask InsertAsync( + IReadOnlyCollection conflicts, + CancellationToken cancellationToken, + IClientSessionHandle? session = null); + + ValueTask> GetConflictsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken, + IClientSessionHandle? session = null); +} + +public sealed class AdvisoryConflictStore : IAdvisoryConflictStore +{ + private readonly IMongoCollection _collection; + + public AdvisoryConflictStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + _collection = database.GetCollection(MongoStorageDefaults.Collections.AdvisoryConflicts); + } + + public async ValueTask InsertAsync( + IReadOnlyCollection conflicts, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(conflicts); + + if (conflicts.Count == 0) + { + return; + } + + var documents = conflicts.Select(AdvisoryConflictDocumentExtensions.FromRecord).ToList(); + var options = new InsertManyOptions { IsOrdered = true }; + + try + { + if (session is null) + { + await _collection.InsertManyAsync(documents, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertManyAsync(session, documents, options, cancellationToken).ConfigureAwait(false); + } + } + catch (MongoBulkWriteException ex) when (ex.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey)) + { + // Conflicts already persisted for this state; ignore duplicates. + } + } + + public async ValueTask> GetConflictsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityKey); + + var filter = Builders.Filter.Eq(document => document.VulnerabilityKey, vulnerabilityKey); + + if (asOf.HasValue) + { + filter &= Builders.Filter.Lte(document => document.AsOf, asOf.Value.UtcDateTime); + } + + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var documents = await find + .SortByDescending(document => document.AsOf) + .ThenByDescending(document => document.RecordedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(static document => document.ToRecord()).ToList(); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentDocument.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentDocument.cs index e4af65cb..6ffa29f8 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentDocument.cs @@ -2,7 +2,7 @@ using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Documents; +namespace StellaOps.Concelier.Storage.Mongo.Documents; [BsonIgnoreExtraElements] public sealed class DocumentDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentRecord.cs similarity index 87% rename from src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentRecord.cs index 1a371362..7268eed3 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentRecord.cs @@ -1,6 +1,6 @@ using MongoDB.Bson; -namespace StellaOps.Feedser.Storage.Mongo.Documents; +namespace StellaOps.Concelier.Storage.Mongo.Documents; public sealed record DocumentRecord( Guid Id, diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentStore.cs similarity index 59% rename from src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentStore.cs index b6c894c8..252928b4 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/DocumentStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Documents/DocumentStore.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Documents; +namespace StellaOps.Concelier.Storage.Mongo.Documents; public sealed class DocumentStore : IDocumentStore { @@ -15,54 +15,73 @@ public sealed class DocumentStore : IDocumentStore _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(record); - - var document = DocumentDocumentExtensions.FromRecord(record); + public async Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DocumentDocumentExtensions.FromRecord(record); var filter = Builders.Filter.Eq(x => x.SourceName, record.SourceName) & Builders.Filter.Eq(x => x.Uri, record.Uri); - var options = new FindOneAndReplaceOptions - { - IsUpsert = true, - ReturnDocument = ReturnDocument.After, - }; - - var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Upserted document {Source}/{Uri}", record.SourceName, record.Uri); - return (replaced ?? document).ToRecord(); - } - - public async Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrEmpty(sourceName); - ArgumentException.ThrowIfNullOrEmpty(uri); - - var filter = Builders.Filter.Eq(x => x.SourceName, sourceName) - & Builders.Filter.Eq(x => x.Uri, uri); - - var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return document?.ToRecord(); - } - - public async Task FindAsync(Guid id, CancellationToken cancellationToken) - { - var idValue = id.ToString(); - var document = await _collection.Find(x => x.Id == idValue).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return document?.ToRecord(); - } - - public async Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrEmpty(status); - - var update = Builders.Update - .Set(x => x.Status, status) - .Set(x => x.LastModified, DateTime.UtcNow); - - var idValue = id.ToString(); - var result = await _collection.UpdateOneAsync(x => x.Id == idValue, update, cancellationToken: cancellationToken).ConfigureAwait(false); - return result.MatchedCount > 0; - } -} + var options = new FindOneAndReplaceOptions + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = session is null + ? await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndReplaceAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted document {Source}/{Uri}", record.SourceName, record.Uri); + return (replaced ?? document).ToRecord(); + } + + public async Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + ArgumentException.ThrowIfNullOrEmpty(uri); + + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName) + & Builders.Filter.Eq(x => x.Uri, uri); + + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task FindAsync(Guid id, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var idValue = id.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, idValue); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrEmpty(status); + + var update = Builders.Update + .Set(x => x.Status, status) + .Set(x => x.LastModified, DateTime.UtcNow); + + var idValue = id.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, idValue); + UpdateResult result; + if (session is null) + { + result = await _collection.UpdateOneAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + result = await _collection.UpdateOneAsync(session, filter, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + return result.MatchedCount > 0; + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Documents/IDocumentStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Documents/IDocumentStore.cs new file mode 100644 index 00000000..60481901 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Documents/IDocumentStore.cs @@ -0,0 +1,14 @@ +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo.Documents; + +public interface IDocumentStore +{ + Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task FindAsync(Guid id, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoDocument.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoDocument.cs index a9121c79..69bb305c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoDocument.cs @@ -2,7 +2,7 @@ using System; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Dtos; +namespace StellaOps.Concelier.Storage.Mongo.Dtos; [BsonIgnoreExtraElements] public sealed class DtoDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoRecord.cs similarity index 76% rename from src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoRecord.cs index 21c4eede..fd8c8faf 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoRecord.cs @@ -1,6 +1,6 @@ using MongoDB.Bson; -namespace StellaOps.Feedser.Storage.Mongo.Dtos; +namespace StellaOps.Concelier.Storage.Mongo.Dtos; public sealed record DtoRecord( Guid Id, diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoStore.cs similarity index 64% rename from src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoStore.cs index 8c7c47ea..4ecffdc0 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/DtoStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Dtos/DtoStore.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Dtos; +namespace StellaOps.Concelier.Storage.Mongo.Dtos; public sealed class DtoStore : IDtoStore { @@ -15,43 +15,52 @@ public sealed class DtoStore : IDtoStore _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(record); - - var document = DtoDocumentExtensions.FromRecord(record); - var documentId = record.DocumentId.ToString(); + public async Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(record); + + var document = DtoDocumentExtensions.FromRecord(record); + var documentId = record.DocumentId.ToString(); var filter = Builders.Filter.Eq(x => x.DocumentId, documentId) & Builders.Filter.Eq(x => x.SourceName, record.SourceName); var options = new FindOneAndReplaceOptions - { - IsUpsert = true, - ReturnDocument = ReturnDocument.After, - }; - - var replaced = await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Upserted DTO for {Source}/{DocumentId}", record.SourceName, record.DocumentId); - return (replaced ?? document).ToRecord(); - } - - public async Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken) - { - var documentIdValue = documentId.ToString(); - var document = await _collection.Find(x => x.DocumentId == documentIdValue) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - return document?.ToRecord(); - } - - public async Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken) - { - var cursor = await _collection.Find(x => x.SourceName == sourceName) - .SortByDescending(x => x.ValidatedAt) - .Limit(limit) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - + { + IsUpsert = true, + ReturnDocument = ReturnDocument.After, + }; + + var replaced = session is null + ? await _collection.FindOneAndReplaceAsync(filter, document, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndReplaceAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Upserted DTO for {Source}/{DocumentId}", record.SourceName, record.DocumentId); + return (replaced ?? document).ToRecord(); + } + + public async Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var documentIdValue = documentId.ToString(); + var filter = Builders.Filter.Eq(x => x.DocumentId, documentIdValue); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var cursor = await query + .SortByDescending(x => x.ValidatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + return cursor.Select(static x => x.ToRecord()).ToArray(); } } diff --git a/src/StellaOps.Concelier.Storage.Mongo/Dtos/IDtoStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Dtos/IDtoStore.cs new file mode 100644 index 00000000..9ce5f7b0 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Dtos/IDtoStore.cs @@ -0,0 +1,12 @@ +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo.Dtos; + +public interface IDtoStore +{ + Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Events/MongoAdvisoryEventRepository.cs b/src/StellaOps.Concelier.Storage.Mongo/Events/MongoAdvisoryEventRepository.cs new file mode 100644 index 00000000..1f923316 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Events/MongoAdvisoryEventRepository.cs @@ -0,0 +1,224 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Conflicts; +using StellaOps.Concelier.Storage.Mongo.Statements; + +namespace StellaOps.Concelier.Storage.Mongo.Events; + +public sealed class MongoAdvisoryEventRepository : IAdvisoryEventRepository +{ + private readonly IAdvisoryStatementStore _statementStore; + private readonly IAdvisoryConflictStore _conflictStore; + + public MongoAdvisoryEventRepository( + IAdvisoryStatementStore statementStore, + IAdvisoryConflictStore conflictStore) + { + _statementStore = statementStore ?? throw new ArgumentNullException(nameof(statementStore)); + _conflictStore = conflictStore ?? throw new ArgumentNullException(nameof(conflictStore)); + } + + public async ValueTask InsertStatementsAsync( + IReadOnlyCollection statements, + CancellationToken cancellationToken) + { + if (statements is null) + { + throw new ArgumentNullException(nameof(statements)); + } + + if (statements.Count == 0) + { + return; + } + + var records = statements + .Select(static entry => + { + var payload = BsonDocument.Parse(entry.CanonicalJson); + return new AdvisoryStatementRecord( + entry.StatementId, + entry.VulnerabilityKey, + entry.AdvisoryKey, + entry.StatementHash.ToArray(), + entry.AsOf, + entry.RecordedAt, + payload, + entry.InputDocumentIds.ToArray()); + }) + .ToList(); + + await _statementStore.InsertAsync(records, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask InsertConflictsAsync( + IReadOnlyCollection conflicts, + CancellationToken cancellationToken) + { + if (conflicts is null) + { + throw new ArgumentNullException(nameof(conflicts)); + } + + if (conflicts.Count == 0) + { + return; + } + + var records = conflicts + .Select(static entry => + { + var payload = BsonDocument.Parse(entry.CanonicalJson); + return new AdvisoryConflictRecord( + entry.ConflictId, + entry.VulnerabilityKey, + entry.ConflictHash.ToArray(), + entry.AsOf, + entry.RecordedAt, + entry.StatementIds.ToArray(), + payload); + }) + .ToList(); + + await _conflictStore.InsertAsync(records, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask> GetStatementsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken) + { + var records = await _statementStore + .GetStatementsAsync(vulnerabilityKey, asOf, cancellationToken) + .ConfigureAwait(false); + + if (records.Count == 0) + { + return Array.Empty(); + } + + var entries = records + .Select(static record => + { + var advisory = CanonicalJsonSerializer.Deserialize(record.Payload.ToJson()); + var canonicalJson = CanonicalJsonSerializer.Serialize(advisory); + + return new AdvisoryStatementEntry( + record.Id, + record.VulnerabilityKey, + record.AdvisoryKey, + canonicalJson, + record.StatementHash.ToImmutableArray(), + record.AsOf, + record.RecordedAt, + record.InputDocumentIds.ToImmutableArray()); + }) + .ToList(); + + return entries; + } + + public async ValueTask> GetConflictsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken) + { + var records = await _conflictStore + .GetConflictsAsync(vulnerabilityKey, asOf, cancellationToken) + .ConfigureAwait(false); + + if (records.Count == 0) + { + return Array.Empty(); + } + + var entries = records + .Select(static record => + { + var canonicalJson = Canonicalize(record.Details); + return new AdvisoryConflictEntry( + record.Id, + record.VulnerabilityKey, + canonicalJson, + record.ConflictHash.ToImmutableArray(), + record.AsOf, + record.RecordedAt, + record.StatementIds.ToImmutableArray()); + }) + .ToList(); + + return entries; + } + private static readonly JsonWriterOptions CanonicalWriterOptions = new() + { + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + Indented = false, + SkipValidation = false, + }; + + private static string Canonicalize(BsonDocument document) + { + using var json = JsonDocument.Parse(document.ToJson()); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream, CanonicalWriterOptions)) + { + WriteCanonical(json.RootElement, writer); + } + + return Encoding.UTF8.GetString(stream.ToArray()); + } + + private static void WriteCanonical(JsonElement element, Utf8JsonWriter writer) + { + switch (element.ValueKind) + { + case JsonValueKind.Object: + writer.WriteStartObject(); + foreach (var property in element.EnumerateObject().OrderBy(static p => p.Name, StringComparer.Ordinal)) + { + writer.WritePropertyName(property.Name); + WriteCanonical(property.Value, writer); + } + writer.WriteEndObject(); + break; + case JsonValueKind.Array: + writer.WriteStartArray(); + foreach (var item in element.EnumerateArray()) + { + WriteCanonical(item, writer); + } + writer.WriteEndArray(); + break; + case JsonValueKind.String: + writer.WriteStringValue(element.GetString()); + break; + case JsonValueKind.Number: + writer.WriteRawValue(element.GetRawText()); + break; + case JsonValueKind.True: + writer.WriteBooleanValue(true); + break; + case JsonValueKind.False: + writer.WriteBooleanValue(false); + break; + case JsonValueKind.Null: + writer.WriteNullValue(); + break; + default: + writer.WriteRawValue(element.GetRawText()); + break; + } + } + +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateDocument.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateDocument.cs index a6e87421..d08e5758 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateDocument.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using System.Linq; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Exporting; +namespace StellaOps.Concelier.Storage.Mongo.Exporting; [BsonIgnoreExtraElements] public sealed class ExportStateDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateManager.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs rename to src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateManager.cs index efd051aa..513c7b4f 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateManager.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateManager.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo.Exporting; +namespace StellaOps.Concelier.Storage.Mongo.Exporting; /// /// Helper for exporters to read and persist their export metadata in Mongo-backed storage. diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateRecord.cs similarity index 85% rename from src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateRecord.cs index 3d72b44e..e7da9625 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateRecord.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.Exporting; +namespace StellaOps.Concelier.Storage.Mongo.Exporting; public sealed record ExportStateRecord( string Id, diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateStore.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateStore.cs index a45e51a9..2062acc4 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/ExportStateStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Exporting/ExportStateStore.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Exporting; +namespace StellaOps.Concelier.Storage.Mongo.Exporting; public sealed class ExportStateStore : IExportStateStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Exporting/IExportStateStore.cs similarity index 78% rename from src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/Exporting/IExportStateStore.cs index 3dad3c23..910a6763 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Exporting/IExportStateStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Exporting/IExportStateStore.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.Exporting; +namespace StellaOps.Concelier.Storage.Mongo.Exporting; public interface IExportStateStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs b/src/StellaOps.Concelier.Storage.Mongo/ISourceStateRepository.cs similarity index 54% rename from src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs rename to src/StellaOps.Concelier.Storage.Mongo/ISourceStateRepository.cs index d5e8a98e..fa1455d0 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ISourceStateRepository.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ISourceStateRepository.cs @@ -1,14 +1,15 @@ using MongoDB.Bson; +using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public interface ISourceStateRepository { - Task TryGetAsync(string sourceName, CancellationToken cancellationToken); + Task TryGetAsync(string sourceName, CancellationToken cancellationToken, IClientSessionHandle? session = null); - Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken); + Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null); - Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken); + Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null); - Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken); + Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken, IClientSessionHandle? session = null); } diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/JobLeaseDocument.cs similarity index 89% rename from src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/JobLeaseDocument.cs index ae8b5b34..97060ec5 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JobLeaseDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JobLeaseDocument.cs @@ -1,7 +1,7 @@ using MongoDB.Bson.Serialization.Attributes; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; [BsonIgnoreExtraElements] public sealed class JobLeaseDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/JobRunDocument.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/JobRunDocument.cs index fabbc3f1..8c49155d 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JobRunDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JobRunDocument.cs @@ -4,9 +4,9 @@ using System.Linq; using System.Text.Json; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; [BsonIgnoreExtraElements] public sealed class JobRunDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/IJpFlagStore.cs similarity index 80% rename from src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/JpFlags/IJpFlagStore.cs index 131ce8b9..25d50dfc 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/IJpFlagStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/IJpFlagStore.cs @@ -1,7 +1,7 @@ using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo.JpFlags; +namespace StellaOps.Concelier.Storage.Mongo.JpFlags; public interface IJpFlagStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagDocument.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagDocument.cs index a493fc51..407b123c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagDocument.cs @@ -1,6 +1,6 @@ using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.JpFlags; +namespace StellaOps.Concelier.Storage.Mongo.JpFlags; [BsonIgnoreExtraElements] public sealed class JpFlagDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagRecord.cs similarity index 85% rename from src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagRecord.cs index 90fa8dc0..292ce7e2 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagRecord.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.JpFlags; +namespace StellaOps.Concelier.Storage.Mongo.JpFlags; /// /// Captures Japan-specific enrichment flags derived from JVN payloads. diff --git a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagStore.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagStore.cs index e5cedaae..efb773f7 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/JpFlags/JpFlagStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/JpFlags/JpFlagStore.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.JpFlags; +namespace StellaOps.Concelier.Storage.Mongo.JpFlags; public sealed class JpFlagStore : IJpFlagStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md b/src/StellaOps.Concelier.Storage.Mongo/MIGRATIONS.md similarity index 60% rename from src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md rename to src/StellaOps.Concelier.Storage.Mongo/MIGRATIONS.md index bca738e1..65bf1698 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MIGRATIONS.md +++ b/src/StellaOps.Concelier.Storage.Mongo/MIGRATIONS.md @@ -1,34 +1,36 @@ # Mongo Schema Migration Playbook -This module owns the persistent shape of Feedser's MongoDB database. Upgrades must be deterministic and safe to run on live replicas. The `MongoMigrationRunner` executes idempotent migrations on startup immediately after the bootstrapper completes its collection and index checks. +This module owns the persistent shape of Concelier's MongoDB database. Upgrades must be deterministic and safe to run on live replicas. The `MongoMigrationRunner` executes idempotent migrations on startup immediately after the bootstrapper completes its collection and index checks. ## Execution Path -1. `StellaOps.Feedser.WebService` calls `MongoBootstrapper.InitializeAsync()` during startup. +1. `StellaOps.Concelier.WebService` calls `MongoBootstrapper.InitializeAsync()` during startup. 2. Once collections and baseline indexes are ensured, the bootstrapper invokes `MongoMigrationRunner.RunAsync()`. 3. Each `IMongoMigration` implementation is sorted by its `Id` (ordinal compare) and executed exactly once. Completion is recorded in the `schema_migrations` collection. 4. Failures surface during startup and prevent the service from serving traffic, matching our "fail-fast" requirement for storage incompatibilities. ## Creating a Migration -1. Implement `IMongoMigration` under `StellaOps.Feedser.Storage.Mongo.Migrations`. Use a monotonically increasing identifier such as `yyyyMMdd_description`. +1. Implement `IMongoMigration` under `StellaOps.Concelier.Storage.Mongo.Migrations`. Use a monotonically increasing identifier such as `yyyyMMdd_description`. 2. Keep the body idempotent: query state first, drop/re-create indexes only when mismatch is detected, and avoid multi-document transactions unless required. 3. Add the migration to DI in `ServiceCollectionExtensions` so it flows into the runner. 4. Write an integration test that exercises the migration against a Mongo2Go instance to validate behaviour. ## Current Migrations -| Id | Description | -| --- | --- | -| `20241005_document_expiry_indexes` | Ensures `document` collection uses the correct TTL/partial index depending on raw document retention settings. | -| `20241005_gridfs_expiry_indexes` | Aligns the GridFS `documents.files` TTL index with retention settings. | +| Id | Description | +| --- | --- | +| `20241005_document_expiry_indexes` | Ensures `document` collection uses the correct TTL/partial index depending on raw document retention settings. | +| `20241005_gridfs_expiry_indexes` | Aligns the GridFS `documents.files` TTL index with retention settings. | +| `20251019_advisory_event_collections` | Creates/aligns indexes for `advisory_statements` and `advisory_conflicts` collections powering the event log + conflict replay pipeline. | ## Operator Runbook -- `schema_migrations` records each applied migration (`_id`, `description`, `appliedAt`). Review this collection when auditing upgrades. -- To re-run a migration in a lab, delete the corresponding document from `schema_migrations` and restart the service. **Do not** do this in production unless the migration body is known to be idempotent and safe. -- When changing retention settings (`RawDocumentRetention`), deploy the new configuration and restart Feedser. The migration runner will adjust indexes on the next boot. -- If migrations fail, restart with `Logging__LogLevel__StellaOps.Feedser.Storage.Mongo.Migrations=Debug` to surface diagnostic output. Remediate underlying index/collection drift before retrying. +- `schema_migrations` records each applied migration (`_id`, `description`, `appliedAt`). Review this collection when auditing upgrades. +- To re-run a migration in a lab, delete the corresponding document from `schema_migrations` and restart the service. **Do not** do this in production unless the migration body is known to be idempotent and safe. +- When changing retention settings (`RawDocumentRetention`), deploy the new configuration and restart Concelier. The migration runner will adjust indexes on the next boot. +- For the event-log collections (`advisory_statements`, `advisory_conflicts`), rollback is simply `db.advisory_statements.drop()` / `db.advisory_conflicts.drop()` followed by a restart if you must revert to the pre-event-log schema (only in labs). Production rollbacks should instead gate merge features that rely on these collections. +- If migrations fail, restart with `Logging__LogLevel__StellaOps.Concelier.Storage.Mongo.Migrations=Debug` to surface diagnostic output. Remediate underlying index/collection drift before retrying. ## Validating an Upgrade diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/IMergeEventStore.cs similarity index 79% rename from src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/MergeEvents/IMergeEventStore.cs index 57df7782..29cececa 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/IMergeEventStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/IMergeEventStore.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; +namespace StellaOps.Concelier.Storage.Mongo.MergeEvents; public interface IMergeEventStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventDocument.cs similarity index 97% rename from src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventDocument.cs index b0db3bb1..38388d73 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventDocument.cs @@ -4,7 +4,7 @@ using System.Linq; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; +namespace StellaOps.Concelier.Storage.Mongo.MergeEvents; [BsonIgnoreExtraElements] public sealed class MergeEventDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventRecord.cs similarity index 80% rename from src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventRecord.cs index a3e02214..80998127 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventRecord.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; +namespace StellaOps.Concelier.Storage.Mongo.MergeEvents; public sealed record MergeEventRecord( Guid Id, diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventStore.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventStore.cs index 3e77e41c..8403ed4c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeEventStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeEventStore.cs @@ -1,7 +1,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; +namespace StellaOps.Concelier.Storage.Mongo.MergeEvents; public sealed class MergeEventStore : IMergeEventStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeFieldDecision.cs similarity index 75% rename from src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs rename to src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeFieldDecision.cs index 2d3f753e..8b558b85 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MergeEvents/MergeFieldDecision.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MergeEvents/MergeFieldDecision.cs @@ -1,8 +1,8 @@ -namespace StellaOps.Feedser.Storage.Mongo.MergeEvents; - -public sealed record MergeFieldDecision( - string Field, - string? SelectedSource, - string DecisionReason, - DateTimeOffset? SelectedModified, - IReadOnlyList ConsideredSources); +namespace StellaOps.Concelier.Storage.Mongo.MergeEvents; + +public sealed record MergeFieldDecision( + string Field, + string? SelectedSource, + string DecisionReason, + DateTimeOffset? SelectedModified, + IReadOnlyList ConsideredSources); diff --git a/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureAdvisoryEventCollectionsMigration.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureAdvisoryEventCollectionsMigration.cs new file mode 100644 index 00000000..cc954e9e --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureAdvisoryEventCollectionsMigration.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo.Migrations; + +public sealed class EnsureAdvisoryEventCollectionsMigration : IMongoMigration +{ + public string Id => "20251019_advisory_event_collections"; + + public string Description => "Ensure advisory_statements and advisory_conflicts indexes exist for event log storage."; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var statements = database.GetCollection(MongoStorageDefaults.Collections.AdvisoryStatements); + var conflicts = database.GetCollection(MongoStorageDefaults.Collections.AdvisoryConflicts); + + var statementIndexes = new List> + { + new( + Builders.IndexKeys.Ascending("vulnerabilityKey").Descending("asOf"), + new CreateIndexOptions { Name = "advisory_statements_vulnerability_asof_desc" }), + new( + Builders.IndexKeys.Ascending("statementHash"), + new CreateIndexOptions { Name = "advisory_statements_statementHash_unique", Unique = true }), + }; + + await statements.Indexes.CreateManyAsync(statementIndexes, cancellationToken).ConfigureAwait(false); + + var conflictIndexes = new List> + { + new( + Builders.IndexKeys.Ascending("vulnerabilityKey").Descending("asOf"), + new CreateIndexOptions { Name = "advisory_conflicts_vulnerability_asof_desc" }), + new( + Builders.IndexKeys.Ascending("conflictHash"), + new CreateIndexOptions { Name = "advisory_conflicts_conflictHash_unique", Unique = true }), + }; + + await conflicts.Indexes.CreateManyAsync(conflictIndexes, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs index 12732a81..f8430a9d 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureDocumentExpiryIndexesMigration.cs @@ -6,7 +6,7 @@ using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Migrations; internal sealed class EnsureDocumentExpiryIndexesMigration : IMongoMigration { diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs index 158f7aa7..48e8525a 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/EnsureGridFsExpiryIndexesMigration.cs @@ -6,7 +6,7 @@ using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Migrations; internal sealed class EnsureGridFsExpiryIndexesMigration : IMongoMigration { diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/IMongoMigration.cs similarity index 87% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/IMongoMigration.cs index 0a0b845c..c8696993 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/IMongoMigration.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/IMongoMigration.cs @@ -1,6 +1,6 @@ using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Migrations; /// /// Represents a single, idempotent MongoDB migration. diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationDocument.cs similarity index 83% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationDocument.cs index 268996e6..1670b820 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationDocument.cs @@ -1,7 +1,7 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Migrations; [BsonIgnoreExtraElements] internal sealed class MongoMigrationDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationRunner.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationRunner.cs index 0256a448..87b38d87 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/MongoMigrationRunner.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/MongoMigrationRunner.cs @@ -6,7 +6,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.Migrations; +namespace StellaOps.Concelier.Storage.Mongo.Migrations; /// /// Executes pending schema migrations tracked inside MongoDB to keep upgrades deterministic. diff --git a/src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs b/src/StellaOps.Concelier.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs rename to src/StellaOps.Concelier.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs index d54dc669..0e5f7e25 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/Migrations/SemVerStyleBackfillMigration.cs @@ -1,81 +1,81 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; - -namespace StellaOps.Feedser.Storage.Mongo.Migrations; - -public sealed class SemVerStyleBackfillMigration : IMongoMigration -{ - private readonly MongoStorageOptions _options; - private readonly ILogger _logger; - - public SemVerStyleBackfillMigration(IOptions options, ILogger logger) - { - _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string Id => "20251011-semver-style-backfill"; - - public string Description => "Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled."; - - public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - if (!_options.EnableSemVerStyle) - { - _logger.LogInformation("SemVer style flag disabled; skipping migration {MigrationId}.", Id); - return; - } - - var collection = database.GetCollection(MongoStorageDefaults.Collections.Advisory); - var filter = Builders.Filter.Or( - Builders.Filter.Exists(doc => doc.NormalizedVersions, false), - Builders.Filter.Where(doc => doc.NormalizedVersions == null || doc.NormalizedVersions.Count == 0)); - - var batchSize = Math.Max(25, _options.BackfillBatchSize); - while (true) - { - var pending = await collection.Find(filter) - .SortBy(doc => doc.AdvisoryKey) - .Limit(batchSize) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - if (pending.Count == 0) - { - break; - } - - var updates = new List>(pending.Count); - foreach (var document in pending) - { - var advisory = CanonicalJsonSerializer.Deserialize(document.Payload.ToJson()); - var normalized = NormalizedVersionDocumentFactory.Create(advisory); - - if (normalized is null || normalized.Count == 0) - { - updates.Add(new UpdateOneModel( - Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), - Builders.Update.Unset(doc => doc.NormalizedVersions))); - continue; - } - - updates.Add(new UpdateOneModel( - Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), - Builders.Update.Set(doc => doc.NormalizedVersions, normalized))); - } - - if (updates.Count > 0) - { - await collection.BulkWriteAsync(updates, cancellationToken: cancellationToken).ConfigureAwait(false); - } - } - } -} +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Storage.Mongo.Advisories; + +namespace StellaOps.Concelier.Storage.Mongo.Migrations; + +public sealed class SemVerStyleBackfillMigration : IMongoMigration +{ + private readonly MongoStorageOptions _options; + private readonly ILogger _logger; + + public SemVerStyleBackfillMigration(IOptions options, ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Id => "20251011-semver-style-backfill"; + + public string Description => "Populate advisory.normalizedVersions for existing documents when SemVer style storage is enabled."; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + if (!_options.EnableSemVerStyle) + { + _logger.LogInformation("SemVer style flag disabled; skipping migration {MigrationId}.", Id); + return; + } + + var collection = database.GetCollection(MongoStorageDefaults.Collections.Advisory); + var filter = Builders.Filter.Or( + Builders.Filter.Exists(doc => doc.NormalizedVersions, false), + Builders.Filter.Where(doc => doc.NormalizedVersions == null || doc.NormalizedVersions.Count == 0)); + + var batchSize = Math.Max(25, _options.BackfillBatchSize); + while (true) + { + var pending = await collection.Find(filter) + .SortBy(doc => doc.AdvisoryKey) + .Limit(batchSize) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (pending.Count == 0) + { + break; + } + + var updates = new List>(pending.Count); + foreach (var document in pending) + { + var advisory = CanonicalJsonSerializer.Deserialize(document.Payload.ToJson()); + var normalized = NormalizedVersionDocumentFactory.Create(advisory); + + if (normalized is null || normalized.Count == 0) + { + updates.Add(new UpdateOneModel( + Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), + Builders.Update.Unset(doc => doc.NormalizedVersions))); + continue; + } + + updates.Add(new UpdateOneModel( + Builders.Filter.Eq(doc => doc.AdvisoryKey, document.AdvisoryKey), + Builders.Update.Set(doc => doc.NormalizedVersions, normalized))); + } + + if (updates.Count > 0) + { + await collection.BulkWriteAsync(updates, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs similarity index 86% rename from src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs index 04006964..2e7ccba4 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoBootstrapper.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoBootstrapper.cs @@ -2,9 +2,9 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Storage.Mongo.Migrations; +using StellaOps.Concelier.Storage.Mongo.Migrations; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; /// /// Ensures required collections and indexes exist before the service begins processing. @@ -25,14 +25,16 @@ public sealed class MongoBootstrapper MongoStorageDefaults.Collections.KevFlag, MongoStorageDefaults.Collections.RuFlags, MongoStorageDefaults.Collections.JpFlags, - MongoStorageDefaults.Collections.PsirtFlags, - MongoStorageDefaults.Collections.MergeEvent, - MongoStorageDefaults.Collections.ExportState, - MongoStorageDefaults.Collections.ChangeHistory, - MongoStorageDefaults.Collections.Locks, - MongoStorageDefaults.Collections.Jobs, - MongoStorageDefaults.Collections.Migrations, - }; + MongoStorageDefaults.Collections.PsirtFlags, + MongoStorageDefaults.Collections.MergeEvent, + MongoStorageDefaults.Collections.ExportState, + MongoStorageDefaults.Collections.ChangeHistory, + MongoStorageDefaults.Collections.AdvisoryStatements, + MongoStorageDefaults.Collections.AdvisoryConflicts, + MongoStorageDefaults.Collections.Locks, + MongoStorageDefaults.Collections.Jobs, + MongoStorageDefaults.Collections.Migrations, + }; private readonly IMongoDatabase _database; private readonly MongoStorageOptions _options; @@ -70,15 +72,17 @@ public sealed class MongoBootstrapper EnsureAdvisoryIndexesAsync(cancellationToken), EnsureDocumentsIndexesAsync(cancellationToken), EnsureDtoIndexesAsync(cancellationToken), - EnsureAliasIndexesAsync(cancellationToken), - EnsureAffectedIndexesAsync(cancellationToken), - EnsureReferenceIndexesAsync(cancellationToken), - EnsureSourceStateIndexesAsync(cancellationToken), - EnsurePsirtFlagIndexesAsync(cancellationToken), - EnsureChangeHistoryIndexesAsync(cancellationToken), - EnsureGridFsIndexesAsync(cancellationToken)).ConfigureAwait(false); - - await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + EnsureAliasIndexesAsync(cancellationToken), + EnsureAffectedIndexesAsync(cancellationToken), + EnsureReferenceIndexesAsync(cancellationToken), + EnsureSourceStateIndexesAsync(cancellationToken), + EnsurePsirtFlagIndexesAsync(cancellationToken), + EnsureAdvisoryStatementIndexesAsync(cancellationToken), + EnsureAdvisoryConflictIndexesAsync(cancellationToken), + EnsureChangeHistoryIndexesAsync(cancellationToken), + EnsureGridFsIndexesAsync(cancellationToken)).ConfigureAwait(false); + + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); _logger.LogInformation("Mongo bootstrapper completed"); } @@ -238,10 +242,10 @@ public sealed class MongoBootstrapper return collection.Indexes.CreateManyAsync(indexes, cancellationToken); } - private Task EnsureReferenceIndexesAsync(CancellationToken cancellationToken) - { - var collection = _database.GetCollection(MongoStorageDefaults.Collections.Reference); - var indexes = new List> + private Task EnsureReferenceIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.Reference); + var indexes = new List> { new( Builders.IndexKeys.Ascending("url"), @@ -250,10 +254,42 @@ public sealed class MongoBootstrapper Builders.IndexKeys.Ascending("advisoryId"), new CreateIndexOptions { Name = "reference_advisoryId" }), }; - - return collection.Indexes.CreateManyAsync(indexes, cancellationToken); - } - + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAdvisoryStatementIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.AdvisoryStatements); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("vulnerabilityKey").Descending("asOf"), + new CreateIndexOptions { Name = "advisory_statements_vulnerability_asof_desc" }), + new( + Builders.IndexKeys.Ascending("statementHash"), + new CreateIndexOptions { Name = "advisory_statements_statementHash_unique", Unique = true }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + + private Task EnsureAdvisoryConflictIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(MongoStorageDefaults.Collections.AdvisoryConflicts); + var indexes = new List> + { + new( + Builders.IndexKeys.Ascending("vulnerabilityKey").Descending("asOf"), + new CreateIndexOptions { Name = "advisory_conflicts_vulnerability_asof_desc" }), + new( + Builders.IndexKeys.Ascending("conflictHash"), + new CreateIndexOptions { Name = "advisory_conflicts_conflictHash_unique", Unique = true }), + }; + + return collection.Indexes.CreateManyAsync(indexes, cancellationToken); + } + private Task EnsureSourceStateIndexesAsync(CancellationToken cancellationToken) { var collection = _database.GetCollection(MongoStorageDefaults.Collections.SourceState); diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoJobStore.cs similarity index 66% rename from src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoJobStore.cs index ce0a3fab..481dfac5 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoJobStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoJobStore.cs @@ -5,9 +5,9 @@ using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Driver; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public sealed class MongoJobStore : IJobStore { @@ -23,129 +23,154 @@ public sealed class MongoJobStore : IJobStore _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken) - { - var runId = Guid.NewGuid(); - var document = JobRunDocumentExtensions.FromRequest(request, runId); + public async Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var runId = Guid.NewGuid(); + var document = JobRunDocumentExtensions.FromRequest(request, runId); + + if (session is null) + { + await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertOneAsync(session, document, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + _logger.LogDebug("Created job run {RunId} for {Kind} with trigger {Trigger}", runId, request.Kind, request.Trigger); + + return document.ToSnapshot(); + } - await _collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Created job run {RunId} for {Kind} with trigger {Trigger}", runId, request.Kind, request.Trigger); - - return document.ToSnapshot(); - } - - public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken) - { - var runIdValue = runId.ToString(); - var filter = Builders.Filter.Eq(x => x.Id, runIdValue) - & Builders.Filter.Eq(x => x.Status, PendingStatus); - - var update = Builders.Update - .Set(x => x.Status, RunningStatus) - .Set(x => x.StartedAt, startedAt.UtcDateTime); - - var result = await _collection.FindOneAndUpdateAsync( - filter, - update, - new FindOneAndUpdateOptions - { - ReturnDocument = ReturnDocument.After, - }, - cancellationToken).ConfigureAwait(false); - - if (result is null) - { - _logger.LogDebug("Failed to start job run {RunId}; status transition rejected", runId); - return null; + public async Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) + & Builders.Filter.Eq(x => x.Status, PendingStatus); + + var update = Builders.Update + .Set(x => x.Status, RunningStatus) + .Set(x => x.StartedAt, startedAt.UtcDateTime); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }; + + var result = session is null + ? await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogDebug("Failed to start job run {RunId}; status transition rejected", runId); + return null; } return result.ToSnapshot(); } - public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken) - { - var runIdValue = runId.ToString(); - var filter = Builders.Filter.Eq(x => x.Id, runIdValue) - & Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); - - var update = Builders.Update - .Set(x => x.Status, completion.Status.ToString()) - .Set(x => x.CompletedAt, completion.CompletedAt.UtcDateTime) - .Set(x => x.Error, completion.Error); - - var result = await _collection.FindOneAndUpdateAsync( - filter, - update, - new FindOneAndUpdateOptions - { - ReturnDocument = ReturnDocument.After, - }, - cancellationToken).ConfigureAwait(false); - - if (result is null) - { - _logger.LogWarning("Failed to mark job run {RunId} as {Status}", runId, completion.Status); - return null; + public async Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var runIdValue = runId.ToString(); + var filter = Builders.Filter.Eq(x => x.Id, runIdValue) + & Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + + var update = Builders.Update + .Set(x => x.Status, completion.Status.ToString()) + .Set(x => x.CompletedAt, completion.CompletedAt.UtcDateTime) + .Set(x => x.Error, completion.Error); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + }; + + var result = session is null + ? await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + + if (result is null) + { + _logger.LogWarning("Failed to mark job run {RunId} as {Status}", runId, completion.Status); + return null; } return result.ToSnapshot(); } - public async Task FindAsync(Guid runId, CancellationToken cancellationToken) - { - var cursor = await _collection.FindAsync(x => x.Id == runId.ToString(), cancellationToken: cancellationToken).ConfigureAwait(false); - var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return document?.ToSnapshot(); - } - - public async Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken) - { - if (limit <= 0) - { - return Array.Empty(); - } - - var filter = string.IsNullOrWhiteSpace(kind) - ? Builders.Filter.Empty - : Builders.Filter.Eq(x => x.Kind, kind); - - var cursor = await _collection.Find(filter) - .SortByDescending(x => x.CreatedAt) - .Limit(limit) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); + public async Task FindAsync(Guid runId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Eq(x => x.Id, runId.ToString()); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToSnapshot(); + } + public async Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (limit <= 0) + { + return Array.Empty(); + } + + var filter = string.IsNullOrWhiteSpace(kind) + ? Builders.Filter.Empty + : Builders.Filter.Eq(x => x.Kind, kind); + + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var cursor = await query + .SortByDescending(x => x.CreatedAt) + .Limit(limit) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); } - public async Task> GetActiveRunsAsync(CancellationToken cancellationToken) - { - var filter = Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); - var cursor = await _collection.Find(filter) - .SortByDescending(x => x.CreatedAt) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); + public async Task> GetActiveRunsAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.In(x => x.Status, new[] { PendingStatus, RunningStatus }); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var cursor = await query + .SortByDescending(x => x.CreatedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return cursor.Select(static doc => doc.ToSnapshot()).ToArray(); } - public async Task GetLastRunAsync(string kind, CancellationToken cancellationToken) - { - var cursor = await _collection.Find(x => x.Kind == kind) - .SortByDescending(x => x.CreatedAt) - .Limit(1) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - + public async Task GetLastRunAsync(string kind, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Eq(x => x.Kind, kind); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var cursor = await query + .SortByDescending(x => x.CreatedAt) + .Limit(1) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + return cursor.FirstOrDefault()?.ToSnapshot(); } - public async Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken) - { - if (kinds is null) - { - throw new ArgumentNullException(nameof(kinds)); - } + public async Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (kinds is null) + { + throw new ArgumentNullException(nameof(kinds)); + } var kindList = kinds .Where(static kind => !string.IsNullOrWhiteSpace(kind)) @@ -168,13 +193,17 @@ public sealed class MongoJobStore : IJobStore var pipeline = new[] { matchStage, sortStage, groupStage }; - var aggregate = await _collection.Aggregate(pipeline) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - var results = new Dictionary(StringComparer.Ordinal); - foreach (var element in aggregate) - { + var aggregateFluent = session is null + ? _collection.Aggregate(pipeline) + : _collection.Aggregate(session, pipeline); + + var aggregate = await aggregateFluent + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var results = new Dictionary(StringComparer.Ordinal); + foreach (var element in aggregate) + { if (!element.TryGetValue("_id", out var idValue) || idValue.BsonType != BsonType.String) { continue; diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoLeaseStore.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoLeaseStore.cs index 5df66eca..6f783fb3 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoLeaseStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoLeaseStore.cs @@ -1,8 +1,8 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public sealed class MongoLeaseStore : ILeaseStore { diff --git a/src/StellaOps.Concelier.Storage.Mongo/MongoSessionProvider.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoSessionProvider.cs new file mode 100644 index 00000000..787192d9 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoSessionProvider.cs @@ -0,0 +1,34 @@ +using Microsoft.Extensions.Options; +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo; + +public interface IMongoSessionProvider +{ + Task StartSessionAsync(CancellationToken cancellationToken = default); +} + +internal sealed class MongoSessionProvider : IMongoSessionProvider +{ + private readonly IMongoClient _client; + private readonly MongoStorageOptions _options; + + public MongoSessionProvider(IMongoClient client, IOptions options) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + } + + public Task StartSessionAsync(CancellationToken cancellationToken = default) + { + var sessionOptions = new ClientSessionOptions + { + DefaultTransactionOptions = new TransactionOptions( + readPreference: ReadPreference.Primary, + readConcern: ReadConcern.Majority, + writeConcern: WriteConcern.WMajority.With(wTimeout: _options.CommandTimeout)) + }; + + return _client.StartSessionAsync(sessionOptions, cancellationToken); + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoSourceStateRepository.cs similarity index 61% rename from src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoSourceStateRepository.cs index f3a88687..93bae1cf 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoSourceStateRepository.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoSourceStateRepository.cs @@ -2,7 +2,7 @@ using Microsoft.Extensions.Logging; using MongoDB.Bson; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public sealed class MongoSourceStateRepository : ISourceStateRepository { @@ -18,32 +18,41 @@ public sealed class MongoSourceStateRepository : ISourceStateRepository _logger = logger ?? throw new ArgumentNullException(nameof(logger)); } - public async Task TryGetAsync(string sourceName, CancellationToken cancellationToken) - { - var cursor = await _collection.FindAsync(x => x.SourceName == sourceName, cancellationToken: cancellationToken).ConfigureAwait(false); - var document = await cursor.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return document?.ToRecord(); - } - - public async Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken) - { - var document = SourceStateDocumentExtensions.FromRecord(record with { UpdatedAt = DateTimeOffset.UtcNow }); - await _collection.ReplaceOneAsync( - x => x.SourceName == record.SourceName, - document, - new ReplaceOptions { IsUpsert = true }, - cancellationToken).ConfigureAwait(false); - - _logger.LogDebug("Upserted source state for {Source}", record.SourceName); - return document.ToRecord(); - } - - public async Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrEmpty(sourceName); - var update = Builders.Update - .Set(x => x.Cursor, cursor ?? new BsonDocument()) - .Set(x => x.LastSuccess, completedAt.UtcDateTime) + public async Task TryGetAsync(string sourceName, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName); + var query = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + var document = await query.FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task UpsertAsync(SourceStateRecord record, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var document = SourceStateDocumentExtensions.FromRecord(record with { UpdatedAt = DateTimeOffset.UtcNow }); + var filter = Builders.Filter.Eq(x => x.SourceName, record.SourceName); + var options = new ReplaceOptions { IsUpsert = true }; + + if (session is null) + { + await _collection.ReplaceOneAsync(filter, document, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, document, options, cancellationToken).ConfigureAwait(false); + } + + _logger.LogDebug("Upserted source state for {Source}", record.SourceName); + return document.ToRecord(); + } + + public async Task UpdateCursorAsync(string sourceName, BsonDocument cursor, DateTimeOffset completedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var update = Builders.Update + .Set(x => x.Cursor, cursor ?? new BsonDocument()) + .Set(x => x.LastSuccess, completedAt.UtcDateTime) .Set(x => x.FailCount, 0) .Set(x => x.BackoffUntil, (DateTime?)null) .Set(x => x.LastFailureReason, null) @@ -52,26 +61,23 @@ public sealed class MongoSourceStateRepository : ISourceStateRepository var options = new FindOneAndUpdateOptions { - ReturnDocument = ReturnDocument.After, - IsUpsert = true, - }; - - var document = await _collection - .FindOneAndUpdateAsync( - x => x.SourceName == sourceName, - update, - options, - cancellationToken) - .ConfigureAwait(false); - return document?.ToRecord(); - } - - public async Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrEmpty(sourceName); - var reasonValue = NormalizeFailureReason(failureReason); - var update = Builders.Update - .Inc(x => x.FailCount, 1) + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName); + var document = session is null + ? await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async Task MarkFailureAsync(string sourceName, DateTimeOffset failedAt, TimeSpan? backoff, string? failureReason, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrEmpty(sourceName); + var reasonValue = NormalizeFailureReason(failureReason); + var update = Builders.Update + .Inc(x => x.FailCount, 1) .Set(x => x.LastFailure, failedAt.UtcDateTime) .Set(x => x.BackoffUntil, backoff.HasValue ? failedAt.UtcDateTime.Add(backoff.Value) : null) .Set(x => x.LastFailureReason, reasonValue) @@ -80,19 +86,16 @@ public sealed class MongoSourceStateRepository : ISourceStateRepository var options = new FindOneAndUpdateOptions { - ReturnDocument = ReturnDocument.After, - IsUpsert = true, - }; - - var document = await _collection - .FindOneAndUpdateAsync( - x => x.SourceName == sourceName, - update, - options, - cancellationToken) - .ConfigureAwait(false); - return document?.ToRecord(); - } + ReturnDocument = ReturnDocument.After, + IsUpsert = true, + }; + + var filter = Builders.Filter.Eq(x => x.SourceName, sourceName); + var document = session is null + ? await _collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false) + : await _collection.FindOneAndUpdateAsync(session, filter, update, options, cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } private static string? NormalizeFailureReason(string? reason) { diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs similarity index 79% rename from src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs index a3c07576..f8f6492c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageDefaults.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageDefaults.cs @@ -1,8 +1,8 @@ -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public static class MongoStorageDefaults { - public const string DefaultDatabaseName = "feedser"; + public const string DefaultDatabaseName = "concelier"; public static class Collections { @@ -18,11 +18,13 @@ public static class MongoStorageDefaults public const string RuFlags = "ru_flags"; public const string JpFlags = "jp_flags"; public const string PsirtFlags = "psirt_flags"; - public const string MergeEvent = "merge_event"; - public const string ExportState = "export_state"; - public const string Locks = "locks"; - public const string Jobs = "jobs"; - public const string Migrations = "schema_migrations"; - public const string ChangeHistory = "source_change_history"; - } -} + public const string MergeEvent = "merge_event"; + public const string ExportState = "export_state"; + public const string Locks = "locks"; + public const string Jobs = "jobs"; + public const string Migrations = "schema_migrations"; + public const string ChangeHistory = "source_change_history"; + public const string AdvisoryStatements = "advisory_statements"; + public const string AdvisoryConflicts = "advisory_conflicts"; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageOptions.cs similarity index 96% rename from src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs rename to src/StellaOps.Concelier.Storage.Mongo/MongoStorageOptions.cs index fe7315b7..0b011f96 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/MongoStorageOptions.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/MongoStorageOptions.cs @@ -1,6 +1,6 @@ using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public sealed class MongoStorageOptions { diff --git a/src/StellaOps.Concelier.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Concelier.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..6164a773 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Concelier.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs similarity index 80% rename from src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs index 568f64d3..a70bf2a7 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/IPsirtFlagStore.cs @@ -1,7 +1,7 @@ using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; +namespace StellaOps.Concelier.Storage.Mongo.PsirtFlags; public interface IPsirtFlagStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs similarity index 93% rename from src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs index e572588e..92f3a3aa 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagDocument.cs @@ -1,6 +1,6 @@ using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; +namespace StellaOps.Concelier.Storage.Mongo.PsirtFlags; [BsonIgnoreExtraElements] public sealed class PsirtFlagDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs similarity index 84% rename from src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs index c3216ff5..4a809f77 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagRecord.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; +namespace StellaOps.Concelier.Storage.Mongo.PsirtFlags; /// /// Describes a PSIRT precedence flag for a canonical advisory. diff --git a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs rename to src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs index 4f42e232..788e07bd 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/PsirtFlags/PsirtFlagStore.cs @@ -3,7 +3,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using MongoDB.Driver; -namespace StellaOps.Feedser.Storage.Mongo.PsirtFlags; +namespace StellaOps.Concelier.Storage.Mongo.PsirtFlags; public sealed class PsirtFlagStore : IPsirtFlagStore { diff --git a/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs b/src/StellaOps.Concelier.Storage.Mongo/RawDocumentRetentionService.cs similarity index 94% rename from src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs rename to src/StellaOps.Concelier.Storage.Mongo/RawDocumentRetentionService.cs index 60027b57..9e99b401 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/RawDocumentRetentionService.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/RawDocumentRetentionService.cs @@ -3,10 +3,10 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Driver; using MongoDB.Driver.GridFS; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; /// /// Periodically purges expired raw documents, associated DTO payloads, and GridFS content. diff --git a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs similarity index 71% rename from src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs rename to src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs index a28b22e7..d726761a 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/ServiceCollectionExtensions.cs @@ -1,21 +1,24 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Driver; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Aliases; -using StellaOps.Feedser.Storage.Mongo.ChangeHistory; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; -using StellaOps.Feedser.Storage.Mongo.Exporting; -using StellaOps.Feedser.Storage.Mongo.JpFlags; -using StellaOps.Feedser.Storage.Mongo.MergeEvents; -using StellaOps.Feedser.Storage.Mongo.PsirtFlags; -using StellaOps.Feedser.Storage.Mongo.Migrations; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Storage.Mongo.Advisories; +using StellaOps.Concelier.Storage.Mongo.Aliases; +using StellaOps.Concelier.Storage.Mongo.ChangeHistory; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; +using StellaOps.Concelier.Storage.Mongo.Exporting; +using StellaOps.Concelier.Storage.Mongo.JpFlags; +using StellaOps.Concelier.Storage.Mongo.MergeEvents; +using StellaOps.Concelier.Storage.Mongo.Conflicts; +using StellaOps.Concelier.Storage.Mongo.PsirtFlags; +using StellaOps.Concelier.Storage.Mongo.Statements; +using StellaOps.Concelier.Storage.Mongo.Events; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Storage.Mongo.Migrations; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public static class ServiceCollectionExtensions { @@ -52,6 +55,8 @@ public static class ServiceCollectionExtensions return database.WithWriteConcern(writeConcern); }); + services.AddScoped(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -64,6 +69,10 @@ public static class ServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.TryAddSingleton(); @@ -81,11 +90,12 @@ public static class ServiceCollectionExtensions services.AddHostedService(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - return services; - } -} + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/SourceStateDocument.cs similarity index 95% rename from src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs rename to src/StellaOps.Concelier.Storage.Mongo/SourceStateDocument.cs index d9dacfac..519a59af 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/SourceStateDocument.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/SourceStateDocument.cs @@ -1,7 +1,7 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; [BsonIgnoreExtraElements] public sealed class SourceStateDocument diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/SourceStateRecord.cs similarity index 84% rename from src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs rename to src/StellaOps.Concelier.Storage.Mongo/SourceStateRecord.cs index 6ea2f339..896ce6ec 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRecord.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/SourceStateRecord.cs @@ -1,6 +1,6 @@ using MongoDB.Bson; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public sealed record SourceStateRecord( string SourceName, diff --git a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs b/src/StellaOps.Concelier.Storage.Mongo/SourceStateRepositoryExtensions.cs similarity index 89% rename from src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs rename to src/StellaOps.Concelier.Storage.Mongo/SourceStateRepositoryExtensions.cs index 7d4bb300..4a28e82c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/SourceStateRepositoryExtensions.cs +++ b/src/StellaOps.Concelier.Storage.Mongo/SourceStateRepositoryExtensions.cs @@ -2,7 +2,7 @@ using System; using System.Threading; using System.Threading.Tasks; -namespace StellaOps.Feedser.Storage.Mongo; +namespace StellaOps.Concelier.Storage.Mongo; public static class SourceStateRepositoryExtensions { diff --git a/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementDocument.cs b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementDocument.cs new file mode 100644 index 00000000..2312e9a8 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementDocument.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Concelier.Storage.Mongo.Statements; + +[BsonIgnoreExtraElements] +public sealed class AdvisoryStatementDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("vulnerabilityKey")] + public string VulnerabilityKey { get; set; } = string.Empty; + + [BsonElement("advisoryKey")] + public string AdvisoryKey { get; set; } = string.Empty; + + [BsonElement("statementHash")] + public byte[] StatementHash { get; set; } = Array.Empty(); + + [BsonElement("asOf")] + public DateTime AsOf { get; set; } + + [BsonElement("recordedAt")] + public DateTime RecordedAt { get; set; } + + [BsonElement("payload")] + public BsonDocument Payload { get; set; } = new(); + + [BsonElement("inputDocuments")] + public List InputDocuments { get; set; } = new(); +} + +internal static class AdvisoryStatementDocumentExtensions +{ + public static AdvisoryStatementDocument FromRecord(AdvisoryStatementRecord record) + => new() + { + Id = record.Id.ToString(), + VulnerabilityKey = record.VulnerabilityKey, + AdvisoryKey = record.AdvisoryKey, + StatementHash = record.StatementHash, + AsOf = record.AsOf.UtcDateTime, + RecordedAt = record.RecordedAt.UtcDateTime, + Payload = (BsonDocument)record.Payload.DeepClone(), + InputDocuments = record.InputDocumentIds.Select(static id => id.ToString()).ToList(), + }; + + public static AdvisoryStatementRecord ToRecord(this AdvisoryStatementDocument document) + => new( + Guid.Parse(document.Id), + document.VulnerabilityKey, + document.AdvisoryKey, + document.StatementHash, + DateTime.SpecifyKind(document.AsOf, DateTimeKind.Utc), + DateTime.SpecifyKind(document.RecordedAt, DateTimeKind.Utc), + (BsonDocument)document.Payload.DeepClone(), + document.InputDocuments.Select(static value => Guid.Parse(value)).ToList()); +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementRecord.cs b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementRecord.cs new file mode 100644 index 00000000..e782054a --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementRecord.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using MongoDB.Bson; + +namespace StellaOps.Concelier.Storage.Mongo.Statements; + +public sealed record AdvisoryStatementRecord( + Guid Id, + string VulnerabilityKey, + string AdvisoryKey, + byte[] StatementHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + BsonDocument Payload, + IReadOnlyList InputDocumentIds); diff --git a/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementStore.cs b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementStore.cs new file mode 100644 index 00000000..87637928 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/Statements/AdvisoryStatementStore.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Concelier.Storage.Mongo.Statements; + +public interface IAdvisoryStatementStore +{ + ValueTask InsertAsync( + IReadOnlyCollection statements, + CancellationToken cancellationToken, + IClientSessionHandle? session = null); + + ValueTask> GetStatementsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken, + IClientSessionHandle? session = null); +} + +public sealed class AdvisoryStatementStore : IAdvisoryStatementStore +{ + private readonly IMongoCollection _collection; + + public AdvisoryStatementStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + _collection = database.GetCollection(MongoStorageDefaults.Collections.AdvisoryStatements); + } + + public async ValueTask InsertAsync( + IReadOnlyCollection statements, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(statements); + + if (statements.Count == 0) + { + return; + } + + var documents = statements.Select(AdvisoryStatementDocumentExtensions.FromRecord).ToList(); + var options = new InsertManyOptions { IsOrdered = true }; + + try + { + if (session is null) + { + await _collection.InsertManyAsync(documents, options, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertManyAsync(session, documents, options, cancellationToken).ConfigureAwait(false); + } + } + catch (MongoBulkWriteException ex) when (ex.WriteErrors.All(error => error.Category == ServerErrorCategory.DuplicateKey)) + { + // All duplicates already exist – safe to ignore for immutable statement log. + } + } + + public async ValueTask> GetStatementsAsync( + string vulnerabilityKey, + DateTimeOffset? asOf, + CancellationToken cancellationToken, + IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityKey); + + var filter = Builders.Filter.Eq(document => document.VulnerabilityKey, vulnerabilityKey); + + if (asOf.HasValue) + { + filter &= Builders.Filter.Lte(document => document.AsOf, asOf.Value.UtcDateTime); + } + + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var documents = await find + .SortByDescending(document => document.AsOf) + .ThenByDescending(document => document.RecordedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return documents.Select(static document => document.ToRecord()).ToList(); + } +} diff --git a/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj b/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj new file mode 100644 index 00000000..57200b41 --- /dev/null +++ b/src/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md b/src/StellaOps.Concelier.Storage.Mongo/TASKS.md similarity index 85% rename from src/StellaOps.Feedser.Storage.Mongo/TASKS.md rename to src/StellaOps.Concelier.Storage.Mongo/TASKS.md index dddad01f..e9ddf5cc 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/TASKS.md +++ b/src/StellaOps.Concelier.Storage.Mongo/TASKS.md @@ -20,5 +20,5 @@ |FEEDSTORAGE-DATA-02-002 Provenance decision persistence|BE-Storage|Models `FEEDMODELS-SCHEMA-01-002`|**DONE (2025-10-12)** – Normalized documents carry decision reasons/source/timestamps with regression coverage verifying SemVer notes + provenance fallbacks.| |FEEDSTORAGE-DATA-02-003 Normalized versions index creation|BE-Storage|Normalization, Mongo bootstrapper|**DONE (2025-10-12)** – Bootstrapper seeds `normalizedVersions.*` indexes when SemVer style is enabled; docs/tests confirm index presence.| |FEEDSTORAGE-DATA-04-001 Advisory payload parity (description/CWEs/canonical metric)|BE-Storage|Models, Core|DONE (2025-10-15) – Mongo payloads round-trip new advisory fields; serializer/tests updated, no migration required beyond optional backfill.| -|FEEDSTORAGE-MONGO-08-001 Causal-consistent session plumbing|BE-Storage|Feedser Core DI|TODO – Introduce scoped MongoDB session provider enabling causal consistency + majority read/write concerns in `AddMongoStorage`; flow optional `IClientSessionHandle` through job/advisory/source state/document stores; add integration test simulating primary election to prove read-your-write + monotonic reads.| -|FEEDSTORAGE-DATA-07-001 Advisory statement & conflict collections|Team Normalization & Storage Backbone|FEEDMERGE-ENGINE-07-001|TODO – Create `advisory_statements` (immutable) and `advisory_conflicts` collections, define `asOf`/`vulnerabilityKey` indexes, and document migration/rollback steps for event-sourced merge.| +|FEEDSTORAGE-MONGO-08-001 Causal-consistent session plumbing|BE-Storage|Concelier Core DI|**DONE (2025-10-19)** – Scoped session provider now caches causal-consistent handles per scope, repositories accept optional sessions end-to-end, and new Mongo session consistency tests cover read-your-write + post-stepdown monotonic reads.| +|FEEDSTORAGE-DATA-07-001 Advisory statement & conflict collections|Team Normalization & Storage Backbone|FEEDMERGE-ENGINE-07-001|**DONE (2025-10-19)** – Added immutable `advisory_statements`/`advisory_conflicts` collections, bootstrapper + migration ensuring vulnerability/asOf + hash indexes, new stores (`AdvisoryStatementStore`, `AdvisoryConflictStore`), and docs outlining rollback. Tests: `dotnet test src/StellaOps.Concelier.Storage.Mongo.Tests/StellaOps.Concelier.Storage.Mongo.Tests.csproj`.| diff --git a/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs b/src/StellaOps.Concelier.Testing/ConnectorTestHarness.cs similarity index 91% rename from src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs rename to src/StellaOps.Concelier.Testing/ConnectorTestHarness.cs index 6e29296d..47146758 100644 --- a/src/StellaOps.Feedser.Testing/ConnectorTestHarness.cs +++ b/src/StellaOps.Concelier.Testing/ConnectorTestHarness.cs @@ -6,14 +6,14 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Feedser.Source.Common.Http; +using StellaOps.Concelier.Connector.Common.Http; using Microsoft.Extensions.Time.Testing; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Testing; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Testing; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Testing; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Testing; -namespace StellaOps.Feedser.Testing; +namespace StellaOps.Concelier.Testing; /// /// Provides a reusable container for connector integration tests with canned HTTP responses and Mongo isolation. diff --git a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs b/src/StellaOps.Concelier.Testing/MongoIntegrationFixture.cs similarity index 94% rename from src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs rename to src/StellaOps.Concelier.Testing/MongoIntegrationFixture.cs index c1843801..76cdc146 100644 --- a/src/StellaOps.Feedser.Testing/MongoIntegrationFixture.cs +++ b/src/StellaOps.Concelier.Testing/MongoIntegrationFixture.cs @@ -6,7 +6,7 @@ using Mongo2Go; using Xunit; using MongoDB.Driver; -namespace StellaOps.Feedser.Testing; +namespace StellaOps.Concelier.Testing; public sealed class MongoIntegrationFixture : IAsyncLifetime { @@ -19,7 +19,7 @@ public sealed class MongoIntegrationFixture : IAsyncLifetime EnsureMongo2GoEnvironment(); Runner = MongoDbRunner.Start(singleNodeReplSet: true); Client = new MongoClient(Runner.ConnectionString); - Database = Client.GetDatabase($"feedser-tests-{Guid.NewGuid():N}"); + Database = Client.GetDatabase($"concelier-tests-{Guid.NewGuid():N}"); return Task.CompletedTask; } diff --git a/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj b/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj similarity index 72% rename from src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj rename to src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj index 2a35b8de..64eb15ce 100644 --- a/src/StellaOps.Feedser.Testing/StellaOps.Feedser.Testing.csproj +++ b/src/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj @@ -14,7 +14,7 @@ - - + + diff --git a/src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs b/src/StellaOps.Concelier.Tests.Shared/AssemblyInfo.cs similarity index 100% rename from src/StellaOps.Feedser.Tests.Shared/AssemblyInfo.cs rename to src/StellaOps.Concelier.Tests.Shared/AssemblyInfo.cs diff --git a/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs b/src/StellaOps.Concelier.Tests.Shared/MongoFixtureCollection.cs similarity index 79% rename from src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs rename to src/StellaOps.Concelier.Tests.Shared/MongoFixtureCollection.cs index 11610647..d479aea2 100644 --- a/src/StellaOps.Feedser.Tests.Shared/MongoFixtureCollection.cs +++ b/src/StellaOps.Concelier.Tests.Shared/MongoFixtureCollection.cs @@ -1,6 +1,6 @@ using Xunit; -namespace StellaOps.Feedser.Testing; +namespace StellaOps.Concelier.Testing; [CollectionDefinition("mongo-fixture", DisableParallelization = true)] public sealed class MongoFixtureCollection : ICollectionFixture; diff --git a/src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs b/src/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs similarity index 56% rename from src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs rename to src/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs index 9c6ebef8..ce9c3bc1 100644 --- a/src/StellaOps.Feedser.WebService.Tests/FeedserOptionsPostConfigureTests.cs +++ b/src/StellaOps.Concelier.WebService.Tests/ConcelierOptionsPostConfigureTests.cs @@ -1,56 +1,56 @@ -using System; -using System.IO; -using StellaOps.Feedser.WebService.Options; -using Xunit; - -namespace StellaOps.Feedser.WebService.Tests; - -public sealed class FeedserOptionsPostConfigureTests -{ - [Fact] - public void Apply_LoadsClientSecretFromRelativeFile() - { - var tempDirectory = Directory.CreateTempSubdirectory(); - try - { - var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret"); - File.WriteAllText(secretPath, " feedser-secret "); - - var options = new FeedserOptions - { - Authority = new FeedserOptions.AuthorityOptions - { - ClientSecretFile = "authority.secret" - } - }; - - FeedserOptionsPostConfigure.Apply(options, tempDirectory.FullName); - - Assert.Equal("feedser-secret", options.Authority.ClientSecret); - } - finally - { - if (Directory.Exists(tempDirectory.FullName)) - { - Directory.Delete(tempDirectory.FullName, recursive: true); - } - } - } - - [Fact] - public void Apply_ThrowsWhenSecretFileMissing() - { - var options = new FeedserOptions - { - Authority = new FeedserOptions.AuthorityOptions - { - ClientSecretFile = "missing.secret" - } - }; - - var exception = Assert.Throws(() => - FeedserOptionsPostConfigure.Apply(options, AppContext.BaseDirectory)); - - Assert.Contains("Authority client secret file", exception.Message); - } -} +using System; +using System.IO; +using StellaOps.Concelier.WebService.Options; +using Xunit; + +namespace StellaOps.Concelier.WebService.Tests; + +public sealed class ConcelierOptionsPostConfigureTests +{ + [Fact] + public void Apply_LoadsClientSecretFromRelativeFile() + { + var tempDirectory = Directory.CreateTempSubdirectory(); + try + { + var secretPath = Path.Combine(tempDirectory.FullName, "authority.secret"); + File.WriteAllText(secretPath, " concelier-secret "); + + var options = new ConcelierOptions + { + Authority = new ConcelierOptions.AuthorityOptions + { + ClientSecretFile = "authority.secret" + } + }; + + ConcelierOptionsPostConfigure.Apply(options, tempDirectory.FullName); + + Assert.Equal("concelier-secret", options.Authority.ClientSecret); + } + finally + { + if (Directory.Exists(tempDirectory.FullName)) + { + Directory.Delete(tempDirectory.FullName, recursive: true); + } + } + } + + [Fact] + public void Apply_ThrowsWhenSecretFileMissing() + { + var options = new ConcelierOptions + { + Authority = new ConcelierOptions.AuthorityOptions + { + ClientSecretFile = "missing.secret" + } + }; + + var exception = Assert.Throws(() => + ConcelierOptionsPostConfigure.Apply(options, AppContext.BaseDirectory)); + + Assert.Contains("Authority client secret file", exception.Message); + } +} diff --git a/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs b/src/StellaOps.Concelier.WebService.Tests/PluginLoaderTests.cs similarity index 77% rename from src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs rename to src/StellaOps.Concelier.WebService.Tests/PluginLoaderTests.cs index b089b983..66ce058e 100644 --- a/src/StellaOps.Feedser.WebService.Tests/PluginLoaderTests.cs +++ b/src/StellaOps.Concelier.WebService.Tests/PluginLoaderTests.cs @@ -1,6 +1,6 @@ using StellaOps.Plugin; -namespace StellaOps.Feedser.WebService.Tests; +namespace StellaOps.Concelier.WebService.Tests; public class PluginLoaderTests { @@ -13,7 +13,7 @@ public class PluginLoaderTests public void ScansConnectorPluginsDirectory() { var services = new NullServices(); - var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "StellaOps.Concelier.PluginBinaries")); var plugins = catalog.GetAvailableConnectorPlugins(services); Assert.NotNull(plugins); } @@ -22,7 +22,7 @@ public class PluginLoaderTests public void ScansExporterPluginsDirectory() { var services = new NullServices(); - var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "PluginBinaries")); + var catalog = new PluginCatalog().AddFromDirectory(Path.Combine(AppContext.BaseDirectory, "StellaOps.Concelier.PluginBinaries")); var plugins = catalog.GetAvailableExporterPlugins(services); Assert.NotNull(plugins); } diff --git a/src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj b/src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj new file mode 100644 index 00000000..796eac59 --- /dev/null +++ b/src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs b/src/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs similarity index 62% rename from src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs rename to src/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs index c80d4287..ea0dae8a 100644 --- a/src/StellaOps.Feedser.WebService.Tests/WebServiceEndpointsTests.cs +++ b/src/StellaOps.Concelier.WebService.Tests/WebServiceEndpointsTests.cs @@ -1,5 +1,7 @@ using System; -using System.Collections.Generic; +using System.Collections.Generic; +using System.Globalization; +using System.IO; using System.Linq; using System.Net; using System.Net.Http.Json; @@ -12,24 +14,26 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Mongo2Go; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.WebService.Jobs; -using StellaOps.Feedser.WebService.Options; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.WebService.Jobs; +using StellaOps.Concelier.WebService.Options; using Xunit.Sdk; using StellaOps.Auth.Abstractions; using StellaOps.Auth.Client; -namespace StellaOps.Feedser.WebService.Tests; +namespace StellaOps.Concelier.WebService.Tests; public sealed class WebServiceEndpointsTests : IAsyncLifetime { private MongoDbRunner _runner = null!; - private FeedserApplicationFactory _factory = null!; + private ConcelierApplicationFactory _factory = null!; public Task InitializeAsync() { _runner = MongoDbRunner.Start(singleNodeReplSet: true); - _factory = new FeedserApplicationFactory(_runner.ConnectionString); + _factory = new ConcelierApplicationFactory(_runner.ConnectionString); return Task.CompletedTask; } @@ -71,9 +75,9 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime } [Fact] - public async Task JobsEndpointsReturnExpectedStatuses() - { - using var client = _factory.CreateClient(); + public async Task JobsEndpointsReturnExpectedStatuses() + { + using var client = _factory.CreateClient(); var definitions = await client.GetAsync("/jobs/definitions"); if (!definitions.IsSuccessStatusCode) @@ -217,24 +221,145 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime } [Fact] - public async Task JobsEndpointsAllowBypassWhenAuthorityEnabled() + public async Task AdvisoryReplayEndpointReturnsLatestStatement() { + var vulnerabilityKey = "CVE-2025-9000"; + var advisory = new Advisory( + advisoryKey: vulnerabilityKey, + title: "Replay Test", + summary: "Example summary", + language: "en", + published: DateTimeOffset.Parse("2025-01-01T00:00:00Z", CultureInfo.InvariantCulture), + modified: DateTimeOffset.Parse("2025-01-02T00:00:00Z", CultureInfo.InvariantCulture), + severity: "medium", + exploitKnown: false, + aliases: new[] { vulnerabilityKey }, + references: Array.Empty(), + affectedPackages: Array.Empty(), + cvssMetrics: Array.Empty(), + provenance: Array.Empty()); + + var statementId = Guid.NewGuid(); + using (var scope = _factory.Services.CreateScope()) + { + var eventLog = scope.ServiceProvider.GetRequiredService(); + var appendRequest = new AdvisoryEventAppendRequest(new[] + { + new AdvisoryStatementInput( + vulnerabilityKey, + advisory, + advisory.Modified ?? advisory.Published ?? DateTimeOffset.UtcNow, + Array.Empty(), + StatementId: statementId, + AdvisoryKey: advisory.AdvisoryKey) + }); + + await eventLog.AppendAsync(appendRequest, CancellationToken.None); + } + + using var client = _factory.CreateClient(); + var response = await client.GetAsync($"/concelier/advisories/{vulnerabilityKey}/replay"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.Equal(vulnerabilityKey, payload!.VulnerabilityKey, ignoreCase: true); + var statement = Assert.Single(payload.Statements); + Assert.Equal(statementId, statement.StatementId); + Assert.Equal(advisory.AdvisoryKey, statement.Advisory.AdvisoryKey); + Assert.False(string.IsNullOrWhiteSpace(statement.StatementHash)); + Assert.True(payload.Conflicts is null || payload.Conflicts!.Count == 0); + } + + [Fact] + public async Task MirrorEndpointsServeConfiguredArtifacts() + { + using var temp = new TempDirectory(); + var exportId = "20251019T120000Z"; + var exportRoot = Path.Combine(temp.Path, exportId); + var mirrorRoot = Path.Combine(exportRoot, "mirror"); + var domainRoot = Path.Combine(mirrorRoot, "primary"); + Directory.CreateDirectory(domainRoot); + + await File.WriteAllTextAsync( + Path.Combine(mirrorRoot, "index.json"), + """{"schemaVersion":1,"domains":[]}"""); + await File.WriteAllTextAsync( + Path.Combine(domainRoot, "manifest.json"), + """{"domainId":"primary"}"""); + await File.WriteAllTextAsync( + Path.Combine(domainRoot, "bundle.json"), + """{"advisories":[]}"""); + await File.WriteAllTextAsync( + Path.Combine(domainRoot, "bundle.json.jws"), + "test-signature"); + var environment = new Dictionary { - ["FEEDSER_AUTHORITY__ENABLED"] = "true", - ["FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", - ["FEEDSER_AUTHORITY__ISSUER"] = "https://authority.example", - ["FEEDSER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", - ["FEEDSER_AUTHORITY__AUDIENCES__0"] = "api://feedser", - ["FEEDSER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, - ["FEEDSER_AUTHORITY__BYPASSNETWORKS__0"] = "127.0.0.1/32", - ["FEEDSER_AUTHORITY__BYPASSNETWORKS__1"] = "::1/128", - ["FEEDSER_AUTHORITY__CLIENTID"] = "feedser-jobs", - ["FEEDSER_AUTHORITY__CLIENTSECRET"] = "test-secret", - ["FEEDSER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, + ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "primary", + ["CONCELIER_MIRROR__DOMAINS__0__DISPLAYNAME"] = "Primary", + ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "false", + ["CONCELIER_MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR"] = "5", + ["CONCELIER_MIRROR__MAXINDEXREQUESTSPERHOUR"] = "5" }; - using var factory = new FeedserApplicationFactory( + using var factory = new ConcelierApplicationFactory(_runner.ConnectionString, environmentOverrides: environment); + using var client = factory.CreateClient(); + + var indexResponse = await client.GetAsync("/concelier/exports/index.json"); + Assert.Equal(HttpStatusCode.OK, indexResponse.StatusCode); + var indexContent = await indexResponse.Content.ReadAsStringAsync(); + Assert.Contains(@"""schemaVersion"":1", indexContent, StringComparison.Ordinal); + + var manifestResponse = await client.GetAsync("/concelier/exports/mirror/primary/manifest.json"); + Assert.Equal(HttpStatusCode.OK, manifestResponse.StatusCode); + var manifestContent = await manifestResponse.Content.ReadAsStringAsync(); + Assert.Contains(@"""domainId"":""primary""", manifestContent, StringComparison.Ordinal); + + var bundleResponse = await client.GetAsync("/concelier/exports/mirror/primary/bundle.json.jws"); + Assert.Equal(HttpStatusCode.OK, bundleResponse.StatusCode); + var signatureContent = await bundleResponse.Content.ReadAsStringAsync(); + Assert.Equal("test-signature", signatureContent); + } + + [Fact] + public async Task MirrorEndpointsEnforceAuthenticationForProtectedDomains() + { + using var temp = new TempDirectory(); + var exportId = "20251019T120000Z"; + var secureRoot = Path.Combine(temp.Path, exportId, "mirror", "secure"); + Directory.CreateDirectory(secureRoot); + + await File.WriteAllTextAsync( + Path.Combine(temp.Path, exportId, "mirror", "index.json"), + """{"schemaVersion":1,"domains":[]}"""); + await File.WriteAllTextAsync( + Path.Combine(secureRoot, "manifest.json"), + """{"domainId":"secure"}"""); + + var environment = new Dictionary + { + ["CONCELIER_MIRROR__ENABLED"] = "true", + ["CONCELIER_MIRROR__EXPORTROOT"] = temp.Path, + ["CONCELIER_MIRROR__ACTIVEEXPORTID"] = exportId, + ["CONCELIER_MIRROR__DOMAINS__0__ID"] = "secure", + ["CONCELIER_MIRROR__DOMAINS__0__REQUIREAUTHENTICATION"] = "true", + ["CONCELIER_MIRROR__DOMAINS__0__MAXDOWNLOADREQUESTSPERHOUR"] = "5", + ["CONCELIER_AUTHORITY__ENABLED"] = "true", + ["CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", + ["CONCELIER_AUTHORITY__ISSUER"] = "https://authority.example", + ["CONCELIER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["CONCELIER_AUTHORITY__AUDIENCES__0"] = "api://concelier", + ["CONCELIER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + ["CONCELIER_AUTHORITY__CLIENTID"] = "concelier-jobs", + ["CONCELIER_AUTHORITY__CLIENTSECRET"] = "secret", + ["CONCELIER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger + }; + + using var factory = new ConcelierApplicationFactory( _runner.ConnectionString, authority => { @@ -243,13 +368,53 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime authority.Issuer = "https://authority.example"; authority.RequireHttpsMetadata = false; authority.Audiences.Clear(); - authority.Audiences.Add("api://feedser"); + authority.Audiences.Add("api://concelier"); authority.RequiredScopes.Clear(); - authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); + authority.ClientId = "concelier-jobs"; + authority.ClientSecret = "secret"; + }, + environment); + + using var client = factory.CreateClient(); + var response = await client.GetAsync("/concelier/exports/mirror/secure/manifest.json"); + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } + + [Fact] + public async Task JobsEndpointsAllowBypassWhenAuthorityEnabled() + { + var environment = new Dictionary + { + ["CONCELIER_AUTHORITY__ENABLED"] = "true", + ["CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", + ["CONCELIER_AUTHORITY__ISSUER"] = "https://authority.example", + ["CONCELIER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["CONCELIER_AUTHORITY__AUDIENCES__0"] = "api://concelier", + ["CONCELIER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + ["CONCELIER_AUTHORITY__BYPASSNETWORKS__0"] = "127.0.0.1/32", + ["CONCELIER_AUTHORITY__BYPASSNETWORKS__1"] = "::1/128", + ["CONCELIER_AUTHORITY__CLIENTID"] = "concelier-jobs", + ["CONCELIER_AUTHORITY__CLIENTSECRET"] = "test-secret", + ["CONCELIER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + }; + + using var factory = new ConcelierApplicationFactory( + _runner.ConnectionString, + authority => + { + authority.Enabled = true; + authority.AllowAnonymousFallback = false; + authority.Issuer = "https://authority.example"; + authority.RequireHttpsMetadata = false; + authority.Audiences.Clear(); + authority.Audiences.Add("api://concelier"); + authority.RequiredScopes.Clear(); + authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); authority.BypassNetworks.Clear(); authority.BypassNetworks.Add("127.0.0.1/32"); authority.BypassNetworks.Add("::1/128"); - authority.ClientId = "feedser-jobs"; + authority.ClientId = "concelier-jobs"; authority.ClientSecret = "test-secret"; }, environment); @@ -263,7 +428,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime Assert.Equal(HttpStatusCode.OK, response.StatusCode); - var auditLogs = factory.LoggerProvider.Snapshot("Feedser.Authorization.Audit"); + var auditLogs = factory.LoggerProvider.Snapshot("Concelier.Authorization.Audit"); var bypassLog = Assert.Single(auditLogs, entry => entry.TryGetState("Bypass", out var state) && state is bool flag && flag); Assert.True(bypassLog.TryGetState("RemoteAddress", out var remoteObj) && string.Equals(remoteObj?.ToString(), "127.0.0.1", StringComparison.Ordinal)); Assert.True(bypassLog.TryGetState("StatusCode", out var statusObj) && Convert.ToInt32(statusObj) == (int)HttpStatusCode.OK); @@ -274,18 +439,18 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { var enforcementEnvironment = new Dictionary { - ["FEEDSER_AUTHORITY__ENABLED"] = "true", - ["FEEDSER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", - ["FEEDSER_AUTHORITY__ISSUER"] = "https://authority.example", - ["FEEDSER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", - ["FEEDSER_AUTHORITY__AUDIENCES__0"] = "api://feedser", - ["FEEDSER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, - ["FEEDSER_AUTHORITY__CLIENTID"] = "feedser-jobs", - ["FEEDSER_AUTHORITY__CLIENTSECRET"] = "test-secret", - ["FEEDSER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, + ["CONCELIER_AUTHORITY__ENABLED"] = "true", + ["CONCELIER_AUTHORITY__ALLOWANONYMOUSFALLBACK"] = "false", + ["CONCELIER_AUTHORITY__ISSUER"] = "https://authority.example", + ["CONCELIER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["CONCELIER_AUTHORITY__AUDIENCES__0"] = "api://concelier", + ["CONCELIER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + ["CONCELIER_AUTHORITY__CLIENTID"] = "concelier-jobs", + ["CONCELIER_AUTHORITY__CLIENTSECRET"] = "test-secret", + ["CONCELIER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, }; - using var factory = new FeedserApplicationFactory( + using var factory = new ConcelierApplicationFactory( _runner.ConnectionString, authority => { @@ -294,16 +459,16 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime authority.Issuer = "https://authority.example"; authority.RequireHttpsMetadata = false; authority.Audiences.Clear(); - authority.Audiences.Add("api://feedser"); + authority.Audiences.Add("api://concelier"); authority.RequiredScopes.Clear(); - authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); authority.BypassNetworks.Clear(); - authority.ClientId = "feedser-jobs"; + authority.ClientId = "concelier-jobs"; authority.ClientSecret = "test-secret"; }, enforcementEnvironment); - var resolved = factory.Services.GetRequiredService>().Value; + var resolved = factory.Services.GetRequiredService>().Value; Assert.False(resolved.Authority.AllowAnonymousFallback); using var client = factory.CreateClient(); @@ -312,7 +477,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); - var auditLogs = factory.LoggerProvider.Snapshot("Feedser.Authorization.Audit"); + var auditLogs = factory.LoggerProvider.Snapshot("Concelier.Authorization.Audit"); var enforcementLog = Assert.Single(auditLogs); Assert.True(enforcementLog.TryGetState("BypassAllowed", out var bypassAllowedObj) && bypassAllowedObj is bool bypassAllowed && bypassAllowed == false); Assert.True(enforcementLog.TryGetState("HasPrincipal", out var principalObj) && principalObj is bool hasPrincipal && hasPrincipal == false); @@ -323,21 +488,21 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { var environment = new Dictionary { - ["FEEDSER_AUTHORITY__ENABLED"] = "true", - ["FEEDSER_AUTHORITY__ISSUER"] = "https://authority.example", - ["FEEDSER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", - ["FEEDSER_AUTHORITY__AUDIENCES__0"] = "api://feedser", - ["FEEDSER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, - ["FEEDSER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.FeedserJobsTrigger, - ["FEEDSER_AUTHORITY__BACKCHANNELTIMEOUTSECONDS"] = "45", - ["FEEDSER_AUTHORITY__RESILIENCE__ENABLERETRIES"] = "true", - ["FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__0"] = "00:00:02", - ["FEEDSER_AUTHORITY__RESILIENCE__RETRYDELAYS__1"] = "00:00:04", - ["FEEDSER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK"] = "false", - ["FEEDSER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE"] = "00:02:30" + ["CONCELIER_AUTHORITY__ENABLED"] = "true", + ["CONCELIER_AUTHORITY__ISSUER"] = "https://authority.example", + ["CONCELIER_AUTHORITY__REQUIREHTTPSMETADATA"] = "false", + ["CONCELIER_AUTHORITY__AUDIENCES__0"] = "api://concelier", + ["CONCELIER_AUTHORITY__REQUIREDSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + ["CONCELIER_AUTHORITY__CLIENTSCOPES__0"] = StellaOpsScopes.ConcelierJobsTrigger, + ["CONCELIER_AUTHORITY__BACKCHANNELTIMEOUTSECONDS"] = "45", + ["CONCELIER_AUTHORITY__RESILIENCE__ENABLERETRIES"] = "true", + ["CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__0"] = "00:00:02", + ["CONCELIER_AUTHORITY__RESILIENCE__RETRYDELAYS__1"] = "00:00:04", + ["CONCELIER_AUTHORITY__RESILIENCE__ALLOWOFFLINECACHEFALLBACK"] = "false", + ["CONCELIER_AUTHORITY__RESILIENCE__OFFLINECACHETOLERANCE"] = "00:02:30" }; - using var factory = new FeedserApplicationFactory( + using var factory = new ConcelierApplicationFactory( _runner.ConnectionString, authority => { @@ -345,11 +510,11 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime authority.Issuer = "https://authority.example"; authority.RequireHttpsMetadata = false; authority.Audiences.Clear(); - authority.Audiences.Add("api://feedser"); + authority.Audiences.Add("api://concelier"); authority.RequiredScopes.Clear(); - authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); authority.ClientScopes.Clear(); - authority.ClientScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + authority.ClientScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); authority.BackchannelTimeoutSeconds = 45; }, environment); @@ -359,13 +524,38 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime Assert.Equal("https://authority.example", options.Authority); Assert.Equal(TimeSpan.FromSeconds(45), options.HttpTimeout); - Assert.Equal(new[] { StellaOpsScopes.FeedserJobsTrigger }, options.NormalizedScopes); + Assert.Equal(new[] { StellaOpsScopes.ConcelierJobsTrigger }, options.NormalizedScopes); Assert.Equal(new[] { TimeSpan.FromSeconds(2), TimeSpan.FromSeconds(4) }, options.NormalizedRetryDelays); Assert.False(options.AllowOfflineCacheFallback); Assert.Equal(TimeSpan.FromSeconds(150), options.OfflineCacheTolerance); } - private sealed class FeedserApplicationFactory : WebApplicationFactory + private sealed record ReplayResponse( + string VulnerabilityKey, + DateTimeOffset? AsOf, + List Statements, + List? Conflicts); + + private sealed record ReplayStatement( + Guid StatementId, + string VulnerabilityKey, + string AdvisoryKey, + Advisory Advisory, + string StatementHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + IReadOnlyList InputDocumentIds); + + private sealed record ReplayConflict( + Guid ConflictId, + string VulnerabilityKey, + IReadOnlyList StatementIds, + string ConflictHash, + DateTimeOffset AsOf, + DateTimeOffset RecordedAt, + string Details); + + private sealed class ConcelierApplicationFactory : WebApplicationFactory { private readonly string _connectionString; private readonly string? _previousDsn; @@ -375,31 +565,31 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime private readonly string? _previousTelemetryLogging; private readonly string? _previousTelemetryTracing; private readonly string? _previousTelemetryMetrics; - private readonly Action? _authorityConfigure; + private readonly Action? _authorityConfigure; private readonly IDictionary _additionalPreviousEnvironment = new Dictionary(StringComparer.OrdinalIgnoreCase); public CollectingLoggerProvider LoggerProvider { get; } = new(); - public FeedserApplicationFactory( + public ConcelierApplicationFactory( string connectionString, - Action? authorityConfigure = null, + Action? authorityConfigure = null, IDictionary? environmentOverrides = null) { _connectionString = connectionString; _authorityConfigure = authorityConfigure; - _previousDsn = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DSN"); - _previousDriver = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__DRIVER"); - _previousTimeout = Environment.GetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS"); - _previousTelemetryEnabled = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED"); - _previousTelemetryLogging = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING"); - _previousTelemetryTracing = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING"); - _previousTelemetryMetrics = Environment.GetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS"); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", connectionString); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", "mongo"); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", "30"); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", "false"); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", "false"); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", "false"); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", "false"); + _previousDsn = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__DSN"); + _previousDriver = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__DRIVER"); + _previousTimeout = Environment.GetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS"); + _previousTelemetryEnabled = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED"); + _previousTelemetryLogging = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING"); + _previousTelemetryTracing = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING"); + _previousTelemetryMetrics = Environment.GetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS"); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DSN", connectionString); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DRIVER", "mongo"); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS", "30"); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", "false"); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", "false"); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", "false"); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", "false"); if (environmentOverrides is not null) { foreach (var kvp in environmentOverrides) @@ -417,7 +607,7 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { var settings = new Dictionary { - ["Plugins:Directory"] = Path.Combine(context.HostingEnvironment.ContentRootPath, "PluginBinaries"), + ["Plugins:Directory"] = Path.Combine(context.HostingEnvironment.ContentRootPath, "StellaOps.Concelier.PluginBinaries"), }; configurationBuilder.AddInMemoryCollection(settings!); @@ -432,17 +622,17 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime { services.AddSingleton(); services.AddSingleton(sp => sp.GetRequiredService()); - services.PostConfigure(options => + services.PostConfigure(options => { options.Storage.Driver = "mongo"; options.Storage.Dsn = _connectionString; options.Storage.CommandTimeoutSeconds = 30; - options.Plugins.Directory ??= Path.Combine(AppContext.BaseDirectory, "PluginBinaries"); + options.Plugins.Directory ??= Path.Combine(AppContext.BaseDirectory, "StellaOps.Concelier.PluginBinaries"); options.Telemetry.Enabled = false; options.Telemetry.EnableLogging = false; options.Telemetry.EnableTracing = false; options.Telemetry.EnableMetrics = false; - options.Authority ??= new FeedserOptions.AuthorityOptions(); + options.Authority ??= new ConcelierOptions.AuthorityOptions(); _authorityConfigure?.Invoke(options.Authority); }); }); @@ -456,13 +646,13 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime protected override void Dispose(bool disposing) { base.Dispose(disposing); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DSN", _previousDsn); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__DRIVER", _previousDriver); - Environment.SetEnvironmentVariable("FEEDSER_STORAGE__COMMANDTIMEOUTSECONDS", _previousTimeout); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLED", _previousTelemetryEnabled); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLELOGGING", _previousTelemetryLogging); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLETRACING", _previousTelemetryTracing); - Environment.SetEnvironmentVariable("FEEDSER_TELEMETRY__ENABLEMETRICS", _previousTelemetryMetrics); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DSN", _previousDsn); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__DRIVER", _previousDriver); + Environment.SetEnvironmentVariable("CONCELIER_STORAGE__COMMANDTIMEOUTSECONDS", _previousTimeout); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLED", _previousTelemetryEnabled); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLELOGGING", _previousTelemetryLogging); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLETRACING", _previousTelemetryTracing); + Environment.SetEnvironmentVariable("CONCELIER_TELEMETRY__ENABLEMETRICS", _previousTelemetryMetrics); foreach (var kvp in _additionalPreviousEnvironment) { Environment.SetEnvironmentVariable(kvp.Key, kvp.Value); @@ -616,6 +806,32 @@ public sealed class WebServiceEndpointsTests : IAsyncLifetime } } } + + private sealed class TempDirectory : IDisposable + { + public string Path { get; } + + public TempDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "concelier-mirror-" + Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture)); + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch + { + // best effort cleanup + } + } + } private sealed record HealthPayload(string Status, DateTimeOffset StartedAt, double UptimeSeconds, StoragePayload Storage, TelemetryPayload Telemetry); diff --git a/src/StellaOps.Feedser.WebService/AGENTS.md b/src/StellaOps.Concelier.WebService/AGENTS.md similarity index 79% rename from src/StellaOps.Feedser.WebService/AGENTS.md rename to src/StellaOps.Concelier.WebService/AGENTS.md index c6bbeabf..e09ae40c 100644 --- a/src/StellaOps.Feedser.WebService/AGENTS.md +++ b/src/StellaOps.Concelier.WebService/AGENTS.md @@ -2,8 +2,8 @@ ## Role Minimal API host wiring configuration, storage, plugin routines, and job endpoints. Operational surface for health, readiness, and job control. ## Scope -- Configuration: appsettings.json + etc/feedser.yaml (yaml path = ../etc/feedser.yaml); bind into FeedserOptions with validation (Only Mongo supported). -- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "feedser"). +- Configuration: appsettings.json + etc/concelier.yaml (yaml path = ../etc/concelier.yaml); bind into ConcelierOptions with validation (Only Mongo supported). +- Mongo: MongoUrl from options.Storage.Dsn; IMongoClient/IMongoDatabase singletons; default database name fallback (options -> URL -> "concelier"). - Services: AddMongoStorage(); AddSourceHttpClients(); RegisterPluginRoutines(configuration, PluginHostOptions). - Bootstrap: MongoBootstrapper.InitializeAsync on startup. - Endpoints (configuration & job control only; root path intentionally unbound): @@ -16,11 +16,11 @@ Minimal API host wiring configuration, storage, plugin routines, and job endpoin - GET /jobs/definitions/{kind}/runs?limit= -> recent runs or 404 if kind unknown. - GET /jobs/active -> currently running. - POST /jobs/{*jobKind} with {trigger?,parameters?} -> 202 Accepted (Location:/jobs/{runId}) | 404 | 409 | 423. -- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "PluginBinaries"; SearchPatterns += "StellaOps.Feedser.Plugin.*.dll"; EnsureDirectoryExists = true. +- PluginHost defaults: BaseDirectory = solution root; PluginsDirectory = "StellaOps.Concelier.PluginBinaries"; SearchPatterns += "StellaOps.Concelier.Plugin.*.dll"; EnsureDirectoryExists = true. ## Participants - Core job system; Storage.Mongo; Source.Common HTTP clients; Exporter and Connector plugin routines discover/register jobs. ## Interfaces & contracts -- Dependency injection boundary for all connectors/exporters; IOptions validated on start. +- Dependency injection boundary for all connectors/exporters; IOptions validated on start. - Cancellation: pass app.Lifetime.ApplicationStopping to bootstrapper. ## In/Out of scope In: hosting, DI composition, REST surface, readiness checks. @@ -29,6 +29,6 @@ Out: business logic of jobs, HTML UI, authn/z (future). - Log startup config (redact DSN credentials), plugin scan results (missing ordered plugins if any). - Structured responses with status codes; no stack traces in HTTP bodies; errors mapped cleanly. ## Tests -- Author and review coverage in `../StellaOps.Feedser.WebService.Tests`. -- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Feedser.Testing`. +- Author and review coverage in `../StellaOps.Concelier.WebService.Tests`. +- Shared fixtures (e.g., `MongoIntegrationFixture`, `ConnectorTestHarness`) live in `../StellaOps.Concelier.Testing`. - Keep fixtures deterministic; match new cases to real-world advisories or regression scenarios. diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs b/src/StellaOps.Concelier.WebService/Diagnostics/HealthContracts.cs similarity index 89% rename from src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs rename to src/StellaOps.Concelier.WebService/Diagnostics/HealthContracts.cs index 63707650..66f70931 100644 --- a/src/StellaOps.Feedser.WebService/Diagnostics/HealthContracts.cs +++ b/src/StellaOps.Concelier.WebService/Diagnostics/HealthContracts.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.WebService.Diagnostics; +namespace StellaOps.Concelier.WebService.Diagnostics; internal sealed record StorageBootstrapHealth( string Driver, diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs b/src/StellaOps.Concelier.WebService/Diagnostics/JobMetrics.cs similarity index 84% rename from src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs rename to src/StellaOps.Concelier.WebService/Diagnostics/JobMetrics.cs index cb85ff95..04ca6f43 100644 --- a/src/StellaOps.Feedser.WebService/Diagnostics/JobMetrics.cs +++ b/src/StellaOps.Concelier.WebService/Diagnostics/JobMetrics.cs @@ -1,10 +1,10 @@ using System.Diagnostics.Metrics; -namespace StellaOps.Feedser.WebService.Diagnostics; +namespace StellaOps.Concelier.WebService.Diagnostics; internal static class JobMetrics { - internal const string MeterName = "StellaOps.Feedser.WebService.Jobs"; + internal const string MeterName = "StellaOps.Concelier.WebService.Jobs"; private static readonly Meter Meter = new(MeterName); diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs b/src/StellaOps.Concelier.WebService/Diagnostics/ProblemTypes.cs similarity index 90% rename from src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs rename to src/StellaOps.Concelier.WebService/Diagnostics/ProblemTypes.cs index 8be947b9..bb3352b8 100644 --- a/src/StellaOps.Feedser.WebService/Diagnostics/ProblemTypes.cs +++ b/src/StellaOps.Concelier.WebService/Diagnostics/ProblemTypes.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.WebService.Diagnostics; +namespace StellaOps.Concelier.WebService.Diagnostics; internal static class ProblemTypes { diff --git a/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs b/src/StellaOps.Concelier.WebService/Diagnostics/ServiceStatus.cs similarity index 94% rename from src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs rename to src/StellaOps.Concelier.WebService/Diagnostics/ServiceStatus.cs index 256d09bf..a7c6fbda 100644 --- a/src/StellaOps.Feedser.WebService/Diagnostics/ServiceStatus.cs +++ b/src/StellaOps.Concelier.WebService/Diagnostics/ServiceStatus.cs @@ -1,6 +1,6 @@ using System.Diagnostics; -namespace StellaOps.Feedser.WebService.Diagnostics; +namespace StellaOps.Concelier.WebService.Diagnostics; internal sealed class ServiceStatus { diff --git a/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs b/src/StellaOps.Concelier.WebService/Extensions/ConfigurationExtensions.cs similarity index 83% rename from src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs rename to src/StellaOps.Concelier.WebService/Extensions/ConfigurationExtensions.cs index 155d7a49..cb8785ba 100644 --- a/src/StellaOps.Feedser.WebService/Extensions/ConfigurationExtensions.cs +++ b/src/StellaOps.Concelier.WebService/Extensions/ConfigurationExtensions.cs @@ -4,11 +4,11 @@ using Microsoft.Extensions.Configuration; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; -namespace StellaOps.Feedser.WebService.Extensions; +namespace StellaOps.Concelier.WebService.Extensions; public static class ConfigurationExtensions { - public static IConfigurationBuilder AddFeedserYaml(this IConfigurationBuilder builder, string path) + public static IConfigurationBuilder AddConcelierYaml(this IConfigurationBuilder builder, string path) { if (builder is null) { diff --git a/src/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs b/src/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs new file mode 100644 index 00000000..91b574ed --- /dev/null +++ b/src/StellaOps.Concelier.WebService/Extensions/JobRegistrationExtensions.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Merge.Jobs; + +namespace StellaOps.Concelier.WebService.Extensions; + +internal static class JobRegistrationExtensions +{ + private sealed record BuiltInJob( + string Kind, + string JobType, + string AssemblyName, + TimeSpan Timeout, + TimeSpan LeaseDuration, + string? CronExpression = null); + + private static readonly IReadOnlyList BuiltInJobs = new List + { + new("source:redhat:fetch", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatFetchJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *"), + new("source:redhat:parse", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatParseJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *"), + new("source:redhat:map", "StellaOps.Concelier.Connector.Distro.RedHat.RedHatMapJob", "StellaOps.Concelier.Connector.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *"), + + new("source:cert-in:fetch", "StellaOps.Concelier.Connector.CertIn.CertInFetchJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:parse", "StellaOps.Concelier.Connector.CertIn.CertInParseJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-in:map", "StellaOps.Concelier.Connector.CertIn.CertInMapJob", "StellaOps.Concelier.Connector.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:cert-fr:fetch", "StellaOps.Concelier.Connector.CertFr.CertFrFetchJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:parse", "StellaOps.Concelier.Connector.CertFr.CertFrParseJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:cert-fr:map", "StellaOps.Concelier.Connector.CertFr.CertFrMapJob", "StellaOps.Concelier.Connector.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:jvn:fetch", "StellaOps.Concelier.Connector.Jvn.JvnFetchJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:parse", "StellaOps.Concelier.Connector.Jvn.JvnParseJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:jvn:map", "StellaOps.Concelier.Connector.Jvn.JvnMapJob", "StellaOps.Concelier.Connector.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:ics-kaspersky:fetch", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyFetchJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:parse", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyParseJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:ics-kaspersky:map", "StellaOps.Concelier.Connector.Ics.Kaspersky.KasperskyMapJob", "StellaOps.Concelier.Connector.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:osv:fetch", "StellaOps.Concelier.Connector.Osv.OsvFetchJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:parse", "StellaOps.Concelier.Connector.Osv.OsvParseJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:osv:map", "StellaOps.Concelier.Connector.Osv.OsvMapJob", "StellaOps.Concelier.Connector.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vmware:fetch", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareFetchJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:parse", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareParseJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vmware:map", "StellaOps.Concelier.Connector.Vndr.Vmware.VmwareMapJob", "StellaOps.Concelier.Connector.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("source:vndr-oracle:fetch", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleFetchJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:parse", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleParseJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + new("source:vndr-oracle:map", "StellaOps.Concelier.Connector.Vndr.Oracle.OracleMapJob", "StellaOps.Concelier.Connector.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), + + new("export:json", "StellaOps.Concelier.Exporter.Json.JsonExportJob", "StellaOps.Concelier.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)), + new("export:trivy-db", "StellaOps.Concelier.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Concelier.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)), + new("merge:reconcile", "StellaOps.Concelier.Merge.Jobs.MergeReconcileJob", "StellaOps.Concelier.Merge", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)) + }; + + public static IServiceCollection AddBuiltInConcelierJobs(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.PostConfigure(options => + { + foreach (var registration in BuiltInJobs) + { + if (options.Definitions.ContainsKey(registration.Kind)) + { + continue; + } + + var jobType = Type.GetType( + $"{registration.JobType}, {registration.AssemblyName}", + throwOnError: false, + ignoreCase: false); + + if (jobType is null) + { + continue; + } + + var timeout = registration.Timeout > TimeSpan.Zero ? registration.Timeout : options.DefaultTimeout; + var lease = registration.LeaseDuration > TimeSpan.Zero ? registration.LeaseDuration : options.DefaultLeaseDuration; + + options.Definitions[registration.Kind] = new JobDefinition( + registration.Kind, + jobType, + timeout, + lease, + registration.CronExpression, + Enabled: true); + } + }); + + return services; + } +} diff --git a/src/StellaOps.Concelier.WebService/Extensions/MirrorEndpointExtensions.cs b/src/StellaOps.Concelier.WebService/Extensions/MirrorEndpointExtensions.cs new file mode 100644 index 00000000..580af391 --- /dev/null +++ b/src/StellaOps.Concelier.WebService/Extensions/MirrorEndpointExtensions.cs @@ -0,0 +1,181 @@ +using System.Globalization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.WebService.Options; +using StellaOps.Concelier.WebService.Services; + +namespace StellaOps.Concelier.WebService.Extensions; + +internal static class MirrorEndpointExtensions +{ + private const string IndexScope = "index"; + private const string DownloadScope = "download"; + + public static void MapConcelierMirrorEndpoints(this WebApplication app, bool authorityConfigured, bool enforceAuthority) + { + app.MapGet("/concelier/exports/index.json", async ( + MirrorFileLocator locator, + MirrorRateLimiter limiter, + IOptionsMonitor optionsMonitor, + HttpContext context, + CancellationToken cancellationToken) => + { + var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions(); + if (!mirrorOptions.Enabled) + { + return Results.NotFound(); + } + + if (!TryAuthorize(mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult)) + { + return unauthorizedResult; + } + + if (!limiter.TryAcquire("__index__", IndexScope, mirrorOptions.MaxIndexRequestsPerHour, out var retryAfter)) + { + ApplyRetryAfter(context.Response, retryAfter); + return Results.StatusCode(StatusCodes.Status429TooManyRequests); + } + + if (!locator.TryResolveIndex(out var path, out _)) + { + return Results.NotFound(); + } + + return await WriteFileAsync(path, context.Response, "application/json").ConfigureAwait(false); + }); + + app.MapGet("/concelier/exports/{**relativePath}", async ( + string? relativePath, + MirrorFileLocator locator, + MirrorRateLimiter limiter, + IOptionsMonitor optionsMonitor, + HttpContext context, + CancellationToken cancellationToken) => + { + var mirrorOptions = optionsMonitor.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions(); + if (!mirrorOptions.Enabled) + { + return Results.NotFound(); + } + + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Results.NotFound(); + } + + if (!locator.TryResolveRelativePath(relativePath, out var path, out _, out var domainId)) + { + return Results.NotFound(); + } + + var domain = FindDomain(mirrorOptions, domainId); + + if (!TryAuthorize(domain?.RequireAuthentication ?? mirrorOptions.RequireAuthentication, enforceAuthority, context, authorityConfigured, out var unauthorizedResult)) + { + return unauthorizedResult; + } + + var limit = domain?.MaxDownloadRequestsPerHour ?? mirrorOptions.MaxIndexRequestsPerHour; + if (!limiter.TryAcquire(domain?.Id ?? "__mirror__", DownloadScope, limit, out var retryAfter)) + { + ApplyRetryAfter(context.Response, retryAfter); + return Results.StatusCode(StatusCodes.Status429TooManyRequests); + } + + var contentType = ResolveContentType(path); + return await WriteFileAsync(path, context.Response, contentType).ConfigureAwait(false); + }); + } + + private static ConcelierOptions.MirrorDomainOptions? FindDomain(ConcelierOptions.MirrorOptions mirrorOptions, string? domainId) + { + if (domainId is null) + { + return null; + } + + foreach (var candidate in mirrorOptions.Domains) + { + if (candidate is null) + { + continue; + } + + if (string.Equals(candidate.Id, domainId, StringComparison.OrdinalIgnoreCase)) + { + return candidate; + } + } + + return null; + } + + private static bool TryAuthorize(bool requireAuthentication, bool enforceAuthority, HttpContext context, bool authorityConfigured, out IResult result) + { + result = Results.Empty; + if (!requireAuthentication) + { + return true; + } + + if (!enforceAuthority || !authorityConfigured) + { + return true; + } + + if (context.User?.Identity?.IsAuthenticated == true) + { + return true; + } + + result = Results.StatusCode(StatusCodes.Status401Unauthorized); + return false; + } + + private static Task WriteFileAsync(string path, HttpResponse response, string contentType) + { + var fileInfo = new FileInfo(path); + if (!fileInfo.Exists) + { + return Task.FromResult(Results.NotFound()); + } + + var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read | FileShare.Delete); + + response.Headers.CacheControl = "public, max-age=60"; + response.Headers.LastModified = fileInfo.LastWriteTimeUtc.ToString("R", CultureInfo.InvariantCulture); + response.ContentLength = fileInfo.Length; + return Task.FromResult(Results.Stream(stream, contentType)); + } + + private static string ResolveContentType(string path) + { + if (path.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + return "application/json"; + } + + if (path.EndsWith(".jws", StringComparison.OrdinalIgnoreCase)) + { + return "application/jose+json"; + } + + return "application/octet-stream"; + } + + private static void ApplyRetryAfter(HttpResponse response, TimeSpan? retryAfter) + { + if (retryAfter is null) + { + return; + } + + var seconds = Math.Max((int)Math.Ceiling(retryAfter.Value.TotalSeconds), 1); + response.Headers.RetryAfter = seconds.ToString(CultureInfo.InvariantCulture); + } +} diff --git a/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs b/src/StellaOps.Concelier.WebService/Extensions/TelemetryExtensions.cs similarity index 81% rename from src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs rename to src/StellaOps.Concelier.WebService/Extensions/TelemetryExtensions.cs index 8d67a09b..bc67e255 100644 --- a/src/StellaOps.Feedser.WebService/Extensions/TelemetryExtensions.cs +++ b/src/StellaOps.Concelier.WebService/Extensions/TelemetryExtensions.cs @@ -10,21 +10,21 @@ using OpenTelemetry.Trace; using Serilog; using Serilog.Core; using Serilog.Events; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Source.Common.Telemetry; -using StellaOps.Feedser.WebService.Diagnostics; -using StellaOps.Feedser.WebService.Options; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Connector.Common.Telemetry; +using StellaOps.Concelier.WebService.Diagnostics; +using StellaOps.Concelier.WebService.Options; -namespace StellaOps.Feedser.WebService.Extensions; +namespace StellaOps.Concelier.WebService.Extensions; public static class TelemetryExtensions { - public static void ConfigureFeedserTelemetry(this WebApplicationBuilder builder, FeedserOptions options) + public static void ConfigureConcelierTelemetry(this WebApplicationBuilder builder, ConcelierOptions options) { ArgumentNullException.ThrowIfNull(builder); ArgumentNullException.ThrowIfNull(options); - var telemetry = options.Telemetry ?? new FeedserOptions.TelemetryOptions(); + var telemetry = options.Telemetry ?? new ConcelierOptions.TelemetryOptions(); if (telemetry.EnableLogging) { @@ -84,11 +84,11 @@ public static class TelemetryExtensions metrics .AddMeter(JobDiagnostics.MeterName) .AddMeter(SourceDiagnostics.MeterName) - .AddMeter("StellaOps.Feedser.Source.CertBund") - .AddMeter("StellaOps.Feedser.Source.Nvd") - .AddMeter("StellaOps.Feedser.Source.Vndr.Chromium") - .AddMeter("StellaOps.Feedser.Source.Vndr.Apple") - .AddMeter("StellaOps.Feedser.Source.Vndr.Adobe") + .AddMeter("StellaOps.Concelier.Connector.CertBund") + .AddMeter("StellaOps.Concelier.Connector.Nvd") + .AddMeter("StellaOps.Concelier.Connector.Vndr.Chromium") + .AddMeter("StellaOps.Concelier.Connector.Vndr.Apple") + .AddMeter("StellaOps.Concelier.Connector.Vndr.Adobe") .AddMeter(JobMetrics.MeterName) .AddAspNetCoreInstrumentation() .AddHttpClientInstrumentation() @@ -99,7 +99,7 @@ public static class TelemetryExtensions } } - private static void ConfigureSerilog(LoggerConfiguration configuration, FeedserOptions.TelemetryOptions telemetry, string environmentName, string applicationName) + private static void ConfigureSerilog(LoggerConfiguration configuration, ConcelierOptions.TelemetryOptions telemetry, string environmentName, string applicationName) { if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogEventLevel level)) { @@ -117,7 +117,7 @@ public static class TelemetryExtensions .WriteTo.Console(outputTemplate: "[{Timestamp:O}] [{Level:u3}] {Message:lj} {Properties}{NewLine}{Exception}"); } - private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing) + private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, TracerProviderBuilder tracing) { if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) { @@ -145,7 +145,7 @@ public static class TelemetryExtensions } } - private static void ConfigureExporters(FeedserOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics) + private static void ConfigureExporters(ConcelierOptions.TelemetryOptions telemetry, MeterProviderBuilder metrics) { if (string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) { @@ -173,7 +173,7 @@ public static class TelemetryExtensions } } - private static string? BuildHeaders(FeedserOptions.TelemetryOptions telemetry) + private static string? BuildHeaders(ConcelierOptions.TelemetryOptions telemetry) { if (telemetry.OtlpHeaders.Count == 0) { diff --git a/src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs b/src/StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs similarity index 87% rename from src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs rename to src/StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs index 3ef28d2b..9f78e59c 100644 --- a/src/StellaOps.Feedser.WebService/Filters/JobAuthorizationAuditFilter.cs +++ b/src/StellaOps.Concelier.WebService/Filters/JobAuthorizationAuditFilter.cs @@ -1,104 +1,104 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net; -using System.Security.Claims; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Auth.Abstractions; -using StellaOps.Feedser.WebService.Options; - -namespace StellaOps.Feedser.WebService.Filters; - -/// -/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage. -/// -public sealed class JobAuthorizationAuditFilter : IEndpointFilter -{ - internal const string LoggerName = "Feedser.Authorization.Audit"; - - public async ValueTask InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next) - { - ArgumentNullException.ThrowIfNull(context); - ArgumentNullException.ThrowIfNull(next); - - var httpContext = context.HttpContext; - var options = httpContext.RequestServices.GetRequiredService>().Value; - var authority = options.Authority; - - if (authority is null || !authority.Enabled) - { - return await next(context).ConfigureAwait(false); - } - - var logger = httpContext.RequestServices - .GetRequiredService() - .CreateLogger(LoggerName); - - var remoteAddress = httpContext.Connection.RemoteIpAddress; - var matcher = new NetworkMaskMatcher(authority.BypassNetworks); - var user = httpContext.User; - var isAuthenticated = user?.Identity?.IsAuthenticated ?? false; - var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress); - - var result = await next(context).ConfigureAwait(false); - - var scopes = ExtractScopes(user); - var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value; - var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value; - - logger.LogInformation( - "Feedser authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}", - httpContext.Request.Path.Value ?? string.Empty, - httpContext.Response.StatusCode, - string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject, - string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId, - scopes.Length == 0 ? "(none)" : string.Join(',', scopes), - bypassUsed, - remoteAddress?.ToString() ?? IPAddress.None.ToString()); - - return result; - } - - private static string[] ExtractScopes(ClaimsPrincipal? principal) - { - if (principal is null) - { - return Array.Empty(); - } - - var values = new HashSet(StringComparer.Ordinal); - - foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem)) - { - if (string.IsNullOrWhiteSpace(claim.Value)) - { - continue; - } - - values.Add(claim.Value); - } - - foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope)) - { - if (string.IsNullOrWhiteSpace(claim.Value)) - { - continue; - } - - var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); - foreach (var part in parts) - { - var normalized = StellaOpsScopes.Normalize(part); - if (!string.IsNullOrEmpty(normalized)) - { - values.Add(normalized); - } - } - } - - return values.Count == 0 ? Array.Empty() : values.ToArray(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Security.Claims; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Abstractions; +using StellaOps.Concelier.WebService.Options; + +namespace StellaOps.Concelier.WebService.Filters; + +/// +/// Emits structured audit logs for job endpoint authorization decisions, including bypass usage. +/// +public sealed class JobAuthorizationAuditFilter : IEndpointFilter +{ + internal const string LoggerName = "Concelier.Authorization.Audit"; + + public async ValueTask InvokeAsync(EndpointFilterInvocationContext context, EndpointFilterDelegate next) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(next); + + var httpContext = context.HttpContext; + var options = httpContext.RequestServices.GetRequiredService>().Value; + var authority = options.Authority; + + if (authority is null || !authority.Enabled) + { + return await next(context).ConfigureAwait(false); + } + + var logger = httpContext.RequestServices + .GetRequiredService() + .CreateLogger(LoggerName); + + var remoteAddress = httpContext.Connection.RemoteIpAddress; + var matcher = new NetworkMaskMatcher(authority.BypassNetworks); + var user = httpContext.User; + var isAuthenticated = user?.Identity?.IsAuthenticated ?? false; + var bypassUsed = !isAuthenticated && matcher.IsAllowed(remoteAddress); + + var result = await next(context).ConfigureAwait(false); + + var scopes = ExtractScopes(user); + var subject = user?.FindFirst(StellaOpsClaimTypes.Subject)?.Value; + var clientId = user?.FindFirst(StellaOpsClaimTypes.ClientId)?.Value; + + logger.LogInformation( + "Concelier authorization audit route={Route} status={StatusCode} subject={Subject} clientId={ClientId} scopes={Scopes} bypass={Bypass} remote={RemoteAddress}", + httpContext.Request.Path.Value ?? string.Empty, + httpContext.Response.StatusCode, + string.IsNullOrWhiteSpace(subject) ? "(anonymous)" : subject, + string.IsNullOrWhiteSpace(clientId) ? "(none)" : clientId, + scopes.Length == 0 ? "(none)" : string.Join(',', scopes), + bypassUsed, + remoteAddress?.ToString() ?? IPAddress.None.ToString()); + + return result; + } + + private static string[] ExtractScopes(ClaimsPrincipal? principal) + { + if (principal is null) + { + return Array.Empty(); + } + + var values = new HashSet(StringComparer.Ordinal); + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.ScopeItem)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + values.Add(claim.Value); + } + + foreach (var claim in principal.FindAll(StellaOpsClaimTypes.Scope)) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + var parts = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + foreach (var part in parts) + { + var normalized = StellaOpsScopes.Normalize(part); + if (!string.IsNullOrEmpty(normalized)) + { + values.Add(normalized); + } + } + } + + return values.Count == 0 ? Array.Empty() : values.ToArray(); + } +} diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs b/src/StellaOps.Concelier.WebService/Jobs/JobDefinitionResponse.cs similarity index 84% rename from src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs rename to src/StellaOps.Concelier.WebService/Jobs/JobDefinitionResponse.cs index 55101484..aece0cee 100644 --- a/src/StellaOps.Feedser.WebService/Jobs/JobDefinitionResponse.cs +++ b/src/StellaOps.Concelier.WebService/Jobs/JobDefinitionResponse.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.WebService.Jobs; +namespace StellaOps.Concelier.WebService.Jobs; public sealed record JobDefinitionResponse( string Kind, diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs b/src/StellaOps.Concelier.WebService/Jobs/JobRunResponse.cs similarity index 86% rename from src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs rename to src/StellaOps.Concelier.WebService/Jobs/JobRunResponse.cs index 9cb60ff0..bb83ff9e 100644 --- a/src/StellaOps.Feedser.WebService/Jobs/JobRunResponse.cs +++ b/src/StellaOps.Concelier.WebService/Jobs/JobRunResponse.cs @@ -1,6 +1,6 @@ -using StellaOps.Feedser.Core.Jobs; +using StellaOps.Concelier.Core.Jobs; -namespace StellaOps.Feedser.WebService.Jobs; +namespace StellaOps.Concelier.WebService.Jobs; public sealed record JobRunResponse( Guid RunId, diff --git a/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs b/src/StellaOps.Concelier.WebService/Jobs/JobTriggerRequest.cs similarity index 77% rename from src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs rename to src/StellaOps.Concelier.WebService/Jobs/JobTriggerRequest.cs index 18c1e443..75c096ff 100644 --- a/src/StellaOps.Feedser.WebService/Jobs/JobTriggerRequest.cs +++ b/src/StellaOps.Concelier.WebService/Jobs/JobTriggerRequest.cs @@ -1,4 +1,4 @@ -namespace StellaOps.Feedser.WebService.Jobs; +namespace StellaOps.Concelier.WebService.Jobs; public sealed class JobTriggerRequest { diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs b/src/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs similarity index 70% rename from src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs rename to src/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs index e16a7567..3caec155 100644 --- a/src/StellaOps.Feedser.WebService/Options/FeedserOptions.cs +++ b/src/StellaOps.Concelier.WebService/Options/ConcelierOptions.cs @@ -1,9 +1,10 @@ using System; using System.Collections.Generic; +using System.Text.Json.Serialization; -namespace StellaOps.Feedser.WebService.Options; +namespace StellaOps.Concelier.WebService.Options; -public sealed class FeedserOptions +public sealed class ConcelierOptions { public StorageOptions Storage { get; set; } = new(); @@ -12,6 +13,8 @@ public sealed class FeedserOptions public TelemetryOptions Telemetry { get; set; } = new(); public AuthorityOptions Authority { get; set; } = new(); + + public MirrorOptions Mirror { get; set; } = new(); public sealed class StorageOptions { @@ -99,4 +102,37 @@ public sealed class FeedserOptions public TimeSpan? OfflineCacheTolerance { get; set; } } } + + public sealed class MirrorOptions + { + public bool Enabled { get; set; } + + public string ExportRoot { get; set; } = System.IO.Path.Combine("exports", "json"); + + public string? ActiveExportId { get; set; } + + public string LatestDirectoryName { get; set; } = "latest"; + + public string MirrorDirectoryName { get; set; } = "mirror"; + + public bool RequireAuthentication { get; set; } + + public int MaxIndexRequestsPerHour { get; set; } = 600; + + public IList Domains { get; } = new List(); + + [JsonIgnore] + public string ExportRootAbsolute { get; internal set; } = string.Empty; + } + + public sealed class MirrorDomainOptions + { + public string Id { get; set; } = string.Empty; + + public string? DisplayName { get; set; } + + public bool RequireAuthentication { get; set; } + + public int MaxDownloadRequestsPerHour { get; set; } = 1200; + } } diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs b/src/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs similarity index 53% rename from src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs rename to src/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs index 6de16e6e..1774df31 100644 --- a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsPostConfigure.cs +++ b/src/StellaOps.Concelier.WebService/Options/ConcelierOptionsPostConfigure.cs @@ -1,46 +1,72 @@ -using System; -using System.IO; - -namespace StellaOps.Feedser.WebService.Options; - -/// -/// Post-configuration helpers for . -/// -public static class FeedserOptionsPostConfigure -{ - /// - /// Applies derived settings that require filesystem access, such as loading client secrets from disk. - /// - /// The options to mutate. - /// Application content root used to resolve relative paths. - public static void Apply(FeedserOptions options, string contentRootPath) - { - ArgumentNullException.ThrowIfNull(options); - - options.Authority ??= new FeedserOptions.AuthorityOptions(); - - var authority = options.Authority; - if (string.IsNullOrWhiteSpace(authority.ClientSecret) - && !string.IsNullOrWhiteSpace(authority.ClientSecretFile)) - { - var resolvedPath = authority.ClientSecretFile!; - if (!Path.IsPathRooted(resolvedPath)) - { - resolvedPath = Path.Combine(contentRootPath, resolvedPath); - } - - if (!File.Exists(resolvedPath)) - { - throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found."); - } - - var secret = File.ReadAllText(resolvedPath).Trim(); - if (string.IsNullOrEmpty(secret)) - { - throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty."); - } - - authority.ClientSecret = secret; - } - } -} +using System; +using System.IO; + +namespace StellaOps.Concelier.WebService.Options; + +/// +/// Post-configuration helpers for . +/// +public static class ConcelierOptionsPostConfigure +{ + /// + /// Applies derived settings that require filesystem access, such as loading client secrets from disk. + /// + /// The options to mutate. + /// Application content root used to resolve relative paths. + public static void Apply(ConcelierOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + + options.Authority ??= new ConcelierOptions.AuthorityOptions(); + + var authority = options.Authority; + if (string.IsNullOrWhiteSpace(authority.ClientSecret) + && !string.IsNullOrWhiteSpace(authority.ClientSecretFile)) + { + var resolvedPath = authority.ClientSecretFile!; + if (!Path.IsPathRooted(resolvedPath)) + { + resolvedPath = Path.Combine(contentRootPath, resolvedPath); + } + + if (!File.Exists(resolvedPath)) + { + throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' was not found."); + } + + var secret = File.ReadAllText(resolvedPath).Trim(); + if (string.IsNullOrEmpty(secret)) + { + throw new InvalidOperationException($"Authority client secret file '{resolvedPath}' is empty."); + } + + authority.ClientSecret = secret; + } + + options.Mirror ??= new ConcelierOptions.MirrorOptions(); + var mirror = options.Mirror; + + if (string.IsNullOrWhiteSpace(mirror.ExportRoot)) + { + mirror.ExportRoot = Path.Combine("exports", "json"); + } + + var resolvedRoot = mirror.ExportRoot; + if (!Path.IsPathRooted(resolvedRoot)) + { + resolvedRoot = Path.Combine(contentRootPath, resolvedRoot); + } + + mirror.ExportRootAbsolute = Path.GetFullPath(resolvedRoot); + + if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName)) + { + mirror.LatestDirectoryName = "latest"; + } + + if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName)) + { + mirror.MirrorDirectoryName = "mirror"; + } + } +} diff --git a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs b/src/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs similarity index 70% rename from src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs rename to src/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs index c66fc520..7491f86f 100644 --- a/src/StellaOps.Feedser.WebService/Options/FeedserOptionsValidator.cs +++ b/src/StellaOps.Concelier.WebService/Options/ConcelierOptionsValidator.cs @@ -3,11 +3,11 @@ using System.Collections.Generic; using Microsoft.Extensions.Logging; using StellaOps.Auth.Abstractions; -namespace StellaOps.Feedser.WebService.Options; +namespace StellaOps.Concelier.WebService.Options; -public static class FeedserOptionsValidator +public static class ConcelierOptionsValidator { - public static void Validate(FeedserOptions options) + public static void Validate(ConcelierOptions options) { ArgumentNullException.ThrowIfNull(options); @@ -26,10 +26,10 @@ public static class FeedserOptionsValidator throw new InvalidOperationException("Command timeout must be greater than zero seconds."); } - options.Telemetry ??= new FeedserOptions.TelemetryOptions(); + options.Telemetry ??= new ConcelierOptions.TelemetryOptions(); - options.Authority ??= new FeedserOptions.AuthorityOptions(); - options.Authority.Resilience ??= new FeedserOptions.AuthorityOptions.ResilienceOptions(); + options.Authority ??= new ConcelierOptions.AuthorityOptions(); + options.Authority.Resilience ??= new ConcelierOptions.AuthorityOptions.ResilienceOptions(); NormalizeList(options.Authority.Audiences, toLower: false); NormalizeList(options.Authority.RequiredScopes, toLower: true); NormalizeList(options.Authority.BypassNetworks, toLower: false); @@ -38,7 +38,7 @@ public static class FeedserOptionsValidator if (options.Authority.RequiredScopes.Count == 0) { - options.Authority.RequiredScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + options.Authority.RequiredScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); } if (options.Authority.ClientScopes.Count == 0) @@ -51,7 +51,7 @@ public static class FeedserOptionsValidator if (options.Authority.ClientScopes.Count == 0) { - options.Authority.ClientScopes.Add(StellaOpsScopes.FeedserJobsTrigger); + options.Authority.ClientScopes.Add(StellaOpsScopes.ConcelierJobsTrigger); } if (options.Authority.BackchannelTimeoutSeconds <= 0) @@ -130,6 +130,9 @@ public static class FeedserOptionsValidator throw new InvalidOperationException("Telemetry OTLP header names must be non-empty."); } } + + options.Mirror ??= new ConcelierOptions.MirrorOptions(); + ValidateMirror(options.Mirror); } private static void NormalizeList(IList values, bool toLower) @@ -166,7 +169,7 @@ public static class FeedserOptionsValidator } } - private static void ValidateResilience(FeedserOptions.AuthorityOptions.ResilienceOptions resilience) + private static void ValidateResilience(ConcelierOptions.AuthorityOptions.ResilienceOptions resilience) { if (resilience.RetryDelays is null) { @@ -186,4 +189,57 @@ public static class FeedserOptionsValidator throw new InvalidOperationException("Authority resilience offlineCacheTolerance must be greater than or equal to zero."); } } + + private static void ValidateMirror(ConcelierOptions.MirrorOptions mirror) + { + if (mirror.MaxIndexRequestsPerHour < 0) + { + throw new InvalidOperationException("Mirror maxIndexRequestsPerHour must be greater than or equal to zero."); + } + + if (string.IsNullOrWhiteSpace(mirror.ExportRoot)) + { + throw new InvalidOperationException("Mirror exportRoot must be configured."); + } + + if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute)) + { + throw new InvalidOperationException("Mirror export root could not be resolved."); + } + + if (string.IsNullOrWhiteSpace(mirror.LatestDirectoryName)) + { + throw new InvalidOperationException("Mirror latestDirectoryName must be provided."); + } + + if (string.IsNullOrWhiteSpace(mirror.MirrorDirectoryName)) + { + throw new InvalidOperationException("Mirror mirrorDirectoryName must be provided."); + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var domain in mirror.Domains) + { + if (string.IsNullOrWhiteSpace(domain.Id)) + { + throw new InvalidOperationException("Mirror domain id must be provided."); + } + + var normalized = domain.Id.Trim(); + if (!seen.Add(normalized)) + { + throw new InvalidOperationException($"Mirror domain id '{normalized}' is duplicated."); + } + + if (domain.MaxDownloadRequestsPerHour < 0) + { + throw new InvalidOperationException($"Mirror domain '{normalized}' maxDownloadRequestsPerHour must be greater than or equal to zero."); + } + } + + if (mirror.Enabled && mirror.Domains.Count == 0) + { + throw new InvalidOperationException("Mirror distribution requires at least one domain when enabled."); + } + } } diff --git a/src/StellaOps.Feedser.WebService/Program.cs b/src/StellaOps.Concelier.WebService/Program.cs similarity index 79% rename from src/StellaOps.Feedser.WebService/Program.cs rename to src/StellaOps.Concelier.WebService/Program.cs index da49b26d..790dc92f 100644 --- a/src/StellaOps.Feedser.WebService/Program.cs +++ b/src/StellaOps.Concelier.WebService/Program.cs @@ -4,8 +4,9 @@ using System.Text; using Microsoft.AspNetCore.Diagnostics; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; using System.Diagnostics; using System.Text.Json; using System.Text.Json.Serialization; @@ -13,16 +14,18 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.WebService.Diagnostics; +using StellaOps.Concelier.Core.Events; +using StellaOps.Concelier.Core.Jobs; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.WebService.Diagnostics; using Serilog; -using StellaOps.Feedser.Merge; -using StellaOps.Feedser.Merge.Services; -using StellaOps.Feedser.WebService.Extensions; -using StellaOps.Feedser.WebService.Jobs; -using StellaOps.Feedser.WebService.Options; -using StellaOps.Feedser.WebService.Filters; +using StellaOps.Concelier.Merge; +using StellaOps.Concelier.Merge.Services; +using StellaOps.Concelier.WebService.Extensions; +using StellaOps.Concelier.WebService.Jobs; +using StellaOps.Concelier.WebService.Options; +using StellaOps.Concelier.WebService.Filters; +using StellaOps.Concelier.WebService.Services; using Serilog.Events; using StellaOps.Plugin.DependencyInjection; using StellaOps.Plugin.Hosting; @@ -33,67 +36,72 @@ using StellaOps.Auth.ServerIntegration; var builder = WebApplication.CreateBuilder(args); -const string JobsPolicyName = "Feedser.Jobs.Trigger"; +const string JobsPolicyName = "Concelier.Jobs.Trigger"; builder.Configuration.AddStellaOpsDefaults(options => { options.BasePath = builder.Environment.ContentRootPath; - options.EnvironmentPrefix = "FEEDSER_"; + options.EnvironmentPrefix = "CONCELIER_"; options.ConfigureBuilder = configurationBuilder => { - configurationBuilder.AddFeedserYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/feedser.yaml")); + configurationBuilder.AddConcelierYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/concelier.yaml")); }; }); var contentRootPath = builder.Environment.ContentRootPath; -var feedserOptions = builder.Configuration.BindOptions(postConfigure: (opts, _) => +var concelierOptions = builder.Configuration.BindOptions(postConfigure: (opts, _) => { - FeedserOptionsPostConfigure.Apply(opts, contentRootPath); - FeedserOptionsValidator.Validate(opts); + ConcelierOptionsPostConfigure.Apply(opts, contentRootPath); + ConcelierOptionsValidator.Validate(opts); }); -builder.Services.AddOptions() +builder.Services.AddOptions() .Bind(builder.Configuration) .PostConfigure(options => { - FeedserOptionsPostConfigure.Apply(options, contentRootPath); - FeedserOptionsValidator.Validate(options); + ConcelierOptionsPostConfigure.Apply(options, contentRootPath); + ConcelierOptionsValidator.Validate(options); }) .ValidateOnStart(); -builder.ConfigureFeedserTelemetry(feedserOptions); - -builder.Services.AddMongoStorage(storageOptions => -{ - storageOptions.ConnectionString = feedserOptions.Storage.Dsn; - storageOptions.DatabaseName = feedserOptions.Storage.Database; - storageOptions.CommandTimeout = TimeSpan.FromSeconds(feedserOptions.Storage.CommandTimeoutSeconds); +builder.ConfigureConcelierTelemetry(concelierOptions); + +builder.Services.TryAddSingleton(_ => TimeProvider.System); +builder.Services.AddMemoryCache(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +builder.Services.AddMongoStorage(storageOptions => +{ + storageOptions.ConnectionString = concelierOptions.Storage.Dsn; + storageOptions.DatabaseName = concelierOptions.Storage.Database; + storageOptions.CommandTimeout = TimeSpan.FromSeconds(concelierOptions.Storage.CommandTimeoutSeconds); }); builder.Services.AddMergeModule(builder.Configuration); builder.Services.AddJobScheduler(); -builder.Services.AddBuiltInFeedserJobs(); +builder.Services.AddBuiltInConcelierJobs(); builder.Services.AddSingleton(sp => new ServiceStatus(sp.GetRequiredService())); -var authorityConfigured = feedserOptions.Authority is { Enabled: true }; +var authorityConfigured = concelierOptions.Authority is { Enabled: true }; if (authorityConfigured) { builder.Services.AddStellaOpsAuthClient(clientOptions => { - clientOptions.Authority = feedserOptions.Authority.Issuer; - clientOptions.ClientId = feedserOptions.Authority.ClientId ?? string.Empty; - clientOptions.ClientSecret = feedserOptions.Authority.ClientSecret; - clientOptions.HttpTimeout = TimeSpan.FromSeconds(feedserOptions.Authority.BackchannelTimeoutSeconds); + clientOptions.Authority = concelierOptions.Authority.Issuer; + clientOptions.ClientId = concelierOptions.Authority.ClientId ?? string.Empty; + clientOptions.ClientSecret = concelierOptions.Authority.ClientSecret; + clientOptions.HttpTimeout = TimeSpan.FromSeconds(concelierOptions.Authority.BackchannelTimeoutSeconds); clientOptions.DefaultScopes.Clear(); - foreach (var scope in feedserOptions.Authority.ClientScopes) + foreach (var scope in concelierOptions.Authority.ClientScopes) { clientOptions.DefaultScopes.Add(scope); } - var resilience = feedserOptions.Authority.Resilience ?? new FeedserOptions.AuthorityOptions.ResilienceOptions(); + var resilience = concelierOptions.Authority.Resilience ?? new ConcelierOptions.AuthorityOptions.ResilienceOptions(); if (resilience.EnableRetries.HasValue) { clientOptions.EnableRetries = resilience.EnableRetries.Value; @@ -124,27 +132,27 @@ if (authorityConfigured) configurationSection: null, configure: resourceOptions => { - resourceOptions.Authority = feedserOptions.Authority.Issuer; - resourceOptions.RequireHttpsMetadata = feedserOptions.Authority.RequireHttpsMetadata; - resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(feedserOptions.Authority.BackchannelTimeoutSeconds); - resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(feedserOptions.Authority.TokenClockSkewSeconds); + resourceOptions.Authority = concelierOptions.Authority.Issuer; + resourceOptions.RequireHttpsMetadata = concelierOptions.Authority.RequireHttpsMetadata; + resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(concelierOptions.Authority.BackchannelTimeoutSeconds); + resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(concelierOptions.Authority.TokenClockSkewSeconds); - if (!string.IsNullOrWhiteSpace(feedserOptions.Authority.MetadataAddress)) + if (!string.IsNullOrWhiteSpace(concelierOptions.Authority.MetadataAddress)) { - resourceOptions.MetadataAddress = feedserOptions.Authority.MetadataAddress; + resourceOptions.MetadataAddress = concelierOptions.Authority.MetadataAddress; } - foreach (var audience in feedserOptions.Authority.Audiences) + foreach (var audience in concelierOptions.Authority.Audiences) { resourceOptions.Audiences.Add(audience); } - foreach (var scope in feedserOptions.Authority.RequiredScopes) + foreach (var scope in concelierOptions.Authority.RequiredScopes) { resourceOptions.RequiredScopes.Add(scope); } - foreach (var network in feedserOptions.Authority.BypassNetworks) + foreach (var network in concelierOptions.Authority.BypassNetworks) { resourceOptions.BypassNetworks.Add(network); } @@ -152,19 +160,19 @@ if (authorityConfigured) builder.Services.AddAuthorization(options => { - options.AddStellaOpsScopePolicy(JobsPolicyName, feedserOptions.Authority.RequiredScopes.ToArray()); + options.AddStellaOpsScopePolicy(JobsPolicyName, concelierOptions.Authority.RequiredScopes.ToArray()); }); } -var pluginHostOptions = BuildPluginOptions(feedserOptions, builder.Environment.ContentRootPath); +var pluginHostOptions = BuildPluginOptions(concelierOptions, builder.Environment.ContentRootPath); builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); builder.Services.AddEndpointsApiExplorer(); var app = builder.Build(); -var resolvedFeedserOptions = app.Services.GetRequiredService>().Value; -var resolvedAuthority = resolvedFeedserOptions.Authority ?? new FeedserOptions.AuthorityOptions(); +var resolvedConcelierOptions = app.Services.GetRequiredService>().Value; +var resolvedAuthority = resolvedConcelierOptions.Authority ?? new ConcelierOptions.AuthorityOptions(); authorityConfigured = resolvedAuthority.Enabled; var enforceAuthority = resolvedAuthority.Enabled && !resolvedAuthority.AllowAnonymousFallback; @@ -174,10 +182,59 @@ if (resolvedAuthority.Enabled && resolvedAuthority.AllowAnonymousFallback) "Authority authentication is configured but anonymous fallback remains enabled. Set authority.allowAnonymousFallback to false before 2025-12-31 to complete the rollout."); } +app.MapConcelierMirrorEndpoints(authorityConfigured, enforceAuthority); + var jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web); jsonOptions.Converters.Add(new JsonStringEnumConverter()); -var loggingEnabled = feedserOptions.Telemetry?.EnableLogging ?? true; +app.MapGet("/concelier/advisories/{vulnerabilityKey}/replay", async ( + string vulnerabilityKey, + DateTimeOffset? asOf, + IAdvisoryEventLog eventLog, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(vulnerabilityKey)) + { + return Results.BadRequest("vulnerabilityKey must be provided."); + } + + var replay = await eventLog.ReplayAsync(vulnerabilityKey.Trim(), asOf, cancellationToken).ConfigureAwait(false); + if (replay.Statements.Length == 0 && replay.Conflicts.Length == 0) + { + return Results.NotFound(); + } + + var response = new + { + replay.VulnerabilityKey, + replay.AsOf, + Statements = replay.Statements.Select(statement => new + { + statement.StatementId, + statement.VulnerabilityKey, + statement.AdvisoryKey, + statement.Advisory, + StatementHash = Convert.ToHexString(statement.StatementHash.ToArray()), + statement.AsOf, + statement.RecordedAt, + InputDocumentIds = statement.InputDocumentIds + }).ToArray(), + Conflicts = replay.Conflicts.Select(conflict => new + { + conflict.ConflictId, + conflict.VulnerabilityKey, + conflict.StatementIds, + ConflictHash = Convert.ToHexString(conflict.ConflictHash.ToArray()), + conflict.AsOf, + conflict.RecordedAt, + Details = conflict.CanonicalJson + }).ToArray() + }; + + return JsonResult(response); +}); + +var loggingEnabled = concelierOptions.Telemetry?.EnableLogging ?? true; if (loggingEnabled) { @@ -238,7 +295,7 @@ if (authorityConfigured) return; } - var optionsMonitor = context.RequestServices.GetRequiredService>().Value.Authority; + var optionsMonitor = context.RequestServices.GetRequiredService>().Value.Authority; if (optionsMonitor is null || !optionsMonitor.Enabled) { return; @@ -253,7 +310,7 @@ if (authorityConfigured) var bypassAllowed = matcher.IsAllowed(remote); logger.LogWarning( - "Feedser authorization denied route={Route} remote={RemoteAddress} bypassAllowed={BypassAllowed} hasPrincipal={HasPrincipal}", + "Concelier authorization denied route={Route} remote={RemoteAddress} bypassAllowed={BypassAllowed} hasPrincipal={HasPrincipal}", context.Request.Path.Value ?? string.Empty, remote?.ToString() ?? "unknown", bypassAllowed, @@ -326,7 +383,7 @@ void ApplyNoCache(HttpResponse response) await InitializeMongoAsync(app); -app.MapGet("/health", (IOptions opts, ServiceStatus status, HttpContext context) => +app.MapGet("/health", (IOptions opts, ServiceStatus status, HttpContext context) => { ApplyNoCache(context.Response); @@ -656,19 +713,20 @@ if (enforceAuthority) await app.RunAsync(); -static PluginHostOptions BuildPluginOptions(FeedserOptions options, string contentRoot) +static PluginHostOptions BuildPluginOptions(ConcelierOptions options, string contentRoot) { var pluginOptions = new PluginHostOptions { - BaseDirectory = options.Plugins.BaseDirectory ?? contentRoot, - PluginsDirectory = options.Plugins.Directory ?? Path.Combine(contentRoot, "PluginBinaries"), + BaseDirectory = options.Plugins.BaseDirectory ?? contentRoot, + PluginsDirectory = options.Plugins.Directory ?? Path.Combine(contentRoot, "StellaOps.Concelier.PluginBinaries"), + PrimaryPrefix = "StellaOps.Concelier", EnsureDirectoryExists = true, RecursiveSearch = false, }; if (options.Plugins.SearchPatterns.Count == 0) { - pluginOptions.SearchPatterns.Add("StellaOps.Feedser.Plugin.*.dll"); + pluginOptions.SearchPatterns.Add("StellaOps.Concelier.Plugin.*.dll"); } else { diff --git a/src/StellaOps.Feedser.WebService/Properties/launchSettings.json b/src/StellaOps.Concelier.WebService/Properties/launchSettings.json similarity index 83% rename from src/StellaOps.Feedser.WebService/Properties/launchSettings.json rename to src/StellaOps.Concelier.WebService/Properties/launchSettings.json index 261bc6f8..6801613e 100644 --- a/src/StellaOps.Feedser.WebService/Properties/launchSettings.json +++ b/src/StellaOps.Concelier.WebService/Properties/launchSettings.json @@ -1,6 +1,6 @@ { "profiles": { - "StellaOps.Feedser.WebService": { + "StellaOps.Concelier.WebService": { "commandName": "Project", "launchBrowser": true, "environmentVariables": { diff --git a/src/StellaOps.Concelier.WebService/Services/MirrorFileLocator.cs b/src/StellaOps.Concelier.WebService/Services/MirrorFileLocator.cs new file mode 100644 index 00000000..2ca013dd --- /dev/null +++ b/src/StellaOps.Concelier.WebService/Services/MirrorFileLocator.cs @@ -0,0 +1,184 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.IO; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Concelier.WebService.Options; + +namespace StellaOps.Concelier.WebService.Services; + +internal sealed class MirrorFileLocator +{ + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + + public MirrorFileLocator(IOptionsMonitor options, ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public bool TryResolveIndex([NotNullWhen(true)] out string? path, [NotNullWhen(true)] out string? exportId) + => TryResolveRelativePath("index.json", out path, out exportId, out _); + + public bool TryResolveRelativePath(string relativePath, [NotNullWhen(true)] out string? fullPath, [NotNullWhen(true)] out string? exportId, out string? domainId) + { + fullPath = null; + exportId = null; + domainId = null; + + var mirror = _options.CurrentValue.Mirror ?? new ConcelierOptions.MirrorOptions(); + if (!mirror.Enabled) + { + return false; + } + + if (!TryResolveExportDirectory(mirror, out var exportDirectory, out exportId)) + { + return false; + } + + var sanitized = SanitizeRelativePath(relativePath); + if (sanitized.Length == 0 || string.Equals(sanitized, "index.json", StringComparison.OrdinalIgnoreCase)) + { + sanitized = $"{mirror.MirrorDirectoryName}/index.json"; + } + + if (!sanitized.StartsWith($"{mirror.MirrorDirectoryName}/", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var candidate = Combine(exportDirectory, sanitized); + if (!CandidateWithinExport(exportDirectory, candidate)) + { + _logger.LogWarning("Rejected mirror export request for path '{RelativePath}' due to traversal attempt.", relativePath); + return false; + } + + if (!File.Exists(candidate)) + { + return false; + } + + // Extract domain id from path mirror//... + var segments = sanitized.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length >= 2) + { + domainId = segments[1]; + } + + fullPath = candidate; + return true; + } + + private bool TryResolveExportDirectory(ConcelierOptions.MirrorOptions mirror, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId) + { + exportDirectory = null; + exportId = null; + + if (string.IsNullOrWhiteSpace(mirror.ExportRootAbsolute)) + { + _logger.LogWarning("Mirror export root is not configured; unable to serve mirror content."); + return false; + } + + var root = mirror.ExportRootAbsolute; + var candidateSegment = string.IsNullOrWhiteSpace(mirror.ActiveExportId) + ? mirror.LatestDirectoryName + : mirror.ActiveExportId!; + + if (TryResolveCandidate(root, candidateSegment, mirror.MirrorDirectoryName, out exportDirectory, out exportId)) + { + return true; + } + + if (!string.Equals(candidateSegment, mirror.LatestDirectoryName, StringComparison.OrdinalIgnoreCase) + && TryResolveCandidate(root, mirror.LatestDirectoryName, mirror.MirrorDirectoryName, out exportDirectory, out exportId)) + { + return true; + } + + try + { + var directories = Directory.Exists(root) + ? Directory.GetDirectories(root) + : Array.Empty(); + + Array.Sort(directories, StringComparer.Ordinal); + Array.Reverse(directories); + + foreach (var directory in directories) + { + if (TryResolveCandidate(root, Path.GetFileName(directory), mirror.MirrorDirectoryName, out exportDirectory, out exportId)) + { + return true; + } + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to enumerate export directories under {Root}.", root); + } + + return false; + } + + private bool TryResolveCandidate(string root, string segment, string mirrorDirectory, [NotNullWhen(true)] out string? exportDirectory, [NotNullWhen(true)] out string? exportId) + { + exportDirectory = null; + exportId = null; + + if (string.IsNullOrWhiteSpace(segment)) + { + return false; + } + + var candidate = Path.Combine(root, segment); + if (!Directory.Exists(candidate)) + { + return false; + } + + var mirrorPath = Path.Combine(candidate, mirrorDirectory); + if (!Directory.Exists(mirrorPath)) + { + return false; + } + + exportDirectory = candidate; + exportId = segment; + return true; + } + + private static string SanitizeRelativePath(string relativePath) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return string.Empty; + } + + var trimmed = relativePath.Replace('\\', '/').Trim().TrimStart('/'); + return trimmed; + } + + private static string Combine(string root, string relativePath) + { + var segments = relativePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length == 0) + { + return Path.GetFullPath(root); + } + + var combinedRelative = Path.Combine(segments); + return Path.GetFullPath(Path.Combine(root, combinedRelative)); + } + + private static bool CandidateWithinExport(string exportDirectory, string candidate) + { + var exportRoot = Path.GetFullPath(exportDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var candidatePath = Path.GetFullPath(candidate); + return candidatePath.StartsWith(exportRoot, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Concelier.WebService/Services/MirrorRateLimiter.cs b/src/StellaOps.Concelier.WebService/Services/MirrorRateLimiter.cs new file mode 100644 index 00000000..4971e93f --- /dev/null +++ b/src/StellaOps.Concelier.WebService/Services/MirrorRateLimiter.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Caching.Memory; + +namespace StellaOps.Concelier.WebService.Services; + +internal sealed class MirrorRateLimiter +{ + private readonly IMemoryCache _cache; + private readonly TimeProvider _timeProvider; + private static readonly TimeSpan Window = TimeSpan.FromHours(1); + + public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider) + { + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter) + { + retryAfter = null; + + if (limit <= 0 || limit == int.MaxValue) + { + return true; + } + + var key = CreateKey(domainId, scope); + var now = _timeProvider.GetUtcNow(); + + var counter = _cache.Get(key); + if (counter is null || now - counter.WindowStart >= Window) + { + counter = new Counter(now, 0); + } + + if (counter.Count >= limit) + { + var windowEnd = counter.WindowStart + Window; + retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero; + return false; + } + + counter = counter with { Count = counter.Count + 1 }; + var absoluteExpiration = counter.WindowStart + Window; + _cache.Set(key, counter, absoluteExpiration); + return true; + } + + private static string CreateKey(string domainId, string scope) + => string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) => + { + state.domainId.AsSpan().CopyTo(span); + span[state.domainId.Length] = '|'; + state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]); + }); + + private sealed record Counter(DateTimeOffset WindowStart, int Count); +} diff --git a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj b/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj similarity index 75% rename from src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj rename to src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj index d63572e7..7fedfa8b 100644 --- a/src/StellaOps.Feedser.WebService/StellaOps.Feedser.WebService.csproj +++ b/src/StellaOps.Concelier.WebService/StellaOps.Concelier.WebService.csproj @@ -5,7 +5,7 @@ enable enable true - StellaOps.Feedser.WebService + StellaOps.Concelier.WebService @@ -21,11 +21,11 @@ - - - - - + + + + + diff --git a/src/StellaOps.Concelier.WebService/TASKS.md b/src/StellaOps.Concelier.WebService/TASKS.md new file mode 100644 index 00000000..95fbbf11 --- /dev/null +++ b/src/StellaOps.Concelier.WebService/TASKS.md @@ -0,0 +1,27 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|FEEDWEB-EVENTS-07-001 Advisory event replay API|Concelier WebService Guild|FEEDCORE-ENGINE-07-001|**DONE (2025-10-19)** – Added `/concelier/advisories/{vulnerabilityKey}/replay` endpoint with optional `asOf`, hex hashes, and conflict payloads; integration covered via `dotnet test src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj`.| +|Bind & validate ConcelierOptions|BE-Base|WebService|DONE – options bound/validated with failure logging.| +|Mongo service wiring|BE-Base|Storage.Mongo|DONE – wiring delegated to `AddMongoStorage`.| +|Bootstrapper execution on start|BE-Base|Storage.Mongo|DONE – startup calls `MongoBootstrapper.InitializeAsync`.| +|Plugin host options finalization|BE-Base|Plugins|DONE – default plugin directories/search patterns configured.| +|Jobs API contract tests|QA|Core|DONE – WebServiceEndpointsTests now cover success payloads, filtering, and trigger outcome mapping.| +|Health/Ready probes|DevOps|Ops|DONE – `/health` and `/ready` endpoints implemented.| +|Serilog + OTEL integration hooks|BE-Base|Observability|DONE – `TelemetryExtensions` wires Serilog + OTEL with configurable exporters.| +|Register built-in jobs (sources/exporters)|BE-Base|Core|DONE – AddBuiltInConcelierJobs adds fallback scheduler definitions for core connectors and exporters via reflection.| +|HTTP problem details consistency|BE-Base|WebService|DONE – API errors now emit RFC7807 responses with trace identifiers and typed problem categories.| +|Request logging and metrics|BE-Base|Observability|DONE – Serilog request logging enabled with enriched context and web.jobs counters published via OpenTelemetry.| +|Endpoint smoke tests (health/ready/jobs error paths)|QA|WebService|DONE – WebServiceEndpointsTests assert success and problem responses for health, ready, and job trigger error paths.| +|Batch job definition last-run lookup|BE-Base|Core|DONE – definitions endpoint now precomputes kinds array and reuses batched last-run dictionary; manual smoke verified via local GET `/jobs/definitions`.| +|Add no-cache headers to health/readiness/jobs APIs|BE-Base|WebService|DONE – helper applies Cache-Control/Pragma/Expires on all health/ready/jobs endpoints; awaiting automated probe tests once connector fixtures stabilize.| +|Authority configuration parity (FSR1)|DevEx/Concelier|Authority options schema|**DONE (2025-10-10)** – Options post-config loads clientSecretFile fallback, validators normalize scopes/audiences, and sample config documents issuer/credential/bypass settings.| +|Document authority toggle & scope requirements|Docs/Concelier|Authority integration|**DOING (2025-10-10)** – Quickstart updated with staging flag, client credentials, env overrides; operator guide refresh pending Docs guild review.| +|Plumb Authority client resilience options|BE-Base|Auth libraries LIB5|**DONE (2025-10-12)** – `Program.cs` wires `authority.resilience.*` + client scopes into `AddStellaOpsAuthClient`; new integration test asserts binding and retries.| +|Author ops guidance for resilience tuning|Docs/Concelier|Plumb Authority client resilience options|**DONE (2025-10-12)** – `docs/21_INSTALL_GUIDE.md` + `docs/ops/concelier-authority-audit-runbook.md` document resilience profiles for connected vs air-gapped installs and reference monitoring cues.| +|Document authority bypass logging patterns|Docs/Concelier|FSR3 logging|**DONE (2025-10-12)** – Updated operator guides clarify `Concelier.Authorization.Audit` fields (route/status/subject/clientId/scopes/bypass/remote) and SIEM triggers.| +|Update Concelier operator guide for enforcement cutoff|Docs/Concelier|FSR1 rollout|**DONE (2025-10-12)** – Installation guide emphasises disabling `allowAnonymousFallback` before 2025-12-31 UTC and connects audit signals to the rollout checklist.| +|Rename plugin drop directory to namespaced path|BE-Base|Plugins|**DONE (2025-10-19)** – Build outputs now target `StellaOps.Concelier.PluginBinaries`/`StellaOps.Authority.PluginBinaries`, plugin host defaults updated, config/docs refreshed, and `dotnet test src/StellaOps.Concelier.WebService.Tests/StellaOps.Concelier.WebService.Tests.csproj --no-restore` covers the change.| +|Authority resilience adoption|Concelier WebService, Docs|Plumb Authority client resilience options|**BLOCKED (2025-10-10)** – Roll out retry/offline knobs to deployment docs and confirm CLI parity once LIB5 lands; unblock after resilience options wired and tested.| +|CONCELIER-WEB-08-201 – Mirror distribution endpoints|Concelier WebService Guild|CONCELIER-EXPORT-08-201, DEVOPS-MIRROR-08-001|DOING (2025-10-19) – HTTP endpoints wired (`/concelier/exports/index.json`, `/concelier/exports/mirror/*`), mirror options bound/validated, and integration tests added; pending auth docs + smoke in ops handbook.| +|Wave 0B readiness checkpoint|Team WebService & Authority|Wave 0A completion|BLOCKED (2025-10-19) – FEEDSTORAGE-MONGO-08-001 closed, but remaining Wave 0A items (AUTH-DPOP-11-001, AUTH-MTLS-11-002, PLUGIN-DI-08-001) still open; maintain current DOING workstreams only.| diff --git a/src/StellaOps.Feedser.sln b/src/StellaOps.Concelier.sln similarity index 81% rename from src/StellaOps.Feedser.sln rename to src/StellaOps.Concelier.sln index e60b3b4e..e6ea515e 100644 --- a/src/StellaOps.Feedser.sln +++ b/src/StellaOps.Concelier.sln @@ -1,944 +1,944 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.0.31903.59 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Acsc", "StellaOps.Feedser.Source.Acsc\StellaOps.Feedser.Source.Acsc.csproj", "{CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{E9DE840D-0760-4324-98E2-7F2CBE06DC1A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{061B0042-9A6C-4CFD-9E48-4D3F3B924442}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Cisa", "StellaOps.Feedser.Source.Ics.Cisa\StellaOps.Feedser.Source.Ics.Cisa.csproj", "{6A301F32-2EEE-491B-9DB9-3BF26D032F07}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{AFCCC916-58E8-4676-AABB-54B04CEA3392}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{BF3DAB2F-E46E-49C1-9BA5-AA389763A632}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{429BAA6A-706D-489A-846F-4B0EF1B15121}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge", "StellaOps.Feedser.Merge\StellaOps.Feedser.Merge.csproj", "{085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json", "StellaOps.Feedser.Exporter.Json\StellaOps.Feedser.Exporter.Json.csproj", "{1C5506B8-C01B-4419-B888-A48F441E0C69}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb", "StellaOps.Feedser.Exporter.TrivyDb\StellaOps.Feedser.Exporter.TrivyDb.csproj", "{4D936BC4-5520-4642-A237-4106E97BC7A0}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{B85C1C0E-B245-44FB-877E-C112DE29041A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService", "StellaOps.Feedser.WebService\StellaOps.Feedser.WebService.csproj", "{2C970A0F-FE3D-425B-B1B3-A008B194F5C2}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cccs", "StellaOps.Feedser.Source.Cccs\StellaOps.Feedser.Source.Cccs.csproj", "{A7035381-6D20-4A07-817B-A324ED735EB3}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian", "StellaOps.Feedser.Source.Distro.Debian\StellaOps.Feedser.Source.Distro.Debian.csproj", "{404F5F6E-37E4-4EF9-B09D-6634366B5D44}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu", "StellaOps.Feedser.Source.Distro.Ubuntu\StellaOps.Feedser.Source.Distro.Ubuntu.csproj", "{1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kisa", "StellaOps.Feedser.Source.Kisa\StellaOps.Feedser.Source.Kisa.csproj", "{23055A20-7079-4336-AD30-EFAA2FA11665}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertCc", "StellaOps.Feedser.Source.CertCc\StellaOps.Feedser.Source.CertCc.csproj", "{C2304954-9B15-4776-8DB6-22E293D311E4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr", "StellaOps.Feedser.Source.CertFr\StellaOps.Feedser.Source.CertFr.csproj", "{E6895821-ED23-46D2-A5DC-06D61F90EC27}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd", "StellaOps.Feedser.Source.Nvd\StellaOps.Feedser.Source.Nvd.csproj", "{378CB675-D70B-4A95-B324-62B67D79AAB7}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle", "StellaOps.Feedser.Source.Vndr.Oracle\StellaOps.Feedser.Source.Vndr.Oracle.csproj", "{53AD2E55-B0F5-46AD-BFE5-82F486371872}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki", "StellaOps.Feedser.Source.Ru.Nkcki\StellaOps.Feedser.Source.Ru.Nkcki.csproj", "{B880C99C-C0BD-4953-95AD-2C76BC43F760}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse", "StellaOps.Feedser.Source.Distro.Suse\StellaOps.Feedser.Source.Distro.Suse.csproj", "{23422F67-C1FB-4FF4-899C-706BCD63D9FD}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu", "StellaOps.Feedser.Source.Ru.Bdu\StellaOps.Feedser.Source.Ru.Bdu.csproj", "{16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev", "StellaOps.Feedser.Source.Kev\StellaOps.Feedser.Source.Kev.csproj", "{20DB9837-715B-4515-98C6-14B50060B765}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky", "StellaOps.Feedser.Source.Ics.Kaspersky\StellaOps.Feedser.Source.Ics.Kaspersky.csproj", "{10849EE2-9F34-4C23-BBB4-916A59CDB7F4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv", "StellaOps.Feedser.Source.Osv\StellaOps.Feedser.Source.Osv.csproj", "{EFB16EDB-78D4-4601-852E-F4B37655FA13}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn", "StellaOps.Feedser.Source.Jvn\StellaOps.Feedser.Source.Jvn.csproj", "{02289F61-0173-42CC-B8F2-25CC53F8E066}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertBund", "StellaOps.Feedser.Source.CertBund\StellaOps.Feedser.Source.CertBund.csproj", "{4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve", "StellaOps.Feedser.Source.Cve\StellaOps.Feedser.Source.Cve.csproj", "{EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco", "StellaOps.Feedser.Source.Vndr.Cisco\StellaOps.Feedser.Source.Vndr.Cisco.csproj", "{19957518-A422-4622-9FD1-621DF3E31869}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Msrc", "StellaOps.Feedser.Source.Vndr.Msrc\StellaOps.Feedser.Source.Vndr.Msrc.csproj", "{69C4C061-F5A0-4EAA-A4CD-9A513523952A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium", "StellaOps.Feedser.Source.Vndr.Chromium\StellaOps.Feedser.Source.Vndr.Chromium.csproj", "{C7F7DE6F-A369-4F43-9864-286DCEC615F8}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Apple", "StellaOps.Feedser.Source.Vndr.Apple\StellaOps.Feedser.Source.Vndr.Apple.csproj", "{1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware", "StellaOps.Feedser.Source.Vndr.Vmware\StellaOps.Feedser.Source.Vndr.Vmware.csproj", "{7255C38D-5A16-4A4D-98CE-CF0FD516B68E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe", "StellaOps.Feedser.Source.Vndr.Adobe\StellaOps.Feedser.Source.Vndr.Adobe.csproj", "{C3A42AA3-800D-4398-A077-5560EE6451EF}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn", "StellaOps.Feedser.Source.CertIn\StellaOps.Feedser.Source.CertIn.csproj", "{5016963A-6FC9-4063-AB83-2D1F9A2BC627}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa", "StellaOps.Feedser.Source.Ghsa\StellaOps.Feedser.Source.Ghsa.csproj", "{72F43F43-F852-487F-8334-91D438CE2F7C}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat", "StellaOps.Feedser.Source.Distro.RedHat\StellaOps.Feedser.Source.Distro.RedHat.csproj", "{A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{F622D38D-DA49-473E-B724-E706F8113CF2}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core.Tests", "StellaOps.Feedser.Core.Tests\StellaOps.Feedser.Core.Tests.csproj", "{3A3D7610-C864-4413-B07E-9E8C2A49A90E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge.Tests", "StellaOps.Feedser.Merge.Tests\StellaOps.Feedser.Merge.Tests.csproj", "{9C4DEE96-CD7D-4AE3-A811-0B48B477003B}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models.Tests", "StellaOps.Feedser.Models.Tests\StellaOps.Feedser.Models.Tests.csproj", "{437B2667-9461-47D2-B75B-4D2E03D69B94}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization.Tests", "StellaOps.Feedser.Normalization.Tests\StellaOps.Feedser.Normalization.Tests.csproj", "{8249DF28-CDAF-4DEF-A912-C27F57B67FD5}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo.Tests", "StellaOps.Feedser.Storage.Mongo.Tests\StellaOps.Feedser.Storage.Mongo.Tests.csproj", "{CBFB015B-C069-475F-A476-D52222729804}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json.Tests", "StellaOps.Feedser.Exporter.Json.Tests\StellaOps.Feedser.Exporter.Json.Tests.csproj", "{2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb.Tests", "StellaOps.Feedser.Exporter.TrivyDb.Tests\StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj", "{3EB22234-642E-4533-BCC3-93E8ED443B1D}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService.Tests", "StellaOps.Feedser.WebService.Tests\StellaOps.Feedser.WebService.Tests.csproj", "{84A5DE81-4444-499A-93BF-6DC4CA72F8D4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common.Tests", "StellaOps.Feedser.Source.Common.Tests\StellaOps.Feedser.Source.Common.Tests.csproj", "{42E21E1D-C3DE-4765-93E9-39391BB5C802}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd.Tests", "StellaOps.Feedser.Source.Nvd.Tests\StellaOps.Feedser.Source.Nvd.Tests.csproj", "{B6E2EE26-B297-4AB9-A47E-A227F5EAE108}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat.Tests", "StellaOps.Feedser.Source.Distro.RedHat.Tests\StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj", "{CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium.Tests", "StellaOps.Feedser.Source.Vndr.Chromium.Tests\StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj", "{2891FCDE-BB89-46F0-A40C-368EF804DB44}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe.Tests", "StellaOps.Feedser.Source.Vndr.Adobe.Tests\StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj", "{B91C60FB-926F-47C3-BFD0-6DD145308344}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle.Tests", "StellaOps.Feedser.Source.Vndr.Oracle.Tests\StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj", "{30DF89D1-D66D-4078-8A3B-951637A42265}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware.Tests", "StellaOps.Feedser.Source.Vndr.Vmware.Tests\StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj", "{6E98C770-72FF-41FA-8C42-30AABAAF5B4E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn.Tests", "StellaOps.Feedser.Source.CertIn.Tests\StellaOps.Feedser.Source.CertIn.Tests.csproj", "{79B36C92-BA93-4406-AB75-6F2282DDFF01}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr.Tests", "StellaOps.Feedser.Source.CertFr.Tests\StellaOps.Feedser.Source.CertFr.Tests.csproj", "{4B60FA53-81F6-4AB6-BE9F-DE0992E11977}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky.Tests", "StellaOps.Feedser.Source.Ics.Kaspersky.Tests\StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj", "{6BBA820B-8443-4832-91C3-3AB002006494}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn.Tests", "StellaOps.Feedser.Source.Jvn.Tests\StellaOps.Feedser.Source.Jvn.Tests.csproj", "{7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv.Tests", "StellaOps.Feedser.Source.Osv.Tests\StellaOps.Feedser.Source.Osv.Tests.csproj", "{F892BFFD-9101-4D59-B6FD-C532EB04D51F}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{EAE910FC-188C-41C3-822A-623964CABE48}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian.Tests", "StellaOps.Feedser.Source.Distro.Debian.Tests\StellaOps.Feedser.Source.Distro.Debian.Tests.csproj", "{BBA5C780-6348-427D-9600-726EAA8963B3}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "StellaOps.Configuration\StellaOps.Configuration.csproj", "{5F44A429-816A-4560-A5AA-61CD23FD8A19}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{544DBB82-4639-4856-A5F2-76828F7A8396}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu.Tests", "StellaOps.Feedser.Source.Ru.Bdu.Tests\StellaOps.Feedser.Source.Ru.Bdu.Tests.csproj", "{C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki.Tests", "StellaOps.Feedser.Source.Ru.Nkcki.Tests\StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj", "{461D4A58-3816-4737-B209-2D1F08B1F4DF}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Debug|x64 = Debug|x64 - Debug|x86 = Debug|x86 - Release|Any CPU = Release|Any CPU - Release|x64 = Release|x64 - Release|x86 = Release|x86 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.ActiveCfg = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.Build.0 = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.ActiveCfg = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.Build.0 = Debug|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.Build.0 = Release|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.ActiveCfg = Release|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.Build.0 = Release|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.ActiveCfg = Release|Any CPU - {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.Build.0 = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.ActiveCfg = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.Build.0 = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.ActiveCfg = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.Build.0 = Debug|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.Build.0 = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.ActiveCfg = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.Build.0 = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.ActiveCfg = Release|Any CPU - {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.Build.0 = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.Build.0 = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.ActiveCfg = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.Build.0 = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.ActiveCfg = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.Build.0 = Debug|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.ActiveCfg = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.Build.0 = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.ActiveCfg = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.Build.0 = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.ActiveCfg = Release|Any CPU - {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.Build.0 = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.ActiveCfg = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.Build.0 = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.ActiveCfg = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.Build.0 = Debug|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.Build.0 = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.ActiveCfg = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.Build.0 = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.ActiveCfg = Release|Any CPU - {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.Build.0 = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.ActiveCfg = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.Build.0 = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.ActiveCfg = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.Build.0 = Debug|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.Build.0 = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.ActiveCfg = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.Build.0 = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.ActiveCfg = Release|Any CPU - {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.Build.0 = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.ActiveCfg = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.Build.0 = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.ActiveCfg = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.Build.0 = Debug|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.Build.0 = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.ActiveCfg = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.Build.0 = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.ActiveCfg = Release|Any CPU - {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.Build.0 = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.Build.0 = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.ActiveCfg = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.Build.0 = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.ActiveCfg = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.Build.0 = Debug|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.ActiveCfg = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.Build.0 = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.ActiveCfg = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.Build.0 = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.ActiveCfg = Release|Any CPU - {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.Build.0 = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.ActiveCfg = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.Build.0 = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.ActiveCfg = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.Build.0 = Debug|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.Build.0 = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.ActiveCfg = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.Build.0 = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.ActiveCfg = Release|Any CPU - {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.Build.0 = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.ActiveCfg = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.Build.0 = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.ActiveCfg = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.Build.0 = Debug|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.Build.0 = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.ActiveCfg = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.Build.0 = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.ActiveCfg = Release|Any CPU - {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.Build.0 = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.ActiveCfg = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.Build.0 = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.ActiveCfg = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.Build.0 = Debug|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.Build.0 = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.ActiveCfg = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.Build.0 = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.ActiveCfg = Release|Any CPU - {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.Build.0 = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.ActiveCfg = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.Build.0 = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.ActiveCfg = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.Build.0 = Debug|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.Build.0 = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.ActiveCfg = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.Build.0 = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.ActiveCfg = Release|Any CPU - {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.Build.0 = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.ActiveCfg = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.Build.0 = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.ActiveCfg = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.Build.0 = Debug|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.Build.0 = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.ActiveCfg = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.Build.0 = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.ActiveCfg = Release|Any CPU - {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.Build.0 = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.ActiveCfg = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.Build.0 = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.ActiveCfg = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.Build.0 = Debug|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.Build.0 = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.ActiveCfg = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.Build.0 = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.ActiveCfg = Release|Any CPU - {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.Build.0 = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.Build.0 = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.ActiveCfg = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.Build.0 = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.ActiveCfg = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.Build.0 = Debug|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.ActiveCfg = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.Build.0 = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.ActiveCfg = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.Build.0 = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.ActiveCfg = Release|Any CPU - {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.Build.0 = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.ActiveCfg = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.Build.0 = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.ActiveCfg = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.Build.0 = Debug|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.Build.0 = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.ActiveCfg = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.Build.0 = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.ActiveCfg = Release|Any CPU - {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.Build.0 = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.Build.0 = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.ActiveCfg = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.Build.0 = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.ActiveCfg = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.Build.0 = Debug|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.ActiveCfg = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.Build.0 = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.ActiveCfg = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.Build.0 = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.ActiveCfg = Release|Any CPU - {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.Build.0 = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.ActiveCfg = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.Build.0 = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.ActiveCfg = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.Build.0 = Debug|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.Build.0 = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.ActiveCfg = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.Build.0 = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.ActiveCfg = Release|Any CPU - {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.Build.0 = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.Build.0 = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.ActiveCfg = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.Build.0 = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.ActiveCfg = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.Build.0 = Debug|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.ActiveCfg = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.Build.0 = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.ActiveCfg = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.Build.0 = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.ActiveCfg = Release|Any CPU - {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.Build.0 = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.Build.0 = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.ActiveCfg = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.Build.0 = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.ActiveCfg = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.Build.0 = Debug|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.ActiveCfg = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.Build.0 = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.ActiveCfg = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.Build.0 = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.ActiveCfg = Release|Any CPU - {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.Build.0 = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.Build.0 = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.ActiveCfg = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.Build.0 = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.ActiveCfg = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.Build.0 = Debug|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.ActiveCfg = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.Build.0 = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.ActiveCfg = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.Build.0 = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.ActiveCfg = Release|Any CPU - {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.Build.0 = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.ActiveCfg = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.Build.0 = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.ActiveCfg = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.Build.0 = Debug|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.Build.0 = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.ActiveCfg = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.Build.0 = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.ActiveCfg = Release|Any CPU - {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.Build.0 = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.ActiveCfg = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.Build.0 = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.ActiveCfg = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.Build.0 = Debug|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.Build.0 = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.ActiveCfg = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.Build.0 = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.ActiveCfg = Release|Any CPU - {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.Build.0 = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.ActiveCfg = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.Build.0 = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.ActiveCfg = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.Build.0 = Debug|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.Build.0 = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.ActiveCfg = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.Build.0 = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.ActiveCfg = Release|Any CPU - {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.Build.0 = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.Build.0 = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.ActiveCfg = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.Build.0 = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.ActiveCfg = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.Build.0 = Debug|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.ActiveCfg = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.Build.0 = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.ActiveCfg = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.Build.0 = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.ActiveCfg = Release|Any CPU - {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.Build.0 = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.ActiveCfg = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.Build.0 = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.ActiveCfg = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.Build.0 = Debug|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.Build.0 = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.ActiveCfg = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.Build.0 = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.ActiveCfg = Release|Any CPU - {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.Build.0 = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.ActiveCfg = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.Build.0 = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.ActiveCfg = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.Build.0 = Debug|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.Build.0 = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.ActiveCfg = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.Build.0 = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.ActiveCfg = Release|Any CPU - {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.Build.0 = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.Build.0 = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.ActiveCfg = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.Build.0 = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.ActiveCfg = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.Build.0 = Debug|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.ActiveCfg = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.Build.0 = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.ActiveCfg = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.Build.0 = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.ActiveCfg = Release|Any CPU - {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.Build.0 = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.ActiveCfg = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.Build.0 = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.ActiveCfg = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.Build.0 = Debug|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.Build.0 = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.ActiveCfg = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.Build.0 = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.ActiveCfg = Release|Any CPU - {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.Build.0 = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.ActiveCfg = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.Build.0 = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.ActiveCfg = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.Build.0 = Debug|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.Build.0 = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.ActiveCfg = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.Build.0 = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.ActiveCfg = Release|Any CPU - {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.Build.0 = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.Build.0 = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.ActiveCfg = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.Build.0 = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.ActiveCfg = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.Build.0 = Debug|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.ActiveCfg = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.Build.0 = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.ActiveCfg = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.Build.0 = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.ActiveCfg = Release|Any CPU - {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.Build.0 = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.ActiveCfg = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.Build.0 = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.ActiveCfg = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.Build.0 = Debug|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.Build.0 = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.ActiveCfg = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.Build.0 = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.ActiveCfg = Release|Any CPU - {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.Build.0 = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.ActiveCfg = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.Build.0 = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.ActiveCfg = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.Build.0 = Debug|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.Build.0 = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.ActiveCfg = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.Build.0 = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.ActiveCfg = Release|Any CPU - {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.Build.0 = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.ActiveCfg = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.Build.0 = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.ActiveCfg = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.Build.0 = Debug|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.Build.0 = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.ActiveCfg = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.Build.0 = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.ActiveCfg = Release|Any CPU - {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.Build.0 = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.ActiveCfg = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.Build.0 = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.ActiveCfg = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.Build.0 = Debug|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.Build.0 = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.ActiveCfg = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.Build.0 = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.ActiveCfg = Release|Any CPU - {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.Build.0 = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.ActiveCfg = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.Build.0 = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.ActiveCfg = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.Build.0 = Debug|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.Build.0 = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.ActiveCfg = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.Build.0 = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.ActiveCfg = Release|Any CPU - {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.Build.0 = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.ActiveCfg = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.Build.0 = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.ActiveCfg = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.Build.0 = Debug|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.Build.0 = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.ActiveCfg = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.Build.0 = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.ActiveCfg = Release|Any CPU - {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.Build.0 = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.ActiveCfg = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.Build.0 = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.ActiveCfg = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.Build.0 = Debug|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.Build.0 = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.ActiveCfg = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.Build.0 = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.ActiveCfg = Release|Any CPU - {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.Build.0 = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.ActiveCfg = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.Build.0 = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.ActiveCfg = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.Build.0 = Debug|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.Build.0 = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.ActiveCfg = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.Build.0 = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.ActiveCfg = Release|Any CPU - {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.Build.0 = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.ActiveCfg = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.Build.0 = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.ActiveCfg = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.Build.0 = Debug|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.Build.0 = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.ActiveCfg = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.Build.0 = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.ActiveCfg = Release|Any CPU - {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.Build.0 = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.ActiveCfg = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.Build.0 = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.ActiveCfg = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.Build.0 = Debug|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.Build.0 = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.ActiveCfg = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.Build.0 = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.ActiveCfg = Release|Any CPU - {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.Build.0 = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.ActiveCfg = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.Build.0 = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.ActiveCfg = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.Build.0 = Debug|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.Build.0 = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.ActiveCfg = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.Build.0 = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.ActiveCfg = Release|Any CPU - {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.Build.0 = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.Build.0 = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.ActiveCfg = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.Build.0 = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.ActiveCfg = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.Build.0 = Debug|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.ActiveCfg = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.Build.0 = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.ActiveCfg = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.Build.0 = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.ActiveCfg = Release|Any CPU - {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.Build.0 = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.ActiveCfg = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.Build.0 = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.ActiveCfg = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.Build.0 = Debug|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.Build.0 = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.ActiveCfg = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.Build.0 = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.ActiveCfg = Release|Any CPU - {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.Build.0 = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.ActiveCfg = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.Build.0 = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.ActiveCfg = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.Build.0 = Debug|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.Build.0 = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.ActiveCfg = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.Build.0 = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.ActiveCfg = Release|Any CPU - {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.Build.0 = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.ActiveCfg = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.Build.0 = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.ActiveCfg = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.Build.0 = Debug|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.Build.0 = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.ActiveCfg = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.Build.0 = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.ActiveCfg = Release|Any CPU - {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.Build.0 = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.ActiveCfg = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.Build.0 = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.ActiveCfg = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.Build.0 = Debug|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.Build.0 = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.ActiveCfg = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.Build.0 = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.ActiveCfg = Release|Any CPU - {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.Build.0 = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.ActiveCfg = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.Build.0 = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.ActiveCfg = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.Build.0 = Debug|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.Build.0 = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.ActiveCfg = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.Build.0 = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.ActiveCfg = Release|Any CPU - {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.Build.0 = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.Build.0 = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.ActiveCfg = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.Build.0 = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.ActiveCfg = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.Build.0 = Debug|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.ActiveCfg = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.Build.0 = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.ActiveCfg = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.Build.0 = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.ActiveCfg = Release|Any CPU - {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.Build.0 = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.ActiveCfg = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.Build.0 = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.ActiveCfg = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.Build.0 = Debug|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.Build.0 = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.ActiveCfg = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.Build.0 = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.ActiveCfg = Release|Any CPU - {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.Build.0 = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.ActiveCfg = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.Build.0 = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.ActiveCfg = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.Build.0 = Debug|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.Build.0 = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.ActiveCfg = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.Build.0 = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.ActiveCfg = Release|Any CPU - {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.Build.0 = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.ActiveCfg = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.Build.0 = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.ActiveCfg = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.Build.0 = Debug|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.Build.0 = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.ActiveCfg = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.Build.0 = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.ActiveCfg = Release|Any CPU - {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.Build.0 = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.ActiveCfg = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.Build.0 = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.ActiveCfg = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.Build.0 = Debug|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.Build.0 = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.ActiveCfg = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.Build.0 = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.ActiveCfg = Release|Any CPU - {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.Build.0 = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.Build.0 = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.ActiveCfg = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.Build.0 = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.ActiveCfg = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.Build.0 = Debug|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.ActiveCfg = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.Build.0 = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.ActiveCfg = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.Build.0 = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.ActiveCfg = Release|Any CPU - {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.Build.0 = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.ActiveCfg = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.Build.0 = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.ActiveCfg = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.Build.0 = Debug|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.Build.0 = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.ActiveCfg = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.Build.0 = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.ActiveCfg = Release|Any CPU - {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.Build.0 = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.Build.0 = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.ActiveCfg = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.Build.0 = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.ActiveCfg = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.Build.0 = Debug|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.ActiveCfg = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.Build.0 = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.ActiveCfg = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.Build.0 = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.ActiveCfg = Release|Any CPU - {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.Build.0 = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.ActiveCfg = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.Build.0 = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.ActiveCfg = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.Build.0 = Debug|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.Build.0 = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.ActiveCfg = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.Build.0 = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.ActiveCfg = Release|Any CPU - {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.Build.0 = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.ActiveCfg = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.Build.0 = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.ActiveCfg = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.Build.0 = Debug|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.Build.0 = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.ActiveCfg = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.Build.0 = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.ActiveCfg = Release|Any CPU - {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.Build.0 = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.ActiveCfg = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.Build.0 = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.ActiveCfg = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.Build.0 = Debug|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.Build.0 = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.ActiveCfg = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.Build.0 = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.ActiveCfg = Release|Any CPU - {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.Build.0 = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.ActiveCfg = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.Build.0 = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.ActiveCfg = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.Build.0 = Debug|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.Build.0 = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.ActiveCfg = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.Build.0 = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.ActiveCfg = Release|Any CPU - {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.Build.0 = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.ActiveCfg = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.Build.0 = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.ActiveCfg = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.Build.0 = Debug|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.Build.0 = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.ActiveCfg = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.Build.0 = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.ActiveCfg = Release|Any CPU - {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.Build.0 = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.ActiveCfg = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.Build.0 = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.ActiveCfg = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.Build.0 = Debug|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.Build.0 = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.ActiveCfg = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.Build.0 = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.ActiveCfg = Release|Any CPU - {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.Build.0 = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.ActiveCfg = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.Build.0 = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.ActiveCfg = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.Build.0 = Debug|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.Build.0 = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.ActiveCfg = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.Build.0 = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.ActiveCfg = Release|Any CPU - {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.Build.0 = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.Build.0 = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.ActiveCfg = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.Build.0 = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.ActiveCfg = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.Build.0 = Debug|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.ActiveCfg = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.Build.0 = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.ActiveCfg = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.Build.0 = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.ActiveCfg = Release|Any CPU - {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.Build.0 = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.Build.0 = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.ActiveCfg = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.Build.0 = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.ActiveCfg = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.Build.0 = Debug|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.ActiveCfg = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.Build.0 = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.ActiveCfg = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.Build.0 = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.ActiveCfg = Release|Any CPU - {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.Build.0 = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x64.ActiveCfg = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x64.Build.0 = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x86.ActiveCfg = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x86.Build.0 = Debug|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|Any CPU.Build.0 = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x64.ActiveCfg = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x64.Build.0 = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x86.ActiveCfg = Release|Any CPU - {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x86.Build.0 = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x64.ActiveCfg = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x64.Build.0 = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x86.ActiveCfg = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x86.Build.0 = Debug|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|Any CPU.Build.0 = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x64.ActiveCfg = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x64.Build.0 = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.ActiveCfg = Release|Any CPU - {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Acsc", "StellaOps.Concelier.Connector.Acsc\StellaOps.Concelier.Connector.Acsc.csproj", "{CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common", "StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj", "{E9DE840D-0760-4324-98E2-7F2CBE06DC1A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{061B0042-9A6C-4CFD-9E48-4D3F3B924442}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Cisa", "StellaOps.Concelier.Connector.Ics.Cisa\StellaOps.Concelier.Connector.Ics.Cisa.csproj", "{6A301F32-2EEE-491B-9DB9-3BF26D032F07}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{AFCCC916-58E8-4676-AABB-54B04CEA3392}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{BF3DAB2F-E46E-49C1-9BA5-AA389763A632}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Normalization", "StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj", "{429BAA6A-706D-489A-846F-4B0EF1B15121}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Merge", "StellaOps.Concelier.Merge\StellaOps.Concelier.Merge.csproj", "{085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.Json", "StellaOps.Concelier.Exporter.Json\StellaOps.Concelier.Exporter.Json.csproj", "{1C5506B8-C01B-4419-B888-A48F441E0C69}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.TrivyDb", "StellaOps.Concelier.Exporter.TrivyDb\StellaOps.Concelier.Exporter.TrivyDb.csproj", "{4D936BC4-5520-4642-A237-4106E97BC7A0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{B85C1C0E-B245-44FB-877E-C112DE29041A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.WebService", "StellaOps.Concelier.WebService\StellaOps.Concelier.WebService.csproj", "{2C970A0F-FE3D-425B-B1B3-A008B194F5C2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Cccs", "StellaOps.Concelier.Connector.Cccs\StellaOps.Concelier.Connector.Cccs.csproj", "{A7035381-6D20-4A07-817B-A324ED735EB3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Debian", "StellaOps.Concelier.Connector.Distro.Debian\StellaOps.Concelier.Connector.Distro.Debian.csproj", "{404F5F6E-37E4-4EF9-B09D-6634366B5D44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Ubuntu", "StellaOps.Concelier.Connector.Distro.Ubuntu\StellaOps.Concelier.Connector.Distro.Ubuntu.csproj", "{1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Kisa", "StellaOps.Concelier.Connector.Kisa\StellaOps.Concelier.Connector.Kisa.csproj", "{23055A20-7079-4336-AD30-EFAA2FA11665}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertCc", "StellaOps.Concelier.Connector.CertCc\StellaOps.Concelier.Connector.CertCc.csproj", "{C2304954-9B15-4776-8DB6-22E293D311E4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertFr", "StellaOps.Concelier.Connector.CertFr\StellaOps.Concelier.Connector.CertFr.csproj", "{E6895821-ED23-46D2-A5DC-06D61F90EC27}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Nvd", "StellaOps.Concelier.Connector.Nvd\StellaOps.Concelier.Connector.Nvd.csproj", "{378CB675-D70B-4A95-B324-62B67D79AAB7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Oracle", "StellaOps.Concelier.Connector.Vndr.Oracle\StellaOps.Concelier.Connector.Vndr.Oracle.csproj", "{53AD2E55-B0F5-46AD-BFE5-82F486371872}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Nkcki", "StellaOps.Concelier.Connector.Ru.Nkcki\StellaOps.Concelier.Connector.Ru.Nkcki.csproj", "{B880C99C-C0BD-4953-95AD-2C76BC43F760}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Suse", "StellaOps.Concelier.Connector.Distro.Suse\StellaOps.Concelier.Connector.Distro.Suse.csproj", "{23422F67-C1FB-4FF4-899C-706BCD63D9FD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Bdu", "StellaOps.Concelier.Connector.Ru.Bdu\StellaOps.Concelier.Connector.Ru.Bdu.csproj", "{16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Kev", "StellaOps.Concelier.Connector.Kev\StellaOps.Concelier.Connector.Kev.csproj", "{20DB9837-715B-4515-98C6-14B50060B765}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Kaspersky", "StellaOps.Concelier.Connector.Ics.Kaspersky\StellaOps.Concelier.Connector.Ics.Kaspersky.csproj", "{10849EE2-9F34-4C23-BBB4-916A59CDB7F4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Osv", "StellaOps.Concelier.Connector.Osv\StellaOps.Concelier.Connector.Osv.csproj", "{EFB16EDB-78D4-4601-852E-F4B37655FA13}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Jvn", "StellaOps.Concelier.Connector.Jvn\StellaOps.Concelier.Connector.Jvn.csproj", "{02289F61-0173-42CC-B8F2-25CC53F8E066}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertBund", "StellaOps.Concelier.Connector.CertBund\StellaOps.Concelier.Connector.CertBund.csproj", "{4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Cve", "StellaOps.Concelier.Connector.Cve\StellaOps.Concelier.Connector.Cve.csproj", "{EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Cisco", "StellaOps.Concelier.Connector.Vndr.Cisco\StellaOps.Concelier.Connector.Vndr.Cisco.csproj", "{19957518-A422-4622-9FD1-621DF3E31869}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Msrc", "StellaOps.Concelier.Connector.Vndr.Msrc\StellaOps.Concelier.Connector.Vndr.Msrc.csproj", "{69C4C061-F5A0-4EAA-A4CD-9A513523952A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Chromium", "StellaOps.Concelier.Connector.Vndr.Chromium\StellaOps.Concelier.Connector.Vndr.Chromium.csproj", "{C7F7DE6F-A369-4F43-9864-286DCEC615F8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Apple", "StellaOps.Concelier.Connector.Vndr.Apple\StellaOps.Concelier.Connector.Vndr.Apple.csproj", "{1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Vmware", "StellaOps.Concelier.Connector.Vndr.Vmware\StellaOps.Concelier.Connector.Vndr.Vmware.csproj", "{7255C38D-5A16-4A4D-98CE-CF0FD516B68E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Adobe", "StellaOps.Concelier.Connector.Vndr.Adobe\StellaOps.Concelier.Connector.Vndr.Adobe.csproj", "{C3A42AA3-800D-4398-A077-5560EE6451EF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertIn", "StellaOps.Concelier.Connector.CertIn\StellaOps.Concelier.Connector.CertIn.csproj", "{5016963A-6FC9-4063-AB83-2D1F9A2BC627}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ghsa", "StellaOps.Concelier.Connector.Ghsa\StellaOps.Concelier.Connector.Ghsa.csproj", "{72F43F43-F852-487F-8334-91D438CE2F7C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.RedHat", "StellaOps.Concelier.Connector.Distro.RedHat\StellaOps.Concelier.Connector.Distro.RedHat.csproj", "{A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{F622D38D-DA49-473E-B724-E706F8113CF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core.Tests", "StellaOps.Concelier.Core.Tests\StellaOps.Concelier.Core.Tests.csproj", "{3A3D7610-C864-4413-B07E-9E8C2A49A90E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Merge.Tests", "StellaOps.Concelier.Merge.Tests\StellaOps.Concelier.Merge.Tests.csproj", "{9C4DEE96-CD7D-4AE3-A811-0B48B477003B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models.Tests", "StellaOps.Concelier.Models.Tests\StellaOps.Concelier.Models.Tests.csproj", "{437B2667-9461-47D2-B75B-4D2E03D69B94}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Normalization.Tests", "StellaOps.Concelier.Normalization.Tests\StellaOps.Concelier.Normalization.Tests.csproj", "{8249DF28-CDAF-4DEF-A912-C27F57B67FD5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo.Tests", "StellaOps.Concelier.Storage.Mongo.Tests\StellaOps.Concelier.Storage.Mongo.Tests.csproj", "{CBFB015B-C069-475F-A476-D52222729804}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.Json.Tests", "StellaOps.Concelier.Exporter.Json.Tests\StellaOps.Concelier.Exporter.Json.Tests.csproj", "{2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.TrivyDb.Tests", "StellaOps.Concelier.Exporter.TrivyDb.Tests\StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj", "{3EB22234-642E-4533-BCC3-93E8ED443B1D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.WebService.Tests", "StellaOps.Concelier.WebService.Tests\StellaOps.Concelier.WebService.Tests.csproj", "{84A5DE81-4444-499A-93BF-6DC4CA72F8D4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common.Tests", "StellaOps.Concelier.Connector.Common.Tests\StellaOps.Concelier.Connector.Common.Tests.csproj", "{42E21E1D-C3DE-4765-93E9-39391BB5C802}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Nvd.Tests", "StellaOps.Concelier.Connector.Nvd.Tests\StellaOps.Concelier.Connector.Nvd.Tests.csproj", "{B6E2EE26-B297-4AB9-A47E-A227F5EAE108}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.RedHat.Tests", "StellaOps.Concelier.Connector.Distro.RedHat.Tests\StellaOps.Concelier.Connector.Distro.RedHat.Tests.csproj", "{CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Chromium.Tests", "StellaOps.Concelier.Connector.Vndr.Chromium.Tests\StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj", "{2891FCDE-BB89-46F0-A40C-368EF804DB44}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Adobe.Tests", "StellaOps.Concelier.Connector.Vndr.Adobe.Tests\StellaOps.Concelier.Connector.Vndr.Adobe.Tests.csproj", "{B91C60FB-926F-47C3-BFD0-6DD145308344}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Oracle.Tests", "StellaOps.Concelier.Connector.Vndr.Oracle.Tests\StellaOps.Concelier.Connector.Vndr.Oracle.Tests.csproj", "{30DF89D1-D66D-4078-8A3B-951637A42265}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Vmware.Tests", "StellaOps.Concelier.Connector.Vndr.Vmware.Tests\StellaOps.Concelier.Connector.Vndr.Vmware.Tests.csproj", "{6E98C770-72FF-41FA-8C42-30AABAAF5B4E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertIn.Tests", "StellaOps.Concelier.Connector.CertIn.Tests\StellaOps.Concelier.Connector.CertIn.Tests.csproj", "{79B36C92-BA93-4406-AB75-6F2282DDFF01}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertFr.Tests", "StellaOps.Concelier.Connector.CertFr.Tests\StellaOps.Concelier.Connector.CertFr.Tests.csproj", "{4B60FA53-81F6-4AB6-BE9F-DE0992E11977}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Kaspersky.Tests", "StellaOps.Concelier.Connector.Ics.Kaspersky.Tests\StellaOps.Concelier.Connector.Ics.Kaspersky.Tests.csproj", "{6BBA820B-8443-4832-91C3-3AB002006494}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Jvn.Tests", "StellaOps.Concelier.Connector.Jvn.Tests\StellaOps.Concelier.Connector.Jvn.Tests.csproj", "{7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Osv.Tests", "StellaOps.Concelier.Connector.Osv.Tests\StellaOps.Concelier.Connector.Osv.Tests.csproj", "{F892BFFD-9101-4D59-B6FD-C532EB04D51F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Testing", "StellaOps.Concelier.Testing\StellaOps.Concelier.Testing.csproj", "{EAE910FC-188C-41C3-822A-623964CABE48}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Debian.Tests", "StellaOps.Concelier.Connector.Distro.Debian.Tests\StellaOps.Concelier.Connector.Distro.Debian.Tests.csproj", "{BBA5C780-6348-427D-9600-726EAA8963B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "StellaOps.Configuration\StellaOps.Configuration.csproj", "{5F44A429-816A-4560-A5AA-61CD23FD8A19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{544DBB82-4639-4856-A5F2-76828F7A8396}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Bdu.Tests", "StellaOps.Concelier.Connector.Ru.Bdu.Tests\StellaOps.Concelier.Connector.Ru.Bdu.Tests.csproj", "{C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Nkcki.Tests", "StellaOps.Concelier.Connector.Ru.Nkcki.Tests\StellaOps.Concelier.Connector.Ru.Nkcki.Tests.csproj", "{461D4A58-3816-4737-B209-2D1F08B1F4DF}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x64.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.ActiveCfg = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Debug|x86.Build.0 = Debug|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|Any CPU.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x64.Build.0 = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.ActiveCfg = Release|Any CPU + {CFD7B267-46B7-4C73-A33A-3E82AD2CFABC}.Release|x86.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x64.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.ActiveCfg = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Debug|x86.Build.0 = Debug|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|Any CPU.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x64.Build.0 = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.ActiveCfg = Release|Any CPU + {E9DE840D-0760-4324-98E2-7F2CBE06DC1A}.Release|x86.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|Any CPU.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x64.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.ActiveCfg = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Debug|x86.Build.0 = Debug|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|Any CPU.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x64.Build.0 = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.ActiveCfg = Release|Any CPU + {061B0042-9A6C-4CFD-9E48-4D3F3B924442}.Release|x86.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x64.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.ActiveCfg = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Debug|x86.Build.0 = Debug|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|Any CPU.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x64.Build.0 = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.ActiveCfg = Release|Any CPU + {6A301F32-2EEE-491B-9DB9-3BF26D032F07}.Release|x86.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x64.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.ActiveCfg = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Debug|x86.Build.0 = Debug|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|Any CPU.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x64.Build.0 = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.ActiveCfg = Release|Any CPU + {AFCCC916-58E8-4676-AABB-54B04CEA3392}.Release|x86.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x64.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.ActiveCfg = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Debug|x86.Build.0 = Debug|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|Any CPU.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x64.Build.0 = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.ActiveCfg = Release|Any CPU + {BF3DAB2F-E46E-49C1-9BA5-AA389763A632}.Release|x86.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|Any CPU.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x64.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.ActiveCfg = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Debug|x86.Build.0 = Debug|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|Any CPU.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x64.Build.0 = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.ActiveCfg = Release|Any CPU + {429BAA6A-706D-489A-846F-4B0EF1B15121}.Release|x86.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x64.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.ActiveCfg = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Debug|x86.Build.0 = Debug|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|Any CPU.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x64.Build.0 = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.ActiveCfg = Release|Any CPU + {085CEC8E-0E10-48E8-89E2-9452CD2E7FA0}.Release|x86.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x64.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Debug|x86.Build.0 = Debug|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|Any CPU.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x64.Build.0 = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.ActiveCfg = Release|Any CPU + {1C5506B8-C01B-4419-B888-A48F441E0C69}.Release|x86.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x64.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.ActiveCfg = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Debug|x86.Build.0 = Debug|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|Any CPU.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x64.Build.0 = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.ActiveCfg = Release|Any CPU + {4D936BC4-5520-4642-A237-4106E97BC7A0}.Release|x86.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x64.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.ActiveCfg = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Debug|x86.Build.0 = Debug|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|Any CPU.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x64.Build.0 = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.ActiveCfg = Release|Any CPU + {B85C1C0E-B245-44FB-877E-C112DE29041A}.Release|x86.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x64.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.ActiveCfg = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Debug|x86.Build.0 = Debug|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|Any CPU.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x64.Build.0 = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.ActiveCfg = Release|Any CPU + {2C970A0F-FE3D-425B-B1B3-A008B194F5C2}.Release|x86.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x64.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.ActiveCfg = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Debug|x86.Build.0 = Debug|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|Any CPU.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x64.Build.0 = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.ActiveCfg = Release|Any CPU + {A7035381-6D20-4A07-817B-A324ED735EB3}.Release|x86.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x64.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.ActiveCfg = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Debug|x86.Build.0 = Debug|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|Any CPU.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x64.Build.0 = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.ActiveCfg = Release|Any CPU + {404F5F6E-37E4-4EF9-B09D-6634366B5D44}.Release|x86.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x64.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.ActiveCfg = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Debug|x86.Build.0 = Debug|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|Any CPU.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x64.Build.0 = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.ActiveCfg = Release|Any CPU + {1BEF4D9D-9EA4-4BE9-9664-F16DC1CA8EEB}.Release|x86.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x64.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.ActiveCfg = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Debug|x86.Build.0 = Debug|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|Any CPU.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x64.Build.0 = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.ActiveCfg = Release|Any CPU + {23055A20-7079-4336-AD30-EFAA2FA11665}.Release|x86.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x64.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.ActiveCfg = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Debug|x86.Build.0 = Debug|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|Any CPU.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x64.Build.0 = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.ActiveCfg = Release|Any CPU + {C2304954-9B15-4776-8DB6-22E293D311E4}.Release|x86.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x64.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.ActiveCfg = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Debug|x86.Build.0 = Debug|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|Any CPU.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x64.Build.0 = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.ActiveCfg = Release|Any CPU + {E6895821-ED23-46D2-A5DC-06D61F90EC27}.Release|x86.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x64.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.ActiveCfg = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Debug|x86.Build.0 = Debug|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|Any CPU.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x64.Build.0 = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.ActiveCfg = Release|Any CPU + {378CB675-D70B-4A95-B324-62B67D79AAB7}.Release|x86.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|Any CPU.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x64.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.ActiveCfg = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Debug|x86.Build.0 = Debug|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|Any CPU.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x64.Build.0 = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.ActiveCfg = Release|Any CPU + {53AD2E55-B0F5-46AD-BFE5-82F486371872}.Release|x86.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x64.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.ActiveCfg = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Debug|x86.Build.0 = Debug|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|Any CPU.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x64.Build.0 = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.ActiveCfg = Release|Any CPU + {B880C99C-C0BD-4953-95AD-2C76BC43F760}.Release|x86.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x64.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.ActiveCfg = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Debug|x86.Build.0 = Debug|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|Any CPU.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x64.Build.0 = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.ActiveCfg = Release|Any CPU + {23422F67-C1FB-4FF4-899C-706BCD63D9FD}.Release|x86.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x64.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.ActiveCfg = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Debug|x86.Build.0 = Debug|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|Any CPU.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x64.Build.0 = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.ActiveCfg = Release|Any CPU + {16AD4AB9-2A80-4CFD-91A7-36CC1FEF439F}.Release|x86.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x64.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.ActiveCfg = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Debug|x86.Build.0 = Debug|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|Any CPU.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x64.Build.0 = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.ActiveCfg = Release|Any CPU + {20DB9837-715B-4515-98C6-14B50060B765}.Release|x86.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x64.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.ActiveCfg = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Debug|x86.Build.0 = Debug|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|Any CPU.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x64.Build.0 = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.ActiveCfg = Release|Any CPU + {10849EE2-9F34-4C23-BBB4-916A59CDB7F4}.Release|x86.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x64.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.ActiveCfg = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Debug|x86.Build.0 = Debug|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|Any CPU.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x64.Build.0 = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.ActiveCfg = Release|Any CPU + {EFB16EDB-78D4-4601-852E-F4B37655FA13}.Release|x86.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x64.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.ActiveCfg = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Debug|x86.Build.0 = Debug|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|Any CPU.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x64.Build.0 = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.ActiveCfg = Release|Any CPU + {02289F61-0173-42CC-B8F2-25CC53F8E066}.Release|x86.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x64.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.ActiveCfg = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Debug|x86.Build.0 = Debug|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|Any CPU.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x64.Build.0 = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.ActiveCfg = Release|Any CPU + {4CE0B67B-2B6D-4D48-9D38-2F1165FD6BF4}.Release|x86.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x64.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.ActiveCfg = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Debug|x86.Build.0 = Debug|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|Any CPU.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x64.Build.0 = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.ActiveCfg = Release|Any CPU + {EB037D9A-EF9C-439D-8A79-4B7D12F9C9D0}.Release|x86.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x64.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.ActiveCfg = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Debug|x86.Build.0 = Debug|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|Any CPU.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x64.Build.0 = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.ActiveCfg = Release|Any CPU + {19957518-A422-4622-9FD1-621DF3E31869}.Release|x86.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x64.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.ActiveCfg = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Debug|x86.Build.0 = Debug|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|Any CPU.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x64.Build.0 = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.ActiveCfg = Release|Any CPU + {69C4C061-F5A0-4EAA-A4CD-9A513523952A}.Release|x86.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x64.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.ActiveCfg = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Debug|x86.Build.0 = Debug|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|Any CPU.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x64.Build.0 = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.ActiveCfg = Release|Any CPU + {C7F7DE6F-A369-4F43-9864-286DCEC615F8}.Release|x86.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x64.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.ActiveCfg = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Debug|x86.Build.0 = Debug|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|Any CPU.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x64.Build.0 = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.ActiveCfg = Release|Any CPU + {1C1593FE-73A4-47E8-A45B-5FC3B0BA7698}.Release|x86.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x64.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.ActiveCfg = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Debug|x86.Build.0 = Debug|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|Any CPU.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x64.Build.0 = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.ActiveCfg = Release|Any CPU + {7255C38D-5A16-4A4D-98CE-CF0FD516B68E}.Release|x86.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x64.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.ActiveCfg = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Debug|x86.Build.0 = Debug|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|Any CPU.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x64.Build.0 = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.ActiveCfg = Release|Any CPU + {C3A42AA3-800D-4398-A077-5560EE6451EF}.Release|x86.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x64.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.ActiveCfg = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Debug|x86.Build.0 = Debug|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|Any CPU.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x64.Build.0 = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.ActiveCfg = Release|Any CPU + {5016963A-6FC9-4063-AB83-2D1F9A2BC627}.Release|x86.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x64.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.ActiveCfg = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Debug|x86.Build.0 = Debug|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|Any CPU.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x64.Build.0 = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.ActiveCfg = Release|Any CPU + {72F43F43-F852-487F-8334-91D438CE2F7C}.Release|x86.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x64.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.ActiveCfg = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Debug|x86.Build.0 = Debug|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|Any CPU.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x64.Build.0 = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.ActiveCfg = Release|Any CPU + {A4DBF88F-34D0-4A05-ACCE-DE080F912FDB}.Release|x86.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x64.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.ActiveCfg = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Debug|x86.Build.0 = Debug|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|Any CPU.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x64.Build.0 = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.ActiveCfg = Release|Any CPU + {F622D38D-DA49-473E-B724-E706F8113CF2}.Release|x86.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x64.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Debug|x86.Build.0 = Debug|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|Any CPU.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x64.Build.0 = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.ActiveCfg = Release|Any CPU + {3A3D7610-C864-4413-B07E-9E8C2A49A90E}.Release|x86.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x64.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.ActiveCfg = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Debug|x86.Build.0 = Debug|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|Any CPU.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x64.Build.0 = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.ActiveCfg = Release|Any CPU + {9C4DEE96-CD7D-4AE3-A811-0B48B477003B}.Release|x86.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x64.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.ActiveCfg = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Debug|x86.Build.0 = Debug|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|Any CPU.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x64.Build.0 = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.ActiveCfg = Release|Any CPU + {437B2667-9461-47D2-B75B-4D2E03D69B94}.Release|x86.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x64.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.ActiveCfg = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Debug|x86.Build.0 = Debug|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|Any CPU.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x64.Build.0 = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.ActiveCfg = Release|Any CPU + {8249DF28-CDAF-4DEF-A912-C27F57B67FD5}.Release|x86.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x64.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.ActiveCfg = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Debug|x86.Build.0 = Debug|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|Any CPU.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x64.Build.0 = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.ActiveCfg = Release|Any CPU + {CBFB015B-C069-475F-A476-D52222729804}.Release|x86.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x64.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.ActiveCfg = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Debug|x86.Build.0 = Debug|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|Any CPU.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x64.Build.0 = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.ActiveCfg = Release|Any CPU + {2A41D9D2-3218-4F12-9C2B-3DB18A8E732E}.Release|x86.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x64.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.ActiveCfg = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Debug|x86.Build.0 = Debug|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|Any CPU.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x64.Build.0 = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.ActiveCfg = Release|Any CPU + {3EB22234-642E-4533-BCC3-93E8ED443B1D}.Release|x86.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x64.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.ActiveCfg = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Debug|x86.Build.0 = Debug|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|Any CPU.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x64.Build.0 = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.ActiveCfg = Release|Any CPU + {84A5DE81-4444-499A-93BF-6DC4CA72F8D4}.Release|x86.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x64.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.ActiveCfg = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Debug|x86.Build.0 = Debug|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|Any CPU.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x64.Build.0 = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.ActiveCfg = Release|Any CPU + {42E21E1D-C3DE-4765-93E9-39391BB5C802}.Release|x86.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x64.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.ActiveCfg = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Debug|x86.Build.0 = Debug|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|Any CPU.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x64.Build.0 = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.ActiveCfg = Release|Any CPU + {B6E2EE26-B297-4AB9-A47E-A227F5EAE108}.Release|x86.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x64.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.ActiveCfg = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Debug|x86.Build.0 = Debug|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|Any CPU.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x64.Build.0 = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.ActiveCfg = Release|Any CPU + {CDB2D636-C82F-43F1-BB30-FFC6258FBAB4}.Release|x86.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x64.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.ActiveCfg = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Debug|x86.Build.0 = Debug|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|Any CPU.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x64.Build.0 = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.ActiveCfg = Release|Any CPU + {2891FCDE-BB89-46F0-A40C-368EF804DB44}.Release|x86.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x64.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.ActiveCfg = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Debug|x86.Build.0 = Debug|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|Any CPU.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x64.Build.0 = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.ActiveCfg = Release|Any CPU + {B91C60FB-926F-47C3-BFD0-6DD145308344}.Release|x86.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|Any CPU.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x64.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.ActiveCfg = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Debug|x86.Build.0 = Debug|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|Any CPU.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x64.Build.0 = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.ActiveCfg = Release|Any CPU + {30DF89D1-D66D-4078-8A3B-951637A42265}.Release|x86.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x64.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.ActiveCfg = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Debug|x86.Build.0 = Debug|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|Any CPU.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x64.Build.0 = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.ActiveCfg = Release|Any CPU + {6E98C770-72FF-41FA-8C42-30AABAAF5B4E}.Release|x86.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|Any CPU.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x64.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.ActiveCfg = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Debug|x86.Build.0 = Debug|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|Any CPU.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x64.Build.0 = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.ActiveCfg = Release|Any CPU + {79B36C92-BA93-4406-AB75-6F2282DDFF01}.Release|x86.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x64.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.ActiveCfg = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Debug|x86.Build.0 = Debug|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|Any CPU.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x64.Build.0 = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.ActiveCfg = Release|Any CPU + {4B60FA53-81F6-4AB6-BE9F-DE0992E11977}.Release|x86.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x64.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.ActiveCfg = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Debug|x86.Build.0 = Debug|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|Any CPU.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x64.Build.0 = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.ActiveCfg = Release|Any CPU + {6BBA820B-8443-4832-91C3-3AB002006494}.Release|x86.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x64.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.ActiveCfg = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Debug|x86.Build.0 = Debug|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|Any CPU.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x64.Build.0 = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.ActiveCfg = Release|Any CPU + {7845AE1C-FBD7-4177-A06F-D7AAE8315DB2}.Release|x86.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x64.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.ActiveCfg = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Debug|x86.Build.0 = Debug|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|Any CPU.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x64.Build.0 = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.ActiveCfg = Release|Any CPU + {F892BFFD-9101-4D59-B6FD-C532EB04D51F}.Release|x86.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x64.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.ActiveCfg = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Debug|x86.Build.0 = Debug|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|Any CPU.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x64.Build.0 = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.ActiveCfg = Release|Any CPU + {EAE910FC-188C-41C3-822A-623964CABE48}.Release|x86.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x64.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Debug|x86.Build.0 = Debug|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|Any CPU.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x64.Build.0 = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.ActiveCfg = Release|Any CPU + {BBA5C780-6348-427D-9600-726EAA8963B3}.Release|x86.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x64.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.ActiveCfg = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Debug|x86.Build.0 = Debug|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|Any CPU.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x64.Build.0 = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.ActiveCfg = Release|Any CPU + {5F44A429-816A-4560-A5AA-61CD23FD8A19}.Release|x86.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x64.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.ActiveCfg = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Debug|x86.Build.0 = Debug|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|Any CPU.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x64.Build.0 = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.ActiveCfg = Release|Any CPU + {20FDC3B4-9908-4ABF-BA1D-50E0B4A64F4B}.Release|x86.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|Any CPU.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x64.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.ActiveCfg = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Debug|x86.Build.0 = Debug|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|Any CPU.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x64.Build.0 = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.ActiveCfg = Release|Any CPU + {544DBB82-4639-4856-A5F2-76828F7A8396}.Release|x86.Build.0 = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x64.ActiveCfg = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x64.Build.0 = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x86.ActiveCfg = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Debug|x86.Build.0 = Debug|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|Any CPU.Build.0 = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x64.ActiveCfg = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x64.Build.0 = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x86.ActiveCfg = Release|Any CPU + {C4B189FA-4268-4B3C-A6B0-C2BB5B96D11A}.Release|x86.Build.0 = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x64.ActiveCfg = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x64.Build.0 = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x86.ActiveCfg = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Debug|x86.Build.0 = Debug|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|Any CPU.Build.0 = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x64.ActiveCfg = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x64.Build.0 = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.ActiveCfg = Release|Any CPU + {461D4A58-3816-4737-B209-2D1F08B1F4DF}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs index 5414ff9b..7e84a69e 100644 --- a/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs +++ b/src/StellaOps.Configuration/StellaOpsAuthorityOptions.cs @@ -1,542 +1,778 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.RateLimiting; -using StellaOps.Authority.Plugins.Abstractions; -using StellaOps.Cryptography; - -namespace StellaOps.Configuration; - -/// -/// Strongly typed configuration for the StellaOps Authority service. -/// -public sealed class StellaOpsAuthorityOptions -{ - private readonly List pluginDirectories = new(); - private readonly List bypassNetworks = new(); - - /// - /// Schema version for downstream consumers to coordinate breaking changes. - /// - public int SchemaVersion { get; set; } = 1; - - /// - /// Absolute issuer URI advertised to clients (e.g. https://authority.stella-ops.local). - /// - public Uri? Issuer { get; set; } - - /// - /// Lifetime for OAuth access tokens issued by Authority. - /// - public TimeSpan AccessTokenLifetime { get; set; } = TimeSpan.FromMinutes(15); - - /// - /// Lifetime for OAuth refresh tokens issued by Authority. - /// - public TimeSpan RefreshTokenLifetime { get; set; } = TimeSpan.FromDays(30); - - /// - /// Lifetime for OpenID Connect identity tokens. - /// - public TimeSpan IdentityTokenLifetime { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Lifetime for OAuth authorization codes. - /// - public TimeSpan AuthorizationCodeLifetime { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Lifetime for OAuth device codes (device authorization flow). - /// - public TimeSpan DeviceCodeLifetime { get; set; } = TimeSpan.FromMinutes(15); - - /// - /// Directories scanned for Authority plugins (absolute or relative to application base path). - /// - public IList PluginDirectories => pluginDirectories; - - /// - /// CIDR blocks permitted to bypass certain authentication policies (e.g. on-host cron). - /// - public IList BypassNetworks => bypassNetworks; - - /// - /// Configuration describing the Authority MongoDB storage. - /// - public AuthorityStorageOptions Storage { get; } = new(); - - /// - /// Bootstrap settings for initial administrative provisioning. - /// - public AuthorityBootstrapOptions Bootstrap { get; } = new(); - - /// - /// Configuration describing available Authority plugins and their manifests. - /// - public AuthorityPluginSettings Plugins { get; } = new(); - - /// - /// Security-related configuration for the Authority host. - /// - public AuthoritySecurityOptions Security { get; } = new(); - - /// - /// Signing options for Authority-generated artefacts (revocation bundles, JWKS). - /// - public AuthoritySigningOptions Signing { get; } = new(); - - /// - /// Validates configured values and normalises collections. - /// - /// Thrown when configuration is invalid. - public void Validate() - { - if (SchemaVersion <= 0) - { - throw new InvalidOperationException("Authority configuration requires a positive schemaVersion."); - } - - if (Issuer is null) - { - throw new InvalidOperationException("Authority configuration requires an issuer URL."); - } - - if (!Issuer.IsAbsoluteUri) - { - throw new InvalidOperationException("Authority issuer must be an absolute URI."); - } - - if (string.Equals(Issuer.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && !Issuer.IsLoopback) - { - throw new InvalidOperationException("Authority issuer must use HTTPS unless running on a loopback interface."); - } - - ValidateLifetime(AccessTokenLifetime, nameof(AccessTokenLifetime), TimeSpan.FromHours(24)); - ValidateLifetime(RefreshTokenLifetime, nameof(RefreshTokenLifetime), TimeSpan.FromDays(365)); - ValidateLifetime(IdentityTokenLifetime, nameof(IdentityTokenLifetime), TimeSpan.FromHours(24)); - ValidateLifetime(AuthorizationCodeLifetime, nameof(AuthorizationCodeLifetime), TimeSpan.FromHours(1)); - ValidateLifetime(DeviceCodeLifetime, nameof(DeviceCodeLifetime), TimeSpan.FromHours(24)); - - NormaliseList(pluginDirectories); - NormaliseList(bypassNetworks); - - Security.Validate(); - Signing.Validate(); - Plugins.NormalizeAndValidate(); - Storage.Validate(); - Bootstrap.Validate(); - } - - private static void ValidateLifetime(TimeSpan value, string propertyName, TimeSpan maximum) - { - if (value <= TimeSpan.Zero) - { - throw new InvalidOperationException($"Authority configuration requires {propertyName} to be greater than zero."); - } - - if (value > maximum) - { - throw new InvalidOperationException($"Authority configuration requires {propertyName} to be less than or equal to {maximum}."); - } - } - - private static void NormaliseList(IList values) - { - if (values.Count == 0) - { - return; - } - - var unique = new HashSet(StringComparer.OrdinalIgnoreCase); - - for (var index = values.Count - 1; index >= 0; index--) - { - var entry = values[index]; - - if (string.IsNullOrWhiteSpace(entry)) - { - values.RemoveAt(index); - continue; - } - - var trimmed = entry.Trim(); - if (!unique.Add(trimmed)) - { - values.RemoveAt(index); - continue; - } - - values[index] = trimmed; - } - } -} - -public sealed class AuthoritySecurityOptions -{ - /// - /// Rate limiting configuration applied to Authority endpoints. - /// - public AuthorityRateLimitingOptions RateLimiting { get; } = new(); - - /// - /// Default password hashing parameters advertised to Authority plug-ins. - /// - public PasswordHashOptions PasswordHashing { get; } = new(); - - internal void Validate() - { - RateLimiting.Validate(); - PasswordHashing.Validate(); - } -} - -public sealed class AuthorityRateLimitingOptions -{ - public AuthorityRateLimitingOptions() - { - Token = new AuthorityEndpointRateLimitOptions - { - PermitLimit = 30, - Window = TimeSpan.FromMinutes(1), - QueueLimit = 0 - }; - - Authorize = new AuthorityEndpointRateLimitOptions - { - PermitLimit = 60, - Window = TimeSpan.FromMinutes(1), - QueueLimit = 10 - }; - - Internal = new AuthorityEndpointRateLimitOptions - { - Enabled = false, - PermitLimit = 5, - Window = TimeSpan.FromMinutes(1), - QueueLimit = 0 - }; - } - - /// - /// Rate limiting configuration applied to the /token endpoint. - /// - public AuthorityEndpointRateLimitOptions Token { get; } - - /// - /// Rate limiting configuration applied to the /authorize endpoint. - /// - public AuthorityEndpointRateLimitOptions Authorize { get; } - - /// - /// Rate limiting configuration applied to /internal endpoints. - /// - public AuthorityEndpointRateLimitOptions Internal { get; } - - internal void Validate() - { - Token.Validate(nameof(Token)); - Authorize.Validate(nameof(Authorize)); - Internal.Validate(nameof(Internal)); - } -} - -public sealed class AuthorityEndpointRateLimitOptions -{ - /// - /// Gets or sets a value indicating whether rate limiting is enabled for the endpoint. - /// - public bool Enabled { get; set; } = true; - - /// - /// Maximum number of requests allowed within the configured window. - /// - public int PermitLimit { get; set; } = 60; - - /// - /// Size of the fixed window applied to the rate limiter. - /// - public TimeSpan Window { get; set; } = TimeSpan.FromMinutes(1); - - /// - /// Maximum number of queued requests awaiting permits. - /// - public int QueueLimit { get; set; } = 0; - - /// - /// Ordering strategy for queued requests. - /// - public QueueProcessingOrder QueueProcessingOrder { get; set; } = QueueProcessingOrder.OldestFirst; - - internal void Validate(string name) - { - if (!Enabled) - { - return; - } - - if (PermitLimit <= 0) - { - throw new InvalidOperationException($"Authority rate limiting '{name}' requires permitLimit to be greater than zero."); - } - - if (QueueLimit < 0) - { - throw new InvalidOperationException($"Authority rate limiting '{name}' queueLimit cannot be negative."); - } - - if (Window <= TimeSpan.Zero || Window > TimeSpan.FromHours(1)) - { - throw new InvalidOperationException($"Authority rate limiting '{name}' window must be greater than zero and no more than one hour."); - } - } -} - -public sealed class AuthorityStorageOptions -{ - /// - /// Mongo connection string used by Authority storage. - /// - public string ConnectionString { get; set; } = string.Empty; - - /// - /// Optional explicit database name override. - /// - public string? DatabaseName { get; set; } - - /// - /// Mongo command timeout. - /// - public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); - - internal void Validate() - { - if (string.IsNullOrWhiteSpace(ConnectionString)) - { - throw new InvalidOperationException("Authority storage requires a Mongo connection string."); - } - - if (CommandTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("Authority storage command timeout must be greater than zero."); - } - } -} - -public sealed class AuthorityBootstrapOptions -{ - /// - /// Enables or disables bootstrap administrative APIs. - /// - public bool Enabled { get; set; } = false; - - /// - /// API key required when invoking bootstrap endpoints. - /// - public string? ApiKey { get; set; } = string.Empty; - - /// - /// Default identity provider used when none is specified in bootstrap requests. - /// - public string? DefaultIdentityProvider { get; set; } = "standard"; - - internal void Validate() - { - if (!Enabled) - { - return; - } - - if (string.IsNullOrWhiteSpace(ApiKey)) - { - throw new InvalidOperationException("Authority bootstrap configuration requires an API key when enabled."); - } - - if (string.IsNullOrWhiteSpace(DefaultIdentityProvider)) - { - throw new InvalidOperationException("Authority bootstrap configuration requires a default identity provider name when enabled."); - } - } -} - -public sealed class AuthorityPluginSettings -{ - private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; - - /// - /// Directory containing per-plugin configuration manifests (relative paths resolved against application base path). - /// - public string ConfigurationDirectory { get; set; } = "../etc/authority.plugins"; - - /// - /// Declarative descriptors for Authority plugins (keyed by logical plugin name). - /// - public IDictionary Descriptors { get; } = new Dictionary(OrdinalIgnoreCase); - - internal void NormalizeAndValidate() - { - if (Descriptors.Count == 0) - { - return; - } - - foreach (var (name, descriptor) in Descriptors.ToArray()) - { - if (descriptor is null) - { - throw new InvalidOperationException($"Authority plugin descriptor '{name}' is null."); - } - - descriptor.Normalize(name); - descriptor.Validate(name); - } - } -} - -public sealed class AuthorityPluginDescriptorOptions -{ - private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; - - private readonly List capabilities = new(); - private readonly Dictionary metadata = new(OrdinalIgnoreCase); - private static readonly HashSet AllowedCapabilities = new( - new[] - { - AuthorityPluginCapabilities.Password, - AuthorityPluginCapabilities.Mfa, - AuthorityPluginCapabilities.ClientProvisioning, - AuthorityPluginCapabilities.Bootstrap - }, - OrdinalIgnoreCase); - - /// - /// Logical type identifier for the plugin (e.g. standard, ldap). - /// - public string? Type { get; set; } - - /// - /// Name of the plugin assembly (without file extension). - /// - public string? AssemblyName { get; set; } - - /// - /// Optional explicit assembly path override; relative paths resolve against plugin directories. - /// - public string? AssemblyPath { get; set; } - - /// - /// Indicates whether the plugin should be enabled. - /// - public bool Enabled { get; set; } = true; - - /// - /// Plugin capability hints surfaced to the Authority host. - /// - public IList Capabilities => capabilities; - - /// - /// Optional metadata (string key/value) passed to plugin implementations. - /// - public IDictionary Metadata => metadata; - - /// - /// Relative path to the plugin-specific configuration file (defaults to <pluginName>.yaml). - /// - public string? ConfigFile { get; set; } - - internal void Normalize(string pluginName) - { - if (string.IsNullOrWhiteSpace(ConfigFile)) - { - ConfigFile = $"{pluginName}.yaml"; - } - else - { - ConfigFile = ConfigFile.Trim(); - } - - Type = string.IsNullOrWhiteSpace(Type) ? pluginName : Type.Trim(); - - if (!string.IsNullOrWhiteSpace(AssemblyName)) - { - AssemblyName = AssemblyName.Trim(); - } - - if (!string.IsNullOrWhiteSpace(AssemblyPath)) - { - AssemblyPath = AssemblyPath.Trim(); - } - - if (capabilities.Count > 0) - { - var seen = new HashSet(OrdinalIgnoreCase); - var unique = new List(capabilities.Count); - - foreach (var entry in capabilities) - { - if (string.IsNullOrWhiteSpace(entry)) - { - continue; - } - - var canonical = entry.Trim().ToLowerInvariant(); - if (seen.Add(canonical)) - { - unique.Add(canonical); - } - } - - unique.Sort(StringComparer.Ordinal); - - capabilities.Clear(); - capabilities.AddRange(unique); - } - } - - internal void Validate(string pluginName) - { - if (string.IsNullOrWhiteSpace(AssemblyName) && string.IsNullOrWhiteSpace(AssemblyPath)) - { - throw new InvalidOperationException($"Authority plugin '{pluginName}' must define either assemblyName or assemblyPath."); - } - - if (string.IsNullOrWhiteSpace(ConfigFile)) - { - throw new InvalidOperationException($"Authority plugin '{pluginName}' must define a configFile."); - } - - if (Path.GetFileName(ConfigFile) != ConfigFile && Path.IsPathRooted(ConfigFile) && !File.Exists(ConfigFile)) - { - throw new InvalidOperationException($"Authority plugin '{pluginName}' specifies configFile '{ConfigFile}' which does not exist."); - } - - foreach (var capability in capabilities) - { - if (!AllowedCapabilities.Contains(capability)) - { - throw new InvalidOperationException($"Authority plugin '{pluginName}' declares unknown capability '{capability}'. Allowed values: password, mfa, clientProvisioning, bootstrap."); - } - } - } - - internal AuthorityPluginManifest ToManifest(string name, string configPath) - { - var capabilitiesSnapshot = capabilities.Count == 0 - ? Array.Empty() - : capabilities.ToArray(); - - var metadataSnapshot = metadata.Count == 0 - ? new Dictionary(OrdinalIgnoreCase) - : new Dictionary(metadata, OrdinalIgnoreCase); - - return new AuthorityPluginManifest( - name, - Type ?? name, - Enabled, - AssemblyName, - AssemblyPath, - capabilitiesSnapshot, - metadataSnapshot, - configPath); - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.RateLimiting; +using StellaOps.Authority.Plugins.Abstractions; +using StellaOps.Cryptography; + +namespace StellaOps.Configuration; + +/// +/// Strongly typed configuration for the StellaOps Authority service. +/// +public sealed class StellaOpsAuthorityOptions +{ + private readonly List pluginDirectories = new(); + private readonly List bypassNetworks = new(); + + /// + /// Schema version for downstream consumers to coordinate breaking changes. + /// + public int SchemaVersion { get; set; } = 1; + + /// + /// Absolute issuer URI advertised to clients (e.g. https://authority.stella-ops.local). + /// + public Uri? Issuer { get; set; } + + /// + /// Lifetime for OAuth access tokens issued by Authority. + /// + public TimeSpan AccessTokenLifetime { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Lifetime for OAuth refresh tokens issued by Authority. + /// + public TimeSpan RefreshTokenLifetime { get; set; } = TimeSpan.FromDays(30); + + /// + /// Lifetime for OpenID Connect identity tokens. + /// + public TimeSpan IdentityTokenLifetime { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Lifetime for OAuth authorization codes. + /// + public TimeSpan AuthorizationCodeLifetime { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Lifetime for OAuth device codes (device authorization flow). + /// + public TimeSpan DeviceCodeLifetime { get; set; } = TimeSpan.FromMinutes(15); + + /// + /// Directories scanned for Authority plugins (absolute or relative to application base path). + /// + public IList PluginDirectories => pluginDirectories; + + /// + /// CIDR blocks permitted to bypass certain authentication policies (e.g. on-host cron). + /// + public IList BypassNetworks => bypassNetworks; + + /// + /// Configuration describing the Authority MongoDB storage. + /// + public AuthorityStorageOptions Storage { get; } = new(); + + /// + /// Bootstrap settings for initial administrative provisioning. + /// + public AuthorityBootstrapOptions Bootstrap { get; } = new(); + + /// + /// Configuration describing available Authority plugins and their manifests. + /// + public AuthorityPluginSettings Plugins { get; } = new(); + + /// + /// Security-related configuration for the Authority host. + /// + public AuthoritySecurityOptions Security { get; } = new(); + + /// + /// Signing options for Authority-generated artefacts (revocation bundles, JWKS). + /// + public AuthoritySigningOptions Signing { get; } = new(); + + /// + /// Validates configured values and normalises collections. + /// + /// Thrown when configuration is invalid. + public void Validate() + { + if (SchemaVersion <= 0) + { + throw new InvalidOperationException("Authority configuration requires a positive schemaVersion."); + } + + if (Issuer is null) + { + throw new InvalidOperationException("Authority configuration requires an issuer URL."); + } + + if (!Issuer.IsAbsoluteUri) + { + throw new InvalidOperationException("Authority issuer must be an absolute URI."); + } + + if (string.Equals(Issuer.Scheme, Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) && !Issuer.IsLoopback) + { + throw new InvalidOperationException("Authority issuer must use HTTPS unless running on a loopback interface."); + } + + ValidateLifetime(AccessTokenLifetime, nameof(AccessTokenLifetime), TimeSpan.FromHours(24)); + ValidateLifetime(RefreshTokenLifetime, nameof(RefreshTokenLifetime), TimeSpan.FromDays(365)); + ValidateLifetime(IdentityTokenLifetime, nameof(IdentityTokenLifetime), TimeSpan.FromHours(24)); + ValidateLifetime(AuthorizationCodeLifetime, nameof(AuthorizationCodeLifetime), TimeSpan.FromHours(1)); + ValidateLifetime(DeviceCodeLifetime, nameof(DeviceCodeLifetime), TimeSpan.FromHours(24)); + + NormaliseList(pluginDirectories); + NormaliseList(bypassNetworks); + + Security.Validate(); + Signing.Validate(); + Plugins.NormalizeAndValidate(); + Storage.Validate(); + Bootstrap.Validate(); + } + + private static void ValidateLifetime(TimeSpan value, string propertyName, TimeSpan maximum) + { + if (value <= TimeSpan.Zero) + { + throw new InvalidOperationException($"Authority configuration requires {propertyName} to be greater than zero."); + } + + if (value > maximum) + { + throw new InvalidOperationException($"Authority configuration requires {propertyName} to be less than or equal to {maximum}."); + } + } + + private static void NormaliseList(IList values) + { + if (values.Count == 0) + { + return; + } + + var unique = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var index = values.Count - 1; index >= 0; index--) + { + var entry = values[index]; + + if (string.IsNullOrWhiteSpace(entry)) + { + values.RemoveAt(index); + continue; + } + + var trimmed = entry.Trim(); + if (!unique.Add(trimmed)) + { + values.RemoveAt(index); + continue; + } + + values[index] = trimmed; + } + } +} + +public sealed class AuthoritySecurityOptions +{ + /// + /// Rate limiting configuration applied to Authority endpoints. + /// + public AuthorityRateLimitingOptions RateLimiting { get; } = new(); + + /// + /// Default password hashing parameters advertised to Authority plug-ins. + /// + public PasswordHashOptions PasswordHashing { get; } = new(); + + /// + /// Sender-constraint configuration (DPoP, mTLS). + /// + public AuthoritySenderConstraintOptions SenderConstraints { get; } = new(); + + internal void Validate() + { + RateLimiting.Validate(); + PasswordHashing.Validate(); + SenderConstraints.Validate(); + } +} + +public sealed class AuthorityRateLimitingOptions +{ + public AuthorityRateLimitingOptions() + { + Token = new AuthorityEndpointRateLimitOptions + { + PermitLimit = 30, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 0 + }; + + Authorize = new AuthorityEndpointRateLimitOptions + { + PermitLimit = 60, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 10 + }; + + Internal = new AuthorityEndpointRateLimitOptions + { + Enabled = false, + PermitLimit = 5, + Window = TimeSpan.FromMinutes(1), + QueueLimit = 0 + }; + } + + /// + /// Rate limiting configuration applied to the /token endpoint. + /// + public AuthorityEndpointRateLimitOptions Token { get; } + + /// + /// Rate limiting configuration applied to the /authorize endpoint. + /// + public AuthorityEndpointRateLimitOptions Authorize { get; } + + /// + /// Rate limiting configuration applied to /internal endpoints. + /// + public AuthorityEndpointRateLimitOptions Internal { get; } + + internal void Validate() + { + Token.Validate(nameof(Token)); + Authorize.Validate(nameof(Authorize)); + Internal.Validate(nameof(Internal)); + } +} + +public sealed class AuthoritySenderConstraintOptions +{ + public AuthoritySenderConstraintOptions() + { + Dpop = new AuthorityDpopOptions(); + Mtls = new AuthorityMtlsOptions(); + } + + public AuthorityDpopOptions Dpop { get; } + + public AuthorityMtlsOptions Mtls { get; } + + internal void Validate() + { + Dpop.Validate(); + Mtls.Validate(); + } +} + +public sealed class AuthorityDpopOptions +{ + private readonly HashSet allowedAlgorithms = new(StringComparer.OrdinalIgnoreCase) + { + "ES256", + "ES384" + }; + + public bool Enabled { get; set; } + + public TimeSpan ProofLifetime { get; set; } = TimeSpan.FromMinutes(2); + + public TimeSpan AllowedClockSkew { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan ReplayWindow { get; set; } = TimeSpan.FromMinutes(5); + + public ISet AllowedAlgorithms => allowedAlgorithms; + + public IReadOnlySet NormalizedAlgorithms { get; private set; } = new HashSet(StringComparer.Ordinal); + + public AuthorityDpopNonceOptions Nonce { get; } = new(); + + internal void Validate() + { + if (ProofLifetime <= TimeSpan.Zero) + { + throw new InvalidOperationException("Dpop.ProofLifetime must be greater than zero."); + } + + if (AllowedClockSkew < TimeSpan.Zero || AllowedClockSkew > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException("Dpop.AllowedClockSkew must be between 0 and 5 minutes."); + } + + if (ReplayWindow < TimeSpan.Zero) + { + throw new InvalidOperationException("Dpop.ReplayWindow must be greater than or equal to zero."); + } + + if (allowedAlgorithms.Count == 0) + { + throw new InvalidOperationException("At least one DPoP algorithm must be configured."); + } + + NormalizedAlgorithms = allowedAlgorithms + .Select(static alg => alg.Trim().ToUpperInvariant()) + .Where(static alg => alg.Length > 0) + .ToHashSet(StringComparer.Ordinal); + + if (NormalizedAlgorithms.Count == 0) + { + throw new InvalidOperationException("Allowed DPoP algorithms cannot be empty after normalization."); + } + + Nonce.Validate(); + } +} + +public sealed class AuthorityDpopNonceOptions +{ + private readonly HashSet requiredAudiences = new(StringComparer.OrdinalIgnoreCase) + { + "signer", + "attestor" + }; + + public bool Enabled { get; set; } = true; + + public TimeSpan Ttl { get; set; } = TimeSpan.FromMinutes(10); + + public int MaxIssuancePerMinute { get; set; } = 120; + + public string Store { get; set; } = "memory"; + + public string? RedisConnectionString { get; set; } + + public ISet RequiredAudiences => requiredAudiences; + + public IReadOnlySet NormalizedAudiences { get; private set; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + internal void Validate() + { + if (Ttl <= TimeSpan.Zero) + { + throw new InvalidOperationException("Dpop.Nonce.Ttl must be greater than zero."); + } + + if (MaxIssuancePerMinute < 1) + { + throw new InvalidOperationException("Dpop.Nonce.MaxIssuancePerMinute must be at least 1."); + } + + if (string.IsNullOrWhiteSpace(Store)) + { + throw new InvalidOperationException("Dpop.Nonce.Store must be specified."); + } + + Store = Store.Trim().ToLowerInvariant(); + + if (Store is not ("memory" or "redis")) + { + throw new InvalidOperationException("Dpop.Nonce.Store must be either 'memory' or 'redis'."); + } + + if (Store == "redis" && string.IsNullOrWhiteSpace(RedisConnectionString)) + { + throw new InvalidOperationException("Dpop.Nonce.RedisConnectionString must be provided when using the 'redis' store."); + } + + NormalizedAudiences = requiredAudiences + .Select(static aud => aud.Trim()) + .Where(static aud => aud.Length > 0) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (NormalizedAudiences.Count == 0) + { + throw new InvalidOperationException("Dpop.Nonce.RequiredAudiences must include at least one audience."); + } + } +} + +public sealed class AuthorityMtlsOptions +{ + private readonly HashSet enforceForAudiences = new(StringComparer.OrdinalIgnoreCase) + { + "signer" + }; + + private readonly HashSet allowedSanTypes = new(StringComparer.OrdinalIgnoreCase) + { + "dns", + "uri" + }; + + public bool Enabled { get; set; } + + public bool RequireChainValidation { get; set; } = true; + + public TimeSpan RotationGrace { get; set; } = TimeSpan.FromMinutes(15); + + public ISet EnforceForAudiences => enforceForAudiences; + + public IReadOnlySet NormalizedAudiences { get; private set; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + public IList AllowedCertificateAuthorities { get; } = new List(); + + public IList AllowedSubjectPatterns { get; } = new List(); + + public ISet AllowedSanTypes => allowedSanTypes; + + public IReadOnlyList NormalizedSubjectPatterns { get; private set; } = Array.Empty(); + + public IReadOnlySet NormalizedSanTypes { get; private set; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + internal void Validate() + { + if (RotationGrace < TimeSpan.Zero) + { + throw new InvalidOperationException("Mtls.RotationGrace must be non-negative."); + } + + NormalizedAudiences = enforceForAudiences + .Select(static aud => aud.Trim()) + .Where(static aud => aud.Length > 0) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (Enabled && NormalizedAudiences.Count == 0) + { + throw new InvalidOperationException("Mtls.EnforceForAudiences must include at least one audience when enabled."); + } + + if (AllowedCertificateAuthorities.Any(static path => string.IsNullOrWhiteSpace(path))) + { + throw new InvalidOperationException("Mtls.AllowedCertificateAuthorities entries must not be empty."); + } + + NormalizedSanTypes = allowedSanTypes + .Select(static value => value.Trim()) + .Where(static value => value.Length > 0) + .Select(static value => value.ToLowerInvariant()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + if (Enabled && NormalizedSanTypes.Count == 0) + { + throw new InvalidOperationException("Mtls.AllowedSanTypes must include at least one entry when enabled."); + } + + var compiledPatterns = new List(AllowedSubjectPatterns.Count); + + foreach (var pattern in AllowedSubjectPatterns) + { + if (string.IsNullOrWhiteSpace(pattern)) + { + throw new InvalidOperationException("Mtls.AllowedSubjectPatterns entries must not be empty."); + } + + try + { + compiledPatterns.Add(new Regex(pattern, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase | RegexOptions.Compiled, TimeSpan.FromMilliseconds(100))); + } + catch (RegexParseException ex) + { + throw new InvalidOperationException($"Mtls.AllowedSubjectPatterns entry '{pattern}' is not a valid regular expression.", ex); + } + } + + NormalizedSubjectPatterns = compiledPatterns; + } +} + +public sealed class AuthorityEndpointRateLimitOptions +{ + /// + /// Gets or sets a value indicating whether rate limiting is enabled for the endpoint. + /// + public bool Enabled { get; set; } = true; + + /// + /// Maximum number of requests allowed within the configured window. + /// + public int PermitLimit { get; set; } = 60; + + /// + /// Size of the fixed window applied to the rate limiter. + /// + public TimeSpan Window { get; set; } = TimeSpan.FromMinutes(1); + + /// + /// Maximum number of queued requests awaiting permits. + /// + public int QueueLimit { get; set; } = 0; + + /// + /// Ordering strategy for queued requests. + /// + public QueueProcessingOrder QueueProcessingOrder { get; set; } = QueueProcessingOrder.OldestFirst; + + internal void Validate(string name) + { + if (!Enabled) + { + return; + } + + if (PermitLimit <= 0) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' requires permitLimit to be greater than zero."); + } + + if (QueueLimit < 0) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' queueLimit cannot be negative."); + } + + if (Window <= TimeSpan.Zero || Window > TimeSpan.FromHours(1)) + { + throw new InvalidOperationException($"Authority rate limiting '{name}' window must be greater than zero and no more than one hour."); + } + } +} + +public sealed class AuthorityStorageOptions +{ + /// + /// Mongo connection string used by Authority storage. + /// + public string ConnectionString { get; set; } = string.Empty; + + /// + /// Optional explicit database name override. + /// + public string? DatabaseName { get; set; } + + /// + /// Mongo command timeout. + /// + public TimeSpan CommandTimeout { get; set; } = TimeSpan.FromSeconds(30); + + internal void Validate() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Authority storage requires a Mongo connection string."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Authority storage command timeout must be greater than zero."); + } + } +} + +public sealed class AuthorityBootstrapOptions +{ + /// + /// Enables or disables bootstrap administrative APIs. + /// + public bool Enabled { get; set; } = false; + + /// + /// API key required when invoking bootstrap endpoints. + /// + public string? ApiKey { get; set; } = string.Empty; + + /// + /// Default identity provider used when none is specified in bootstrap requests. + /// + public string? DefaultIdentityProvider { get; set; } = "standard"; + + internal void Validate() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(ApiKey)) + { + throw new InvalidOperationException("Authority bootstrap configuration requires an API key when enabled."); + } + + if (string.IsNullOrWhiteSpace(DefaultIdentityProvider)) + { + throw new InvalidOperationException("Authority bootstrap configuration requires a default identity provider name when enabled."); + } + } +} + +public sealed class AuthorityPluginSettings +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + /// + /// Directory containing per-plugin configuration manifests (relative paths resolved against application base path). + /// + public string ConfigurationDirectory { get; set; } = "../etc/authority.plugins"; + + /// + /// Declarative descriptors for Authority plugins (keyed by logical plugin name). + /// + public IDictionary Descriptors { get; } = new Dictionary(OrdinalIgnoreCase); + + internal void NormalizeAndValidate() + { + if (Descriptors.Count == 0) + { + return; + } + + foreach (var (name, descriptor) in Descriptors.ToArray()) + { + if (descriptor is null) + { + throw new InvalidOperationException($"Authority plugin descriptor '{name}' is null."); + } + + descriptor.Normalize(name); + descriptor.Validate(name); + } + } +} + +public sealed class AuthorityPluginDescriptorOptions +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + private readonly List capabilities = new(); + private readonly Dictionary metadata = new(OrdinalIgnoreCase); + private static readonly HashSet AllowedCapabilities = new( + new[] + { + AuthorityPluginCapabilities.Password, + AuthorityPluginCapabilities.Mfa, + AuthorityPluginCapabilities.ClientProvisioning, + AuthorityPluginCapabilities.Bootstrap + }, + OrdinalIgnoreCase); + + /// + /// Logical type identifier for the plugin (e.g. standard, ldap). + /// + public string? Type { get; set; } + + /// + /// Name of the plugin assembly (without file extension). + /// + public string? AssemblyName { get; set; } + + /// + /// Optional explicit assembly path override; relative paths resolve against plugin directories. + /// + public string? AssemblyPath { get; set; } + + /// + /// Indicates whether the plugin should be enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Plugin capability hints surfaced to the Authority host. + /// + public IList Capabilities => capabilities; + + /// + /// Optional metadata (string key/value) passed to plugin implementations. + /// + public IDictionary Metadata => metadata; + + /// + /// Relative path to the plugin-specific configuration file (defaults to <pluginName>.yaml). + /// + public string? ConfigFile { get; set; } + + internal void Normalize(string pluginName) + { + if (string.IsNullOrWhiteSpace(ConfigFile)) + { + ConfigFile = $"{pluginName}.yaml"; + } + else + { + ConfigFile = ConfigFile.Trim(); + } + + Type = string.IsNullOrWhiteSpace(Type) ? pluginName : Type.Trim(); + + if (!string.IsNullOrWhiteSpace(AssemblyName)) + { + AssemblyName = AssemblyName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(AssemblyPath)) + { + AssemblyPath = AssemblyPath.Trim(); + } + + if (capabilities.Count > 0) + { + var seen = new HashSet(OrdinalIgnoreCase); + var unique = new List(capabilities.Count); + + foreach (var entry in capabilities) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + var canonical = entry.Trim().ToLowerInvariant(); + if (seen.Add(canonical)) + { + unique.Add(canonical); + } + } + + unique.Sort(StringComparer.Ordinal); + + capabilities.Clear(); + capabilities.AddRange(unique); + } + } + + internal void Validate(string pluginName) + { + if (string.IsNullOrWhiteSpace(AssemblyName) && string.IsNullOrWhiteSpace(AssemblyPath)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' must define either assemblyName or assemblyPath."); + } + + if (string.IsNullOrWhiteSpace(ConfigFile)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' must define a configFile."); + } + + if (Path.GetFileName(ConfigFile) != ConfigFile && Path.IsPathRooted(ConfigFile) && !File.Exists(ConfigFile)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' specifies configFile '{ConfigFile}' which does not exist."); + } + + foreach (var capability in capabilities) + { + if (!AllowedCapabilities.Contains(capability)) + { + throw new InvalidOperationException($"Authority plugin '{pluginName}' declares unknown capability '{capability}'. Allowed values: password, mfa, clientProvisioning, bootstrap."); + } + } + } + + internal AuthorityPluginManifest ToManifest(string name, string configPath) + { + var capabilitiesSnapshot = capabilities.Count == 0 + ? Array.Empty() + : capabilities.ToArray(); + + var metadataSnapshot = metadata.Count == 0 + ? new Dictionary(OrdinalIgnoreCase) + : new Dictionary(metadata, OrdinalIgnoreCase); + + return new AuthorityPluginManifest( + name, + Type ?? name, + Enabled, + AssemblyName, + AssemblyPath, + capabilitiesSnapshot, + metadataSnapshot, + configPath); + } +} diff --git a/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleCryptoServiceCollectionExtensions.cs b/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleCryptoServiceCollectionExtensions.cs new file mode 100644 index 00000000..525ad865 --- /dev/null +++ b/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleCryptoServiceCollectionExtensions.cs @@ -0,0 +1,21 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Cryptography; + +namespace StellaOps.Cryptography.Plugin.BouncyCastle; + +/// +/// Dependency injection helpers for registering the BouncyCastle Ed25519 crypto provider. +/// +public static class BouncyCastleCryptoServiceCollectionExtensions +{ + public static IServiceCollection AddBouncyCastleEd25519Provider(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + + return services; + } +} diff --git a/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleEd25519CryptoProvider.cs b/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleEd25519CryptoProvider.cs new file mode 100644 index 00000000..1b91f5dd --- /dev/null +++ b/src/StellaOps.Cryptography.Plugin.BouncyCastle/BouncyCastleEd25519CryptoProvider.cs @@ -0,0 +1,211 @@ +using System.Collections.Concurrent; +using Microsoft.IdentityModel.Tokens; +using Org.BouncyCastle.Crypto.Parameters; +using Org.BouncyCastle.Crypto.Signers; +using StellaOps.Cryptography; + +namespace StellaOps.Cryptography.Plugin.BouncyCastle; + +/// +/// Ed25519 signing provider backed by BouncyCastle primitives. +/// +public sealed class BouncyCastleEd25519CryptoProvider : ICryptoProvider +{ + private static readonly HashSet SupportedAlgorithms = new(StringComparer.OrdinalIgnoreCase) + { + SignatureAlgorithms.Ed25519, + SignatureAlgorithms.EdDsa + }; + + private static readonly string[] DefaultKeyOps = { "sign", "verify" }; + + private readonly ConcurrentDictionary signingKeys = new(StringComparer.Ordinal); + + public string Name => "bouncycastle.ed25519"; + + public bool Supports(CryptoCapability capability, string algorithmId) + { + if (string.IsNullOrWhiteSpace(algorithmId)) + { + return false; + } + + return capability switch + { + CryptoCapability.Signing or CryptoCapability.Verification => SupportedAlgorithms.Contains(algorithmId), + _ => false + }; + } + + public IPasswordHasher GetPasswordHasher(string algorithmId) + => throw new NotSupportedException("BouncyCastle provider does not expose password hashing capabilities."); + + public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference) + { + ArgumentException.ThrowIfNullOrWhiteSpace(algorithmId); + ArgumentNullException.ThrowIfNull(keyReference); + + if (!signingKeys.TryGetValue(keyReference.KeyId, out var entry)) + { + throw new KeyNotFoundException($"Signing key '{keyReference.KeyId}' is not registered with provider '{Name}'."); + } + + EnsureAlgorithmSupported(algorithmId); + var normalized = NormalizeAlgorithm(algorithmId); + if (!string.Equals(entry.Descriptor.AlgorithmId, normalized, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Signing key '{keyReference.KeyId}' is registered for algorithm '{entry.Descriptor.AlgorithmId}', not '{algorithmId}'."); + } + + return new Ed25519SignerWrapper(entry); + } + + public void UpsertSigningKey(CryptoSigningKey signingKey) + { + ArgumentNullException.ThrowIfNull(signingKey); + EnsureAlgorithmSupported(signingKey.AlgorithmId); + + if (signingKey.Kind != CryptoSigningKeyKind.Raw) + { + throw new InvalidOperationException($"Provider '{Name}' requires raw Ed25519 private key material."); + } + + var privateKey = NormalizePrivateKey(signingKey.PrivateKey); + var publicKey = NormalizePublicKey(signingKey.PublicKey, privateKey); + + var privateKeyParameters = new Ed25519PrivateKeyParameters(privateKey, 0); + var publicKeyParameters = new Ed25519PublicKeyParameters(publicKey, 0); + + var descriptor = new CryptoSigningKey( + signingKey.Reference, + NormalizeAlgorithm(signingKey.AlgorithmId), + privateKey, + signingKey.CreatedAt, + signingKey.ExpiresAt, + publicKey, + signingKey.Metadata); + + signingKeys.AddOrUpdate( + signingKey.Reference.KeyId, + _ => new KeyEntry(descriptor, privateKeyParameters, publicKeyParameters), + (_, _) => new KeyEntry(descriptor, privateKeyParameters, publicKeyParameters)); + } + + public bool RemoveSigningKey(string keyId) + { + if (string.IsNullOrWhiteSpace(keyId)) + { + return false; + } + + return signingKeys.TryRemove(keyId, out _); + } + + public IReadOnlyCollection GetSigningKeys() + => signingKeys.Values.Select(static entry => entry.Descriptor).ToArray(); + + private static void EnsureAlgorithmSupported(string algorithmId) + { + if (!SupportedAlgorithms.Contains(algorithmId)) + { + throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider 'bouncycastle.ed25519'."); + } + } + + private static string NormalizeAlgorithm(string algorithmId) + => string.Equals(algorithmId, SignatureAlgorithms.EdDsa, StringComparison.OrdinalIgnoreCase) + ? SignatureAlgorithms.Ed25519 + : SignatureAlgorithms.Ed25519; + + private static byte[] NormalizePrivateKey(ReadOnlyMemory privateKey) + { + var span = privateKey.Span; + return span.Length switch + { + 32 => span.ToArray(), + 64 => span[..32].ToArray(), + _ => throw new InvalidOperationException("Ed25519 private key must be 32 or 64 bytes.") + }; + } + + private static byte[] NormalizePublicKey(ReadOnlyMemory publicKey, byte[] privateKey) + { + if (publicKey.IsEmpty) + { + var privateParams = new Ed25519PrivateKeyParameters(privateKey, 0); + return privateParams.GeneratePublicKey().GetEncoded(); + } + + if (publicKey.Span.Length != 32) + { + throw new InvalidOperationException("Ed25519 public key must be 32 bytes."); + } + + return publicKey.ToArray(); + } + + private sealed record KeyEntry( + CryptoSigningKey Descriptor, + Ed25519PrivateKeyParameters PrivateKey, + Ed25519PublicKeyParameters PublicKey); + + private sealed class Ed25519SignerWrapper : ICryptoSigner + { + private readonly KeyEntry entry; + + public Ed25519SignerWrapper(KeyEntry entry) + { + this.entry = entry ?? throw new ArgumentNullException(nameof(entry)); + } + + public string KeyId => entry.Descriptor.Reference.KeyId; + + public string AlgorithmId => entry.Descriptor.AlgorithmId; + + public ValueTask SignAsync(ReadOnlyMemory data, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var signer = new Ed25519Signer(); + var buffer = data.ToArray(); + signer.Init(true, entry.PrivateKey); + signer.BlockUpdate(buffer, 0, buffer.Length); + var signature = signer.GenerateSignature(); + return ValueTask.FromResult(signature); + } + + public ValueTask VerifyAsync(ReadOnlyMemory data, ReadOnlyMemory signature, CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var verifier = new Ed25519Signer(); + var buffer = data.ToArray(); + verifier.Init(false, entry.PublicKey); + verifier.BlockUpdate(buffer, 0, buffer.Length); + var verified = verifier.VerifySignature(signature.ToArray()); + return ValueTask.FromResult(verified); + } + + public JsonWebKey ExportPublicJsonWebKey() + { + var jwk = new JsonWebKey + { + Kid = entry.Descriptor.Reference.KeyId, + Alg = SignatureAlgorithms.EdDsa, + Kty = "OKP", + Use = JsonWebKeyUseNames.Sig, + Crv = "Ed25519" + }; + + foreach (var op in DefaultKeyOps) + { + jwk.KeyOps.Add(op); + } + + jwk.X = Base64UrlEncoder.Encode(entry.PublicKey.GetEncoded()); + + return jwk; + } + } +} diff --git a/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj b/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj new file mode 100644 index 00000000..f0180975 --- /dev/null +++ b/src/StellaOps.Cryptography.Plugin.BouncyCastle/StellaOps.Cryptography.Plugin.BouncyCastle.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Cryptography.Tests/BouncyCastleEd25519CryptoProviderTests.cs b/src/StellaOps.Cryptography.Tests/BouncyCastleEd25519CryptoProviderTests.cs new file mode 100644 index 00000000..b91e6748 --- /dev/null +++ b/src/StellaOps.Cryptography.Tests/BouncyCastleEd25519CryptoProviderTests.cs @@ -0,0 +1,52 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Cryptography; +using StellaOps.Cryptography.DependencyInjection; +using StellaOps.Cryptography.Plugin.BouncyCastle; +using Xunit; + +namespace StellaOps.Cryptography.Tests; + +public sealed class BouncyCastleEd25519CryptoProviderTests +{ + [Fact] + public async Task SignAndVerify_WithBouncyCastleProvider_Succeeds() + { + var services = new ServiceCollection(); + services.AddStellaOpsCrypto(); + services.AddBouncyCastleEd25519Provider(); + + using var provider = services.BuildServiceProvider(); + var registry = provider.GetRequiredService(); + var bcProvider = provider.GetServices() + .OfType() + .Single(); + + var keyId = "ed25519-unit-test"; + var privateKeyBytes = Enumerable.Range(0, 32).Select(i => (byte)(i + 1)).ToArray(); + var keyReference = new CryptoKeyReference(keyId, bcProvider.Name); + var signingKey = new CryptoSigningKey( + keyReference, + SignatureAlgorithms.Ed25519, + privateKeyBytes, + createdAt: DateTimeOffset.UtcNow); + + bcProvider.UpsertSigningKey(signingKey); + + var resolution = registry.ResolveSigner( + CryptoCapability.Signing, + SignatureAlgorithms.Ed25519, + keyReference, + bcProvider.Name); + + var payload = new byte[] { 0x01, 0x02, 0x03, 0x04 }; + var signature = await resolution.Signer.SignAsync(payload); + + Assert.True(await resolution.Signer.VerifyAsync(payload, signature)); + + var jwk = resolution.Signer.ExportPublicJsonWebKey(); + Assert.Equal("OKP", jwk.Kty); + Assert.Equal("Ed25519", jwk.Crv); + Assert.Equal(SignatureAlgorithms.EdDsa, jwk.Alg); + Assert.Equal(keyId, jwk.Kid); + } +} diff --git a/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj b/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj index 67024af6..3a92e15c 100644 --- a/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj +++ b/src/StellaOps.Cryptography.Tests/StellaOps.Cryptography.Tests.csproj @@ -1,14 +1,16 @@ - - - net10.0 - enable - enable - false - - - $(DefineConstants);STELLAOPS_CRYPTO_SODIUM - - - - - + + + net10.0 + enable + enable + false + + + $(DefineConstants);STELLAOPS_CRYPTO_SODIUM + + + + + + + diff --git a/src/StellaOps.Cryptography/AGENTS.md b/src/StellaOps.Cryptography/AGENTS.md index b1124feb..d1ba83d9 100644 --- a/src/StellaOps.Cryptography/AGENTS.md +++ b/src/StellaOps.Cryptography/AGENTS.md @@ -1,21 +1,22 @@ -# Team 8 — Security Guild (Authority & Shared Crypto) - -## Role - -Team 8 owns the end-to-end security posture for StellaOps Authority and its consumers. That includes password hashing policy, audit/event hygiene, rate-limit & lockout rules, revocation distribution, and sovereign cryptography abstractions that allow alternative algorithm suites (e.g., GOST) without touching feature code. - -## Operational Boundaries - -- Primary workspace: `src/StellaOps.Cryptography`, `src/StellaOps.Authority.Plugin.Standard`, `src/StellaOps.Authority.Storage.Mongo`, and Authority host (`src/StellaOps.Authority/StellaOps.Authority`). -- Coordinate cross-module changes via TASKS.md updates and PR descriptions. -- Never bypass deterministic behaviour (sorted keys, stable timestamps). -- Tests live alongside owning projects (`*.Tests`). Extend goldens instead of rewriting. - -## Expectations - -- Default to Argon2id (Konscious) for password hashing; PBKDF2 only for legacy verification with transparent rehash on success. -- Emit structured security events with minimal PII and clear correlation IDs. -- Rate-limit `/token` and bootstrap endpoints once CORE8 hooks are available. -- Deliver offline revocation bundles signed with detached JWS and provide a verification script. -- Maintain `docs/security/authority-threat-model.md` and ensure mitigations are tracked. -- All crypto consumption flows through `StellaOps.Cryptography` abstractions to enable sovereign crypto providers. +# Team 8 — Security Guild (Authority & Shared Crypto) + +## Role + +Team 8 owns the end-to-end security posture for StellaOps Authority and its consumers. That includes password hashing policy, audit/event hygiene, rate-limit & lockout rules, revocation distribution, and sovereign cryptography abstractions that allow alternative algorithm suites (e.g., GOST) without touching feature code. + +## Operational Boundaries + +- Primary workspace: `src/StellaOps.Cryptography`, `src/StellaOps.Authority.Plugin.Standard`, `src/StellaOps.Authority.Storage.Mongo`, and Authority host (`src/StellaOps.Authority/StellaOps.Authority`). +- Coordinate cross-module changes via TASKS.md updates and PR descriptions. +- Never bypass deterministic behaviour (sorted keys, stable timestamps). +- Tests live alongside owning projects (`*.Tests`). Extend goldens instead of rewriting. + +## Expectations + +- Default to Argon2id (Konscious) for password hashing; PBKDF2 only for legacy verification with transparent rehash on success. +- Emit structured security events with minimal PII and clear correlation IDs. +- Rate-limit `/token` and bootstrap endpoints once CORE8 hooks are available. +- Deliver offline revocation bundles signed with detached JWS and provide a verification script. +- Maintain `docs/security/authority-threat-model.md` and ensure mitigations are tracked. +- All crypto consumption flows through `StellaOps.Cryptography` abstractions to enable sovereign crypto providers. +- Every new cryptographic algorithm, dependency, or acceleration path ships as an `ICryptoProvider` plug-in under `StellaOps.Cryptography.*`; feature code must never bind directly to third-party crypto libraries. diff --git a/src/StellaOps.Cryptography/CryptoSigningKey.cs b/src/StellaOps.Cryptography/CryptoSigningKey.cs index 13d3fe27..aa1e98e4 100644 --- a/src/StellaOps.Cryptography/CryptoSigningKey.cs +++ b/src/StellaOps.Cryptography/CryptoSigningKey.cs @@ -1,105 +1,176 @@ -using System; -using System.Collections.Generic; -using System.Collections.ObjectModel; -using System.Linq; -using System.Security.Cryptography; - -namespace StellaOps.Cryptography; - -/// -/// Represents asymmetric signing key material managed by a crypto provider. -/// -public sealed class CryptoSigningKey -{ - private static readonly ReadOnlyDictionary EmptyMetadata = - new(new Dictionary(StringComparer.OrdinalIgnoreCase)); - - public CryptoSigningKey( - CryptoKeyReference reference, - string algorithmId, - in ECParameters privateParameters, - DateTimeOffset createdAt, - DateTimeOffset? expiresAt = null, - IReadOnlyDictionary? metadata = null) - { - Reference = reference ?? throw new ArgumentNullException(nameof(reference)); - - if (string.IsNullOrWhiteSpace(algorithmId)) - { - throw new ArgumentException("Algorithm identifier is required.", nameof(algorithmId)); - } - - if (privateParameters.D is null || privateParameters.D.Length == 0) - { - throw new ArgumentException("Private key parameters must include the scalar component.", nameof(privateParameters)); - } - - AlgorithmId = algorithmId; - CreatedAt = createdAt; - ExpiresAt = expiresAt; - - PrivateParameters = CloneParameters(privateParameters, includePrivate: true); - PublicParameters = CloneParameters(privateParameters, includePrivate: false); - Metadata = metadata is null - ? EmptyMetadata - : new ReadOnlyDictionary(metadata.ToDictionary( - static pair => pair.Key, - static pair => pair.Value, - StringComparer.OrdinalIgnoreCase)); - } - - /// - /// Gets the key reference (id + provider hint). - /// - public CryptoKeyReference Reference { get; } - - /// - /// Gets the algorithm identifier (e.g., ES256). - /// - public string AlgorithmId { get; } - - /// - /// Gets the private EC parameters (cloned). - /// - public ECParameters PrivateParameters { get; } - - /// - /// Gets the public EC parameters (cloned, no private scalar). - /// - public ECParameters PublicParameters { get; } - - /// - /// Gets the timestamp when the key was created/imported. - /// - public DateTimeOffset CreatedAt { get; } - - /// - /// Gets the optional expiry timestamp for the key. - /// - public DateTimeOffset? ExpiresAt { get; } - - /// - /// Gets arbitrary metadata entries associated with the key. - /// - public IReadOnlyDictionary Metadata { get; } - - private static ECParameters CloneParameters(ECParameters source, bool includePrivate) - { - var clone = new ECParameters - { - Curve = source.Curve, - Q = new ECPoint - { - X = source.Q.X is null ? null : (byte[])source.Q.X.Clone(), - Y = source.Q.Y is null ? null : (byte[])source.Q.Y.Clone() - } - }; - - if (includePrivate && source.D is not null) - { - clone.D = (byte[])source.D.Clone(); - } - - return clone; - } -} +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Security.Cryptography; + +namespace StellaOps.Cryptography; + +/// +/// Describes the underlying key material for a . +/// +public enum CryptoSigningKeyKind +{ + Ec, + Raw +} + +/// +/// Represents asymmetric signing key material managed by a crypto provider. +/// +public sealed class CryptoSigningKey +{ + private static readonly ReadOnlyDictionary EmptyMetadata = + new(new Dictionary(StringComparer.OrdinalIgnoreCase)); + private static readonly byte[] EmptyKey = Array.Empty(); + + private readonly byte[] privateKeyBytes; + private readonly byte[] publicKeyBytes; + + public CryptoSigningKey( + CryptoKeyReference reference, + string algorithmId, + in ECParameters privateParameters, + DateTimeOffset createdAt, + DateTimeOffset? expiresAt = null, + IReadOnlyDictionary? metadata = null) + { + Reference = reference ?? throw new ArgumentNullException(nameof(reference)); + + if (string.IsNullOrWhiteSpace(algorithmId)) + { + throw new ArgumentException("Algorithm identifier is required.", nameof(algorithmId)); + } + + if (privateParameters.D is null || privateParameters.D.Length == 0) + { + throw new ArgumentException("Private key parameters must include the scalar component.", nameof(privateParameters)); + } + + AlgorithmId = algorithmId; + CreatedAt = createdAt; + ExpiresAt = expiresAt; + Kind = CryptoSigningKeyKind.Ec; + + privateKeyBytes = EmptyKey; + publicKeyBytes = EmptyKey; + + PrivateParameters = CloneParameters(privateParameters, includePrivate: true); + PublicParameters = CloneParameters(privateParameters, includePrivate: false); + Metadata = metadata is null + ? EmptyMetadata + : new ReadOnlyDictionary(metadata.ToDictionary( + static pair => pair.Key, + static pair => pair.Value, + StringComparer.OrdinalIgnoreCase)); + } + + public CryptoSigningKey( + CryptoKeyReference reference, + string algorithmId, + ReadOnlyMemory privateKey, + DateTimeOffset createdAt, + DateTimeOffset? expiresAt = null, + ReadOnlyMemory publicKey = default, + IReadOnlyDictionary? metadata = null) + { + Reference = reference ?? throw new ArgumentNullException(nameof(reference)); + + if (string.IsNullOrWhiteSpace(algorithmId)) + { + throw new ArgumentException("Algorithm identifier is required.", nameof(algorithmId)); + } + + if (privateKey.IsEmpty) + { + throw new ArgumentException("Private key material must be provided.", nameof(privateKey)); + } + + AlgorithmId = algorithmId; + CreatedAt = createdAt; + ExpiresAt = expiresAt; + Kind = CryptoSigningKeyKind.Raw; + + privateKeyBytes = privateKey.ToArray(); + publicKeyBytes = publicKey.IsEmpty ? EmptyKey : publicKey.ToArray(); + + PrivateParameters = default; + PublicParameters = default; + Metadata = metadata is null + ? EmptyMetadata + : new ReadOnlyDictionary(metadata.ToDictionary( + static pair => pair.Key, + static pair => pair.Value, + StringComparer.OrdinalIgnoreCase)); + } + + /// + /// Gets the key reference (id + provider hint). + /// + public CryptoKeyReference Reference { get; } + + /// + /// Gets the algorithm identifier (e.g., ES256). + /// + public string AlgorithmId { get; } + + /// + /// Gets the private EC parameters (cloned). + /// + public ECParameters PrivateParameters { get; } + + /// + /// Gets the public EC parameters (cloned, no private scalar). + /// + public ECParameters PublicParameters { get; } + + /// + /// Indicates the underlying key material representation. + /// + public CryptoSigningKeyKind Kind { get; } + + /// + /// Gets the raw private key bytes when available (empty for EC-backed keys). + /// + public ReadOnlyMemory PrivateKey => privateKeyBytes; + + /// + /// Gets the raw public key bytes when available (empty for EC-backed keys or when not supplied). + /// + public ReadOnlyMemory PublicKey => publicKeyBytes; + + /// + /// Gets the timestamp when the key was created/imported. + /// + public DateTimeOffset CreatedAt { get; } + + /// + /// Gets the optional expiry timestamp for the key. + /// + public DateTimeOffset? ExpiresAt { get; } + + /// + /// Gets arbitrary metadata entries associated with the key. + /// + public IReadOnlyDictionary Metadata { get; } + + private static ECParameters CloneParameters(ECParameters source, bool includePrivate) + { + var clone = new ECParameters + { + Curve = source.Curve, + Q = new ECPoint + { + X = source.Q.X is null ? null : (byte[])source.Q.X.Clone(), + Y = source.Q.Y is null ? null : (byte[])source.Q.Y.Clone() + } + }; + + if (includePrivate && source.D is not null) + { + clone.D = (byte[])source.D.Clone(); + } + + return clone; + } +} diff --git a/src/StellaOps.Cryptography/DefaultCryptoProvider.cs b/src/StellaOps.Cryptography/DefaultCryptoProvider.cs index 68066062..5f2d0ac3 100644 --- a/src/StellaOps.Cryptography/DefaultCryptoProvider.cs +++ b/src/StellaOps.Cryptography/DefaultCryptoProvider.cs @@ -1,129 +1,133 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography; - -namespace StellaOps.Cryptography; - -/// -/// Default in-process crypto provider exposing password hashing capabilities. -/// -public sealed class DefaultCryptoProvider : ICryptoProvider -{ - private readonly ConcurrentDictionary passwordHashers; - private readonly ConcurrentDictionary signingKeys; - private static readonly HashSet SupportedSigningAlgorithms = new(StringComparer.OrdinalIgnoreCase) - { - SignatureAlgorithms.Es256 - }; - - public DefaultCryptoProvider() - { - passwordHashers = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - signingKeys = new ConcurrentDictionary(StringComparer.Ordinal); - - var argon = new Argon2idPasswordHasher(); - var pbkdf2 = new Pbkdf2PasswordHasher(); - - passwordHashers.TryAdd(PasswordHashAlgorithm.Argon2id.ToString(), argon); - passwordHashers.TryAdd(PasswordHashAlgorithms.Argon2id, argon); - passwordHashers.TryAdd(PasswordHashAlgorithm.Pbkdf2.ToString(), pbkdf2); - passwordHashers.TryAdd(PasswordHashAlgorithms.Pbkdf2Sha256, pbkdf2); - } - - public string Name => "default"; - - public bool Supports(CryptoCapability capability, string algorithmId) - { - if (string.IsNullOrWhiteSpace(algorithmId)) - { - return false; - } - - return capability switch - { - CryptoCapability.PasswordHashing => passwordHashers.ContainsKey(algorithmId), - CryptoCapability.Signing or CryptoCapability.Verification => SupportedSigningAlgorithms.Contains(algorithmId), - _ => false - }; - } - - public IPasswordHasher GetPasswordHasher(string algorithmId) - { - if (!Supports(CryptoCapability.PasswordHashing, algorithmId)) - { - throw new InvalidOperationException($"Password hashing algorithm '{algorithmId}' is not supported by provider '{Name}'."); - } - - return passwordHashers[algorithmId]; - } - - public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference) - { - ArgumentNullException.ThrowIfNull(keyReference); - - if (!Supports(CryptoCapability.Signing, algorithmId)) - { - throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider '{Name}'."); - } - - if (!signingKeys.TryGetValue(keyReference.KeyId, out var signingKey)) - { - throw new KeyNotFoundException($"Signing key '{keyReference.KeyId}' is not registered with provider '{Name}'."); - } - - if (!string.Equals(signingKey.AlgorithmId, algorithmId, StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException( - $"Signing key '{keyReference.KeyId}' is registered for algorithm '{signingKey.AlgorithmId}', not '{algorithmId}'."); - } - - return EcdsaSigner.Create(signingKey); - } - - public void UpsertSigningKey(CryptoSigningKey signingKey) - { - ArgumentNullException.ThrowIfNull(signingKey); - EnsureSigningSupported(signingKey.AlgorithmId); - ValidateSigningKey(signingKey); - - signingKeys.AddOrUpdate(signingKey.Reference.KeyId, signingKey, (_, _) => signingKey); - } - - public bool RemoveSigningKey(string keyId) - { - if (string.IsNullOrWhiteSpace(keyId)) - { - return false; - } - - return signingKeys.TryRemove(keyId, out _); - } - - public IReadOnlyCollection GetSigningKeys() - => signingKeys.Values.ToArray(); - - private static void EnsureSigningSupported(string algorithmId) - { - if (!SupportedSigningAlgorithms.Contains(algorithmId)) - { - throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider 'default'."); - } - } - - private static void ValidateSigningKey(CryptoSigningKey signingKey) - { - if (!string.Equals(signingKey.AlgorithmId, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException($"Only ES256 signing keys are currently supported by provider 'default'."); - } - - var expected = ECCurve.NamedCurves.nistP256; - var curve = signingKey.PrivateParameters.Curve; - if (!curve.IsNamed || !string.Equals(curve.Oid.Value, expected.Oid.Value, StringComparison.Ordinal)) - { - throw new InvalidOperationException("ES256 signing keys must use the NIST P-256 curve."); - } - } -} +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; + +namespace StellaOps.Cryptography; + +/// +/// Default in-process crypto provider exposing password hashing capabilities. +/// +public sealed class DefaultCryptoProvider : ICryptoProvider +{ + private readonly ConcurrentDictionary passwordHashers; + private readonly ConcurrentDictionary signingKeys; + private static readonly HashSet SupportedSigningAlgorithms = new(StringComparer.OrdinalIgnoreCase) + { + SignatureAlgorithms.Es256 + }; + + public DefaultCryptoProvider() + { + passwordHashers = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + signingKeys = new ConcurrentDictionary(StringComparer.Ordinal); + + var argon = new Argon2idPasswordHasher(); + var pbkdf2 = new Pbkdf2PasswordHasher(); + + passwordHashers.TryAdd(PasswordHashAlgorithm.Argon2id.ToString(), argon); + passwordHashers.TryAdd(PasswordHashAlgorithms.Argon2id, argon); + passwordHashers.TryAdd(PasswordHashAlgorithm.Pbkdf2.ToString(), pbkdf2); + passwordHashers.TryAdd(PasswordHashAlgorithms.Pbkdf2Sha256, pbkdf2); + } + + public string Name => "default"; + + public bool Supports(CryptoCapability capability, string algorithmId) + { + if (string.IsNullOrWhiteSpace(algorithmId)) + { + return false; + } + + return capability switch + { + CryptoCapability.PasswordHashing => passwordHashers.ContainsKey(algorithmId), + CryptoCapability.Signing or CryptoCapability.Verification => SupportedSigningAlgorithms.Contains(algorithmId), + _ => false + }; + } + + public IPasswordHasher GetPasswordHasher(string algorithmId) + { + if (!Supports(CryptoCapability.PasswordHashing, algorithmId)) + { + throw new InvalidOperationException($"Password hashing algorithm '{algorithmId}' is not supported by provider '{Name}'."); + } + + return passwordHashers[algorithmId]; + } + + public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference) + { + ArgumentNullException.ThrowIfNull(keyReference); + + if (!Supports(CryptoCapability.Signing, algorithmId)) + { + throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider '{Name}'."); + } + + if (!signingKeys.TryGetValue(keyReference.KeyId, out var signingKey)) + { + throw new KeyNotFoundException($"Signing key '{keyReference.KeyId}' is not registered with provider '{Name}'."); + } + + if (!string.Equals(signingKey.AlgorithmId, algorithmId, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Signing key '{keyReference.KeyId}' is registered for algorithm '{signingKey.AlgorithmId}', not '{algorithmId}'."); + } + + return EcdsaSigner.Create(signingKey); + } + + public void UpsertSigningKey(CryptoSigningKey signingKey) + { + ArgumentNullException.ThrowIfNull(signingKey); + EnsureSigningSupported(signingKey.AlgorithmId); + if (signingKey.Kind != CryptoSigningKeyKind.Ec) + { + throw new InvalidOperationException($"Provider '{Name}' only accepts EC signing keys."); + } + ValidateSigningKey(signingKey); + + signingKeys.AddOrUpdate(signingKey.Reference.KeyId, signingKey, (_, _) => signingKey); + } + + public bool RemoveSigningKey(string keyId) + { + if (string.IsNullOrWhiteSpace(keyId)) + { + return false; + } + + return signingKeys.TryRemove(keyId, out _); + } + + public IReadOnlyCollection GetSigningKeys() + => signingKeys.Values.ToArray(); + + private static void EnsureSigningSupported(string algorithmId) + { + if (!SupportedSigningAlgorithms.Contains(algorithmId)) + { + throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider 'default'."); + } + } + + private static void ValidateSigningKey(CryptoSigningKey signingKey) + { + if (!string.Equals(signingKey.AlgorithmId, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Only ES256 signing keys are currently supported by provider 'default'."); + } + + var expected = ECCurve.NamedCurves.nistP256; + var curve = signingKey.PrivateParameters.Curve; + if (!curve.IsNamed || !string.Equals(curve.Oid.Value, expected.Oid.Value, StringComparison.Ordinal)) + { + throw new InvalidOperationException("ES256 signing keys must use the NIST P-256 curve."); + } + } +} diff --git a/src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs b/src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs index 9670c6f8..87df8bee 100644 --- a/src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs +++ b/src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs @@ -64,6 +64,10 @@ public sealed class LibsodiumCryptoProvider : ICryptoProvider { ArgumentNullException.ThrowIfNull(signingKey); EnsureAlgorithmSupported(signingKey.AlgorithmId); + if (signingKey.Kind != CryptoSigningKeyKind.Ec) + { + throw new InvalidOperationException($"Provider '{Name}' only accepts EC signing keys."); + } signingKeys.AddOrUpdate(signingKey.Reference.KeyId, signingKey, (_, _) => signingKey); } diff --git a/src/StellaOps.Cryptography/SignatureAlgorithms.cs b/src/StellaOps.Cryptography/SignatureAlgorithms.cs index 0299f503..6667209c 100644 --- a/src/StellaOps.Cryptography/SignatureAlgorithms.cs +++ b/src/StellaOps.Cryptography/SignatureAlgorithms.cs @@ -1,11 +1,13 @@ -namespace StellaOps.Cryptography; - -/// -/// Known signature algorithm identifiers. -/// -public static class SignatureAlgorithms -{ - public const string Es256 = "ES256"; - public const string Es384 = "ES384"; - public const string Es512 = "ES512"; -} +namespace StellaOps.Cryptography; + +/// +/// Known signature algorithm identifiers. +/// +public static class SignatureAlgorithms +{ + public const string Es256 = "ES256"; + public const string Es384 = "ES384"; + public const string Es512 = "ES512"; + public const string Ed25519 = "ED25519"; + public const string EdDsa = "EdDSA"; +} diff --git a/src/StellaOps.Cryptography/TASKS.md b/src/StellaOps.Cryptography/TASKS.md index 89bc0dec..d2692761 100644 --- a/src/StellaOps.Cryptography/TASKS.md +++ b/src/StellaOps.Cryptography/TASKS.md @@ -1,44 +1,51 @@ -# Team 8 — Security Guild Task Board (UTC 2025-10-10) - -| ID | Status | Owner | Description | Dependencies | Exit Criteria | -|----|--------|-------|-------------|--------------|---------------| -| SEC1.A | DONE (2025-10-11) | Security Guild | Introduce `Argon2idPasswordHasher` backed by Konscious defaults. Wire options into `StandardPluginOptions` (`PasswordHashOptions`) and `StellaOpsAuthorityOptions.Security.PasswordHashing`. | PLG3, CORE3 | ✅ Hashes emit PHC string `$argon2id$v=19$m=19456,t=2,p=1$...`; ✅ `NeedsRehash` promotes PBKDF2 → Argon2; ✅ Integration tests cover tamper, legacy rehash, perf p95 < 250 ms. | -| SEC1.B | DONE (2025-10-12) | Security Guild | Add compile-time switch to enable libsodium/Core variants later (`STELLAOPS_CRYPTO_SODIUM`). Document build variable. | SEC1.A | ✅ Conditional compilation path compiles; ✅ README snippet in `docs/security/password-hashing.md`. | -| SEC2.A | DONE (2025-10-13) | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5–CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. | -| SEC2.B | DONE (2025-10-13) | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. | -| SEC3.A | DONE (2025-10-12) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. | -| SEC3.B | DONE (2025-10-13) | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. | -| SEC4.A | DONE (2025-10-12) | Security Guild + DevOps | Define revocation JSON schema (`revocation_bundle.schema.json`) and detached JWS workflow. | CORE9, OPS3 | ✅ Schema + sample committed; ✅ CLI command `stellaops auth revoke export` scaffolded with acceptance tests; ✅ Verification script + docs. | -| SEC4.B | DONE (2025-10-12) | Security Guild | Integrate signing keys with crypto provider abstraction (initially ES256 via BCL). | SEC4.A, D5 | ✅ `ICryptoProvider.GetSigner` stub + default BCL signer; ✅ Unit tests verifying signature roundtrip. | -| SEC5.A | DONE (2025-10-12) | Security Guild | Author STRIDE threat model (`docs/security/authority-threat-model.md`) covering token, bootstrap, revocation, CLI, plugin surfaces. | All SEC1–SEC4 in progress | ✅ DFDs + trust boundaries drawn; ✅ Risk table with owners/actions; ✅ Follow-up backlog issues created. | -| SEC5.B | DONE (2025-10-14) | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. | -| SEC5.B1 | DONE (2025-10-14) | Security Guild + Authority Core | Introduce `LibsodiumCryptoProvider` implementing ECDSA signing/verification via libsodium, register under feature flag, and validate against existing ES256 fixtures. | SEC5.B | ✅ Provider resolves via `ICryptoProviderRegistry`; ✅ Integration tests cover sign/verify parity with default provider; ✅ Fallback to managed provider documented. | -| SEC5.B2 | DONE (2025-10-14) | Security Guild + DevEx/CLI | Extend `stellaops auth revoke verify` to detect provider metadata, reuse registry for verification, and document CLI workflow. | SEC5.B | ✅ CLI uses registry signers for verification; ✅ End-to-end test invokes verify against sample bundle; ✅ docs/11_AUTHORITY.md references CLI procedure. | -| SEC5.C | DONE (2025-10-14) | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. | -| SEC5.D | DONE (2025-10-14) | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. | -> Remark (2025-10-14): Cleanup service wired to store; background sweep + invite audit tests added. -| SEC5.E | DONE (2025-10-14) | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. | -> Remark (2025-10-14): Token usage metadata persisted with replay audits + handler/unit coverage. -| SEC5.F | DONE (2025-10-14) | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. | -> Remark (2025-10-14): Analyzer surfaces warnings during CLI load; docs updated with mitigation steps. -| SEC5.G | DONE (2025-10-14) | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. | -> Remark (2025-10-14): Offline kit docs include manifest verification workflow; attestation artifacts referenced. -| SEC5.H | DONE (2025-10-13) | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. | -| D5.A | DONE (2025-10-12) | Security Guild | Flesh out `StellaOps.Cryptography` provider registry, policy, and DI helpers enabling sovereign crypto selection. | SEC1.A, SEC4.B | ✅ `ICryptoProviderRegistry` implementation with provider selection rules; ✅ `StellaOps.Cryptography.DependencyInjection` extensions; ✅ Tests covering fallback ordering. | - -> Remark (2025-10-13, SEC2.B): Coordinated with Authority Core — audit sinks now receive `/token` success/failure events; awaiting host test suite once signing fixture lands. -> -> Remark (2025-10-13, SEC3.B): Pinged Docs & Plugin guilds — rate limit guidance published in `docs/security/rate-limits.md` and flagged for PLG6.DOC copy lift. -> -> Remark (2025-10-13, SEC5.B): Split follow-up into SEC5.B1 (libsodium provider) and SEC5.B2 (CLI verification) after scoping registry integration; work not yet started. - -## Notes -- Target Argon2 parameters follow OWASP Cheat Sheet (memory ≈ 19 MiB, iterations 2, parallelism 1). Allow overrides via configuration. -- When CORE8 lands, pair with Team 2 to expose request context information required by the rate limiter (client_id enrichment). -- Revocation bundle must be consumable offline; include issue timestamp, signing key metadata, and reasons. -- All crypto usage in Authority code should funnel through the new abstractions (`ICryptoProvider`), enabling future CryptoPro/OpenSSL providers. - -## Done Definition -- Code merges include unit/integration tests and documentation updates. -- `TASKS.md` status transitions (TODO → DOING → DONE/BLOCKED) must happen in the same PR as the work. -- Prior to marking DONE: run `dotnet test` for touched solutions and attach excerpt to PR description. +# Team 8 — Security Guild Task Board (UTC 2025-10-10) + +| ID | Status | Owner | Description | Dependencies | Exit Criteria | +|----|--------|-------|-------------|--------------|---------------| +| SEC1.A | DONE (2025-10-11) | Security Guild | Introduce `Argon2idPasswordHasher` backed by Konscious defaults. Wire options into `StandardPluginOptions` (`PasswordHashOptions`) and `StellaOpsAuthorityOptions.Security.PasswordHashing`. | PLG3, CORE3 | ✅ Hashes emit PHC string `$argon2id$v=19$m=19456,t=2,p=1$...`; ✅ `NeedsRehash` promotes PBKDF2 → Argon2; ✅ Integration tests cover tamper, legacy rehash, perf p95 < 250 ms. | +| SEC1.B | DONE (2025-10-12) | Security Guild | Add compile-time switch to enable libsodium/Core variants later (`STELLAOPS_CRYPTO_SODIUM`). Document build variable. | SEC1.A | ✅ Conditional compilation path compiles; ✅ README snippet in `docs/security/password-hashing.md`. | +| SEC2.A | DONE (2025-10-13) | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5–CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. | +| SEC2.B | DONE (2025-10-13) | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. | +| SEC3.A | DONE (2025-10-12) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. | +| SEC3.B | DONE (2025-10-13) | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. | +| SEC4.A | DONE (2025-10-12) | Security Guild + DevOps | Define revocation JSON schema (`revocation_bundle.schema.json`) and detached JWS workflow. | CORE9, OPS3 | ✅ Schema + sample committed; ✅ CLI command `stellaops auth revoke export` scaffolded with acceptance tests; ✅ Verification script + docs. | +| SEC4.B | DONE (2025-10-12) | Security Guild | Integrate signing keys with crypto provider abstraction (initially ES256 via BCL). | SEC4.A, D5 | ✅ `ICryptoProvider.GetSigner` stub + default BCL signer; ✅ Unit tests verifying signature roundtrip. | +| SEC5.A | DONE (2025-10-12) | Security Guild | Author STRIDE threat model (`docs/security/authority-threat-model.md`) covering token, bootstrap, revocation, CLI, plugin surfaces. | All SEC1–SEC4 in progress | ✅ DFDs + trust boundaries drawn; ✅ Risk table with owners/actions; ✅ Follow-up backlog issues created. | +| SEC5.B | DONE (2025-10-14) | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. | +| SEC5.B1 | DONE (2025-10-14) | Security Guild + Authority Core | Introduce `LibsodiumCryptoProvider` implementing ECDSA signing/verification via libsodium, register under feature flag, and validate against existing ES256 fixtures. | SEC5.B | ✅ Provider resolves via `ICryptoProviderRegistry`; ✅ Integration tests cover sign/verify parity with default provider; ✅ Fallback to managed provider documented. | +| SEC5.B2 | DONE (2025-10-14) | Security Guild + DevEx/CLI | Extend `stellaops auth revoke verify` to detect provider metadata, reuse registry for verification, and document CLI workflow. | SEC5.B | ✅ CLI uses registry signers for verification; ✅ End-to-end test invokes verify against sample bundle; ✅ docs/11_AUTHORITY.md references CLI procedure. | +| SEC5.C | DONE (2025-10-14) | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. | +| SEC5.D | DONE (2025-10-14) | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. | +> Remark (2025-10-14): Cleanup service wired to store; background sweep + invite audit tests added. +| SEC5.E | DONE (2025-10-14) | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. | +> Remark (2025-10-14): Token usage metadata persisted with replay audits + handler/unit coverage. +| SEC5.F | DONE (2025-10-14) | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. | +> Remark (2025-10-14): Analyzer surfaces warnings during CLI load; docs updated with mitigation steps. +| SEC5.G | DONE (2025-10-14) | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. | +> Remark (2025-10-14): Offline kit docs include manifest verification workflow; attestation artifacts referenced. +| SEC5.H | DONE (2025-10-13) | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. | +| D5.A | DONE (2025-10-12) | Security Guild | Flesh out `StellaOps.Cryptography` provider registry, policy, and DI helpers enabling sovereign crypto selection. | SEC1.A, SEC4.B | ✅ `ICryptoProviderRegistry` implementation with provider selection rules; ✅ `StellaOps.Cryptography.DependencyInjection` extensions; ✅ Tests covering fallback ordering. | +| SEC6.A | DONE (2025-10-19) | Security Guild | Ship BouncyCastle-backed Ed25519 signing as a `StellaOps.Cryptography` plug-in and migrate Scanner WebService signing to consume the provider registry; codify the plug-in rule in AGENTS.
          2025-10-19: Added `StellaOps.Cryptography.Plugin.BouncyCastle`, updated DI and ReportSigner, captured provider tests (`BouncyCastleEd25519CryptoProviderTests`). | D5.A | ✅ Plug-in registered via DI (`AddStellaOpsCrypto` + `AddBouncyCastleEd25519Provider`); ✅ Report signer resolves keys through registry; ✅ Unit tests cover Ed25519 sign/verify via provider. | + +> Remark (2025-10-13, SEC2.B): Coordinated with Authority Core — audit sinks now receive `/token` success/failure events; awaiting host test suite once signing fixture lands. +> +> Remark (2025-10-13, SEC3.B): Pinged Docs & Plugin guilds — rate limit guidance published in `docs/security/rate-limits.md` and flagged for PLG6.DOC copy lift. +> +> Remark (2025-10-13, SEC5.B): Split follow-up into SEC5.B1 (libsodium provider) and SEC5.B2 (CLI verification) after scoping registry integration; work not yet started. + +> Remark (2025-10-13, SEC2.B): Coordinated with Authority Core — audit sinks now receive `/token` success/failure events; awaiting host test suite once signing fixture lands. +> +> Remark (2025-10-13, SEC3.B): Pinged Docs & Plugin guilds — rate limit guidance published in `docs/security/rate-limits.md` and flagged for PLG6.DOC copy lift. +> +> Remark (2025-10-13, SEC5.B): Split follow-up into SEC5.B1 (libsodium provider) and SEC5.B2 (CLI verification) after scoping registry integration; work not yet started. + +## Notes +- Target Argon2 parameters follow OWASP Cheat Sheet (memory ≈ 19 MiB, iterations 2, parallelism 1). Allow overrides via configuration. +- When CORE8 lands, pair with Team 2 to expose request context information required by the rate limiter (client_id enrichment). +- Revocation bundle must be consumable offline; include issue timestamp, signing key metadata, and reasons. +- All crypto usage in Authority code should funnel through the new abstractions (`ICryptoProvider`), enabling future CryptoPro/OpenSSL providers. + +## Done Definition +- Code merges include unit/integration tests and documentation updates. +- `TASKS.md` status transitions (TODO → DOING → DONE/BLOCKED) must happen in the same PR as the work. +- Prior to marking DONE: run `dotnet test` for touched solutions and attach excerpt to PR description. diff --git a/src/StellaOps.DependencyInjection/ServiceBindingAttribute.cs b/src/StellaOps.DependencyInjection/ServiceBindingAttribute.cs new file mode 100644 index 00000000..7129fbc4 --- /dev/null +++ b/src/StellaOps.DependencyInjection/ServiceBindingAttribute.cs @@ -0,0 +1,64 @@ +using System; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.DependencyInjection; + +/// +/// Declares how a plug-in type should be registered with the host dependency injection container. +/// +[AttributeUsage(AttributeTargets.Class, AllowMultiple = true, Inherited = false)] +public sealed class ServiceBindingAttribute : Attribute +{ + /// + /// Creates a binding that registers the decorated type as itself with a singleton lifetime. + /// + public ServiceBindingAttribute() + : this(null, ServiceLifetime.Singleton) + { + } + + /// + /// Creates a binding that registers the decorated type as itself with the specified lifetime. + /// + public ServiceBindingAttribute(ServiceLifetime lifetime) + : this(null, lifetime) + { + } + + /// + /// Creates a binding that registers the decorated type as the specified service type with a singleton lifetime. + /// + public ServiceBindingAttribute(Type serviceType) + : this(serviceType, ServiceLifetime.Singleton) + { + } + + /// + /// Creates a binding that registers the decorated type as the specified service type. + /// + public ServiceBindingAttribute(Type? serviceType, ServiceLifetime lifetime) + { + ServiceType = serviceType; + Lifetime = lifetime; + } + + /// + /// The service contract that should resolve to the decorated implementation. When null, the implementation registers itself. + /// + public Type? ServiceType { get; } + + /// + /// The lifetime that should be used when registering the decorated implementation. + /// + public ServiceLifetime Lifetime { get; } + + /// + /// Indicates whether existing descriptors for the same service type should be removed before this binding is applied. + /// + public bool ReplaceExisting { get; init; } + + /// + /// When true, the implementation is also registered as itself even if a service type is specified. + /// + public bool RegisterAsSelf { get; init; } +} diff --git a/src/StellaOps.Vexer.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs b/src/StellaOps.Excititor.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs similarity index 88% rename from src/StellaOps.Vexer.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs rename to src/StellaOps.Excititor.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs index 16f9061b..2e2999ad 100644 --- a/src/StellaOps.Vexer.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs +++ b/src/StellaOps.Excititor.ArtifactStores.S3.Tests/S3ArtifactClientTests.cs @@ -1,38 +1,38 @@ -using Amazon.S3; -using Amazon.S3.Model; -using Moq; -using StellaOps.Vexer.ArtifactStores.S3; -using StellaOps.Vexer.Export; - -namespace StellaOps.Vexer.ArtifactStores.S3.Tests; - -public sealed class S3ArtifactClientTests -{ - [Fact] - public async Task ObjectExistsAsync_ReturnsTrue_WhenMetadataSucceeds() - { - var mock = new Mock(); - mock.Setup(x => x.GetObjectMetadataAsync("bucket", "key", default)).ReturnsAsync(new GetObjectMetadataResponse - { - HttpStatusCode = System.Net.HttpStatusCode.OK, - }); - - var client = new S3ArtifactClient(mock.Object, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); - var exists = await client.ObjectExistsAsync("bucket", "key", default); - Assert.True(exists); - } - - [Fact] - public async Task PutObjectAsync_MapsMetadata() - { - var mock = new Mock(); - mock.Setup(x => x.PutObjectAsync(It.IsAny(), default)) - .ReturnsAsync(new PutObjectResponse()); - - var client = new S3ArtifactClient(mock.Object, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); - using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); - await client.PutObjectAsync("bucket", "key", stream, new Dictionary { ["a"] = "b" }, default); - - mock.Verify(x => x.PutObjectAsync(It.Is(r => r.Metadata["a"] == "b"), default), Times.Once); - } -} +using Amazon.S3; +using Amazon.S3.Model; +using Moq; +using StellaOps.Excititor.ArtifactStores.S3; +using StellaOps.Excititor.Export; + +namespace StellaOps.Excititor.ArtifactStores.S3.Tests; + +public sealed class S3ArtifactClientTests +{ + [Fact] + public async Task ObjectExistsAsync_ReturnsTrue_WhenMetadataSucceeds() + { + var mock = new Mock(); + mock.Setup(x => x.GetObjectMetadataAsync("bucket", "key", default)).ReturnsAsync(new GetObjectMetadataResponse + { + HttpStatusCode = System.Net.HttpStatusCode.OK, + }); + + var client = new S3ArtifactClient(mock.Object, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + var exists = await client.ObjectExistsAsync("bucket", "key", default); + Assert.True(exists); + } + + [Fact] + public async Task PutObjectAsync_MapsMetadata() + { + var mock = new Mock(); + mock.Setup(x => x.PutObjectAsync(It.IsAny(), default)) + .ReturnsAsync(new PutObjectResponse()); + + var client = new S3ArtifactClient(mock.Object, Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance); + using var stream = new MemoryStream(new byte[] { 1, 2, 3 }); + await client.PutObjectAsync("bucket", "key", stream, new Dictionary { ["a"] = "b" }, default); + + mock.Verify(x => x.PutObjectAsync(It.Is(r => r.Metadata["a"] == "b"), default), Times.Once); + } +} diff --git a/src/StellaOps.Vexer.ArtifactStores.S3.Tests/StellaOps.Vexer.ArtifactStores.S3.Tests.csproj b/src/StellaOps.Excititor.ArtifactStores.S3.Tests/StellaOps.Excititor.ArtifactStores.S3.Tests.csproj similarity index 75% rename from src/StellaOps.Vexer.ArtifactStores.S3.Tests/StellaOps.Vexer.ArtifactStores.S3.Tests.csproj rename to src/StellaOps.Excititor.ArtifactStores.S3.Tests/StellaOps.Excititor.ArtifactStores.S3.Tests.csproj index 43bbc0f4..94ed72f0 100644 --- a/src/StellaOps.Vexer.ArtifactStores.S3.Tests/StellaOps.Vexer.ArtifactStores.S3.Tests.csproj +++ b/src/StellaOps.Excititor.ArtifactStores.S3.Tests/StellaOps.Excititor.ArtifactStores.S3.Tests.csproj @@ -1,15 +1,15 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Vexer.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs similarity index 89% rename from src/StellaOps.Vexer.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs index 483956b4..716985ec 100644 --- a/src/StellaOps.Vexer.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.ArtifactStores.S3/Extensions/ServiceCollectionExtensions.cs @@ -1,38 +1,38 @@ -using Amazon; -using Amazon.Runtime; -using Amazon.S3; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Export; - -namespace StellaOps.Vexer.ArtifactStores.S3.Extensions; - -public static class ServiceCollectionExtensions -{ - public static IServiceCollection AddVexS3ArtifactClient(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(configure); - - services.Configure(configure); - services.AddSingleton(CreateS3Client); - services.AddSingleton(); - return services; - } - - private static IAmazonS3 CreateS3Client(IServiceProvider provider) - { - var options = provider.GetRequiredService>().Value; - var config = new AmazonS3Config - { - RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region), - ForcePathStyle = options.ForcePathStyle, - }; - - if (!string.IsNullOrWhiteSpace(options.ServiceUrl)) - { - config.ServiceURL = options.ServiceUrl; - } - - return new AmazonS3Client(config); - } -} +using Amazon; +using Amazon.Runtime; +using Amazon.S3; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Export; + +namespace StellaOps.Excititor.ArtifactStores.S3.Extensions; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddVexS3ArtifactClient(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + services.Configure(configure); + services.AddSingleton(CreateS3Client); + services.AddSingleton(); + return services; + } + + private static IAmazonS3 CreateS3Client(IServiceProvider provider) + { + var options = provider.GetRequiredService>().Value; + var config = new AmazonS3Config + { + RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region), + ForcePathStyle = options.ForcePathStyle, + }; + + if (!string.IsNullOrWhiteSpace(options.ServiceUrl)) + { + config.ServiceURL = options.ServiceUrl; + } + + return new AmazonS3Client(config); + } +} diff --git a/src/StellaOps.Vexer.ArtifactStores.S3/S3ArtifactClient.cs b/src/StellaOps.Excititor.ArtifactStores.S3/S3ArtifactClient.cs similarity index 94% rename from src/StellaOps.Vexer.ArtifactStores.S3/S3ArtifactClient.cs rename to src/StellaOps.Excititor.ArtifactStores.S3/S3ArtifactClient.cs index 96494fd2..0ec894ec 100644 --- a/src/StellaOps.Vexer.ArtifactStores.S3/S3ArtifactClient.cs +++ b/src/StellaOps.Excititor.ArtifactStores.S3/S3ArtifactClient.cs @@ -1,85 +1,85 @@ -using Amazon.S3; -using Amazon.S3.Model; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Export; - -namespace StellaOps.Vexer.ArtifactStores.S3; - -public sealed class S3ArtifactClientOptions -{ - public string Region { get; set; } = "us-east-1"; - - public string? ServiceUrl { get; set; } - = null; - - public bool ForcePathStyle { get; set; } - = true; -} - -public sealed class S3ArtifactClient : IS3ArtifactClient -{ - private readonly IAmazonS3 _s3; - private readonly ILogger _logger; - - public S3ArtifactClient(IAmazonS3 s3, ILogger logger) - { - _s3 = s3 ?? throw new ArgumentNullException(nameof(s3)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken) - { - try - { - var metadata = await _s3.GetObjectMetadataAsync(bucketName, key, cancellationToken).ConfigureAwait(false); - return metadata.HttpStatusCode == System.Net.HttpStatusCode.OK; - } - catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) - { - return false; - } - } - - public async Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken) - { - var request = new PutObjectRequest - { - BucketName = bucketName, - Key = key, - InputStream = content, - AutoCloseStream = false, - }; - - foreach (var kvp in metadata) - { - request.Metadata[kvp.Key] = kvp.Value; - } - - await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Uploaded object {Bucket}/{Key}", bucketName, key); - } - - public async Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken) - { - try - { - var response = await _s3.GetObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false); - var buffer = new MemoryStream(); - await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); - buffer.Position = 0; - return buffer; - } - catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) - { - _logger.LogDebug("Object {Bucket}/{Key} not found", bucketName, key); - return null; - } - } - - public async Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken) - { - await _s3.DeleteObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false); - _logger.LogDebug("Deleted object {Bucket}/{Key}", bucketName, key); - } -} +using Amazon.S3; +using Amazon.S3.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Export; + +namespace StellaOps.Excititor.ArtifactStores.S3; + +public sealed class S3ArtifactClientOptions +{ + public string Region { get; set; } = "us-east-1"; + + public string? ServiceUrl { get; set; } + = null; + + public bool ForcePathStyle { get; set; } + = true; +} + +public sealed class S3ArtifactClient : IS3ArtifactClient +{ + private readonly IAmazonS3 _s3; + private readonly ILogger _logger; + + public S3ArtifactClient(IAmazonS3 s3, ILogger logger) + { + _s3 = s3 ?? throw new ArgumentNullException(nameof(s3)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken) + { + try + { + var metadata = await _s3.GetObjectMetadataAsync(bucketName, key, cancellationToken).ConfigureAwait(false); + return metadata.HttpStatusCode == System.Net.HttpStatusCode.OK; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return false; + } + } + + public async Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken) + { + var request = new PutObjectRequest + { + BucketName = bucketName, + Key = key, + InputStream = content, + AutoCloseStream = false, + }; + + foreach (var kvp in metadata) + { + request.Metadata[kvp.Key] = kvp.Value; + } + + await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Uploaded object {Bucket}/{Key}", bucketName, key); + } + + public async Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken) + { + try + { + var response = await _s3.GetObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false); + var buffer = new MemoryStream(); + await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + return buffer; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + _logger.LogDebug("Object {Bucket}/{Key} not found", bucketName, key); + return null; + } + } + + public async Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken) + { + await _s3.DeleteObjectAsync(bucketName, key, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Deleted object {Bucket}/{Key}", bucketName, key); + } +} diff --git a/src/StellaOps.Vexer.ArtifactStores.S3/StellaOps.Vexer.ArtifactStores.S3.csproj b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj similarity index 77% rename from src/StellaOps.Vexer.ArtifactStores.S3/StellaOps.Vexer.ArtifactStores.S3.csproj rename to src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj index 507c79e3..3e4e5d38 100644 --- a/src/StellaOps.Vexer.ArtifactStores.S3/StellaOps.Vexer.ArtifactStores.S3.csproj +++ b/src/StellaOps.Excititor.ArtifactStores.S3/StellaOps.Excititor.ArtifactStores.S3.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Attestation.Tests/StellaOps.Excititor.Attestation.Tests.csproj b/src/StellaOps.Excititor.Attestation.Tests/StellaOps.Excititor.Attestation.Tests.csproj new file mode 100644 index 00000000..a521425c --- /dev/null +++ b/src/StellaOps.Excititor.Attestation.Tests/StellaOps.Excititor.Attestation.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + preview + enable + enable + true + + + + + + diff --git a/src/StellaOps.Vexer.Attestation.Tests/VexAttestationClientTests.cs b/src/StellaOps.Excititor.Attestation.Tests/VexAttestationClientTests.cs similarity index 91% rename from src/StellaOps.Vexer.Attestation.Tests/VexAttestationClientTests.cs rename to src/StellaOps.Excititor.Attestation.Tests/VexAttestationClientTests.cs index 1a27252b..7b066f1e 100644 --- a/src/StellaOps.Vexer.Attestation.Tests/VexAttestationClientTests.cs +++ b/src/StellaOps.Excititor.Attestation.Tests/VexAttestationClientTests.cs @@ -1,81 +1,81 @@ -using System.Collections.Immutable; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Attestation.Dsse; -using StellaOps.Vexer.Attestation.Signing; -using StellaOps.Vexer.Attestation.Transparency; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation.Tests; - -public sealed class VexAttestationClientTests -{ - [Fact] - public async Task SignAsync_ReturnsEnvelopeDigestAndDiagnostics() - { - var signer = new FakeSigner(); - var builder = new VexDsseBuilder(signer, NullLogger.Instance); - var options = Options.Create(new VexAttestationClientOptions()); - var client = new VexAttestationClient(builder, options, NullLogger.Instance); - - var request = new VexAttestationRequest( - ExportId: "exports/456", - QuerySignature: new VexQuerySignature("filters"), - Artifact: new VexContentAddress("sha256", "deadbeef"), - Format: VexExportFormat.Json, - CreatedAt: DateTimeOffset.UtcNow, - SourceProviders: ImmutableArray.Create("vendor"), - Metadata: ImmutableDictionary.Empty); - - var response = await client.SignAsync(request, CancellationToken.None); - - Assert.NotNull(response.Attestation); - Assert.NotNull(response.Attestation.EnvelopeDigest); - Assert.True(response.Diagnostics.ContainsKey("envelope")); - } - - [Fact] - public async Task SignAsync_SubmitsToTransparencyLog_WhenConfigured() - { - var signer = new FakeSigner(); - var builder = new VexDsseBuilder(signer, NullLogger.Instance); - var options = Options.Create(new VexAttestationClientOptions()); - var transparency = new FakeTransparencyLogClient(); - var client = new VexAttestationClient(builder, options, NullLogger.Instance, transparencyLogClient: transparency); - - var request = new VexAttestationRequest( - ExportId: "exports/789", - QuerySignature: new VexQuerySignature("filters"), - Artifact: new VexContentAddress("sha256", "deadbeef"), - Format: VexExportFormat.Json, - CreatedAt: DateTimeOffset.UtcNow, - SourceProviders: ImmutableArray.Create("vendor"), - Metadata: ImmutableDictionary.Empty); - - var response = await client.SignAsync(request, CancellationToken.None); - - Assert.NotNull(response.Attestation.Rekor); - Assert.True(response.Diagnostics.ContainsKey("rekorLocation")); - Assert.True(transparency.SubmitCalled); - } - - private sealed class FakeSigner : IVexSigner - { - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexSignedPayload("signature", "key")); - } - - private sealed class FakeTransparencyLogClient : ITransparencyLogClient - { - public bool SubmitCalled { get; private set; } - - public ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken) - { - SubmitCalled = true; - return ValueTask.FromResult(new TransparencyLogEntry(Guid.NewGuid().ToString(), "https://rekor.example/entries/123", "23", null)); - } - - public ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken) - => ValueTask.FromResult(true); - } -} +using System.Collections.Immutable; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Attestation.Dsse; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Attestation.Transparency; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation.Tests; + +public sealed class VexAttestationClientTests +{ + [Fact] + public async Task SignAsync_ReturnsEnvelopeDigestAndDiagnostics() + { + var signer = new FakeSigner(); + var builder = new VexDsseBuilder(signer, NullLogger.Instance); + var options = Options.Create(new VexAttestationClientOptions()); + var client = new VexAttestationClient(builder, options, NullLogger.Instance); + + var request = new VexAttestationRequest( + ExportId: "exports/456", + QuerySignature: new VexQuerySignature("filters"), + Artifact: new VexContentAddress("sha256", "deadbeef"), + Format: VexExportFormat.Json, + CreatedAt: DateTimeOffset.UtcNow, + SourceProviders: ImmutableArray.Create("vendor"), + Metadata: ImmutableDictionary.Empty); + + var response = await client.SignAsync(request, CancellationToken.None); + + Assert.NotNull(response.Attestation); + Assert.NotNull(response.Attestation.EnvelopeDigest); + Assert.True(response.Diagnostics.ContainsKey("envelope")); + } + + [Fact] + public async Task SignAsync_SubmitsToTransparencyLog_WhenConfigured() + { + var signer = new FakeSigner(); + var builder = new VexDsseBuilder(signer, NullLogger.Instance); + var options = Options.Create(new VexAttestationClientOptions()); + var transparency = new FakeTransparencyLogClient(); + var client = new VexAttestationClient(builder, options, NullLogger.Instance, transparencyLogClient: transparency); + + var request = new VexAttestationRequest( + ExportId: "exports/789", + QuerySignature: new VexQuerySignature("filters"), + Artifact: new VexContentAddress("sha256", "deadbeef"), + Format: VexExportFormat.Json, + CreatedAt: DateTimeOffset.UtcNow, + SourceProviders: ImmutableArray.Create("vendor"), + Metadata: ImmutableDictionary.Empty); + + var response = await client.SignAsync(request, CancellationToken.None); + + Assert.NotNull(response.Attestation.Rekor); + Assert.True(response.Diagnostics.ContainsKey("rekorLocation")); + Assert.True(transparency.SubmitCalled); + } + + private sealed class FakeSigner : IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload("signature", "key")); + } + + private sealed class FakeTransparencyLogClient : ITransparencyLogClient + { + public bool SubmitCalled { get; private set; } + + public ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken) + { + SubmitCalled = true; + return ValueTask.FromResult(new TransparencyLogEntry(Guid.NewGuid().ToString(), "https://rekor.example/entries/123", "23", null)); + } + + public ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken) + => ValueTask.FromResult(true); + } +} diff --git a/src/StellaOps.Vexer.Attestation.Tests/VexDsseBuilderTests.cs b/src/StellaOps.Excititor.Attestation.Tests/VexDsseBuilderTests.cs similarity index 86% rename from src/StellaOps.Vexer.Attestation.Tests/VexDsseBuilderTests.cs rename to src/StellaOps.Excititor.Attestation.Tests/VexDsseBuilderTests.cs index 408d5dfd..e0901b3c 100644 --- a/src/StellaOps.Vexer.Attestation.Tests/VexDsseBuilderTests.cs +++ b/src/StellaOps.Excititor.Attestation.Tests/VexDsseBuilderTests.cs @@ -1,52 +1,52 @@ -using System.Collections.Immutable; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Attestation.Dsse; -using StellaOps.Vexer.Attestation.Models; -using StellaOps.Vexer.Attestation.Signing; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation.Tests; - -public sealed class VexDsseBuilderTests -{ - [Fact] - public async Task CreateEnvelopeAsync_ProducesDeterministicPayload() - { - var signer = new FakeSigner("signature-value", "key-1"); - var builder = new VexDsseBuilder(signer, NullLogger.Instance); - - var request = new VexAttestationRequest( - ExportId: "exports/123", - QuerySignature: new VexQuerySignature("filters"), - Artifact: new VexContentAddress("sha256", "deadbeef"), - Format: VexExportFormat.Json, - CreatedAt: DateTimeOffset.UtcNow, - SourceProviders: ImmutableArray.Create("vendor"), - Metadata: ImmutableDictionary.Empty); - - var envelope = await builder.CreateEnvelopeAsync(request, request.Metadata, CancellationToken.None); - - Assert.Equal("application/vnd.in-toto+json", envelope.PayloadType); - Assert.Single(envelope.Signatures); - Assert.Equal("signature-value", envelope.Signatures[0].Signature); - Assert.Equal("key-1", envelope.Signatures[0].KeyId); - - var digest = VexDsseBuilder.ComputeEnvelopeDigest(envelope); - Assert.StartsWith("sha256:", digest); - } - - private sealed class FakeSigner : IVexSigner - { - private readonly string _signature; - private readonly string _keyId; - - public FakeSigner(string signature, string keyId) - { - _signature = signature; - _keyId = keyId; - } - - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexSignedPayload(_signature, _keyId)); - } -} +using System.Collections.Immutable; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Attestation.Dsse; +using StellaOps.Excititor.Attestation.Models; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation.Tests; + +public sealed class VexDsseBuilderTests +{ + [Fact] + public async Task CreateEnvelopeAsync_ProducesDeterministicPayload() + { + var signer = new FakeSigner("signature-value", "key-1"); + var builder = new VexDsseBuilder(signer, NullLogger.Instance); + + var request = new VexAttestationRequest( + ExportId: "exports/123", + QuerySignature: new VexQuerySignature("filters"), + Artifact: new VexContentAddress("sha256", "deadbeef"), + Format: VexExportFormat.Json, + CreatedAt: DateTimeOffset.UtcNow, + SourceProviders: ImmutableArray.Create("vendor"), + Metadata: ImmutableDictionary.Empty); + + var envelope = await builder.CreateEnvelopeAsync(request, request.Metadata, CancellationToken.None); + + Assert.Equal("application/vnd.in-toto+json", envelope.PayloadType); + Assert.Single(envelope.Signatures); + Assert.Equal("signature-value", envelope.Signatures[0].Signature); + Assert.Equal("key-1", envelope.Signatures[0].KeyId); + + var digest = VexDsseBuilder.ComputeEnvelopeDigest(envelope); + Assert.StartsWith("sha256:", digest); + } + + private sealed class FakeSigner : IVexSigner + { + private readonly string _signature; + private readonly string _keyId; + + public FakeSigner(string signature, string keyId) + { + _signature = signature; + _keyId = keyId; + } + + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload(_signature, _keyId)); + } +} diff --git a/src/StellaOps.Vexer.Attestation/AGENTS.md b/src/StellaOps.Excititor.Attestation/AGENTS.md similarity index 82% rename from src/StellaOps.Vexer.Attestation/AGENTS.md rename to src/StellaOps.Excititor.Attestation/AGENTS.md index 43d87380..5ce4b59f 100644 --- a/src/StellaOps.Vexer.Attestation/AGENTS.md +++ b/src/StellaOps.Excititor.Attestation/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Builds and verifies in-toto/DSSE attestations for Vexer exports and integrates with Rekor v2 transparency logs. -## Scope -- Attestation envelope builders, signing workflows (keyless/keyed), and predicate model definitions. -- Rekor v2 client implementation (submit, verify, poll inclusion) with retry/backoff policies. -- Verification utilities reused by Worker for periodic revalidation. -- Configuration bindings for signer identity, Rekor endpoints, and offline bundle operation. -## Participants -- Export module calls into this layer to generate attestations after export artifacts are produced. -- WebService and Worker consume verification helpers to ensure stored envelopes remain valid. -- CLI `vexer verify` leverages verification services through WebService endpoints. -## Interfaces & contracts -- `IExportAttestor`, `ITransparencyLogClient`, predicate DTOs, and verification result records. -- Extension methods to register attestation services in DI across WebService/Worker. -## In/Out of scope -In: attestation creation, verification, Rekor integration, signer configuration. -Out: export artifact generation, storage persistence, CLI interaction layers. -## Observability & security expectations -- Structured logs for signing/verification with envelope digest, Rekor URI, and latency; never log private keys. -- Metrics for attestation successes/failures and Rekor submission durations. -## Tests -- Unit tests and integration stubs (with fake Rekor) will live in `../StellaOps.Vexer.Attestation.Tests`. +# AGENTS +## Role +Builds and verifies in-toto/DSSE attestations for Excititor exports and integrates with Rekor v2 transparency logs. +## Scope +- Attestation envelope builders, signing workflows (keyless/keyed), and predicate model definitions. +- Rekor v2 client implementation (submit, verify, poll inclusion) with retry/backoff policies. +- Verification utilities reused by Worker for periodic revalidation. +- Configuration bindings for signer identity, Rekor endpoints, and offline bundle operation. +## Participants +- Export module calls into this layer to generate attestations after export artifacts are produced. +- WebService and Worker consume verification helpers to ensure stored envelopes remain valid. +- CLI `excititor verify` leverages verification services through WebService endpoints. +## Interfaces & contracts +- `IExportAttestor`, `ITransparencyLogClient`, predicate DTOs, and verification result records. +- Extension methods to register attestation services in DI across WebService/Worker. +## In/Out of scope +In: attestation creation, verification, Rekor integration, signer configuration. +Out: export artifact generation, storage persistence, CLI interaction layers. +## Observability & security expectations +- Structured logs for signing/verification with envelope digest, Rekor URI, and latency; never log private keys. +- Metrics for attestation successes/failures and Rekor submission durations. +## Tests +- Unit tests and integration stubs (with fake Rekor) will live in `../StellaOps.Excititor.Attestation.Tests`. diff --git a/src/StellaOps.Vexer.Attestation/Dsse/DsseEnvelope.cs b/src/StellaOps.Excititor.Attestation/Dsse/DsseEnvelope.cs similarity index 88% rename from src/StellaOps.Vexer.Attestation/Dsse/DsseEnvelope.cs rename to src/StellaOps.Excititor.Attestation/Dsse/DsseEnvelope.cs index 65b7f93c..ebab157c 100644 --- a/src/StellaOps.Vexer.Attestation/Dsse/DsseEnvelope.cs +++ b/src/StellaOps.Excititor.Attestation/Dsse/DsseEnvelope.cs @@ -1,13 +1,13 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Vexer.Attestation.Dsse; - -public sealed record DsseEnvelope( - [property: JsonPropertyName("payload")] string Payload, - [property: JsonPropertyName("payloadType")] string PayloadType, - [property: JsonPropertyName("signatures")] IReadOnlyList Signatures); - -public sealed record DsseSignature( - [property: JsonPropertyName("sig")] string Signature, - [property: JsonPropertyName("keyid")] string? KeyId); +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Excititor.Attestation.Dsse; + +public sealed record DsseEnvelope( + [property: JsonPropertyName("payload")] string Payload, + [property: JsonPropertyName("payloadType")] string PayloadType, + [property: JsonPropertyName("signatures")] IReadOnlyList Signatures); + +public sealed record DsseSignature( + [property: JsonPropertyName("sig")] string Signature, + [property: JsonPropertyName("keyid")] string? KeyId); diff --git a/src/StellaOps.Vexer.Attestation/Dsse/VexDsseBuilder.cs b/src/StellaOps.Excititor.Attestation/Dsse/VexDsseBuilder.cs similarity index 92% rename from src/StellaOps.Vexer.Attestation/Dsse/VexDsseBuilder.cs rename to src/StellaOps.Excititor.Attestation/Dsse/VexDsseBuilder.cs index 64e846c0..8e7f36e1 100644 --- a/src/StellaOps.Vexer.Attestation/Dsse/VexDsseBuilder.cs +++ b/src/StellaOps.Excititor.Attestation/Dsse/VexDsseBuilder.cs @@ -1,83 +1,83 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Attestation.Models; -using StellaOps.Vexer.Attestation.Signing; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation.Dsse; - -public sealed class VexDsseBuilder -{ - private const string PayloadType = "application/vnd.in-toto+json"; - - private readonly IVexSigner _signer; - private readonly ILogger _logger; - private readonly JsonSerializerOptions _serializerOptions; - - public VexDsseBuilder(IVexSigner signer, ILogger logger) - { - _signer = signer ?? throw new ArgumentNullException(nameof(signer)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _serializerOptions = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.Never, - WriteIndented = false, - }; - _serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); - } - - public async ValueTask CreateEnvelopeAsync( - VexAttestationRequest request, - IReadOnlyDictionary? metadata, - CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(request); - - var predicate = VexAttestationPredicate.FromRequest(request, metadata); - var subject = new VexInTotoSubject( - request.ExportId, - new Dictionary(StringComparer.Ordinal) - { - { request.Artifact.Algorithm.ToLowerInvariant(), request.Artifact.Digest } - }); - - var statement = new VexInTotoStatement( - VexInTotoStatement.InTotoType, - "https://stella-ops.org/attestations/vex-export", - new[] { subject }, - predicate); - - var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions); - var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false); - - var envelope = new DsseEnvelope( - Convert.ToBase64String(payloadBytes), - PayloadType, - new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) }); - - _logger.LogDebug("DSSE envelope created for export {ExportId}", request.ExportId); - return envelope; - } - - public static string ComputeEnvelopeDigest(DsseEnvelope envelope) - { - ArgumentNullException.ThrowIfNull(envelope); - var envelopeJson = JsonSerializer.Serialize(envelope, new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }); - var bytes = Encoding.UTF8.GetBytes(envelopeJson); - var hash = SHA256.HashData(bytes); - return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); - } -} +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Attestation.Models; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation.Dsse; + +public sealed class VexDsseBuilder +{ + private const string PayloadType = "application/vnd.in-toto+json"; + + private readonly IVexSigner _signer; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _serializerOptions; + + public VexDsseBuilder(IVexSigner signer, ILogger logger) + { + _signer = signer ?? throw new ArgumentNullException(nameof(signer)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _serializerOptions = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = false, + }; + _serializerOptions.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + } + + public async ValueTask CreateEnvelopeAsync( + VexAttestationRequest request, + IReadOnlyDictionary? metadata, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + var predicate = VexAttestationPredicate.FromRequest(request, metadata); + var subject = new VexInTotoSubject( + request.ExportId, + new Dictionary(StringComparer.Ordinal) + { + { request.Artifact.Algorithm.ToLowerInvariant(), request.Artifact.Digest } + }); + + var statement = new VexInTotoStatement( + VexInTotoStatement.InTotoType, + "https://stella-ops.org/attestations/vex-export", + new[] { subject }, + predicate); + + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(statement, _serializerOptions); + var signatureResult = await _signer.SignAsync(payloadBytes, cancellationToken).ConfigureAwait(false); + + var envelope = new DsseEnvelope( + Convert.ToBase64String(payloadBytes), + PayloadType, + new[] { new DsseSignature(signatureResult.Signature, signatureResult.KeyId) }); + + _logger.LogDebug("DSSE envelope created for export {ExportId}", request.ExportId); + return envelope; + } + + public static string ComputeEnvelopeDigest(DsseEnvelope envelope) + { + ArgumentNullException.ThrowIfNull(envelope); + var envelopeJson = JsonSerializer.Serialize(envelope, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }); + var bytes = Encoding.UTF8.GetBytes(envelopeJson); + var hash = SHA256.HashData(bytes); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Excititor.Attestation/EXCITITOR-ATTEST-01-003-plan.md b/src/StellaOps.Excititor.Attestation/EXCITITOR-ATTEST-01-003-plan.md new file mode 100644 index 00000000..0039088f --- /dev/null +++ b/src/StellaOps.Excititor.Attestation/EXCITITOR-ATTEST-01-003-plan.md @@ -0,0 +1,149 @@ +# EXCITITOR-ATTEST-01-003 - Verification & Observability Plan + +- **Date:** 2025-10-19 +- **Status:** Draft +- **Owner:** Team Excititor Attestation +- **Related tasks:** EXCITITOR-ATTEST-01-003 (Wave 0), EXCITITOR-WEB-01-003/004, EXCITITOR-WORKER-01-003 +- **Prerequisites satisfied:** EXCITITOR-ATTEST-01-002 (Rekor v2 client integration) + +## 1. Objectives + +1. Provide deterministic attestation verification helpers consumable by Excititor WebService (`/excititor/verify`, `/excititor/export*`) and Worker re-verification loops. +2. Surface structured diagnostics for success, soft failures, and hard failures (signature mismatch, Rekor gaps, artifact digest drift). +3. Emit observability signals (logs, metrics, optional tracing) that can run offline and degrade gracefully when transparency services are unreachable. +4. Add regression tests (unit + integration) covering positive path, negative path, and offline fallback scenarios. + +## 2. Deliverables + +- `IVexAttestationVerifier` abstraction + `VexAttestationVerifier` implementation inside `StellaOps.Excititor.Attestation`, encapsulating DSSE validation, predicate checks, artifact digest confirmation, Rekor inclusion verification, and deterministic diagnostics. +- DI wiring (extension method) for registering verifier + instrumentation dependencies alongside the existing signer/rekor client. +- Shared `VexAttestationDiagnostics` record describing normalized diagnostic keys consumed by Worker/WebService logging. +- Metrics utility (`AttestationMetrics`) exposing counters/histograms via `System.Diagnostics.Metrics`, exported under `StellaOps.Excititor.Attestation` meter. +- Activity source (`AttestationActivitySource`) for optional tracing spans around sign/verify operations. +- Documentation updates (`EXCITITOR-ATTEST-01-003-plan.md`, `TASKS.md` notes) describing instrumentation + test expectations. +- Test coverage in `StellaOps.Excititor.Attestation.Tests` (unit) and scaffolding notes for WebService/Worker integration tests. + +## 3. Verification Flow + +### 3.1 Inputs + +- `VexAttestationRequest` from Core (contains export identifiers, artifact digest, metadata, source providers). +- Optional Rekor reference from previous signing (`VexAttestationMetadata.Rekor`). +- Configured policies (tolerated clock skew, Rekor verification toggle, offline mode flag, maximum metadata drift). + +### 3.2 Steps + +1. **Envelope decode** - retrieve DSSE envelope + predicate from storage (Worker) or request payload (WebService), canonicalize JSON, compute digest, compare with metadata `envelopeDigest`. +2. **Subject validation** - ensure subject digest matches exported artifact digest (algorithm & value) and export identifier matches `request.ExportId`. +3. **Signature verification** - delegate to signer/verifier abstraction (cosign/x509) using configured trust anchors; record `signature_state` diagnostic (verified, skipped_offline, failed). +4. **Provenance checks** - confirm predicate type (`https://stella-ops.org/attestations/vex-export`) and metadata shape; enforce deterministic timestamp tolerance. +5. **Transparency log** - if Rekor reference present and verification enabled, call `ITransparencyLogClient.VerifyAsync` with retry/backoff budget; support offline bypass with diagnostic `rekor_state=unreachable`. +6. **Result aggregation** - produce `VexAttestationVerification` containing `IsValid` flag and diagnostics map (includes `failure_reason` when invalid). + +### 3.3 Failure Categories & Handling + +| Category | Detection | Handling | +|---|---|---| +| Signature mismatch | Signer verification failure or subject digest mismatch | Mark invalid, emit warning log, increment `verify.failed` counter with `reason=signature_mismatch`. | +| Rekor absence/stale | Rekor verify returns false | Mark invalid unless offline mode configured; log with correlation ID; `reason=rekor_missing`. | +| Predicate schema drift | Predicate type or required fields missing | Mark invalid, include `reason=predicate_invalid`. | +| Time skew | `signedAt` older than policy threshold | Mark invalid (hard) or warn (soft) per options; include `reason=stale_attestation`. | +| Unexpected metadata | Unknown export format, provider mismatch | Mark invalid; `reason=metadata_mismatch`. | +| Offline Rekor | HTTP client throws | Mark soft failure if `AllowOfflineTransparency` true; degrade metrics with `rekor_state=offline`. | + +## 4. Observability + +### 4.1 Metrics (Meter name: `StellaOps.Excititor.Attestation`) + +| Metric | Type | Dimensions | Description | +|---|---|---|---| +| `stellaops.excititor.attestation.verify.total` | Counter | `result` (`success`/`failure`/`soft_failure`), `component` (`webservice`/`worker`), `reverify` (`true`/`false`) | Counts verification attempts. | +| `stellaops.excititor.attestation.verify.duration.ms` | Histogram | `component`, `result` | Measures end-to-end verification latency. | +| `stellaops.excititor.attestation.verify.rekor.calls` | Counter | `result` (`verified`/`unreachable`/`skipped`) | Rekor verification outcomes. | +| `stellaops.excititor.attestation.verify.cache.hit` | Counter | `hit` (`true`/`false`) | Tracks reuse of cached verification results (Worker loop). | + +Metrics must register via static helper using `Meter` and support offline operation (no exporter dependency). Histogram records double milliseconds; use `Stopwatch.GetElapsedTime` for monotonic timing. + +### 4.2 Logging + +- Use structured logs (`ILogger`) with event IDs: `AttestationVerified` (Information), `AttestationVerificationFailed` (Warning), `AttestationVerificationError` (Error). +- Include correlation ID (`request.QuerySignature.Value`), `exportId`, `envelopeDigest`, `rekorLocation`, `reason`, and `durationMs`. +- Avoid logging private keys or full envelope; log envelope digest only. For debug builds, gate optional envelope JSON behind `LogLevel.Trace` and configuration flag. + +### 4.3 Tracing + +- Activity source name `StellaOps.Excititor.Attestation` with spans `attestation.verify` (parent from WebService request or Worker job) including tags: `stellaops.export_id`, `stellaops.result`, `stellaops.rekor.state`. +- Propagate Activity through Rekor client via `HttpClient` instrumentation (auto instrumentation available). + +## 5. Integration Points + +### 5.1 WebService + +- Inject `IVexAttestationVerifier` into export endpoints and `/excititor/verify` handler. +- Persist verification result diagnostics alongside response payload for deterministic clients. +- Return HTTP 200 with `{ valid: true }` when verified; 409 for invalid attestation with diagnostics JSON; 503 when Rekor unreachable and offline override disabled. +- Add caching for idempotent verification (e.g., by envelope digest) to reduce Rekor calls and surface via metrics. + +### 5.2 Worker + +- Schedule background job (`EXCITITOR-WORKER-01-003`) to re-verify stored attestations on TTL (default 12h) using new verifier; on failure, flag export for re-sign and notify via event bus (future task). +- Emit logs/metrics with `component=worker`; include job IDs and next scheduled run. +- Provide cancellation-aware loops (respect `CancellationToken`) and deterministic order (sorted by export id). + +### 5.3 Storage / Cache Hooks + +- Store latest verification status and diagnostics in attestation metadata collection (Mongo) keyed by `envelopeDigest` + `artifactDigest` to avoid duplicate work. +- Expose read API (via WebService) for clients to fetch last verification timestamp + result. + +## 6. Test Strategy + +### 6.1 Unit Tests (`StellaOps.Excititor.Attestation.Tests`) + +- `VexAttestationVerifierTests.VerifyAsync_Succeeds_WhenSignatureAndRekorValid` - uses fake signer/verifier + in-memory Rekor client returning success. +- `...ReturnsSoftFailure_WhenRekorOfflineAndAllowed` - ensure `IsValid=true`, diagnostic `rekor_state=offline`, metric increments `result=soft_failure`. +- `...Fails_WhenDigestMismatch` - ensures invalid result, log entry recorded, metrics increment `result=failure` with `reason=signature_mismatch`. +- `...Fails_WhenPredicateTypeUnexpected` - invalid with `reason=predicate_invalid`. +- `...RespectsCancellation` - cancellation token triggered before Rekor call results in `OperationCanceledException` and no metrics increments beyond started attempt. + +### 6.2 WebService Integration Tests (`StellaOps.Excititor.WebService.Tests`) + +- `VerifyEndpoint_Returns200_OnValidAttestation` - mocks verifier to return success, asserts response payload, metrics stub invoked. +- `VerifyEndpoint_Returns409_OnInvalid` - invalid diag forwarded, ensures logging occurs. +- `ExportEndpoint_IncludesVerificationDiagnostics` - ensures signed export responses include last verification metadata. + +### 6.3 Worker Tests (`StellaOps.Excititor.Worker.Tests`) + +- `ReverificationJob_RequeuesOnFailure` - invalid result triggers requeue/backoff. +- `ReverificationJob_PersistsStatusAndMetrics` - success path updates repository & metrics. + +### 6.4 Determinism/Regression + +- Golden test verifying that identical inputs produce identical diagnostics dictionaries (sorted keys). +- Ensure metrics dimensions remain stable via snapshot test (e.g., capturing tags in fake meter listener). + +## 7. Implementation Sequencing + +1. Introduce verifier abstraction + implementation with basic tests (signature + Rekor success/failure). +2. Add observability helpers (metrics, activity, logging) and wire into verifier; extend tests to assert instrumentation (using in-memory listener/log sink). +3. Update WebService DI/service layer to use verifier; craft endpoint integration tests. +4. Update Worker scheduling code to call verifier & emit metrics. +5. Wire persistence/caching and document configuration knobs (retry, offline, TTL). +6. Finalize documentation (architecture updates, runbook entries) before closing task. + +## 8. Configuration Defaults + +- `AttestationVerificationOptions` (new): `RequireRekor=true`, `AllowOfflineTransparency=false`, `MaxClockSkew=PT5M`, `ReverifyInterval=PT12H`, `CacheWindow=PT1H`. +- Options bind from configuration section `Excititor:Attestation` across WebService/Worker; offline kit ships defaults. + +## 9. Open Questions + +- Should verification gracefully accept legacy predicate types (pre-1.0) or hard fail? (Proposed: allow via allowlist with warning diagnostics.) +- Do we need cross-module eventing when verification fails (e.g., notify Export module) or is logging sufficient in Wave 0? (Proposed: log + metrics now, escalate in later wave.) +- Confirm whether Worker re-verification writes to Mongo or triggers Export module to re-sign artifacts automatically; placeholder: record status + timestamp only. + +## 10. Acceptance Criteria + +- Plan approved by Attestation + WebService + Worker leads. +- Metrics/logging names peer-reviewed to avoid collisions. +- Test backlog items entered into respective `TASKS.md` once implementation starts. +- Documentation (this plan) linked from `TASKS.md` notes for discoverability. diff --git a/src/StellaOps.Vexer.Attestation/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs similarity index 77% rename from src/StellaOps.Vexer.Attestation/Extensions/ServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs index ac48256c..d903af5d 100644 --- a/src/StellaOps.Vexer.Attestation/Extensions/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Attestation/Extensions/ServiceCollectionExtensions.cs @@ -1,24 +1,24 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Vexer.Attestation.Dsse; -using StellaOps.Vexer.Attestation.Transparency; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation.Extensions; - -public static class VexAttestationServiceCollectionExtensions -{ - public static IServiceCollection AddVexAttestation(this IServiceCollection services) - { - services.AddSingleton(); - services.AddSingleton(); - return services; - } - - public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(configure); - services.Configure(configure); - services.AddHttpClient(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Attestation.Dsse; +using StellaOps.Excititor.Attestation.Transparency; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation.Extensions; + +public static class VexAttestationServiceCollectionExtensions +{ + public static IServiceCollection AddVexAttestation(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + return services; + } + + public static IServiceCollection AddVexRekorClient(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + services.Configure(configure); + services.AddHttpClient(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Attestation/Models/VexAttestationPredicate.cs b/src/StellaOps.Excititor.Attestation/Models/VexAttestationPredicate.cs similarity index 91% rename from src/StellaOps.Vexer.Attestation/Models/VexAttestationPredicate.cs rename to src/StellaOps.Excititor.Attestation/Models/VexAttestationPredicate.cs index c879b069..33ef69a3 100644 --- a/src/StellaOps.Vexer.Attestation/Models/VexAttestationPredicate.cs +++ b/src/StellaOps.Excititor.Attestation/Models/VexAttestationPredicate.cs @@ -1,44 +1,44 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Text.Json.Serialization; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation.Models; - -public sealed record VexAttestationPredicate( - string ExportId, - string QuerySignature, - string ArtifactAlgorithm, - string ArtifactDigest, - VexExportFormat Format, - DateTimeOffset CreatedAt, - IReadOnlyList SourceProviders, - IReadOnlyDictionary Metadata) -{ - public static VexAttestationPredicate FromRequest( - VexAttestationRequest request, - IReadOnlyDictionary? metadata = null) - => new( - request.ExportId, - request.QuerySignature.Value, - request.Artifact.Algorithm, - request.Artifact.Digest, - request.Format, - request.CreatedAt, - request.SourceProviders, - metadata is null ? ImmutableDictionary.Empty : metadata.ToImmutableDictionary(StringComparer.Ordinal)); -} - -public sealed record VexInTotoSubject( - string Name, - IReadOnlyDictionary Digest); - -public sealed record VexInTotoStatement( - [property: JsonPropertyName("_type")] string Type, - string PredicateType, - IReadOnlyList Subject, - VexAttestationPredicate Predicate) -{ - public static readonly string InTotoType = "https://in-toto.io/Statement/v0.1"; -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation.Models; + +public sealed record VexAttestationPredicate( + string ExportId, + string QuerySignature, + string ArtifactAlgorithm, + string ArtifactDigest, + VexExportFormat Format, + DateTimeOffset CreatedAt, + IReadOnlyList SourceProviders, + IReadOnlyDictionary Metadata) +{ + public static VexAttestationPredicate FromRequest( + VexAttestationRequest request, + IReadOnlyDictionary? metadata = null) + => new( + request.ExportId, + request.QuerySignature.Value, + request.Artifact.Algorithm, + request.Artifact.Digest, + request.Format, + request.CreatedAt, + request.SourceProviders, + metadata is null ? ImmutableDictionary.Empty : metadata.ToImmutableDictionary(StringComparer.Ordinal)); +} + +public sealed record VexInTotoSubject( + string Name, + IReadOnlyDictionary Digest); + +public sealed record VexInTotoStatement( + [property: JsonPropertyName("_type")] string Type, + string PredicateType, + IReadOnlyList Subject, + VexAttestationPredicate Predicate) +{ + public static readonly string InTotoType = "https://in-toto.io/Statement/v0.1"; +} diff --git a/src/StellaOps.Vexer.Attestation/Signing/IVexSigner.cs b/src/StellaOps.Excititor.Attestation/Signing/IVexSigner.cs similarity index 81% rename from src/StellaOps.Vexer.Attestation/Signing/IVexSigner.cs rename to src/StellaOps.Excititor.Attestation/Signing/IVexSigner.cs index d2371425..a46a303b 100644 --- a/src/StellaOps.Vexer.Attestation/Signing/IVexSigner.cs +++ b/src/StellaOps.Excititor.Attestation/Signing/IVexSigner.cs @@ -1,12 +1,12 @@ -using System; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Vexer.Attestation.Signing; - -public sealed record VexSignedPayload(string Signature, string? KeyId); - -public interface IVexSigner -{ - ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken); -} +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Attestation.Signing; + +public sealed record VexSignedPayload(string Signature, string? KeyId); + +public interface IVexSigner +{ + ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Attestation/StellaOps.Vexer.Attestation.csproj b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj similarity index 76% rename from src/StellaOps.Vexer.Attestation/StellaOps.Vexer.Attestation.csproj rename to src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj index 8874903c..0294085d 100644 --- a/src/StellaOps.Vexer.Attestation/StellaOps.Vexer.Attestation.csproj +++ b/src/StellaOps.Excititor.Attestation/StellaOps.Excititor.Attestation.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Attestation/TASKS.md b/src/StellaOps.Excititor.Attestation/TASKS.md new file mode 100644 index 00000000..18da5104 --- /dev/null +++ b/src/StellaOps.Excititor.Attestation/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-ATTEST-01-001 – In-toto predicate & DSSE builder|Team Excititor Attestation|EXCITITOR-CORE-01-001|**DONE (2025-10-16)** – Added deterministic in-toto predicate/statement models, DSSE envelope builder wired to signer abstraction, and attestation client producing metadata + diagnostics.| +|EXCITITOR-ATTEST-01-002 – Rekor v2 client integration|Team Excititor Attestation|EXCITITOR-ATTEST-01-001|**DONE (2025-10-16)** – Implemented Rekor HTTP client with retry/backoff, transparency log abstraction, DI helpers, and attestation client integration capturing Rekor metadata + diagnostics.| +|EXCITITOR-ATTEST-01-003 – Verification suite & observability|Team Excititor Attestation|EXCITITOR-ATTEST-01-002|DOING (2025-10-19) – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests. Draft plan logged in `EXCITITOR-ATTEST-01-003-plan.md` (2025-10-19).| diff --git a/src/StellaOps.Vexer.Attestation/Transparency/ITransparencyLogClient.cs b/src/StellaOps.Excititor.Attestation/Transparency/ITransparencyLogClient.cs similarity index 78% rename from src/StellaOps.Vexer.Attestation/Transparency/ITransparencyLogClient.cs rename to src/StellaOps.Excititor.Attestation/Transparency/ITransparencyLogClient.cs index de87adc9..0f01e442 100644 --- a/src/StellaOps.Vexer.Attestation/Transparency/ITransparencyLogClient.cs +++ b/src/StellaOps.Excititor.Attestation/Transparency/ITransparencyLogClient.cs @@ -1,14 +1,14 @@ -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Vexer.Attestation.Dsse; - -namespace StellaOps.Vexer.Attestation.Transparency; - -public sealed record TransparencyLogEntry(string Id, string Location, string? LogIndex, string? InclusionProofUrl); - -public interface ITransparencyLogClient -{ - ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken); - - ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken); -} +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Excititor.Attestation.Dsse; + +namespace StellaOps.Excititor.Attestation.Transparency; + +public sealed record TransparencyLogEntry(string Id, string Location, string? LogIndex, string? InclusionProofUrl); + +public interface ITransparencyLogClient +{ + ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken); + + ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClient.cs b/src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClient.cs similarity index 95% rename from src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClient.cs rename to src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClient.cs index e851f82a..7963ffc9 100644 --- a/src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClient.cs +++ b/src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClient.cs @@ -1,91 +1,91 @@ -using System.Net.Http.Json; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Attestation.Dsse; - -namespace StellaOps.Vexer.Attestation.Transparency; - -internal sealed class RekorHttpClient : ITransparencyLogClient -{ - private readonly HttpClient _httpClient; - private readonly RekorHttpClientOptions _options; - private readonly ILogger _logger; - - public RekorHttpClient(HttpClient httpClient, IOptions options, ILogger logger) - { - _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); - ArgumentNullException.ThrowIfNull(options); - _options = options.Value; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - - if (!string.IsNullOrWhiteSpace(_options.BaseAddress)) - { - _httpClient.BaseAddress = new Uri(_options.BaseAddress, UriKind.Absolute); - } - - if (!string.IsNullOrWhiteSpace(_options.ApiKey)) - { - _httpClient.DefaultRequestHeaders.Add("Authorization", _options.ApiKey); - } - } - - public async ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(envelope); - var payload = JsonSerializer.Serialize(envelope); - using var content = new StringContent(payload); - content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); - - HttpResponseMessage? response = null; - for (var attempt = 0; attempt < _options.RetryCount; attempt++) - { - response = await _httpClient.PostAsync("/api/v2/log/entries", content, cancellationToken).ConfigureAwait(false); - if (response.IsSuccessStatusCode) - { - break; - } - - _logger.LogWarning("Rekor submission failed with status {Status}; attempt {Attempt}", response.StatusCode, attempt + 1); - if (attempt + 1 < _options.RetryCount) - { - await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false); - } - } - - if (response is null || !response.IsSuccessStatusCode) - { - throw new HttpRequestException($"Failed to submit attestation to Rekor ({response?.StatusCode})."); - } - - var entryLocation = response.Headers.Location?.ToString() ?? string.Empty; - var body = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - var entry = ParseEntryLocation(entryLocation, body); - _logger.LogInformation("Rekor entry recorded at {Location}", entry.Location); - return entry; - } - - public async ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(entryLocation)) - { - return false; - } - - var response = await _httpClient.GetAsync(entryLocation, cancellationToken).ConfigureAwait(false); - return response.IsSuccessStatusCode; - } - - private static TransparencyLogEntry ParseEntryLocation(string location, JsonElement body) - { - var id = body.TryGetProperty("uuid", out var uuid) ? uuid.GetString() ?? string.Empty : Guid.NewGuid().ToString(); - var logIndex = body.TryGetProperty("logIndex", out var logIndexElement) ? logIndexElement.GetString() : null; - string? inclusionProof = null; - if (body.TryGetProperty("verification", out var verification) && verification.TryGetProperty("inclusionProof", out var inclusion)) - { - inclusionProof = inclusion.GetProperty("logIndex").GetRawText(); - } - - return new TransparencyLogEntry(id, location, logIndex, inclusionProof); - } -} +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Attestation.Dsse; + +namespace StellaOps.Excititor.Attestation.Transparency; + +internal sealed class RekorHttpClient : ITransparencyLogClient +{ + private readonly HttpClient _httpClient; + private readonly RekorHttpClientOptions _options; + private readonly ILogger _logger; + + public RekorHttpClient(HttpClient httpClient, IOptions options, ILogger logger) + { + _httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (!string.IsNullOrWhiteSpace(_options.BaseAddress)) + { + _httpClient.BaseAddress = new Uri(_options.BaseAddress, UriKind.Absolute); + } + + if (!string.IsNullOrWhiteSpace(_options.ApiKey)) + { + _httpClient.DefaultRequestHeaders.Add("Authorization", _options.ApiKey); + } + } + + public async ValueTask SubmitAsync(DsseEnvelope envelope, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(envelope); + var payload = JsonSerializer.Serialize(envelope); + using var content = new StringContent(payload); + content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"); + + HttpResponseMessage? response = null; + for (var attempt = 0; attempt < _options.RetryCount; attempt++) + { + response = await _httpClient.PostAsync("/api/v2/log/entries", content, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + break; + } + + _logger.LogWarning("Rekor submission failed with status {Status}; attempt {Attempt}", response.StatusCode, attempt + 1); + if (attempt + 1 < _options.RetryCount) + { + await Task.Delay(_options.RetryDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (response is null || !response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Failed to submit attestation to Rekor ({response?.StatusCode})."); + } + + var entryLocation = response.Headers.Location?.ToString() ?? string.Empty; + var body = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var entry = ParseEntryLocation(entryLocation, body); + _logger.LogInformation("Rekor entry recorded at {Location}", entry.Location); + return entry; + } + + public async ValueTask VerifyAsync(string entryLocation, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(entryLocation)) + { + return false; + } + + var response = await _httpClient.GetAsync(entryLocation, cancellationToken).ConfigureAwait(false); + return response.IsSuccessStatusCode; + } + + private static TransparencyLogEntry ParseEntryLocation(string location, JsonElement body) + { + var id = body.TryGetProperty("uuid", out var uuid) ? uuid.GetString() ?? string.Empty : Guid.NewGuid().ToString(); + var logIndex = body.TryGetProperty("logIndex", out var logIndexElement) ? logIndexElement.GetString() : null; + string? inclusionProof = null; + if (body.TryGetProperty("verification", out var verification) && verification.TryGetProperty("inclusionProof", out var inclusion)) + { + inclusionProof = inclusion.GetProperty("logIndex").GetRawText(); + } + + return new TransparencyLogEntry(id, location, logIndex, inclusionProof); + } +} diff --git a/src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClientOptions.cs b/src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClientOptions.cs similarity index 81% rename from src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClientOptions.cs rename to src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClientOptions.cs index 1565e260..435ca7cc 100644 --- a/src/StellaOps.Vexer.Attestation/Transparency/RekorHttpClientOptions.cs +++ b/src/StellaOps.Excititor.Attestation/Transparency/RekorHttpClientOptions.cs @@ -1,13 +1,13 @@ -namespace StellaOps.Vexer.Attestation.Transparency; - -public sealed class RekorHttpClientOptions -{ - public string BaseAddress { get; set; } = "https://rekor.sigstore.dev"; - - public string? ApiKey { get; set; } - = null; - - public int RetryCount { get; set; } = 3; - - public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(2); -} +namespace StellaOps.Excititor.Attestation.Transparency; + +public sealed class RekorHttpClientOptions +{ + public string BaseAddress { get; set; } = "https://rekor.sigstore.dev"; + + public string? ApiKey { get; set; } + = null; + + public int RetryCount { get; set; } = 3; + + public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(2); +} diff --git a/src/StellaOps.Vexer.Attestation/VexAttestationClient.cs b/src/StellaOps.Excititor.Attestation/VexAttestationClient.cs similarity index 91% rename from src/StellaOps.Vexer.Attestation/VexAttestationClient.cs rename to src/StellaOps.Excititor.Attestation/VexAttestationClient.cs index 94f6f214..01bf817e 100644 --- a/src/StellaOps.Vexer.Attestation/VexAttestationClient.cs +++ b/src/StellaOps.Excititor.Attestation/VexAttestationClient.cs @@ -1,108 +1,108 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Attestation.Dsse; -using StellaOps.Vexer.Attestation.Models; -using StellaOps.Vexer.Attestation.Signing; -using StellaOps.Vexer.Attestation.Transparency; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Attestation; - -public sealed class VexAttestationClientOptions -{ - public IReadOnlyDictionary DefaultMetadata { get; set; } = ImmutableDictionary.Empty; -} - -public sealed class VexAttestationClient : IVexAttestationClient -{ - private readonly VexDsseBuilder _builder; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly IReadOnlyDictionary _defaultMetadata; - private readonly ITransparencyLogClient? _transparencyLogClient; - - public VexAttestationClient( - VexDsseBuilder builder, - IOptions options, - ILogger logger, - TimeProvider? timeProvider = null, - ITransparencyLogClient? transparencyLogClient = null) - { - _builder = builder ?? throw new ArgumentNullException(nameof(builder)); - ArgumentNullException.ThrowIfNull(options); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? TimeProvider.System; - _defaultMetadata = options.Value.DefaultMetadata; - _transparencyLogClient = transparencyLogClient; - } - - public async ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(request); - var mergedMetadata = MergeMetadata(request.Metadata, _defaultMetadata); - - var envelope = await _builder.CreateEnvelopeAsync(request, mergedMetadata, cancellationToken).ConfigureAwait(false); - var envelopeDigest = VexDsseBuilder.ComputeEnvelopeDigest(envelope); - var signedAt = _timeProvider.GetUtcNow(); - - var diagnosticsBuilder = ImmutableDictionary.Empty - .Add("envelope", JsonSerializer.Serialize(envelope)) - .Add("predicateType", "https://stella-ops.org/attestations/vex-export"); - - VexRekorReference? rekorReference = null; - if (_transparencyLogClient is not null) - { - try - { - var entry = await _transparencyLogClient.SubmitAsync(envelope, cancellationToken).ConfigureAwait(false); - rekorReference = new VexRekorReference("0.2", entry.Location, entry.LogIndex, entry.InclusionProofUrl is not null ? new Uri(entry.InclusionProofUrl) : null); - diagnosticsBuilder = diagnosticsBuilder.Add("rekorLocation", entry.Location); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to submit attestation to Rekor transparency log"); - throw; - } - } - - var metadata = new VexAttestationMetadata( - predicateType: "https://stella-ops.org/attestations/vex-export", - rekor: rekorReference, - envelopeDigest: envelopeDigest, - signedAt: signedAt); - - _logger.LogInformation("Generated DSSE envelope for export {ExportId} ({Digest})", request.ExportId, envelopeDigest); - - return new VexAttestationResponse(metadata, diagnosticsBuilder); - } - - public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) - { - // Placeholder until verification flow is implemented in VEXER-ATTEST-01-003. - return ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); - } - - private static IReadOnlyDictionary MergeMetadata( - IReadOnlyDictionary requestMetadata, - IReadOnlyDictionary defaults) - { - if (defaults.Count == 0) - { - return requestMetadata; - } - - var merged = new Dictionary(defaults, StringComparer.Ordinal); - foreach (var kvp in requestMetadata) - { - merged[kvp.Key] = kvp.Value; - } - - return merged.ToImmutableDictionary(StringComparer.Ordinal); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Attestation.Dsse; +using StellaOps.Excititor.Attestation.Models; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Attestation.Transparency; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Attestation; + +public sealed class VexAttestationClientOptions +{ + public IReadOnlyDictionary DefaultMetadata { get; set; } = ImmutableDictionary.Empty; +} + +public sealed class VexAttestationClient : IVexAttestationClient +{ + private readonly VexDsseBuilder _builder; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly IReadOnlyDictionary _defaultMetadata; + private readonly ITransparencyLogClient? _transparencyLogClient; + + public VexAttestationClient( + VexDsseBuilder builder, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null, + ITransparencyLogClient? transparencyLogClient = null) + { + _builder = builder ?? throw new ArgumentNullException(nameof(builder)); + ArgumentNullException.ThrowIfNull(options); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _defaultMetadata = options.Value.DefaultMetadata; + _transparencyLogClient = transparencyLogClient; + } + + public async ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + var mergedMetadata = MergeMetadata(request.Metadata, _defaultMetadata); + + var envelope = await _builder.CreateEnvelopeAsync(request, mergedMetadata, cancellationToken).ConfigureAwait(false); + var envelopeDigest = VexDsseBuilder.ComputeEnvelopeDigest(envelope); + var signedAt = _timeProvider.GetUtcNow(); + + var diagnosticsBuilder = ImmutableDictionary.Empty + .Add("envelope", JsonSerializer.Serialize(envelope)) + .Add("predicateType", "https://stella-ops.org/attestations/vex-export"); + + VexRekorReference? rekorReference = null; + if (_transparencyLogClient is not null) + { + try + { + var entry = await _transparencyLogClient.SubmitAsync(envelope, cancellationToken).ConfigureAwait(false); + rekorReference = new VexRekorReference("0.2", entry.Location, entry.LogIndex, entry.InclusionProofUrl is not null ? new Uri(entry.InclusionProofUrl) : null); + diagnosticsBuilder = diagnosticsBuilder.Add("rekorLocation", entry.Location); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to submit attestation to Rekor transparency log"); + throw; + } + } + + var metadata = new VexAttestationMetadata( + predicateType: "https://stella-ops.org/attestations/vex-export", + rekor: rekorReference, + envelopeDigest: envelopeDigest, + signedAt: signedAt); + + _logger.LogInformation("Generated DSSE envelope for export {ExportId} ({Digest})", request.ExportId, envelopeDigest); + + return new VexAttestationResponse(metadata, diagnosticsBuilder); + } + + public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + // Placeholder until verification flow is implemented in EXCITITOR-ATTEST-01-003. + return ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); + } + + private static IReadOnlyDictionary MergeMetadata( + IReadOnlyDictionary requestMetadata, + IReadOnlyDictionary defaults) + { + if (defaults.Count == 0) + { + return requestMetadata; + } + + var merged = new Dictionary(defaults, StringComparer.Ordinal); + foreach (var kvp in requestMetadata) + { + merged[kvp.Key] = kvp.Value; + } + + return merged.ToImmutableDictionary(StringComparer.Ordinal); + } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md b/src/StellaOps.Excititor.Connectors.Abstractions/AGENTS.md similarity index 68% rename from src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md rename to src/StellaOps.Excititor.Connectors.Abstractions/AGENTS.md index dadb0d0d..7d428357 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.Abstractions/AGENTS.md @@ -1,22 +1,22 @@ -# AGENTS -## Role -Defines shared connector infrastructure for Vexer, including base contexts, result contracts, configuration binding, and helper utilities reused by all connector plug-ins. -## Scope -- `IVexConnector` context implementation, raw store helpers, verification hooks, and telemetry utilities. -- Configuration primitives (YAML parsing, secrets handling guidelines) and options validation. -- Connector lifecycle helpers for retries, paging, `.well-known` discovery, and resume markers. -- Documentation for connector packaging, plugin manifest metadata, and DI registration (see `docs/dev/30_VEXER_CONNECTOR_GUIDE.md` and `docs/dev/templates/vexer-connector/`). -## Participants -- All Vexer connector projects reference this module to obtain base classes and context services. -- WebService/Worker instantiate connectors via plugin loader leveraging abstractions defined here. -## Interfaces & contracts -- Connector context, result, and telemetry interfaces; `VexConnectorDescriptor`, `VexConnectorBase`, options binder/validators, authentication helpers. -- Utility classes for HTTP clients, throttling, and deterministic logging. -## In/Out of scope -In: shared abstractions, helper utilities, configuration binding, documentation for connector authors. -Out: provider-specific logic (implemented in individual connector modules), storage persistence, HTTP host code. -## Observability & security expectations -- Provide structured logging helpers, correlation IDs, and metrics instrumentation toggles for connectors. -- Enforce redaction of secrets in logs and config dumps. -## Tests -- Abstraction/unit tests will live in `../StellaOps.Vexer.Connectors.Abstractions.Tests`, covering default behaviors and sample harness. +# AGENTS +## Role +Defines shared connector infrastructure for Excititor, including base contexts, result contracts, configuration binding, and helper utilities reused by all connector plug-ins. +## Scope +- `IVexConnector` context implementation, raw store helpers, verification hooks, and telemetry utilities. +- Configuration primitives (YAML parsing, secrets handling guidelines) and options validation. +- Connector lifecycle helpers for retries, paging, `.well-known` discovery, and resume markers. +- Documentation for connector packaging, plugin manifest metadata, and DI registration (see `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md` and `docs/dev/templates/excititor-connector/`). +## Participants +- All Excititor connector projects reference this module to obtain base classes and context services. +- WebService/Worker instantiate connectors via plugin loader leveraging abstractions defined here. +## Interfaces & contracts +- Connector context, result, and telemetry interfaces; `VexConnectorDescriptor`, `VexConnectorBase`, options binder/validators, authentication helpers. +- Utility classes for HTTP clients, throttling, and deterministic logging. +## In/Out of scope +In: shared abstractions, helper utilities, configuration binding, documentation for connector authors. +Out: provider-specific logic (implemented in individual connector modules), storage persistence, HTTP host code. +## Observability & security expectations +- Provide structured logging helpers, correlation IDs, and metrics instrumentation toggles for connectors. +- Enforce redaction of secrets in logs and config dumps. +## Tests +- Abstraction/unit tests will live in `../StellaOps.Excititor.Connectors.Abstractions.Tests`, covering default behaviors and sample harness. diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.Abstractions/IVexConnectorOptionsValidator.cs similarity index 84% rename from src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/IVexConnectorOptionsValidator.cs index 007004d7..06c4562f 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/IVexConnectorOptionsValidator.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/IVexConnectorOptionsValidator.cs @@ -1,12 +1,12 @@ -using System.Collections.Generic; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Custom validator hook executed after connector options are bound. -/// -/// Connector-specific options type. -public interface IVexConnectorOptionsValidator -{ - void Validate(VexConnectorDescriptor descriptor, TOptions options, IList errors); -} +using System.Collections.Generic; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Custom validator hook executed after connector options are bound. +/// +/// Connector-specific options type. +public interface IVexConnectorOptionsValidator +{ + void Validate(VexConnectorDescriptor descriptor, TOptions options, IList errors); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj similarity index 84% rename from src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj rename to src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj index 63f633c3..733276ad 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/StellaOps.Vexer.Connectors.Abstractions.csproj +++ b/src/StellaOps.Excititor.Connectors.Abstractions/StellaOps.Excititor.Connectors.Abstractions.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md b/src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md new file mode 100644 index 00000000..a25d1b10 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Abstractions/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-ABS-01-001 – Connector context & base classes|Team Excititor Connectors|EXCITITOR-CORE-01-003|**DONE (2025-10-17)** – Added `StellaOps.Excititor.Connectors.Abstractions` project with `VexConnectorBase`, deterministic logging scopes, metadata builder helpers, and connector descriptors; docs updated to highlight the shared abstractions.| +|EXCITITOR-CONN-ABS-01-002 – YAML options & validation|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Delivered `VexConnectorOptionsBinder` + binder options/validators, environment-variable expansion, data-annotation checks, and custom validation hooks with documentation updates covering the workflow.| +|EXCITITOR-CONN-ABS-01-003 – Plugin packaging & docs|Team Excititor Connectors|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Authored `docs/dev/30_EXCITITOR_CONNECTOR_GUIDE.md`, added quick-start template under `docs/dev/templates/excititor-connector/`, and updated module docs to reference the packaging workflow.| diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorBase.cs similarity index 93% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorBase.cs index ba312419..fa34a719 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorBase.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorBase.cs @@ -1,99 +1,99 @@ -using System.Collections.Immutable; -using System.Security.Cryptography; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Convenience base class for implementing . -/// -public abstract class VexConnectorBase : IVexConnector -{ - protected VexConnectorBase(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider? timeProvider = null) - { - Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor)); - Logger = logger ?? throw new ArgumentNullException(nameof(logger)); - TimeProvider = timeProvider ?? TimeProvider.System; - } - - /// - public string Id => Descriptor.Id; - - /// - public VexProviderKind Kind => Descriptor.Kind; - - protected VexConnectorDescriptor Descriptor { get; } - - protected ILogger Logger { get; } - - protected TimeProvider TimeProvider { get; } - - protected DateTimeOffset UtcNow() => TimeProvider.GetUtcNow(); - - protected VexRawDocument CreateRawDocument( - VexDocumentFormat format, - Uri sourceUri, - ReadOnlyMemory content, - ImmutableDictionary? metadata = null) - { - if (sourceUri is null) - { - throw new ArgumentNullException(nameof(sourceUri)); - } - - var digest = ComputeSha256(content.Span); - var captured = TimeProvider.GetUtcNow(); - return new VexRawDocument( - Descriptor.Id, - format, - sourceUri, - captured, - digest, - content, - metadata ?? ImmutableDictionary.Empty); - } - - protected IDisposable BeginConnectorScope(string operation, IReadOnlyDictionary? metadata = null) - => VexConnectorLogScope.Begin(Logger, Descriptor, operation, metadata); - - protected void LogConnectorEvent(LogLevel level, string eventName, string message, IReadOnlyDictionary? metadata = null, Exception? exception = null) - { - using var scope = BeginConnectorScope(eventName, metadata); - if (exception is null) - { - Logger.Log(level, "{Message}", message); - } - else - { - Logger.Log(level, exception, "{Message}", message); - } - } - - protected ImmutableDictionary BuildMetadata(Action configure) - { - ArgumentNullException.ThrowIfNull(configure); - var builder = new VexConnectorMetadataBuilder(); - configure(builder); - return builder.Build(); - } - - private static string ComputeSha256(ReadOnlySpan content) - { - Span buffer = stackalloc byte[32]; - if (SHA256.TryHashData(content, buffer, out _)) - { - return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); - } - - using var sha = SHA256.Create(); - var hash = sha.ComputeHash(content.ToArray()); - return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); - } - - public abstract ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken); - - public abstract IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken); - - public abstract ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); -} +using System.Collections.Immutable; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Convenience base class for implementing . +/// +public abstract class VexConnectorBase : IVexConnector +{ + protected VexConnectorBase(VexConnectorDescriptor descriptor, ILogger logger, TimeProvider? timeProvider = null) + { + Descriptor = descriptor ?? throw new ArgumentNullException(nameof(descriptor)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + TimeProvider = timeProvider ?? TimeProvider.System; + } + + /// + public string Id => Descriptor.Id; + + /// + public VexProviderKind Kind => Descriptor.Kind; + + public VexConnectorDescriptor Descriptor { get; } + + protected ILogger Logger { get; } + + protected TimeProvider TimeProvider { get; } + + protected DateTimeOffset UtcNow() => TimeProvider.GetUtcNow(); + + protected VexRawDocument CreateRawDocument( + VexDocumentFormat format, + Uri sourceUri, + ReadOnlyMemory content, + ImmutableDictionary? metadata = null) + { + if (sourceUri is null) + { + throw new ArgumentNullException(nameof(sourceUri)); + } + + var digest = ComputeSha256(content.Span); + var captured = TimeProvider.GetUtcNow(); + return new VexRawDocument( + Descriptor.Id, + format, + sourceUri, + captured, + digest, + content, + metadata ?? ImmutableDictionary.Empty); + } + + protected IDisposable BeginConnectorScope(string operation, IReadOnlyDictionary? metadata = null) + => VexConnectorLogScope.Begin(Logger, Descriptor, operation, metadata); + + protected void LogConnectorEvent(LogLevel level, string eventName, string message, IReadOnlyDictionary? metadata = null, Exception? exception = null) + { + using var scope = BeginConnectorScope(eventName, metadata); + if (exception is null) + { + Logger.Log(level, "{Message}", message); + } + else + { + Logger.Log(level, exception, "{Message}", message); + } + } + + protected ImmutableDictionary BuildMetadata(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new VexConnectorMetadataBuilder(); + configure(builder); + return builder.Build(); + } + + private static string ComputeSha256(ReadOnlySpan content) + { + Span buffer = stackalloc byte[32]; + if (SHA256.TryHashData(content, buffer, out _)) + { + return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); + } + + using var sha = SHA256.Create(); + var hash = sha.ComputeHash(content.ToArray()); + return "sha256:" + Convert.ToHexString(hash).ToLowerInvariant(); + } + + public abstract ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken); + + public abstract IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken); + + public abstract ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorDescriptor.cs similarity index 88% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorDescriptor.cs index 4e7eb211..2f380bd4 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorDescriptor.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorDescriptor.cs @@ -1,54 +1,54 @@ -using System.Collections.Immutable; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Static descriptor for a Vexer connector plug-in. -/// -public sealed record VexConnectorDescriptor -{ - public VexConnectorDescriptor(string id, VexProviderKind kind, string displayName) - { - if (string.IsNullOrWhiteSpace(id)) - { - throw new ArgumentException("Connector id must be provided.", nameof(id)); - } - - Id = id; - Kind = kind; - DisplayName = string.IsNullOrWhiteSpace(displayName) ? id : displayName; - } - - /// - /// Stable connector identifier (matches provider id). - /// - public string Id { get; } - - /// - /// Provider kind served by the connector. - /// - public VexProviderKind Kind { get; } - - /// - /// Human friendly name used in logs/diagnostics. - /// - public string DisplayName { get; } - - /// - /// Optional friendly description. - /// - public string? Description { get; init; } - - /// - /// Document formats the connector is expected to emit. - /// - public ImmutableArray SupportedFormats { get; init; } = ImmutableArray.Empty; - - /// - /// Optional tags surfaced in diagnostics (e.g. "beta", "offline"). - /// - public ImmutableArray Tags { get; init; } = ImmutableArray.Empty; - - public override string ToString() => $"{Id} ({Kind})"; -} +using System.Collections.Immutable; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Static descriptor for a Excititor connector plug-in. +/// +public sealed record VexConnectorDescriptor +{ + public VexConnectorDescriptor(string id, VexProviderKind kind, string displayName) + { + if (string.IsNullOrWhiteSpace(id)) + { + throw new ArgumentException("Connector id must be provided.", nameof(id)); + } + + Id = id; + Kind = kind; + DisplayName = string.IsNullOrWhiteSpace(displayName) ? id : displayName; + } + + /// + /// Stable connector identifier (matches provider id). + /// + public string Id { get; } + + /// + /// Provider kind served by the connector. + /// + public VexProviderKind Kind { get; } + + /// + /// Human friendly name used in logs/diagnostics. + /// + public string DisplayName { get; } + + /// + /// Optional friendly description. + /// + public string? Description { get; init; } + + /// + /// Document formats the connector is expected to emit. + /// + public ImmutableArray SupportedFormats { get; init; } = ImmutableArray.Empty; + + /// + /// Optional tags surfaced in diagnostics (e.g. "beta", "offline"). + /// + public ImmutableArray Tags { get; init; } = ImmutableArray.Empty; + + public override string ToString() => $"{Id} ({Kind})"; +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorLogScope.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorLogScope.cs index 5cc8da82..268aed54 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorLogScope.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorLogScope.cs @@ -1,50 +1,50 @@ -using System.Linq; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Helper to establish deterministic logging scopes for connector operations. -/// -public static class VexConnectorLogScope -{ - public static IDisposable Begin(ILogger logger, VexConnectorDescriptor descriptor, string operation, IReadOnlyDictionary? metadata = null) - { - ArgumentNullException.ThrowIfNull(logger); - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentException.ThrowIfNullOrEmpty(operation); - - var scopeValues = new List> - { - new("vex.connector.id", descriptor.Id), - new("vex.connector.kind", descriptor.Kind.ToString()), - new("vex.connector.operation", operation), - }; - - if (!string.Equals(descriptor.DisplayName, descriptor.Id, StringComparison.Ordinal)) - { - scopeValues.Add(new KeyValuePair("vex.connector.displayName", descriptor.DisplayName)); - } - - if (!string.IsNullOrWhiteSpace(descriptor.Description)) - { - scopeValues.Add(new KeyValuePair("vex.connector.description", descriptor.Description)); - } - - if (!descriptor.Tags.IsDefaultOrEmpty) - { - scopeValues.Add(new KeyValuePair("vex.connector.tags", string.Join(",", descriptor.Tags))); - } - - if (metadata is not null) - { - foreach (var kvp in metadata.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) - { - scopeValues.Add(new KeyValuePair($"vex.{kvp.Key}", kvp.Value)); - } - } - - return logger.BeginScope(scopeValues)!; - } -} +using System.Linq; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Helper to establish deterministic logging scopes for connector operations. +/// +public static class VexConnectorLogScope +{ + public static IDisposable Begin(ILogger logger, VexConnectorDescriptor descriptor, string operation, IReadOnlyDictionary? metadata = null) + { + ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentException.ThrowIfNullOrEmpty(operation); + + var scopeValues = new List> + { + new("vex.connector.id", descriptor.Id), + new("vex.connector.kind", descriptor.Kind.ToString()), + new("vex.connector.operation", operation), + }; + + if (!string.Equals(descriptor.DisplayName, descriptor.Id, StringComparison.Ordinal)) + { + scopeValues.Add(new KeyValuePair("vex.connector.displayName", descriptor.DisplayName)); + } + + if (!string.IsNullOrWhiteSpace(descriptor.Description)) + { + scopeValues.Add(new KeyValuePair("vex.connector.description", descriptor.Description)); + } + + if (!descriptor.Tags.IsDefaultOrEmpty) + { + scopeValues.Add(new KeyValuePair("vex.connector.tags", string.Join(",", descriptor.Tags))); + } + + if (metadata is not null) + { + foreach (var kvp in metadata.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + scopeValues.Add(new KeyValuePair($"vex.{kvp.Key}", kvp.Value)); + } + } + + return logger.BeginScope(scopeValues)!; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorMetadataBuilder.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorMetadataBuilder.cs index 8404ab94..d05a8819 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorMetadataBuilder.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorMetadataBuilder.cs @@ -1,37 +1,37 @@ -using System.Collections.Immutable; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Builds deterministic metadata dictionaries for raw documents and logging scopes. -/// -public sealed class VexConnectorMetadataBuilder -{ - private readonly SortedDictionary _values = new(StringComparer.Ordinal); - - public VexConnectorMetadataBuilder Add(string key, string? value) - { - if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value)) - { - _values[key] = value!; - } - - return this; - } - - public VexConnectorMetadataBuilder Add(string key, DateTimeOffset value) - => Add(key, value.ToUniversalTime().ToString("O")); - - public VexConnectorMetadataBuilder AddRange(IEnumerable> items) - { - foreach (var item in items) - { - Add(item.Key, item.Value); - } - - return this; - } - - public ImmutableDictionary Build() - => _values.ToImmutableDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); -} +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Builds deterministic metadata dictionaries for raw documents and logging scopes. +/// +public sealed class VexConnectorMetadataBuilder +{ + private readonly SortedDictionary _values = new(StringComparer.Ordinal); + + public VexConnectorMetadataBuilder Add(string key, string? value) + { + if (!string.IsNullOrWhiteSpace(key) && !string.IsNullOrWhiteSpace(value)) + { + _values[key] = value!; + } + + return this; + } + + public VexConnectorMetadataBuilder Add(string key, DateTimeOffset value) + => Add(key, value.ToUniversalTime().ToString("O")); + + public VexConnectorMetadataBuilder AddRange(IEnumerable> items) + { + foreach (var item in items) + { + Add(item.Key, item.Value); + } + + return this; + } + + public ImmutableDictionary Build() + => _values.ToImmutableDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal); +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinder.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinder.cs index d307fbca..dc89c417 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinder.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinder.cs @@ -1,157 +1,157 @@ -using System.Collections.Immutable; -using System.ComponentModel.DataAnnotations; -using System.Linq; -using Microsoft.Extensions.Configuration; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Provides strongly typed binding and validation for connector options. -/// -public static class VexConnectorOptionsBinder -{ - public static TOptions Bind( - VexConnectorDescriptor descriptor, - VexConnectorSettings settings, - VexConnectorOptionsBinderOptions? options = null, - IEnumerable>? validators = null) - where TOptions : class, new() - { - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentNullException.ThrowIfNull(settings); - - var binderSettings = options ?? new VexConnectorOptionsBinderOptions(); - var transformed = TransformValues(settings, binderSettings); - - var configuration = BuildConfiguration(transformed); - - var result = new TOptions(); - var errors = new List(); - - try - { - configuration.Bind( - result, - binderOptions => binderOptions.ErrorOnUnknownConfiguration = !binderSettings.AllowUnknownKeys); - } - catch (InvalidOperationException ex) when (!binderSettings.AllowUnknownKeys) - { - errors.Add(ex.Message); - } - - binderSettings.PostConfigure?.Invoke(result); - - if (binderSettings.ValidateDataAnnotations) - { - ValidateDataAnnotations(result, errors); - } - - if (validators is not null) - { - foreach (var validator in validators) - { - validator?.Validate(descriptor, result, errors); - } - } - - if (errors.Count > 0) - { - throw new VexConnectorOptionsValidationException(descriptor.Id, errors); - } - - return result; - } - - private static ImmutableDictionary TransformValues( - VexConnectorSettings settings, - VexConnectorOptionsBinderOptions binderOptions) - { - var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); - foreach (var kvp in settings.Values) - { - var value = kvp.Value; - if (binderOptions.TrimWhitespace && value is not null) - { - value = value.Trim(); - } - - if (binderOptions.TreatEmptyAsNull && string.IsNullOrEmpty(value)) - { - value = null; - } - - if (value is not null && binderOptions.ExpandEnvironmentVariables) - { - value = Environment.ExpandEnvironmentVariables(value); - } - - if (binderOptions.ValueTransformer is not null) - { - value = binderOptions.ValueTransformer.Invoke(kvp.Key, value); - } - - builder[kvp.Key] = value; - } - - return builder.ToImmutable(); - } - - private static IConfiguration BuildConfiguration(ImmutableDictionary values) - { - var sources = new List>(); - foreach (var kvp in values) - { - if (kvp.Value is not null) - { - sources.Add(new KeyValuePair(kvp.Key, kvp.Value)); - } - } - - var configurationBuilder = new ConfigurationBuilder(); - configurationBuilder.Add(new DictionaryConfigurationSource(sources)); - return configurationBuilder.Build(); - } - - private static void ValidateDataAnnotations(TOptions options, IList errors) - { - var validationResults = new List(); - var validationContext = new ValidationContext(options!); - if (!Validator.TryValidateObject(options!, validationContext, validationResults, validateAllProperties: true)) - { - foreach (var validationResult in validationResults) - { - if (!string.IsNullOrWhiteSpace(validationResult.ErrorMessage)) - { - errors.Add(validationResult.ErrorMessage); - } - } - } - } - - private sealed class DictionaryConfigurationSource : IConfigurationSource - { - private readonly IReadOnlyList> _data; - - public DictionaryConfigurationSource(IEnumerable> data) - { - _data = data?.ToList() ?? new List>(); - } - - public IConfigurationProvider Build(IConfigurationBuilder builder) => new DictionaryConfigurationProvider(_data); - } - - private sealed class DictionaryConfigurationProvider : ConfigurationProvider - { - public DictionaryConfigurationProvider(IEnumerable> data) - { - foreach (var pair in data) - { - if (pair.Value is not null) - { - Data[pair.Key] = pair.Value; - } - } - } - } -} +using System.Collections.Immutable; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using Microsoft.Extensions.Configuration; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Provides strongly typed binding and validation for connector options. +/// +public static class VexConnectorOptionsBinder +{ + public static TOptions Bind( + VexConnectorDescriptor descriptor, + VexConnectorSettings settings, + VexConnectorOptionsBinderOptions? options = null, + IEnumerable>? validators = null) + where TOptions : class, new() + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(settings); + + var binderSettings = options ?? new VexConnectorOptionsBinderOptions(); + var transformed = TransformValues(settings, binderSettings); + + var configuration = BuildConfiguration(transformed); + + var result = new TOptions(); + var errors = new List(); + + try + { + configuration.Bind( + result, + binderOptions => binderOptions.ErrorOnUnknownConfiguration = !binderSettings.AllowUnknownKeys); + } + catch (InvalidOperationException ex) when (!binderSettings.AllowUnknownKeys) + { + errors.Add(ex.Message); + } + + binderSettings.PostConfigure?.Invoke(result); + + if (binderSettings.ValidateDataAnnotations) + { + ValidateDataAnnotations(result, errors); + } + + if (validators is not null) + { + foreach (var validator in validators) + { + validator?.Validate(descriptor, result, errors); + } + } + + if (errors.Count > 0) + { + throw new VexConnectorOptionsValidationException(descriptor.Id, errors); + } + + return result; + } + + private static ImmutableDictionary TransformValues( + VexConnectorSettings settings, + VexConnectorOptionsBinderOptions binderOptions) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var kvp in settings.Values) + { + var value = kvp.Value; + if (binderOptions.TrimWhitespace && value is not null) + { + value = value.Trim(); + } + + if (binderOptions.TreatEmptyAsNull && string.IsNullOrEmpty(value)) + { + value = null; + } + + if (value is not null && binderOptions.ExpandEnvironmentVariables) + { + value = Environment.ExpandEnvironmentVariables(value); + } + + if (binderOptions.ValueTransformer is not null) + { + value = binderOptions.ValueTransformer.Invoke(kvp.Key, value); + } + + builder[kvp.Key] = value; + } + + return builder.ToImmutable(); + } + + private static IConfiguration BuildConfiguration(ImmutableDictionary values) + { + var sources = new List>(); + foreach (var kvp in values) + { + if (kvp.Value is not null) + { + sources.Add(new KeyValuePair(kvp.Key, kvp.Value)); + } + } + + var configurationBuilder = new ConfigurationBuilder(); + configurationBuilder.Add(new DictionaryConfigurationSource(sources)); + return configurationBuilder.Build(); + } + + private static void ValidateDataAnnotations(TOptions options, IList errors) + { + var validationResults = new List(); + var validationContext = new ValidationContext(options!); + if (!Validator.TryValidateObject(options!, validationContext, validationResults, validateAllProperties: true)) + { + foreach (var validationResult in validationResults) + { + if (!string.IsNullOrWhiteSpace(validationResult.ErrorMessage)) + { + errors.Add(validationResult.ErrorMessage); + } + } + } + } + + private sealed class DictionaryConfigurationSource : IConfigurationSource + { + private readonly IReadOnlyList> _data; + + public DictionaryConfigurationSource(IEnumerable> data) + { + _data = data?.ToList() ?? new List>(); + } + + public IConfigurationProvider Build(IConfigurationBuilder builder) => new DictionaryConfigurationProvider(_data); + } + + private sealed class DictionaryConfigurationProvider : ConfigurationProvider + { + public DictionaryConfigurationProvider(IEnumerable> data) + { + foreach (var pair in data) + { + if (pair.Value is not null) + { + Data[pair.Key] = pair.Value; + } + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs similarity index 93% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs index c4710999..33067caf 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsBinderOptions.cs @@ -1,45 +1,45 @@ -namespace StellaOps.Vexer.Connectors.Abstractions; - -/// -/// Customisation options for connector options binding. -/// -public sealed class VexConnectorOptionsBinderOptions -{ - /// - /// Indicates whether environment variables should be expanded in option values. - /// Defaults to true. - /// - public bool ExpandEnvironmentVariables { get; set; } = true; - - /// - /// When true the binder trims whitespace around option values. - /// - public bool TrimWhitespace { get; set; } = true; - - /// - /// Converts empty strings to null before binding. Default: true. - /// - public bool TreatEmptyAsNull { get; set; } = true; - - /// - /// When false, binding fails if unknown configuration keys are provided. - /// Default: true (permitting unknown keys). - /// - public bool AllowUnknownKeys { get; set; } = true; - - /// - /// Enables validation after binding. - /// Default: true. - /// - public bool ValidateDataAnnotations { get; set; } = true; - - /// - /// Optional post-configuration callback executed after binding. - /// - public Action? PostConfigure { get; set; } - - /// - /// Optional hook to transform raw configuration values before binding. - /// - public Func? ValueTransformer { get; set; } -} +namespace StellaOps.Excititor.Connectors.Abstractions; + +/// +/// Customisation options for connector options binding. +/// +public sealed class VexConnectorOptionsBinderOptions +{ + /// + /// Indicates whether environment variables should be expanded in option values. + /// Defaults to true. + /// + public bool ExpandEnvironmentVariables { get; set; } = true; + + /// + /// When true the binder trims whitespace around option values. + /// + public bool TrimWhitespace { get; set; } = true; + + /// + /// Converts empty strings to null before binding. Default: true. + /// + public bool TreatEmptyAsNull { get; set; } = true; + + /// + /// When false, binding fails if unknown configuration keys are provided. + /// Default: true (permitting unknown keys). + /// + public bool AllowUnknownKeys { get; set; } = true; + + /// + /// Enables validation after binding. + /// Default: true. + /// + public bool ValidateDataAnnotations { get; set; } = true; + + /// + /// Optional post-configuration callback executed after binding. + /// + public Action? PostConfigure { get; set; } + + /// + /// Optional hook to transform raw configuration values before binding. + /// + public Func? ValueTransformer { get; set; } +} diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsValidationException.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs rename to src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsValidationException.cs index 4bd9b48c..164c470b 100644 --- a/src/StellaOps.Vexer.Connectors.Abstractions/VexConnectorOptionsValidationException.cs +++ b/src/StellaOps.Excititor.Connectors.Abstractions/VexConnectorOptionsValidationException.cs @@ -1,36 +1,36 @@ -using System.Collections.Immutable; - -namespace StellaOps.Vexer.Connectors.Abstractions; - -public sealed class VexConnectorOptionsValidationException : Exception -{ - public VexConnectorOptionsValidationException( - string connectorId, - IEnumerable errors) - : base(BuildMessage(connectorId, errors)) - { - ConnectorId = connectorId; - Errors = errors?.ToImmutableArray() ?? ImmutableArray.Empty; - } - - public string ConnectorId { get; } - - public ImmutableArray Errors { get; } - - private static string BuildMessage(string connectorId, IEnumerable errors) - { - var builder = new System.Text.StringBuilder(); - builder.Append("Connector options validation failed for '"); - builder.Append(connectorId); - builder.Append("'."); - - var list = errors?.ToImmutableArray() ?? ImmutableArray.Empty; - if (!list.IsDefaultOrEmpty) - { - builder.Append(" Errors: "); - builder.Append(string.Join("; ", list)); - } - - return builder.ToString(); - } -} +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Connectors.Abstractions; + +public sealed class VexConnectorOptionsValidationException : Exception +{ + public VexConnectorOptionsValidationException( + string connectorId, + IEnumerable errors) + : base(BuildMessage(connectorId, errors)) + { + ConnectorId = connectorId; + Errors = errors?.ToImmutableArray() ?? ImmutableArray.Empty; + } + + public string ConnectorId { get; } + + public ImmutableArray Errors { get; } + + private static string BuildMessage(string connectorId, IEnumerable errors) + { + var builder = new System.Text.StringBuilder(); + builder.Append("Connector options validation failed for '"); + builder.Append(connectorId); + builder.Append("'."); + + var list = errors?.ToImmutableArray() ?? ImmutableArray.Empty; + if (!list.IsDefaultOrEmpty) + { + builder.Append(" Errors: "); + builder.Append(string.Join("; ", list)); + } + + return builder.ToString(); + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs index 792f3326..9a0561c8 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Connectors/CiscoCsafConnectorTests.cs @@ -1,214 +1,214 @@ -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Cisco.CSAF; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; -using System.Collections.Immutable; -using System.IO.Abstractions.TestingHelpers; -using Xunit; -using System.Threading; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.Connectors; - -public sealed class CiscoCsafConnectorTests -{ - [Fact] - public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState() - { - var responses = new Dictionary> - { - [new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses(""" - { - "metadata": { - "publisher": { - "name": "Cisco", - "category": "vendor", - "contact_details": { "id": "vexer:cisco" } - } - }, - "distributions": { - "directories": [ "https://api.cisco.test/csaf/" ] - } - } - """), - [new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses(""" - { - "advisories": [ - { - "id": "cisco-sa-2025", - "url": "https://api.cisco.test/csaf/cisco-sa-2025.json", - "published": "2025-10-01T00:00:00Z", - "lastModified": "2025-10-02T00:00:00Z", - "sha256": "cafebabe" - } - ] - } - """), - [new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }") - }; - - var handler = new RoutingHttpMessageHandler(responses); - var httpClient = new HttpClient(handler); - var factory = new SingleHttpClientFactory(httpClient); - var metadataLoader = new CiscoProviderMetadataLoader( - factory, - new MemoryCache(new MemoryCacheOptions()), - Options.Create(new CiscoConnectorOptions - { - MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json", - PersistOfflineSnapshot = false, - }), - NullLogger.Instance, - new MockFileSystem()); - - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new CiscoCsafConnector( - metadataLoader, - factory, - stateRepository, - new[] { new CiscoConnectorOptionsValidator() }, - NullLogger.Instance, - TimeProvider.System); - - var settings = new VexConnectorSettings(ImmutableDictionary.Empty); - await connector.ValidateAsync(settings, CancellationToken.None); - - var sink = new InMemoryRawSink(); - var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().HaveCount(1); - sink.Documents.Should().HaveCount(1); - stateRepository.CurrentState.Should().NotBeNull(); - stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1); - - // second run should not refetch documents - sink.Documents.Clear(); - documents.Clear(); - - await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(doc); - } - - documents.Should().BeEmpty(); - sink.Documents.Should().BeEmpty(); - } - - private static Queue QueueResponses(string payload) - => new(new[] - { - new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - } - }); - - private sealed class RoutingHttpMessageHandler : HttpMessageHandler - { - private readonly Dictionary> _responses; - - public RoutingHttpMessageHandler(Dictionary> responses) - { - _responses = responses; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0) - { - var response = queue.Peek(); - return Task.FromResult(response.Clone()); - } - - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent($"No response configured for {request.RequestUri}"), - }); - } - } - - private sealed class SingleHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? CurrentState { get; private set; } - - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - => ValueTask.FromResult(CurrentState); - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - CurrentState = state; - return ValueTask.CompletedTask; - } - } - - private sealed class InMemoryRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } -} - -internal static class HttpResponseMessageExtensions -{ - public static HttpResponseMessage Clone(this HttpResponseMessage response) - { - var clone = new HttpResponseMessage(response.StatusCode); - foreach (var header in response.Headers) - { - clone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - if (response.Content is not null) - { - var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); - clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType); - } - - return clone; - } -} +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Cisco.CSAF; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.Collections.Immutable; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Connectors; + +public sealed class CiscoCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_NewAdvisory_StoresDocumentAndUpdatesState() + { + var responses = new Dictionary> + { + [new Uri("https://api.cisco.test/.well-known/csaf/provider-metadata.json")] = QueueResponses(""" + { + "metadata": { + "publisher": { + "name": "Cisco", + "category": "vendor", + "contact_details": { "id": "excititor:cisco" } + } + }, + "distributions": { + "directories": [ "https://api.cisco.test/csaf/" ] + } + } + """), + [new Uri("https://api.cisco.test/csaf/index.json")] = QueueResponses(""" + { + "advisories": [ + { + "id": "cisco-sa-2025", + "url": "https://api.cisco.test/csaf/cisco-sa-2025.json", + "published": "2025-10-01T00:00:00Z", + "lastModified": "2025-10-02T00:00:00Z", + "sha256": "cafebabe" + } + ] + } + """), + [new Uri("https://api.cisco.test/csaf/cisco-sa-2025.json")] = QueueResponses("{ \"document\": \"payload\" }") + }; + + var handler = new RoutingHttpMessageHandler(responses); + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var metadataLoader = new CiscoProviderMetadataLoader( + factory, + new MemoryCache(new MemoryCacheOptions()), + Options.Create(new CiscoConnectorOptions + { + MetadataUri = "https://api.cisco.test/.well-known/csaf/provider-metadata.json", + PersistOfflineSnapshot = false, + }), + NullLogger.Instance, + new MockFileSystem()); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new CiscoCsafConnector( + metadataLoader, + factory, + stateRepository, + new[] { new CiscoConnectorOptionsValidator() }, + NullLogger.Instance, + TimeProvider.System); + + var settings = new VexConnectorSettings(ImmutableDictionary.Empty); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().HaveCount(1); + + // second run should not refetch documents + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + } + + private static Queue QueueResponses(string payload) + => new(new[] + { + new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + } + }); + + private sealed class RoutingHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary> _responses; + + public RoutingHttpMessageHandler(Dictionary> responses) + { + _responses = responses; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var queue) && queue.Count > 0) + { + var response = queue.Peek(); + return Task.FromResult(response.Clone()); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No response configured for {request.RequestUri}"), + }); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? CurrentState { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + => ValueTask.FromResult(CurrentState); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + CurrentState = state; + return ValueTask.CompletedTask; + } + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} + +internal static class HttpResponseMessageExtensions +{ + public static HttpResponseMessage Clone(this HttpResponseMessage response) + { + var clone = new HttpResponseMessage(response.StatusCode); + foreach (var header in response.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (response.Content is not null) + { + var payload = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); + clone.Content = new StringContent(payload, Encoding.UTF8, response.Content.Headers.ContentType?.MediaType); + } + + return clone; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs similarity index 89% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs index 59b0031d..45cee8dd 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/Metadata/CiscoProviderMetadataLoaderTests.cs @@ -1,149 +1,149 @@ -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; -using StellaOps.Vexer.Core; -using System.IO.Abstractions.TestingHelpers; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.Metadata; - -public sealed class CiscoProviderMetadataLoaderTests -{ - [Fact] - public async Task LoadAsync_FetchesFromNetworkWithBearerToken() - { - var payload = """ - { - "metadata": { - "publisher": { - "name": "Cisco CSAF", - "category": "vendor", - "contact_details": { - "id": "vexer:cisco" - } - } - }, - "distributions": { - "directories": [ - "https://api.security.cisco.com/csaf/v2/advisories/" - ] - }, - "discovery": { - "well_known": "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", - "rolie": "https://api.security.cisco.com/csaf/rolie/feed" - }, - "trust": { - "weight": 0.9, - "cosign": { - "issuer": "https://oidc.security.cisco.com", - "identity_pattern": "spiffe://cisco/*" - }, - "pgp_fingerprints": [ "1234ABCD" ] - } - } - """; - - HttpRequestMessage? capturedRequest = null; - var handler = new FakeHttpMessageHandler(request => - { - capturedRequest = request; - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - Headers = { ETag = new System.Net.Http.Headers.EntityTagHeaderValue("\"etag1\"") } - }; - }); - - var httpClient = new HttpClient(handler); - var factory = new SingleHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var options = Options.Create(new CiscoConnectorOptions - { - ApiToken = "token-123", - MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", - }); - var fileSystem = new MockFileSystem(); - var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); - - var result = await loader.LoadAsync(CancellationToken.None); - - result.Provider.Id.Should().Be("vexer:cisco"); - result.Provider.BaseUris.Should().ContainSingle(uri => uri.ToString() == "https://api.security.cisco.com/csaf/v2/advisories/"); - result.Provider.Discovery.RolIeService.Should().Be(new Uri("https://api.security.cisco.com/csaf/rolie/feed")); - result.ServedFromCache.Should().BeFalse(); - capturedRequest.Should().NotBeNull(); - capturedRequest!.Headers.Authorization.Should().NotBeNull(); - capturedRequest.Headers.Authorization!.Parameter.Should().Be("token-123"); - } - - [Fact] - public async Task LoadAsync_FallsBackToOfflineSnapshot() - { - var payload = """ - { - "metadata": { - "publisher": { - "name": "Cisco CSAF", - "category": "vendor", - "contact_details": { - "id": "vexer:cisco" - } - } - } - } - """; - - var handler = new FakeHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)); - var httpClient = new HttpClient(handler); - var factory = new SingleHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var options = Options.Create(new CiscoConnectorOptions - { - MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", - PreferOfflineSnapshot = true, - OfflineSnapshotPath = "/snapshots/cisco.json", - }); - var fileSystem = new MockFileSystem(new Dictionary - { - ["/snapshots/cisco.json"] = new MockFileData(payload), - }); - var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); - - var result = await loader.LoadAsync(CancellationToken.None); - - result.FromOfflineSnapshot.Should().BeTrue(); - result.Provider.Id.Should().Be("vexer:cisco"); - } - - private sealed class SingleHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class FakeHttpMessageHandler : HttpMessageHandler - { - private readonly Func _responder; - - public FakeHttpMessageHandler(Func responder) - { - _responder = responder; - } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - return Task.FromResult(_responder(request)); - } - } -} +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Excititor.Core; +using System.IO.Abstractions.TestingHelpers; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.Metadata; + +public sealed class CiscoProviderMetadataLoaderTests +{ + [Fact] + public async Task LoadAsync_FetchesFromNetworkWithBearerToken() + { + var payload = """ + { + "metadata": { + "publisher": { + "name": "Cisco CSAF", + "category": "vendor", + "contact_details": { + "id": "excititor:cisco" + } + } + }, + "distributions": { + "directories": [ + "https://api.security.cisco.com/csaf/v2/advisories/" + ] + }, + "discovery": { + "well_known": "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + "rolie": "https://api.security.cisco.com/csaf/rolie/feed" + }, + "trust": { + "weight": 0.9, + "cosign": { + "issuer": "https://oidc.security.cisco.com", + "identity_pattern": "spiffe://cisco/*" + }, + "pgp_fingerprints": [ "1234ABCD" ] + } + } + """; + + HttpRequestMessage? capturedRequest = null; + var handler = new FakeHttpMessageHandler(request => + { + capturedRequest = request; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + Headers = { ETag = new System.Net.Http.Headers.EntityTagHeaderValue("\"etag1\"") } + }; + }); + + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new CiscoConnectorOptions + { + ApiToken = "token-123", + MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + }); + var fileSystem = new MockFileSystem(); + var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + result.Provider.Id.Should().Be("excititor:cisco"); + result.Provider.BaseUris.Should().ContainSingle(uri => uri.ToString() == "https://api.security.cisco.com/csaf/v2/advisories/"); + result.Provider.Discovery.RolIeService.Should().Be(new Uri("https://api.security.cisco.com/csaf/rolie/feed")); + result.ServedFromCache.Should().BeFalse(); + capturedRequest.Should().NotBeNull(); + capturedRequest!.Headers.Authorization.Should().NotBeNull(); + capturedRequest.Headers.Authorization!.Parameter.Should().Be("token-123"); + } + + [Fact] + public async Task LoadAsync_FallsBackToOfflineSnapshot() + { + var payload = """ + { + "metadata": { + "publisher": { + "name": "Cisco CSAF", + "category": "vendor", + "contact_details": { + "id": "excititor:cisco" + } + } + } + } + """; + + var handler = new FakeHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError)); + var httpClient = new HttpClient(handler); + var factory = new SingleHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new CiscoConnectorOptions + { + MetadataUri = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json", + PreferOfflineSnapshot = true, + OfflineSnapshotPath = "/snapshots/cisco.json", + }); + var fileSystem = new MockFileSystem(new Dictionary + { + ["/snapshots/cisco.json"] = new MockFileData(payload), + }); + var loader = new CiscoProviderMetadataLoader(factory, cache, options, NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + result.FromOfflineSnapshot.Should().BeTrue(); + result.Provider.Id.Should().Be("excititor:cisco"); + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class FakeHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responder; + + public FakeHttpMessageHandler(Func responder) + { + _responder = responder; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(_responder(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.csproj similarity index 78% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.csproj index 1e11d778..00fd2ae2 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests/StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests/StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/AGENTS.md b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/AGENTS.md similarity index 95% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/AGENTS.md index fb3df82b..0c632160 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector responsible for ingesting Cisco CSAF VEX advisories and handing raw documents to normalizers with Cisco-specific metadata. -## Scope -- Discovery of Cisco CSAF collection endpoints, authentication (when required), and pagination routines. -- HTTP retries/backoff, checksum verification, and document deduplication before storage. -- Mapping Cisco advisory identifiers, product hierarchies, and severity hints into connector metadata. -- Surfacing provider trust configuration aligned with policy expectations. -## Participants -- Worker drives scheduled pulls; WebService may trigger manual runs. -- CSAF normalizer consumes raw documents to emit claims. -- Policy module references connector trust hints (e.g., Cisco signing identities). -## Interfaces & contracts -- Implements `IVexConnector` using shared abstractions for HTTP/resume handling. -- Provides options for API tokens, rate limits, and concurrency. -## In/Out of scope -In: data fetching, provider metadata, retry controls, raw document persistence. -Out: normalization/export, attestation, Mongo wiring (handled in other modules). -## Observability & security expectations -- Log fetch batches with document counts/durations; mask credentials. -- Emit metrics for rate-limit hits, retries, and quarantine events. -## Tests -- Unit tests plus HTTP harness fixtures will live in `../StellaOps.Vexer.Connectors.Cisco.CSAF.Tests`. +# AGENTS +## Role +Connector responsible for ingesting Cisco CSAF VEX advisories and handing raw documents to normalizers with Cisco-specific metadata. +## Scope +- Discovery of Cisco CSAF collection endpoints, authentication (when required), and pagination routines. +- HTTP retries/backoff, checksum verification, and document deduplication before storage. +- Mapping Cisco advisory identifiers, product hierarchies, and severity hints into connector metadata. +- Surfacing provider trust configuration aligned with policy expectations. +## Participants +- Worker drives scheduled pulls; WebService may trigger manual runs. +- CSAF normalizer consumes raw documents to emit claims. +- Policy module references connector trust hints (e.g., Cisco signing identities). +## Interfaces & contracts +- Implements `IVexConnector` using shared abstractions for HTTP/resume handling. +- Provides options for API tokens, rate limits, and concurrency. +## In/Out of scope +In: data fetching, provider metadata, retry controls, raw document persistence. +Out: normalization/export, attestation, Mongo wiring (handled in other modules). +## Observability & security expectations +- Log fetch batches with document counts/durations; mask credentials. +- Emit metrics for rate-limit hits, retries, and quarantine events. +## Tests +- Unit tests plus HTTP harness fixtures will live in `../StellaOps.Excititor.Connectors.Cisco.CSAF.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs similarity index 89% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs index a9829c3c..063e2475 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/CiscoCsafConnector.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/CiscoCsafConnector.cs @@ -1,247 +1,257 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF; - -public sealed class CiscoCsafConnector : VexConnectorBase -{ - private static readonly VexConnectorDescriptor DescriptorInstance = new( - id: "vexer:cisco", - kind: VexProviderKind.Vendor, - displayName: "Cisco CSAF") - { - Tags = ImmutableArray.Create("cisco", "csaf"), - }; - - private readonly CiscoProviderMetadataLoader _metadataLoader; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IVexConnectorStateRepository _stateRepository; - private readonly IEnumerable> _validators; - private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); - - private CiscoConnectorOptions? _options; - private CiscoProviderMetadataResult? _providerMetadata; - - public CiscoCsafConnector( - CiscoProviderMetadataLoader metadataLoader, - IHttpClientFactory httpClientFactory, - IVexConnectorStateRepository stateRepository, - IEnumerable>? validators, - ILogger logger, - TimeProvider timeProvider) - : base(DescriptorInstance, logger, timeProvider) - { - _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); - LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary - { - ["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length, - ["fromOffline"] = _providerMetadata.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_providerMetadata is null) - { - _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); - } - - var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); - var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; - var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); - var digestList = new List(knownDigests); - var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; - var latestTimestamp = state?.LastUpdated ?? since; - var stateChanged = false; - - var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); - foreach (var directory in _providerMetadata.Provider.BaseUris) - { - await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false)) - { - var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue; - if (published <= since) - { - continue; - } - - using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - contentResponse.EnsureSuccessStatusCode(); - var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - - var rawDocument = CreateRawDocument( - VexDocumentFormat.Csaf, - advisory.DocumentUri, - payload, - BuildMetadata(builder => builder - .Add("cisco.csaf.advisoryId", advisory.Id) - .Add("cisco.csaf.revision", advisory.Revision) - .Add("cisco.csaf.published", advisory.Published?.ToString("O")) - .Add("cisco.csaf.modified", advisory.LastModified?.ToString("O")) - .Add("cisco.csaf.sha256", advisory.Sha256))); - - if (!digestSet.Add(rawDocument.Digest)) - { - continue; - } - - await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); - digestList.Add(rawDocument.Digest); - stateChanged = true; - if (published > latestTimestamp) - { - latestTimestamp = published; - } - - yield return rawDocument; - } - } - - if (stateChanged) - { - var newState = new VexConnectorState( - Descriptor.Id, - latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, - digestList.ToImmutableArray()); - await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); - } - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing."); - - private async IAsyncEnumerable EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var nextUri = BuildIndexUri(directory, null); - while (nextUri is not null) - { - using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var page = JsonSerializer.Deserialize(json, _serializerOptions); - if (page?.Advisories is null) - { - yield break; - } - - foreach (var advisory in page.Advisories) - { - if (string.IsNullOrWhiteSpace(advisory.Url)) - { - continue; - } - - if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri)) - { - continue; - } - - if (!documentUri.IsAbsoluteUri) - { - documentUri = new Uri(directory, documentUri); - } - - yield return new CiscoAdvisoryEntry( - advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(), - documentUri, - advisory.Revision, - advisory.Published, - advisory.LastModified, - advisory.Sha256); - } - - nextUri = ResolveNextUri(directory, page.Next); - } - } - - private static Uri BuildIndexUri(Uri directory, string? relative) - { - if (string.IsNullOrWhiteSpace(relative)) - { - var baseText = directory.ToString(); - if (!baseText.EndsWith('/')) - { - baseText += "/"; - } - - return new Uri(new Uri(baseText, UriKind.Absolute), "index.json"); - } - - if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute)) - { - return absolute; - } - - var baseTextRelative = directory.ToString(); - if (!baseTextRelative.EndsWith('/')) - { - baseTextRelative += "/"; - } - - return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative); - } - - private static Uri? ResolveNextUri(Uri directory, string? next) - { - if (string.IsNullOrWhiteSpace(next)) - { - return null; - } - - return BuildIndexUri(directory, next); - } - - private sealed record CiscoAdvisoryIndex - { - public List? Advisories { get; init; } - public string? Next { get; init; } - } - - private sealed record CiscoAdvisory - { - public string? Id { get; init; } - public string? Url { get; init; } - public string? Revision { get; init; } - public DateTimeOffset? Published { get; init; } - public DateTimeOffset? LastModified { get; init; } - public string? Sha256 { get; init; } - } - - private sealed record CiscoAdvisoryEntry( - string Id, - Uri DocumentUri, - string? Revision, - DateTimeOffset? Published, - DateTimeOffset? LastModified, - string? Sha256); -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF; + +public sealed class CiscoCsafConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:cisco", + kind: VexProviderKind.Vendor, + displayName: "Cisco CSAF") + { + Tags = ImmutableArray.Create("cisco", "csaf"), + }; + + private readonly CiscoProviderMetadataLoader _metadataLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IEnumerable> _validators; + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + private CiscoConnectorOptions? _options; + private CiscoProviderMetadataResult? _providerMetadata; + + public CiscoCsafConnector( + CiscoProviderMetadataLoader metadataLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + IEnumerable>? validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Cisco CSAF metadata loaded.", new Dictionary + { + ["baseUriCount"] = _providerMetadata.Provider.BaseUris.Length, + ["fromOffline"] = _providerMetadata.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_providerMetadata is null) + { + _providerMetadata = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var digestList = new List(knownDigests); + var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; + var latestTimestamp = state?.LastUpdated ?? since; + var stateChanged = false; + + var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); + foreach (var directory in _providerMetadata.Provider.BaseUris) + { + await foreach (var advisory in EnumerateCatalogAsync(client, directory, cancellationToken).ConfigureAwait(false)) + { + var published = advisory.LastModified ?? advisory.Published ?? DateTimeOffset.MinValue; + if (published <= since) + { + continue; + } + + using var contentResponse = await client.GetAsync(advisory.DocumentUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + contentResponse.EnsureSuccessStatusCode(); + var payload = await contentResponse.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + + var rawDocument = CreateRawDocument( + VexDocumentFormat.Csaf, + advisory.DocumentUri, + payload, + BuildMetadata(builder => builder + .Add("cisco.csaf.advisoryId", advisory.Id) + .Add("cisco.csaf.revision", advisory.Revision) + .Add("cisco.csaf.published", advisory.Published?.ToString("O")) + .Add("cisco.csaf.modified", advisory.LastModified?.ToString("O")) + .Add("cisco.csaf.sha256", advisory.Sha256))); + + if (!digestSet.Add(rawDocument.Digest)) + { + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + if (published > latestTimestamp) + { + latestTimestamp = published; + } + + yield return rawDocument; + } + } + + if (stateChanged) + { + var baseState = state ?? new VexConnectorState( + Descriptor.Id, + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var newState = baseState with + { + LastUpdated = latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, + DocumentDigests = digestList.ToImmutableArray(), + }; + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("CiscoCsafConnector relies on CSAF normalizers for document processing."); + + private async IAsyncEnumerable EnumerateCatalogAsync(HttpClient client, Uri directory, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var nextUri = BuildIndexUri(directory, null); + while (nextUri is not null) + { + using var response = await client.GetAsync(nextUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var page = JsonSerializer.Deserialize(json, _serializerOptions); + if (page?.Advisories is null) + { + yield break; + } + + foreach (var advisory in page.Advisories) + { + if (string.IsNullOrWhiteSpace(advisory.Url)) + { + continue; + } + + if (!Uri.TryCreate(advisory.Url, UriKind.RelativeOrAbsolute, out var documentUri)) + { + continue; + } + + if (!documentUri.IsAbsoluteUri) + { + documentUri = new Uri(directory, documentUri); + } + + yield return new CiscoAdvisoryEntry( + advisory.Id ?? documentUri.Segments.LastOrDefault()?.Trim('/') ?? documentUri.ToString(), + documentUri, + advisory.Revision, + advisory.Published, + advisory.LastModified, + advisory.Sha256); + } + + nextUri = ResolveNextUri(directory, page.Next); + } + } + + private static Uri BuildIndexUri(Uri directory, string? relative) + { + if (string.IsNullOrWhiteSpace(relative)) + { + var baseText = directory.ToString(); + if (!baseText.EndsWith('/')) + { + baseText += "/"; + } + + return new Uri(new Uri(baseText, UriKind.Absolute), "index.json"); + } + + if (Uri.TryCreate(relative, UriKind.Absolute, out var absolute)) + { + return absolute; + } + + var baseTextRelative = directory.ToString(); + if (!baseTextRelative.EndsWith('/')) + { + baseTextRelative += "/"; + } + + return new Uri(new Uri(baseTextRelative, UriKind.Absolute), relative); + } + + private static Uri? ResolveNextUri(Uri directory, string? next) + { + if (string.IsNullOrWhiteSpace(next)) + { + return null; + } + + return BuildIndexUri(directory, next); + } + + private sealed record CiscoAdvisoryIndex + { + public List? Advisories { get; init; } + public string? Next { get; init; } + } + + private sealed record CiscoAdvisory + { + public string? Id { get; init; } + public string? Url { get; init; } + public string? Revision { get; init; } + public DateTimeOffset? Published { get; init; } + public DateTimeOffset? LastModified { get; init; } + public string? Sha256 { get; init; } + } + + private sealed record CiscoAdvisoryEntry( + string Id, + Uri DocumentUri, + string? Revision, + DateTimeOffset? Published, + DateTimeOffset? LastModified, + string? Sha256); +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs index 20ee0ce5..e0ad295d 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptions.cs @@ -1,58 +1,58 @@ -using System.ComponentModel.DataAnnotations; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; - -public sealed class CiscoConnectorOptions : IValidatableObject -{ - public const string HttpClientName = "cisco-csaf"; - - /// - /// Endpoint for Cisco CSAF provider metadata discovery. - /// - [Required] - public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json"; - - /// - /// Optional bearer token used when Cisco endpoints require authentication. - /// - public string? ApiToken { get; set; } - - /// - /// How long provider metadata remains cached. - /// - public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); - - /// - /// Whether to prefer offline snapshots when fetching metadata. - /// - public bool PreferOfflineSnapshot { get; set; } - - /// - /// When set, provider metadata will be persisted to the given file path. - /// - public bool PersistOfflineSnapshot { get; set; } - - public string? OfflineSnapshotPath { get; set; } - - public IEnumerable Validate(ValidationContext validationContext) - { - if (string.IsNullOrWhiteSpace(MetadataUri)) - { - yield return new ValidationResult("MetadataUri must be provided.", new[] { nameof(MetadataUri) }); - } - else if (!Uri.TryCreate(MetadataUri, UriKind.Absolute, out _)) - { - yield return new ValidationResult("MetadataUri must be an absolute URI.", new[] { nameof(MetadataUri) }); - } - - if (MetadataCacheDuration <= TimeSpan.Zero) - { - yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) }); - } - - if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) }); - } - } -} +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; + +public sealed class CiscoConnectorOptions : IValidatableObject +{ + public const string HttpClientName = "cisco-csaf"; + + /// + /// Endpoint for Cisco CSAF provider metadata discovery. + /// + [Required] + public string MetadataUri { get; set; } = "https://api.security.cisco.com/.well-known/csaf/provider-metadata.json"; + + /// + /// Optional bearer token used when Cisco endpoints require authentication. + /// + public string? ApiToken { get; set; } + + /// + /// How long provider metadata remains cached. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); + + /// + /// Whether to prefer offline snapshots when fetching metadata. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// When set, provider metadata will be persisted to the given file path. + /// + public bool PersistOfflineSnapshot { get; set; } + + public string? OfflineSnapshotPath { get; set; } + + public IEnumerable Validate(ValidationContext validationContext) + { + if (string.IsNullOrWhiteSpace(MetadataUri)) + { + yield return new ValidationResult("MetadataUri must be provided.", new[] { nameof(MetadataUri) }); + } + else if (!Uri.TryCreate(MetadataUri, UriKind.Absolute, out _)) + { + yield return new ValidationResult("MetadataUri must be an absolute URI.", new[] { nameof(MetadataUri) }); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + yield return new ValidationResult("MetadataCacheDuration must be greater than zero.", new[] { nameof(MetadataCacheDuration) }); + } + + if (PersistOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + yield return new ValidationResult("OfflineSnapshotPath must be provided when PersistOfflineSnapshot is enabled.", new[] { nameof(OfflineSnapshotPath) }); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs similarity index 86% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs index 6599b60a..eb013e7d 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Configuration/CiscoConnectorOptionsValidator.cs @@ -1,25 +1,25 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using StellaOps.Vexer.Connectors.Abstractions; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; - -public sealed class CiscoConnectorOptionsValidator : IVexConnectorOptionsValidator -{ - public void Validate(VexConnectorDescriptor descriptor, CiscoConnectorOptions options, IList errors) - { - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentNullException.ThrowIfNull(options); - ArgumentNullException.ThrowIfNull(errors); - - var validationResults = new List(); - if (!Validator.TryValidateObject(options, new ValidationContext(options), validationResults, validateAllProperties: true)) - { - foreach (var result in validationResults) - { - errors.Add(result.ErrorMessage ?? "Cisco connector options validation failed."); - } - } - } -} +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; + +public sealed class CiscoConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + public void Validate(VexConnectorDescriptor descriptor, CiscoConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + var validationResults = new List(); + if (!Validator.TryValidateObject(options, new ValidationContext(options), validationResults, validateAllProperties: true)) + { + foreach (var result in validationResults) + { + errors.Add(result.ErrorMessage ?? "Cisco connector options validation failed."); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs similarity index 85% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs index a8ce354c..a01e4f42 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/DependencyInjection/CiscoConnectorServiceCollectionExtensions.cs @@ -1,52 +1,52 @@ -using System.ComponentModel.DataAnnotations; -using System.Net.Http.Headers; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.DependencyInjection; - -public static class CiscoConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.AddOptions() - .Configure(options => - { - configure?.Invoke(options); - }) - .PostConfigure(options => - { - Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); - }); - - services.TryAddSingleton(); - services.TryAddSingleton(); - services.TryAddEnumerable(ServiceDescriptor.Singleton, CiscoConnectorOptionsValidator>()); - - services.AddHttpClient(CiscoConnectorOptions.HttpClientName) - .ConfigureHttpClient((provider, client) => - { - var options = provider.GetRequiredService>().Value; - client.Timeout = TimeSpan.FromSeconds(30); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - if (!string.IsNullOrWhiteSpace(options.ApiToken)) - { - client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken); - } - }); - - services.AddSingleton(); - services.AddSingleton(); - - return services; - } -} +using System.ComponentModel.DataAnnotations; +using System.Net.Http.Headers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.DependencyInjection; + +public static class CiscoConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddCiscoCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }) + .PostConfigure(options => + { + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton, CiscoConnectorOptionsValidator>()); + + services.AddHttpClient(CiscoConnectorOptions.HttpClientName) + .ConfigureHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().Value; + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + if (!string.IsNullOrWhiteSpace(options.ApiToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", options.ApiToken); + } + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs index 5d5d7438..3e227654 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/Metadata/CiscoProviderMetadataLoader.cs @@ -1,332 +1,332 @@ -using System.Collections.Immutable; -using System.Net; -using System.Net.Http.Headers; -using System.Text.Json; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.Cisco.CSAF.Configuration; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata; - -public sealed class CiscoProviderMetadataLoader -{ - public const string CacheKey = "StellaOps.Vexer.Connectors.Cisco.CSAF.Metadata"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _memoryCache; - private readonly ILogger _logger; - private readonly CiscoConnectorOptions _options; - private readonly IFileSystem _fileSystem; - private readonly JsonSerializerOptions _serializerOptions; - private readonly SemaphoreSlim _semaphore = new(1, 1); - - public CiscoProviderMetadataLoader( - IHttpClientFactory httpClientFactory, - IMemoryCache memoryCache, - IOptions options, - ILogger logger, - IFileSystem? fileSystem = null) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - ArgumentNullException.ThrowIfNull(options); - _options = options.Value ?? throw new ArgumentNullException(nameof(options)); - _fileSystem = fileSystem ?? new FileSystem(); - _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - }; - } - - public async Task LoadAsync(CancellationToken cancellationToken) - { - if (_memoryCache.TryGetValue(CacheKey, out var cached) && cached is not null && !cached.IsExpired()) - { - _logger.LogDebug("Returning cached Cisco provider metadata (expires {Expires}).", cached.ExpiresAt); - return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); - } - - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_memoryCache.TryGetValue(CacheKey, out cached) && cached is not null && !cached.IsExpired()) - { - return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); - } - - CacheEntry? previous = cached; - - if (!_options.PreferOfflineSnapshot) - { - var network = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); - if (network is not null) - { - StoreCache(network); - return new CiscoProviderMetadataResult(network.Provider, network.FetchedAt, false, false); - } - } - - var offline = TryLoadFromOffline(); - if (offline is not null) - { - var entry = offline with - { - FetchedAt = DateTimeOffset.UtcNow, - ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - FromOffline = true, - }; - StoreCache(entry); - return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false); - } - - throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot."); - } - finally - { - _semaphore.Release(); - } - } - - private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) - { - try - { - var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); - using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); - if (!string.IsNullOrWhiteSpace(_options.ApiToken)) - { - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiToken); - } - - if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) - { - request.Headers.IfNoneMatch.Add(etag); - } - - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) - { - _logger.LogDebug("Cisco provider metadata not modified (etag {ETag}).", previous.ETag); - return previous with - { - FetchedAt = DateTimeOffset.UtcNow, - ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - }; - } - - response.EnsureSuccessStatusCode(); - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var provider = ParseProvider(payload); - var etagHeader = response.Headers.ETag?.ToString(); - - if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) - { - try - { - _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); - _logger.LogDebug("Persisted Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to persist Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); - } - } - - return new CacheEntry( - provider, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - etagHeader, - FromOffline: false); - } - catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) - { - _logger.LogWarning(ex, "Failed to fetch Cisco provider metadata from {Uri}; falling back to offline snapshot when available.", _options.MetadataUri); - return null; - } - } - - private CacheEntry? TryLoadFromOffline() - { - if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) - { - return null; - } - - if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) - { - _logger.LogWarning("Cisco offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); - return null; - } - - try - { - var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); - var provider = ParseProvider(payload); - return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, null, true); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load Cisco provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); - return null; - } - } - - private VexProvider ParseProvider(string payload) - { - if (string.IsNullOrWhiteSpace(payload)) - { - throw new InvalidOperationException("Cisco provider metadata payload was empty."); - } - - ProviderMetadataDocument? document; - try - { - document = JsonSerializer.Deserialize(payload, _serializerOptions); - } - catch (JsonException ex) - { - throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex); - } - - if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id)) - { - throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier."); - } - - var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe); - var trust = document.Trust is null - ? VexProviderTrust.Default - : new VexProviderTrust( - document.Trust.Weight ?? 1.0, - document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty), - document.Trust.PgpFingerprints ?? Enumerable.Empty()); - - var directories = document.Distributions?.Directories is null - ? Enumerable.Empty() - : document.Distributions.Directories - .Where(static s => !string.IsNullOrWhiteSpace(s)) - .Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null) - .Where(static uri => uri is not null)! - .Select(static uri => uri!); - - return new VexProvider( - id: document.Metadata.Publisher.ContactDetails.Id, - displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id, - kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub, - baseUris: directories, - discovery: discovery, - trust: trust, - enabled: true); - } - - private void StoreCache(CacheEntry entry) - { - var options = new MemoryCacheEntryOptions - { - AbsoluteExpiration = entry.ExpiresAt, - }; - _memoryCache.Set(CacheKey, entry, options); - } - - private sealed record CacheEntry( - VexProvider Provider, - DateTimeOffset FetchedAt, - DateTimeOffset ExpiresAt, - string? ETag, - bool FromOffline) - { - public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; - } -} - -public sealed record CiscoProviderMetadataResult( - VexProvider Provider, - DateTimeOffset FetchedAt, - bool FromOfflineSnapshot, - bool ServedFromCache); - -#region document models - -internal sealed class ProviderMetadataDocument -{ - [System.Text.Json.Serialization.JsonPropertyName("metadata")] - public ProviderMetadataMetadata Metadata { get; set; } = new(); - - [System.Text.Json.Serialization.JsonPropertyName("discovery")] - public ProviderMetadataDiscovery? Discovery { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("trust")] - public ProviderMetadataTrust? Trust { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("distributions")] - public ProviderMetadataDistributions? Distributions { get; set; } -} - -internal sealed class ProviderMetadataMetadata -{ - [System.Text.Json.Serialization.JsonPropertyName("publisher")] - public ProviderMetadataPublisher Publisher { get; set; } = new(); -} - -internal sealed class ProviderMetadataPublisher -{ - [System.Text.Json.Serialization.JsonPropertyName("name")] - public string? Name { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("category")] - public string? Category { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("contact_details")] - public ProviderMetadataPublisherContact ContactDetails { get; set; } = new(); -} - -internal sealed class ProviderMetadataPublisherContact -{ - [System.Text.Json.Serialization.JsonPropertyName("id")] - public string? Id { get; set; } -} - -internal sealed class ProviderMetadataDiscovery -{ - [System.Text.Json.Serialization.JsonPropertyName("well_known")] - public Uri? WellKnown { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("rolie")] - public Uri? RolIe { get; set; } -} - -internal sealed class ProviderMetadataTrust -{ - [System.Text.Json.Serialization.JsonPropertyName("weight")] - public double? Weight { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("cosign")] - public ProviderMetadataTrustCosign? Cosign { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("pgp_fingerprints")] - public string[]? PgpFingerprints { get; set; } -} - -internal sealed class ProviderMetadataTrustCosign -{ - [System.Text.Json.Serialization.JsonPropertyName("issuer")] - public string? Issuer { get; set; } - - [System.Text.Json.Serialization.JsonPropertyName("identity_pattern")] - public string? IdentityPattern { get; set; } -} - -internal sealed class ProviderMetadataDistributions -{ - [System.Text.Json.Serialization.JsonPropertyName("directories")] - public string[]? Directories { get; set; } -} - -#endregion +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Cisco.CSAF.Configuration; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata; + +public sealed class CiscoProviderMetadataLoader +{ + public const string CacheKey = "StellaOps.Excititor.Connectors.Cisco.CSAF.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly ILogger _logger; + private readonly CiscoConnectorOptions _options; + private readonly IFileSystem _fileSystem; + private readonly JsonSerializerOptions _serializerOptions; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public CiscoProviderMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? new FileSystem(); + _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + } + + public async Task LoadAsync(CancellationToken cancellationToken) + { + if (_memoryCache.TryGetValue(CacheKey, out var cached) && cached is not null && !cached.IsExpired()) + { + _logger.LogDebug("Returning cached Cisco provider metadata (expires {Expires}).", cached.ExpiresAt); + return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(CacheKey, out cached) && cached is not null && !cached.IsExpired()) + { + return new CiscoProviderMetadataResult(cached.Provider, cached.FetchedAt, cached.FromOffline, true); + } + + CacheEntry? previous = cached; + + if (!_options.PreferOfflineSnapshot) + { + var network = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); + if (network is not null) + { + StoreCache(network); + return new CiscoProviderMetadataResult(network.Provider, network.FetchedAt, false, false); + } + } + + var offline = TryLoadFromOffline(); + if (offline is not null) + { + var entry = offline with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + FromOffline = true, + }; + StoreCache(entry); + return new CiscoProviderMetadataResult(entry.Provider, entry.FetchedAt, true, false); + } + + throw new InvalidOperationException("Unable to load Cisco CSAF provider metadata from network or offline snapshot."); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(CiscoConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); + if (!string.IsNullOrWhiteSpace(_options.ApiToken)) + { + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _options.ApiToken); + } + + if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Cisco provider metadata not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var provider = ParseProvider(payload); + var etagHeader = response.Headers.ETag?.ToString(); + + if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + try + { + _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Cisco metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + } + + return new CacheEntry( + provider, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + etagHeader, + FromOffline: false); + } + catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Cisco provider metadata from {Uri}; falling back to offline snapshot when available.", _options.MetadataUri); + return null; + } + } + + private CacheEntry? TryLoadFromOffline() + { + if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) + { + _logger.LogWarning("Cisco offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); + var provider = ParseProvider(payload); + return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, null, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Cisco provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); + return null; + } + } + + private VexProvider ParseProvider(string payload) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Cisco provider metadata payload was empty."); + } + + ProviderMetadataDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, _serializerOptions); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Failed to parse Cisco provider metadata.", ex); + } + + if (document?.Metadata?.Publisher?.ContactDetails is null || string.IsNullOrWhiteSpace(document.Metadata.Publisher.ContactDetails.Id)) + { + throw new InvalidOperationException("Cisco provider metadata did not include a publisher identifier."); + } + + var discovery = new VexProviderDiscovery(document.Discovery?.WellKnown, document.Discovery?.RolIe); + var trust = document.Trust is null + ? VexProviderTrust.Default + : new VexProviderTrust( + document.Trust.Weight ?? 1.0, + document.Trust.Cosign is null ? null : new VexCosignTrust(document.Trust.Cosign.Issuer ?? string.Empty, document.Trust.Cosign.IdentityPattern ?? string.Empty), + document.Trust.PgpFingerprints ?? Enumerable.Empty()); + + var directories = document.Distributions?.Directories is null + ? Enumerable.Empty() + : document.Distributions.Directories + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => Uri.TryCreate(s, UriKind.Absolute, out var uri) ? uri : null) + .Where(static uri => uri is not null)! + .Select(static uri => uri!); + + return new VexProvider( + id: document.Metadata.Publisher.ContactDetails.Id, + displayName: document.Metadata.Publisher.Name ?? document.Metadata.Publisher.ContactDetails.Id, + kind: document.Metadata.Publisher.Category?.Equals("vendor", StringComparison.OrdinalIgnoreCase) == true ? VexProviderKind.Vendor : VexProviderKind.Hub, + baseUris: directories, + discovery: discovery, + trust: trust, + enabled: true); + } + + private void StoreCache(CacheEntry entry) + { + var options = new MemoryCacheEntryOptions + { + AbsoluteExpiration = entry.ExpiresAt, + }; + _memoryCache.Set(CacheKey, entry, options); + } + + private sealed record CacheEntry( + VexProvider Provider, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOffline) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record CiscoProviderMetadataResult( + VexProvider Provider, + DateTimeOffset FetchedAt, + bool FromOfflineSnapshot, + bool ServedFromCache); + +#region document models + +internal sealed class ProviderMetadataDocument +{ + [System.Text.Json.Serialization.JsonPropertyName("metadata")] + public ProviderMetadataMetadata Metadata { get; set; } = new(); + + [System.Text.Json.Serialization.JsonPropertyName("discovery")] + public ProviderMetadataDiscovery? Discovery { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("trust")] + public ProviderMetadataTrust? Trust { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("distributions")] + public ProviderMetadataDistributions? Distributions { get; set; } +} + +internal sealed class ProviderMetadataMetadata +{ + [System.Text.Json.Serialization.JsonPropertyName("publisher")] + public ProviderMetadataPublisher Publisher { get; set; } = new(); +} + +internal sealed class ProviderMetadataPublisher +{ + [System.Text.Json.Serialization.JsonPropertyName("name")] + public string? Name { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("category")] + public string? Category { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("contact_details")] + public ProviderMetadataPublisherContact ContactDetails { get; set; } = new(); +} + +internal sealed class ProviderMetadataPublisherContact +{ + [System.Text.Json.Serialization.JsonPropertyName("id")] + public string? Id { get; set; } +} + +internal sealed class ProviderMetadataDiscovery +{ + [System.Text.Json.Serialization.JsonPropertyName("well_known")] + public Uri? WellKnown { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("rolie")] + public Uri? RolIe { get; set; } +} + +internal sealed class ProviderMetadataTrust +{ + [System.Text.Json.Serialization.JsonPropertyName("weight")] + public double? Weight { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("cosign")] + public ProviderMetadataTrustCosign? Cosign { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("pgp_fingerprints")] + public string[]? PgpFingerprints { get; set; } +} + +internal sealed class ProviderMetadataTrustCosign +{ + [System.Text.Json.Serialization.JsonPropertyName("issuer")] + public string? Issuer { get; set; } + + [System.Text.Json.Serialization.JsonPropertyName("identity_pattern")] + public string? IdentityPattern { get; set; } +} + +internal sealed class ProviderMetadataDistributions +{ + [System.Text.Json.Serialization.JsonPropertyName("directories")] + public string[]? Directories { get; set; } +} + +#endregion diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj similarity index 60% rename from src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj rename to src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj index d7480a49..4a15c1f6 100644 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/StellaOps.Vexer.Connectors.Cisco.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/StellaOps.Excititor.Connectors.Cisco.CSAF.csproj @@ -1,20 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md new file mode 100644 index 00000000..ae214bb6 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Cisco.CSAF/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-CISCO-01-001 – Endpoint discovery & auth plumbing|Team Excititor Connectors – Cisco|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `CiscoProviderMetadataLoader` with bearer token support, offline snapshot fallback, DI helpers, and tests covering network/offline discovery to unblock subsequent fetch work.| +|EXCITITOR-CONN-CISCO-01-002 – CSAF pull loop & pagination|Team Excititor Connectors – Cisco|EXCITITOR-CONN-CISCO-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Implemented paginated advisory fetch using provider directories, raw document persistence with dedupe/state tracking, offline resiliency, and unit coverage.| +|EXCITITOR-CONN-CISCO-01-003 – Provider trust metadata|Team Excititor Connectors – Cisco|EXCITITOR-CONN-CISCO-01-002, EXCITITOR-POLICY-01-001|**DOING (2025-10-19)** – Prereqs confirmed (both DONE); implementing cosign/PGP trust metadata emission and advisory provenance hints for policy weighting.| diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs index cea0cb3c..6525a04b 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Authentication/MsrcTokenProviderTests.cs @@ -1,176 +1,176 @@ -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using NSubstitute; -using StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; -using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; -using System.IO.Abstractions.TestingHelpers; -using Xunit; - -namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.Authentication; - -public sealed class MsrcTokenProviderTests -{ - [Fact] - public async Task GetAccessTokenAsync_CachesUntilExpiry() - { - var handler = new TestHttpMessageHandler(new[] - { - CreateTokenResponse("token-1"), - CreateTokenResponse("token-2"), - }); - var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = Options.Create(new MsrcConnectorOptions - { - TenantId = "contoso.onmicrosoft.com", - ClientId = "client-id", - ClientSecret = "secret", - }); - - var timeProvider = new AdjustableTimeProvider(); - var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); - - var first = await provider.GetAccessTokenAsync(CancellationToken.None); - first.Value.Should().Be("token-1"); - handler.InvocationCount.Should().Be(1); - - var second = await provider.GetAccessTokenAsync(CancellationToken.None); - second.Value.Should().Be("token-1"); - handler.InvocationCount.Should().Be(1); - } - - [Fact] - public async Task GetAccessTokenAsync_RefreshesWhenExpired() - { - var handler = new TestHttpMessageHandler(new[] - { - CreateTokenResponse("token-1", expiresIn: 120), - CreateTokenResponse("token-2", expiresIn: 3600), - }); - var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = Options.Create(new MsrcConnectorOptions - { - TenantId = "contoso.onmicrosoft.com", - ClientId = "client-id", - ClientSecret = "secret", - ExpiryLeewaySeconds = 60, - }); - - var timeProvider = new AdjustableTimeProvider(); - var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); - - var first = await provider.GetAccessTokenAsync(CancellationToken.None); - first.Value.Should().Be("token-1"); - handler.InvocationCount.Should().Be(1); - - timeProvider.Advance(TimeSpan.FromMinutes(2)); - var second = await provider.GetAccessTokenAsync(CancellationToken.None); - second.Value.Should().Be("token-2"); - handler.InvocationCount.Should().Be(2); - } - - [Fact] - public async Task GetAccessTokenAsync_OfflineStaticToken() - { - var factory = Substitute.For(); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = Options.Create(new MsrcConnectorOptions - { - PreferOfflineToken = true, - StaticAccessToken = "offline-token", - }); - - var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); - var token = await provider.GetAccessTokenAsync(CancellationToken.None); - token.Value.Should().Be("offline-token"); - token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); - } - - [Fact] - public async Task GetAccessTokenAsync_OfflineFileToken() - { - var factory = Substitute.For(); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var offlinePath = fileSystem.Path.Combine("/tokens", "msrc.txt"); - fileSystem.AddFile(offlinePath, new MockFileData("file-token")); - - var options = Options.Create(new MsrcConnectorOptions - { - PreferOfflineToken = true, - OfflineTokenPath = offlinePath, - }); - - var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); - var token = await provider.GetAccessTokenAsync(CancellationToken.None); - token.Value.Should().Be("file-token"); - token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); - } - - private static HttpResponseMessage CreateTokenResponse(string token, int expiresIn = 3600) - { - var json = $"{{\"access_token\":\"{token}\",\"token_type\":\"Bearer\",\"expires_in\":{expiresIn}}}"; - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(json, Encoding.UTF8, "application/json"), - }; - } - - private sealed class AdjustableTimeProvider : TimeProvider - { - private DateTimeOffset _now = DateTimeOffset.UtcNow; - - public override DateTimeOffset GetUtcNow() => _now; - - public void Advance(TimeSpan span) => _now = _now.Add(span); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Queue _responses; - - public TestHttpMessageHandler(IEnumerable responses) - { - _responses = new Queue(responses); - } - - public int InvocationCount { get; private set; } - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - InvocationCount++; - if (_responses.Count == 0) - { - return Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError) - { - Content = new StringContent("no responses remaining"), - }); - } - - return Task.FromResult(_responses.Dequeue()); - } - } -} +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using NSubstitute; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Authentication; + +public sealed class MsrcTokenProviderTests +{ + [Fact] + public async Task GetAccessTokenAsync_CachesUntilExpiry() + { + var handler = new TestHttpMessageHandler(new[] + { + CreateTokenResponse("token-1"), + CreateTokenResponse("token-2"), + }); + var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + TenantId = "contoso.onmicrosoft.com", + ClientId = "client-id", + ClientSecret = "secret", + }); + + var timeProvider = new AdjustableTimeProvider(); + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); + + var first = await provider.GetAccessTokenAsync(CancellationToken.None); + first.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + + var second = await provider.GetAccessTokenAsync(CancellationToken.None); + second.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + } + + [Fact] + public async Task GetAccessTokenAsync_RefreshesWhenExpired() + { + var handler = new TestHttpMessageHandler(new[] + { + CreateTokenResponse("token-1", expiresIn: 120), + CreateTokenResponse("token-2", expiresIn: 3600), + }); + var client = new HttpClient(handler) { BaseAddress = new Uri("https://login.microsoftonline.com/") }; + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + TenantId = "contoso.onmicrosoft.com", + ClientId = "client-id", + ClientSecret = "secret", + ExpiryLeewaySeconds = 60, + }); + + var timeProvider = new AdjustableTimeProvider(); + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance, timeProvider); + + var first = await provider.GetAccessTokenAsync(CancellationToken.None); + first.Value.Should().Be("token-1"); + handler.InvocationCount.Should().Be(1); + + timeProvider.Advance(TimeSpan.FromMinutes(2)); + var second = await provider.GetAccessTokenAsync(CancellationToken.None); + second.Value.Should().Be("token-2"); + handler.InvocationCount.Should().Be(2); + } + + [Fact] + public async Task GetAccessTokenAsync_OfflineStaticToken() + { + var factory = Substitute.For(); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = Options.Create(new MsrcConnectorOptions + { + PreferOfflineToken = true, + StaticAccessToken = "offline-token", + }); + + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); + var token = await provider.GetAccessTokenAsync(CancellationToken.None); + token.Value.Should().Be("offline-token"); + token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); + } + + [Fact] + public async Task GetAccessTokenAsync_OfflineFileToken() + { + var factory = Substitute.For(); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine("/tokens", "msrc.txt"); + fileSystem.AddFile(offlinePath, new MockFileData("file-token")); + + var options = Options.Create(new MsrcConnectorOptions + { + PreferOfflineToken = true, + OfflineTokenPath = offlinePath, + }); + + var provider = new MsrcTokenProvider(factory, cache, fileSystem, options, NullLogger.Instance); + var token = await provider.GetAccessTokenAsync(CancellationToken.None); + token.Value.Should().Be("file-token"); + token.ExpiresAt.Should().Be(DateTimeOffset.MaxValue); + } + + private static HttpResponseMessage CreateTokenResponse(string token, int expiresIn = 3600) + { + var json = $"{{\"access_token\":\"{token}\",\"token_type\":\"Bearer\",\"expires_in\":{expiresIn}}}"; + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + + public void Advance(TimeSpan span) => _now = _now.Add(span); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue _responses; + + public TestHttpMessageHandler(IEnumerable responses) + { + _responses = new Queue(responses); + } + + public int InvocationCount { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (_responses.Count == 0) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("no responses remaining"), + }); + } + + return Task.FromResult(_responses.Dequeue()); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs new file mode 100644 index 00000000..c28dd4e1 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/Connectors/MsrcCsafConnectorTests.cs @@ -0,0 +1,363 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Compression; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.MSRC.CSAF; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using Xunit; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.Connectors; + +public sealed class MsrcCsafConnectorTests +{ + private static readonly VexConnectorDescriptor Descriptor = new("excititor:msrc", VexProviderKind.Vendor, "MSRC CSAF"); + + [Fact] + public async Task FetchAsync_EmitsDocumentAndPersistsState() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0001", + "vulnerabilityId": "ADV-0001", + "severity": "Critical", + "releaseDate": "2025-10-17T00:00:00Z", + "lastModifiedDate": "2025-10-18T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0001.json" + } + ] + } + """; + + var csaf = """{"document":{"title":"Example"}}"""; + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + var emitted = documents[0]; + emitted.SourceUri.Should().Be(new Uri("https://example.com/csaf/ADV-0001.json")); + emitted.Metadata["msrc.vulnerabilityId"].Should().Be("ADV-0001"); + emitted.Metadata["msrc.csaf.format"].Should().Be("json"); + emitted.Metadata.Should().NotContainKey("excititor.quarantine.reason"); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 18, 0, 0, 0, TimeSpan.Zero)); + stateRepository.State.DocumentDigests.Should().HaveCount(1); + } + + [Fact] + public async Task FetchAsync_SkipsDocumentsWithExistingDigest() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0001", + "vulnerabilityId": "ADV-0001", + "lastModifiedDate": "2025-10-18T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0001.json" + } + ] + } + """; + + var csaf = """{"document":{"title":"Example"}}"""; + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 15, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var firstPass = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + firstPass.Add(document); + } + + firstPass.Should().HaveCount(1); + stateRepository.State.Should().NotBeNull(); + var persistedState = stateRepository.State!; + + handler.Reset( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csaf, "application/json")); + + sink.Documents.Clear(); + var secondPass = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + secondPass.Add(document); + } + + secondPass.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().Equal(persistedState.DocumentDigests); + } + + [Fact] + public async Task FetchAsync_QuarantinesInvalidCsafPayload() + { + var summary = """ + { + "value": [ + { + "id": "ADV-0002", + "vulnerabilityId": "ADV-0002", + "lastModifiedDate": "2025-10-19T00:00:00Z", + "cvrfUrl": "https://example.com/csaf/ADV-0002.zip" + } + ] + } + """; + + var csafZip = CreateZip("document.json", "{ invalid json "); + var handler = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, summary, "application/json"), + _ => Response(HttpStatusCode.OK, csafZip, "application/zip")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var stateRepository = new InMemoryConnectorStateRepository(); + var options = Options.Create(CreateOptions()); + var connector = new MsrcCsafConnector( + factory, + new StubTokenProvider(), + stateRepository, + options, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(VexConnectorSettings.Empty, CancellationToken.None); + + var sink = new CapturingRawSink(); + var context = new VexConnectorContext( + Since: new DateTimeOffset(2025, 10, 17, 0, 0, 0, TimeSpan.Zero), + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().HaveCount(1); + sink.Documents[0].Metadata["excititor.quarantine.reason"].Should().Contain("JSON parse failed"); + sink.Documents[0].Metadata["msrc.csaf.format"].Should().Be("zip"); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().HaveCount(1); + } + + private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) + => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + + private static HttpResponseMessage Response(HttpStatusCode statusCode, byte[] content, string contentType) + { + var response = new HttpResponseMessage(statusCode); + response.Content = new ByteArrayContent(content); + response.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); + return response; + } + + private static MsrcConnectorOptions CreateOptions() + => new() + { + BaseUri = new Uri("https://example.com/", UriKind.Absolute), + TenantId = Guid.NewGuid().ToString(), + ClientId = "client-id", + ClientSecret = "secret", + Scope = MsrcConnectorOptions.DefaultScope, + PageSize = 5, + MaxAdvisoriesPerFetch = 5, + RequestDelay = TimeSpan.Zero, + RetryBaseDelay = TimeSpan.FromMilliseconds(10), + MaxRetryAttempts = 2, + }; + + private static byte[] CreateZip(string entryName, string content) + { + using var buffer = new MemoryStream(); + using (var archive = new ZipArchive(buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + var entry = archive.CreateEntry(entryName); + using var writer = new StreamWriter(entry.Open(), Encoding.UTF8); + writer.Write(content); + } + + return buffer.ToArray(); + } + + private sealed class StubTokenProvider : IMsrcTokenProvider + { + public ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) + => ValueTask.FromResult(new MsrcAccessToken("token", "Bearer", DateTimeOffset.MaxValue)); + } + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + => ValueTask.FromResult(State); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + State = state; + return ValueTask.CompletedTask; + } + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + public void Reset(params Func[] responders) + { + _responders.Clear(); + foreach (var responder in responders) + { + _responders.Enqueue(responder); + } + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for MSRC connector test request."); + } + + var responder = _responders.Count > 1 ? _responders.Dequeue() : _responders.Peek(); + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj similarity index 78% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj index b3eb395f..15b12b14 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests/StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests/StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj @@ -1,18 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/AGENTS.md b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/AGENTS.md similarity index 94% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF/AGENTS.md index b96c34a8..7d4abf20 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector for Microsoft Security Response Center (MSRC) CSAF advisories, handling authenticated downloads, throttling, and raw document persistence. -## Scope -- MSRC API onboarding (AAD client credentials), metadata discovery, and CSAF listing retrieval. -- Download pipeline with retry/backoff, checksum validation, and document deduplication. -- Mapping MSRC-specific identifiers (CVE, ADV, KB) and remediation guidance into connector metadata. -- Emitting trust metadata (AAD issuer, signing certificates) for policy weighting. -## Participants -- Worker schedules MSRC pulls honoring rate limits; WebService may trigger manual runs for urgent updates. -- CSAF normalizer processes retrieved documents into claims. -- Policy subsystem references connector trust hints for consensus scoring. -## Interfaces & contracts -- Implements `IVexConnector`, requires configuration options for tenant/client/secret or managed identity. -- Uses shared HTTP helpers, resume markers, and telemetry from Abstractions module. -## In/Out of scope -In: authenticated fetching, raw document storage, metadata mapping, retry logic. -Out: normalization/export, attestation, storage implementations (handled elsewhere). -## Observability & security expectations -- Log request batches, rate-limit responses, and token refresh events without leaking secrets. -- Track metrics for documents fetched, retries, and failure categories. -## Tests -- Connector tests with mocked MSRC endpoints and AAD token flow will live in `../StellaOps.Vexer.Connectors.MSRC.CSAF.Tests`. +# AGENTS +## Role +Connector for Microsoft Security Response Center (MSRC) CSAF advisories, handling authenticated downloads, throttling, and raw document persistence. +## Scope +- MSRC API onboarding (AAD client credentials), metadata discovery, and CSAF listing retrieval. +- Download pipeline with retry/backoff, checksum validation, and document deduplication. +- Mapping MSRC-specific identifiers (CVE, ADV, KB) and remediation guidance into connector metadata. +- Emitting trust metadata (AAD issuer, signing certificates) for policy weighting. +## Participants +- Worker schedules MSRC pulls honoring rate limits; WebService may trigger manual runs for urgent updates. +- CSAF normalizer processes retrieved documents into claims. +- Policy subsystem references connector trust hints for consensus scoring. +## Interfaces & contracts +- Implements `IVexConnector`, requires configuration options for tenant/client/secret or managed identity. +- Uses shared HTTP helpers, resume markers, and telemetry from Abstractions module. +## In/Out of scope +In: authenticated fetching, raw document storage, metadata mapping, retry logic. +Out: normalization/export, attestation, storage implementations (handled elsewhere). +## Observability & security expectations +- Log request batches, rate-limit responses, and token refresh events without leaking secrets. +- Track metrics for documents fetched, retries, and failure categories. +## Tests +- Connector tests with mocked MSRC endpoints and AAD token flow will live in `../StellaOps.Excititor.Connectors.MSRC.CSAF.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs index c201e8dd..e404556c 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/Authentication/MsrcTokenProvider.cs @@ -1,185 +1,185 @@ -using System; -using System.Collections.Generic; -using System.IO.Abstractions; -using System.Net.Http; -using System.Net.Http.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; - -namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; - -public interface IMsrcTokenProvider -{ - ValueTask GetAccessTokenAsync(CancellationToken cancellationToken); -} - -public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable -{ - private const string CachePrefix = "StellaOps.Vexer.Connectors.MSRC.CSAF.Token"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _cache; - private readonly IFileSystem _fileSystem; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly MsrcConnectorOptions _options; - private readonly SemaphoreSlim _refreshLock = new(1, 1); - - public MsrcTokenProvider( - IHttpClientFactory httpClientFactory, - IMemoryCache cache, - IFileSystem fileSystem, - IOptions options, - ILogger logger, - TimeProvider? timeProvider = null) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _cache = cache ?? throw new ArgumentNullException(nameof(cache)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - ArgumentNullException.ThrowIfNull(options); - _options = options.Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(_fileSystem); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public async ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) - { - if (_options.PreferOfflineToken) - { - return LoadOfflineToken(); - } - - var cacheKey = CreateCacheKey(); - if (_cache.TryGetValue(cacheKey, out var cachedToken) && - cachedToken is not null && - !cachedToken.IsExpired(_timeProvider.GetUtcNow())) - { - return cachedToken; - } - - await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_cache.TryGetValue(cacheKey, out cachedToken) && - cachedToken is not null && - !cachedToken.IsExpired(_timeProvider.GetUtcNow())) - { - return cachedToken; - } - - var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false); - var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue - ? (DateTimeOffset?)null - : token.ExpiresAt; - - var options = new MemoryCacheEntryOptions(); - if (absoluteExpiration.HasValue) - { - options.AbsoluteExpiration = absoluteExpiration.Value; - } - - _cache.Set(cacheKey, token, options); - return token; - } - finally - { - _refreshLock.Release(); - } - } - - private MsrcAccessToken LoadOfflineToken() - { - if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken)) - { - return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue); - } - - if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath)) - { - throw new InvalidOperationException("Offline token mode is enabled but no token was provided."); - } - - if (!_fileSystem.File.Exists(_options.OfflineTokenPath)) - { - throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist."); - } - - var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim(); - if (string.IsNullOrEmpty(token)) - { - throw new InvalidOperationException("Offline token file was empty."); - } - - return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue); - } - - private async Task RequestTokenAsync(CancellationToken cancellationToken) - { - _logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId); - - var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName); - using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) - { - Content = new FormUrlEncodedContent(new Dictionary - { - ["client_id"] = _options.ClientId, - ["client_secret"] = _options.ClientSecret!, - ["grant_type"] = "client_credentials", - ["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope, - }), - }; - - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}"); - } - - var tokenResponse = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException("Token endpoint returned an empty payload."); - - if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken)) - { - throw new InvalidOperationException("Token endpoint response did not include an access_token."); - } - - var now = _timeProvider.GetUtcNow(); - var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds - ? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds) - : now.AddMinutes(5); - - return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt); - } - - private string CreateCacheKey() - => $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}"; - - private Uri BuildTokenUri() - => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); - - public void Dispose() => _refreshLock.Dispose(); - - private sealed record TokenResponse - { - [JsonPropertyName("access_token")] - public string? AccessToken { get; init; } - - [JsonPropertyName("token_type")] - public string? TokenType { get; init; } - - [JsonPropertyName("expires_in")] - public int ExpiresIn { get; init; } - } -} - -public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt) -{ - public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; -} +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using System.Net.Http; +using System.Net.Http.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; + +public interface IMsrcTokenProvider +{ + ValueTask GetAccessTokenAsync(CancellationToken cancellationToken); +} + +public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable +{ + private const string CachePrefix = "StellaOps.Excititor.Connectors.MSRC.CSAF.Token"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly MsrcConnectorOptions _options; + private readonly SemaphoreSlim _refreshLock = new(1, 1); + + public MsrcTokenProvider( + IHttpClientFactory httpClientFactory, + IMemoryCache cache, + IFileSystem fileSystem, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value ?? throw new ArgumentNullException(nameof(options)); + _options.Validate(_fileSystem); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask GetAccessTokenAsync(CancellationToken cancellationToken) + { + if (_options.PreferOfflineToken) + { + return LoadOfflineToken(); + } + + var cacheKey = CreateCacheKey(); + if (_cache.TryGetValue(cacheKey, out var cachedToken) && + cachedToken is not null && + !cachedToken.IsExpired(_timeProvider.GetUtcNow())) + { + return cachedToken; + } + + await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(cacheKey, out cachedToken) && + cachedToken is not null && + !cachedToken.IsExpired(_timeProvider.GetUtcNow())) + { + return cachedToken; + } + + var token = await RequestTokenAsync(cancellationToken).ConfigureAwait(false); + var absoluteExpiration = token.ExpiresAt == DateTimeOffset.MaxValue + ? (DateTimeOffset?)null + : token.ExpiresAt; + + var options = new MemoryCacheEntryOptions(); + if (absoluteExpiration.HasValue) + { + options.AbsoluteExpiration = absoluteExpiration.Value; + } + + _cache.Set(cacheKey, token, options); + return token; + } + finally + { + _refreshLock.Release(); + } + } + + private MsrcAccessToken LoadOfflineToken() + { + if (!string.IsNullOrWhiteSpace(_options.StaticAccessToken)) + { + return new MsrcAccessToken(_options.StaticAccessToken!, "Bearer", DateTimeOffset.MaxValue); + } + + if (string.IsNullOrWhiteSpace(_options.OfflineTokenPath)) + { + throw new InvalidOperationException("Offline token mode is enabled but no token was provided."); + } + + if (!_fileSystem.File.Exists(_options.OfflineTokenPath)) + { + throw new InvalidOperationException($"Offline token path '{_options.OfflineTokenPath}' does not exist."); + } + + var token = _fileSystem.File.ReadAllText(_options.OfflineTokenPath).Trim(); + if (string.IsNullOrEmpty(token)) + { + throw new InvalidOperationException("Offline token file was empty."); + } + + return new MsrcAccessToken(token, "Bearer", DateTimeOffset.MaxValue); + } + + private async Task RequestTokenAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Fetching MSRC AAD access token for tenant {TenantId}.", _options.TenantId); + + var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.TokenClientName); + using var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) + { + Content = new FormUrlEncodedContent(new Dictionary + { + ["client_id"] = _options.ClientId, + ["client_secret"] = _options.ClientSecret!, + ["grant_type"] = "client_credentials", + ["scope"] = string.IsNullOrWhiteSpace(_options.Scope) ? MsrcConnectorOptions.DefaultScope : _options.Scope, + }), + }; + + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to acquire MSRC access token ({(int)response.StatusCode}). Response: {payload}"); + } + + var tokenResponse = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false) + ?? throw new InvalidOperationException("Token endpoint returned an empty payload."); + + if (string.IsNullOrWhiteSpace(tokenResponse.AccessToken)) + { + throw new InvalidOperationException("Token endpoint response did not include an access_token."); + } + + var now = _timeProvider.GetUtcNow(); + var expiresAt = tokenResponse.ExpiresIn > _options.ExpiryLeewaySeconds + ? now.AddSeconds(tokenResponse.ExpiresIn - _options.ExpiryLeewaySeconds) + : now.AddMinutes(5); + + return new MsrcAccessToken(tokenResponse.AccessToken!, tokenResponse.TokenType ?? "Bearer", expiresAt); + } + + private string CreateCacheKey() + => $"{CachePrefix}:{_options.TenantId}:{_options.ClientId}:{_options.Scope}"; + + private Uri BuildTokenUri() + => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); + + public void Dispose() => _refreshLock.Dispose(); + + private sealed record TokenResponse + { + [JsonPropertyName("access_token")] + public string? AccessToken { get; init; } + + [JsonPropertyName("token_type")] + public string? TokenType { get; init; } + + [JsonPropertyName("expires_in")] + public int ExpiresIn { get; init; } + } +} + +public sealed record MsrcAccessToken(string Value, string Type, DateTimeOffset ExpiresAt) +{ + public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; +} diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs new file mode 100644 index 00000000..aaafb148 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs @@ -0,0 +1,211 @@ +using System; +using System.Globalization; +using System.IO; +using System.IO.Abstractions; +using System.Linq; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; + +public sealed class MsrcConnectorOptions +{ + public const string TokenClientName = "excititor.connector.msrc.token"; + public const string DefaultScope = "https://api.msrc.microsoft.com/.default"; + public const string ApiClientName = "excititor.connector.msrc.api"; + public const string DefaultBaseUri = "https://api.msrc.microsoft.com/sug/v2.0/"; + public const string DefaultLocale = "en-US"; + public const string DefaultApiVersion = "2024-08-01"; + + /// + /// Azure AD tenant identifier (GUID or domain). + /// + public string TenantId { get; set; } = string.Empty; + + /// + /// Azure AD application (client) identifier. + /// + public string ClientId { get; set; } = string.Empty; + + /// + /// Azure AD application secret for client credential flow. + /// + public string? ClientSecret { get; set; } + /// + /// OAuth scope requested for MSRC API access. + /// + public string Scope { get; set; } = DefaultScope; + + /// + /// When true, token acquisition is skipped and the connector expects offline handling. + /// + public bool PreferOfflineToken { get; set; } + /// + /// Optional path to a pre-provisioned bearer token used when is enabled. + /// + public string? OfflineTokenPath { get; set; } + /// + /// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles). + /// + public string? StaticAccessToken { get; set; } + /// + /// Minimum buffer (seconds) subtracted from token expiry before refresh. + /// + public int ExpiryLeewaySeconds { get; set; } = 60; + + /// + /// Base URI for MSRC Security Update Guide API. + /// + public Uri BaseUri { get; set; } = new(DefaultBaseUri, UriKind.Absolute); + + /// + /// Locale requested when fetching summaries. + /// + public string Locale { get; set; } = DefaultLocale; + + /// + /// API version appended to MSRC requests. + /// + public string ApiVersion { get; set; } = DefaultApiVersion; + + /// + /// Page size used while enumerating summaries. + /// + public int PageSize { get; set; } = 100; + + /// + /// Maximum CSAF advisories fetched per connector run. + /// + public int MaxAdvisoriesPerFetch { get; set; } = 200; + + /// + /// Overlap window applied when resuming from the last modified cursor. + /// + public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10); + + /// + /// Delay between CSAF downloads to respect rate limits. + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + /// + /// Maximum retry attempts for summary/detail fetch operations. + /// + public int MaxRetryAttempts { get; set; } = 3; + + /// + /// Base delay applied between retries (jitter handled by connector). + /// + public TimeSpan RetryBaseDelay { get; set; } = TimeSpan.FromSeconds(2); + + /// + /// Optional lower bound for initial synchronisation when no cursor is stored. + /// + public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30); + + /// + /// Maximum number of document digests persisted for deduplication. + /// + public int MaxTrackedDigests { get; set; } = 2048; + + public void Validate(IFileSystem? fileSystem = null) + { + if (PreferOfflineToken) + { + if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken)) + { + throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled."); + } + } + else + { + if (string.IsNullOrWhiteSpace(TenantId)) + { + throw new InvalidOperationException("TenantId is required when not operating in offline token mode."); + } + + if (string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("ClientId is required when not operating in offline token mode."); + } + + if (string.IsNullOrWhiteSpace(ClientSecret)) + { + throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode."); + } + } + + if (string.IsNullOrWhiteSpace(Scope)) + { + Scope = DefaultScope; + } + + if (ExpiryLeewaySeconds < 10) + { + ExpiryLeewaySeconds = 10; + } + + if (BaseUri is null || !BaseUri.IsAbsoluteUri) + { + throw new InvalidOperationException("BaseUri must be an absolute URI."); + } + + if (string.IsNullOrWhiteSpace(Locale)) + { + throw new InvalidOperationException("Locale must be provided."); + } + + if (!CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException($"Locale '{Locale}' is not recognised."); + } + + if (string.IsNullOrWhiteSpace(ApiVersion)) + { + throw new InvalidOperationException("ApiVersion must be provided."); + } + + if (PageSize <= 0 || PageSize > 500) + { + throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500."); + } + + if (MaxAdvisoriesPerFetch <= 0) + { + throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); + } + + if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6)) + { + throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours."); + } + + if (RequestDelay < TimeSpan.Zero || RequestDelay > TimeSpan.FromSeconds(10)) + { + throw new InvalidOperationException($"{nameof(RequestDelay)} must be between 0 and 10 seconds."); + } + + if (MaxRetryAttempts <= 0 || MaxRetryAttempts > 10) + { + throw new InvalidOperationException($"{nameof(MaxRetryAttempts)} must be between 1 and 10."); + } + + if (RetryBaseDelay < TimeSpan.Zero || RetryBaseDelay > TimeSpan.FromMinutes(5)) + { + throw new InvalidOperationException($"{nameof(RetryBaseDelay)} must be between 0 and 5 minutes."); + } + + if (MaxTrackedDigests <= 0 || MaxTrackedDigests > 10000) + { + throw new InvalidOperationException($"{nameof(MaxTrackedDigests)} must be between 1 and 10000."); + } + + if (!string.IsNullOrWhiteSpace(OfflineTokenPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineTokenPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs similarity index 52% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs index 633e647c..28fec447 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/DependencyInjection/MsrcConnectorServiceCollectionExtensions.cs @@ -1,40 +1,58 @@ -using System; -using System.Net; -using System.Net.Http; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using StellaOps.Vexer.Connectors.MSRC.CSAF.Authentication; -using StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.MSRC.CSAF.DependencyInjection; - -public static class MsrcConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddOptions() - .Configure(options => configure?.Invoke(options)); - - services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(30); - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.MSRC.CSAF/1.0"); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - }) - .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.All, - }); - - services.AddSingleton(); - - return services; - } -} +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using System.IO.Abstractions; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF.DependencyInjection; + +public static class MsrcConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddMsrcCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddHttpClient(MsrcConnectorOptions.TokenClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddHttpClient(MsrcConnectorOptions.ApiClientName) + .ConfigureHttpClient((provider, client) => + { + var options = provider.GetRequiredService>().Value; + client.BaseAddress = options.BaseUri; + client.Timeout = TimeSpan.FromSeconds(60); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.MSRC.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs new file mode 100644 index 00000000..21e558e8 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/MsrcCsafConnector.cs @@ -0,0 +1,581 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Authentication; +using StellaOps.Excititor.Connectors.MSRC.CSAF.Configuration; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.MSRC.CSAF; + +public sealed class MsrcCsafConnector : VexConnectorBase +{ + private const string QuarantineMetadataKey = "excititor.quarantine.reason"; + private const string FormatMetadataKey = "msrc.csaf.format"; + private const string VulnerabilityMetadataKey = "msrc.vulnerabilityId"; + private const string AdvisoryIdMetadataKey = "msrc.advisoryId"; + private const string LastModifiedMetadataKey = "msrc.lastModified"; + private const string ReleaseDateMetadataKey = "msrc.releaseDate"; + private const string CvssSeverityMetadataKey = "msrc.severity"; + private const string CvrfUrlMetadataKey = "msrc.cvrfUrl"; + + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:msrc", + kind: VexProviderKind.Vendor, + displayName: "Microsoft MSRC CSAF") + { + Description = "Authenticated connector for Microsoft Security Response Center CSAF advisories.", + SupportedFormats = ImmutableArray.Create(VexDocumentFormat.Csaf), + Tags = ImmutableArray.Create("microsoft", "csaf", "vendor"), + }; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMsrcTokenProvider _tokenProvider; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + + private MsrcConnectorOptions? _validatedOptions; + + public MsrcCsafConnector( + IHttpClientFactory httpClientFactory, + IMsrcTokenProvider tokenProvider, + IVexConnectorStateRepository stateRepository, + IOptions options, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); + options.Validate(); + _validatedOptions = options; + + LogConnectorEvent( + LogLevel.Information, + "validate", + "Validated MSRC CSAF connector options.", + new Dictionary + { + ["baseUri"] = options.BaseUri.ToString(), + ["locale"] = options.Locale, + ["apiVersion"] = options.ApiVersion, + ["pageSize"] = options.PageSize, + ["maxAdvisories"] = options.MaxAdvisoriesPerFetch, + }); + + return ValueTask.CompletedTask; + } + + public override async IAsyncEnumerable FetchAsync( + VexConnectorContext context, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var options = EnsureOptionsValidated(); + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var (from, to) = CalculateWindow(context.Since, state, options); + + LogConnectorEvent( + LogLevel.Information, + "fetch.window", + $"Fetching MSRC CSAF advisories updated between {from:O} and {to:O}.", + new Dictionary + { + ["from"] = from, + ["to"] = to, + ["cursorOverlapSeconds"] = options.CursorOverlap.TotalSeconds, + }); + + var client = await CreateAuthenticatedClientAsync(options, cancellationToken).ConfigureAwait(false); + + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var digestList = new List(knownDigests); + var latest = state?.LastUpdated ?? from; + var fetched = 0; + var stateChanged = false; + + await foreach (var summary in EnumerateSummariesAsync(client, options, from, to, cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (fetched >= options.MaxAdvisoriesPerFetch) + { + break; + } + + if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) + { + LogConnectorEvent(LogLevel.Debug, "skip.no-cvrf", $"Skipping MSRC advisory {summary.Id} because no CSAF URL was provided."); + continue; + } + + var documentUri = ResolveCvrfUri(options.BaseUri, summary.CvrfUrl); + + VexRawDocument? rawDocument = null; + try + { + rawDocument = await DownloadCsafAsync(client, summary, documentUri, options, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.error", $"Failed to download MSRC CSAF package {documentUri}.", new Dictionary + { + ["advisoryId"] = summary.Id, + ["vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, + }, ex); + + await Task.Delay(GetRetryDelay(options, 1), cancellationToken).ConfigureAwait(false); + continue; + } + + if (!digestSet.Add(rawDocument.Digest)) + { + LogConnectorEvent(LogLevel.Debug, "skip.duplicate", $"Skipping MSRC CSAF package {documentUri} because it was already processed."); + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + fetched++; + + latest = DetermineLatest(summary, latest) ?? latest; + + var quarantineReason = rawDocument.Metadata.TryGetValue(QuarantineMetadataKey, out var reason) ? reason : null; + if (quarantineReason is not null) + { + LogConnectorEvent(LogLevel.Warning, "quarantine", $"Quarantined MSRC CSAF package {documentUri} ({quarantineReason})."); + continue; + } + + yield return rawDocument; + + if (options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (stateChanged) + { + if (digestList.Count > options.MaxTrackedDigests) + { + var trimmed = digestList.Count - options.MaxTrackedDigests; + digestList.RemoveRange(0, trimmed); + } + + var baseState = state ?? new VexConnectorState( + Descriptor.Id, + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var newState = baseState with + { + LastUpdated = latest == DateTimeOffset.MinValue ? state?.LastUpdated : latest, + DocumentDigests = digestList.ToImmutableArray(), + }; + + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + + LogConnectorEvent( + LogLevel.Information, + "fetch.completed", + $"MSRC CSAF fetch completed with {fetched} new documents.", + new Dictionary + { + ["fetched"] = fetched, + ["stateChanged"] = stateChanged, + ["lastUpdated"] = latest, + }); + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("MSRC CSAF connector relies on CSAF normalizers for document processing."); + + private async Task DownloadCsafAsync( + HttpClient client, + MsrcVulnerabilitySummary summary, + Uri documentUri, + MsrcConnectorOptions options, + CancellationToken cancellationToken) + { + using var response = await SendWithRetryAsync( + client, + () => new HttpRequestMessage(HttpMethod.Get, documentUri), + options, + cancellationToken).ConfigureAwait(false); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + + var validation = ValidateCsafPayload(payload); + var metadata = BuildMetadata(builder => + { + builder.Add(AdvisoryIdMetadataKey, summary.Id); + builder.Add(VulnerabilityMetadataKey, summary.VulnerabilityId ?? summary.Id); + builder.Add(CvrfUrlMetadataKey, documentUri.ToString()); + builder.Add(FormatMetadataKey, validation.Format); + + if (!string.IsNullOrWhiteSpace(summary.Severity)) + { + builder.Add(CvssSeverityMetadataKey, summary.Severity); + } + + if (summary.LastModifiedDate is not null) + { + builder.Add(LastModifiedMetadataKey, summary.LastModifiedDate.Value.ToString("O")); + } + + if (summary.ReleaseDate is not null) + { + builder.Add(ReleaseDateMetadataKey, summary.ReleaseDate.Value.ToString("O")); + } + + if (!string.IsNullOrWhiteSpace(validation.QuarantineReason)) + { + builder.Add(QuarantineMetadataKey, validation.QuarantineReason); + } + + if (response.Headers.ETag is not null) + { + builder.Add("http.etag", response.Headers.ETag.Tag); + } + + if (response.Content.Headers.LastModified is { } lastModified) + { + builder.Add("http.lastModified", lastModified.ToString("O")); + } + }); + + return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, payload, metadata); + } + + private async Task CreateAuthenticatedClientAsync(MsrcConnectorOptions options, CancellationToken cancellationToken) + { + var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); + var client = _httpClientFactory.CreateClient(MsrcConnectorOptions.ApiClientName); + + client.DefaultRequestHeaders.Remove("Authorization"); + client.DefaultRequestHeaders.Add("Authorization", $"{token.Type} {token.Value}"); + client.DefaultRequestHeaders.Remove("Accept-Language"); + client.DefaultRequestHeaders.Add("Accept-Language", options.Locale); + client.DefaultRequestHeaders.Remove("api-version"); + client.DefaultRequestHeaders.Add("api-version", options.ApiVersion); + client.DefaultRequestHeaders.Remove("Accept"); + client.DefaultRequestHeaders.Add("Accept", "application/json"); + + return client; + } + + private async Task SendWithRetryAsync( + HttpClient client, + Func requestFactory, + MsrcConnectorOptions options, + CancellationToken cancellationToken) + { + Exception? lastError = null; + HttpResponseMessage? response = null; + + for (var attempt = 1; attempt <= options.MaxRetryAttempts; attempt++) + { + response?.Dispose(); + using var request = requestFactory(); + try + { + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + return response; + } + + if (!ShouldRetry(response.StatusCode) || attempt == options.MaxRetryAttempts) + { + response.EnsureSuccessStatusCode(); + } + } + catch (Exception ex) when (IsTransient(ex) && attempt < options.MaxRetryAttempts) + { + lastError = ex; + LogConnectorEvent(LogLevel.Warning, "retry", $"Retrying MSRC request (attempt {attempt}/{options.MaxRetryAttempts}).", exception: ex); + } + catch (Exception ex) + { + response?.Dispose(); + throw; + } + + await Task.Delay(GetRetryDelay(options, attempt), cancellationToken).ConfigureAwait(false); + } + + response?.Dispose(); + throw lastError ?? new InvalidOperationException("MSRC request retries exhausted."); + } + + private TimeSpan GetRetryDelay(MsrcConnectorOptions options, int attempt) + { + var baseDelay = options.RetryBaseDelay.TotalMilliseconds; + var multiplier = Math.Pow(2, Math.Max(0, attempt - 1)); + var jitter = Random.Shared.NextDouble() * baseDelay * 0.25; + var delayMs = Math.Min(baseDelay * multiplier + jitter, TimeSpan.FromMinutes(5).TotalMilliseconds); + return TimeSpan.FromMilliseconds(delayMs); + } + + private async IAsyncEnumerable EnumerateSummariesAsync( + HttpClient client, + MsrcConnectorOptions options, + DateTimeOffset from, + DateTimeOffset to, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + var fetched = 0; + var requestUri = BuildSummaryUri(options, from, to); + + while (requestUri is not null && fetched < options.MaxAdvisoriesPerFetch) + { + using var response = await SendWithRetryAsync( + client, + () => new HttpRequestMessage(HttpMethod.Get, requestUri), + options, + cancellationToken).ConfigureAwait(false); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var payload = await JsonSerializer.DeserializeAsync(stream, _serializerOptions, cancellationToken).ConfigureAwait(false) + ?? new MsrcSummaryResponse(); + + foreach (var summary in payload.Value) + { + if (string.IsNullOrWhiteSpace(summary.CvrfUrl)) + { + continue; + } + + yield return summary; + fetched++; + + if (fetched >= options.MaxAdvisoriesPerFetch) + { + yield break; + } + } + + if (string.IsNullOrWhiteSpace(payload.NextLink)) + { + break; + } + + if (!Uri.TryCreate(payload.NextLink, UriKind.Absolute, out requestUri)) + { + LogConnectorEvent(LogLevel.Warning, "pagination.invalid", $"MSRC pagination returned invalid next link '{payload.NextLink}'."); + break; + } + } + } + + private static Uri BuildSummaryUri(MsrcConnectorOptions options, DateTimeOffset from, DateTimeOffset to) + { + var baseText = options.BaseUri.ToString().TrimEnd('/'); + var builder = new StringBuilder(baseText.Length + 128); + builder.Append(baseText); + if (!baseText.EndsWith("/vulnerabilities", StringComparison.OrdinalIgnoreCase)) + { + builder.Append("/vulnerabilities"); + } + + builder.Append("?"); + builder.Append("$top=").Append(options.PageSize); + builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(from.ToUniversalTime().ToString("O"))); + builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(to.ToUniversalTime().ToString("O"))); + builder.Append("&$orderby=lastModifiedDate"); + builder.Append("&locale=").Append(Uri.EscapeDataString(options.Locale)); + builder.Append("&api-version=").Append(Uri.EscapeDataString(options.ApiVersion)); + + return new Uri(builder.ToString(), UriKind.Absolute); + } + + private (DateTimeOffset From, DateTimeOffset To) CalculateWindow( + DateTimeOffset? contextSince, + VexConnectorState? state, + MsrcConnectorOptions options) + { + var now = UtcNow(); + var since = contextSince ?? state?.LastUpdated ?? options.InitialLastModified ?? now.AddDays(-30); + + if (state?.LastUpdated is { } persisted && persisted > since) + { + since = persisted; + } + + if (options.CursorOverlap > TimeSpan.Zero) + { + since = since.Add(-options.CursorOverlap); + } + + if (since < now.AddYears(-20)) + { + since = now.AddYears(-20); + } + + return (since, now); + } + + private static bool ShouldRetry(HttpStatusCode statusCode) + => statusCode == HttpStatusCode.TooManyRequests || + (int)statusCode >= 500; + + private static bool IsTransient(Exception exception) + => exception is HttpRequestException or IOException or TaskCanceledException; + + private static Uri ResolveCvrfUri(Uri baseUri, string cvrfUrl) + => Uri.TryCreate(cvrfUrl, UriKind.Absolute, out var absolute) + ? absolute + : new Uri(baseUri, cvrfUrl); + + private static CsafValidationResult ValidateCsafPayload(ReadOnlyMemory payload) + { + try + { + if (IsZip(payload.Span)) + { + using var zipStream = new MemoryStream(payload.ToArray(), writable: false); + using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read, leaveOpen: true); + var entry = archive.Entries.FirstOrDefault(e => e.Name.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + ?? archive.Entries.FirstOrDefault(); + if (entry is null) + { + return new CsafValidationResult("zip", "Zip archive did not contain any entries."); + } + + using var entryStream = entry.Open(); + using var reader = new StreamReader(entryStream, Encoding.UTF8); + using var json = JsonDocument.Parse(reader.ReadToEnd()); + return CsafValidationResult.Valid("zip"); + } + + if (IsGzip(payload.Span)) + { + using var input = new MemoryStream(payload.ToArray(), writable: false); + using var gzip = new GZipStream(input, CompressionMode.Decompress); + using var reader = new StreamReader(gzip, Encoding.UTF8); + using var json = JsonDocument.Parse(reader.ReadToEnd()); + return CsafValidationResult.Valid("gzip"); + } + + using var jsonDocument = JsonDocument.Parse(payload.Span); + return CsafValidationResult.Valid("json"); + } + catch (JsonException ex) + { + return new CsafValidationResult("json", $"JSON parse failed: {ex.Message}"); + } + catch (InvalidDataException ex) + { + return new CsafValidationResult("invalid", ex.Message); + } + catch (EndOfStreamException ex) + { + return new CsafValidationResult("invalid", ex.Message); + } + } + + private static bool IsZip(ReadOnlySpan content) + => content.Length > 3 && content[0] == 0x50 && content[1] == 0x4B; + + private static bool IsGzip(ReadOnlySpan content) + => content.Length > 2 && content[0] == 0x1F && content[1] == 0x8B; + + private static DateTimeOffset? DetermineLatest(MsrcVulnerabilitySummary summary, DateTimeOffset? current) + { + var candidate = summary.LastModifiedDate ?? summary.ReleaseDate; + if (candidate is null) + { + return current; + } + + if (current is null || candidate > current) + { + return candidate; + } + + return current; + } + + private MsrcConnectorOptions EnsureOptionsValidated() + { + if (_validatedOptions is not null) + { + return _validatedOptions; + } + + var options = _options.Value ?? throw new InvalidOperationException("MSRC connector options were not registered."); + options.Validate(); + _validatedOptions = options; + return options; + } + + private sealed record CsafValidationResult(string Format, string? QuarantineReason) + { + public static CsafValidationResult Valid(string format) => new(format, null); + } +} + +internal sealed record MsrcSummaryResponse +{ + [JsonPropertyName("value")] + public List Value { get; init; } = new(); + + [JsonPropertyName("@odata.nextLink")] + public string? NextLink { get; init; } +} + +internal sealed record MsrcVulnerabilitySummary +{ + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("vulnerabilityId")] + public string? VulnerabilityId { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("releaseDate")] + public DateTimeOffset? ReleaseDate { get; init; } + + [JsonPropertyName("lastModifiedDate")] + public DateTimeOffset? LastModifiedDate { get; init; } + + [JsonPropertyName("cvrfUrl")] + public string? CvrfUrl { get; init; } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj similarity index 66% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj rename to src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj index 3c27086b..b3e1398a 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/StellaOps.Excititor.Connectors.MSRC.CSAF.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md new file mode 100644 index 00000000..ac9a9094 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.MSRC.CSAF/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-MS-01-001 – AAD onboarding & token cache|Team Excititor Connectors – MSRC|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.| +|EXCITITOR-CONN-MS-01-002 – CSAF download pipeline|Team Excititor Connectors – MSRC|EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** – Prereqs verified (EXCITITOR-CONN-MS-01-001, EXCITITOR-STORAGE-01-003); drafting fetch/retry plan and storage wiring before implementation of CSAF package download, checksum validation, and quarantine flows.| +|EXCITITOR-CONN-MS-01-003 – Trust metadata & provenance hints|Team Excititor Connectors – MSRC|EXCITITOR-CONN-MS-01-002, EXCITITOR-POLICY-01-001|TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration.| diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Configuration/OciOpenVexAttestationConnectorOptionsValidatorTests.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Configuration/OciOpenVexAttestationConnectorOptionsValidatorTests.cs new file mode 100644 index 00000000..b50d6009 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Configuration/OciOpenVexAttestationConnectorOptionsValidatorTests.cs @@ -0,0 +1,76 @@ +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using StellaOps.Excititor.Core; +using System.Collections.Generic; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.Configuration; + +public sealed class OciOpenVexAttestationConnectorOptionsValidatorTests +{ + [Fact] + public void Validate_WithValidConfiguration_Succeeds() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/offline/registry.example.com/repo/latest/openvex-attestations.tgz"] = new MockFileData(string.Empty), + }); + + var validator = new OciOpenVexAttestationConnectorOptionsValidator(fileSystem); + var options = new OciOpenVexAttestationConnectorOptions + { + AllowHttpRegistries = true, + }; + + options.Images.Add(new OciImageSubscriptionOptions + { + Reference = "registry.example.com/repo/image:latest", + OfflineBundlePath = "/offline/registry.example.com/repo/latest/openvex-attestations.tgz", + }); + + options.Registry.Username = "user"; + options.Registry.Password = "pass"; + + options.Cosign.Mode = CosignCredentialMode.None; + + var errors = new List(); + + validator.Validate(new VexConnectorDescriptor("id", VexProviderKind.Attestation, "display"), options, errors); + + errors.Should().BeEmpty(); + } + + [Fact] + public void Validate_WhenImagesMissing_AddsError() + { + var validator = new OciOpenVexAttestationConnectorOptionsValidator(new MockFileSystem()); + var options = new OciOpenVexAttestationConnectorOptions(); + + var errors = new List(); + + validator.Validate(new VexConnectorDescriptor("id", VexProviderKind.Attestation, "display"), options, errors); + + errors.Should().ContainSingle().Which.Should().Contain("At least one OCI image reference must be configured."); + } + + [Fact] + public void Validate_WhenDigestMalformed_AddsError() + { + var validator = new OciOpenVexAttestationConnectorOptionsValidator(new MockFileSystem()); + var options = new OciOpenVexAttestationConnectorOptions(); + options.Images.Add(new OciImageSubscriptionOptions + { + Reference = "registry.test/repo/image@sha256:not-a-digest", + }); + + var errors = new List(); + + validator.Validate(new VexConnectorDescriptor("id", VexProviderKind.Attestation, "display"), options, errors); + + errors.Should().ContainSingle(); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs new file mode 100644 index 00000000..134c0fc8 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Connector/OciOpenVexAttestationConnectorTests.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; +using StellaOps.Excititor.Core; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.Connector; + +public sealed class OciOpenVexAttestationConnectorTests +{ + [Fact] + public async Task FetchAsync_WithOfflineBundle_EmitsRawDocument() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var httpClient = new HttpClient(new StubHttpMessageHandler()) + { + BaseAddress = new System.Uri("https://registry.example.com/") + }; + + var httpFactory = new SingleClientHttpClientFactory(httpClient); + var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); + + var connector = new OciOpenVexAttestationConnector( + discovery, + fetcher, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add("Images:0:Reference", "registry.example.com/repo/image:latest") + .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") + .Add("Offline:PreferOffline", "true") + .Add("Offline:AllowNetworkFallback", "false") + .Add("Cosign:Mode", "None"); + + var settings = new VexConnectorSettings(settingsValues); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new CapturingRawSink(); + var verifier = new CapturingSignatureVerifier(); + var context = new VexConnectorContext( + Since: null, + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: verifier, + Normalizers: new NoopNormalizerRouter(), + Services: new Microsoft.Extensions.DependencyInjection.ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + documents[0].Format.Should().Be(VexDocumentFormat.OciAttestation); + documents[0].Metadata.Should().ContainKey("oci.attestation.sourceKind").WhoseValue.Should().Be("offline"); + documents[0].Metadata.Should().ContainKey("vex.provenance.sourceKind").WhoseValue.Should().Be("offline"); + documents[0].Metadata.Should().ContainKey("vex.provenance.registryAuthMode").WhoseValue.Should().Be("Anonymous"); + verifier.VerifyCalls.Should().Be(1); + } + + [Fact] + public async Task FetchAsync_WithSignatureMetadata_EnrichesProvenance() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/attestation.json"] = new MockFileData("{\"payload\":\"\",\"payloadType\":\"application/vnd.in-toto+json\",\"signatures\":[{\"sig\":\"\"}]}"), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var httpClient = new HttpClient(new StubHttpMessageHandler()) + { + BaseAddress = new System.Uri("https://registry.example.com/") + }; + + var httpFactory = new SingleClientHttpClientFactory(httpClient); + var discovery = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + var fetcher = new OciAttestationFetcher(httpFactory, fileSystem, NullLogger.Instance); + + var connector = new OciOpenVexAttestationConnector( + discovery, + fetcher, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add("Images:0:Reference", "registry.example.com/repo/image:latest") + .Add("Images:0:OfflineBundlePath", "/bundles/attestation.json") + .Add("Offline:PreferOffline", "true") + .Add("Offline:AllowNetworkFallback", "false") + .Add("Cosign:Mode", "Keyless") + .Add("Cosign:Keyless:Issuer", "https://issuer.example.com") + .Add("Cosign:Keyless:Subject", "subject@example.com"); + + var settings = new VexConnectorSettings(settingsValues); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new CapturingRawSink(); + var verifier = new CapturingSignatureVerifier + { + Result = new VexSignatureMetadata( + type: "cosign", + subject: "sig-subject", + issuer: "sig-issuer", + keyId: "key-id", + verifiedAt: DateTimeOffset.UtcNow, + transparencyLogReference: "rekor://entry/123") + }; + + var context = new VexConnectorContext( + Since: null, + Settings: VexConnectorSettings.Empty, + RawSink: sink, + SignatureVerifier: verifier, + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + var metadata = documents[0].Metadata; + metadata.Should().Contain("vex.signature.type", "cosign"); + metadata.Should().Contain("vex.signature.subject", "sig-subject"); + metadata.Should().Contain("vex.signature.issuer", "sig-issuer"); + metadata.Should().Contain("vex.signature.keyId", "key-id"); + metadata.Should().ContainKey("vex.signature.verifiedAt"); + metadata.Should().Contain("vex.signature.transparencyLogReference", "rekor://entry/123"); + metadata.Should().Contain("vex.provenance.cosign.mode", "Keyless"); + metadata.Should().Contain("vex.provenance.cosign.issuer", "https://issuer.example.com"); + metadata.Should().Contain("vex.provenance.cosign.subject", "subject@example.com"); + verifier.VerifyCalls.Should().Be(1); + } + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class CapturingSignatureVerifier : IVexSignatureVerifier + { + public int VerifyCalls { get; private set; } + + public VexSignatureMetadata? Result { get; set; } + + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + { + VerifyCalls++; + return ValueTask.FromResult(Result); + } + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + RequestMessage = request + }); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Discovery/OciAttestationDiscoveryServiceTests.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Discovery/OciAttestationDiscoveryServiceTests.cs new file mode 100644 index 00000000..a18668fc --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/Discovery/OciAttestationDiscoveryServiceTests.cs @@ -0,0 +1,83 @@ +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using System.Collections.Generic; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.Discovery; + +public sealed class OciAttestationDiscoveryServiceTests +{ + [Fact] + public async Task LoadAsync_ResolvesOfflinePaths() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/registry.example.com/repo/image/latest/openvex-attestations.tgz"] = new MockFileData(string.Empty), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var service = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + + var options = new OciOpenVexAttestationConnectorOptions + { + AllowHttpRegistries = true, + }; + + options.Images.Add(new OciImageSubscriptionOptions + { + Reference = "registry.example.com/repo/image:latest", + }); + + options.Offline.RootDirectory = "/bundles"; + options.Cosign.Mode = CosignCredentialMode.None; + + var result = await service.LoadAsync(options, CancellationToken.None); + + result.Targets.Should().ContainSingle(); + result.Targets[0].OfflineBundle.Should().NotBeNull(); + var offline = result.Targets[0].OfflineBundle!; + offline.Exists.Should().BeTrue(); + var expectedPath = fileSystem.Path.Combine( + fileSystem.Path.GetFullPath("/bundles"), + "registry.example.com", + "repo", + "image", + "latest", + "openvex-attestations.tgz"); + offline.Path.Should().Be(expectedPath); + } + + [Fact] + public async Task LoadAsync_CachesResults() + { + var fileSystem = new MockFileSystem(new Dictionary + { + ["/bundles/registry.example.com/repo/image/latest/openvex-attestations.tgz"] = new MockFileData(string.Empty), + }); + + using var cache = new MemoryCache(new MemoryCacheOptions()); + var service = new OciAttestationDiscoveryService(cache, fileSystem, NullLogger.Instance); + + var options = new OciOpenVexAttestationConnectorOptions + { + AllowHttpRegistries = true, + }; + + options.Images.Add(new OciImageSubscriptionOptions + { + Reference = "registry.example.com/repo/image:latest", + }); + + options.Offline.RootDirectory = "/bundles"; + options.Cosign.Mode = CosignCredentialMode.None; + + var first = await service.LoadAsync(options, CancellationToken.None); + var second = await service.LoadAsync(options, CancellationToken.None); + + ReferenceEquals(first, second).Should().BeTrue(); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj new file mode 100644 index 00000000..9955c490 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + NU1903 + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/AGENTS.md b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/AGENTS.md similarity index 93% rename from src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/AGENTS.md rename to src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/AGENTS.md index f1e74ebb..19e43061 100644 --- a/src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector for OCI registry OpenVEX attestations, discovering images, downloading attestations, and projecting statements into raw storage. -## Scope -- OCI registry discovery, authentication (cosign OIDC/key), and ref resolution for provided image digests/tags. -- Fetching DSSE envelopes, verifying signatures (delegated to Attestation module), and persisting raw statements. -- Mapping OCI manifest metadata (repository, digest, subject) to connector provenance. -- Managing offline bundles that seed attestations without registry access. -## Participants -- Worker schedules polls for configured registries/images; WebService supports manual refresh. -- OpenVEX normalizer consumes statements to create claims. -- Attestation module is reused to verify upstream envelopes prior to storage. -## Interfaces & contracts -- Implements `IVexConnector` with options for image list, auth, parallelism, and offline file seeds. -- Utilizes shared abstractions for retries, telemetry, and resume markers. -## In/Out of scope -In: OCI interaction, attestation retrieval, verification trigger, raw persistence. -Out: normalization/export, policy evaluation, storage implementation. -## Observability & security expectations -- Log image references, attestation counts, verification outcomes; redact credentials. -- Emit metrics for attestation reuse ratio, verification duration, and failures. -## Tests -- Connector tests with mock OCI registry/attestation responses will live in `../StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest.Tests`. +# AGENTS +## Role +Connector for OCI registry OpenVEX attestations, discovering images, downloading attestations, and projecting statements into raw storage. +## Scope +- OCI registry discovery, authentication (cosign OIDC/key), and ref resolution for provided image digests/tags. +- Fetching DSSE envelopes, verifying signatures (delegated to Attestation module), and persisting raw statements. +- Mapping OCI manifest metadata (repository, digest, subject) to connector provenance. +- Managing offline bundles that seed attestations without registry access. +## Participants +- Worker schedules polls for configured registries/images; WebService supports manual refresh. +- OpenVEX normalizer consumes statements to create claims. +- Attestation module is reused to verify upstream envelopes prior to storage. +## Interfaces & contracts +- Implements `IVexConnector` with options for image list, auth, parallelism, and offline file seeds. +- Utilizes shared abstractions for retries, telemetry, and resume markers. +## In/Out of scope +In: OCI interaction, attestation retrieval, verification trigger, raw persistence. +Out: normalization/export, policy evaluation, storage implementation. +## Observability & security expectations +- Log image references, attestation counts, verification outcomes; redact credentials. +- Emit metrics for attestation reuse ratio, verification duration, and failures. +## Tests +- Connector tests with mock OCI registry/attestation responses will live in `../StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Tests`. diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciCosignAuthority.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciCosignAuthority.cs new file mode 100644 index 00000000..a5a14372 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciCosignAuthority.cs @@ -0,0 +1,110 @@ +using System; +using System.IO.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; + +public sealed record CosignKeylessIdentity( + string Issuer, + string Subject, + Uri? FulcioUrl, + Uri? RekorUrl, + string? ClientId, + string? ClientSecret, + string? Audience, + string? IdentityToken); + +public sealed record CosignKeyPairIdentity( + string PrivateKeyPath, + string? Password, + string? CertificatePath, + Uri? RekorUrl, + string? FulcioRootPath); + +public sealed record OciCosignAuthority( + CosignCredentialMode Mode, + CosignKeylessIdentity? Keyless, + CosignKeyPairIdentity? KeyPair, + bool RequireSignature, + TimeSpan VerifyTimeout); + +public static class OciCosignAuthorityFactory +{ + public static OciCosignAuthority Create(OciCosignVerificationOptions options, IFileSystem? fileSystem = null) + { + ArgumentNullException.ThrowIfNull(options); + + CosignKeylessIdentity? keyless = null; + CosignKeyPairIdentity? keyPair = null; + + switch (options.Mode) + { + case CosignCredentialMode.None: + break; + + case CosignCredentialMode.Keyless: + keyless = CreateKeyless(options.Keyless); + break; + + case CosignCredentialMode.KeyPair: + keyPair = CreateKeyPair(options.KeyPair, fileSystem); + break; + + default: + throw new InvalidOperationException($"Unsupported Cosign credential mode '{options.Mode}'."); + } + + return new OciCosignAuthority( + Mode: options.Mode, + Keyless: keyless, + KeyPair: keyPair, + RequireSignature: options.RequireSignature, + VerifyTimeout: options.VerifyTimeout); + } + + private static CosignKeylessIdentity CreateKeyless(CosignKeylessOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + Uri? fulcio = null; + Uri? rekor = null; + + if (!string.IsNullOrWhiteSpace(options.FulcioUrl)) + { + fulcio = new Uri(options.FulcioUrl, UriKind.Absolute); + } + + if (!string.IsNullOrWhiteSpace(options.RekorUrl)) + { + rekor = new Uri(options.RekorUrl, UriKind.Absolute); + } + + return new CosignKeylessIdentity( + Issuer: options.Issuer!, + Subject: options.Subject!, + FulcioUrl: fulcio, + RekorUrl: rekor, + ClientId: options.ClientId, + ClientSecret: options.ClientSecret, + Audience: options.Audience, + IdentityToken: options.IdentityToken); + } + + private static CosignKeyPairIdentity CreateKeyPair(CosignKeyPairOptions options, IFileSystem? fileSystem) + { + ArgumentNullException.ThrowIfNull(options); + + Uri? rekor = null; + if (!string.IsNullOrWhiteSpace(options.RekorUrl)) + { + rekor = new Uri(options.RekorUrl, UriKind.Absolute); + } + + return new CosignKeyPairIdentity( + PrivateKeyPath: options.PrivateKeyPath!, + Password: options.Password, + CertificatePath: options.CertificatePath, + RekorUrl: rekor, + FulcioRootPath: options.FulcioRootPath); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciRegistryAuthorization.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciRegistryAuthorization.cs new file mode 100644 index 00000000..abddfaef --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Authentication/OciRegistryAuthorization.cs @@ -0,0 +1,59 @@ +using System; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; + +public enum OciRegistryAuthMode +{ + Anonymous = 0, + Basic = 1, + IdentityToken = 2, + RefreshToken = 3, +} + +public sealed record OciRegistryAuthorization( + string? RegistryAuthority, + OciRegistryAuthMode Mode, + string? Username, + string? Password, + string? IdentityToken, + string? RefreshToken, + bool AllowAnonymousFallback) +{ + public static OciRegistryAuthorization Create(OciRegistryAuthenticationOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var mode = OciRegistryAuthMode.Anonymous; + string? username = null; + string? password = null; + string? identityToken = null; + string? refreshToken = null; + + if (!string.IsNullOrWhiteSpace(options.IdentityToken)) + { + mode = OciRegistryAuthMode.IdentityToken; + identityToken = options.IdentityToken; + } + else if (!string.IsNullOrWhiteSpace(options.RefreshToken)) + { + mode = OciRegistryAuthMode.RefreshToken; + refreshToken = options.RefreshToken; + } + else if (!string.IsNullOrWhiteSpace(options.Username)) + { + mode = OciRegistryAuthMode.Basic; + username = options.Username; + password = options.Password; + } + + return new OciRegistryAuthorization( + RegistryAuthority: options.RegistryAuthority, + Mode: mode, + Username: username, + Password: password, + IdentityToken: identityToken, + RefreshToken: refreshToken, + AllowAnonymousFallback: options.AllowAnonymousFallback); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptions.cs new file mode 100644 index 00000000..edcd1fc8 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptions.cs @@ -0,0 +1,321 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; + +public sealed class OciOpenVexAttestationConnectorOptions +{ + public const string HttpClientName = "excititor.connector.oci.openvex.attest"; + + public IList Images { get; } = new List(); + + public OciRegistryAuthenticationOptions Registry { get; } = new(); + + public OciCosignVerificationOptions Cosign { get; } = new(); + + public OciOfflineBundleOptions Offline { get; } = new(); + + public TimeSpan DiscoveryCacheDuration { get; set; } = TimeSpan.FromMinutes(15); + + public int MaxParallelResolutions { get; set; } = 4; + + public bool AllowHttpRegistries { get; set; } + + public void Validate(IFileSystem? fileSystem = null) + { + if (Images.Count == 0) + { + throw new InvalidOperationException("At least one OCI image reference must be configured."); + } + + foreach (var image in Images) + { + image.Validate(); + } + + if (MaxParallelResolutions <= 0 || MaxParallelResolutions > 32) + { + throw new InvalidOperationException("MaxParallelResolutions must be between 1 and 32."); + } + + if (DiscoveryCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("DiscoveryCacheDuration must be a positive time span."); + } + + Registry.Validate(); + Cosign.Validate(fileSystem); + Offline.Validate(fileSystem); + + if (!AllowHttpRegistries && Images.Any(i => i.Reference is not null && i.Reference.StartsWith("http://", StringComparison.OrdinalIgnoreCase))) + { + throw new InvalidOperationException("HTTP (non-TLS) registries are disabled. Enable AllowHttpRegistries to permit them."); + } + } +} + +public sealed class OciImageSubscriptionOptions +{ + private OciImageReference? _parsedReference; + + /// + /// Gets or sets the OCI reference (e.g. registry.example.com/repository:tag or registry.example.com/repository@sha256:abcdef). + /// + public string? Reference { get; set; } + + /// + /// Optional friendly name used in logs when referencing this subscription. + /// + public string? DisplayName { get; set; } + + /// + /// Optional file path for an offline attestation bundle associated with this image. + /// + public string? OfflineBundlePath { get; set; } + + /// + /// Optional override for the expected subject digest. When provided, discovery will verify resolved digests match. + /// + public string? ExpectedSubjectDigest { get; set; } + + internal OciImageReference? ParsedReference => _parsedReference; + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Reference)) + { + throw new InvalidOperationException("Image Reference is required for OCI OpenVEX attestation connector."); + } + + _parsedReference = OciImageReferenceParser.Parse(Reference); + + if (!string.IsNullOrWhiteSpace(ExpectedSubjectDigest)) + { + if (!ExpectedSubjectDigest.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("ExpectedSubjectDigest must start with 'sha256:'."); + } + + if (ExpectedSubjectDigest.Length != "sha256:".Length + 64) + { + throw new InvalidOperationException("ExpectedSubjectDigest must contain a 64-character hexadecimal hash."); + } + } + } +} + +public sealed class OciRegistryAuthenticationOptions +{ + /// + /// Optional registry authority filter (e.g. registry.example.com:5000). When set it must match image references. + /// + public string? RegistryAuthority { get; set; } + + public string? Username { get; set; } + + public string? Password { get; set; } + + public string? IdentityToken { get; set; } + + public string? RefreshToken { get; set; } + + public bool AllowAnonymousFallback { get; set; } = true; + + public void Validate() + { + var hasUser = !string.IsNullOrWhiteSpace(Username); + var hasPassword = !string.IsNullOrWhiteSpace(Password); + var hasIdentityToken = !string.IsNullOrWhiteSpace(IdentityToken); + var hasRefreshToken = !string.IsNullOrWhiteSpace(RefreshToken); + + if (hasIdentityToken && (hasUser || hasPassword)) + { + throw new InvalidOperationException("IdentityToken cannot be combined with Username/Password for OCI registry authentication."); + } + + if (hasRefreshToken && (hasUser || hasPassword)) + { + throw new InvalidOperationException("RefreshToken cannot be combined with Username/Password for OCI registry authentication."); + } + + if (hasUser != hasPassword) + { + throw new InvalidOperationException("Username and Password must be provided together for OCI registry authentication."); + } + + if (!string.IsNullOrWhiteSpace(RegistryAuthority) && RegistryAuthority.Contains('/', StringComparison.Ordinal)) + { + throw new InvalidOperationException("RegistryAuthority must not contain path segments."); + } + } +} + +public sealed class OciCosignVerificationOptions +{ + public CosignCredentialMode Mode { get; set; } = CosignCredentialMode.Keyless; + + public CosignKeylessOptions Keyless { get; } = new(); + + public CosignKeyPairOptions KeyPair { get; } = new(); + + public bool RequireSignature { get; set; } = true; + + public TimeSpan VerifyTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public void Validate(IFileSystem? fileSystem = null) + { + if (VerifyTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("VerifyTimeout must be a positive time span."); + } + + switch (Mode) + { + case CosignCredentialMode.None: + break; + + case CosignCredentialMode.Keyless: + Keyless.Validate(); + break; + + case CosignCredentialMode.KeyPair: + KeyPair.Validate(fileSystem); + break; + + default: + throw new InvalidOperationException($"Unsupported Cosign credential mode '{Mode}'."); + } + } +} + +public enum CosignCredentialMode +{ + None = 0, + Keyless = 1, + KeyPair = 2, +} + +public sealed class CosignKeylessOptions +{ + public string? Issuer { get; set; } + + public string? Subject { get; set; } + + public string? FulcioUrl { get; set; } = "https://fulcio.sigstore.dev"; + + public string? RekorUrl { get; set; } = "https://rekor.sigstore.dev"; + + public string? ClientId { get; set; } + + public string? ClientSecret { get; set; } + + public string? Audience { get; set; } + + public string? IdentityToken { get; set; } + + public void Validate() + { + if (string.IsNullOrWhiteSpace(Issuer)) + { + throw new InvalidOperationException("Cosign keyless Issuer must be provided."); + } + + if (string.IsNullOrWhiteSpace(Subject)) + { + throw new InvalidOperationException("Cosign keyless Subject must be provided."); + } + + if (!string.IsNullOrWhiteSpace(FulcioUrl) && !Uri.TryCreate(FulcioUrl, UriKind.Absolute, out var fulcio)) + { + throw new InvalidOperationException("FulcioUrl must be an absolute URI when provided."); + } + + if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out var rekor)) + { + throw new InvalidOperationException("RekorUrl must be an absolute URI when provided."); + } + + if (!string.IsNullOrWhiteSpace(ClientSecret) && string.IsNullOrWhiteSpace(ClientId)) + { + throw new InvalidOperationException("Cosign keyless ClientId must be provided when ClientSecret is specified."); + } + } +} + +public sealed class CosignKeyPairOptions +{ + public string? PrivateKeyPath { get; set; } + + public string? Password { get; set; } + + public string? CertificatePath { get; set; } + + public string? RekorUrl { get; set; } + + public string? FulcioRootPath { get; set; } + + public void Validate(IFileSystem? fileSystem = null) + { + if (string.IsNullOrWhiteSpace(PrivateKeyPath)) + { + throw new InvalidOperationException("PrivateKeyPath must be provided for Cosign key pair mode."); + } + + var fs = fileSystem ?? new FileSystem(); + if (!fs.File.Exists(PrivateKeyPath)) + { + throw new InvalidOperationException($"Cosign private key file not found: {PrivateKeyPath}"); + } + + if (!string.IsNullOrWhiteSpace(CertificatePath) && !fs.File.Exists(CertificatePath)) + { + throw new InvalidOperationException($"Cosign certificate file not found: {CertificatePath}"); + } + + if (!string.IsNullOrWhiteSpace(FulcioRootPath) && !fs.File.Exists(FulcioRootPath)) + { + throw new InvalidOperationException($"Cosign Fulcio root file not found: {FulcioRootPath}"); + } + + if (!string.IsNullOrWhiteSpace(RekorUrl) && !Uri.TryCreate(RekorUrl, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("RekorUrl must be an absolute URI when provided for Cosign key pair mode."); + } + } +} + +public sealed class OciOfflineBundleOptions +{ + public string? RootDirectory { get; set; } + + public bool PreferOffline { get; set; } + + public bool AllowNetworkFallback { get; set; } = true; + + public string? DefaultBundleFileName { get; set; } = "openvex-attestations.tgz"; + + public bool RequireBundles { get; set; } + + public void Validate(IFileSystem? fileSystem = null) + { + if (string.IsNullOrWhiteSpace(RootDirectory)) + { + return; + } + + var fs = fileSystem ?? new FileSystem(); + if (!fs.Directory.Exists(RootDirectory)) + { + if (PreferOffline || RequireBundles) + { + throw new InvalidOperationException($"Offline bundle root directory '{RootDirectory}' does not exist."); + } + + fs.Directory.CreateDirectory(RootDirectory); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptionsValidator.cs new file mode 100644 index 00000000..146fbf2a --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Configuration/OciOpenVexAttestationConnectorOptionsValidator.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; + +public sealed class OciOpenVexAttestationConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public OciOpenVexAttestationConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate( + VexConnectorDescriptor descriptor, + OciOpenVexAttestationConnectorOptions options, + IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/DependencyInjection/OciOpenVexAttestationConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/DependencyInjection/OciOpenVexAttestationConnectorServiceCollectionExtensions.cs new file mode 100644 index 00000000..926a11f4 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/DependencyInjection/OciOpenVexAttestationConnectorServiceCollectionExtensions.cs @@ -0,0 +1,52 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.DependencyInjection; + +public static class OciOpenVexAttestationConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddOciOpenVexAttestationConnector( + this IServiceCollection services, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }); + + services.AddSingleton, OciOpenVexAttestationConnectorOptionsValidator>(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddHttpClient(OciOpenVexAttestationConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.cncf.openvex.v1+json"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + return services; + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryResult.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryResult.cs new file mode 100644 index 00000000..59cfc9f8 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryResult.cs @@ -0,0 +1,11 @@ +using System.Collections.Immutable; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +public sealed record OciAttestationDiscoveryResult( + ImmutableArray Targets, + OciRegistryAuthorization RegistryAuthorization, + OciCosignAuthority CosignAuthority, + bool PreferOffline, + bool AllowNetworkFallback); diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryService.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryService.cs new file mode 100644 index 00000000..ad35cf9c --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationDiscoveryService.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +public sealed class OciAttestationDiscoveryService +{ + private readonly IMemoryCache _memoryCache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + + public OciAttestationDiscoveryService( + IMemoryCache memoryCache, + IFileSystem fileSystem, + ILogger logger) + { + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task LoadAsync( + OciOpenVexAttestationConnectorOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + cancellationToken.ThrowIfCancellationRequested(); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out OciAttestationDiscoveryResult? cached) && cached is not null) + { + _logger.LogDebug("Using cached OCI attestation discovery result for {ImageCount} images.", cached.Targets.Length); + return Task.FromResult(cached); + } + + var targets = new List(options.Images.Count); + foreach (var image in options.Images) + { + cancellationToken.ThrowIfCancellationRequested(); + + var parsed = image.ParsedReference ?? OciImageReferenceParser.Parse(image.Reference!); + var offlinePath = ResolveOfflinePath(options, image, parsed); + + OciOfflineBundleReference? offline = null; + if (!string.IsNullOrWhiteSpace(offlinePath)) + { + var fullPath = _fileSystem.Path.GetFullPath(offlinePath!); + var exists = _fileSystem.File.Exists(fullPath) || _fileSystem.Directory.Exists(fullPath); + + if (!exists && options.Offline.RequireBundles) + { + throw new InvalidOperationException($"Required offline bundle '{fullPath}' for reference '{parsed.Canonical}' was not found."); + } + + offline = new OciOfflineBundleReference(fullPath, exists, image.ExpectedSubjectDigest); + } + + targets.Add(new OciAttestationTarget(parsed, image.ExpectedSubjectDigest, offline)); + } + + var authorization = OciRegistryAuthorization.Create(options.Registry); + var cosignAuthority = OciCosignAuthorityFactory.Create(options.Cosign, _fileSystem); + + var result = new OciAttestationDiscoveryResult( + targets.ToImmutableArray(), + authorization, + cosignAuthority, + options.Offline.PreferOffline, + options.Offline.AllowNetworkFallback); + + _memoryCache.Set(cacheKey, result, options.DiscoveryCacheDuration); + + return Task.FromResult(result); + } + + private string? ResolveOfflinePath( + OciOpenVexAttestationConnectorOptions options, + OciImageSubscriptionOptions image, + OciImageReference parsed) + { + if (!string.IsNullOrWhiteSpace(image.OfflineBundlePath)) + { + return image.OfflineBundlePath; + } + + if (string.IsNullOrWhiteSpace(options.Offline.RootDirectory)) + { + return null; + } + + var root = options.Offline.RootDirectory!; + var segments = new List { SanitizeSegment(parsed.Registry) }; + + var repositoryParts = parsed.Repository.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (repositoryParts.Length == 0) + { + segments.Add(SanitizeSegment(parsed.Repository)); + } + else + { + foreach (var part in repositoryParts) + { + segments.Add(SanitizeSegment(part)); + } + } + + var versionSegment = parsed.Digest is not null + ? SanitizeSegment(parsed.Digest) + : SanitizeSegment(parsed.Tag ?? "latest"); + + segments.Add(versionSegment); + + var combined = _fileSystem.Path.Combine(new[] { root }.Concat(segments).ToArray()); + + if (!string.IsNullOrWhiteSpace(options.Offline.DefaultBundleFileName)) + { + combined = _fileSystem.Path.Combine(combined, options.Offline.DefaultBundleFileName!); + } + + return combined; + } + + private static string SanitizeSegment(string value) + { + if (string.IsNullOrEmpty(value)) + { + return "_"; + } + + var builder = new StringBuilder(value.Length); + foreach (var ch in value) + { + if (char.IsLetterOrDigit(ch) || ch is '-' or '_' or '.') + { + builder.Append(ch); + } + else + { + builder.Append('_'); + } + } + + return builder.Length == 0 ? "_" : builder.ToString(); + } + + private static string CreateCacheKey(OciOpenVexAttestationConnectorOptions options) + { + using var sha = SHA256.Create(); + var builder = new StringBuilder(); + builder.AppendLine("oci-openvex-attest"); + builder.AppendLine(options.MaxParallelResolutions.ToString()); + builder.AppendLine(options.AllowHttpRegistries.ToString()); + builder.AppendLine(options.Offline.PreferOffline.ToString()); + builder.AppendLine(options.Offline.AllowNetworkFallback.ToString()); + + foreach (var image in options.Images) + { + builder.AppendLine(image.Reference ?? string.Empty); + builder.AppendLine(image.ExpectedSubjectDigest ?? string.Empty); + builder.AppendLine(image.OfflineBundlePath ?? string.Empty); + } + + if (!string.IsNullOrWhiteSpace(options.Offline.RootDirectory)) + { + builder.AppendLine(options.Offline.RootDirectory); + builder.AppendLine(options.Offline.DefaultBundleFileName ?? string.Empty); + } + + builder.AppendLine(options.Registry.RegistryAuthority ?? string.Empty); + builder.AppendLine(options.Registry.AllowAnonymousFallback.ToString()); + + var bytes = Encoding.UTF8.GetBytes(builder.ToString()); + var hashBytes = sha.ComputeHash(bytes); + return Convert.ToHexString(hashBytes); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationTarget.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationTarget.cs new file mode 100644 index 00000000..eec49fce --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciAttestationTarget.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +public sealed record OciAttestationTarget( + OciImageReference Image, + string? ExpectedSubjectDigest, + OciOfflineBundleReference? OfflineBundle); diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReference.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReference.cs new file mode 100644 index 00000000..ce333b45 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReference.cs @@ -0,0 +1,27 @@ +using System; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +public sealed record OciImageReference(string Registry, string Repository, string? Tag, string? Digest, string OriginalReference, string Scheme = "https") +{ + public string Canonical => + Digest is not null + ? $"{Registry}/{Repository}@{Digest}" + : Tag is not null + ? $"{Registry}/{Repository}:{Tag}" + : $"{Registry}/{Repository}"; + + public bool HasDigest => !string.IsNullOrWhiteSpace(Digest); + + public bool HasTag => !string.IsNullOrWhiteSpace(Tag); + + public OciImageReference WithDigest(string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + throw new ArgumentException("Digest must be provided.", nameof(digest)); + } + + return this with { Digest = digest }; + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReferenceParser.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReferenceParser.cs new file mode 100644 index 00000000..8d02bb9c --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciImageReferenceParser.cs @@ -0,0 +1,129 @@ +using System; +using System.Text.RegularExpressions; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +internal static class OciImageReferenceParser +{ + private static readonly Regex DigestRegex = new(@"^(?[A-Za-z0-9+._-]+):(?[A-Fa-f0-9]{32,})$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex RepositoryRegex = new(@"^[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|[-]*)[a-z0-9]+)*)*$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + public static OciImageReference Parse(string reference) + { + if (string.IsNullOrWhiteSpace(reference)) + { + throw new InvalidOperationException("OCI reference cannot be empty."); + } + + var trimmed = reference.Trim(); + string original = trimmed; + + var scheme = "https"; + if (trimmed.StartsWith("oci://", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed.Substring("oci://".Length); + } + + if (trimmed.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed.Substring("https://".Length); + scheme = "https"; + } + else if (trimmed.StartsWith("http://", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed.Substring("http://".Length); + scheme = "http"; + } + + var firstSlash = trimmed.IndexOf('/'); + if (firstSlash <= 0) + { + throw new InvalidOperationException($"OCI reference '{reference}' must include a registry and repository component."); + } + + var registry = trimmed[..firstSlash]; + var remainder = trimmed[(firstSlash + 1)..]; + + if (!LooksLikeRegistry(registry)) + { + throw new InvalidOperationException($"OCI reference '{reference}' is missing an explicit registry component."); + } + + string? digest = null; + string? tag = null; + + var digestIndex = remainder.IndexOf('@'); + if (digestIndex >= 0) + { + digest = remainder[(digestIndex + 1)..]; + remainder = remainder[..digestIndex]; + + if (!DigestRegex.IsMatch(digest)) + { + throw new InvalidOperationException($"Digest segment '{digest}' is not a valid OCI digest."); + } + } + + var tagIndex = remainder.LastIndexOf(':'); + if (tagIndex >= 0) + { + tag = remainder[(tagIndex + 1)..]; + remainder = remainder[..tagIndex]; + + if (string.IsNullOrWhiteSpace(tag)) + { + throw new InvalidOperationException("OCI tag segment cannot be empty."); + } + + if (tag.Contains('/', StringComparison.Ordinal)) + { + throw new InvalidOperationException("OCI tag segment cannot contain '/'."); + } + } + + var repository = remainder; + if (string.IsNullOrWhiteSpace(repository)) + { + throw new InvalidOperationException("OCI repository segment cannot be empty."); + } + + if (!RepositoryRegex.IsMatch(repository)) + { + throw new InvalidOperationException($"Repository segment '{repository}' is not valid per OCI distribution rules."); + } + + return new OciImageReference( + Registry: registry, + Repository: repository, + Tag: tag, + Digest: digest, + OriginalReference: original, + Scheme: scheme); + } + + private static bool LooksLikeRegistry(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (value.Equals("localhost", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (value.Contains('.', StringComparison.Ordinal) || value.Contains(':', StringComparison.Ordinal)) + { + return true; + } + + // IPv4/IPv6 simplified check + if (value.Length >= 3 && char.IsDigit(value[0])) + { + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciOfflineBundleReference.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciOfflineBundleReference.cs new file mode 100644 index 00000000..412f5399 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Discovery/OciOfflineBundleReference.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +public sealed record OciOfflineBundleReference(string Path, bool Exists, string? ExpectedSubjectDigest); diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciArtifactDescriptor.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciArtifactDescriptor.cs new file mode 100644 index 00000000..7c0abe7e --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciArtifactDescriptor.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; + +internal sealed record OciArtifactDescriptor( + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("artifactType")] string? ArtifactType, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("annotations")] IReadOnlyDictionary? Annotations); + +internal sealed record OciReferrerIndex( + [property: JsonPropertyName("referrers")] IReadOnlyList Referrers); diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationDocument.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationDocument.cs new file mode 100644 index 00000000..558fbf3f --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationDocument.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; + +public sealed record OciAttestationDocument( + Uri SourceUri, + ReadOnlyMemory Content, + ImmutableDictionary Metadata, + string? SubjectDigest, + string? ArtifactDigest, + string? ArtifactType, + string SourceKind); diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationFetcher.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationFetcher.cs new file mode 100644 index 00000000..0078204c --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciAttestationFetcher.cs @@ -0,0 +1,258 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.IO.Abstractions; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Net.Http; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using System.Formats.Tar; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; + +public sealed class OciAttestationFetcher +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + + public OciAttestationFetcher( + IHttpClientFactory httpClientFactory, + IFileSystem fileSystem, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async IAsyncEnumerable FetchAsync( + OciAttestationDiscoveryResult discovery, + OciOpenVexAttestationConnectorOptions options, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(discovery); + ArgumentNullException.ThrowIfNull(options); + + foreach (var target in discovery.Targets) + { + cancellationToken.ThrowIfCancellationRequested(); + + bool yieldedOffline = false; + if (target.OfflineBundle is not null && target.OfflineBundle.Exists) + { + await foreach (var offlineDocument in ReadOfflineAsync(target, cancellationToken)) + { + yieldedOffline = true; + yield return offlineDocument; + } + + if (!discovery.AllowNetworkFallback) + { + continue; + } + } + + if (discovery.PreferOffline && yieldedOffline && !discovery.AllowNetworkFallback) + { + continue; + } + + if (!discovery.PreferOffline || discovery.AllowNetworkFallback || !yieldedOffline) + { + await foreach (var registryDocument in FetchFromRegistryAsync(discovery, options, target, cancellationToken)) + { + yield return registryDocument; + } + } + } + } + + private async IAsyncEnumerable ReadOfflineAsync( + OciAttestationTarget target, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var offline = target.OfflineBundle!; + var path = _fileSystem.Path.GetFullPath(offline.Path); + + if (!_fileSystem.File.Exists(path)) + { + if (offline.Exists) + { + _logger.LogWarning("Offline bundle {Path} disappeared before processing.", path); + } + yield break; + } + + var extension = _fileSystem.Path.GetExtension(path).ToLowerInvariant(); + var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest; + + if (string.Equals(extension, ".json", StringComparison.OrdinalIgnoreCase) || + string.Equals(extension, ".dsse", StringComparison.OrdinalIgnoreCase)) + { + var bytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + var metadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest); + yield return new OciAttestationDocument( + new Uri(path, UriKind.Absolute), + bytes, + metadata, + subjectDigest, + null, + null, + "offline"); + yield break; + } + + if (string.Equals(extension, ".tgz", StringComparison.OrdinalIgnoreCase) || + string.Equals(extension, ".gz", StringComparison.OrdinalIgnoreCase) || + string.Equals(extension, ".tar", StringComparison.OrdinalIgnoreCase)) + { + await foreach (var document in ReadTarArchiveAsync(target, path, subjectDigest, cancellationToken)) + { + yield return document; + } + yield break; + } + + // Default: treat as binary blob. + var fallbackBytes = await _fileSystem.File.ReadAllBytesAsync(path, cancellationToken).ConfigureAwait(false); + var fallbackMetadata = BuildOfflineMetadata(target, path, entryName: null, subjectDigest); + yield return new OciAttestationDocument( + new Uri(path, UriKind.Absolute), + fallbackBytes, + fallbackMetadata, + subjectDigest, + null, + null, + "offline"); + } + + private async IAsyncEnumerable ReadTarArchiveAsync( + OciAttestationTarget target, + string path, + string? subjectDigest, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + await using var fileStream = _fileSystem.File.OpenRead(path); + Stream archiveStream = fileStream; + + if (path.EndsWith(".gz", StringComparison.OrdinalIgnoreCase) || + path.EndsWith(".tgz", StringComparison.OrdinalIgnoreCase)) + { + archiveStream = new GZipStream(fileStream, CompressionMode.Decompress, leaveOpen: false); + } + + using var tarReader = new TarReader(archiveStream, leaveOpen: false); + TarEntry? entry; + + while ((entry = await tarReader.GetNextEntryAsync(copyData: false, cancellationToken).ConfigureAwait(false)) is not null) + { + if (entry.EntryType is not TarEntryType.RegularFile || entry.DataStream is null) + { + continue; + } + + await using var entryStream = entry.DataStream; + using var buffer = new MemoryStream(); + await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + + var metadata = BuildOfflineMetadata(target, path, entry.Name, subjectDigest); + var sourceUri = new Uri($"{_fileSystem.Path.GetFullPath(path)}#{entry.Name}", UriKind.Absolute); + yield return new OciAttestationDocument( + sourceUri, + buffer.ToArray(), + metadata, + subjectDigest, + null, + null, + "offline"); + } + } + + private async IAsyncEnumerable FetchFromRegistryAsync( + OciAttestationDiscoveryResult discovery, + OciOpenVexAttestationConnectorOptions options, + OciAttestationTarget target, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + var registryClient = new OciRegistryClient( + _httpClientFactory, + _logger, + discovery.RegistryAuthorization, + options); + + var subjectDigest = target.Image.Digest ?? target.ExpectedSubjectDigest; + if (string.IsNullOrWhiteSpace(subjectDigest)) + { + subjectDigest = await registryClient.ResolveDigestAsync(target.Image, cancellationToken).ConfigureAwait(false); + } + + if (string.IsNullOrWhiteSpace(subjectDigest)) + { + _logger.LogWarning("Unable to resolve subject digest for {Reference}; skipping registry fetch.", target.Image.Canonical); + yield break; + } + + if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest) && + !string.Equals(target.ExpectedSubjectDigest, subjectDigest, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Resolved digest {Resolved} does not match expected digest {Expected} for {Reference}.", + subjectDigest, + target.ExpectedSubjectDigest, + target.Image.Canonical); + } + + var descriptors = await registryClient.ListReferrersAsync(target.Image, subjectDigest, cancellationToken).ConfigureAwait(false); + if (descriptors.Count == 0) + { + yield break; + } + + foreach (var descriptor in descriptors) + { + cancellationToken.ThrowIfCancellationRequested(); + var document = await registryClient.DownloadAttestationAsync(target.Image, descriptor, subjectDigest, cancellationToken).ConfigureAwait(false); + if (document is not null) + { + yield return document; + } + } + } + + private static ImmutableDictionary BuildOfflineMetadata( + OciAttestationTarget target, + string bundlePath, + string? entryName, + string? subjectDigest) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + builder["oci.image.registry"] = target.Image.Registry; + builder["oci.image.repository"] = target.Image.Repository; + builder["oci.image.reference"] = target.Image.Canonical; + if (!string.IsNullOrWhiteSpace(subjectDigest)) + { + builder["oci.image.subjectDigest"] = subjectDigest; + } + if (!string.IsNullOrWhiteSpace(target.ExpectedSubjectDigest)) + { + builder["oci.image.expectedSubjectDigest"] = target.ExpectedSubjectDigest!; + } + + builder["oci.attestation.sourceKind"] = "offline"; + builder["oci.attestation.source"] = bundlePath; + if (!string.IsNullOrWhiteSpace(entryName)) + { + builder["oci.attestation.bundleEntry"] = entryName!; + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciRegistryClient.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciRegistryClient.cs new file mode 100644 index 00000000..e1f3510c --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/Fetch/OciRegistryClient.cs @@ -0,0 +1,362 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Authentication; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; + +internal sealed class OciRegistryClient +{ + private const string ManifestMediaType = "application/vnd.oci.image.manifest.v1+json"; + private const string ReferrersArtifactType = "application/vnd.dsse.envelope.v1+json"; + private const string DsseMediaType = "application/vnd.dsse.envelope.v1+json"; + private const string OpenVexMediaType = "application/vnd.cncf.openvex.v1+json"; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly IHttpClientFactory _httpClientFactory; + private readonly ILogger _logger; + private readonly OciRegistryAuthorization _authorization; + private readonly OciOpenVexAttestationConnectorOptions _options; + + public OciRegistryClient( + IHttpClientFactory httpClientFactory, + ILogger logger, + OciRegistryAuthorization authorization, + OciOpenVexAttestationConnectorOptions options) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _authorization = authorization ?? throw new ArgumentNullException(nameof(authorization)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public async Task ResolveDigestAsync(OciImageReference image, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(image); + + if (image.HasDigest) + { + return image.Digest; + } + + var requestUri = BuildRegistryUri(image, $"manifests/{EscapeReference(image.Tag ?? "latest")}"); + + async Task RequestFactory() + { + var request = new HttpRequestMessage(HttpMethod.Head, requestUri); + request.Headers.Accept.ParseAdd(ManifestMediaType); + ApplyAuthentication(request); + return await Task.FromResult(request).ConfigureAwait(false); + } + + using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogWarning("Failed to resolve digest for {Reference}; registry returned 404.", image.Canonical); + return null; + } + + response.EnsureSuccessStatusCode(); + } + + if (response.Headers.TryGetValues("Docker-Content-Digest", out var values)) + { + var digest = values.FirstOrDefault(); + if (!string.IsNullOrWhiteSpace(digest)) + { + return digest; + } + } + + // Manifest may have been returned without digest header; fall back to GET. + async Task ManifestRequestFactory() + { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Accept.ParseAdd(ManifestMediaType); + ApplyAuthentication(request); + return await Task.FromResult(request).ConfigureAwait(false); + } + + using var manifestResponse = await SendAsync(ManifestRequestFactory, cancellationToken).ConfigureAwait(false); + manifestResponse.EnsureSuccessStatusCode(); + + if (manifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var manifestValues)) + { + return manifestValues.FirstOrDefault(); + } + + _logger.LogWarning("Registry {Registry} did not provide Docker-Content-Digest header for {Reference}.", image.Registry, image.Canonical); + return null; + } + + public async Task> ListReferrersAsync( + OciImageReference image, + string subjectDigest, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(subjectDigest); + + var query = $"artifactType={Uri.EscapeDataString(ReferrersArtifactType)}"; + var requestUri = BuildRegistryUri(image, $"referrers/{subjectDigest}", query); + + async Task RequestFactory() + { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + ApplyAuthentication(request); + request.Headers.Accept.ParseAdd("application/json"); + return await Task.FromResult(request).ConfigureAwait(false); + } + + using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogDebug("Registry returned 404 for referrers on {Subject}.", subjectDigest); + return Array.Empty(); + } + + response.EnsureSuccessStatusCode(); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var index = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); + return index?.Referrers ?? Array.Empty(); + } + + public async Task DownloadAttestationAsync( + OciImageReference image, + OciArtifactDescriptor descriptor, + string subjectDigest, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(descriptor); + + if (!IsSupportedDescriptor(descriptor)) + { + return null; + } + + var requestUri = BuildRegistryUri(image, $"blobs/{descriptor.Digest}"); + + async Task RequestFactory() + { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + ApplyAuthentication(request); + request.Headers.Accept.ParseAdd(descriptor.MediaType ?? "application/octet-stream"); + return await Task.FromResult(request).ConfigureAwait(false); + } + + using var response = await SendAsync(RequestFactory, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + if (response.StatusCode == HttpStatusCode.NotFound) + { + _logger.LogWarning("Registry returned 404 while downloading attestation {Digest} for {Subject}.", descriptor.Digest, subjectDigest); + return null; + } + + response.EnsureSuccessStatusCode(); + } + + var buffer = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var metadata = BuildMetadata(image, descriptor, "registry", requestUri.ToString(), subjectDigest); + return new OciAttestationDocument( + requestUri, + buffer, + metadata, + subjectDigest, + descriptor.Digest, + descriptor.ArtifactType, + "registry"); + } + + private static bool IsSupportedDescriptor(OciArtifactDescriptor descriptor) + { + if (descriptor is null) + { + return false; + } + + if (!string.IsNullOrWhiteSpace(descriptor.ArtifactType) && + descriptor.ArtifactType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + if (!string.IsNullOrWhiteSpace(descriptor.MediaType) && + (descriptor.MediaType.Equals(DsseMediaType, StringComparison.OrdinalIgnoreCase) || + descriptor.MediaType.Equals(OpenVexMediaType, StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + return false; + } + + private async Task SendAsync( + Func> requestFactory, + CancellationToken cancellationToken) + { + const int maxAttempts = 3; + TimeSpan delay = TimeSpan.FromSeconds(1); + Exception? lastError = null; + + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + using var request = await requestFactory().ConfigureAwait(false); + var client = _httpClientFactory.CreateClient(OciOpenVexAttestationConnectorOptions.HttpClientName); + + try + { + var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + return response; + } + + if (response.StatusCode == HttpStatusCode.Unauthorized) + { + if (_authorization.Mode == OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback) + { + var message = $"Registry request to {request.RequestUri} was unauthorized and anonymous fallback is disabled."; + response.Dispose(); + throw new HttpRequestException(message); + } + + lastError = new HttpRequestException($"Registry returned 401 Unauthorized for {request.RequestUri}."); + } + else if ((int)response.StatusCode >= 500 || response.StatusCode == (HttpStatusCode)429) + { + lastError = new HttpRequestException($"Registry returned status {(int)response.StatusCode} ({response.ReasonPhrase}) for {request.RequestUri}."); + } + else + { + response.EnsureSuccessStatusCode(); + } + + response.Dispose(); + } + catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException) + { + lastError = ex; + } + + if (attempt < maxAttempts) + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay = TimeSpan.FromSeconds(Math.Min(delay.TotalSeconds * 2, 10)); + } + } + + throw new HttpRequestException("Failed to execute OCI registry request after multiple attempts.", lastError); + } + + private void ApplyAuthentication(HttpRequestMessage request) + { + switch (_authorization.Mode) + { + case OciRegistryAuthMode.Basic when + !string.IsNullOrEmpty(_authorization.Username) && + !string.IsNullOrEmpty(_authorization.Password): + var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{_authorization.Username}:{_authorization.Password}")); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + break; + case OciRegistryAuthMode.IdentityToken when !string.IsNullOrWhiteSpace(_authorization.IdentityToken): + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.IdentityToken); + break; + case OciRegistryAuthMode.RefreshToken when !string.IsNullOrWhiteSpace(_authorization.RefreshToken): + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authorization.RefreshToken); + break; + default: + if (_authorization.Mode != OciRegistryAuthMode.Anonymous && !_authorization.AllowAnonymousFallback) + { + _logger.LogDebug("No authentication header applied for request to {Uri} (mode {Mode}).", request.RequestUri, _authorization.Mode); + } + break; + } + } + + private Uri BuildRegistryUri(OciImageReference image, string relativePath, string? query = null) + { + var scheme = image.Scheme; + if (!string.Equals(scheme, "https", StringComparison.OrdinalIgnoreCase) && !_options.AllowHttpRegistries) + { + throw new InvalidOperationException($"HTTP access to registry '{image.Registry}' is disabled. Set AllowHttpRegistries to true to enable."); + } + + var builder = new UriBuilder($"{scheme}://{image.Registry}") + { + Path = $"v2/{BuildRepositoryPath(image.Repository)}/{relativePath}" + }; + + if (!string.IsNullOrWhiteSpace(query)) + { + builder.Query = query; + } + + return builder.Uri; + } + + private static string BuildRepositoryPath(string repository) + { + var segments = repository.Split('/', StringSplitOptions.RemoveEmptyEntries); + return string.Join('/', segments.Select(Uri.EscapeDataString)); + } + + private static string EscapeReference(string reference) + { + return Uri.EscapeDataString(reference); + } + + private static ImmutableDictionary BuildMetadata( + OciImageReference image, + OciArtifactDescriptor descriptor, + string sourceKind, + string sourcePath, + string subjectDigest) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + builder["oci.image.registry"] = image.Registry; + builder["oci.image.repository"] = image.Repository; + builder["oci.image.reference"] = image.Canonical; + builder["oci.image.subjectDigest"] = subjectDigest; + builder["oci.attestation.sourceKind"] = sourceKind; + builder["oci.attestation.source"] = sourcePath; + builder["oci.attestation.artifactDigest"] = descriptor.Digest; + builder["oci.attestation.mediaType"] = descriptor.MediaType ?? string.Empty; + builder["oci.attestation.artifactType"] = descriptor.ArtifactType ?? string.Empty; + builder["oci.attestation.size"] = descriptor.Size.ToString(CultureInfo.InvariantCulture); + + if (descriptor.Annotations is not null) + { + foreach (var annotation in descriptor.Annotations) + { + builder[$"oci.attestation.annotations.{annotation.Key}"] = annotation.Value; + } + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/OciOpenVexAttestationConnector.cs b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/OciOpenVexAttestationConnector.cs new file mode 100644 index 00000000..9c331621 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/OciOpenVexAttestationConnector.cs @@ -0,0 +1,221 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Configuration; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Discovery; +using StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.Fetch; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest; + +public sealed class OciOpenVexAttestationConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor StaticDescriptor = new( + id: "excititor:oci.openvex.attest", + kind: VexProviderKind.Attestation, + displayName: "OCI OpenVEX Attestations") + { + Tags = ImmutableArray.Create("oci", "openvex", "attestation", "cosign", "offline"), + }; + + private readonly OciAttestationDiscoveryService _discoveryService; + private readonly OciAttestationFetcher _fetcher; + private readonly IEnumerable> _validators; + + private OciOpenVexAttestationConnectorOptions? _options; + private OciAttestationDiscoveryResult? _discovery; + + public OciOpenVexAttestationConnector( + OciAttestationDiscoveryService discoveryService, + OciAttestationFetcher fetcher, + ILogger logger, + TimeProvider timeProvider, + IEnumerable>? validators = null) + : base(StaticDescriptor, logger, timeProvider) + { + _discoveryService = discoveryService ?? throw new ArgumentNullException(nameof(discoveryService)); + _fetcher = fetcher ?? throw new ArgumentNullException(nameof(fetcher)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Information, "validate", "Resolved OCI attestation targets.", new Dictionary + { + ["targets"] = _discovery.Targets.Length, + ["offlinePreferred"] = _discovery.PreferOffline, + ["allowNetworkFallback"] = _discovery.AllowNetworkFallback, + ["authMode"] = _discovery.RegistryAuthorization.Mode.ToString(), + ["cosignMode"] = _discovery.CosignAuthority.Mode.ToString(), + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_discovery is null) + { + _discovery = await _discoveryService.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + var documentCount = 0; + await foreach (var document in _fetcher.FetchAsync(_discovery, _options, cancellationToken)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var verificationDocument = CreateRawDocument( + VexDocumentFormat.OciAttestation, + document.SourceUri, + document.Content, + document.Metadata); + + var signatureMetadata = await context.SignatureVerifier.VerifyAsync(verificationDocument, cancellationToken).ConfigureAwait(false); + if (signatureMetadata is not null) + { + LogConnectorEvent(LogLevel.Debug, "signature", "Signature metadata captured for attestation.", new Dictionary + { + ["subject"] = signatureMetadata.Subject, + ["type"] = signatureMetadata.Type, + }); + } + + var enrichedMetadata = BuildProvenanceMetadata(document, signatureMetadata); + var rawDocument = CreateRawDocument( + VexDocumentFormat.OciAttestation, + document.SourceUri, + document.Content, + enrichedMetadata); + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + documentCount++; + yield return rawDocument; + } + + LogConnectorEvent(LogLevel.Information, "fetch", "OCI attestation fetch completed.", new Dictionary + { + ["documents"] = documentCount, + ["since"] = context.Since?.ToString("O"), + }); + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("Attestation documents rely on dedicated normalizers, to be wired in EXCITITOR-CONN-OCI-01-002."); + + public OciAttestationDiscoveryResult? GetCachedDiscovery() => _discovery; + + private ImmutableDictionary BuildProvenanceMetadata(OciAttestationDocument document, VexSignatureMetadata? signature) + { + var builder = document.Metadata.ToBuilder(); + + if (!string.IsNullOrWhiteSpace(document.SourceKind)) + { + builder["vex.provenance.sourceKind"] = document.SourceKind; + } + + if (!string.IsNullOrWhiteSpace(document.SubjectDigest)) + { + builder["vex.provenance.subjectDigest"] = document.SubjectDigest!; + } + + if (!string.IsNullOrWhiteSpace(document.ArtifactDigest)) + { + builder["vex.provenance.artifactDigest"] = document.ArtifactDigest!; + } + + if (!string.IsNullOrWhiteSpace(document.ArtifactType)) + { + builder["vex.provenance.artifactType"] = document.ArtifactType!; + } + + if (_discovery is not null) + { + builder["vex.provenance.registryAuthMode"] = _discovery.RegistryAuthorization.Mode.ToString(); + var registryAuthority = _discovery.RegistryAuthorization.RegistryAuthority; + if (string.IsNullOrWhiteSpace(registryAuthority)) + { + if (builder.TryGetValue("oci.image.registry", out var metadataRegistry) && !string.IsNullOrWhiteSpace(metadataRegistry)) + { + registryAuthority = metadataRegistry; + } + } + + if (!string.IsNullOrWhiteSpace(registryAuthority)) + { + builder["vex.provenance.registryAuthority"] = registryAuthority!; + } + + builder["vex.provenance.cosign.mode"] = _discovery.CosignAuthority.Mode.ToString(); + + if (_discovery.CosignAuthority.Keyless is not null) + { + var keyless = _discovery.CosignAuthority.Keyless; + builder["vex.provenance.cosign.issuer"] = keyless!.Issuer; + builder["vex.provenance.cosign.subject"] = keyless.Subject; + if (keyless.FulcioUrl is not null) + { + builder["vex.provenance.cosign.fulcioUrl"] = keyless.FulcioUrl!.ToString(); + } + + if (keyless.RekorUrl is not null) + { + builder["vex.provenance.cosign.rekorUrl"] = keyless.RekorUrl!.ToString(); + } + } + else if (_discovery.CosignAuthority.KeyPair is not null) + { + var keyPair = _discovery.CosignAuthority.KeyPair; + builder["vex.provenance.cosign.keyPair"] = "true"; + if (keyPair!.RekorUrl is not null) + { + builder["vex.provenance.cosign.rekorUrl"] = keyPair.RekorUrl!.ToString(); + } + } + } + + if (signature is not null) + { + builder["vex.signature.type"] = signature.Type; + if (!string.IsNullOrWhiteSpace(signature.Subject)) + { + builder["vex.signature.subject"] = signature.Subject!; + } + + if (!string.IsNullOrWhiteSpace(signature.Issuer)) + { + builder["vex.signature.issuer"] = signature.Issuer!; + } + + if (!string.IsNullOrWhiteSpace(signature.KeyId)) + { + builder["vex.signature.keyId"] = signature.KeyId!; + } + + if (signature.VerifiedAt is not null) + { + builder["vex.signature.verifiedAt"] = signature.VerifiedAt.Value.ToString("O"); + } + + if (!string.IsNullOrWhiteSpace(signature.TransparencyLogReference)) + { + builder["vex.signature.transparencyLogReference"] = signature.TransparencyLogReference!; + } + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj similarity index 63% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj rename to src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj index a99a942f..4aea0c00 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/StellaOps.Vexer.Connectors.Oracle.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest.csproj @@ -1,18 +1,20 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + NU1903 + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md new file mode 100644 index 00000000..5bd99b10 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.OCI.OpenVEX.Attest/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-OCI-01-001 – OCI discovery & auth plumbing|Team Excititor Connectors – OCI|EXCITITOR-CONN-ABS-01-001|DONE (2025-10-18) – Added connector skeleton, options/validators, discovery caching, cosign/auth descriptors, offline bundle resolution, DI wiring, and regression tests.| +|EXCITITOR-CONN-OCI-01-002 – Attestation fetch & verify loop|Team Excititor Connectors – OCI|EXCITITOR-CONN-OCI-01-001, EXCITITOR-ATTEST-01-002|DONE (2025-10-18) – Added offline/registry fetch services, DSSE retrieval with retries, signature verification callout, and raw persistence coverage.| +|EXCITITOR-CONN-OCI-01-003 – Provenance metadata & policy hooks|Team Excititor Connectors – OCI|EXCITITOR-CONN-OCI-01-002, EXCITITOR-POLICY-01-001|DONE (2025-10-18) – Enriched attestation metadata with provenance hints, cosign expectations, registry auth context, and signature diagnostics for policy consumption.| diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs new file mode 100644 index 00000000..0b8155bf --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Connectors/OracleCsafConnectorTests.cs @@ -0,0 +1,311 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Oracle.CSAF; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Connectors; + +public sealed class OracleCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_NewEntry_PersistsDocumentAndUpdatesState() + { + var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); + var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var payloadDigest = ComputeDigest(payload); + var snapshotPath = "/snapshots/oracle-catalog.json"; + var fileSystem = new MockFileSystem(); + fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, payloadDigest, "2025-10-15T00:00:00Z"))); + + var handler = new StubHttpMessageHandler(new Dictionary + { + [documentUri] = CreateResponse(payload), + }); + var httpClient = new HttpClient(handler); + var httpFactory = new SingleHttpClientFactory(httpClient); + var loader = new OracleCatalogLoader( + httpFactory, + new MemoryCache(new MemoryCacheOptions()), + fileSystem, + NullLogger.Instance, + TimeProvider.System); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new OracleCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { new OracleConnectorOptionsValidator(fileSystem) }, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") + .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) + .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); + var settings = new VexConnectorSettings(settingsValues); + + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext( + Since: null, + Settings: settings, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + documents[0].Digest.Should().Be(payloadDigest); + documents[0].Metadata["oracle.csaf.entryId"].Should().Be("CPU2025Oct"); + documents[0].Metadata["oracle.csaf.sha256"].Should().Be(payloadDigest); + + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.DocumentDigests.Should().ContainSingle().Which.Should().Be(payloadDigest); + + handler.GetCallCount(documentUri).Should().Be(1); + + // second run should short-circuit without downloading again + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + handler.GetCallCount(documentUri).Should().Be(1); + } + + [Fact] + public async Task FetchAsync_ChecksumMismatch_SkipsDocument() + { + var documentUri = new Uri("https://oracle.example/security/csaf/cpu2025oct.json"); + var payload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var snapshotPath = "/snapshots/oracle-catalog.json"; + var fileSystem = new MockFileSystem(); + fileSystem.AddFile(snapshotPath, new MockFileData(BuildOfflineSnapshot(documentUri, "deadbeef", "2025-10-15T00:00:00Z"))); + + var handler = new StubHttpMessageHandler(new Dictionary + { + [documentUri] = CreateResponse(payload), + }); + var httpClient = new HttpClient(handler); + var httpFactory = new SingleHttpClientFactory(httpClient); + var loader = new OracleCatalogLoader( + httpFactory, + new MemoryCache(new MemoryCacheOptions()), + fileSystem, + NullLogger.Instance, + TimeProvider.System); + + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new OracleCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { new OracleConnectorOptionsValidator(fileSystem) }, + NullLogger.Instance, + TimeProvider.System); + + var settingsValues = ImmutableDictionary.Empty + .Add(nameof(OracleConnectorOptions.PreferOfflineSnapshot), "true") + .Add(nameof(OracleConnectorOptions.OfflineSnapshotPath), snapshotPath) + .Add(nameof(OracleConnectorOptions.PersistOfflineSnapshot), "false"); + var settings = new VexConnectorSettings(settingsValues); + + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext( + Since: null, + Settings: settings, + RawSink: sink, + SignatureVerifier: new NoopSignatureVerifier(), + Normalizers: new NoopNormalizerRouter(), + Services: new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.State.Should().BeNull(); + handler.GetCallCount(documentUri).Should().Be(1); + } + + private static HttpResponseMessage CreateResponse(byte[] payload) + => new(HttpStatusCode.OK) + { + Content = new ByteArrayContent(payload) + { + Headers = + { + ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/json"), + } + } + }; + + private static string ComputeDigest(byte[] payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return "sha256:" + Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private static string BuildOfflineSnapshot(Uri documentUri, string sha256, string publishedAt) + { + var snapshot = new + { + metadata = new + { + generatedAt = "2025-10-14T12:00:00Z", + entries = new[] + { + new + { + id = "CPU2025Oct", + title = "Oracle Critical Patch Update Advisory - October 2025", + documentUri = documentUri.ToString(), + publishedAt, + revision = publishedAt, + sha256, + size = 1024, + products = new[] { "Oracle Database" } + } + }, + cpuSchedule = Array.Empty() + }, + fetchedAt = "2025-10-14T12:00:00Z" + }; + + return JsonSerializer.Serialize(snapshot, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + } + + private sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + private readonly Dictionary _callCounts = new(); + + public StubHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int GetCallCount(Uri uri) => _callCounts.TryGetValue(uri, out var count) ? count : 0; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri is null || !_responses.TryGetValue(request.RequestUri, out var response)) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound)); + } + + _callCounts.TryGetValue(request.RequestUri, out var count); + _callCounts[request.RequestUri] = count + 1; + return Task.FromResult(response.Clone()); + } + } + + private sealed class SingleHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + => ValueTask.FromResult(State); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + State = state; + return ValueTask.CompletedTask; + } + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} + +internal static class HttpResponseMessageExtensions +{ + public static HttpResponseMessage Clone(this HttpResponseMessage response) + { + var clone = new HttpResponseMessage(response.StatusCode); + foreach (var header in response.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (response.Content is not null) + { + var payload = response.Content.ReadAsByteArrayAsync().GetAwaiter().GetResult(); + var mediaType = response.Content.Headers.ContentType?.MediaType ?? "application/json"; + clone.Content = new ByteArrayContent(payload); + clone.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(mediaType); + } + + return clone; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs index de05d2d4..173d3df7 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/Metadata/OracleCatalogLoaderTests.cs @@ -1,205 +1,205 @@ -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; -using System.IO.Abstractions.TestingHelpers; -using Xunit; -using System.Threading; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.Metadata; - -public sealed class OracleCatalogLoaderTests -{ - private const string SampleCatalog = """ - { - "generated": "2025-09-30T18:00:00Z", - "catalog": [ - { - "id": "CPU2025Oct", - "title": "Oracle Critical Patch Update Advisory - October 2025", - "published": "2025-10-15T00:00:00Z", - "revision": "2025-10-15T00:00:00Z", - "document": { - "url": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", - "sha256": "abc123", - "size": 1024 - }, - "products": ["Oracle Database", "Java SE"] - } - ], - "schedule": [ - { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } - ] - } - """; - - private const string SampleCalendar = """ - { - "cpuWindows": [ - { "name": "2026-Jan", "releaseDate": "2026-01-21T00:00:00Z" } - ] - } - """; - - [Fact] - public async Task LoadAsync_FetchesAndCachesCatalog() - { - var handler = new TestHttpMessageHandler(new Dictionary - { - [new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json")] = CreateResponse(SampleCatalog), - [new Uri("https://www.oracle.com/security-alerts/cpu/cal.json")] = CreateResponse(SampleCalendar), - }); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); - - var options = new OracleConnectorOptions - { - CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), - CpuCalendarUri = new Uri("https://www.oracle.com/security-alerts/cpu/cal.json"), - OfflineSnapshotPath = "/snapshots/oracle-catalog.json", - }; - - var result = await loader.LoadAsync(options, CancellationToken.None); - result.Metadata.Entries.Should().HaveCount(1); - result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2025-Oct"); - result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2026-Jan"); - result.FromCache.Should().BeFalse(); - fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); - - handler.ResetInvocationCount(); - var cached = await loader.LoadAsync(options, CancellationToken.None); - cached.FromCache.Should().BeTrue(); - handler.InvocationCount.Should().Be(0); - } - - [Fact] - public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() - { - var handler = new TestHttpMessageHandler(new Dictionary()); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var offlineJson = """ - { - "metadata": { - "generatedAt": "2025-09-30T18:00:00Z", - "entries": [ - { - "id": "CPU2025Oct", - "title": "Oracle Critical Patch Update Advisory - October 2025", - "documentUri": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", - "publishedAt": "2025-10-15T00:00:00Z", - "revision": "2025-10-15T00:00:00Z", - "sha256": "abc123", - "size": 1024, - "products": [ "Oracle Database" ] - } - ], - "cpuSchedule": [ - { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } - ] - }, - "fetchedAt": "2025-10-01T00:00:00Z" - } - """; - fileSystem.AddFile("/snapshots/oracle-catalog.json", new MockFileData(offlineJson)); - var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); - - var options = new OracleConnectorOptions - { - CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), - OfflineSnapshotPath = "/snapshots/oracle-catalog.json", - PreferOfflineSnapshot = true, - }; - - var result = await loader.LoadAsync(options, CancellationToken.None); - result.FromOfflineSnapshot.Should().BeTrue(); - result.Metadata.Entries.Should().NotBeEmpty(); - } - - [Fact] - public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() - { - var handler = new TestHttpMessageHandler(new Dictionary()); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance); - - var options = new OracleConnectorOptions - { - CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), - PreferOfflineSnapshot = true, - OfflineSnapshotPath = "/missing/oracle-catalog.json", - }; - - await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); - } - - private static HttpResponseMessage CreateResponse(string payload) - => new(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class AdjustableTimeProvider : TimeProvider - { - private DateTimeOffset _now = DateTimeOffset.UtcNow; - - public override DateTimeOffset GetUtcNow() => _now; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Dictionary _responses; - - public TestHttpMessageHandler(Dictionary responses) - { - _responses = responses; - } - - public int InvocationCount { get; private set; } - - public void ResetInvocationCount() => InvocationCount = 0; - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - InvocationCount++; - if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) - { - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - return new HttpResponseMessage(response.StatusCode) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - } - - return new HttpResponseMessage(HttpStatusCode.InternalServerError) - { - Content = new StringContent("unexpected request"), - }; - } - } -} +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.Metadata; + +public sealed class OracleCatalogLoaderTests +{ + private const string SampleCatalog = """ + { + "generated": "2025-09-30T18:00:00Z", + "catalog": [ + { + "id": "CPU2025Oct", + "title": "Oracle Critical Patch Update Advisory - October 2025", + "published": "2025-10-15T00:00:00Z", + "revision": "2025-10-15T00:00:00Z", + "document": { + "url": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", + "sha256": "abc123", + "size": 1024 + }, + "products": ["Oracle Database", "Java SE"] + } + ], + "schedule": [ + { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } + ] + } + """; + + private const string SampleCalendar = """ + { + "cpuWindows": [ + { "name": "2026-Jan", "releaseDate": "2026-01-21T00:00:00Z" } + ] + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesCatalog() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json")] = CreateResponse(SampleCatalog), + [new Uri("https://www.oracle.com/security-alerts/cpu/cal.json")] = CreateResponse(SampleCalendar), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + CpuCalendarUri = new Uri("https://www.oracle.com/security-alerts/cpu/cal.json"), + OfflineSnapshotPath = "/snapshots/oracle-catalog.json", + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.Metadata.Entries.Should().HaveCount(1); + result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2025-Oct"); + result.Metadata.CpuSchedule.Should().Contain(r => r.Window == "2026-Jan"); + result.FromCache.Should().BeFalse(); + fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); + + handler.ResetInvocationCount(); + var cached = await loader.LoadAsync(options, CancellationToken.None); + cached.FromCache.Should().BeTrue(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlineJson = """ + { + "metadata": { + "generatedAt": "2025-09-30T18:00:00Z", + "entries": [ + { + "id": "CPU2025Oct", + "title": "Oracle Critical Patch Update Advisory - October 2025", + "documentUri": "https://updates.oracle.com/cpu/2025-10/cpu2025oct.json", + "publishedAt": "2025-10-15T00:00:00Z", + "revision": "2025-10-15T00:00:00Z", + "sha256": "abc123", + "size": 1024, + "products": [ "Oracle Database" ] + } + ], + "cpuSchedule": [ + { "window": "2025-Oct", "releaseDate": "2025-10-15T00:00:00Z" } + ] + }, + "fetchedAt": "2025-10-01T00:00:00Z" + } + """; + fileSystem.AddFile("/snapshots/oracle-catalog.json", new MockFileData(offlineJson)); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + OfflineSnapshotPath = "/snapshots/oracle-catalog.json", + PreferOfflineSnapshot = true, + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Entries.Should().NotBeEmpty(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new OracleCatalogLoader(factory, cache, fileSystem, NullLogger.Instance); + + var options = new OracleConnectorOptions + { + CatalogUri = new Uri("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"), + PreferOfflineSnapshot = true, + OfflineSnapshotPath = "/missing/oracle-catalog.json", + }; + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private static HttpResponseMessage CreateResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + + public TestHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int InvocationCount { get; private set; } + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(response.StatusCode) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("unexpected request"), + }; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj similarity index 76% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj index 68fa2841..d100d43e 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests/StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/AGENTS.md b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/AGENTS.md similarity index 95% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF/AGENTS.md index fab25f9c..d65ccf14 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector for Oracle CSAF advisories, including CPU and other bulletin releases, projecting documents into raw storage for normalization. -## Scope -- Discovery of Oracle CSAF catalogue, navigation of quarterly CPU bundles, and delta detection. -- HTTP fetch with retry/backoff, checksum validation, and deduplication across revisions. -- Mapping Oracle advisory metadata (CPU ID, component families) into connector context. -- Publishing trust metadata (PGP keys/cosign options) aligned with policy expectations. -## Participants -- Worker orchestrates regular pulls respecting Oracle publication cadence; WebService offers manual triggers. -- CSAF normalizer processes raw documents to claims. -- Policy engine leverages trust metadata and provenance hints. -## Interfaces & contracts -- Implements `IVexConnector` using shared abstractions for HTTP/resume and telemetry. -- Configuration options for CPU schedule, credentials (if required), and offline snapshot ingestion. -## In/Out of scope -In: fetching, metadata mapping, raw persistence, trust hints. -Out: normalization, storage internals, export/attestation flows. -## Observability & security expectations -- Log CPU release windows, document counts, and fetch durations; redact any secrets. -- Emit metrics for deduped vs new documents and quarantine rates. -## Tests -- Harness tests with mocked Oracle catalogues will live in `../StellaOps.Vexer.Connectors.Oracle.CSAF.Tests`. +# AGENTS +## Role +Connector for Oracle CSAF advisories, including CPU and other bulletin releases, projecting documents into raw storage for normalization. +## Scope +- Discovery of Oracle CSAF catalogue, navigation of quarterly CPU bundles, and delta detection. +- HTTP fetch with retry/backoff, checksum validation, and deduplication across revisions. +- Mapping Oracle advisory metadata (CPU ID, component families) into connector context. +- Publishing trust metadata (PGP keys/cosign options) aligned with policy expectations. +## Participants +- Worker orchestrates regular pulls respecting Oracle publication cadence; WebService offers manual triggers. +- CSAF normalizer processes raw documents to claims. +- Policy engine leverages trust metadata and provenance hints. +## Interfaces & contracts +- Implements `IVexConnector` using shared abstractions for HTTP/resume and telemetry. +- Configuration options for CPU schedule, credentials (if required), and offline snapshot ingestion. +## In/Out of scope +In: fetching, metadata mapping, raw persistence, trust hints. +Out: normalization, storage internals, export/attestation flows. +## Observability & security expectations +- Log CPU release windows, document counts, and fetch durations; redact any secrets. +- Emit metrics for deduped vs new documents and quarantine rates. +## Tests +- Harness tests with mocked Oracle catalogues will live in `../StellaOps.Excititor.Connectors.Oracle.CSAF.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs index aea25b80..44c4e2f0 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptions.cs @@ -1,85 +1,85 @@ -using System; -using System.IO; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; - -public sealed class OracleConnectorOptions -{ - public const string HttpClientName = "vexer.connector.oracle.catalog"; - - /// - /// Oracle CSAF catalog endpoint hosting advisory metadata. - /// - public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"); - - /// - /// Optional CPU calendar endpoint providing upcoming release dates. - /// - public Uri? CpuCalendarUri { get; set; } - /// - /// Duration the discovery metadata should be cached before refresh. - /// - public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); - - /// - /// When true, the loader will prefer offline snapshot data over network fetches. - /// - public bool PreferOfflineSnapshot { get; set; } - /// - /// Optional file path for persisting or ingesting catalog snapshots. - /// - public string? OfflineSnapshotPath { get; set; } - /// - /// Enables writing fresh catalog responses to . - /// - public bool PersistOfflineSnapshot { get; set; } = true; - - /// - /// Optional request delay when iterating catalogue entries (for rate limiting). - /// - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - public void Validate(IFileSystem? fileSystem = null) - { - if (CatalogUri is null || !CatalogUri.IsAbsoluteUri) - { - throw new InvalidOperationException("CatalogUri must be an absolute URI."); - } - - if (CatalogUri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException("CatalogUri must use HTTP or HTTPS."); - } - - if (CpuCalendarUri is not null && (!CpuCalendarUri.IsAbsoluteUri || CpuCalendarUri.Scheme is not ("http" or "https"))) - { - throw new InvalidOperationException("CpuCalendarUri must be an absolute HTTP(S) URI when provided."); - } - - if (MetadataCacheDuration <= TimeSpan.Zero) - { - throw new InvalidOperationException("MetadataCacheDuration must be positive."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("RequestDelay cannot be negative."); - } - - if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); - } - - if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - var fs = fileSystem ?? new FileSystem(); - var directory = Path.GetDirectoryName(OfflineSnapshotPath); - if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) - { - fs.Directory.CreateDirectory(directory); - } - } - } -} +using System; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; + +public sealed class OracleConnectorOptions +{ + public const string HttpClientName = "excititor.connector.oracle.catalog"; + + /// + /// Oracle CSAF catalog endpoint hosting advisory metadata. + /// + public Uri CatalogUri { get; set; } = new("https://www.oracle.com/security-alerts/cpu/csaf/catalog.json"); + + /// + /// Optional CPU calendar endpoint providing upcoming release dates. + /// + public Uri? CpuCalendarUri { get; set; } + /// + /// Duration the discovery metadata should be cached before refresh. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(6); + + /// + /// When true, the loader will prefer offline snapshot data over network fetches. + /// + public bool PreferOfflineSnapshot { get; set; } + /// + /// Optional file path for persisting or ingesting catalog snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + /// + /// Enables writing fresh catalog responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Optional request delay when iterating catalogue entries (for rate limiting). + /// + public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); + + public void Validate(IFileSystem? fileSystem = null) + { + if (CatalogUri is null || !CatalogUri.IsAbsoluteUri) + { + throw new InvalidOperationException("CatalogUri must be an absolute URI."); + } + + if (CatalogUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("CatalogUri must use HTTP or HTTPS."); + } + + if (CpuCalendarUri is not null && (!CpuCalendarUri.IsAbsoluteUri || CpuCalendarUri.Scheme is not ("http" or "https"))) + { + throw new InvalidOperationException("CpuCalendarUri must be an absolute HTTP(S) URI when provided."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (RequestDelay < TimeSpan.Zero) + { + throw new InvalidOperationException("RequestDelay cannot be negative."); + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs similarity index 84% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs index 13574413..3e659130 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Configuration/OracleConnectorOptionsValidator.cs @@ -1,32 +1,32 @@ -using System; -using System.Collections.Generic; -using System.IO.Abstractions; -using StellaOps.Vexer.Connectors.Abstractions; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; - -public sealed class OracleConnectorOptionsValidator : IVexConnectorOptionsValidator -{ - private readonly IFileSystem _fileSystem; - - public OracleConnectorOptionsValidator(IFileSystem fileSystem) - { - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - } - - public void Validate(VexConnectorDescriptor descriptor, OracleConnectorOptions options, IList errors) - { - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentNullException.ThrowIfNull(options); - ArgumentNullException.ThrowIfNull(errors); - - try - { - options.Validate(_fileSystem); - } - catch (Exception ex) - { - errors.Add(ex.Message); - } - } -} +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; + +public sealed class OracleConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public OracleConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, OracleConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs similarity index 80% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs index af8aa641..6f5df95a 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/DependencyInjection/OracleConnectorServiceCollectionExtensions.cs @@ -1,45 +1,45 @@ -using System; -using System.Net; -using System.Net.Http; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF.DependencyInjection; - -public static class OracleConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddOracleCsafConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddOptions() - .Configure(options => configure?.Invoke(options)); - - services.AddSingleton, OracleConnectorOptionsValidator>(); - - services.AddHttpClient(OracleConnectorOptions.HttpClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(60); - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.Oracle.CSAF/1.0"); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - }) - .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.All, - }); - - services.AddSingleton(); - services.AddSingleton(); - - return services; - } -} +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.DependencyInjection; + +public static class OracleConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddOracleCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddSingleton, OracleConnectorOptionsValidator>(); + + services.AddHttpClient(OracleConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(60); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Oracle.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs similarity index 96% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs rename to src/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs index 8268aff2..6936fb24 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/Metadata/OracleCatalogLoader.cs @@ -1,418 +1,418 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO.Abstractions; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; - -public sealed class OracleCatalogLoader -{ - public const string CachePrefix = "StellaOps.Vexer.Connectors.Oracle.CSAF.Catalog"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _memoryCache; - private readonly IFileSystem _fileSystem; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly SemaphoreSlim _semaphore = new(1, 1); - private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); - - public OracleCatalogLoader( - IHttpClientFactory httpClientFactory, - IMemoryCache memoryCache, - IFileSystem fileSystem, - ILogger logger, - TimeProvider? timeProvider = null) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public async Task LoadAsync(OracleConnectorOptions options, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(options); - options.Validate(_fileSystem); - - var cacheKey = CreateCacheKey(options); - if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) - { - return cached.ToResult(fromCache: true); - } - - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) - { - return cached.ToResult(fromCache: true); - } - - CacheEntry? entry = null; - if (options.PreferOfflineSnapshot) - { - entry = LoadFromOffline(options); - if (entry is null) - { - throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline snapshot was found or could be loaded."); - } - } - else - { - entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) - ?? LoadFromOffline(options); - } - - if (entry is null) - { - throw new InvalidOperationException("Unable to load Oracle CSAF catalog from network or offline snapshot."); - } - - var expiration = entry.MetadataCacheDuration == TimeSpan.Zero - ? (DateTimeOffset?)null - : _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); - - var cacheEntryOptions = new MemoryCacheEntryOptions(); - if (expiration.HasValue) - { - cacheEntryOptions.AbsoluteExpiration = expiration.Value; - } - - _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheEntryOptions); - return entry.ToResult(fromCache: false); - } - finally - { - _semaphore.Release(); - } - } - - private async Task TryFetchFromNetworkAsync(OracleConnectorOptions options, CancellationToken cancellationToken) - { - try - { - var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName); - using var response = await client.GetAsync(options.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - var catalogPayload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - - string? calendarPayload = null; - if (options.CpuCalendarUri is not null) - { - try - { - using var calendarResponse = await client.GetAsync(options.CpuCalendarUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - calendarResponse.EnsureSuccessStatusCode(); - calendarPayload = await calendarResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, "Failed to fetch Oracle CPU calendar from {Uri}; continuing without schedule.", options.CpuCalendarUri); - } - } - - var metadata = ParseMetadata(catalogPayload, calendarPayload); - var fetchedAt = _timeProvider.GetUtcNow(); - var entry = new CacheEntry(metadata, fetchedAt, fetchedAt, options.MetadataCacheDuration, false); - - PersistSnapshotIfNeeded(options, metadata, fetchedAt); - return entry; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, "Failed to fetch Oracle CSAF catalog from {Uri}; attempting offline fallback if available.", options.CatalogUri); - return null; - } - } - - private CacheEntry? LoadFromOffline(OracleConnectorOptions options) - { - if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return null; - } - - if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) - { - _logger.LogWarning("Oracle offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); - return null; - } - - try - { - var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); - var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); - if (snapshot is null) - { - throw new InvalidOperationException("Offline snapshot payload was empty."); - } - - return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load Oracle CSAF catalog from offline snapshot {Path}.", options.OfflineSnapshotPath); - return null; - } - } - - private OracleCatalogMetadata ParseMetadata(string catalogPayload, string? calendarPayload) - { - if (string.IsNullOrWhiteSpace(catalogPayload)) - { - throw new InvalidOperationException("Oracle catalog payload was empty."); - } - - using var document = JsonDocument.Parse(catalogPayload); - var root = document.RootElement; - - var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) - ? generated - : _timeProvider.GetUtcNow(); - - var entries = ParseEntries(root); - var schedule = ParseSchedule(root); - - if (!string.IsNullOrWhiteSpace(calendarPayload)) - { - schedule = MergeSchedule(schedule, calendarPayload); - } - - return new OracleCatalogMetadata(generatedAt, entries, schedule); - } - - private ImmutableArray ParseEntries(JsonElement root) - { - if (!root.TryGetProperty("catalog", out var catalogElement) || catalogElement.ValueKind is not JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var entry in catalogElement.EnumerateArray()) - { - var id = entry.TryGetProperty("id", out var idElement) && idElement.ValueKind == JsonValueKind.String ? idElement.GetString() : null; - var title = entry.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String ? titleElement.GetString() : null; - if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(title)) - { - continue; - } - - DateTimeOffset publishedAt = default; - if (entry.TryGetProperty("published", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(publishedElement.GetString(), out var publishedParsed)) - { - publishedAt = publishedParsed; - } - - string? revision = null; - if (entry.TryGetProperty("revision", out var revisionElement) && revisionElement.ValueKind == JsonValueKind.String) - { - revision = revisionElement.GetString(); - } - - ImmutableArray products = ImmutableArray.Empty; - if (entry.TryGetProperty("products", out var productsElement)) - { - products = ParseStringArray(productsElement); - } - - Uri? documentUri = null; - string? sha256 = null; - long? size = null; - - if (entry.TryGetProperty("document", out var documentElement) && documentElement.ValueKind == JsonValueKind.Object) - { - if (documentElement.TryGetProperty("url", out var urlElement) && urlElement.ValueKind == JsonValueKind.String && Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var parsedUri)) - { - documentUri = parsedUri; - } - - if (documentElement.TryGetProperty("sha256", out var hashElement) && hashElement.ValueKind == JsonValueKind.String) - { - sha256 = hashElement.GetString(); - } - - if (documentElement.TryGetProperty("size", out var sizeElement) && sizeElement.ValueKind == JsonValueKind.Number && sizeElement.TryGetInt64(out var parsedSize)) - { - size = parsedSize; - } - } - - if (documentUri is null) - { - continue; - } - - builder.Add(new OracleCatalogEntry(id!, title!, documentUri, publishedAt, revision, sha256, size, products)); - } - - return builder.ToImmutable(); - } - - private ImmutableArray ParseSchedule(JsonElement root) - { - if (!root.TryGetProperty("schedule", out var scheduleElement) || scheduleElement.ValueKind is not JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var item in scheduleElement.EnumerateArray()) - { - var window = item.TryGetProperty("window", out var windowElement) && windowElement.ValueKind == JsonValueKind.String ? windowElement.GetString() : null; - if (string.IsNullOrWhiteSpace(window)) - { - continue; - } - - DateTimeOffset releaseDate = default; - if (item.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) - { - releaseDate = parsed; - } - - builder.Add(new OracleCpuRelease(window!, releaseDate)); - } - - return builder.ToImmutable(); - } - - private ImmutableArray MergeSchedule(ImmutableArray existing, string calendarPayload) - { - try - { - using var document = JsonDocument.Parse(calendarPayload); - var root = document.RootElement; - if (!root.TryGetProperty("cpuWindows", out var windowsElement) || windowsElement.ValueKind is not JsonValueKind.Array) - { - return existing; - } - - var builder = existing.ToBuilder(); - var known = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var item in builder) - { - known.Add(item.Window); - } - - foreach (var windowElement in windowsElement.EnumerateArray()) - { - var name = windowElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; - if (string.IsNullOrWhiteSpace(name)) - { - continue; - } - - if (!known.Add(name)) - { - continue; - } - - DateTimeOffset releaseDate = default; - if (windowElement.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) - { - releaseDate = parsed; - } - - builder.Add(new OracleCpuRelease(name!, releaseDate)); - } - - return builder.ToImmutable(); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, "Failed to parse Oracle CPU calendar payload; continuing with existing schedule data."); - return existing; - } - } - - private ImmutableArray ParseStringArray(JsonElement element) - { - if (element.ValueKind is not JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var item in element.EnumerateArray()) - { - if (item.ValueKind == JsonValueKind.String) - { - var value = item.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - builder.Add(value); - } - } - } - - return builder.ToImmutable(); - } - - private void PersistSnapshotIfNeeded(OracleConnectorOptions options, OracleCatalogMetadata metadata, DateTimeOffset fetchedAt) - { - if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return; - } - - try - { - var snapshot = new OracleCatalogSnapshot(metadata, fetchedAt); - var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); - _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); - _logger.LogDebug("Persisted Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to persist Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); - } - } - - private static string CreateCacheKey(OracleConnectorOptions options) - => $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}"; - - private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) - { - public bool IsExpired(DateTimeOffset now) - => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; - - public OracleCatalogResult ToResult(bool fromCache) - => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); - } - - private sealed record OracleCatalogSnapshot(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt); -} - -public sealed record OracleCatalogMetadata( - DateTimeOffset GeneratedAt, - ImmutableArray Entries, - ImmutableArray CpuSchedule); - -public sealed record OracleCatalogEntry( - string Id, - string Title, - Uri DocumentUri, - DateTimeOffset PublishedAt, - string? Revision, - string? Sha256, - long? Size, - ImmutableArray Products); - -public sealed record OracleCpuRelease(string Window, DateTimeOffset ReleaseDate); - -public sealed record OracleCatalogResult( - OracleCatalogMetadata Metadata, - DateTimeOffset FetchedAt, - bool FromCache, - bool FromOfflineSnapshot); +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; + +public sealed class OracleCatalogLoader +{ + public const string CachePrefix = "StellaOps.Excititor.Connectors.Oracle.CSAF.Catalog"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + public OracleCatalogLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IFileSystem fileSystem, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task LoadAsync(OracleConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + options.Validate(_fileSystem); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + CacheEntry? entry = null; + if (options.PreferOfflineSnapshot) + { + entry = LoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline snapshot was found or could be loaded."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) + ?? LoadFromOffline(options); + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Oracle CSAF catalog from network or offline snapshot."); + } + + var expiration = entry.MetadataCacheDuration == TimeSpan.Zero + ? (DateTimeOffset?)null + : _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); + + var cacheEntryOptions = new MemoryCacheEntryOptions(); + if (expiration.HasValue) + { + cacheEntryOptions.AbsoluteExpiration = expiration.Value; + } + + _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheEntryOptions); + return entry.ToResult(fromCache: false); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(OracleConnectorOptions options, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName); + using var response = await client.GetAsync(options.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var catalogPayload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + string? calendarPayload = null; + if (options.CpuCalendarUri is not null) + { + try + { + using var calendarResponse = await client.GetAsync(options.CpuCalendarUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + calendarResponse.EnsureSuccessStatusCode(); + calendarPayload = await calendarResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Oracle CPU calendar from {Uri}; continuing without schedule.", options.CpuCalendarUri); + } + } + + var metadata = ParseMetadata(catalogPayload, calendarPayload); + var fetchedAt = _timeProvider.GetUtcNow(); + var entry = new CacheEntry(metadata, fetchedAt, fetchedAt, options.MetadataCacheDuration, false); + + PersistSnapshotIfNeeded(options, metadata, fetchedAt); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Oracle CSAF catalog from {Uri}; attempting offline fallback if available.", options.CatalogUri); + return null; + } + } + + private CacheEntry? LoadFromOffline(OracleConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Oracle offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); + if (snapshot is null) + { + throw new InvalidOperationException("Offline snapshot payload was empty."); + } + + return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Oracle CSAF catalog from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private OracleCatalogMetadata ParseMetadata(string catalogPayload, string? calendarPayload) + { + if (string.IsNullOrWhiteSpace(catalogPayload)) + { + throw new InvalidOperationException("Oracle catalog payload was empty."); + } + + using var document = JsonDocument.Parse(catalogPayload); + var root = document.RootElement; + + var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) + ? generated + : _timeProvider.GetUtcNow(); + + var entries = ParseEntries(root); + var schedule = ParseSchedule(root); + + if (!string.IsNullOrWhiteSpace(calendarPayload)) + { + schedule = MergeSchedule(schedule, calendarPayload); + } + + return new OracleCatalogMetadata(generatedAt, entries, schedule); + } + + private ImmutableArray ParseEntries(JsonElement root) + { + if (!root.TryGetProperty("catalog", out var catalogElement) || catalogElement.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var entry in catalogElement.EnumerateArray()) + { + var id = entry.TryGetProperty("id", out var idElement) && idElement.ValueKind == JsonValueKind.String ? idElement.GetString() : null; + var title = entry.TryGetProperty("title", out var titleElement) && titleElement.ValueKind == JsonValueKind.String ? titleElement.GetString() : null; + if (string.IsNullOrWhiteSpace(id) || string.IsNullOrWhiteSpace(title)) + { + continue; + } + + DateTimeOffset publishedAt = default; + if (entry.TryGetProperty("published", out var publishedElement) && publishedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(publishedElement.GetString(), out var publishedParsed)) + { + publishedAt = publishedParsed; + } + + string? revision = null; + if (entry.TryGetProperty("revision", out var revisionElement) && revisionElement.ValueKind == JsonValueKind.String) + { + revision = revisionElement.GetString(); + } + + ImmutableArray products = ImmutableArray.Empty; + if (entry.TryGetProperty("products", out var productsElement)) + { + products = ParseStringArray(productsElement); + } + + Uri? documentUri = null; + string? sha256 = null; + long? size = null; + + if (entry.TryGetProperty("document", out var documentElement) && documentElement.ValueKind == JsonValueKind.Object) + { + if (documentElement.TryGetProperty("url", out var urlElement) && urlElement.ValueKind == JsonValueKind.String && Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var parsedUri)) + { + documentUri = parsedUri; + } + + if (documentElement.TryGetProperty("sha256", out var hashElement) && hashElement.ValueKind == JsonValueKind.String) + { + sha256 = hashElement.GetString(); + } + + if (documentElement.TryGetProperty("size", out var sizeElement) && sizeElement.ValueKind == JsonValueKind.Number && sizeElement.TryGetInt64(out var parsedSize)) + { + size = parsedSize; + } + } + + if (documentUri is null) + { + continue; + } + + builder.Add(new OracleCatalogEntry(id!, title!, documentUri, publishedAt, revision, sha256, size, products)); + } + + return builder.ToImmutable(); + } + + private ImmutableArray ParseSchedule(JsonElement root) + { + if (!root.TryGetProperty("schedule", out var scheduleElement) || scheduleElement.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in scheduleElement.EnumerateArray()) + { + var window = item.TryGetProperty("window", out var windowElement) && windowElement.ValueKind == JsonValueKind.String ? windowElement.GetString() : null; + if (string.IsNullOrWhiteSpace(window)) + { + continue; + } + + DateTimeOffset releaseDate = default; + if (item.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) + { + releaseDate = parsed; + } + + builder.Add(new OracleCpuRelease(window!, releaseDate)); + } + + return builder.ToImmutable(); + } + + private ImmutableArray MergeSchedule(ImmutableArray existing, string calendarPayload) + { + try + { + using var document = JsonDocument.Parse(calendarPayload); + var root = document.RootElement; + if (!root.TryGetProperty("cpuWindows", out var windowsElement) || windowsElement.ValueKind is not JsonValueKind.Array) + { + return existing; + } + + var builder = existing.ToBuilder(); + var known = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var item in builder) + { + known.Add(item.Window); + } + + foreach (var windowElement in windowsElement.EnumerateArray()) + { + var name = windowElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; + if (string.IsNullOrWhiteSpace(name)) + { + continue; + } + + if (!known.Add(name)) + { + continue; + } + + DateTimeOffset releaseDate = default; + if (windowElement.TryGetProperty("releaseDate", out var releaseElement) && releaseElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(releaseElement.GetString(), out var parsed)) + { + releaseDate = parsed; + } + + builder.Add(new OracleCpuRelease(name!, releaseDate)); + } + + return builder.ToImmutable(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to parse Oracle CPU calendar payload; continuing with existing schedule data."); + return existing; + } + } + + private ImmutableArray ParseStringArray(JsonElement element) + { + if (element.ValueKind is not JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in element.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) + { + var value = item.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + builder.Add(value); + } + } + } + + return builder.ToImmutable(); + } + + private void PersistSnapshotIfNeeded(OracleConnectorOptions options, OracleCatalogMetadata metadata, DateTimeOffset fetchedAt) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var snapshot = new OracleCatalogSnapshot(metadata, fetchedAt); + var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Oracle CSAF catalog snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private static string CreateCacheKey(OracleConnectorOptions options) + => $"{CachePrefix}:{options.CatalogUri}:{options.CpuCalendarUri}"; + + private sealed record CacheEntry(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) + { + public bool IsExpired(DateTimeOffset now) + => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; + + public OracleCatalogResult ToResult(bool fromCache) + => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); + } + + private sealed record OracleCatalogSnapshot(OracleCatalogMetadata Metadata, DateTimeOffset FetchedAt); +} + +public sealed record OracleCatalogMetadata( + DateTimeOffset GeneratedAt, + ImmutableArray Entries, + ImmutableArray CpuSchedule); + +public sealed record OracleCatalogEntry( + string Id, + string Title, + Uri DocumentUri, + DateTimeOffset PublishedAt, + string? Revision, + string? Sha256, + long? Size, + ImmutableArray Products); + +public sealed record OracleCpuRelease(string Window, DateTimeOffset ReleaseDate); + +public sealed record OracleCatalogResult( + OracleCatalogMetadata Metadata, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs new file mode 100644 index 00000000..c86c0a6e --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/OracleCsafConnector.cs @@ -0,0 +1,360 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Oracle.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.Oracle.CSAF; + +public sealed class OracleCsafConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:oracle", + kind: VexProviderKind.Vendor, + displayName: "Oracle CSAF") + { + Tags = ImmutableArray.Create("oracle", "csaf", "cpu"), + }; + + private readonly OracleCatalogLoader _catalogLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IEnumerable> _validators; + + private OracleConnectorOptions? _options; + private OracleCatalogResult? _catalog; + + public OracleCsafConnector( + OracleCatalogLoader catalogLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + IEnumerable> validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary + { + ["catalogEntryCount"] = _catalog.Metadata.Entries.Length, + ["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length, + ["fromOffline"] = _catalog.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + _catalog ??= await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + + var entries = _catalog.Metadata.Entries + .OrderBy(static entry => entry.PublishedAt == default ? DateTimeOffset.MinValue : entry.PublishedAt) + .ToImmutableArray(); + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var since = ResolveSince(context.Since, state?.LastUpdated); + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var digestList = new List(knownDigests); + var latestPublished = state?.LastUpdated ?? since ?? DateTimeOffset.MinValue; + var stateChanged = false; + + var client = _httpClientFactory.CreateClient(OracleConnectorOptions.HttpClientName); + + LogConnectorEvent(LogLevel.Information, "fetch.begin", "Starting Oracle CSAF catalogue iteration.", new Dictionary + { + ["since"] = since?.ToString("O"), + ["entryCount"] = entries.Length, + }); + + foreach (var entry in entries) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (ShouldSkipEntry(entry, since)) + { + continue; + } + + var expectedDigest = NormalizeDigest(entry.Sha256); + if (expectedDigest is not null && digestSet.Contains(expectedDigest)) + { + latestPublished = UpdateLatest(latestPublished, entry.PublishedAt); + LogConnectorEvent(LogLevel.Debug, "fetch.skip.cached", "Skipping Oracle CSAF entry because digest already processed.", new Dictionary + { + ["entryId"] = entry.Id, + ["digest"] = expectedDigest, + }); + continue; + } + + var rawDocument = await DownloadEntryAsync(client, entry, cancellationToken).ConfigureAwait(false); + if (rawDocument is null) + { + continue; + } + + if (expectedDigest is not null && !string.Equals(rawDocument.Digest, expectedDigest, StringComparison.OrdinalIgnoreCase)) + { + LogConnectorEvent(LogLevel.Warning, "fetch.checksum_mismatch", "Oracle CSAF document checksum mismatch; document skipped.", new Dictionary + { + ["entryId"] = entry.Id, + ["expected"] = expectedDigest, + ["actual"] = rawDocument.Digest, + ["documentUri"] = entry.DocumentUri.ToString(), + }); + continue; + } + + if (!digestSet.Add(rawDocument.Digest)) + { + LogConnectorEvent(LogLevel.Debug, "fetch.skip.duplicate", "Oracle CSAF document digest already ingested.", new Dictionary + { + ["entryId"] = entry.Id, + ["digest"] = rawDocument.Digest, + }); + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + latestPublished = UpdateLatest(latestPublished, entry.PublishedAt); + + LogConnectorEvent(LogLevel.Information, "fetch.document_ingested", "Oracle CSAF document stored.", new Dictionary + { + ["entryId"] = entry.Id, + ["digest"] = rawDocument.Digest, + ["documentUri"] = entry.DocumentUri.ToString(), + ["publishedAt"] = entry.PublishedAt.ToString("O"), + }); + + yield return rawDocument; + + if (_options.RequestDelay > TimeSpan.Zero) + { + await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false); + } + } + + if (stateChanged) + { + var baseState = state ?? new VexConnectorState( + Descriptor.Id, + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var newState = baseState with + { + LastUpdated = latestPublished == DateTimeOffset.MinValue ? baseState.LastUpdated : latestPublished, + DocumentDigests = digestList.ToImmutableArray(), + }; + + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + + var ingestedCount = digestList.Count - knownDigests.Length; + LogConnectorEvent(LogLevel.Information, "fetch.complete", "Oracle CSAF fetch completed.", new Dictionary + { + ["stateChanged"] = stateChanged, + ["documentsProcessed"] = ingestedCount, + ["latestPublished"] = latestPublished == DateTimeOffset.MinValue ? null : latestPublished.ToString("O"), + }); + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers."); + + public OracleCatalogResult? GetCachedCatalog() => _catalog; + + private static DateTimeOffset? ResolveSince(DateTimeOffset? contextSince, DateTimeOffset? stateSince) + { + if (contextSince is null) + { + return stateSince; + } + + if (stateSince is null) + { + return contextSince; + } + + return stateSince > contextSince ? stateSince : contextSince; + } + + private static bool ShouldSkipEntry(OracleCatalogEntry entry, DateTimeOffset? since) + { + if (since is null) + { + return false; + } + + if (entry.PublishedAt == default) + { + return false; + } + + return entry.PublishedAt <= since; + } + + private async Task DownloadEntryAsync(HttpClient client, OracleCatalogEntry entry, CancellationToken cancellationToken) + { + if (entry.DocumentUri is null) + { + LogConnectorEvent(LogLevel.Warning, "fetch.skip.missing_uri", "Oracle CSAF entry missing document URI; skipping.", new Dictionary + { + ["entryId"] = entry.Id, + }); + return null; + } + + var payload = await DownloadWithRetryAsync(client, entry.DocumentUri, cancellationToken).ConfigureAwait(false); + if (payload is null) + { + return null; + } + + var metadata = BuildMetadata(builder => + { + builder.Add("oracle.csaf.entryId", entry.Id); + builder.Add("oracle.csaf.title", entry.Title); + builder.Add("oracle.csaf.revision", entry.Revision); + if (entry.PublishedAt != default) + { + builder.Add("oracle.csaf.published", entry.PublishedAt.ToString("O")); + } + + builder.Add("oracle.csaf.sha256", NormalizeDigest(entry.Sha256)); + builder.Add("oracle.csaf.size", entry.Size?.ToString(CultureInfo.InvariantCulture)); + if (!entry.Products.IsDefaultOrEmpty) + { + builder.Add("oracle.csaf.products", string.Join(",", entry.Products)); + } + }); + + return CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload.AsMemory(), metadata); + } + + private async Task DownloadWithRetryAsync(HttpClient client, Uri uri, CancellationToken cancellationToken) + { + const int maxAttempts = 3; + var delay = TimeSpan.FromSeconds(1); + + for (var attempt = 1; attempt <= maxAttempts; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + using var response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + if (IsTransient(response.StatusCode) && attempt < maxAttempts) + { + LogConnectorEvent(LogLevel.Warning, "fetch.retry.status", "Oracle CSAF document request returned transient status; retrying.", new Dictionary + { + ["status"] = (int)response.StatusCode, + ["attempt"] = attempt, + ["uri"] = uri.ToString(), + }); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay = delay + delay; + continue; + } + + response.EnsureSuccessStatusCode(); + } + + var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + return bytes; + } + catch (Exception ex) when (IsTransient(ex) && attempt < maxAttempts) + { + LogConnectorEvent(LogLevel.Warning, "fetch.retry.exception", "Oracle CSAF document request failed; retrying.", new Dictionary + { + ["attempt"] = attempt, + ["uri"] = uri.ToString(), + ["exception"] = ex.GetType().Name, + }); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + delay = delay + delay; + } + } + + LogConnectorEvent(LogLevel.Error, "fetch.failed", "Oracle CSAF document could not be retrieved after retries.", new Dictionary + { + ["uri"] = uri.ToString(), + }); + + return null; + } + + private static bool IsTransient(Exception exception) + => exception is HttpRequestException or IOException or TaskCanceledException; + + private static bool IsTransient(HttpStatusCode statusCode) + { + var status = (int)statusCode; + return status is >= 500 or 408 or 429; + } + + private static string? NormalizeDigest(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var trimmed = digest.Trim(); + if (!trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + trimmed = "sha256:" + trimmed; + } + + return trimmed.ToLowerInvariant(); + } + + private static DateTimeOffset UpdateLatest(DateTimeOffset current, DateTimeOffset published) + { + if (published == default) + { + return current; + } + + return published > current ? published : current; + } +} diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj new file mode 100644 index 00000000..4a15c1f6 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/StellaOps.Excititor.Connectors.Oracle.CSAF.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md new file mode 100644 index 00000000..d7a04609 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Oracle.CSAF/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-ORACLE-01-001 – Oracle CSAF catalogue discovery|Team Excititor Connectors – Oracle|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-19)** – Implemented cached Oracle CSAF catalog loader with CPU calendar merge, offline snapshot ingest/persist, options validation + DI wiring, and regression tests; prerequisite EXCITITOR-CONN-ABS-01-001 verified DONE per Sprint 5 log (2025-10-19).| +|EXCITITOR-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline|Team Excititor Connectors – Oracle|EXCITITOR-CONN-ORACLE-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-19)** – Added Oracle CSAF fetch loop with retry/backoff, checksum validation, resume-aware state persistence, digest dedupe, configurable throttling, and raw storage wiring; regression tests cover new ingestion and mismatch handling.| +|EXCITITOR-CONN-ORACLE-01-003 – Trust metadata + provenance|Team Excititor Connectors – Oracle|EXCITITOR-CONN-ORACLE-01-002, EXCITITOR-POLICY-01-001|TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting.| diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs index 22765fd2..46ecd46e 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Connectors/RedHatCsafConnectorTests.cs @@ -1,277 +1,277 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.Connectors; - -public sealed class RedHatCsafConnectorTests -{ - private static readonly VexConnectorDescriptor Descriptor = new("vexer:redhat", VexProviderKind.Distro, "Red Hat CSAF"); - - [Fact] - public async Task FetchAsync_EmitsDocumentsAfterSince() - { - var metadata = """ - { - "metadata": { - "provider": { "name": "Red Hat Product Security" } - }, - "distributions": [ - { "directory": "https://example.com/security/data/csaf/v2/advisories/" } - ], - "rolie": { - "feeds": [ - { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } - ] - } - } - """; - - var feed = """ - - - urn:redhat:1 - 2025-10-16T10:00:00Z - - - - urn:redhat:2 - 2025-10-17T10:00:00Z - - - - """; - - var handler = TestHttpMessageHandler.Create( - request => Response(HttpStatusCode.OK, metadata, "application/json"), - request => Response(HttpStatusCode.OK, feed, "application/atom+xml"), - request => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://example.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var options = Options.Create(new RedHatConnectorOptions()); - var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); - var stateRepository = new InMemoryConnectorStateRepository(); - var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); - - var rawSink = new CapturingRawSink(); - var context = new VexConnectorContext( - new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero), - VexConnectorSettings.Empty, - rawSink, - new NoopSignatureVerifier(), - new NoopNormalizerRouter(), - new ServiceCollection().BuildServiceProvider()); - - var results = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - results.Add(document); - } - - Assert.Single(results); - Assert.Single(rawSink.Documents); - Assert.Equal("https://example.com/doc2.json", results[0].SourceUri.ToString()); - Assert.Equal("https://example.com/doc2.json", rawSink.Documents[0].SourceUri.ToString()); - Assert.Equal(3, handler.CallCount); - stateRepository.State.Should().NotBeNull(); - stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 17, 10, 0, 0, TimeSpan.Zero)); - stateRepository.State.DocumentDigests.Should().HaveCount(1); - } - - [Fact] - public async Task FetchAsync_UsesStateToSkipDuplicateDocuments() - { - var metadata = """ - { - "metadata": { - "provider": { "name": "Red Hat Product Security" } - }, - "distributions": [ - { "directory": "https://example.com/security/data/csaf/v2/advisories/" } - ], - "rolie": { - "feeds": [ - { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } - ] - } - } - """; - - var feed = """ - - - urn:redhat:1 - 2025-10-17T10:00:00Z - - - - """; - - var handler1 = TestHttpMessageHandler.Create( - _ => Response(HttpStatusCode.OK, metadata, "application/json"), - _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), - _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); - - var stateRepository = new InMemoryConnectorStateRepository(); - await ExecuteFetchAsync(handler1, stateRepository); - - stateRepository.State.Should().NotBeNull(); - var previousState = stateRepository.State!; - - var handler2 = TestHttpMessageHandler.Create( - _ => Response(HttpStatusCode.OK, metadata, "application/json"), - _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), - _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); - - var (results, rawSink) = await ExecuteFetchAsync(handler2, stateRepository); - - results.Should().BeEmpty(); - rawSink.Documents.Should().BeEmpty(); - stateRepository.State!.DocumentDigests.Should().Equal(previousState.DocumentDigests); - } - - private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) - => new(statusCode) - { - Content = new StringContent(content, Encoding.UTF8, contentType), - }; - - private sealed class CapturingRawSink : IVexRawDocumentSink - { - public List Documents { get; } = new(); - - public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - Documents.Add(document); - return ValueTask.CompletedTask; - } - } - - private sealed class NoopSignatureVerifier : IVexSignatureVerifier - { - public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class NoopNormalizerRouter : IVexNormalizerRouter - { - public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Queue> _responders; - - private TestHttpMessageHandler(IEnumerable> responders) - { - _responders = new Queue>(responders); - } - - public int CallCount { get; private set; } - - public static TestHttpMessageHandler Create(params Func[] responders) - => new(responders); - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - CallCount++; - if (_responders.Count == 0) - { - throw new InvalidOperationException("No responder configured for request."); - } - - var responder = _responders.Count > 1 - ? _responders.Dequeue() - : _responders.Peek(); - - var response = responder(request); - response.RequestMessage = request; - return Task.FromResult(response); - } - } - - private static async Task<(List Documents, CapturingRawSink Sink)> ExecuteFetchAsync( - TestHttpMessageHandler handler, - InMemoryConnectorStateRepository stateRepository) - { - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://example.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var options = Options.Create(new RedHatConnectorOptions()); - var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); - var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); - - var rawSink = new CapturingRawSink(); - var context = new VexConnectorContext( - null, - VexConnectorSettings.Empty, - rawSink, - new NoopSignatureVerifier(), - new NoopNormalizerRouter(), - new ServiceCollection().BuildServiceProvider()); - - var documents = new List(); - await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) - { - documents.Add(document); - } - - return (documents, rawSink); - } - - private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository - { - public VexConnectorState? State { get; private set; } - - public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - { - if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) - { - return ValueTask.FromResult(State); - } - - return ValueTask.FromResult(null); - } - - public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - State = state; - return ValueTask.CompletedTask; - } - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Connectors; + +public sealed class RedHatCsafConnectorTests +{ + private static readonly VexConnectorDescriptor Descriptor = new("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF"); + + [Fact] + public async Task FetchAsync_EmitsDocumentsAfterSince() + { + var metadata = """ + { + "metadata": { + "provider": { "name": "Red Hat Product Security" } + }, + "distributions": [ + { "directory": "https://example.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + var feed = """ + + + urn:redhat:1 + 2025-10-16T10:00:00Z + + + + urn:redhat:2 + 2025-10-17T10:00:00Z + + + + """; + + var handler = TestHttpMessageHandler.Create( + request => Response(HttpStatusCode.OK, metadata, "application/json"), + request => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + request => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new RedHatConnectorOptions()); + var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); + + var rawSink = new CapturingRawSink(); + var context = new VexConnectorContext( + new DateTimeOffset(2025, 10, 16, 12, 0, 0, TimeSpan.Zero), + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider()); + + var results = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + results.Add(document); + } + + Assert.Single(results); + Assert.Single(rawSink.Documents); + Assert.Equal("https://example.com/doc2.json", results[0].SourceUri.ToString()); + Assert.Equal("https://example.com/doc2.json", rawSink.Documents[0].SourceUri.ToString()); + Assert.Equal(3, handler.CallCount); + stateRepository.State.Should().NotBeNull(); + stateRepository.State!.LastUpdated.Should().Be(new DateTimeOffset(2025, 10, 17, 10, 0, 0, TimeSpan.Zero)); + stateRepository.State.DocumentDigests.Should().HaveCount(1); + } + + [Fact] + public async Task FetchAsync_UsesStateToSkipDuplicateDocuments() + { + var metadata = """ + { + "metadata": { + "provider": { "name": "Red Hat Product Security" } + }, + "distributions": [ + { "directory": "https://example.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://example.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + var feed = """ + + + urn:redhat:1 + 2025-10-17T10:00:00Z + + + + """; + + var handler1 = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, metadata, "application/json"), + _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var stateRepository = new InMemoryConnectorStateRepository(); + await ExecuteFetchAsync(handler1, stateRepository); + + stateRepository.State.Should().NotBeNull(); + var previousState = stateRepository.State!; + + var handler2 = TestHttpMessageHandler.Create( + _ => Response(HttpStatusCode.OK, metadata, "application/json"), + _ => Response(HttpStatusCode.OK, feed, "application/atom+xml"), + _ => Response(HttpStatusCode.OK, "{ \"csaf\": 1 }", "application/json")); + + var (results, rawSink) = await ExecuteFetchAsync(handler2, stateRepository); + + results.Should().BeEmpty(); + rawSink.Documents.Should().BeEmpty(); + stateRepository.State!.DocumentDigests.Should().Equal(previousState.DocumentDigests); + } + + private static HttpResponseMessage Response(HttpStatusCode statusCode, string content, string contentType) + => new(statusCode) + { + Content = new StringContent(content, Encoding.UTF8, contentType), + }; + + private sealed class CapturingRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public int CallCount { get; private set; } + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + CallCount++; + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for request."); + } + + var responder = _responders.Count > 1 + ? _responders.Dequeue() + : _responders.Peek(); + + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } + + private static async Task<(List Documents, CapturingRawSink Sink)> ExecuteFetchAsync( + TestHttpMessageHandler handler, + InMemoryConnectorStateRepository stateRepository) + { + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://example.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var options = Options.Create(new RedHatConnectorOptions()); + var metadataLoader = new RedHatProviderMetadataLoader(factory, cache, options, NullLogger.Instance); + var connector = new RedHatCsafConnector(Descriptor, metadataLoader, factory, stateRepository, NullLogger.Instance, TimeProvider.System); + + var rawSink = new CapturingRawSink(); + var context = new VexConnectorContext( + null, + VexConnectorSettings.Empty, + rawSink, + new NoopSignatureVerifier(), + new NoopNormalizerRouter(), + new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var document in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(document); + } + + return (documents, rawSink); + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? State { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + { + if (State is not null && string.Equals(State.ConnectorId, connectorId, StringComparison.OrdinalIgnoreCase)) + { + return ValueTask.FromResult(State); + } + + return ValueTask.FromResult(null); + } + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + State = state; + return ValueTask.CompletedTask; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs index 95d1dec9..d33b882f 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/Metadata/RedHatProviderMetadataLoaderTests.cs @@ -1,235 +1,235 @@ -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; -using System.IO.Abstractions.TestingHelpers; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.Metadata; - -public sealed class RedHatProviderMetadataLoaderTests -{ - private const string SampleJson = """ - { - "metadata": { - "provider": { - "name": "Red Hat Product Security" - } - }, - "distributions": [ - { "directory": "https://access.redhat.com/security/data/csaf/v2/advisories/" } - ], - "rolie": { - "feeds": [ - { "url": "https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom" } - ] - } - } - """; - - [Fact] - public async Task LoadAsync_FetchesMetadataAndCaches() - { - var handler = TestHttpMessageHandler.RespondWith(_ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); - return response; - }); - - var httpClient = new HttpClient(handler) - { - BaseAddress = new Uri("https://access.redhat.com/"), - }; - - var factory = new SingleClientHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = new RedHatConnectorOptions - { - MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), - OfflineSnapshotPath = fileSystem.Path.Combine("/offline", "redhat-provider.json"), - CosignIssuer = "https://sigstore.dev/redhat", - CosignIdentityPattern = "^spiffe://redhat/.+$", - }; - options.PgpFingerprints.Add("A1B2C3D4E5F6"); - options.Validate(fileSystem); - - var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); - - var result = await loader.LoadAsync(CancellationToken.None); - - Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); - Assert.False(result.FromCache); - Assert.False(result.FromOfflineSnapshot); - Assert.Single(result.Provider.BaseUris); - Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/", result.Provider.BaseUris[0].ToString()); - Assert.Equal("https://access.redhat.com/.well-known/csaf/provider-metadata.json", result.Provider.Discovery.WellKnownMetadata?.ToString()); - Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom", result.Provider.Discovery.RolIeService?.ToString()); - Assert.Equal(1.0, result.Provider.Trust.Weight); - Assert.NotNull(result.Provider.Trust.Cosign); - Assert.Equal("https://sigstore.dev/redhat", result.Provider.Trust.Cosign!.Issuer); - Assert.Equal("^spiffe://redhat/.+$", result.Provider.Trust.Cosign.IdentityPattern); - Assert.Contains("A1B2C3D4E5F6", result.Provider.Trust.PgpFingerprints); - Assert.True(fileSystem.FileExists(options.OfflineSnapshotPath)); - Assert.Equal(1, handler.CallCount); - - var second = await loader.LoadAsync(CancellationToken.None); - Assert.True(second.FromCache); - Assert.False(second.FromOfflineSnapshot); - Assert.Equal(1, handler.CallCount); - } - - [Fact] - public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() - { - var handler = TestHttpMessageHandler.RespondWith(_ => throw new InvalidOperationException("HTTP should not be called")); - - var httpClient = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - - var fileSystem = new MockFileSystem(new Dictionary - { - ["/snapshots/redhat.json"] = new MockFileData(SampleJson), - }); - - var options = new RedHatConnectorOptions - { - MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), - OfflineSnapshotPath = "/snapshots/redhat.json", - PreferOfflineSnapshot = true, - PersistOfflineSnapshot = false, - }; - options.Validate(fileSystem); - - var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); - - var result = await loader.LoadAsync(CancellationToken.None); - - Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); - Assert.False(result.FromCache); - Assert.True(result.FromOfflineSnapshot); - Assert.Equal(0, handler.CallCount); - - var second = await loader.LoadAsync(CancellationToken.None); - Assert.True(second.FromCache); - Assert.True(second.FromOfflineSnapshot); - Assert.Equal(0, handler.CallCount); - } - - [Fact] - public async Task LoadAsync_UsesETagForConditionalRequest() - { - var handler = TestHttpMessageHandler.Create( - _ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); - return response; - }, - request => - { - Assert.Contains(request.Headers.IfNoneMatch, etag => etag.Tag == "\"abc\""); - return new HttpResponseMessage(HttpStatusCode.NotModified); - }); - - var httpClient = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(httpClient); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = new RedHatConnectorOptions - { - MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), - OfflineSnapshotPath = "/offline/redhat.json", - MetadataCacheDuration = TimeSpan.FromMinutes(1), - }; - options.Validate(fileSystem); - - var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); - - var first = await loader.LoadAsync(CancellationToken.None); - Assert.False(first.FromCache); - Assert.False(first.FromOfflineSnapshot); - - Assert.True(cache.TryGetValue(RedHatProviderMetadataLoader.CacheKey, out var entryObj)); - Assert.NotNull(entryObj); - - var entryType = entryObj!.GetType(); - var provider = entryType.GetProperty("Provider")!.GetValue(entryObj); - var fetchedAt = entryType.GetProperty("FetchedAt")!.GetValue(entryObj); - var etag = entryType.GetProperty("ETag")!.GetValue(entryObj); - var fromOffline = entryType.GetProperty("FromOffline")!.GetValue(entryObj); - - var expiredEntry = Activator.CreateInstance(entryType, provider, fetchedAt, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1), etag, fromOffline); - cache.Set(RedHatProviderMetadataLoader.CacheKey, expiredEntry!, new MemoryCacheEntryOptions - { - AbsoluteExpiration = DateTimeOffset.UtcNow + TimeSpan.FromMinutes(1), - }); - - var second = await loader.LoadAsync(CancellationToken.None); - - var third = await loader.LoadAsync(CancellationToken.None); - Assert.True(third.FromCache); - - Assert.Equal(2, handler.CallCount); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Queue> _responders; - - private TestHttpMessageHandler(IEnumerable> responders) - { - _responders = new Queue>(responders); - } - - public int CallCount { get; private set; } - - public static TestHttpMessageHandler RespondWith(Func responder) - => new(new[] { responder }); - - public static TestHttpMessageHandler Create(params Func[] responders) - => new(responders); - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - CallCount++; - if (_responders.Count == 0) - { - throw new InvalidOperationException("No responder configured for request."); - } - - var responder = _responders.Count > 1 - ? _responders.Dequeue() - : _responders.Peek(); - - var response = responder(request); - response.RequestMessage = request; - return Task.FromResult(response); - } - } -} +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.Metadata; + +public sealed class RedHatProviderMetadataLoaderTests +{ + private const string SampleJson = """ + { + "metadata": { + "provider": { + "name": "Red Hat Product Security" + } + }, + "distributions": [ + { "directory": "https://access.redhat.com/security/data/csaf/v2/advisories/" } + ], + "rolie": { + "feeds": [ + { "url": "https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom" } + ] + } + } + """; + + [Fact] + public async Task LoadAsync_FetchesMetadataAndCaches() + { + var handler = TestHttpMessageHandler.RespondWith(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }); + + var httpClient = new HttpClient(handler) + { + BaseAddress = new Uri("https://access.redhat.com/"), + }; + + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = fileSystem.Path.Combine("/offline", "redhat-provider.json"), + CosignIssuer = "https://sigstore.dev/redhat", + CosignIdentityPattern = "^spiffe://redhat/.+$", + }; + options.PgpFingerprints.Add("A1B2C3D4E5F6"); + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); + Assert.False(result.FromCache); + Assert.False(result.FromOfflineSnapshot); + Assert.Single(result.Provider.BaseUris); + Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/", result.Provider.BaseUris[0].ToString()); + Assert.Equal("https://access.redhat.com/.well-known/csaf/provider-metadata.json", result.Provider.Discovery.WellKnownMetadata?.ToString()); + Assert.Equal("https://access.redhat.com/security/data/csaf/v2/advisories/rolie/feed.atom", result.Provider.Discovery.RolIeService?.ToString()); + Assert.Equal(1.0, result.Provider.Trust.Weight); + Assert.NotNull(result.Provider.Trust.Cosign); + Assert.Equal("https://sigstore.dev/redhat", result.Provider.Trust.Cosign!.Issuer); + Assert.Equal("^spiffe://redhat/.+$", result.Provider.Trust.Cosign.IdentityPattern); + Assert.Contains("A1B2C3D4E5F6", result.Provider.Trust.PgpFingerprints); + Assert.True(fileSystem.FileExists(options.OfflineSnapshotPath)); + Assert.Equal(1, handler.CallCount); + + var second = await loader.LoadAsync(CancellationToken.None); + Assert.True(second.FromCache); + Assert.False(second.FromOfflineSnapshot); + Assert.Equal(1, handler.CallCount); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new InvalidOperationException("HTTP should not be called")); + + var httpClient = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + + var fileSystem = new MockFileSystem(new Dictionary + { + ["/snapshots/redhat.json"] = new MockFileData(SampleJson), + }); + + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = "/snapshots/redhat.json", + PreferOfflineSnapshot = true, + PersistOfflineSnapshot = false, + }; + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var result = await loader.LoadAsync(CancellationToken.None); + + Assert.Equal("Red Hat Product Security", result.Provider.DisplayName); + Assert.False(result.FromCache); + Assert.True(result.FromOfflineSnapshot); + Assert.Equal(0, handler.CallCount); + + var second = await loader.LoadAsync(CancellationToken.None); + Assert.True(second.FromCache); + Assert.True(second.FromOfflineSnapshot); + Assert.Equal(0, handler.CallCount); + } + + [Fact] + public async Task LoadAsync_UsesETagForConditionalRequest() + { + var handler = TestHttpMessageHandler.Create( + _ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleJson, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }, + request => + { + Assert.Contains(request.Headers.IfNoneMatch, etag => etag.Tag == "\"abc\""); + return new HttpResponseMessage(HttpStatusCode.NotModified); + }); + + var httpClient = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RedHatConnectorOptions + { + MetadataUri = new Uri("https://access.redhat.com/.well-known/csaf/provider-metadata.json"), + OfflineSnapshotPath = "/offline/redhat.json", + MetadataCacheDuration = TimeSpan.FromMinutes(1), + }; + options.Validate(fileSystem); + + var loader = new RedHatProviderMetadataLoader(factory, cache, Options.Create(options), NullLogger.Instance, fileSystem); + + var first = await loader.LoadAsync(CancellationToken.None); + Assert.False(first.FromCache); + Assert.False(first.FromOfflineSnapshot); + + Assert.True(cache.TryGetValue(RedHatProviderMetadataLoader.CacheKey, out var entryObj)); + Assert.NotNull(entryObj); + + var entryType = entryObj!.GetType(); + var provider = entryType.GetProperty("Provider")!.GetValue(entryObj); + var fetchedAt = entryType.GetProperty("FetchedAt")!.GetValue(entryObj); + var etag = entryType.GetProperty("ETag")!.GetValue(entryObj); + var fromOffline = entryType.GetProperty("FromOffline")!.GetValue(entryObj); + + var expiredEntry = Activator.CreateInstance(entryType, provider, fetchedAt, DateTimeOffset.UtcNow - TimeSpan.FromSeconds(1), etag, fromOffline); + cache.Set(RedHatProviderMetadataLoader.CacheKey, expiredEntry!, new MemoryCacheEntryOptions + { + AbsoluteExpiration = DateTimeOffset.UtcNow + TimeSpan.FromMinutes(1), + }); + + var second = await loader.LoadAsync(CancellationToken.None); + + var third = await loader.LoadAsync(CancellationToken.None); + Assert.True(third.FromCache); + + Assert.Equal(2, handler.CallCount); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responders; + + private TestHttpMessageHandler(IEnumerable> responders) + { + _responders = new Queue>(responders); + } + + public int CallCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responder) + => new(new[] { responder }); + + public static TestHttpMessageHandler Create(params Func[] responders) + => new(responders); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + CallCount++; + if (_responders.Count == 0) + { + throw new InvalidOperationException("No responder configured for request."); + } + + var responder = _responders.Count > 1 + ? _responders.Dequeue() + : _responders.Peek(); + + var response = responder(request); + response.RequestMessage = request; + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.csproj similarity index 66% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.csproj index 07cb02f5..4fa46301 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests/StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/AGENTS.md similarity index 89% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/AGENTS.md index 5c2528d0..648bb1e1 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/AGENTS.md @@ -1,25 +1,25 @@ -# AGENTS -## Role -Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization. -## Scope -- Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches. -- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots. -- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage. -- Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents. -- Emitting structured telemetry and resume markers for incremental pulls. -- Supplying Red Hat-specific trust overrides and provenance hints to normalization. -## Participants -- Worker schedules pulls using this connector; WebService triggers ad-hoc runs. -- CSAF normalizer consumes fetched documents to produce claims. -- Policy/consensus rely on Red Hat trust metadata captured here. -## Interfaces & contracts -- Implements `IVexConnector` with Red Hat-specific options (parallelism, token auth if configured). -- Uses abstractions from `StellaOps.Vexer.Connectors.Abstractions` for HTTP/resume helpers. -## In/Out of scope -In: data acquisition, HTTP retries, raw document persistence, provider metadata population. -Out: normalization, storage internals, attestation, general connector abstractions (covered elsewhere). -## Observability & security expectations -- Log provider metadata URL, revision ids, fetch durations; redact tokens. -- Emit counters for documents fetched, skipped (304), quarantined. -## Tests -- Connector harness tests (mock HTTP) and resume regression cases will live in `../StellaOps.Vexer.Connectors.RedHat.CSAF.Tests`. +# AGENTS +## Role +Connector for Red Hat CSAF VEX feeds, fetching provider metadata, CSAF documents, and projecting them into raw storage for normalization. +## Scope +- Discovery via `/.well-known/csaf/provider-metadata.json`, scheduling windows, and ETag-aware HTTP fetches. +- `RedHatProviderMetadataLoader` handles `.well-known` metadata with caching, schema validation, and offline snapshots. +- `RedHatCsafConnector` consumes ROLIE feeds to fetch incremental CSAF documents, honours `context.Since`, and streams raw advisories to storage. +- Mapping Red Hat CSAF specifics (product tree aliases, RHSA identifiers, revision history) into raw documents. +- Emitting structured telemetry and resume markers for incremental pulls. +- Supplying Red Hat-specific trust overrides and provenance hints to normalization. +## Participants +- Worker schedules pulls using this connector; WebService triggers ad-hoc runs. +- CSAF normalizer consumes fetched documents to produce claims. +- Policy/consensus rely on Red Hat trust metadata captured here. +## Interfaces & contracts +- Implements `IVexConnector` with Red Hat-specific options (parallelism, token auth if configured). +- Uses abstractions from `StellaOps.Excititor.Connectors.Abstractions` for HTTP/resume helpers. +## In/Out of scope +In: data acquisition, HTTP retries, raw document persistence, provider metadata population. +Out: normalization, storage internals, attestation, general connector abstractions (covered elsewhere). +## Observability & security expectations +- Log provider metadata URL, revision ids, fetch durations; redact tokens. +- Emit counters for documents fetched, skipped (304), quarantined. +## Tests +- Connector harness tests (mock HTTP) and resume regression cases will live in `../StellaOps.Excititor.Connectors.RedHat.CSAF.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs similarity index 93% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs index 9d9bf1ab..eb694a74 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/Configuration/RedHatConnectorOptions.cs @@ -1,104 +1,104 @@ -using System.Collections.Generic; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; - -public sealed class RedHatConnectorOptions -{ - public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json"); - - /// - /// HTTP client name registered for the connector. - /// - public const string HttpClientName = "vexer.connector.redhat"; - - /// - /// URI of the CSAF provider metadata document. - /// - public Uri MetadataUri { get; set; } = DefaultMetadataUri; - - /// - /// Duration to cache loaded metadata before refreshing. - /// - public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1); - - /// - /// Optional file path used to store or source offline metadata snapshots. - /// - public string? OfflineSnapshotPath { get; set; } - - /// - /// When true, the loader prefers the offline snapshot without attempting a network fetch. - /// - public bool PreferOfflineSnapshot { get; set; } - - /// - /// Enables writing fresh metadata responses to . - /// - public bool PersistOfflineSnapshot { get; set; } = true; - - /// - /// Explicit trust weight override applied to the provider entry. - /// - public double TrustWeight { get; set; } = 1.0; - - /// - /// Sigstore/Cosign issuer used to verify CSAF signatures, if published. - /// - public string? CosignIssuer { get; set; } = "https://access.redhat.com"; - - /// - /// Identity pattern matched against the Cosign certificate subject. - /// - public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$"; - - /// - /// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts. - /// - public IList PgpFingerprints { get; } = new List(); - - public void Validate(IFileSystem? fileSystem = null) - { - if (MetadataUri is null || !MetadataUri.IsAbsoluteUri) - { - throw new InvalidOperationException("Metadata URI must be absolute."); - } - - if (MetadataUri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS."); - } - - if (MetadataCacheDuration <= TimeSpan.Zero) - { - throw new InvalidOperationException("Metadata cache duration must be positive."); - } - - if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - var fs = fileSystem ?? new FileSystem(); - var directory = Path.GetDirectoryName(OfflineSnapshotPath); - if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) - { - fs.Directory.CreateDirectory(directory); - } - } - - if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0) - { - TrustWeight = 1.0; - } - else if (TrustWeight > 1.0) - { - TrustWeight = 1.0; - } - - if (CosignIssuer is not null) - { - if (string.IsNullOrWhiteSpace(CosignIdentityPattern)) - { - throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); - } - } - } -} +using System.Collections.Generic; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; + +public sealed class RedHatConnectorOptions +{ + public static readonly Uri DefaultMetadataUri = new("https://access.redhat.com/.well-known/csaf/provider-metadata.json"); + + /// + /// HTTP client name registered for the connector. + /// + public const string HttpClientName = "excititor.connector.redhat"; + + /// + /// URI of the CSAF provider metadata document. + /// + public Uri MetadataUri { get; set; } = DefaultMetadataUri; + + /// + /// Duration to cache loaded metadata before refreshing. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(1); + + /// + /// Optional file path used to store or source offline metadata snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + + /// + /// When true, the loader prefers the offline snapshot without attempting a network fetch. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// Enables writing fresh metadata responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Explicit trust weight override applied to the provider entry. + /// + public double TrustWeight { get; set; } = 1.0; + + /// + /// Sigstore/Cosign issuer used to verify CSAF signatures, if published. + /// + public string? CosignIssuer { get; set; } = "https://access.redhat.com"; + + /// + /// Identity pattern matched against the Cosign certificate subject. + /// + public string? CosignIdentityPattern { get; set; } = "^https://access\\.redhat\\.com/.+$"; + + /// + /// Optional list of PGP fingerprints recognised for Red Hat CSAF artifacts. + /// + public IList PgpFingerprints { get; } = new List(); + + public void Validate(IFileSystem? fileSystem = null) + { + if (MetadataUri is null || !MetadataUri.IsAbsoluteUri) + { + throw new InvalidOperationException("Metadata URI must be absolute."); + } + + if (MetadataUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("Metadata URI must use HTTP or HTTPS."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("Metadata cache duration must be positive."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + + if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight) || TrustWeight <= 0) + { + TrustWeight = 1.0; + } + else if (TrustWeight > 1.0) + { + TrustWeight = 1.0; + } + + if (CosignIssuer is not null) + { + if (string.IsNullOrWhiteSpace(CosignIdentityPattern)) + { + throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs similarity index 80% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs index ed46bc70..4fe8e399 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/DependencyInjection/RedHatConnectorServiceCollectionExtensions.cs @@ -1,45 +1,45 @@ -using System.Net; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; - -public static class RedHatConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.AddOptions() - .Configure(options => - { - configure?.Invoke(options); - }) - .PostConfigure(options => options.Validate()); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(30); - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.RedHat/1.0"); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - }) - .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.All, - }); - - services.AddSingleton(); - services.AddSingleton(); - - return services; - } -} +using System.Net; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection; + +public static class RedHatConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddRedHatCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }) + .PostConfigure(options => options.Validate()); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddHttpClient(RedHatConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.RedHat/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs index 8a38db61..0cab13ac 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/Metadata/RedHatProviderMetadataLoader.cs @@ -1,312 +1,312 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; - -public sealed class RedHatProviderMetadataLoader -{ - public const string CacheKey = "StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _cache; - private readonly ILogger _logger; - private readonly RedHatConnectorOptions _options; - private readonly IFileSystem _fileSystem; - private readonly JsonSerializerOptions _serializerOptions; - private readonly SemaphoreSlim _refreshSemaphore = new(1, 1); - - public RedHatProviderMetadataLoader( - IHttpClientFactory httpClientFactory, - IMemoryCache memoryCache, - IOptions options, - ILogger logger, - IFileSystem? fileSystem = null) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _fileSystem = fileSystem ?? new FileSystem(); - _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - }; - } - - public async Task LoadAsync(CancellationToken cancellationToken) - { - if (_cache.TryGetValue(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired()) - { - _logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt); - return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline); - } - - await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_cache.TryGetValue(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired()) - { - return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline); - } - - CacheEntry? previous = cached; - - // Attempt live fetch unless offline preferred. - if (!_options.PreferOfflineSnapshot) - { - var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); - if (httpResult is not null) - { - StoreCache(httpResult); - return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false); - } - } - - var offlineResult = TryLoadFromOffline(); - if (offlineResult is not null) - { - var offlineEntry = offlineResult with - { - FetchedAt = DateTimeOffset.UtcNow, - ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - FromOffline = true, - }; - StoreCache(offlineEntry); - return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true); - } - - throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot."); - } - finally - { - _refreshSemaphore.Release(); - } - } - - private void StoreCache(CacheEntry entry) - { - var cacheEntryOptions = new MemoryCacheEntryOptions - { - AbsoluteExpiration = entry.ExpiresAt, - }; - _cache.Set(CacheKey, entry, cacheEntryOptions); - } - - private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) - { - try - { - var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); - using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); - if (!string.IsNullOrWhiteSpace(previous?.ETag)) - { - if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) - { - request.Headers.IfNoneMatch.Add(etag); - } - } - - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) - { - _logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag); - return previous with - { - FetchedAt = DateTimeOffset.UtcNow, - ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - }; - } - - response.EnsureSuccessStatusCode(); - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - - var provider = ParseAndValidate(payload); - var etagHeader = response.Headers.ETag?.ToString(); - - if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) - { - try - { - _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); - _logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); - } - } - - return new CacheEntry( - provider, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow + _options.MetadataCacheDuration, - etagHeader, - FromOffline: false); - } - catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) - { - _logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri); - return null; - } - } - - private CacheEntry? TryLoadFromOffline() - { - if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) - { - return null; - } - - if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) - { - _logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); - return null; - } - - try - { - var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); - var provider = ParseAndValidate(payload); - return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); - return null; - } - } - - private VexProvider ParseAndValidate(string payload) - { - if (string.IsNullOrWhiteSpace(payload)) - { - throw new InvalidOperationException("Provider metadata payload was empty."); - } - - ProviderMetadataDocument? document; - try - { - document = JsonSerializer.Deserialize(payload, _serializerOptions); - } - catch (JsonException ex) - { - throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex); - } - - if (document is null) - { - throw new InvalidOperationException("Provider metadata payload was null after parsing."); - } - - if (document.Metadata?.Provider?.Name is null) - { - throw new InvalidOperationException("Provider metadata missing provider name."); - } - - var distributions = document.Distributions? - .Select(static d => d.Directory) - .Where(static s => !string.IsNullOrWhiteSpace(s)) - .Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory))) - .ToImmutableArray() ?? ImmutableArray.Empty; - - if (distributions.IsDefaultOrEmpty) - { - throw new InvalidOperationException("Provider metadata did not include any valid distribution directories."); - } - - Uri? rolieFeed = null; - if (document.Rolie?.Feeds is not null) - { - foreach (var feed in document.Rolie.Feeds) - { - if (!string.IsNullOrWhiteSpace(feed.Url)) - { - rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url"); - break; - } - } - } - - var trust = BuildTrust(); - return new VexProvider( - id: "vexer:redhat", - displayName: document.Metadata.Provider.Name, - kind: VexProviderKind.Distro, - baseUris: distributions, - discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed), - trust: trust); - } - - private VexProviderTrust BuildTrust() - { - VexCosignTrust? cosign = null; - if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern)) - { - cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!); - } - - return new VexProviderTrust( - _options.TrustWeight, - cosign, - _options.PgpFingerprints); - } - - private static Uri CreateUri(string value, string propertyName) - { - if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI."); - } - - return uri; - } - - private sealed record ProviderMetadataDocument( - [property: JsonPropertyName("metadata")] ProviderMetadata? Metadata, - [property: JsonPropertyName("distributions")] IReadOnlyList? Distributions, - [property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie); - - private sealed record ProviderMetadata( - [property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider); - - private sealed record ProviderMetadataProvider( - [property: JsonPropertyName("name")] string? Name); - - private sealed record ProviderMetadataDistribution( - [property: JsonPropertyName("directory")] string? Directory); - - private sealed record ProviderMetadataRolie( - [property: JsonPropertyName("feeds")] IReadOnlyList? Feeds); - - private sealed record ProviderMetadataRolieFeed( - [property: JsonPropertyName("url")] string? Url); - - private sealed record CacheEntry( - VexProvider Provider, - DateTimeOffset FetchedAt, - DateTimeOffset ExpiresAt, - string? ETag, - bool FromOffline) - { - public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; - } -} - -public sealed record RedHatProviderMetadataResult( - VexProvider Provider, - DateTimeOffset FetchedAt, - bool FromCache, - bool FromOfflineSnapshot); +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; + +public sealed class RedHatProviderMetadataLoader +{ + public const string CacheKey = "StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly ILogger _logger; + private readonly RedHatConnectorOptions _options; + private readonly IFileSystem _fileSystem; + private readonly JsonSerializerOptions _serializerOptions; + private readonly SemaphoreSlim _refreshSemaphore = new(1, 1); + + public RedHatProviderMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); + _fileSystem = fileSystem ?? new FileSystem(); + _serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + }; + } + + public async Task LoadAsync(CancellationToken cancellationToken) + { + if (_cache.TryGetValue(CacheKey, out var cached) && cached is { } cachedEntry && !cachedEntry.IsExpired()) + { + _logger.LogDebug("Returning cached Red Hat provider metadata (expires {Expires}).", cachedEntry.ExpiresAt); + return new RedHatProviderMetadataResult(cachedEntry.Provider, cachedEntry.FetchedAt, true, cachedEntry.FromOffline); + } + + await _refreshSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(CacheKey, out cached) && cached is { } cachedAfterLock && !cachedAfterLock.IsExpired()) + { + return new RedHatProviderMetadataResult(cachedAfterLock.Provider, cachedAfterLock.FetchedAt, true, cachedAfterLock.FromOffline); + } + + CacheEntry? previous = cached; + + // Attempt live fetch unless offline preferred. + if (!_options.PreferOfflineSnapshot) + { + var httpResult = await TryFetchFromNetworkAsync(previous, cancellationToken).ConfigureAwait(false); + if (httpResult is not null) + { + StoreCache(httpResult); + return new RedHatProviderMetadataResult(httpResult.Provider, httpResult.FetchedAt, false, false); + } + } + + var offlineResult = TryLoadFromOffline(); + if (offlineResult is not null) + { + var offlineEntry = offlineResult with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + FromOffline = true, + }; + StoreCache(offlineEntry); + return new RedHatProviderMetadataResult(offlineEntry.Provider, offlineEntry.FetchedAt, false, true); + } + + throw new InvalidOperationException("Unable to load Red Hat CSAF provider metadata from network or offline snapshot."); + } + finally + { + _refreshSemaphore.Release(); + } + } + + private void StoreCache(CacheEntry entry) + { + var cacheEntryOptions = new MemoryCacheEntryOptions + { + AbsoluteExpiration = entry.ExpiresAt, + }; + _cache.Set(CacheKey, entry, cacheEntryOptions); + } + + private async Task TryFetchFromNetworkAsync(CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, _options.MetadataUri); + if (!string.IsNullOrWhiteSpace(previous?.ETag)) + { + if (EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Red Hat provider metadata not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + var provider = ParseAndValidate(payload); + var etagHeader = response.Headers.ETag?.ToString(); + + if (_options.PersistOfflineSnapshot && !string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + try + { + _fileSystem.File.WriteAllText(_options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Red Hat metadata snapshot to {Path}.", _options.OfflineSnapshotPath); + } + } + + return new CacheEntry( + provider, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + _options.MetadataCacheDuration, + etagHeader, + FromOffline: false); + } + catch (Exception ex) when (ex is not OperationCanceledException && !_options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Red Hat provider metadata from {Uri}, will attempt offline snapshot.", _options.MetadataUri); + return null; + } + } + + private CacheEntry? TryLoadFromOffline() + { + if (string.IsNullOrWhiteSpace(_options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(_options.OfflineSnapshotPath)) + { + _logger.LogWarning("Offline snapshot path {Path} does not exist.", _options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(_options.OfflineSnapshotPath); + var provider = ParseAndValidate(payload); + return new CacheEntry(provider, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow + _options.MetadataCacheDuration, ETag: null, FromOffline: true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Red Hat provider metadata from offline snapshot {Path}.", _options.OfflineSnapshotPath); + return null; + } + } + + private VexProvider ParseAndValidate(string payload) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Provider metadata payload was empty."); + } + + ProviderMetadataDocument? document; + try + { + document = JsonSerializer.Deserialize(payload, _serializerOptions); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Provider metadata payload could not be parsed.", ex); + } + + if (document is null) + { + throw new InvalidOperationException("Provider metadata payload was null after parsing."); + } + + if (document.Metadata?.Provider?.Name is null) + { + throw new InvalidOperationException("Provider metadata missing provider name."); + } + + var distributions = document.Distributions? + .Select(static d => d.Directory) + .Where(static s => !string.IsNullOrWhiteSpace(s)) + .Select(static s => CreateUri(s!, nameof(ProviderMetadataDistribution.Directory))) + .ToImmutableArray() ?? ImmutableArray.Empty; + + if (distributions.IsDefaultOrEmpty) + { + throw new InvalidOperationException("Provider metadata did not include any valid distribution directories."); + } + + Uri? rolieFeed = null; + if (document.Rolie?.Feeds is not null) + { + foreach (var feed in document.Rolie.Feeds) + { + if (!string.IsNullOrWhiteSpace(feed.Url)) + { + rolieFeed = CreateUri(feed.Url, "rolie.feeds[].url"); + break; + } + } + } + + var trust = BuildTrust(); + return new VexProvider( + id: "excititor:redhat", + displayName: document.Metadata.Provider.Name, + kind: VexProviderKind.Distro, + baseUris: distributions, + discovery: new VexProviderDiscovery(_options.MetadataUri, rolieFeed), + trust: trust); + } + + private VexProviderTrust BuildTrust() + { + VexCosignTrust? cosign = null; + if (!string.IsNullOrWhiteSpace(_options.CosignIssuer) && !string.IsNullOrWhiteSpace(_options.CosignIdentityPattern)) + { + cosign = new VexCosignTrust(_options.CosignIssuer!, _options.CosignIdentityPattern!); + } + + return new VexProviderTrust( + _options.TrustWeight, + cosign, + _options.PgpFingerprints); + } + + private static Uri CreateUri(string value, string propertyName) + { + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException($"Provider metadata field '{propertyName}' must be an absolute HTTP(S) URI."); + } + + return uri; + } + + private sealed record ProviderMetadataDocument( + [property: JsonPropertyName("metadata")] ProviderMetadata? Metadata, + [property: JsonPropertyName("distributions")] IReadOnlyList? Distributions, + [property: JsonPropertyName("rolie")] ProviderMetadataRolie? Rolie); + + private sealed record ProviderMetadata( + [property: JsonPropertyName("provider")] ProviderMetadataProvider? Provider); + + private sealed record ProviderMetadataProvider( + [property: JsonPropertyName("name")] string? Name); + + private sealed record ProviderMetadataDistribution( + [property: JsonPropertyName("directory")] string? Directory); + + private sealed record ProviderMetadataRolie( + [property: JsonPropertyName("feeds")] IReadOnlyList? Feeds); + + private sealed record ProviderMetadataRolieFeed( + [property: JsonPropertyName("url")] string? Url); + + private sealed record CacheEntry( + VexProvider Provider, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOffline) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record RedHatProviderMetadataResult( + VexProvider Provider, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs similarity index 88% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs index b4cca425..6b9046ba 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/RedHatCsafConnector.cs +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/RedHatCsafConnector.cs @@ -1,186 +1,196 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Net.Http; -using System.Runtime.CompilerServices; -using System.Text.Json; -using System.Xml.Linq; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Configuration; -using StellaOps.Vexer.Connectors.RedHat.CSAF.Metadata; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Connectors.RedHat.CSAF; - -public sealed class RedHatCsafConnector : VexConnectorBase -{ - private readonly RedHatProviderMetadataLoader _metadataLoader; - private readonly IHttpClientFactory _httpClientFactory; - private readonly IVexConnectorStateRepository _stateRepository; - public RedHatCsafConnector( - VexConnectorDescriptor descriptor, - RedHatProviderMetadataLoader metadataLoader, - IHttpClientFactory httpClientFactory, - IVexConnectorStateRepository stateRepository, - ILogger logger, - TimeProvider timeProvider) - : base(descriptor, logger, timeProvider) - { - _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); - } - - public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - // No connector-specific settings yet. - return ValueTask.CompletedTask; - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); - if (metadataResult.Provider.Discovery.RolIeService is null) - { - throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed."); - } - - var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); - - var sinceTimestamp = context.Since; - if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp)) - { - sinceTimestamp = persisted; - } - - var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; - var digestList = new List(knownDigests); - var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); - var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue; - var stateChanged = false; - - foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false)) - { - if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp) - { - continue; - } - - if (entry.DocumentUri is null) - { - Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id); - continue; - } - - var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false); - - if (!digestSet.Add(rawDocument.Digest)) - { - Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest); - continue; - } - - await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); - digestList.Add(rawDocument.Digest); - stateChanged = true; - - if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated) - { - latestUpdated = entryUpdated; - } - - yield return rawDocument; - } - - if (stateChanged) - { - var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated; - var updatedState = new VexConnectorState( - Descriptor.Id, - newLastUpdated, - digestList.ToImmutableArray()); - - await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false); - } - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - { - // This connector relies on format-specific normalizers registered elsewhere. - throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component."); - } - - private async Task> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); - using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - var document = XDocument.Load(stream); - var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom"; - - var entries = document.Root? - .Elements(ns + "entry") - .Select(e => new RolieEntry( - Id: (string?)e.Element(ns + "id"), - Updated: ParseUpdated((string?)e.Element(ns + "updated")), - DocumentUri: ParseDocumentLink(e, ns))) - .Where(entry => entry.Id is not null && entry.Updated is not null) - .OrderBy(entry => entry.Updated) - .ToList() ?? new List(); - - return entries; - } - - private static DateTimeOffset? ParseUpdated(string? value) - => DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; - - private static Uri? ParseDocumentLink(XElement entry, XNamespace ns) - { - var linkElements = entry.Elements(ns + "link"); - foreach (var link in linkElements) - { - var rel = (string?)link.Attribute("rel"); - var href = (string?)link.Attribute("href"); - if (string.IsNullOrWhiteSpace(href)) - { - continue; - } - - if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase)) - { - if (Uri.TryCreate(href, UriKind.Absolute, out var uri)) - { - return uri; - } - } - } - - return null; - } - - private async Task DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken) - { - var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI."); - - var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); - using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - var metadata = BuildMetadata(builder => builder - .Add("redhat.csaf.entryId", entry.Id) - .Add("redhat.csaf.documentUri", documentUri.ToString()) - .Add("redhat.csaf.updated", entry.Updated?.ToString("O"))); - - return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata); - } - - private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri); -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Xml.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Configuration; +using StellaOps.Excititor.Connectors.RedHat.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.RedHat.CSAF; + +public sealed class RedHatCsafConnector : VexConnectorBase +{ + private readonly RedHatProviderMetadataLoader _metadataLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + public RedHatCsafConnector( + VexConnectorDescriptor descriptor, + RedHatProviderMetadataLoader metadataLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + ILogger logger, + TimeProvider timeProvider) + : base(descriptor, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + } + + public override ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + // No connector-specific settings yet. + return ValueTask.CompletedTask; + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var metadataResult = await _metadataLoader.LoadAsync(cancellationToken).ConfigureAwait(false); + if (metadataResult.Provider.Discovery.RolIeService is null) + { + throw new InvalidOperationException("Red Hat provider metadata did not specify a ROLIE feed."); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + + var sinceTimestamp = context.Since; + if (state?.LastUpdated is { } persisted && (sinceTimestamp is null || persisted > sinceTimestamp)) + { + sinceTimestamp = persisted; + } + + var knownDigests = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestList = new List(knownDigests); + var digestSet = new HashSet(knownDigests, StringComparer.OrdinalIgnoreCase); + var latestUpdated = state?.LastUpdated ?? sinceTimestamp ?? DateTimeOffset.MinValue; + var stateChanged = false; + + foreach (var entry in await FetchRolieEntriesAsync(metadataResult.Provider.Discovery.RolIeService, cancellationToken).ConfigureAwait(false)) + { + if (sinceTimestamp is not null && entry.Updated is DateTimeOffset updated && updated <= sinceTimestamp) + { + continue; + } + + if (entry.DocumentUri is null) + { + Logger.LogDebug("Skipping ROLIE entry {Id} because no document link was provided.", entry.Id); + continue; + } + + var rawDocument = await DownloadCsafDocumentAsync(entry, cancellationToken).ConfigureAwait(false); + + if (!digestSet.Add(rawDocument.Digest)) + { + Logger.LogDebug("Skipping CSAF document {Uri} because digest {Digest} was already processed.", rawDocument.SourceUri, rawDocument.Digest); + continue; + } + + await context.RawSink.StoreAsync(rawDocument, cancellationToken).ConfigureAwait(false); + digestList.Add(rawDocument.Digest); + stateChanged = true; + + if (entry.Updated is DateTimeOffset entryUpdated && entryUpdated > latestUpdated) + { + latestUpdated = entryUpdated; + } + + yield return rawDocument; + } + + if (stateChanged) + { + var newLastUpdated = latestUpdated == DateTimeOffset.MinValue ? state?.LastUpdated : latestUpdated; + var baseState = state ?? new VexConnectorState( + Descriptor.Id, + null, + ImmutableArray.Empty, + ImmutableDictionary.Empty, + null, + 0, + null, + null); + var updatedState = baseState with + { + LastUpdated = newLastUpdated, + DocumentDigests = digestList.ToImmutableArray(), + }; + + await _stateRepository.SaveAsync(updatedState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + // This connector relies on format-specific normalizers registered elsewhere. + throw new NotSupportedException("RedHatCsafConnector does not perform in-line normalization; use the CSAF normalizer component."); + } + + private async Task> FetchRolieEntriesAsync(Uri feedUri, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var response = await client.GetAsync(feedUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + var document = XDocument.Load(stream); + var ns = document.Root?.Name.Namespace ?? "http://www.w3.org/2005/Atom"; + + var entries = document.Root? + .Elements(ns + "entry") + .Select(e => new RolieEntry( + Id: (string?)e.Element(ns + "id"), + Updated: ParseUpdated((string?)e.Element(ns + "updated")), + DocumentUri: ParseDocumentLink(e, ns))) + .Where(entry => entry.Id is not null && entry.Updated is not null) + .OrderBy(entry => entry.Updated) + .ToList() ?? new List(); + + return entries; + } + + private static DateTimeOffset? ParseUpdated(string? value) + => DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + + private static Uri? ParseDocumentLink(XElement entry, XNamespace ns) + { + var linkElements = entry.Elements(ns + "link"); + foreach (var link in linkElements) + { + var rel = (string?)link.Attribute("rel"); + var href = (string?)link.Attribute("href"); + if (string.IsNullOrWhiteSpace(href)) + { + continue; + } + + if (rel is null || rel.Equals("enclosure", StringComparison.OrdinalIgnoreCase) || rel.Equals("alternate", StringComparison.OrdinalIgnoreCase)) + { + if (Uri.TryCreate(href, UriKind.Absolute, out var uri)) + { + return uri; + } + } + } + + return null; + } + + private async Task DownloadCsafDocumentAsync(RolieEntry entry, CancellationToken cancellationToken) + { + var documentUri = entry.DocumentUri ?? throw new InvalidOperationException("ROLIE entry missing document URI."); + + var client = _httpClientFactory.CreateClient(RedHatConnectorOptions.HttpClientName); + using var response = await client.GetAsync(documentUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var metadata = BuildMetadata(builder => builder + .Add("redhat.csaf.entryId", entry.Id) + .Add("redhat.csaf.documentUri", documentUri.ToString()) + .Add("redhat.csaf.updated", entry.Updated?.ToString("O"))); + + return CreateRawDocument(VexDocumentFormat.Csaf, documentUri, contentBytes, metadata); + } + + private sealed record RolieEntry(string? Id, DateTimeOffset? Updated, Uri? DocumentUri); +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj similarity index 66% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj rename to src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj index 3c27086b..b3e1398a 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/StellaOps.Vexer.Connectors.RedHat.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/StellaOps.Excititor.Connectors.RedHat.CSAF.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md new file mode 100644 index 00000000..f200e26c --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.RedHat.CSAF/TASKS.md @@ -0,0 +1,10 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-RH-01-001 – Provider metadata discovery|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `RedHatProviderMetadataLoader` with HTTP/ETag caching, offline snapshot handling, and validation; exposed DI helper + tests covering live, cached, and offline scenarios.| +|EXCITITOR-CONN-RH-01-002 – Incremental CSAF pulls|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Implemented `RedHatCsafConnector` with ROLIE feed parsing, incremental filtering via `context.Since`, CSAF document download + metadata capture, and persistence through `IVexRawDocumentSink`; tests cover live fetch/cache/offline scenarios with ETag handling.| +|EXCITITOR-CONN-RH-01-003 – Trust metadata emission|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-POLICY-01-001|**DONE (2025-10-17)** – Provider metadata loader now emits trust overrides (weight, cosign issuer/pattern, PGP fingerprints) and the connector surfaces provenance hints for policy/consensus layers.| +|EXCITITOR-CONN-RH-01-004 – Resume state persistence|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Connector now loads/saves resume state via `IVexConnectorStateRepository`, tracking last update timestamp and recent document digests to avoid duplicate CSAF ingestion; regression covers state persistence and duplicate skips.| +|EXCITITOR-CONN-RH-01-005 – Worker/WebService integration|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002|**DONE (2025-10-17)** – Worker/WebService now call `AddRedHatCsafConnector`, register the connector + state repo, and default worker scheduling adds the `excititor:redhat` provider so background jobs and orchestration can activate the connector without extra wiring.| +|EXCITITOR-CONN-RH-01-006 – CSAF normalization parity tests|Team Excititor Connectors – Red Hat|EXCITITOR-CONN-RH-01-002, EXCITITOR-FMT-CSAF-01-001|**DONE (2025-10-17)** – Added RHSA fixture-driven regression verifying CSAF normalizer retains Red Hat product metadata, tracking fields, and timestamps (`rhsa-sample.json` + `CsafNormalizerTests.NormalizeAsync_PreservesRedHatSpecificMetadata`).| diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs similarity index 93% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs index 3ca92c92..afc33dfd 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Authentication/RancherHubTokenProviderTests.cs @@ -1,138 +1,138 @@ -using System; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; -using Xunit; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.Authentication; - -public sealed class RancherHubTokenProviderTests -{ - private const string TokenResponse = "{\"access_token\":\"abc123\",\"token_type\":\"Bearer\",\"expires_in\":3600}"; - - [Fact] - public async Task GetAccessTokenAsync_RequestsAndCachesToken() - { - var handler = TestHttpMessageHandler.RespondWith(request => - { - request.Headers.Authorization.Should().NotBeNull(); - request.Content.Should().NotBeNull(); - - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(TokenResponse, Encoding.UTF8, "application/json"), - }; - }); - - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://identity.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - - var options = new RancherHubConnectorOptions - { - ClientId = "client", - ClientSecret = "secret", - TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), - Audience = "https://vexhub.suse.com", - }; - options.Scopes.Clear(); - options.Scopes.Add("hub.read"); - options.Scopes.Add("hub.events"); - - var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); - token.Should().NotBeNull(); - token!.Value.Should().Be("abc123"); - - var cached = await provider.GetAccessTokenAsync(options, CancellationToken.None); - cached.Should().NotBeNull(); - handler.InvocationCount.Should().Be(1); - } - - [Fact] - public async Task GetAccessTokenAsync_ReturnsNullWhenOfflinePreferred() - { - var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://identity.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - var options = new RancherHubConnectorOptions - { - PreferOfflineSnapshot = true, - ClientId = "client", - ClientSecret = "secret", - TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), - }; - - var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); - token.Should().BeNull(); - handler.InvocationCount.Should().Be(0); - } - - [Fact] - public async Task GetAccessTokenAsync_ReturnsNullWithoutCredentials() - { - var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://identity.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - var options = new RancherHubConnectorOptions(); - - var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); - token.Should().BeNull(); - handler.InvocationCount.Should().Be(0); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Func _responseFactory; - - private TestHttpMessageHandler(Func responseFactory) - { - _responseFactory = responseFactory; - } - - public int InvocationCount { get; private set; } - - public static TestHttpMessageHandler RespondWith(Func responseFactory) - => new(responseFactory); - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - InvocationCount++; - return Task.FromResult(_responseFactory(request)); - } - } -} +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using Xunit; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.Authentication; + +public sealed class RancherHubTokenProviderTests +{ + private const string TokenResponse = "{\"access_token\":\"abc123\",\"token_type\":\"Bearer\",\"expires_in\":3600}"; + + [Fact] + public async Task GetAccessTokenAsync_RequestsAndCachesToken() + { + var handler = TestHttpMessageHandler.RespondWith(request => + { + request.Headers.Authorization.Should().NotBeNull(); + request.Content.Should().NotBeNull(); + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(TokenResponse, Encoding.UTF8, "application/json"), + }; + }); + + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + + var options = new RancherHubConnectorOptions + { + ClientId = "client", + ClientSecret = "secret", + TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), + Audience = "https://vexhub.suse.com", + }; + options.Scopes.Clear(); + options.Scopes.Add("hub.read"); + options.Scopes.Add("hub.events"); + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().NotBeNull(); + token!.Value.Should().Be("abc123"); + + var cached = await provider.GetAccessTokenAsync(options, CancellationToken.None); + cached.Should().NotBeNull(); + handler.InvocationCount.Should().Be(1); + } + + [Fact] + public async Task GetAccessTokenAsync_ReturnsNullWhenOfflinePreferred() + { + var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var options = new RancherHubConnectorOptions + { + PreferOfflineSnapshot = true, + ClientId = "client", + ClientSecret = "secret", + TokenEndpoint = new Uri("https://identity.suse.com/oauth/token"), + }; + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().BeNull(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task GetAccessTokenAsync_ReturnsNullWithoutCredentials() + { + var handler = TestHttpMessageHandler.RespondWith(_ => new HttpResponseMessage(HttpStatusCode.OK)); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://identity.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var provider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var options = new RancherHubConnectorOptions(); + + var token = await provider.GetAccessTokenAsync(options, CancellationToken.None); + token.Should().BeNull(); + handler.InvocationCount.Should().Be(0); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responseFactory; + + private TestHttpMessageHandler(Func responseFactory) + { + _responseFactory = responseFactory; + } + + public int InvocationCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responseFactory) + => new(responseFactory); + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + return Task.FromResult(_responseFactory(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs similarity index 93% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs index b47da23c..1be19ac6 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/Metadata/RancherHubMetadataLoaderTests.cs @@ -1,178 +1,178 @@ -using System; -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; -using System.IO.Abstractions.TestingHelpers; -using System.Threading; -using Xunit; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.Metadata; - -public sealed class RancherHubMetadataLoaderTests -{ - private const string SampleDiscovery = """ - { - "hubId": "vexer:suse.rancher", - "title": "SUSE Rancher VEX Hub", - "subscription": { - "eventsUri": "https://vexhub.suse.com/api/v1/events", - "checkpointUri": "https://vexhub.suse.com/api/v1/checkpoints", - "requiresAuthentication": true, - "channels": ["rke2", "k3s"], - "scopes": ["hub.read", "hub.events"] - }, - "authentication": { - "tokenUri": "https://identity.suse.com/oauth2/token", - "audience": "https://vexhub.suse.com" - }, - "offline": { - "snapshotUri": "https://downloads.suse.com/vexhub/snapshot.json", - "sha256": "deadbeef", - "updated": "2025-10-10T12:00:00Z" - } - } - """; - - [Fact] - public async Task LoadAsync_FetchesAndCachesMetadata() - { - var handler = TestHttpMessageHandler.RespondWith(_ => - { - var response = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(SampleDiscovery, Encoding.UTF8, "application/json"), - }; - response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); - return response; - }); - - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://vexhub.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); - var options = new RancherHubConnectorOptions - { - DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), - OfflineSnapshotPath = offlinePath, - }; - - var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); - - var result = await loader.LoadAsync(options, CancellationToken.None); - - result.FromCache.Should().BeFalse(); - result.FromOfflineSnapshot.Should().BeFalse(); - result.Metadata.Provider.DisplayName.Should().Be("SUSE Rancher VEX Hub"); - result.Metadata.Subscription.EventsUri.Should().Be(new Uri("https://vexhub.suse.com/api/v1/events")); - result.Metadata.Authentication.TokenEndpoint.Should().Be(new Uri("https://identity.suse.com/oauth2/token")); - - // Second call should be served from cache (no additional HTTP invocation). - handler.ResetInvocationCount(); - await loader.LoadAsync(options, CancellationToken.None); - handler.InvocationCount.Should().Be(0); - } - - [Fact] - public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() - { - var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://vexhub.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); - fileSystem.AddFile(offlinePath, new MockFileData(SampleDiscovery)); - - var options = new RancherHubConnectorOptions - { - DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), - OfflineSnapshotPath = offlinePath, - }; - - var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); - - var result = await loader.LoadAsync(options, CancellationToken.None); - result.FromOfflineSnapshot.Should().BeTrue(); - result.Metadata.Subscription.RequiresAuthentication.Should().BeTrue(); - result.Metadata.OfflineSnapshot.Should().NotBeNull(); - } - - [Fact] - public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() - { - var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); - var client = new HttpClient(handler) - { - BaseAddress = new Uri("https://vexhub.suse.com"), - }; - - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var options = new RancherHubConnectorOptions - { - DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), - OfflineSnapshotPath = "/offline/missing.json", - PreferOfflineSnapshot = true, - }; - - var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); - var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); - - await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); - } - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Func _responseFactory; - - private TestHttpMessageHandler(Func responseFactory) - { - _responseFactory = responseFactory; - } - - public int InvocationCount { get; private set; } - - public static TestHttpMessageHandler RespondWith(Func responseFactory) - => new(responseFactory); - - public void ResetInvocationCount() => InvocationCount = 0; - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - InvocationCount++; - return Task.FromResult(_responseFactory(request)); - } - } -} +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; +using System.IO.Abstractions.TestingHelpers; +using System.Threading; +using Xunit; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.Metadata; + +public sealed class RancherHubMetadataLoaderTests +{ + private const string SampleDiscovery = """ + { + "hubId": "excititor:suse.rancher", + "title": "SUSE Rancher VEX Hub", + "subscription": { + "eventsUri": "https://vexhub.suse.com/api/v1/events", + "checkpointUri": "https://vexhub.suse.com/api/v1/checkpoints", + "requiresAuthentication": true, + "channels": ["rke2", "k3s"], + "scopes": ["hub.read", "hub.events"] + }, + "authentication": { + "tokenUri": "https://identity.suse.com/oauth2/token", + "audience": "https://vexhub.suse.com" + }, + "offline": { + "snapshotUri": "https://downloads.suse.com/vexhub/snapshot.json", + "sha256": "deadbeef", + "updated": "2025-10-10T12:00:00Z" + } + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesMetadata() + { + var handler = TestHttpMessageHandler.RespondWith(_ => + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(SampleDiscovery, Encoding.UTF8, "application/json"), + }; + response.Headers.ETag = new EntityTagHeaderValue("\"abc\""); + return response; + }); + + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = offlinePath, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + var result = await loader.LoadAsync(options, CancellationToken.None); + + result.FromCache.Should().BeFalse(); + result.FromOfflineSnapshot.Should().BeFalse(); + result.Metadata.Provider.DisplayName.Should().Be("SUSE Rancher VEX Hub"); + result.Metadata.Subscription.EventsUri.Should().Be(new Uri("https://vexhub.suse.com/api/v1/events")); + result.Metadata.Authentication.TokenEndpoint.Should().Be(new Uri("https://identity.suse.com/oauth2/token")); + + // Second call should be served from cache (no additional HTTP invocation). + handler.ResetInvocationCount(); + await loader.LoadAsync(options, CancellationToken.None); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenNetworkFails() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var offlinePath = fileSystem.Path.Combine(@"C:\offline", "rancher-hub.json"); + fileSystem.AddFile(offlinePath, new MockFileData(SampleDiscovery)); + + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = offlinePath, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Subscription.RequiresAuthentication.Should().BeTrue(); + result.Metadata.OfflineSnapshot.Should().NotBeNull(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenOfflinePreferredButMissing() + { + var handler = TestHttpMessageHandler.RespondWith(_ => throw new HttpRequestException("network down")); + var client = new HttpClient(handler) + { + BaseAddress = new Uri("https://vexhub.suse.com"), + }; + + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var options = new RancherHubConnectorOptions + { + DiscoveryUri = new Uri("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"), + OfflineSnapshotPath = "/offline/missing.json", + PreferOfflineSnapshot = true, + }; + + var tokenProvider = new RancherHubTokenProvider(factory, cache, NullLogger.Instance); + var loader = new RancherHubMetadataLoader(factory, cache, tokenProvider, fileSystem, NullLogger.Instance); + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Func _responseFactory; + + private TestHttpMessageHandler(Func responseFactory) + { + _responseFactory = responseFactory; + } + + public int InvocationCount { get; private set; } + + public static TestHttpMessageHandler RespondWith(Func responseFactory) + => new(responseFactory); + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + return Task.FromResult(_responseFactory(request)); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.csproj similarity index 65% rename from src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.csproj index e7a249c3..428b8ce0 100644 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests/StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/AGENTS.md b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/AGENTS.md similarity index 93% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/AGENTS.md rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/AGENTS.md index 3455eba0..8bd774ca 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector targeting SUSE Rancher VEX Hub feeds, ingesting hub events and translating them into raw documents for normalization. -## Scope -- Hub discovery, authentication, and subscription handling for Rancher VEX updates. -- HTTP/WebSocket (if provided) ingestion, checkpoint tracking, and deduplication. -- Mapping Rancher-specific status fields and product identifiers into connector metadata. -- Integration with offline bundles to allow snapshot imports. -## Participants -- Worker manages scheduled syncs using this connector; WebService can trigger manual reconcile pulls. -- Normalizers convert retrieved documents via CSAF/OpenVEX workflows depending on payload. -- Policy module uses trust metadata produced here for weight evaluation. -## Interfaces & contracts -- Implements `IVexConnector` with options for hub URL, credentials, and poll intervals. -- Uses shared abstractions for resume markers and telemetry. -## In/Out of scope -In: hub connectivity, message processing, raw persistence, provider metadata. -Out: normalization/export tasks, storage layer implementation, attestation. -## Observability & security expectations -- Log subscription IDs, batch sizes, and checkpoint updates while redacting secrets. -- Emit metrics for messages processed, lag, and retries. -## Tests -- Connector harness tests with simulated hub responses will live in `../StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests`. +# AGENTS +## Role +Connector targeting SUSE Rancher VEX Hub feeds, ingesting hub events and translating them into raw documents for normalization. +## Scope +- Hub discovery, authentication, and subscription handling for Rancher VEX updates. +- HTTP/WebSocket (if provided) ingestion, checkpoint tracking, and deduplication. +- Mapping Rancher-specific status fields and product identifiers into connector metadata. +- Integration with offline bundles to allow snapshot imports. +## Participants +- Worker manages scheduled syncs using this connector; WebService can trigger manual reconcile pulls. +- Normalizers convert retrieved documents via CSAF/OpenVEX workflows depending on payload. +- Policy module uses trust metadata produced here for weight evaluation. +## Interfaces & contracts +- Implements `IVexConnector` with options for hub URL, credentials, and poll intervals. +- Uses shared abstractions for resume markers and telemetry. +## In/Out of scope +In: hub connectivity, message processing, raw persistence, provider metadata. +Out: normalization/export tasks, storage layer implementation, attestation. +## Observability & security expectations +- Log subscription IDs, batch sizes, and checkpoint updates while redacting secrets. +- Emit metrics for messages processed, lag, and retries. +## Tests +- Connector harness tests with simulated hub responses will live in `../StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs index 6b5ff948..d823ee46 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Authentication/RancherHubTokenProvider.cs @@ -1,171 +1,171 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; - -public sealed class RancherHubTokenProvider -{ - private const string CachePrefix = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Token"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _cache; - private readonly ILogger _logger; - private readonly SemaphoreSlim _semaphore = new(1, 1); - - public RancherHubTokenProvider(IHttpClientFactory httpClientFactory, IMemoryCache cache, ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _cache = cache ?? throw new ArgumentNullException(nameof(cache)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask GetAccessTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(options); - - if (options.PreferOfflineSnapshot) - { - _logger.LogDebug("Skipping token request because PreferOfflineSnapshot is enabled."); - return null; - } - - var hasCredentials = !string.IsNullOrWhiteSpace(options.ClientId) && - !string.IsNullOrWhiteSpace(options.ClientSecret) && - options.TokenEndpoint is not null; - - if (!hasCredentials) - { - if (!options.AllowAnonymousDiscovery) - { - _logger.LogDebug("No Rancher hub credentials configured; proceeding without Authorization header."); - } - - return null; - } - - var cacheKey = $"{CachePrefix}:{options.ClientId}"; - if (_cache.TryGetValue(cacheKey, out var cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) - { - return cachedToken; - } - - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_cache.TryGetValue(cacheKey, out cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) - { - return cachedToken; - } - - var token = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); - if (token is not null) - { - var lifetime = token.ExpiresAt - DateTimeOffset.UtcNow; - if (lifetime <= TimeSpan.Zero) - { - lifetime = TimeSpan.FromMinutes(5); - } - - var absoluteExpiration = lifetime > TimeSpan.FromSeconds(30) - ? DateTimeOffset.UtcNow + lifetime - TimeSpan.FromSeconds(30) - : DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); - - _cache.Set(cacheKey, token, new MemoryCacheEntryOptions - { - AbsoluteExpiration = absoluteExpiration, - }); - } - - return token; - } - finally - { - _semaphore.Release(); - } - } - - private async Task RequestTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) - { - using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); - request.Headers.Accept.ParseAdd("application/json"); - - var parameters = new Dictionary - { - ["grant_type"] = "client_credentials", - }; - - if (options.Scopes.Count > 0) - { - parameters["scope"] = string.Join(' ', options.Scopes); - } - - if (!string.IsNullOrWhiteSpace(options.Audience)) - { - parameters["audience"] = options.Audience!; - } - - if (string.Equals(options.ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal)) - { - var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.ClientId}:{options.ClientSecret}")); - request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); - } - else - { - parameters["client_id"] = options.ClientId!; - parameters["client_secret"] = options.ClientSecret!; - } - - request.Content = new FormUrlEncodedContent(parameters); - - var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - throw new InvalidOperationException($"Failed to acquire Rancher hub access token ({response.StatusCode}): {payload}"); - } - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); - var root = document.RootElement; - - if (!root.TryGetProperty("access_token", out var accessTokenProperty) || accessTokenProperty.ValueKind is not JsonValueKind.String) - { - throw new InvalidOperationException("Token endpoint response missing access_token."); - } - - var token = accessTokenProperty.GetString(); - if (string.IsNullOrWhiteSpace(token)) - { - throw new InvalidOperationException("Token endpoint response contained an empty access_token."); - } - - var tokenType = root.TryGetProperty("token_type", out var tokenTypeElement) && tokenTypeElement.ValueKind is JsonValueKind.String - ? tokenTypeElement.GetString() ?? "Bearer" - : "Bearer"; - - var expires = root.TryGetProperty("expires_in", out var expiresElement) && - expiresElement.ValueKind is JsonValueKind.Number && - expiresElement.TryGetInt32(out var expiresSeconds) - ? DateTimeOffset.UtcNow + TimeSpan.FromSeconds(Math.Max(30, expiresSeconds)) - : DateTimeOffset.UtcNow + TimeSpan.FromMinutes(30); - - _logger.LogDebug("Acquired Rancher hub access token (expires {Expires}).", expires); - - return new RancherHubAccessToken(token, tokenType, expires); - } -} - -public sealed record RancherHubAccessToken(string Value, string TokenType, DateTimeOffset ExpiresAt) -{ - public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt - TimeSpan.FromMinutes(1); -} +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; + +public sealed class RancherHubTokenProvider +{ + private const string CachePrefix = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Token"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _cache; + private readonly ILogger _logger; + private readonly SemaphoreSlim _semaphore = new(1, 1); + + public RancherHubTokenProvider(IHttpClientFactory httpClientFactory, IMemoryCache cache, ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask GetAccessTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.PreferOfflineSnapshot) + { + _logger.LogDebug("Skipping token request because PreferOfflineSnapshot is enabled."); + return null; + } + + var hasCredentials = !string.IsNullOrWhiteSpace(options.ClientId) && + !string.IsNullOrWhiteSpace(options.ClientSecret) && + options.TokenEndpoint is not null; + + if (!hasCredentials) + { + if (!options.AllowAnonymousDiscovery) + { + _logger.LogDebug("No Rancher hub credentials configured; proceeding without Authorization header."); + } + + return null; + } + + var cacheKey = $"{CachePrefix}:{options.ClientId}"; + if (_cache.TryGetValue(cacheKey, out var cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) + { + return cachedToken; + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cache.TryGetValue(cacheKey, out cachedToken) && cachedToken is not null && !cachedToken.IsExpired()) + { + return cachedToken; + } + + var token = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var lifetime = token.ExpiresAt - DateTimeOffset.UtcNow; + if (lifetime <= TimeSpan.Zero) + { + lifetime = TimeSpan.FromMinutes(5); + } + + var absoluteExpiration = lifetime > TimeSpan.FromSeconds(30) + ? DateTimeOffset.UtcNow + lifetime - TimeSpan.FromSeconds(30) + : DateTimeOffset.UtcNow + TimeSpan.FromSeconds(10); + + _cache.Set(cacheKey, token, new MemoryCacheEntryOptions + { + AbsoluteExpiration = absoluteExpiration, + }); + } + + return token; + } + finally + { + _semaphore.Release(); + } + } + + private async Task RequestTokenAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); + request.Headers.Accept.ParseAdd("application/json"); + + var parameters = new Dictionary + { + ["grant_type"] = "client_credentials", + }; + + if (options.Scopes.Count > 0) + { + parameters["scope"] = string.Join(' ', options.Scopes); + } + + if (!string.IsNullOrWhiteSpace(options.Audience)) + { + parameters["audience"] = options.Audience!; + } + + if (string.Equals(options.ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal)) + { + var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{options.ClientId}:{options.ClientSecret}")); + request.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials); + } + else + { + parameters["client_id"] = options.ClientId!; + parameters["client_secret"] = options.ClientSecret!; + } + + request.Content = new FormUrlEncodedContent(parameters); + + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Failed to acquire Rancher hub access token ({response.StatusCode}): {payload}"); + } + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + var root = document.RootElement; + + if (!root.TryGetProperty("access_token", out var accessTokenProperty) || accessTokenProperty.ValueKind is not JsonValueKind.String) + { + throw new InvalidOperationException("Token endpoint response missing access_token."); + } + + var token = accessTokenProperty.GetString(); + if (string.IsNullOrWhiteSpace(token)) + { + throw new InvalidOperationException("Token endpoint response contained an empty access_token."); + } + + var tokenType = root.TryGetProperty("token_type", out var tokenTypeElement) && tokenTypeElement.ValueKind is JsonValueKind.String + ? tokenTypeElement.GetString() ?? "Bearer" + : "Bearer"; + + var expires = root.TryGetProperty("expires_in", out var expiresElement) && + expiresElement.ValueKind is JsonValueKind.Number && + expiresElement.TryGetInt32(out var expiresSeconds) + ? DateTimeOffset.UtcNow + TimeSpan.FromSeconds(Math.Max(30, expiresSeconds)) + : DateTimeOffset.UtcNow + TimeSpan.FromMinutes(30); + + _logger.LogDebug("Acquired Rancher hub access token (expires {Expires}).", expires); + + return new RancherHubAccessToken(token, tokenType, expires); + } +} + +public sealed record RancherHubAccessToken(string Value, string TokenType, DateTimeOffset ExpiresAt) +{ + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt - TimeSpan.FromMinutes(1); +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs index 83e0bea1..28422914 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptions.cs @@ -1,186 +1,186 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; - -public sealed class RancherHubConnectorOptions -{ - public static readonly Uri DefaultDiscoveryUri = new("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"); - - /// - /// HTTP client name registered for the connector. - /// - public const string HttpClientName = "vexer.connector.suse.rancherhub"; - - /// - /// URI for the Rancher VEX hub discovery document. - /// - public Uri DiscoveryUri { get; set; } = DefaultDiscoveryUri; - - /// - /// Optional OAuth2/OIDC token endpoint used for hub authentication. - /// - public Uri? TokenEndpoint { get; set; } - - /// - /// Client identifier used when requesting hub access tokens. - /// - public string? ClientId { get; set; } - - /// - /// Client secret used when requesting hub access tokens. - /// - public string? ClientSecret { get; set; } - - /// - /// OAuth scopes requested for hub access; defaults align with Rancher hub reader role. - /// - public IList Scopes { get; } = new List { "hub.read" }; - - /// - /// Optional audience claim passed when requesting tokens (client credential grant). - /// - public string? Audience { get; set; } - - /// - /// Preferred authentication scheme. Supported: client_secret_basic (default) or client_secret_post. - /// - public string ClientAuthenticationScheme { get; set; } = "client_secret_basic"; - - /// - /// Duration to cache discovery metadata before re-fetching. - /// - public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromMinutes(30); - - /// - /// Optional file path for discovery metadata snapshots. - /// - public string? OfflineSnapshotPath { get; set; } - - /// - /// When true, the loader prefers the offline snapshot prior to attempting network discovery. - /// - public bool PreferOfflineSnapshot { get; set; } - - /// - /// Enables persisting freshly fetched discovery documents to . - /// - public bool PersistOfflineSnapshot { get; set; } = true; - - /// - /// Weight applied to the provider entry; hubs default below direct vendor feeds. - /// - public double TrustWeight { get; set; } = 0.6; - - /// - /// Optional Sigstore/Cosign issuer for verifying hub-delivered attestations. - /// - public string? CosignIssuer { get; set; } - - /// - /// Cosign identity pattern matched against transparency log subjects. - /// - public string? CosignIdentityPattern { get; set; } - - /// - /// Additional trusted PGP fingerprints declared by the hub. - /// - public IList PgpFingerprints { get; } = new List(); - - /// - /// Allows falling back to unauthenticated discovery requests when credentials are absent. - /// - public bool AllowAnonymousDiscovery { get; set; } - - public void Validate(IFileSystem? fileSystem = null) - { - if (DiscoveryUri is null || !DiscoveryUri.IsAbsoluteUri) - { - throw new InvalidOperationException("DiscoveryUri must be an absolute URI."); - } - - if (DiscoveryUri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException("DiscoveryUri must use HTTP or HTTPS."); - } - - if (MetadataCacheDuration <= TimeSpan.Zero) - { - throw new InvalidOperationException("MetadataCacheDuration must be positive."); - } - - if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - var fs = fileSystem ?? new FileSystem(); - var directory = Path.GetDirectoryName(OfflineSnapshotPath); - if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) - { - fs.Directory.CreateDirectory(directory); - } - } - - if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); - } - - var hasClientId = !string.IsNullOrWhiteSpace(ClientId); - var hasClientSecret = !string.IsNullOrWhiteSpace(ClientSecret); - var hasTokenEndpoint = TokenEndpoint is not null; - if (hasClientId || hasClientSecret || hasTokenEndpoint) - { - if (!(hasClientId && hasClientSecret && hasTokenEndpoint)) - { - throw new InvalidOperationException("ClientId, ClientSecret, and TokenEndpoint must be provided together for authenticated discovery."); - } - - if (TokenEndpoint is not null && (!TokenEndpoint.IsAbsoluteUri || TokenEndpoint.Scheme is not ("http" or "https"))) - { - throw new InvalidOperationException("TokenEndpoint must be an absolute HTTP(S) URI."); - } - } - - if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight)) - { - TrustWeight = 0.6; - } - else if (TrustWeight <= 0) - { - TrustWeight = 0.1; - } - else if (TrustWeight > 1.0) - { - TrustWeight = 1.0; - } - - if (!string.IsNullOrWhiteSpace(CosignIssuer) && string.IsNullOrWhiteSpace(CosignIdentityPattern)) - { - throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); - } - - if (!string.Equals(ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal) && - !string.Equals(ClientAuthenticationScheme, "client_secret_post", StringComparison.Ordinal)) - { - throw new InvalidOperationException("ClientAuthenticationScheme must be 'client_secret_basic' or 'client_secret_post'."); - } - - // Remove any empty scopes to avoid token request issues. - if (Scopes.Count > 0) - { - for (var i = Scopes.Count - 1; i >= 0; i--) - { - if (string.IsNullOrWhiteSpace(Scopes[i])) - { - Scopes.RemoveAt(i); - } - } - } - - if (Scopes.Count == 0) - { - Scopes.Add("hub.read"); - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; + +public sealed class RancherHubConnectorOptions +{ + public static readonly Uri DefaultDiscoveryUri = new("https://vexhub.suse.com/.well-known/vex/rancher-hub.json"); + + /// + /// HTTP client name registered for the connector. + /// + public const string HttpClientName = "excititor.connector.suse.rancherhub"; + + /// + /// URI for the Rancher VEX hub discovery document. + /// + public Uri DiscoveryUri { get; set; } = DefaultDiscoveryUri; + + /// + /// Optional OAuth2/OIDC token endpoint used for hub authentication. + /// + public Uri? TokenEndpoint { get; set; } + + /// + /// Client identifier used when requesting hub access tokens. + /// + public string? ClientId { get; set; } + + /// + /// Client secret used when requesting hub access tokens. + /// + public string? ClientSecret { get; set; } + + /// + /// OAuth scopes requested for hub access; defaults align with Rancher hub reader role. + /// + public IList Scopes { get; } = new List { "hub.read" }; + + /// + /// Optional audience claim passed when requesting tokens (client credential grant). + /// + public string? Audience { get; set; } + + /// + /// Preferred authentication scheme. Supported: client_secret_basic (default) or client_secret_post. + /// + public string ClientAuthenticationScheme { get; set; } = "client_secret_basic"; + + /// + /// Duration to cache discovery metadata before re-fetching. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromMinutes(30); + + /// + /// Optional file path for discovery metadata snapshots. + /// + public string? OfflineSnapshotPath { get; set; } + + /// + /// When true, the loader prefers the offline snapshot prior to attempting network discovery. + /// + public bool PreferOfflineSnapshot { get; set; } + + /// + /// Enables persisting freshly fetched discovery documents to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + /// + /// Weight applied to the provider entry; hubs default below direct vendor feeds. + /// + public double TrustWeight { get; set; } = 0.6; + + /// + /// Optional Sigstore/Cosign issuer for verifying hub-delivered attestations. + /// + public string? CosignIssuer { get; set; } + + /// + /// Cosign identity pattern matched against transparency log subjects. + /// + public string? CosignIdentityPattern { get; set; } + + /// + /// Additional trusted PGP fingerprints declared by the hub. + /// + public IList PgpFingerprints { get; } = new List(); + + /// + /// Allows falling back to unauthenticated discovery requests when credentials are absent. + /// + public bool AllowAnonymousDiscovery { get; set; } + + public void Validate(IFileSystem? fileSystem = null) + { + if (DiscoveryUri is null || !DiscoveryUri.IsAbsoluteUri) + { + throw new InvalidOperationException("DiscoveryUri must be an absolute URI."); + } + + if (DiscoveryUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("DiscoveryUri must use HTTP or HTTPS."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + var hasClientId = !string.IsNullOrWhiteSpace(ClientId); + var hasClientSecret = !string.IsNullOrWhiteSpace(ClientSecret); + var hasTokenEndpoint = TokenEndpoint is not null; + if (hasClientId || hasClientSecret || hasTokenEndpoint) + { + if (!(hasClientId && hasClientSecret && hasTokenEndpoint)) + { + throw new InvalidOperationException("ClientId, ClientSecret, and TokenEndpoint must be provided together for authenticated discovery."); + } + + if (TokenEndpoint is not null && (!TokenEndpoint.IsAbsoluteUri || TokenEndpoint.Scheme is not ("http" or "https"))) + { + throw new InvalidOperationException("TokenEndpoint must be an absolute HTTP(S) URI."); + } + } + + if (double.IsNaN(TrustWeight) || double.IsInfinity(TrustWeight)) + { + TrustWeight = 0.6; + } + else if (TrustWeight <= 0) + { + TrustWeight = 0.1; + } + else if (TrustWeight > 1.0) + { + TrustWeight = 1.0; + } + + if (!string.IsNullOrWhiteSpace(CosignIssuer) && string.IsNullOrWhiteSpace(CosignIdentityPattern)) + { + throw new InvalidOperationException("CosignIdentityPattern must be provided when CosignIssuer is specified."); + } + + if (!string.Equals(ClientAuthenticationScheme, "client_secret_basic", StringComparison.Ordinal) && + !string.Equals(ClientAuthenticationScheme, "client_secret_post", StringComparison.Ordinal)) + { + throw new InvalidOperationException("ClientAuthenticationScheme must be 'client_secret_basic' or 'client_secret_post'."); + } + + // Remove any empty scopes to avoid token request issues. + if (Scopes.Count > 0) + { + for (var i = Scopes.Count - 1; i >= 0; i--) + { + if (string.IsNullOrWhiteSpace(Scopes[i])) + { + Scopes.RemoveAt(i); + } + } + } + + if (Scopes.Count == 0) + { + Scopes.Add("hub.read"); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs similarity index 84% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs index 4c29f5c5..d8b992a0 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Configuration/RancherHubConnectorOptionsValidator.cs @@ -1,32 +1,32 @@ -using System; -using System.Collections.Generic; -using System.IO.Abstractions; -using StellaOps.Vexer.Connectors.Abstractions; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; - -public sealed class RancherHubConnectorOptionsValidator : IVexConnectorOptionsValidator -{ - private readonly IFileSystem _fileSystem; - - public RancherHubConnectorOptionsValidator(IFileSystem fileSystem) - { - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - } - - public void Validate(VexConnectorDescriptor descriptor, RancherHubConnectorOptions options, IList errors) - { - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentNullException.ThrowIfNull(options); - ArgumentNullException.ThrowIfNull(errors); - - try - { - options.Validate(_fileSystem); - } - catch (Exception ex) - { - errors.Add(ex.Message); - } - } -} +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; + +public sealed class RancherHubConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public RancherHubConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, RancherHubConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs similarity index 77% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs index 3e8b23e7..ca6289e3 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/DependencyInjection/RancherHubConnectorServiceCollectionExtensions.cs @@ -1,49 +1,49 @@ -using System; -using System.Net; -using System.Net.Http; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.DependencyInjection; - -public static class RancherHubConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddRancherHubConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddOptions() - .Configure(options => - { - configure?.Invoke(options); - }); - - services.AddSingleton, RancherHubConnectorOptionsValidator>(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - - services.AddHttpClient(RancherHubConnectorOptions.HttpClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(30); - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/1.0"); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - }) - .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.All, - }); - - return services; - } -} +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.DependencyInjection; + +public static class RancherHubConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddRancherHubConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => + { + configure?.Invoke(options); + }); + + services.AddSingleton, RancherHubConnectorOptionsValidator>(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + services.AddHttpClient(RancherHubConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(30); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + return services; + } +} diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Design/EXCITITOR-CONN-SUSE-01-002.md b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Design/EXCITITOR-CONN-SUSE-01-002.md new file mode 100644 index 00000000..d731a57d --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Design/EXCITITOR-CONN-SUSE-01-002.md @@ -0,0 +1,127 @@ +# EXCITITOR-CONN-SUSE-01-002 — Checkpointed Event Ingestion (Design) + +**Status:** draft • **Updated:** 2025-10-19 +**Scope:** StellaOps.Excititor.Connectors.SUSE.RancherVEXHub + +## Goals + +- Stream Rancher VEX Hub events deterministically, supporting cold start and incremental resumes. +- Persist checkpoints so subsequent runs (worker/manual CLI) resume where the previous execution stopped. +- Deduplicate hub payloads using cryptographic digests while keeping a short history (≤ 200 entries) to align with `IVexConnectorStateRepository` constraints. +- Quarantine malformed/unfetchable events without blocking healthy ones, making failures observable for operators. +- Remain offline-friendly: work from discovery metadata snapshots and cached checkpoints without live network calls when configured. + +## Assumed Event Model + +Discovery metadata supplies `subscription.eventsUri` and (optionally) `subscription.checkpointUri`. Rancher emits JSON event batches over HTTP(S): + +```json +{ + "cursor": "opaque-offset-123", + "events": [ + { + "id": "evt-2025-10-19T12:42:30Z-001", + "type": "vex.statement.published", + "channel": "rancher/rke2", + "publishedAt": "2025-10-19T12:42:30Z", + "document": { + "uri": "https://hub.suse.example/events/evt-.../statement.json", + "sha256": "ab12...", + "format": "csaf" + } + } + ] +} +``` + +Key properties assumed per discovery schema validation: + +- `cursor` advances monotonically and can be replayed via `?cursor=` or a POST to `checkpointUri`. +- Events carry a `document.uri` (absolute HTTP(S) URI) and an optional digest (`document.sha256`). When absent, a digest is computed after download. +- `publishedAt` is UTC and stable; it is used as `VexConnectorState.LastUpdated` fallback when no checkpoint is provided. +- Optional `channels` allow filtering (`channels=foo,bar`) to minimise payloads. + +The connector must tolerate missing fields by quaratining the raw envelope. + +## Flow Overview + +1. **Load connector state** from `IVexConnectorStateRepository` keyed by `Descriptor.Id`. + - `LastUpdated` stores the last successfully processed `publishedAt`. + - `DocumentDigests` stores: + - Last checkpoint token entry prefixed `checkpoint:` (only most recent kept). + - Recent raw document digests for deduping. +2. **Resolve resume parameters**: + - Start cursor: explicit CLI `context.Since` overrides persisted checkpoint. + - If checkpoint exists, call `eventsUri?cursor=`; else pass `since=` (from `state.LastUpdated` or `context.Since`). + - Limit channels if discovery enumerated them and options specify `RancherHubConnectorOptions.EnabledChannels` (future option). +3. **Fetch batches** in a deterministic, cancellation-aware loop: + - Send GETs with `pageSize` cap (default 200) and follow `nextCursor`/pagination until exhaustion. + - For each batch log metrics (`eventCount`, `cursor`, `fromOffline` flag). +4. **Process events**: + - Validate minimal shape (id, document uri). Missing/invalid fields => log warning + quarantine JSON payload. + - Fetch document content via shared HTTP client. Respect optional digests (compare after download). + - Build raw metadata: event ID, channel, publishedAt, checkpoint cursor (if provided), offline flag. + - Deduplicate using `HashSet` seeded with persisted digests; skip duplicates without re-writing state. + - Push valid documents to `context.RawSink.StoreAsync` and yield them downstream. + - Capture latest `publishedAt` and `cursor` for state update. +5. **Quarantine path**: + - Serialize offending envelope into UTF-8 JSON (`application/vnd.stella.quarantine+json` metadata flag). + - Persist via `context.RawSink.StoreAsync` using format `VexDocumentFormat.Json` and metadata `{"rancher.event.quarantine":"true"}` to allow downstream filtering/reporting. +6. **Persist state** once the batch completes or on graceful cancellation: + - Update `LastUpdated` with max `publishedAt` processed. + - Rebuild digest window (most recent ≤ 200). + - Store latest checkpoint token (if hub supplied one) as first digest entry `checkpoint:` for quick retrieval. + +## Key Types & Components + +```csharp +internal sealed record RancherHubEventEnvelope( + string Id, + string? Type, + string Channel, + DateTimeOffset PublishedAt, + Uri DocumentUri, + string? DocumentDigest, + string? DocumentFormat); + +internal sealed record RancherHubCheckpointState( + string? Cursor, + DateTimeOffset? LatestPublishedAt, + ImmutableArray Digests); +``` + +- `RancherHubEventClient` (new) encapsulates HTTP paging, cursor handling, and offline replay (reading bundled snapshot JSON when `PreferOfflineSnapshot` enabled). +- `RancherHubCheckpointManager` (new) reads/writes `VexConnectorState`, encoding checkpoint token under the `checkpoint:` prefix and trimming digest history. + +## Deduplication Strategy + +- Primary key: document SHA-256 digest (hub-provided or computed). Fallback: `event.Id` when digest missing (encoded as `event:` entry). +- Persist dedupe keys via `DocumentDigests` to short-circuit duplicates on next run. Keep insertion order for deterministic state updates. +- When offline snapshot is replayed, skip dedupe reset—reused digests still apply. + +## Quarantine Semantics + +- Trigger conditions: + - JSON envelope missing required fields. + - Document fetch returns non-success HTTP code. + - Digest mismatch between declared `document.sha256` and computed value. +- Action: create `VexRawDocument` with metadata: + - `rancher.event.id`, `rancher.event.channel`, `rancher.event.type`, `rancher.event.error`. + - `rancher.event.quarantine=true` flag for downstream routing. + - Content: original envelope JSON (or error stub when fetch failed). +- Quarantine entries count toward dedupe history using a synthetic digest `quarantine:` to prevent repeated attempts until manual intervention. + +## Cancellation & Determinism + +- Each HTTP call honours `CancellationToken`. +- Loop checkpoints after each processed batch; if cancellation occurs mid-batch, state updates only include successfully handled documents to preserve deterministic replays. +- Sorting: events processed in ascending `publishedAt` (or server-provided order). Within batch, maintain original order to avoid digest reshuffling. + +## Open Questions / Follow-ups + +- Confirm exact Rancher event schema (pending coordination with SUSE PSIRT) and adjust parser accordingly. +- Validate whether `checkpointUri` requires POST with body `{ "cursor": "..."} ` or simple GET. +- Decide on channel filtering surface area (option flag vs. discovery default). +- Establish metrics contract once observability task (future) starts. + +Until those are resolved the implementation will keep parser tolerant with detailed logging and quarantine coverage so future adjustments are low risk. diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs new file mode 100644 index 00000000..cf94f5eb --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventClient.cs @@ -0,0 +1,311 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; + +internal sealed class RancherHubEventClient +{ + private readonly IHttpClientFactory _httpClientFactory; + private readonly RancherHubTokenProvider _tokenProvider; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly JsonDocumentOptions _documentOptions = new() + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }; + + private const string CheckpointPrefix = "checkpoint"; + + public RancherHubEventClient( + IHttpClientFactory httpClientFactory, + RancherHubTokenProvider tokenProvider, + IFileSystem fileSystem, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async IAsyncEnumerable FetchEventBatchesAsync( + RancherHubConnectorOptions options, + RancherHubMetadata metadata, + string? cursor, + DateTimeOffset? since, + ImmutableArray channels, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(metadata); + + if (options.PreferOfflineSnapshot && metadata.OfflineSnapshot is not null) + { + var offline = await LoadOfflineSnapshotAsync(metadata.OfflineSnapshot, cancellationToken).ConfigureAwait(false); + if (offline is not null) + { + yield return offline; + } + + yield break; + } + + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + var currentCursor = cursor; + var currentSince = since; + var firstRequest = true; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + var requestUri = BuildRequestUri(metadata.Subscription.EventsUri, currentCursor, currentSince, channels); + using var request = await CreateRequestAsync(options, metadata, requestUri, cancellationToken).ConfigureAwait(false); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new InvalidOperationException($"Rancher hub events request failed ({(int)response.StatusCode} {response.StatusCode}). Payload: {payload}"); + } + + var json = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var batch = ParseBatch(json, fromOfflineSnapshot: false); + + yield return batch; + + if (string.IsNullOrWhiteSpace(batch.NextCursor)) + { + break; + } + + if (!firstRequest && string.Equals(batch.NextCursor, currentCursor, StringComparison.Ordinal)) + { + _logger.LogWarning("Detected stable cursor {Cursor}; stopping to avoid loop.", batch.NextCursor); + break; + } + + currentCursor = batch.NextCursor; + currentSince = null; // cursor supersedes since parameter + firstRequest = false; + } + } + + private async Task LoadOfflineSnapshotAsync(RancherHubOfflineSnapshotMetadata offline, CancellationToken cancellationToken) + { + try + { + string payload; + if (offline.SnapshotUri.Scheme.Equals("file", StringComparison.OrdinalIgnoreCase)) + { + var path = offline.SnapshotUri.LocalPath; + payload = await _fileSystem.File.ReadAllTextAsync(path, Encoding.UTF8, cancellationToken).ConfigureAwait(false); + } + else + { + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var response = await client.GetAsync(offline.SnapshotUri, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + + if (!string.IsNullOrWhiteSpace(offline.Sha256)) + { + var computed = ComputeSha256(payload); + if (!string.Equals(computed, offline.Sha256, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning( + "Offline snapshot digest mismatch (expected {Expected}, computed {Computed}); proceeding anyway.", + offline.Sha256, + computed); + } + } + + return ParseBatch(payload, fromOfflineSnapshot: true); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to load Rancher hub offline snapshot from {Uri}.", offline.SnapshotUri); + return null; + } + } + + private async Task CreateRequestAsync(RancherHubConnectorOptions options, RancherHubMetadata metadata, Uri requestUri, CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Get, requestUri); + request.Headers.Accept.ParseAdd("application/json"); + + if (metadata.Subscription.RequiresAuthentication) + { + var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; + request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); + } + } + + return request; + } + + private RancherHubEventBatch ParseBatch(string payload, bool fromOfflineSnapshot) + { + using var document = JsonDocument.Parse(payload, _documentOptions); + var root = document.RootElement; + + var cursor = ReadString(root, "cursor", "currentCursor", "checkpoint"); + var nextCursor = ReadString(root, "nextCursor", "next", "continuation", "continuationToken"); + var eventsElement = TryGetProperty(root, "events", "items", "data") ?? default; + var events = ImmutableArray.CreateBuilder(); + + if (eventsElement.ValueKind == JsonValueKind.Array) + { + foreach (var item in eventsElement.EnumerateArray()) + { + events.Add(ParseEvent(item)); + } + } + + return new RancherHubEventBatch(cursor, nextCursor, events.ToImmutable(), fromOfflineSnapshot, payload); + } + + private RancherHubEventRecord ParseEvent(JsonElement element) + { + var rawJson = element.GetRawText(); + var id = ReadString(element, "id", "eventId", "uuid"); + var type = ReadString(element, "type", "eventType"); + var channel = ReadString(element, "channel", "product", "stream"); + var publishedAt = ParseDate(ReadString(element, "publishedAt", "timestamp", "createdAt")); + + Uri? documentUri = null; + string? documentDigest = null; + string? documentFormat = null; + + var documentElement = TryGetProperty(element, "document", "payload", "statement"); + if (documentElement.HasValue) + { + documentUri = ParseUri(ReadString(documentElement.Value, "uri", "url", "href")); + documentDigest = ReadString(documentElement.Value, "sha256", "digest", "checksum"); + documentFormat = ReadString(documentElement.Value, "format", "kind", "type"); + } + else + { + documentUri = ParseUri(ReadString(element, "documentUri", "uri", "url")); + documentDigest = ReadString(element, "documentSha256", "sha256", "digest"); + documentFormat = ReadString(element, "documentFormat", "format"); + } + + return new RancherHubEventRecord(rawJson, id, type, channel, publishedAt, documentUri, documentDigest, documentFormat); + } + + private static Uri? ParseUri(string? value) + => string.IsNullOrWhiteSpace(value) ? null : Uri.TryCreate(value, UriKind.Absolute, out var uri) ? uri : null; + + private static DateTimeOffset? ParseDate(string? value) + => string.IsNullOrWhiteSpace(value) ? null : DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + + private static string? ReadString(JsonElement element, params string[] propertyNames) + { + var property = TryGetProperty(element, propertyNames); + if (!property.HasValue || property.Value.ValueKind is not JsonValueKind.String) + { + return null; + } + + var value = property.Value.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames) + { + foreach (var propertyName in propertyNames) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined) + { + return property; + } + } + + return null; + } + + private static string BuildQueryString(Dictionary parameters) + { + if (parameters.Count == 0) + { + return string.Empty; + } + + var builder = new StringBuilder(); + var first = true; + foreach (var kvp in parameters) + { + if (string.IsNullOrEmpty(kvp.Value)) + { + continue; + } + + if (!first) + { + builder.Append('&'); + } + builder.Append(Uri.EscapeDataString(kvp.Key)); + builder.Append('='); + builder.Append(Uri.EscapeDataString(kvp.Value)); + first = false; + } + + return builder.ToString(); + } + + private static Uri BuildRequestUri(Uri baseUri, string? cursor, DateTimeOffset? since, ImmutableArray channels) + { + var builder = new UriBuilder(baseUri); + var parameters = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(cursor)) + { + parameters["cursor"] = cursor; + } + else if (since is not null) + { + parameters["since"] = since.Value.ToUniversalTime().ToString("O"); + } + + if (!channels.IsDefaultOrEmpty && channels.Length > 0) + { + parameters["channels"] = string.Join(',', channels); + } + + var query = BuildQueryString(parameters); + builder.Query = string.IsNullOrEmpty(query) ? null : query; + return builder.Uri; + } + + private static string ComputeSha256(string payload) + { + var bytes = Encoding.UTF8.GetBytes(payload); + Span hash = stackalloc byte[32]; + if (SHA256.TryHashData(bytes, hash, out _)) + { + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + using var sha = SHA256.Create(); + return Convert.ToHexString(sha.ComputeHash(bytes)).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs new file mode 100644 index 00000000..d1434a3e --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Events/RancherHubEventModels.cs @@ -0,0 +1,21 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; + +internal sealed record RancherHubEventRecord( + string RawJson, + string? Id, + string? Type, + string? Channel, + DateTimeOffset? PublishedAt, + Uri? DocumentUri, + string? DocumentDigest, + string? DocumentFormat); + +internal sealed record RancherHubEventBatch( + string? Cursor, + string? NextCursor, + ImmutableArray Events, + bool FromOfflineSnapshot, + string RawPayload); diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs index 930f8192..9a60041c 100644 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/Metadata/RancherHubMetadataLoader.cs @@ -1,455 +1,455 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO.Abstractions; -using System.Net; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Authentication; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; - -public sealed class RancherHubMetadataLoader -{ - public const string CachePrefix = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _memoryCache; - private readonly RancherHubTokenProvider _tokenProvider; - private readonly IFileSystem _fileSystem; - private readonly ILogger _logger; - private readonly SemaphoreSlim _semaphore = new(1, 1); - private readonly JsonDocumentOptions _documentOptions; - - public RancherHubMetadataLoader( - IHttpClientFactory httpClientFactory, - IMemoryCache memoryCache, - RancherHubTokenProvider tokenProvider, - IFileSystem fileSystem, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); - _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _documentOptions = new JsonDocumentOptions - { - CommentHandling = JsonCommentHandling.Skip, - AllowTrailingCommas = true, - }; - } - - public async Task LoadAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(options); - - var cacheKey = CreateCacheKey(options); - if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired()) - { - _logger.LogDebug("Returning cached Rancher hub metadata (expires {Expires}).", cached.ExpiresAt); - return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); - } - - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired()) - { - return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); - } - - CacheEntry? previous = cached; - CacheEntry? entry = null; - - if (options.PreferOfflineSnapshot) - { - entry = TryLoadFromOffline(options); - if (entry is null) - { - throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline discovery snapshot was found."); - } - } - else - { - entry = await TryFetchFromNetworkAsync(options, previous, cancellationToken).ConfigureAwait(false); - if (entry is null) - { - entry = TryLoadFromOffline(options); - } - } - - if (entry is null) - { - throw new InvalidOperationException("Unable to load Rancher hub discovery metadata from network or offline snapshot."); - } - - _memoryCache.Set(cacheKey, entry, new MemoryCacheEntryOptions - { - AbsoluteExpirationRelativeToNow = options.MetadataCacheDuration, - }); - - return new RancherHubMetadataResult(entry.Metadata, entry.FetchedAt, FromCache: false, entry.FromOfflineSnapshot); - } - finally - { - _semaphore.Release(); - } - } - - private async Task TryFetchFromNetworkAsync(RancherHubConnectorOptions options, CacheEntry? previous, CancellationToken cancellationToken) - { - try - { - var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); - using var request = new HttpRequestMessage(HttpMethod.Get, options.DiscoveryUri); - request.Headers.Accept.ParseAdd("application/json"); - - if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) - { - request.Headers.IfNoneMatch.Add(etag); - } - - var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false); - if (token is not null) - { - var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; - request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); - } - - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) - { - _logger.LogDebug("Rancher hub discovery document not modified (etag {ETag}).", previous.ETag); - return previous with - { - FetchedAt = DateTimeOffset.UtcNow, - ExpiresAt = DateTimeOffset.UtcNow + options.MetadataCacheDuration, - FromOfflineSnapshot = false, - }; - } - - response.EnsureSuccessStatusCode(); - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var metadata = ParseMetadata(payload, options); - var entry = new CacheEntry( - metadata, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow + options.MetadataCacheDuration, - response.Headers.ETag?.ToString(), - FromOfflineSnapshot: false, - Payload: payload); - - PersistOfflineSnapshot(options, payload); - return entry; - } - catch (Exception ex) when (ex is not OperationCanceledException && !options.PreferOfflineSnapshot) - { - _logger.LogWarning(ex, "Failed to fetch Rancher hub discovery document; attempting offline snapshot fallback."); - return null; - } - } - - private CacheEntry? TryLoadFromOffline(RancherHubConnectorOptions options) - { - if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return null; - } - - if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) - { - _logger.LogWarning("Rancher hub offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); - return null; - } - - try - { - var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); - var metadata = ParseMetadata(payload, options); - return new CacheEntry( - metadata, - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow + options.MetadataCacheDuration, - ETag: null, - FromOfflineSnapshot: true, - Payload: payload); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load Rancher hub discovery metadata from offline snapshot {Path}.", options.OfflineSnapshotPath); - return null; - } - } - - private void PersistOfflineSnapshot(RancherHubConnectorOptions options, string payload) - { - if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return; - } - - try - { - var directory = _fileSystem.Path.GetDirectoryName(options.OfflineSnapshotPath); - if (!string.IsNullOrWhiteSpace(directory)) - { - _fileSystem.Directory.CreateDirectory(directory); - } - - _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); - _logger.LogDebug("Persisted Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to persist Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); - } - } - - private RancherHubMetadata ParseMetadata(string payload, RancherHubConnectorOptions options) - { - if (string.IsNullOrWhiteSpace(payload)) - { - throw new InvalidOperationException("Rancher hub discovery payload was empty."); - } - - try - { - using var document = JsonDocument.Parse(payload, _documentOptions); - var root = document.RootElement; - - var hubId = ReadString(root, "hubId") ?? "vexer:suse:rancher"; - var title = ReadString(root, "title") ?? ReadString(root, "displayName") ?? "SUSE Rancher VEX Hub"; - var baseUri = ReadUri(root, "baseUri"); - - var subscriptionElement = TryGetProperty(root, "subscription"); - if (!subscriptionElement.HasValue) - { - throw new InvalidOperationException("Discovery payload missing subscription section."); - } - - var subscription = subscriptionElement.Value; - var eventsUri = ReadRequiredUri(subscription, "eventsUri", "eventsUrl", "eventsEndpoint"); - var checkpointUri = ReadUri(subscription, "checkpointUri", "checkpointUrl", "checkpointEndpoint"); - var channels = ReadStringArray(subscription, "channels", "defaultChannels", "products"); - var scopes = ReadStringArray(subscription, "scopes", "defaultScopes"); - var requiresAuth = ReadBoolean(subscription, "requiresAuthentication", defaultValue: options.TokenEndpoint is not null); - - var authenticationElement = TryGetProperty(root, "authentication"); - var tokenEndpointFromMetadata = authenticationElement.HasValue - ? ReadUri(authenticationElement.Value, "tokenUri", "tokenEndpoint") ?? options.TokenEndpoint - : options.TokenEndpoint; - var audience = authenticationElement.HasValue - ? ReadString(authenticationElement.Value, "audience", "aud") ?? options.Audience - : options.Audience; - - var offlineElement = TryGetProperty(root, "offline", "snapshot"); - var offlineSnapshot = offlineElement.HasValue - ? BuildOfflineSnapshot(offlineElement.Value, options) - : null; - - var provider = BuildProvider(hubId, title, baseUri, eventsUri, options); - var subscriptionMetadata = new RancherHubSubscriptionMetadata(eventsUri, checkpointUri, channels, scopes, requiresAuth); - var authenticationMetadata = new RancherHubAuthenticationMetadata(tokenEndpointFromMetadata, audience); - - return new RancherHubMetadata(provider, subscriptionMetadata, authenticationMetadata, offlineSnapshot); - } - catch (JsonException ex) - { - throw new InvalidOperationException("Failed to parse Rancher hub discovery payload.", ex); - } - } - - private RancherHubOfflineSnapshotMetadata? BuildOfflineSnapshot(JsonElement element, RancherHubConnectorOptions options) - { - var snapshotUri = ReadUri(element, "snapshotUri", "uri", "url"); - if (snapshotUri is null) - { - return null; - } - - var checksum = ReadString(element, "sha256", "checksum", "digest"); - DateTimeOffset? updatedAt = null; - var updatedString = ReadString(element, "updated", "lastModified", "timestamp"); - if (!string.IsNullOrWhiteSpace(updatedString) && DateTimeOffset.TryParse(updatedString, out var parsed)) - { - updatedAt = parsed; - } - - return new RancherHubOfflineSnapshotMetadata(snapshotUri, checksum, updatedAt); - } - - private VexProvider BuildProvider(string hubId, string title, Uri? baseUri, Uri eventsUri, RancherHubConnectorOptions options) - { - var baseUris = new List(); - if (baseUri is not null) - { - baseUris.Add(baseUri); - } - baseUris.Add(eventsUri); - - VexCosignTrust? cosign = null; - if (!string.IsNullOrWhiteSpace(options.CosignIssuer) && !string.IsNullOrWhiteSpace(options.CosignIdentityPattern)) - { - cosign = new VexCosignTrust(options.CosignIssuer!, options.CosignIdentityPattern!); - } - - var trust = new VexProviderTrust(options.TrustWeight, cosign, options.PgpFingerprints); - return new VexProvider(hubId, title, VexProviderKind.Hub, baseUris, new VexProviderDiscovery(options.DiscoveryUri, null), trust); - } - - private static string CreateCacheKey(RancherHubConnectorOptions options) - => $"{CachePrefix}:{options.DiscoveryUri}"; - - private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames) - { - foreach (var name in propertyNames) - { - if (element.TryGetProperty(name, out var value) && value.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined) - { - return value; - } - } - - return null; - } - - private static string? ReadString(JsonElement element, params string[] propertyNames) - { - var property = TryGetProperty(element, propertyNames); - if (property is null || property.Value.ValueKind is not JsonValueKind.String) - { - return null; - } - - var value = property.Value.GetString(); - return string.IsNullOrWhiteSpace(value) ? null : value; - } - - private static bool ReadBoolean(JsonElement element, string propertyName, bool defaultValue) - { - if (!element.TryGetProperty(propertyName, out var property)) - { - return defaultValue; - } - - return property.ValueKind switch - { - JsonValueKind.True => true, - JsonValueKind.False => false, - JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed, - _ => defaultValue, - }; - } - - private static ImmutableArray ReadStringArray(JsonElement element, params string[] propertyNames) - { - var property = TryGetProperty(element, propertyNames); - if (property is null) - { - return ImmutableArray.Empty; - } - - if (property.Value.ValueKind is JsonValueKind.Array) - { - var builder = ImmutableArray.CreateBuilder(); - foreach (var item in property.Value.EnumerateArray()) - { - if (item.ValueKind is JsonValueKind.String) - { - var value = item.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - builder.Add(value!); - } - } - } - - return builder.Count == 0 ? ImmutableArray.Empty : builder.ToImmutable(); - } - - if (property.Value.ValueKind is JsonValueKind.String) - { - var single = property.Value.GetString(); - return string.IsNullOrWhiteSpace(single) - ? ImmutableArray.Empty - : ImmutableArray.Create(single!); - } - - return ImmutableArray.Empty; - } - - private static Uri? ReadUri(JsonElement element, params string[] propertyNames) - { - var value = ReadString(element, propertyNames); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException($"Discovery field '{string.Join("/", propertyNames)}' must be an absolute HTTP(S) URI."); - } - - return uri; - } - - private static Uri ReadRequiredUri(JsonElement element, params string[] propertyNames) - { - var uri = ReadUri(element, propertyNames); - if (uri is null) - { - throw new InvalidOperationException($"Discovery payload missing required URI field '{string.Join("/", propertyNames)}'."); - } - - return uri; - } - - private sealed record CacheEntry( - RancherHubMetadata Metadata, - DateTimeOffset FetchedAt, - DateTimeOffset ExpiresAt, - string? ETag, - bool FromOfflineSnapshot, - string? Payload) - { - public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; - } -} - -public sealed record RancherHubMetadata( - VexProvider Provider, - RancherHubSubscriptionMetadata Subscription, - RancherHubAuthenticationMetadata Authentication, - RancherHubOfflineSnapshotMetadata? OfflineSnapshot); - -public sealed record RancherHubSubscriptionMetadata( - Uri EventsUri, - Uri? CheckpointUri, - ImmutableArray Channels, - ImmutableArray Scopes, - bool RequiresAuthentication); - -public sealed record RancherHubAuthenticationMetadata( - Uri? TokenEndpoint, - string? Audience); - -public sealed record RancherHubOfflineSnapshotMetadata( - Uri SnapshotUri, - string? Sha256, - DateTimeOffset? UpdatedAt); - -public sealed record RancherHubMetadataResult( - RancherHubMetadata Metadata, - DateTimeOffset FetchedAt, - bool FromCache, - bool FromOfflineSnapshot); +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; + +public sealed class RancherHubMetadataLoader +{ + public const string CachePrefix = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly RancherHubTokenProvider _tokenProvider; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonDocumentOptions _documentOptions; + + public RancherHubMetadataLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + RancherHubTokenProvider tokenProvider, + IFileSystem fileSystem, + ILogger logger) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _documentOptions = new JsonDocumentOptions + { + CommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }; + } + + public async Task LoadAsync(RancherHubConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired()) + { + _logger.LogDebug("Returning cached Rancher hub metadata (expires {Expires}).", cached.ExpiresAt); + return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired()) + { + return new RancherHubMetadataResult(cached.Metadata, cached.FetchedAt, FromCache: true, cached.FromOfflineSnapshot); + } + + CacheEntry? previous = cached; + CacheEntry? entry = null; + + if (options.PreferOfflineSnapshot) + { + entry = TryLoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline discovery snapshot was found."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, previous, cancellationToken).ConfigureAwait(false); + if (entry is null) + { + entry = TryLoadFromOffline(options); + } + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Rancher hub discovery metadata from network or offline snapshot."); + } + + _memoryCache.Set(cacheKey, entry, new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = options.MetadataCacheDuration, + }); + + return new RancherHubMetadataResult(entry.Metadata, entry.FetchedAt, FromCache: false, entry.FromOfflineSnapshot); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(RancherHubConnectorOptions options, CacheEntry? previous, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, options.DiscoveryUri); + request.Headers.Accept.ParseAdd("application/json"); + + if (!string.IsNullOrWhiteSpace(previous?.ETag) && EntityTagHeaderValue.TryParse(previous.ETag, out var etag)) + { + request.Headers.IfNoneMatch.Add(etag); + } + + var token = await _tokenProvider.GetAccessTokenAsync(options, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; + request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); + } + + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified && previous is not null) + { + _logger.LogDebug("Rancher hub discovery document not modified (etag {ETag}).", previous.ETag); + return previous with + { + FetchedAt = DateTimeOffset.UtcNow, + ExpiresAt = DateTimeOffset.UtcNow + options.MetadataCacheDuration, + FromOfflineSnapshot = false, + }; + } + + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var metadata = ParseMetadata(payload, options); + var entry = new CacheEntry( + metadata, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + options.MetadataCacheDuration, + response.Headers.ETag?.ToString(), + FromOfflineSnapshot: false, + Payload: payload); + + PersistOfflineSnapshot(options, payload); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException && !options.PreferOfflineSnapshot) + { + _logger.LogWarning(ex, "Failed to fetch Rancher hub discovery document; attempting offline snapshot fallback."); + return null; + } + } + + private CacheEntry? TryLoadFromOffline(RancherHubConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Rancher hub offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var metadata = ParseMetadata(payload, options); + return new CacheEntry( + metadata, + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow + options.MetadataCacheDuration, + ETag: null, + FromOfflineSnapshot: true, + Payload: payload); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Rancher hub discovery metadata from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private void PersistOfflineSnapshot(RancherHubConnectorOptions options, string payload) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var directory = _fileSystem.Path.GetDirectoryName(options.OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory)) + { + _fileSystem.Directory.CreateDirectory(directory); + } + + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Rancher hub discovery snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private RancherHubMetadata ParseMetadata(string payload, RancherHubConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Rancher hub discovery payload was empty."); + } + + try + { + using var document = JsonDocument.Parse(payload, _documentOptions); + var root = document.RootElement; + + var hubId = ReadString(root, "hubId") ?? "excititor:suse:rancher"; + var title = ReadString(root, "title") ?? ReadString(root, "displayName") ?? "SUSE Rancher VEX Hub"; + var baseUri = ReadUri(root, "baseUri"); + + var subscriptionElement = TryGetProperty(root, "subscription"); + if (!subscriptionElement.HasValue) + { + throw new InvalidOperationException("Discovery payload missing subscription section."); + } + + var subscription = subscriptionElement.Value; + var eventsUri = ReadRequiredUri(subscription, "eventsUri", "eventsUrl", "eventsEndpoint"); + var checkpointUri = ReadUri(subscription, "checkpointUri", "checkpointUrl", "checkpointEndpoint"); + var channels = ReadStringArray(subscription, "channels", "defaultChannels", "products"); + var scopes = ReadStringArray(subscription, "scopes", "defaultScopes"); + var requiresAuth = ReadBoolean(subscription, "requiresAuthentication", defaultValue: options.TokenEndpoint is not null); + + var authenticationElement = TryGetProperty(root, "authentication"); + var tokenEndpointFromMetadata = authenticationElement.HasValue + ? ReadUri(authenticationElement.Value, "tokenUri", "tokenEndpoint") ?? options.TokenEndpoint + : options.TokenEndpoint; + var audience = authenticationElement.HasValue + ? ReadString(authenticationElement.Value, "audience", "aud") ?? options.Audience + : options.Audience; + + var offlineElement = TryGetProperty(root, "offline", "snapshot"); + var offlineSnapshot = offlineElement.HasValue + ? BuildOfflineSnapshot(offlineElement.Value, options) + : null; + + var provider = BuildProvider(hubId, title, baseUri, eventsUri, options); + var subscriptionMetadata = new RancherHubSubscriptionMetadata(eventsUri, checkpointUri, channels, scopes, requiresAuth); + var authenticationMetadata = new RancherHubAuthenticationMetadata(tokenEndpointFromMetadata, audience); + + return new RancherHubMetadata(provider, subscriptionMetadata, authenticationMetadata, offlineSnapshot); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Failed to parse Rancher hub discovery payload.", ex); + } + } + + private RancherHubOfflineSnapshotMetadata? BuildOfflineSnapshot(JsonElement element, RancherHubConnectorOptions options) + { + var snapshotUri = ReadUri(element, "snapshotUri", "uri", "url"); + if (snapshotUri is null) + { + return null; + } + + var checksum = ReadString(element, "sha256", "checksum", "digest"); + DateTimeOffset? updatedAt = null; + var updatedString = ReadString(element, "updated", "lastModified", "timestamp"); + if (!string.IsNullOrWhiteSpace(updatedString) && DateTimeOffset.TryParse(updatedString, out var parsed)) + { + updatedAt = parsed; + } + + return new RancherHubOfflineSnapshotMetadata(snapshotUri, checksum, updatedAt); + } + + private VexProvider BuildProvider(string hubId, string title, Uri? baseUri, Uri eventsUri, RancherHubConnectorOptions options) + { + var baseUris = new List(); + if (baseUri is not null) + { + baseUris.Add(baseUri); + } + baseUris.Add(eventsUri); + + VexCosignTrust? cosign = null; + if (!string.IsNullOrWhiteSpace(options.CosignIssuer) && !string.IsNullOrWhiteSpace(options.CosignIdentityPattern)) + { + cosign = new VexCosignTrust(options.CosignIssuer!, options.CosignIdentityPattern!); + } + + var trust = new VexProviderTrust(options.TrustWeight, cosign, options.PgpFingerprints); + return new VexProvider(hubId, title, VexProviderKind.Hub, baseUris, new VexProviderDiscovery(options.DiscoveryUri, null), trust); + } + + private static string CreateCacheKey(RancherHubConnectorOptions options) + => $"{CachePrefix}:{options.DiscoveryUri}"; + + private static JsonElement? TryGetProperty(JsonElement element, params string[] propertyNames) + { + foreach (var name in propertyNames) + { + if (element.TryGetProperty(name, out var value) && value.ValueKind is not JsonValueKind.Null and not JsonValueKind.Undefined) + { + return value; + } + } + + return null; + } + + private static string? ReadString(JsonElement element, params string[] propertyNames) + { + var property = TryGetProperty(element, propertyNames); + if (property is null || property.Value.ValueKind is not JsonValueKind.String) + { + return null; + } + + var value = property.Value.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + private static bool ReadBoolean(JsonElement element, string propertyName, bool defaultValue) + { + if (!element.TryGetProperty(propertyName, out var property)) + { + return defaultValue; + } + + return property.ValueKind switch + { + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.String when bool.TryParse(property.GetString(), out var parsed) => parsed, + _ => defaultValue, + }; + } + + private static ImmutableArray ReadStringArray(JsonElement element, params string[] propertyNames) + { + var property = TryGetProperty(element, propertyNames); + if (property is null) + { + return ImmutableArray.Empty; + } + + if (property.Value.ValueKind is JsonValueKind.Array) + { + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in property.Value.EnumerateArray()) + { + if (item.ValueKind is JsonValueKind.String) + { + var value = item.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + builder.Add(value!); + } + } + } + + return builder.Count == 0 ? ImmutableArray.Empty : builder.ToImmutable(); + } + + if (property.Value.ValueKind is JsonValueKind.String) + { + var single = property.Value.GetString(); + return string.IsNullOrWhiteSpace(single) + ? ImmutableArray.Empty + : ImmutableArray.Create(single!); + } + + return ImmutableArray.Empty; + } + + private static Uri? ReadUri(JsonElement element, params string[] propertyNames) + { + var value = ReadString(element, propertyNames); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException($"Discovery field '{string.Join("/", propertyNames)}' must be an absolute HTTP(S) URI."); + } + + return uri; + } + + private static Uri ReadRequiredUri(JsonElement element, params string[] propertyNames) + { + var uri = ReadUri(element, propertyNames); + if (uri is null) + { + throw new InvalidOperationException($"Discovery payload missing required URI field '{string.Join("/", propertyNames)}'."); + } + + return uri; + } + + private sealed record CacheEntry( + RancherHubMetadata Metadata, + DateTimeOffset FetchedAt, + DateTimeOffset ExpiresAt, + string? ETag, + bool FromOfflineSnapshot, + string? Payload) + { + public bool IsExpired() => DateTimeOffset.UtcNow >= ExpiresAt; + } +} + +public sealed record RancherHubMetadata( + VexProvider Provider, + RancherHubSubscriptionMetadata Subscription, + RancherHubAuthenticationMetadata Authentication, + RancherHubOfflineSnapshotMetadata? OfflineSnapshot); + +public sealed record RancherHubSubscriptionMetadata( + Uri EventsUri, + Uri? CheckpointUri, + ImmutableArray Channels, + ImmutableArray Scopes, + bool RequiresAuthentication); + +public sealed record RancherHubAuthenticationMetadata( + Uri? TokenEndpoint, + string? Audience); + +public sealed record RancherHubOfflineSnapshotMetadata( + Uri SnapshotUri, + string? Sha256, + DateTimeOffset? UpdatedAt); + +public sealed record RancherHubMetadataResult( + RancherHubMetadata Metadata, + DateTimeOffset FetchedAt, + bool FromCache, + bool FromOfflineSnapshot); diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs new file mode 100644 index 00000000..bca605af --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs @@ -0,0 +1,344 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Authentication; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Configuration; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Events; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Metadata; +using StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub; + +public sealed class RancherHubConnector : VexConnectorBase +{ + private static readonly VexConnectorDescriptor StaticDescriptor = new( + id: "excititor:suse.rancher", + kind: VexProviderKind.Hub, + displayName: "SUSE Rancher VEX Hub") + { + Tags = ImmutableArray.Create("hub", "suse", "offline"), + }; + + private readonly RancherHubMetadataLoader _metadataLoader; + private readonly RancherHubEventClient _eventClient; + private readonly RancherHubCheckpointManager _checkpointManager; + private readonly RancherHubTokenProvider _tokenProvider; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IEnumerable> _validators; + + private RancherHubConnectorOptions? _options; + private RancherHubMetadataResult? _metadata; + + public RancherHubConnector( + RancherHubMetadataLoader metadataLoader, + RancherHubEventClient eventClient, + RancherHubCheckpointManager checkpointManager, + RancherHubTokenProvider tokenProvider, + IHttpClientFactory httpClientFactory, + ILogger logger, + TimeProvider timeProvider, + IEnumerable>? validators = null) + : base(StaticDescriptor, logger, timeProvider) + { + _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); + _eventClient = eventClient ?? throw new ArgumentNullException(nameof(eventClient)); + _checkpointManager = checkpointManager ?? throw new ArgumentNullException(nameof(checkpointManager)); + _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary + { + ["discoveryUri"] = _options.DiscoveryUri.ToString(), + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, + ["fromOffline"] = _metadata.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_metadata is null) + { + _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + var checkpoint = await _checkpointManager.LoadAsync(Descriptor.Id, context, cancellationToken).ConfigureAwait(false); + var digestHistory = checkpoint.Digests.ToList(); + var dedupeSet = new HashSet(checkpoint.Digests, StringComparer.OrdinalIgnoreCase); + var latestCursor = checkpoint.Cursor; + var latestPublishedAt = checkpoint.LastPublishedAt ?? checkpoint.EffectiveSince; + var stateChanged = false; + + LogConnectorEvent(LogLevel.Information, "fetch_start", "Starting Rancher hub event ingestion.", new Dictionary + { + ["since"] = checkpoint.EffectiveSince?.ToString("O"), + ["cursor"] = checkpoint.Cursor, + ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), + ["offline"] = checkpoint.Cursor is null && _options.PreferOfflineSnapshot, + }); + + await foreach (var batch in _eventClient.FetchEventBatchesAsync( + _options, + _metadata.Metadata, + checkpoint.Cursor, + checkpoint.EffectiveSince, + _metadata.Metadata.Subscription.Channels, + cancellationToken).ConfigureAwait(false)) + { + LogConnectorEvent(LogLevel.Debug, "batch", "Processing Rancher hub batch.", new Dictionary + { + ["cursor"] = batch.Cursor, + ["nextCursor"] = batch.NextCursor, + ["count"] = batch.Events.Length, + ["offline"] = batch.FromOfflineSnapshot, + }); + + if (!string.IsNullOrWhiteSpace(batch.NextCursor) && !string.Equals(batch.NextCursor, latestCursor, StringComparison.Ordinal)) + { + latestCursor = batch.NextCursor; + stateChanged = true; + } + else if (string.IsNullOrWhiteSpace(latestCursor) && !string.IsNullOrWhiteSpace(batch.Cursor)) + { + latestCursor = batch.Cursor; + } + + foreach (var record in batch.Events) + { + cancellationToken.ThrowIfCancellationRequested(); + + var result = await ProcessEventAsync(record, batch, context, dedupeSet, digestHistory, cancellationToken).ConfigureAwait(false); + if (result.ProcessedDocument is not null) + { + yield return result.ProcessedDocument; + stateChanged = true; + if (result.PublishedAt is { } published && (latestPublishedAt is null || published > latestPublishedAt)) + { + latestPublishedAt = published; + } + } + else if (result.Quarantined) + { + stateChanged = true; + } + } + } + + if (stateChanged || !string.Equals(latestCursor, checkpoint.Cursor, StringComparison.Ordinal) || latestPublishedAt != checkpoint.LastPublishedAt) + { + await _checkpointManager.SaveAsync( + Descriptor.Id, + latestCursor, + latestPublishedAt, + digestHistory.ToImmutableArray(), + cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); + + public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata; + + private async Task ProcessEventAsync( + RancherHubEventRecord record, + RancherHubEventBatch batch, + VexConnectorContext context, + HashSet dedupeSet, + List digestHistory, + CancellationToken cancellationToken) + { + var quarantineKey = BuildQuarantineKey(record); + if (dedupeSet.Contains(quarantineKey)) + { + return EventProcessingResult.QuarantinedOnly; + } + + if (record.DocumentUri is null || string.IsNullOrWhiteSpace(record.Id)) + { + await QuarantineAsync(record, batch, "missing documentUri or id", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + + var client = _httpClientFactory.CreateClient(RancherHubConnectorOptions.HttpClientName); + using var request = await CreateDocumentRequestAsync(record.DocumentUri, cancellationToken).ConfigureAwait(false); + using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + await QuarantineAsync(record, batch, $"document fetch failed ({(int)response.StatusCode} {response.StatusCode})", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + + var contentBytes = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + var publishedAt = record.PublishedAt ?? UtcNow(); + var metadata = BuildMetadata(builder => builder + .Add("rancher.event.id", record.Id) + .Add("rancher.event.type", record.Type) + .Add("rancher.event.channel", record.Channel) + .Add("rancher.event.published", publishedAt) + .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) + .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false") + .Add("rancher.event.declaredDigest", record.DocumentDigest)); + + var format = ResolveFormat(record.DocumentFormat); + var document = CreateRawDocument(format, record.DocumentUri, contentBytes, metadata); + + if (!string.IsNullOrWhiteSpace(record.DocumentDigest)) + { + var declared = NormalizeDigest(record.DocumentDigest); + var computed = NormalizeDigest(document.Digest); + if (!string.Equals(declared, computed, StringComparison.OrdinalIgnoreCase)) + { + await QuarantineAsync(record, batch, $"digest mismatch (declared {record.DocumentDigest}, computed {document.Digest})", context, cancellationToken).ConfigureAwait(false); + AddQuarantineDigest(quarantineKey, dedupeSet, digestHistory); + return EventProcessingResult.QuarantinedOnly; + } + } + + if (!dedupeSet.Add(document.Digest)) + { + return EventProcessingResult.Skipped; + } + + digestHistory.Add(document.Digest); + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + return new EventProcessingResult(document, false, publishedAt); + } + + private async Task CreateDocumentRequestAsync(Uri documentUri, CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Get, documentUri); + if (_metadata?.Metadata.Subscription.RequiresAuthentication ?? false) + { + var token = await _tokenProvider.GetAccessTokenAsync(_options!, cancellationToken).ConfigureAwait(false); + if (token is not null) + { + var scheme = string.IsNullOrWhiteSpace(token.TokenType) ? "Bearer" : token.TokenType; + request.Headers.Authorization = new AuthenticationHeaderValue(scheme, token.Value); + } + } + + return request; + } + + private async Task QuarantineAsync( + RancherHubEventRecord record, + RancherHubEventBatch batch, + string reason, + VexConnectorContext context, + CancellationToken cancellationToken) + { + var metadata = BuildMetadata(builder => builder + .Add("rancher.event.id", record.Id) + .Add("rancher.event.type", record.Type) + .Add("rancher.event.channel", record.Channel) + .Add("rancher.event.quarantine", "true") + .Add("rancher.event.error", reason) + .Add("rancher.event.cursor", batch.NextCursor ?? batch.Cursor) + .Add("rancher.event.offline", batch.FromOfflineSnapshot ? "true" : "false")); + + var sourceUri = record.DocumentUri ?? _metadata?.Metadata.Subscription.EventsUri ?? _options!.DiscoveryUri; + var payload = Encoding.UTF8.GetBytes(record.RawJson); + var document = CreateRawDocument(VexDocumentFormat.Csaf, sourceUri, payload, metadata); + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + + LogConnectorEvent(LogLevel.Warning, "quarantine", "Rancher hub event moved to quarantine.", new Dictionary + { + ["eventId"] = record.Id ?? "(missing)", + ["reason"] = reason, + }); + } + + private static void AddQuarantineDigest(string key, HashSet dedupeSet, List digestHistory) + { + if (dedupeSet.Add(key)) + { + digestHistory.Add(key); + } + } + + private static string BuildQuarantineKey(RancherHubEventRecord record) + { + if (!string.IsNullOrWhiteSpace(record.Id)) + { + return $"quarantine:{record.Id}"; + } + + Span hash = stackalloc byte[32]; + var bytes = Encoding.UTF8.GetBytes(record.RawJson); + if (!SHA256.TryHashData(bytes, hash, out _)) + { + using var sha = SHA256.Create(); + hash = sha.ComputeHash(bytes); + } + + return $"quarantine:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static string NormalizeDigest(string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return digest; + } + + var trimmed = digest.Trim(); + return trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) + ? trimmed.ToLowerInvariant() + : $"sha256:{trimmed.ToLowerInvariant()}"; + } + + private static VexDocumentFormat ResolveFormat(string? format) + { + if (string.IsNullOrWhiteSpace(format)) + { + return VexDocumentFormat.Csaf; + } + + return format.ToLowerInvariant() switch + { + "csaf" or "csaf_json" or "json" => VexDocumentFormat.Csaf, + "cyclonedx" or "cyclonedx_vex" => VexDocumentFormat.CycloneDx, + "openvex" => VexDocumentFormat.OpenVex, + "oci" or "oci_attestation" or "attestation" => VexDocumentFormat.OciAttestation, + _ => VexDocumentFormat.Csaf, + }; + } + + private sealed record EventProcessingResult(VexRawDocument? ProcessedDocument, bool Quarantined, DateTimeOffset? PublishedAt) + { + public static EventProcessingResult QuarantinedOnly { get; } = new(null, true, null); + + public static EventProcessingResult Skipped { get; } = new(null, false, null); + } +} diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs new file mode 100644 index 00000000..73c22674 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/State/RancherHubCheckpointManager.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.State; + +internal sealed record RancherHubCheckpointState( + string? Cursor, + DateTimeOffset? LastPublishedAt, + DateTimeOffset? EffectiveSince, + ImmutableArray Digests); + +internal sealed class RancherHubCheckpointManager +{ + private const string CheckpointPrefix = "checkpoint:"; + private readonly IVexConnectorStateRepository _repository; + + public RancherHubCheckpointManager(IVexConnectorStateRepository repository) + { + _repository = repository ?? throw new ArgumentNullException(nameof(repository)); + } + + public async ValueTask LoadAsync(string connectorId, VexConnectorContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var state = await _repository.GetAsync(connectorId, cancellationToken).ConfigureAwait(false); + var cursor = ExtractCursor(state?.DocumentDigests ?? ImmutableArray.Empty); + var digests = ExtractDigests(state?.DocumentDigests ?? ImmutableArray.Empty); + var lastPublishedAt = state?.LastUpdated; + var effectiveSince = context.Since; + + if (context.Settings.Values.TryGetValue("checkpoint", out var checkpointOverride) && !string.IsNullOrWhiteSpace(checkpointOverride)) + { + cursor = checkpointOverride; + digests = ImmutableArray.Empty; + } + + if (effectiveSince is null && lastPublishedAt is not null) + { + effectiveSince = lastPublishedAt; + } + + if (effectiveSince is not null && lastPublishedAt is not null && effectiveSince < lastPublishedAt) + { + digests = ImmutableArray.Empty; + } + + return new RancherHubCheckpointState(cursor, lastPublishedAt, effectiveSince, digests); + } + + public ValueTask SaveAsync(string connectorId, string? cursor, DateTimeOffset? lastPublishedAt, ImmutableArray digests, CancellationToken cancellationToken) + { + var entries = ImmutableArray.CreateBuilder(); + if (!string.IsNullOrWhiteSpace(cursor)) + { + entries.Add($"{CheckpointPrefix}{cursor}"); + } + + foreach (var digest in digests) + { + if (string.IsNullOrWhiteSpace(digest)) + { + continue; + } + + if (digest.StartsWith(CheckpointPrefix, StringComparison.Ordinal)) + { + continue; + } + + entries.Add(digest); + } + + var state = new VexConnectorState(connectorId, lastPublishedAt, entries.ToImmutable()); + return _repository.SaveAsync(state, cancellationToken); + } + + private static string? ExtractCursor(ImmutableArray digests) + { + foreach (var entry in digests) + { + if (entry.StartsWith(CheckpointPrefix, StringComparison.Ordinal)) + { + return entry[CheckpointPrefix.Length..]; + } + } + + return null; + } + + private static ImmutableArray ExtractDigests(ImmutableArray digests) + => digests.Where(d => !d.StartsWith(CheckpointPrefix, StringComparison.Ordinal)).ToImmutableArray(); +} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj similarity index 66% rename from src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj rename to src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj index a99a942f..b3e1398a 100644 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/StellaOps.Vexer.Connectors.MSRC.CSAF.csproj +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj @@ -1,18 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md new file mode 100644 index 00000000..56ba1654 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.SUSE.RancherVEXHub/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-SUSE-01-001 – Rancher hub discovery & auth|Team Excititor Connectors – SUSE|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Rancher hub options/token provider, discovery metadata loader with offline snapshots + caching, connector shell, DI wiring, and unit tests covering network/offline paths.| +|EXCITITOR-CONN-SUSE-01-002 – Checkpointed event ingestion|Team Excititor Connectors – SUSE|EXCITITOR-CONN-SUSE-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads.
          2025-10-19: Prereqs EXCITITOR-CONN-SUSE-01-001 & EXCITITOR-STORAGE-01-003 confirmed complete; initiating checkpoint/resume implementation plan.| +|EXCITITOR-CONN-SUSE-01-003 – Trust metadata & policy hints|Team Excititor Connectors – SUSE|EXCITITOR-CONN-SUSE-01-002, EXCITITOR-POLICY-01-001|TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine.| diff --git a/src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md b/src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md new file mode 100644 index 00000000..43064222 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.StellaOpsMirror/TASKS.md @@ -0,0 +1,7 @@ +# StellaOps Mirror VEX Connector Task Board (Sprint 7) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| EXCITITOR-CONN-STELLA-07-001 | TODO | Excititor Connectors – Stella | EXCITITOR-EXPORT-01-007 | Implement mirror fetch client consuming `https://.stella-ops.org/excititor/exports/index.json`, validating signatures/digests, storing raw consensus bundles with provenance. | Fetch job downloads mirror manifest, verifies DSSE/signature, stores raw documents + provenance; unit tests cover happy path and tampered manifest failure. | +| EXCITITOR-CONN-STELLA-07-002 | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-001 | Normalize mirror bundles into VexClaim sets referencing original provider metadata and mirror provenance. | Normalizer emits VexClaims with mirror provenance + policy metadata, fixtures assert deterministic output parity vs local exports. | +| EXCITITOR-CONN-STELLA-07-003 | TODO | Excititor Connectors – Stella | EXCITITOR-CONN-STELLA-07-002 | Implement incremental cursor handling per-export digest, support resume, and document configuration for downstream Excititor mirrors. | Connector resumes from last export digest, handles delta/export rotation, docs show configuration; integration test covers resume + new export ingest. | diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs new file mode 100644 index 00000000..84db1062 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Connectors/UbuntuCsafConnectorTests.cs @@ -0,0 +1,309 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; +using System.IO.Abstractions.TestingHelpers; +using Xunit; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Connectors; + +public sealed class UbuntuCsafConnectorTests +{ + [Fact] + public async Task FetchAsync_IngestsNewDocument_UpdatesStateAndUsesEtag() + { + var baseUri = new Uri("https://ubuntu.test/security/csaf/"); + var indexUri = new Uri(baseUri, "index.json"); + var catalogUri = new Uri(baseUri, "stable/catalog.json"); + var advisoryUri = new Uri(baseUri, "stable/USN-2025-0001.json"); + + var manifest = CreateTestManifest(advisoryUri, "USN-2025-0001", "2025-10-18T00:00:00Z"); + var documentPayload = Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"); + var documentSha = ComputeSha256(documentPayload); + + var indexJson = manifest.IndexJson; + var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", documentSha, StringComparison.Ordinal); + var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, documentPayload, expectedEtag: "etag-123"); + + var httpClient = new HttpClient(handler); + var httpFactory = new SingleClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); + + var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); + var stateRepository = new InMemoryConnectorStateRepository(); + var connector = new UbuntuCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { optionsValidator }, + NullLogger.Instance, + TimeProvider.System); + + var settings = new VexConnectorSettings(ImmutableDictionary.Empty); + await connector.ValidateAsync(settings, CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().HaveCount(1); + sink.Documents.Should().HaveCount(1); + var stored = sink.Documents.Single(); + stored.Digest.Should().Be($"sha256:{documentSha}"); + stored.Metadata.TryGetValue("ubuntu.etag", out var storedEtag).Should().BeTrue(); + storedEtag.Should().Be("etag-123"); + + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().Contain($"sha256:{documentSha}"); + stateRepository.CurrentState.DocumentDigests.Should().Contain($"etag:{advisoryUri}|etag-123"); + stateRepository.CurrentState.LastUpdated.Should().Be(DateTimeOffset.Parse("2025-10-18T00:00:00Z")); + + handler.DocumentRequestCount.Should().Be(1); + + // Second run: Expect connector to send If-None-Match and skip download via 304. + sink.Documents.Clear(); + documents.Clear(); + + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + handler.DocumentRequestCount.Should().Be(2); + handler.SeenIfNoneMatch.Should().Contain("\"etag-123\""); + } + + [Fact] + public async Task FetchAsync_SkipsWhenChecksumMismatch() + { + var baseUri = new Uri("https://ubuntu.test/security/csaf/"); + var indexUri = new Uri(baseUri, "index.json"); + var catalogUri = new Uri(baseUri, "stable/catalog.json"); + var advisoryUri = new Uri(baseUri, "stable/USN-2025-0002.json"); + + var manifest = CreateTestManifest(advisoryUri, "USN-2025-0002", "2025-10-18T00:00:00Z"); + var indexJson = manifest.IndexJson; + var catalogJson = manifest.CatalogJson.Replace("{{SHA256}}", new string('a', 64), StringComparison.Ordinal); + var handler = new UbuntuTestHttpHandler(indexUri, indexJson, catalogUri, catalogJson, advisoryUri, Encoding.UTF8.GetBytes("{\"document\":\"payload\"}"), expectedEtag: "etag-999"); + + var httpClient = new HttpClient(handler); + var httpFactory = new SingleClientFactory(httpClient); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(httpFactory, cache, fileSystem, NullLogger.Instance, TimeProvider.System); + var optionsValidator = new UbuntuConnectorOptionsValidator(fileSystem); + var stateRepository = new InMemoryConnectorStateRepository(); + + var connector = new UbuntuCsafConnector( + loader, + httpFactory, + stateRepository, + new[] { optionsValidator }, + NullLogger.Instance, + TimeProvider.System); + + await connector.ValidateAsync(new VexConnectorSettings(ImmutableDictionary.Empty), CancellationToken.None); + + var sink = new InMemoryRawSink(); + var context = new VexConnectorContext(null, VexConnectorSettings.Empty, sink, new NoopSignatureVerifier(), new NoopNormalizerRouter(), new ServiceCollection().BuildServiceProvider()); + + var documents = new List(); + await foreach (var doc in connector.FetchAsync(context, CancellationToken.None)) + { + documents.Add(doc); + } + + documents.Should().BeEmpty(); + sink.Documents.Should().BeEmpty(); + stateRepository.CurrentState.Should().NotBeNull(); + stateRepository.CurrentState!.DocumentDigests.Should().BeEmpty(); + handler.DocumentRequestCount.Should().Be(1); + } + + private static (string IndexJson, string CatalogJson) CreateTestManifest(Uri advisoryUri, string advisoryId, string timestamp) + { + var indexJson = $$""" + { + "generated": "2025-10-18T00:00:00Z", + "channels": [ + { + "name": "stable", + "catalogUrl": "{{advisoryUri.GetLeftPart(UriPartial.Authority)}}/security/csaf/stable/catalog.json", + "sha256": "ignore" + } + ] + } + """; + + var catalogJson = $$""" + { + "resources": [ + { + "id": "{{advisoryId}}", + "type": "csaf", + "url": "{{advisoryUri}}", + "last_modified": "{{timestamp}}", + "hashes": { + "sha256": "{{SHA256}}" + }, + "etag": "\"etag-123\"", + "title": "{{advisoryId}}" + } + ] + } + """; + + return (indexJson, catalogJson); + } + + private static string ComputeSha256(ReadOnlySpan payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private sealed class SingleClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class UbuntuTestHttpHandler : HttpMessageHandler + { + private readonly Uri _indexUri; + private readonly string _indexPayload; + private readonly Uri _catalogUri; + private readonly string _catalogPayload; + private readonly Uri _documentUri; + private readonly byte[] _documentPayload; + private readonly string _expectedEtag; + + public int DocumentRequestCount { get; private set; } + public List SeenIfNoneMatch { get; } = new(); + + public UbuntuTestHttpHandler(Uri indexUri, string indexPayload, Uri catalogUri, string catalogPayload, Uri documentUri, byte[] documentPayload, string expectedEtag) + { + _indexUri = indexUri; + _indexPayload = indexPayload; + _catalogUri = catalogUri; + _catalogPayload = catalogPayload; + _documentUri = documentUri; + _documentPayload = documentPayload; + _expectedEtag = expectedEtag; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.RequestUri == _indexUri) + { + return Task.FromResult(CreateJsonResponse(_indexPayload)); + } + + if (request.RequestUri == _catalogUri) + { + return Task.FromResult(CreateJsonResponse(_catalogPayload)); + } + + if (request.RequestUri == _documentUri) + { + DocumentRequestCount++; + if (request.Headers.IfNoneMatch is { Count: > 0 }) + { + var header = request.Headers.IfNoneMatch.First().ToString(); + SeenIfNoneMatch.Add(header); + if (header.Trim('"') == _expectedEtag || header == $"\"{_expectedEtag}\"") + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotModified)); + } + } + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(_documentPayload), + }; + response.Headers.ETag = new EntityTagHeaderValue($"\"{_expectedEtag}\""); + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + return Task.FromResult(response); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) + { + Content = new StringContent($"No response configured for {request.RequestUri}"), + }); + } + + private static HttpResponseMessage CreateJsonResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + private sealed class InMemoryConnectorStateRepository : IVexConnectorStateRepository + { + public VexConnectorState? CurrentState { get; private set; } + + public ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) + => ValueTask.FromResult(CurrentState); + + public ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) + { + CurrentState = state; + return ValueTask.CompletedTask; + } + } + + private sealed class InMemoryRawSink : IVexRawDocumentSink + { + public List Documents { get; } = new(); + + public ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + { + Documents.Add(document); + return ValueTask.CompletedTask; + } + } + + private sealed class NoopSignatureVerifier : IVexSignatureVerifier + { + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class NoopNormalizerRouter : IVexNormalizerRouter + { + public ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexClaimBatch(document, ImmutableArray.Empty, ImmutableDictionary.Empty)); + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs similarity index 94% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs index 363fb91d..9f868e22 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/Metadata/UbuntuCatalogLoaderTests.cs @@ -1,172 +1,172 @@ -using System.Collections.Generic; -using System.Net; -using System.Net.Http; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; -using System.IO.Abstractions.TestingHelpers; -using Xunit; -using System.Threading; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.Metadata; - -public sealed class UbuntuCatalogLoaderTests -{ - private const string SampleIndex = """ - { - "generated": "2025-10-10T00:00:00Z", - "channels": [ - { - "name": "stable", - "catalogUrl": "https://ubuntu.com/security/csaf/stable/catalog.json", - "sha256": "abc", - "lastUpdated": "2025-10-09T10:00:00Z" - }, - { - "name": "esm", - "catalogUrl": "https://ubuntu.com/security/csaf/esm/catalog.json", - "sha256": "def", - "lastUpdated": "2025-10-08T10:00:00Z" - } - ] - } - """; - - [Fact] - public async Task LoadAsync_FetchesAndCachesIndex() - { - var handler = new TestHttpMessageHandler(new Dictionary - { - [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), - }); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); - - var options = new UbuntuConnectorOptions - { - IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), - OfflineSnapshotPath = "/snapshots/ubuntu-index.json", - }; - - var result = await loader.LoadAsync(options, CancellationToken.None); - result.Metadata.Channels.Should().HaveCount(1); - result.Metadata.Channels[0].Name.Should().Be("stable"); - fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); - result.FromCache.Should().BeFalse(); - - handler.ResetInvocationCount(); - var cached = await loader.LoadAsync(options, CancellationToken.None); - cached.FromCache.Should().BeTrue(); - handler.InvocationCount.Should().Be(0); - } - - [Fact] - public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() - { - var handler = new TestHttpMessageHandler(new Dictionary()); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - fileSystem.AddFile("/snapshots/ubuntu-index.json", new MockFileData($"{{\"metadata\":{SampleIndex},\"fetchedAt\":\"2025-10-10T00:00:00Z\"}}")); - var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); - - var options = new UbuntuConnectorOptions - { - IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), - OfflineSnapshotPath = "/snapshots/ubuntu-index.json", - PreferOfflineSnapshot = true, - Channels = { "stable" } - }; - - var result = await loader.LoadAsync(options, CancellationToken.None); - result.FromOfflineSnapshot.Should().BeTrue(); - result.Metadata.Channels.Should().NotBeEmpty(); - } - - [Fact] - public async Task LoadAsync_ThrowsWhenNoChannelsMatch() - { - var handler = new TestHttpMessageHandler(new Dictionary - { - [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), - }); - var client = new HttpClient(handler); - var factory = new SingleClientHttpClientFactory(client); - var cache = new MemoryCache(new MemoryCacheOptions()); - var fileSystem = new MockFileSystem(); - var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); - - var options = new UbuntuConnectorOptions - { - IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), - }; - options.Channels.Clear(); - options.Channels.Add("nonexistent"); - - await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); - } - - private static HttpResponseMessage CreateResponse(string payload) - => new(HttpStatusCode.OK) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - - private sealed class SingleClientHttpClientFactory : IHttpClientFactory - { - private readonly HttpClient _client; - - public SingleClientHttpClientFactory(HttpClient client) - { - _client = client; - } - - public HttpClient CreateClient(string name) => _client; - } - - private sealed class AdjustableTimeProvider : TimeProvider - { - private DateTimeOffset _now = DateTimeOffset.UtcNow; - - public override DateTimeOffset GetUtcNow() => _now; - } - - private sealed class TestHttpMessageHandler : HttpMessageHandler - { - private readonly Dictionary _responses; - - public TestHttpMessageHandler(Dictionary responses) - { - _responses = responses; - } - - public int InvocationCount { get; private set; } - - public void ResetInvocationCount() => InvocationCount = 0; - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - InvocationCount++; - if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) - { - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - return new HttpResponseMessage(response.StatusCode) - { - Content = new StringContent(payload, Encoding.UTF8, "application/json"), - }; - } - - return new HttpResponseMessage(HttpStatusCode.InternalServerError) - { - Content = new StringContent("unexpected request"), - }; - } - } -} +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using System.IO.Abstractions.TestingHelpers; +using Xunit; +using System.Threading; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.Metadata; + +public sealed class UbuntuCatalogLoaderTests +{ + private const string SampleIndex = """ + { + "generated": "2025-10-10T00:00:00Z", + "channels": [ + { + "name": "stable", + "catalogUrl": "https://ubuntu.com/security/csaf/stable/catalog.json", + "sha256": "abc", + "lastUpdated": "2025-10-09T10:00:00Z" + }, + { + "name": "esm", + "catalogUrl": "https://ubuntu.com/security/csaf/esm/catalog.json", + "sha256": "def", + "lastUpdated": "2025-10-08T10:00:00Z" + } + ] + } + """; + + [Fact] + public async Task LoadAsync_FetchesAndCachesIndex() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + OfflineSnapshotPath = "/snapshots/ubuntu-index.json", + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.Metadata.Channels.Should().HaveCount(1); + result.Metadata.Channels[0].Name.Should().Be("stable"); + fileSystem.FileExists(options.OfflineSnapshotPath!).Should().BeTrue(); + result.FromCache.Should().BeFalse(); + + handler.ResetInvocationCount(); + var cached = await loader.LoadAsync(options, CancellationToken.None); + cached.FromCache.Should().BeTrue(); + handler.InvocationCount.Should().Be(0); + } + + [Fact] + public async Task LoadAsync_UsesOfflineSnapshotWhenPreferred() + { + var handler = new TestHttpMessageHandler(new Dictionary()); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + fileSystem.AddFile("/snapshots/ubuntu-index.json", new MockFileData($"{{\"metadata\":{SampleIndex},\"fetchedAt\":\"2025-10-10T00:00:00Z\"}}")); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + OfflineSnapshotPath = "/snapshots/ubuntu-index.json", + PreferOfflineSnapshot = true, + Channels = { "stable" } + }; + + var result = await loader.LoadAsync(options, CancellationToken.None); + result.FromOfflineSnapshot.Should().BeTrue(); + result.Metadata.Channels.Should().NotBeEmpty(); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenNoChannelsMatch() + { + var handler = new TestHttpMessageHandler(new Dictionary + { + [new Uri("https://ubuntu.com/security/csaf/index.json")] = CreateResponse(SampleIndex), + }); + var client = new HttpClient(handler); + var factory = new SingleClientHttpClientFactory(client); + var cache = new MemoryCache(new MemoryCacheOptions()); + var fileSystem = new MockFileSystem(); + var loader = new UbuntuCatalogLoader(factory, cache, fileSystem, NullLogger.Instance, new AdjustableTimeProvider()); + + var options = new UbuntuConnectorOptions + { + IndexUri = new Uri("https://ubuntu.com/security/csaf/index.json"), + }; + options.Channels.Clear(); + options.Channels.Add("nonexistent"); + + await Assert.ThrowsAsync(() => loader.LoadAsync(options, CancellationToken.None)); + } + + private static HttpResponseMessage CreateResponse(string payload) + => new(HttpStatusCode.OK) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + + private sealed class SingleClientHttpClientFactory : IHttpClientFactory + { + private readonly HttpClient _client; + + public SingleClientHttpClientFactory(HttpClient client) + { + _client = client; + } + + public HttpClient CreateClient(string name) => _client; + } + + private sealed class AdjustableTimeProvider : TimeProvider + { + private DateTimeOffset _now = DateTimeOffset.UtcNow; + + public override DateTimeOffset GetUtcNow() => _now; + } + + private sealed class TestHttpMessageHandler : HttpMessageHandler + { + private readonly Dictionary _responses; + + public TestHttpMessageHandler(Dictionary responses) + { + _responses = responses; + } + + public int InvocationCount { get; private set; } + + public void ResetInvocationCount() => InvocationCount = 0; + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + InvocationCount++; + if (request.RequestUri is not null && _responses.TryGetValue(request.RequestUri, out var response)) + { + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + return new HttpResponseMessage(response.StatusCode) + { + Content = new StringContent(payload, Encoding.UTF8, "application/json"), + }; + } + + return new HttpResponseMessage(HttpStatusCode.InternalServerError) + { + Content = new StringContent("unexpected request"), + }; + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj similarity index 76% rename from src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj index 78a8cb0a..ce08adbf 100644 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests/StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests/StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/AGENTS.md b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/AGENTS.md similarity index 94% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/AGENTS.md index 18ec016f..a17155ad 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Connector for Ubuntu CSAF advisories (USN VEX data), managing discovery, incremental pulls, and raw document persistence. -## Scope -- Ubuntu CSAF metadata discovery, release channel awareness, and pagination handling. -- HTTP client with retries/backoff, checksum validation, and deduplication. -- Mapping Ubuntu identifiers (USN numbers, package metadata) into connector metadata for downstream policy. -- Emitting trust configuration (GPG fingerprints, cosign options) for policy weighting. -## Participants -- Worker schedules regular pulls; WebService can initiate manual ingest/resume. -- CSAF normalizer converts raw documents into claims. -- Policy engine leverages connector-supplied trust metadata. -## Interfaces & contracts -- Implements `IVexConnector`, using shared abstractions for HTTP/resume markers and telemetry. -- Provides options for release channels (stable/LTS) and offline seed bundles. -## In/Out of scope -In: data fetching, metadata mapping, raw persistence, trust hints. -Out: normalization/export, storage internals, attestation. -## Observability & security expectations -- Log release window fetch metrics, rate limits, and deduplication stats; mask secrets. -- Emit counters for newly ingested vs unchanged USNs and quota usage. -## Tests -- Connector tests with mocked Ubuntu CSAF endpoints will live in `../StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests`. +# AGENTS +## Role +Connector for Ubuntu CSAF advisories (USN VEX data), managing discovery, incremental pulls, and raw document persistence. +## Scope +- Ubuntu CSAF metadata discovery, release channel awareness, and pagination handling. +- HTTP client with retries/backoff, checksum validation, and deduplication. +- Mapping Ubuntu identifiers (USN numbers, package metadata) into connector metadata for downstream policy. +- Emitting trust configuration (GPG fingerprints, cosign options) for policy weighting. +## Participants +- Worker schedules regular pulls; WebService can initiate manual ingest/resume. +- CSAF normalizer converts raw documents into claims. +- Policy engine leverages connector-supplied trust metadata. +## Interfaces & contracts +- Implements `IVexConnector`, using shared abstractions for HTTP/resume markers and telemetry. +- Provides options for release channels (stable/LTS) and offline seed bundles. +## In/Out of scope +In: data fetching, metadata mapping, raw persistence, trust hints. +Out: normalization/export, storage internals, attestation. +## Observability & security expectations +- Log release window fetch metrics, rate limits, and deduplication stats; mask secrets. +- Emit counters for newly ingested vs unchanged USNs and quota usage. +## Tests +- Connector tests with mocked Ubuntu CSAF endpoints will live in `../StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests`. diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs similarity index 92% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs index d575b61c..dc54bc47 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptions.cs @@ -1,90 +1,90 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; - -public sealed class UbuntuConnectorOptions -{ - public const string HttpClientName = "vexer.connector.ubuntu.catalog"; - - /// - /// Root index that lists Ubuntu CSAF channels. - /// - public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json"); - - /// - /// Channels to include (e.g. stable, esm, lts). - /// - public IList Channels { get; } = new List { "stable" }; - - /// - /// Duration to cache discovery metadata. - /// - public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(4); - - /// - /// Prefer offline snapshot when available. - /// - public bool PreferOfflineSnapshot { get; set; } - /// - /// Optional file path for offline index snapshot. - /// - public string? OfflineSnapshotPath { get; set; } - /// - /// Controls persistence of network responses to . - /// - public bool PersistOfflineSnapshot { get; set; } = true; - - public void Validate(IFileSystem? fileSystem = null) - { - if (IndexUri is null || !IndexUri.IsAbsoluteUri) - { - throw new InvalidOperationException("IndexUri must be an absolute URI."); - } - - if (IndexUri.Scheme is not ("http" or "https")) - { - throw new InvalidOperationException("IndexUri must use HTTP or HTTPS."); - } - - if (Channels.Count == 0) - { - throw new InvalidOperationException("At least one channel must be specified."); - } - - for (var i = Channels.Count - 1; i >= 0; i--) - { - if (string.IsNullOrWhiteSpace(Channels[i])) - { - Channels.RemoveAt(i); - } - } - - if (Channels.Count == 0) - { - throw new InvalidOperationException("Channel names cannot be empty."); - } - - if (MetadataCacheDuration <= TimeSpan.Zero) - { - throw new InvalidOperationException("MetadataCacheDuration must be positive."); - } - - if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); - } - - if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) - { - var fs = fileSystem ?? new FileSystem(); - var directory = Path.GetDirectoryName(OfflineSnapshotPath); - if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) - { - fs.Directory.CreateDirectory(directory); - } - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; + +public sealed class UbuntuConnectorOptions +{ + public const string HttpClientName = "excititor.connector.ubuntu.catalog"; + + /// + /// Root index that lists Ubuntu CSAF channels. + /// + public Uri IndexUri { get; set; } = new("https://ubuntu.com/security/csaf/index.json"); + + /// + /// Channels to include (e.g. stable, esm, lts). + /// + public IList Channels { get; } = new List { "stable" }; + + /// + /// Duration to cache discovery metadata. + /// + public TimeSpan MetadataCacheDuration { get; set; } = TimeSpan.FromHours(4); + + /// + /// Prefer offline snapshot when available. + /// + public bool PreferOfflineSnapshot { get; set; } + /// + /// Optional file path for offline index snapshot. + /// + public string? OfflineSnapshotPath { get; set; } + /// + /// Controls persistence of network responses to . + /// + public bool PersistOfflineSnapshot { get; set; } = true; + + public void Validate(IFileSystem? fileSystem = null) + { + if (IndexUri is null || !IndexUri.IsAbsoluteUri) + { + throw new InvalidOperationException("IndexUri must be an absolute URI."); + } + + if (IndexUri.Scheme is not ("http" or "https")) + { + throw new InvalidOperationException("IndexUri must use HTTP or HTTPS."); + } + + if (Channels.Count == 0) + { + throw new InvalidOperationException("At least one channel must be specified."); + } + + for (var i = Channels.Count - 1; i >= 0; i--) + { + if (string.IsNullOrWhiteSpace(Channels[i])) + { + Channels.RemoveAt(i); + } + } + + if (Channels.Count == 0) + { + throw new InvalidOperationException("Channel names cannot be empty."); + } + + if (MetadataCacheDuration <= TimeSpan.Zero) + { + throw new InvalidOperationException("MetadataCacheDuration must be positive."); + } + + if (PreferOfflineSnapshot && string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + throw new InvalidOperationException("OfflineSnapshotPath must be provided when PreferOfflineSnapshot is enabled."); + } + + if (!string.IsNullOrWhiteSpace(OfflineSnapshotPath)) + { + var fs = fileSystem ?? new FileSystem(); + var directory = Path.GetDirectoryName(OfflineSnapshotPath); + if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) + { + fs.Directory.CreateDirectory(directory); + } + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs similarity index 84% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs index 1adb98fd..05c3c6da 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Configuration/UbuntuConnectorOptionsValidator.cs @@ -1,32 +1,32 @@ -using System; -using System.Collections.Generic; -using System.IO.Abstractions; -using StellaOps.Vexer.Connectors.Abstractions; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; - -public sealed class UbuntuConnectorOptionsValidator : IVexConnectorOptionsValidator -{ - private readonly IFileSystem _fileSystem; - - public UbuntuConnectorOptionsValidator(IFileSystem fileSystem) - { - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - } - - public void Validate(VexConnectorDescriptor descriptor, UbuntuConnectorOptions options, IList errors) - { - ArgumentNullException.ThrowIfNull(descriptor); - ArgumentNullException.ThrowIfNull(options); - ArgumentNullException.ThrowIfNull(errors); - - try - { - options.Validate(_fileSystem); - } - catch (Exception ex) - { - errors.Add(ex.Message); - } - } -} +using System; +using System.Collections.Generic; +using System.IO.Abstractions; +using StellaOps.Excititor.Connectors.Abstractions; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; + +public sealed class UbuntuConnectorOptionsValidator : IVexConnectorOptionsValidator +{ + private readonly IFileSystem _fileSystem; + + public UbuntuConnectorOptionsValidator(IFileSystem fileSystem) + { + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + } + + public void Validate(VexConnectorDescriptor descriptor, UbuntuConnectorOptions options, IList errors) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(errors); + + try + { + options.Validate(_fileSystem); + } + catch (Exception ex) + { + errors.Add(ex.Message); + } + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs similarity index 80% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs index 42d759c8..775bd5b7 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/DependencyInjection/UbuntuConnectorServiceCollectionExtensions.cs @@ -1,45 +1,45 @@ -using System; -using System.Net; -using System.Net.Http; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; -using StellaOps.Vexer.Core; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.DependencyInjection; - -public static class UbuntuConnectorServiceCollectionExtensions -{ - public static IServiceCollection AddUbuntuCsafConnector(this IServiceCollection services, Action? configure = null) - { - ArgumentNullException.ThrowIfNull(services); - - services.TryAddSingleton(); - services.TryAddSingleton(); - - services.AddOptions() - .Configure(options => configure?.Invoke(options)); - - services.AddSingleton, UbuntuConnectorOptionsValidator>(); - - services.AddHttpClient(UbuntuConnectorOptions.HttpClientName, client => - { - client.Timeout = TimeSpan.FromSeconds(60); - client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Vexer.Connectors.Ubuntu.CSAF/1.0"); - client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); - }) - .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.All, - }); - - services.AddSingleton(); - services.AddSingleton(); - - return services; - } -} +using System; +using System.Net; +using System.Net.Http; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Excititor.Core; +using System.IO.Abstractions; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.DependencyInjection; + +public static class UbuntuConnectorServiceCollectionExtensions +{ + public static IServiceCollection AddUbuntuCsafConnector(this IServiceCollection services, Action? configure = null) + { + ArgumentNullException.ThrowIfNull(services); + + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.AddOptions() + .Configure(options => configure?.Invoke(options)); + + services.AddSingleton, UbuntuConnectorOptionsValidator>(); + + services.AddHttpClient(UbuntuConnectorOptions.HttpClientName, client => + { + client.Timeout = TimeSpan.FromSeconds(60); + client.DefaultRequestHeaders.UserAgent.ParseAdd("StellaOps.Excititor.Connectors.Ubuntu.CSAF/1.0"); + client.DefaultRequestHeaders.Accept.ParseAdd("application/json"); + }) + .ConfigurePrimaryHttpMessageHandler(static () => new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }); + + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs similarity index 95% rename from src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs rename to src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs index 890c3420..27d1935a 100644 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/Metadata/UbuntuCatalogLoader.cs @@ -1,248 +1,248 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO.Abstractions; -using System.Net; -using System.Net.Http; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; - -public sealed class UbuntuCatalogLoader -{ - public const string CachePrefix = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Index"; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMemoryCache _memoryCache; - private readonly IFileSystem _fileSystem; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - private readonly SemaphoreSlim _semaphore = new(1, 1); - private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); - - public UbuntuCatalogLoader( - IHttpClientFactory httpClientFactory, - IMemoryCache memoryCache, - IFileSystem fileSystem, - ILogger logger, - TimeProvider? timeProvider = null) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); - _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public async Task LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(options); - options.Validate(_fileSystem); - - var cacheKey = CreateCacheKey(options); - if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) - { - return cached.ToResult(fromCache: true); - } - - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) - { - return cached.ToResult(fromCache: true); - } - - CacheEntry? entry = null; - if (options.PreferOfflineSnapshot) - { - entry = LoadFromOffline(options); - if (entry is null) - { - throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found."); - } - } - else - { - entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) - ?? LoadFromOffline(options); - } - - if (entry is null) - { - throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot."); - } - - var cacheOptions = new MemoryCacheEntryOptions(); - if (entry.MetadataCacheDuration > TimeSpan.Zero) - { - cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); - } - - _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions); - return entry.ToResult(fromCache: false); - } - finally - { - _semaphore.Release(); - } - } - - private async Task TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) - { - try - { - var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); - using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - - var metadata = ParseMetadata(payload, options.Channels); - var now = _timeProvider.GetUtcNow(); - var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false); - - PersistSnapshotIfNeeded(options, metadata, now); - return entry; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri); - return null; - } - } - - private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options) - { - if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return null; - } - - if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) - { - _logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); - return null; - } - - try - { - var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); - var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); - if (snapshot is null) - { - throw new InvalidOperationException("Offline snapshot payload was empty."); - } - - return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath); - return null; - } - } - - private UbuntuCatalogMetadata ParseMetadata(string payload, IList channels) - { - if (string.IsNullOrWhiteSpace(payload)) - { - throw new InvalidOperationException("Ubuntu index payload was empty."); - } - - using var document = JsonDocument.Parse(payload); - var root = document.RootElement; - - var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) - ? generated - : _timeProvider.GetUtcNow(); - - var channelSet = new HashSet(channels, StringComparer.OrdinalIgnoreCase); - - if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array) - { - throw new InvalidOperationException("Ubuntu index did not include a channels array."); - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var channelElement in channelsElement.EnumerateArray()) - { - var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; - if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name)) - { - continue; - } - - if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri)) - { - _logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name); - continue; - } - - string? sha256 = null; - if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String) - { - sha256 = shaElement.GetString(); - } - - DateTimeOffset? lastUpdated = null; - if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated)) - { - lastUpdated = updated; - } - - builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated)); - } - - if (builder.Count == 0) - { - throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index."); - } - - return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable()); - } - - private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt) - { - if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) - { - return; - } - - try - { - var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt); - var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); - _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); - _logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); - } - } - - private static string CreateCacheKey(UbuntuConnectorOptions options) - => $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}"; - - private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) - { - public bool IsExpired(DateTimeOffset now) - => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; - - public UbuntuCatalogResult ToResult(bool fromCache) - => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); - } - - private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt); -} - -public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray Channels); - -public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated); - -public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot); +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO.Abstractions; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; + +public sealed class UbuntuCatalogLoader +{ + public const string CachePrefix = "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Index"; + + private readonly IHttpClientFactory _httpClientFactory; + private readonly IMemoryCache _memoryCache; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly JsonSerializerOptions _serializerOptions = new(JsonSerializerDefaults.Web); + + public UbuntuCatalogLoader( + IHttpClientFactory httpClientFactory, + IMemoryCache memoryCache, + IFileSystem fileSystem, + ILogger logger, + TimeProvider? timeProvider = null) + { + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _memoryCache = memoryCache ?? throw new ArgumentNullException(nameof(memoryCache)); + _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task LoadAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + options.Validate(_fileSystem); + + var cacheKey = CreateCacheKey(options); + if (_memoryCache.TryGetValue(cacheKey, out var cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_memoryCache.TryGetValue(cacheKey, out cached) && cached is not null && !cached.IsExpired(_timeProvider.GetUtcNow())) + { + return cached.ToResult(fromCache: true); + } + + CacheEntry? entry = null; + if (options.PreferOfflineSnapshot) + { + entry = LoadFromOffline(options); + if (entry is null) + { + throw new InvalidOperationException("PreferOfflineSnapshot is enabled but no offline Ubuntu snapshot was found."); + } + } + else + { + entry = await TryFetchFromNetworkAsync(options, cancellationToken).ConfigureAwait(false) + ?? LoadFromOffline(options); + } + + if (entry is null) + { + throw new InvalidOperationException("Unable to load Ubuntu CSAF index from network or offline snapshot."); + } + + var cacheOptions = new MemoryCacheEntryOptions(); + if (entry.MetadataCacheDuration > TimeSpan.Zero) + { + cacheOptions.AbsoluteExpiration = _timeProvider.GetUtcNow().Add(entry.MetadataCacheDuration); + } + + _memoryCache.Set(cacheKey, entry with { CachedAt = _timeProvider.GetUtcNow() }, cacheOptions); + return entry.ToResult(fromCache: false); + } + finally + { + _semaphore.Release(); + } + } + + private async Task TryFetchFromNetworkAsync(UbuntuConnectorOptions options, CancellationToken cancellationToken) + { + try + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + using var response = await client.GetAsync(options.IndexUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var payload = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + + var metadata = ParseMetadata(payload, options.Channels); + var now = _timeProvider.GetUtcNow(); + var entry = new CacheEntry(metadata, now, now, options.MetadataCacheDuration, false); + + PersistSnapshotIfNeeded(options, metadata, now); + return entry; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Failed to fetch Ubuntu CSAF index from {Uri}; attempting offline fallback if available.", options.IndexUri); + return null; + } + } + + private CacheEntry? LoadFromOffline(UbuntuConnectorOptions options) + { + if (string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return null; + } + + if (!_fileSystem.File.Exists(options.OfflineSnapshotPath)) + { + _logger.LogWarning("Ubuntu offline snapshot path {Path} does not exist.", options.OfflineSnapshotPath); + return null; + } + + try + { + var payload = _fileSystem.File.ReadAllText(options.OfflineSnapshotPath); + var snapshot = JsonSerializer.Deserialize(payload, _serializerOptions); + if (snapshot is null) + { + throw new InvalidOperationException("Offline snapshot payload was empty."); + } + + return new CacheEntry(snapshot.Metadata, snapshot.FetchedAt, _timeProvider.GetUtcNow(), options.MetadataCacheDuration, true); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load Ubuntu CSAF index from offline snapshot {Path}.", options.OfflineSnapshotPath); + return null; + } + } + + private UbuntuCatalogMetadata ParseMetadata(string payload, IList channels) + { + if (string.IsNullOrWhiteSpace(payload)) + { + throw new InvalidOperationException("Ubuntu index payload was empty."); + } + + using var document = JsonDocument.Parse(payload); + var root = document.RootElement; + + var generatedAt = root.TryGetProperty("generated", out var generatedElement) && generatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(generatedElement.GetString(), out var generated) + ? generated + : _timeProvider.GetUtcNow(); + + var channelSet = new HashSet(channels, StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("channels", out var channelsElement) || channelsElement.ValueKind is not JsonValueKind.Array) + { + throw new InvalidOperationException("Ubuntu index did not include a channels array."); + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var channelElement in channelsElement.EnumerateArray()) + { + var name = channelElement.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String ? nameElement.GetString() : null; + if (string.IsNullOrWhiteSpace(name) || !channelSet.Contains(name)) + { + continue; + } + + if (!channelElement.TryGetProperty("catalogUrl", out var urlElement) || urlElement.ValueKind != JsonValueKind.String || !Uri.TryCreate(urlElement.GetString(), UriKind.Absolute, out var catalogUri)) + { + _logger.LogWarning("Channel {Channel} did not specify a valid catalogUrl.", name); + continue; + } + + string? sha256 = null; + if (channelElement.TryGetProperty("sha256", out var shaElement) && shaElement.ValueKind == JsonValueKind.String) + { + sha256 = shaElement.GetString(); + } + + DateTimeOffset? lastUpdated = null; + if (channelElement.TryGetProperty("lastUpdated", out var updatedElement) && updatedElement.ValueKind == JsonValueKind.String && DateTimeOffset.TryParse(updatedElement.GetString(), out var updated)) + { + lastUpdated = updated; + } + + builder.Add(new UbuChannelCatalog(name!, catalogUri, sha256, lastUpdated)); + } + + if (builder.Count == 0) + { + throw new InvalidOperationException("None of the requested Ubuntu channels were present in the index."); + } + + return new UbuntuCatalogMetadata(generatedAt, builder.ToImmutable()); + } + + private void PersistSnapshotIfNeeded(UbuntuConnectorOptions options, UbuntuCatalogMetadata metadata, DateTimeOffset fetchedAt) + { + if (!options.PersistOfflineSnapshot || string.IsNullOrWhiteSpace(options.OfflineSnapshotPath)) + { + return; + } + + try + { + var snapshot = new UbuntuCatalogSnapshot(metadata, fetchedAt); + var payload = JsonSerializer.Serialize(snapshot, _serializerOptions); + _fileSystem.File.WriteAllText(options.OfflineSnapshotPath, payload); + _logger.LogDebug("Persisted Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to persist Ubuntu CSAF index snapshot to {Path}.", options.OfflineSnapshotPath); + } + } + + private static string CreateCacheKey(UbuntuConnectorOptions options) + => $"{CachePrefix}:{options.IndexUri}:{string.Join(',', options.Channels)}"; + + private sealed record CacheEntry(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, DateTimeOffset CachedAt, TimeSpan MetadataCacheDuration, bool FromOfflineSnapshot) + { + public bool IsExpired(DateTimeOffset now) + => MetadataCacheDuration > TimeSpan.Zero && now >= CachedAt + MetadataCacheDuration; + + public UbuntuCatalogResult ToResult(bool fromCache) + => new(Metadata, FetchedAt, fromCache, FromOfflineSnapshot); + } + + private sealed record UbuntuCatalogSnapshot(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt); +} + +public sealed record UbuntuCatalogMetadata(DateTimeOffset GeneratedAt, ImmutableArray Channels); + +public sealed record UbuChannelCatalog(string Name, Uri CatalogUri, string? Sha256, DateTimeOffset? LastUpdated); + +public sealed record UbuntuCatalogResult(UbuntuCatalogMetadata Metadata, DateTimeOffset FetchedAt, bool FromCache, bool FromOfflineSnapshot); diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj new file mode 100644 index 00000000..4a15c1f6 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md new file mode 100644 index 00000000..58e6b8e3 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/TASKS.md @@ -0,0 +1,8 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CONN-UBUNTU-01-001 – Ubuntu CSAF discovery & channels|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.| +|EXCITITOR-CONN-UBUNTU-01-002 – Incremental fetch & deduplication|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-UBUNTU-01-001, EXCITITOR-STORAGE-01-003|**DOING (2025-10-19)** – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.| +|EXCITITOR-CONN-UBUNTU-01-003 – Trust metadata & provenance|Team Excititor Connectors – Ubuntu|EXCITITOR-CONN-UBUNTU-01-002, EXCITITOR-POLICY-01-001|TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.| +> Remark (2025-10-19, EXCITITOR-CONN-UBUNTU-01-002): Prerequisites EXCITITOR-CONN-UBUNTU-01-001 and EXCITITOR-STORAGE-01-003 verified as **DONE**; advancing to DOING per Wave 0 kickoff. diff --git a/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs new file mode 100644 index 00000000..9592ba75 --- /dev/null +++ b/src/StellaOps.Excititor.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs @@ -0,0 +1,502 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Configuration; +using StellaOps.Excititor.Connectors.Ubuntu.CSAF.Metadata; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Connectors.Ubuntu.CSAF; + +public sealed class UbuntuCsafConnector : VexConnectorBase +{ + private const string EtagTokenPrefix = "etag:"; + + private static readonly VexConnectorDescriptor DescriptorInstance = new( + id: "excititor:ubuntu", + kind: VexProviderKind.Distro, + displayName: "Ubuntu CSAF") + { + Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"), + }; + + private readonly UbuntuCatalogLoader _catalogLoader; + private readonly IHttpClientFactory _httpClientFactory; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IEnumerable> _validators; + + private UbuntuConnectorOptions? _options; + private UbuntuCatalogResult? _catalog; + + public UbuntuCsafConnector( + UbuntuCatalogLoader catalogLoader, + IHttpClientFactory httpClientFactory, + IVexConnectorStateRepository stateRepository, + IEnumerable> validators, + ILogger logger, + TimeProvider timeProvider) + : base(DescriptorInstance, logger, timeProvider) + { + _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); + _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _validators = validators ?? Array.Empty>(); + } + + public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) + { + _options = VexConnectorOptionsBinder.Bind( + Descriptor, + settings, + validators: _validators); + + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary + { + ["channelCount"] = _catalog.Metadata.Channels.Length, + ["fromOffline"] = _catalog.FromOfflineSnapshot, + }); + } + + public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + if (_options is null) + { + throw new InvalidOperationException("Connector must be validated before fetch operations."); + } + + if (_catalog is null) + { + _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); + } + + var state = await _stateRepository.GetAsync(Descriptor.Id, cancellationToken).ConfigureAwait(false); + var knownTokens = state?.DocumentDigests ?? ImmutableArray.Empty; + var digestSet = new HashSet(StringComparer.OrdinalIgnoreCase); + var tokenSet = new HashSet(StringComparer.Ordinal); + var tokenList = new List(knownTokens.Length + 16); + var etagMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var token in knownTokens) + { + tokenSet.Add(token); + tokenList.Add(token); + if (TryParseEtagToken(token, out var uri, out var etag)) + { + etagMap[uri] = etag; + } + else + { + digestSet.Add(token); + } + } + + var since = context.Since ?? state?.LastUpdated ?? DateTimeOffset.MinValue; + var latestTimestamp = state?.LastUpdated ?? since; + var stateChanged = false; + + foreach (var channel in _catalog.Metadata.Channels) + { + await foreach (var entry in EnumerateChannelResourcesAsync(channel, cancellationToken).ConfigureAwait(false)) + { + var entryTimestamp = entry.LastModified ?? channel.LastUpdated ?? _catalog.Metadata.GeneratedAt; + if (entryTimestamp <= since) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + var expectedDigest = entry.Sha256 is null ? null : NormalizeDigest(entry.Sha256); + if (expectedDigest is not null && digestSet.Contains(expectedDigest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + etagMap.TryGetValue(entry.DocumentUri.ToString(), out var knownEtag); + + var download = await DownloadDocumentAsync(entry, knownEtag, cancellationToken).ConfigureAwait(false); + if (download is null) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + var document = download.Document; + if (!digestSet.Add(document.Digest)) + { + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + continue; + } + + await context.RawSink.StoreAsync(document, cancellationToken).ConfigureAwait(false); + if (tokenSet.Add(document.Digest)) + { + tokenList.Add(document.Digest); + } + + if (!string.IsNullOrWhiteSpace(download.ETag)) + { + var etagValue = download.ETag!; + etagMap[entry.DocumentUri.ToString()] = etagValue; + var etagToken = CreateEtagToken(entry.DocumentUri, etagValue); + if (tokenSet.Add(etagToken)) + { + tokenList.Add(etagToken); + } + } + + stateChanged = true; + if (entryTimestamp > latestTimestamp) + { + latestTimestamp = entryTimestamp; + } + + yield return document; + } + } + + if (stateChanged || latestTimestamp > (state?.LastUpdated ?? DateTimeOffset.MinValue)) + { + var newState = new VexConnectorState( + Descriptor.Id, + latestTimestamp == DateTimeOffset.MinValue ? state?.LastUpdated : latestTimestamp, + tokenList.ToImmutableArray()); + + await _stateRepository.SaveAsync(newState, cancellationToken).ConfigureAwait(false); + } + } + + public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing."); + + public UbuntuCatalogResult? GetCachedCatalog() => _catalog; + + private async IAsyncEnumerable EnumerateChannelResourcesAsync(UbuChannelCatalog channel, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + HttpResponseMessage? response = null; + try + { + response = await client.GetAsync(channel.CatalogUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var document = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (!document.RootElement.TryGetProperty("resources", out var resourcesElement) || resourcesElement.ValueKind != JsonValueKind.Array) + { + LogConnectorEvent(LogLevel.Warning, "fetch.catalog.empty", "Ubuntu CSAF channel catalog missing 'resources' array.", new Dictionary + { + ["channel"] = channel.Name, + ["catalog"] = channel.CatalogUri.ToString(), + }); + yield break; + } + + foreach (var resource in resourcesElement.EnumerateArray()) + { + var type = GetString(resource, "type"); + if (type is not null && !type.Equals("csaf", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var uriText = GetString(resource, "url") + ?? GetString(resource, "canonical") + ?? GetString(resource, "download") + ?? GetString(resource, "uri"); + + if (uriText is null || !Uri.TryCreate(uriText, UriKind.Absolute, out var documentUri)) + { + continue; + } + + var sha256 = TryGetHash(resource); + var etag = GetString(resource, "etag"); + var lastModified = ParseDate(resource, "last_modified") + ?? ParseDate(resource, "published") + ?? ParseDate(resource, "released") + ?? channel.LastUpdated; + var title = GetString(resource, "title"); + var version = GetString(resource, "version"); + var advisoryId = GetString(resource, "id") ?? ExtractAdvisoryId(documentUri, title); + + yield return new UbuntuCatalogEntry( + channel.Name, + advisoryId, + documentUri, + sha256, + etag, + lastModified, + title, + version); + } + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.catalog.failure", "Failed to enumerate Ubuntu CSAF channel catalog.", new Dictionary + { + ["channel"] = channel.Name, + ["catalog"] = channel.CatalogUri.ToString(), + }, ex); + } + finally + { + response?.Dispose(); + } + } + + private async Task DownloadDocumentAsync(UbuntuCatalogEntry entry, string? knownEtag, CancellationToken cancellationToken) + { + var client = _httpClientFactory.CreateClient(UbuntuConnectorOptions.HttpClientName); + using var request = new HttpRequestMessage(HttpMethod.Get, entry.DocumentUri); + if (!string.IsNullOrWhiteSpace(knownEtag)) + { + request.Headers.IfNoneMatch.TryParseAdd(EnsureQuoted(knownEtag)); + } + + HttpResponseMessage? response = null; + try + { + response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode == HttpStatusCode.NotModified) + { + LogConnectorEvent(LogLevel.Debug, "fetch.document.not_modified", "Ubuntu CSAF document not modified per ETag.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["etag"] = knownEtag, + }); + return null; + } + + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); + if (entry.Sha256 is not null) + { + var expected = NormalizeDigest(entry.Sha256); + var actual = "sha256:" + ComputeSha256Hex(payload); + if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase)) + { + LogConnectorEvent(LogLevel.Warning, "fetch.document.checksum_mismatch", "Ubuntu CSAF document checksum mismatch; skipping document.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + ["expected"] = expected, + ["actual"] = actual, + }); + return null; + } + } + + var etagHeader = response.Headers.ETag?.Tag; + var etagValue = !string.IsNullOrWhiteSpace(etagHeader) + ? Unquote(etagHeader!) + : entry.ETag is null ? null : Unquote(entry.ETag); + + var metadata = BuildMetadata(builder => + { + builder.Add("ubuntu.channel", entry.Channel); + builder.Add("ubuntu.uri", entry.DocumentUri.ToString()); + if (!string.IsNullOrWhiteSpace(entry.AdvisoryId)) + { + builder.Add("ubuntu.advisoryId", entry.AdvisoryId); + } + + if (!string.IsNullOrWhiteSpace(entry.Title)) + { + builder.Add("ubuntu.title", entry.Title!); + } + + if (!string.IsNullOrWhiteSpace(entry.Version)) + { + builder.Add("ubuntu.version", entry.Version!); + } + + if (entry.LastModified is { } modified) + { + builder.Add("ubuntu.lastModified", modified.ToString("O")); + } + + if (entry.Sha256 is not null) + { + builder.Add("ubuntu.sha256", NormalizeDigest(entry.Sha256)); + } + + if (!string.IsNullOrWhiteSpace(etagValue)) + { + builder.Add("ubuntu.etag", etagValue!); + } + }); + + var document = CreateRawDocument(VexDocumentFormat.Csaf, entry.DocumentUri, payload, metadata); + return new DownloadResult(document, etagValue); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogConnectorEvent(LogLevel.Warning, "fetch.document.failure", "Failed to download Ubuntu CSAF document.", new Dictionary + { + ["uri"] = entry.DocumentUri.ToString(), + }, ex); + return null; + } + finally + { + response?.Dispose(); + } + } + + private static string NormalizeDigest(string value) + { + var trimmed = value.Trim(); + if (trimmed.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase)) + { + trimmed = trimmed[7..]; + } + + return "sha256:" + trimmed.Replace(" ", string.Empty, StringComparison.Ordinal).ToLowerInvariant(); + } + + private static string ComputeSha256Hex(ReadOnlySpan payload) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(payload, buffer); + return Convert.ToHexString(buffer).ToLowerInvariant(); + } + + private static string? GetString(JsonElement element, string propertyName) + { + if (element.TryGetProperty(propertyName, out var property) && property.ValueKind == JsonValueKind.String) + { + var value = property.GetString(); + return string.IsNullOrWhiteSpace(value) ? null : value; + } + + return null; + } + + private static string? TryGetHash(JsonElement resource) + { + if (resource.TryGetProperty("hashes", out var hashesElement) && hashesElement.ValueKind == JsonValueKind.Object) + { + if (hashesElement.TryGetProperty("sha256", out var hash) && hash.ValueKind == JsonValueKind.String) + { + var value = hash.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + return value; + } + } + } + + return GetString(resource, "sha256"); + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var text = GetString(element, propertyName); + if (text is null) + { + return null; + } + + return DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal, out var value) + ? value + : (DateTimeOffset?)null; + } + + private static string ExtractAdvisoryId(Uri uri, string? title) + { + if (!string.IsNullOrWhiteSpace(title)) + { + return title!; + } + + var segments = uri.Segments; + if (segments.Length > 0) + { + var candidate = segments[^1].Trim('/'); + if (candidate.EndsWith(".json", StringComparison.OrdinalIgnoreCase)) + { + candidate = candidate[..^5]; + } + + if (!string.IsNullOrWhiteSpace(candidate)) + { + return candidate; + } + } + + return uri.AbsolutePath.Trim('/'); + } + + private static string EnsureQuoted(string value) + { + var trimmed = value.Trim(); + return trimmed.StartsWith('"') ? trimmed : $"\"{trimmed}\""; + } + + private static string Unquote(string value) + => value.Trim().Trim('"'); + + private static string CreateEtagToken(Uri uri, string etag) + => $"{EtagTokenPrefix}{uri}|{etag}"; + + private static bool TryParseEtagToken(string token, out string uri, out string etag) + { + uri = string.Empty; + etag = string.Empty; + if (!token.StartsWith(EtagTokenPrefix, StringComparison.Ordinal)) + { + return false; + } + + var separatorIndex = token.IndexOf('|', EtagTokenPrefix.Length); + if (separatorIndex < 0 || separatorIndex == EtagTokenPrefix.Length) + { + return false; + } + + uri = token[EtagTokenPrefix.Length..separatorIndex]; + etag = token[(separatorIndex + 1)..]; + return !string.IsNullOrWhiteSpace(uri) && !string.IsNullOrWhiteSpace(etag); + } + + private sealed record UbuntuCatalogEntry( + string Channel, + string? AdvisoryId, + Uri DocumentUri, + string? Sha256, + string? ETag, + DateTimeOffset? LastModified, + string? Title, + string? Version); + + private sealed record DownloadResult(VexRawDocument Document, string? ETag); +} diff --git a/src/StellaOps.Vexer.Attestation.Tests/StellaOps.Vexer.Attestation.Tests.csproj b/src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj similarity index 61% rename from src/StellaOps.Vexer.Attestation.Tests/StellaOps.Vexer.Attestation.Tests.csproj rename to src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj index 6f4e0b68..04b33f0e 100644 --- a/src/StellaOps.Vexer.Attestation.Tests/StellaOps.Vexer.Attestation.Tests.csproj +++ b/src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj @@ -1,13 +1,13 @@ - - - net10.0 - preview - enable - enable - true - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + diff --git a/src/StellaOps.Vexer.Core.Tests/VexCanonicalJsonSerializerTests.cs b/src/StellaOps.Excititor.Core.Tests/VexCanonicalJsonSerializerTests.cs similarity index 62% rename from src/StellaOps.Vexer.Core.Tests/VexCanonicalJsonSerializerTests.cs rename to src/StellaOps.Excititor.Core.Tests/VexCanonicalJsonSerializerTests.cs index 3212681f..851f01bd 100644 --- a/src/StellaOps.Vexer.Core.Tests/VexCanonicalJsonSerializerTests.cs +++ b/src/StellaOps.Excititor.Core.Tests/VexCanonicalJsonSerializerTests.cs @@ -1,126 +1,161 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using StellaOps.Vexer.Core; -using Xunit; - -namespace StellaOps.Vexer.Core.Tests; - -public sealed class VexCanonicalJsonSerializerTests -{ - [Fact] - public void SerializeClaim_ProducesDeterministicOrder() - { - var product = new VexProduct( - key: "pkg:redhat/demo", - name: "Demo App", - version: "1.2.3", - purl: "pkg:rpm/redhat/demo@1.2.3", - cpe: "cpe:2.3:a:redhat:demo:1.2.3", - componentIdentifiers: new[] { "componentB", "componentA" }); - - var document = new VexClaimDocument( - format: VexDocumentFormat.Csaf, - digest: "sha256:6d5a", - sourceUri: new Uri("https://example.org/vex/csaf.json"), - revision: "2024-09-15", - signature: new VexSignatureMetadata( - type: "pgp", - subject: "CN=Red Hat", - issuer: "CN=Red Hat Root", - keyId: "0xABCD", - verifiedAt: new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero))); - - var claim = new VexClaim( - vulnerabilityId: "CVE-2025-12345", - providerId: "redhat", - product: product, - status: VexClaimStatus.NotAffected, - document: document, - firstSeen: new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero), - lastSeen: new DateTimeOffset(2025, 10, 11, 12, 0, 0, TimeSpan.Zero), - justification: VexJustification.ComponentNotPresent, - detail: "Package not shipped in this channel.", - confidence: new VexConfidence("high", 0.95, "policy/default"), - additionalMetadata: ImmutableDictionary.Empty - .Add("source", "csaf") - .Add("revision", "2024-09-15")); - - var json = VexCanonicalJsonSerializer.Serialize(claim); - - Assert.Equal( - "{\"vulnerabilityId\":\"CVE-2025-12345\",\"providerId\":\"redhat\",\"product\":{\"key\":\"pkg:redhat/demo\",\"name\":\"Demo App\",\"version\":\"1.2.3\",\"purl\":\"pkg:rpm/redhat/demo@1.2.3\",\"cpe\":\"cpe:2.3:a:redhat:demo:1.2.3\",\"componentIdentifiers\":[\"componentA\",\"componentB\"]},\"status\":\"not_affected\",\"justification\":\"component_not_present\",\"detail\":\"Package not shipped in this channel.\",\"document\":{\"format\":\"csaf\",\"digest\":\"sha256:6d5a\",\"sourceUri\":\"https://example.org/vex/csaf.json\",\"revision\":\"2024-09-15\",\"signature\":{\"type\":\"pgp\",\"subject\":\"CN=Red Hat\",\"issuer\":\"CN=Red Hat Root\",\"keyId\":\"0xABCD\",\"verifiedAt\":\"2025-10-14T09:30:00+00:00\",\"transparencyLogReference\":null}},\"firstSeen\":\"2025-10-10T12:00:00+00:00\",\"lastSeen\":\"2025-10-11T12:00:00+00:00\",\"confidence\":{\"level\":\"high\",\"score\":0.95,\"method\":\"policy/default\"},\"additionalMetadata\":{\"revision\":\"2024-09-15\",\"source\":\"csaf\"}}", - json); - } - - [Fact] - public void QuerySignature_FromFilters_SortsAndNormalizesKeys() - { - var signature = VexQuerySignature.FromFilters(new[] - { - new KeyValuePair(" provider ", " redhat "), - new KeyValuePair("vulnId", "CVE-2025-12345"), - new KeyValuePair("provider", "canonical"), - }); - - Assert.Equal("provider=canonical&provider=redhat&vulnId=CVE-2025-12345", signature.Value); - } - - [Fact] - public void SerializeExportManifest_OrdersArraysAndNestedObjects() - { - var manifest = new VexExportManifest( - exportId: "export/2025/10/15/1", - querySignature: new VexQuerySignature("provider=redhat&format=consensus"), - format: VexExportFormat.OpenVex, - createdAt: new DateTimeOffset(2025, 10, 15, 8, 45, 0, TimeSpan.Zero), - artifact: new VexContentAddress("sha256", "abcd1234"), - claimCount: 42, - sourceProviders: new[] { "cisco", "redhat", "redhat" }, - fromCache: true, - consensusRevision: "rev-7", - attestation: new VexAttestationMetadata( - predicateType: "https://in-toto.io/Statement/v0.1", - rekor: new VexRekorReference("v2", "rekor://uuid/1234", "17", new Uri("https://rekor.example/log/17")), - envelopeDigest: "sha256:deadbeef", - signedAt: new DateTimeOffset(2025, 10, 15, 8, 46, 0, TimeSpan.Zero)), - sizeBytes: 4096); - - var json = VexCanonicalJsonSerializer.SerializeIndented(manifest); - - const string expected = """ - { - "exportId": "export/2025/10/15/1", - "querySignature": { - "value": "provider=redhat&format=consensus" - }, - "format": "openvex", - "createdAt": "2025-10-15T08:45:00+00:00", - "artifact": { - "algorithm": "sha256", - "digest": "abcd1234" - }, - "claimCount": 42, - "fromCache": true, - "sourceProviders": [ - "cisco", - "redhat" - ], - "consensusRevision": "rev-7", - "attestation": { - "predicateType": "https://in-toto.io/Statement/v0.1", - "rekor": { - "apiVersion": "v2", - "location": "rekor://uuid/1234", - "logIndex": "17", - "inclusionProofUri": "https://rekor.example/log/17" - }, - "envelopeDigest": "sha256:deadbeef", - "signedAt": "2025-10-15T08:46:00+00:00" - }, - "sizeBytes": 4096 - } - """; - - Assert.Equal(expected.ReplaceLineEndings(), json.ReplaceLineEndings()); - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using StellaOps.Excititor.Core; +using Xunit; + +namespace StellaOps.Excititor.Core.Tests; + +public sealed class VexCanonicalJsonSerializerTests +{ + [Fact] + public void SerializeClaim_ProducesDeterministicOrder() + { + var product = new VexProduct( + key: "pkg:redhat/demo", + name: "Demo App", + version: "1.2.3", + purl: "pkg:rpm/redhat/demo@1.2.3", + cpe: "cpe:2.3:a:redhat:demo:1.2.3", + componentIdentifiers: new[] { "componentB", "componentA" }); + + var document = new VexClaimDocument( + format: VexDocumentFormat.Csaf, + digest: "sha256:6d5a", + sourceUri: new Uri("https://example.org/vex/csaf.json"), + revision: "2024-09-15", + signature: new VexSignatureMetadata( + type: "pgp", + subject: "CN=Red Hat", + issuer: "CN=Red Hat Root", + keyId: "0xABCD", + verifiedAt: new DateTimeOffset(2025, 10, 14, 9, 30, 0, TimeSpan.Zero))); + + var claim = new VexClaim( + vulnerabilityId: "CVE-2025-12345", + providerId: "redhat", + product: product, + status: VexClaimStatus.NotAffected, + document: document, + firstSeen: new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero), + lastSeen: new DateTimeOffset(2025, 10, 11, 12, 0, 0, TimeSpan.Zero), + justification: VexJustification.ComponentNotPresent, + detail: "Package not shipped in this channel.", + confidence: new VexConfidence("high", 0.95, "policy/default"), + signals: new VexSignalSnapshot( + new VexSeveritySignal("CVSS:3.1", 7.5, label: "high", vector: "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"), + kev: true, + epss: 0.42), + additionalMetadata: ImmutableDictionary.Empty + .Add("source", "csaf") + .Add("revision", "2024-09-15")); + + var json = VexCanonicalJsonSerializer.Serialize(claim); + + Assert.Equal( + "{\"vulnerabilityId\":\"CVE-2025-12345\",\"providerId\":\"redhat\",\"product\":{\"key\":\"pkg:redhat/demo\",\"name\":\"Demo App\",\"version\":\"1.2.3\",\"purl\":\"pkg:rpm/redhat/demo@1.2.3\",\"cpe\":\"cpe:2.3:a:redhat:demo:1.2.3\",\"componentIdentifiers\":[\"componentA\",\"componentB\"]},\"status\":\"not_affected\",\"justification\":\"component_not_present\",\"detail\":\"Package not shipped in this channel.\",\"signals\":{\"severity\":{\"scheme\":\"CVSS:3.1\",\"score\":7.5,\"label\":\"high\",\"vector\":\"CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H\"},\"kev\":true,\"epss\":0.42},\"document\":{\"format\":\"csaf\",\"digest\":\"sha256:6d5a\",\"sourceUri\":\"https://example.org/vex/csaf.json\",\"revision\":\"2024-09-15\",\"signature\":{\"type\":\"pgp\",\"subject\":\"CN=Red Hat\",\"issuer\":\"CN=Red Hat Root\",\"keyId\":\"0xABCD\",\"verifiedAt\":\"2025-10-14T09:30:00+00:00\",\"transparencyLogReference\":null}},\"firstSeen\":\"2025-10-10T12:00:00+00:00\",\"lastSeen\":\"2025-10-11T12:00:00+00:00\",\"confidence\":{\"level\":\"high\",\"score\":0.95,\"method\":\"policy/default\"},\"additionalMetadata\":{\"revision\":\"2024-09-15\",\"source\":\"csaf\"}}", + json); + } + + [Fact] + public void SerializeConsensus_IncludesSignalsInOrder() + { + var product = new VexProduct("pkg:demo/app", "Demo App"); + var sources = new[] + { + new VexConsensusSource("redhat", VexClaimStatus.Affected, "sha256:redhat", 1.0), + }; + + var consensus = new VexConsensus( + "CVE-2025-9999", + product, + VexConsensusStatus.Affected, + new DateTimeOffset(2025, 10, 15, 12, 0, 0, TimeSpan.Zero), + sources, + signals: new VexSignalSnapshot( + new VexSeveritySignal("stellaops:v1", score: 9.1, label: "critical"), + kev: false, + epss: 0.67), + policyVersion: "baseline/v1", + summary: "Affected due to vendor advisory.", + policyRevisionId: "rev-1", + policyDigest: "sha256:abcd"); + + var json = VexCanonicalJsonSerializer.Serialize(consensus); + + Assert.Equal( + "{\"vulnerabilityId\":\"CVE-2025-9999\",\"product\":{\"key\":\"pkg:demo/app\",\"name\":\"Demo App\",\"version\":null,\"purl\":null,\"cpe\":null,\"componentIdentifiers\":[]},\"status\":\"affected\",\"calculatedAt\":\"2025-10-15T12:00:00+00:00\",\"sources\":[{\"providerId\":\"redhat\",\"status\":\"affected\",\"documentDigest\":\"sha256:redhat\",\"weight\":1,\"justification\":null,\"detail\":null,\"confidence\":null}],\"conflicts\":[],\"signals\":{\"severity\":{\"scheme\":\"stellaops:v1\",\"score\":9.1,\"label\":\"critical\",\"vector\":null},\"kev\":false,\"epss\":0.67},\"policyVersion\":\"baseline/v1\",\"summary\":\"Affected due to vendor advisory.\",\"policyDigest\":\"sha256:abcd\",\"policyRevisionId\":\"rev-1\"}", + json); + } + + [Fact] + public void QuerySignature_FromFilters_SortsAndNormalizesKeys() + { + var signature = VexQuerySignature.FromFilters(new[] + { + new KeyValuePair(" provider ", " redhat "), + new KeyValuePair("vulnId", "CVE-2025-12345"), + new KeyValuePair("provider", "canonical"), + }); + + Assert.Equal("provider=canonical&provider=redhat&vulnId=CVE-2025-12345", signature.Value); + } + + [Fact] + public void SerializeExportManifest_OrdersArraysAndNestedObjects() + { + var manifest = new VexExportManifest( + exportId: "export/2025/10/15/1", + querySignature: new VexQuerySignature("provider=redhat&format=consensus"), + format: VexExportFormat.OpenVex, + createdAt: new DateTimeOffset(2025, 10, 15, 8, 45, 0, TimeSpan.Zero), + artifact: new VexContentAddress("sha256", "abcd1234"), + claimCount: 42, + sourceProviders: new[] { "cisco", "redhat", "redhat" }, + fromCache: true, + consensusRevision: "rev-7", + attestation: new VexAttestationMetadata( + predicateType: "https://in-toto.io/Statement/v0.1", + rekor: new VexRekorReference("v2", "rekor://uuid/1234", "17", new Uri("https://rekor.example/log/17")), + envelopeDigest: "sha256:deadbeef", + signedAt: new DateTimeOffset(2025, 10, 15, 8, 46, 0, TimeSpan.Zero)), + sizeBytes: 4096); + + var json = VexCanonicalJsonSerializer.SerializeIndented(manifest); + + const string expected = """ + { + "exportId": "export/2025/10/15/1", + "querySignature": { + "value": "provider=redhat&format=consensus" + }, + "format": "openvex", + "createdAt": "2025-10-15T08:45:00+00:00", + "artifact": { + "algorithm": "sha256", + "digest": "abcd1234" + }, + "claimCount": 42, + "fromCache": true, + "sourceProviders": [ + "cisco", + "redhat" + ], + "consensusRevision": "rev-7", + "attestation": { + "predicateType": "https://in-toto.io/Statement/v0.1", + "rekor": { + "apiVersion": "v2", + "location": "rekor://uuid/1234", + "logIndex": "17", + "inclusionProofUri": "https://rekor.example/log/17" + }, + "envelopeDigest": "sha256:deadbeef", + "signedAt": "2025-10-15T08:46:00+00:00" + }, + "sizeBytes": 4096 + } + """; + + Assert.Equal(expected.ReplaceLineEndings(), json.ReplaceLineEndings()); + } +} diff --git a/src/StellaOps.Vexer.Core.Tests/VexConsensusResolverTests.cs b/src/StellaOps.Excititor.Core.Tests/VexConsensusResolverTests.cs similarity index 85% rename from src/StellaOps.Vexer.Core.Tests/VexConsensusResolverTests.cs rename to src/StellaOps.Excititor.Core.Tests/VexConsensusResolverTests.cs index 1396d0b9..6f3e615e 100644 --- a/src/StellaOps.Vexer.Core.Tests/VexConsensusResolverTests.cs +++ b/src/StellaOps.Excititor.Core.Tests/VexConsensusResolverTests.cs @@ -1,200 +1,227 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using StellaOps.Vexer.Core; -using Xunit; - -namespace StellaOps.Vexer.Core.Tests; - -public sealed class VexConsensusResolverTests -{ - private static readonly VexProduct DemoProduct = new( - key: "pkg:demo/app", - name: "Demo App", - version: "1.0.0", - purl: "pkg:demo/app@1.0.0", - cpe: "cpe:2.3:a:demo:app:1.0.0"); - - [Fact] - public void Resolve_SingleAcceptedClaim_SelectsStatus() - { - var provider = CreateProvider("redhat", VexProviderKind.Vendor); - var claim = CreateClaim( - "CVE-2025-0001", - provider.Id, - VexClaimStatus.Affected, - justification: null); - - var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); - - var result = resolver.Resolve(new VexConsensusRequest( - claim.VulnerabilityId, - DemoProduct, - new[] { claim }, - new Dictionary { [provider.Id] = provider }, - DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); - - Assert.Equal(VexConsensusStatus.Affected, result.Consensus.Status); - Assert.Equal("baseline/v1", result.Consensus.PolicyVersion); - Assert.Single(result.Consensus.Sources); - Assert.Empty(result.Consensus.Conflicts); - Assert.NotNull(result.Consensus.Summary); - Assert.Contains("affected", result.Consensus.Summary!, StringComparison.Ordinal); - - var decision = Assert.Single(result.DecisionLog); - Assert.True(decision.Included); - Assert.Equal(provider.Id, decision.ProviderId); - Assert.Null(decision.Reason); - } - - [Fact] - public void Resolve_NotAffectedWithoutJustification_IsRejected() - { - var provider = CreateProvider("cisco", VexProviderKind.Vendor); - var claim = CreateClaim( - "CVE-2025-0002", - provider.Id, - VexClaimStatus.NotAffected, - justification: null); - - var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); - - var result = resolver.Resolve(new VexConsensusRequest( - claim.VulnerabilityId, - DemoProduct, - new[] { claim }, - new Dictionary { [provider.Id] = provider }, - DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); - - Assert.Equal(VexConsensusStatus.UnderInvestigation, result.Consensus.Status); - Assert.Empty(result.Consensus.Sources); - var conflict = Assert.Single(result.Consensus.Conflicts); - Assert.Equal("missing_justification", conflict.Reason); - - var decision = Assert.Single(result.DecisionLog); - Assert.False(decision.Included); - Assert.Equal("missing_justification", decision.Reason); - } - - [Fact] - public void Resolve_MajorityWeightWins_WithConflictingSources() - { - var vendor = CreateProvider("redhat", VexProviderKind.Vendor); - var distro = CreateProvider("fedora", VexProviderKind.Distro); - - var claims = new[] - { - CreateClaim( - "CVE-2025-0003", - vendor.Id, - VexClaimStatus.Affected, - detail: "Vendor advisory", - documentDigest: "sha256:vendor"), - CreateClaim( - "CVE-2025-0003", - distro.Id, - VexClaimStatus.NotAffected, - justification: VexJustification.ComponentNotPresent, - detail: "Distro package not shipped", - documentDigest: "sha256:distro"), - }; - - var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); - - var result = resolver.Resolve(new VexConsensusRequest( - "CVE-2025-0003", - DemoProduct, - claims, - new Dictionary - { - [vendor.Id] = vendor, - [distro.Id] = distro, - }, - DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); - - Assert.Equal(VexConsensusStatus.Affected, result.Consensus.Status); - Assert.Equal(2, result.Consensus.Sources.Length); - Assert.Equal(1.0, result.Consensus.Sources.First(s => s.ProviderId == vendor.Id).Weight); - Assert.Contains(result.Consensus.Conflicts, c => c.ProviderId == distro.Id && c.Reason == "status_conflict"); - Assert.NotNull(result.Consensus.Summary); - Assert.Contains("affected", result.Consensus.Summary!, StringComparison.Ordinal); - } - - [Fact] - public void Resolve_TieFallsBackToUnderInvestigation() - { - var hub = CreateProvider("hub", VexProviderKind.Hub); - var platform = CreateProvider("platform", VexProviderKind.Platform); - - var claims = new[] - { - CreateClaim( - "CVE-2025-0004", - hub.Id, - VexClaimStatus.Affected, - detail: "Hub escalation", - documentDigest: "sha256:hub"), - CreateClaim( - "CVE-2025-0004", - platform.Id, - VexClaimStatus.NotAffected, - justification: VexJustification.ProtectedByMitigatingControl, - detail: "Runtime mitigations", - documentDigest: "sha256:platform"), - }; - - var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy( - new VexConsensusPolicyOptions( - hubWeight: 0.5, - platformWeight: 0.5))); - - var result = resolver.Resolve(new VexConsensusRequest( - "CVE-2025-0004", - DemoProduct, - claims, - new Dictionary - { - [hub.Id] = hub, - [platform.Id] = platform, - }, - DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); - - Assert.Equal(VexConsensusStatus.UnderInvestigation, result.Consensus.Status); - Assert.Equal(2, result.Consensus.Conflicts.Length); - Assert.NotNull(result.Consensus.Summary); - Assert.Contains("No majority consensus", result.Consensus.Summary!, StringComparison.Ordinal); - } - - private static VexProvider CreateProvider(string id, VexProviderKind kind) - => new( - id, - displayName: id.ToUpperInvariant(), - kind, - baseUris: Array.Empty(), - trust: new VexProviderTrust(weight: 1.0, cosign: null)); - - private static VexClaim CreateClaim( - string vulnerabilityId, - string providerId, - VexClaimStatus status, - VexJustification? justification = null, - string? detail = null, - string? documentDigest = null) - => new( - vulnerabilityId, - providerId, - DemoProduct, - status, - new VexClaimDocument( - VexDocumentFormat.Csaf, - documentDigest ?? $"sha256:{providerId}", - new Uri($"https://example.org/{providerId}/{vulnerabilityId}.json"), - "1"), - firstSeen: DateTimeOffset.Parse("2025-10-10T12:00:00Z"), - lastSeen: DateTimeOffset.Parse("2025-10-11T12:00:00Z"), - justification, - detail, - confidence: null, - additionalMetadata: ImmutableDictionary.Empty); -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Excititor.Core; +using Xunit; + +namespace StellaOps.Excititor.Core.Tests; + +public sealed class VexConsensusResolverTests +{ + private static readonly VexProduct DemoProduct = new( + key: "pkg:demo/app", + name: "Demo App", + version: "1.0.0", + purl: "pkg:demo/app@1.0.0", + cpe: "cpe:2.3:a:demo:app:1.0.0"); + + [Fact] + public void Resolve_SingleAcceptedClaim_SelectsStatus() + { + var provider = CreateProvider("redhat", VexProviderKind.Vendor); + var claim = CreateClaim( + "CVE-2025-0001", + provider.Id, + VexClaimStatus.Affected, + justification: null); + + var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); + + var result = resolver.Resolve(new VexConsensusRequest( + claim.VulnerabilityId, + DemoProduct, + new[] { claim }, + new Dictionary { [provider.Id] = provider }, + DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); + + Assert.Equal(VexConsensusStatus.Affected, result.Consensus.Status); + Assert.Equal("baseline/v1", result.Consensus.PolicyVersion); + Assert.Single(result.Consensus.Sources); + Assert.Empty(result.Consensus.Conflicts); + Assert.NotNull(result.Consensus.Summary); + Assert.Contains("affected", result.Consensus.Summary!, StringComparison.Ordinal); + + var decision = Assert.Single(result.DecisionLog); + Assert.True(decision.Included); + Assert.Equal(provider.Id, decision.ProviderId); + Assert.Null(decision.Reason); + } + + [Fact] + public void Resolve_NotAffectedWithoutJustification_IsRejected() + { + var provider = CreateProvider("cisco", VexProviderKind.Vendor); + var claim = CreateClaim( + "CVE-2025-0002", + provider.Id, + VexClaimStatus.NotAffected, + justification: null); + + var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); + + var result = resolver.Resolve(new VexConsensusRequest( + claim.VulnerabilityId, + DemoProduct, + new[] { claim }, + new Dictionary { [provider.Id] = provider }, + DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); + + Assert.Equal(VexConsensusStatus.UnderInvestigation, result.Consensus.Status); + Assert.Empty(result.Consensus.Sources); + var conflict = Assert.Single(result.Consensus.Conflicts); + Assert.Equal("missing_justification", conflict.Reason); + + var decision = Assert.Single(result.DecisionLog); + Assert.False(decision.Included); + Assert.Equal("missing_justification", decision.Reason); + } + + [Fact] + public void Resolve_MajorityWeightWins_WithConflictingSources() + { + var vendor = CreateProvider("redhat", VexProviderKind.Vendor); + var distro = CreateProvider("fedora", VexProviderKind.Distro); + + var claims = new[] + { + CreateClaim( + "CVE-2025-0003", + vendor.Id, + VexClaimStatus.Affected, + detail: "Vendor advisory", + documentDigest: "sha256:vendor"), + CreateClaim( + "CVE-2025-0003", + distro.Id, + VexClaimStatus.NotAffected, + justification: VexJustification.ComponentNotPresent, + detail: "Distro package not shipped", + documentDigest: "sha256:distro"), + }; + + var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy()); + + var result = resolver.Resolve(new VexConsensusRequest( + "CVE-2025-0003", + DemoProduct, + claims, + new Dictionary + { + [vendor.Id] = vendor, + [distro.Id] = distro, + }, + DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); + + Assert.Equal(VexConsensusStatus.Affected, result.Consensus.Status); + Assert.Equal(2, result.Consensus.Sources.Length); + Assert.Equal(1.0, result.Consensus.Sources.First(s => s.ProviderId == vendor.Id).Weight); + Assert.Contains(result.Consensus.Conflicts, c => c.ProviderId == distro.Id && c.Reason == "status_conflict"); + Assert.NotNull(result.Consensus.Summary); + Assert.Contains("affected", result.Consensus.Summary!, StringComparison.Ordinal); + } + + [Fact] + public void Resolve_TieFallsBackToUnderInvestigation() + { + var hub = CreateProvider("hub", VexProviderKind.Hub); + var platform = CreateProvider("platform", VexProviderKind.Platform); + + var claims = new[] + { + CreateClaim( + "CVE-2025-0004", + hub.Id, + VexClaimStatus.Affected, + detail: "Hub escalation", + documentDigest: "sha256:hub"), + CreateClaim( + "CVE-2025-0004", + platform.Id, + VexClaimStatus.NotAffected, + justification: VexJustification.ProtectedByMitigatingControl, + detail: "Runtime mitigations", + documentDigest: "sha256:platform"), + }; + + var resolver = new VexConsensusResolver(new BaselineVexConsensusPolicy( + new VexConsensusPolicyOptions( + hubWeight: 0.5, + platformWeight: 0.5))); + + var result = resolver.Resolve(new VexConsensusRequest( + "CVE-2025-0004", + DemoProduct, + claims, + new Dictionary + { + [hub.Id] = hub, + [platform.Id] = platform, + }, + DateTimeOffset.Parse("2025-10-15T12:00:00Z"))); + + Assert.Equal(VexConsensusStatus.UnderInvestigation, result.Consensus.Status); + Assert.Equal(2, result.Consensus.Conflicts.Length); + Assert.NotNull(result.Consensus.Summary); + Assert.Contains("No majority consensus", result.Consensus.Summary!, StringComparison.Ordinal); + } + + [Fact] + public void Resolve_RespectsRaisedWeightCeiling() + { + var provider = CreateProvider("vendor", VexProviderKind.Vendor); + var claim = CreateClaim( + "CVE-2025-0100", + provider.Id, + VexClaimStatus.Affected, + documentDigest: "sha256:vendor"); + + var policy = new BaselineVexConsensusPolicy(new VexConsensusPolicyOptions( + vendorWeight: 1.4, + weightCeiling: 2.0)); + var resolver = new VexConsensusResolver(policy); + + var result = resolver.Resolve(new VexConsensusRequest( + claim.VulnerabilityId, + DemoProduct, + new[] { claim }, + new Dictionary { [provider.Id] = provider }, + DateTimeOffset.Parse("2025-10-15T12:00:00Z"), + WeightCeiling: 2.0)); + + var source = Assert.Single(result.Consensus.Sources); + Assert.Equal(1.4, source.Weight); + } + + private static VexProvider CreateProvider(string id, VexProviderKind kind) + => new( + id, + displayName: id.ToUpperInvariant(), + kind, + baseUris: Array.Empty(), + trust: new VexProviderTrust(weight: 1.0, cosign: null)); + + private static VexClaim CreateClaim( + string vulnerabilityId, + string providerId, + VexClaimStatus status, + VexJustification? justification = null, + string? detail = null, + string? documentDigest = null) + => new( + vulnerabilityId, + providerId, + DemoProduct, + status, + new VexClaimDocument( + VexDocumentFormat.Csaf, + documentDigest ?? $"sha256:{providerId}", + new Uri($"https://example.org/{providerId}/{vulnerabilityId}.json"), + "1"), + firstSeen: DateTimeOffset.Parse("2025-10-10T12:00:00Z"), + lastSeen: DateTimeOffset.Parse("2025-10-11T12:00:00Z"), + justification, + detail, + confidence: null, + additionalMetadata: ImmutableDictionary.Empty); +} diff --git a/src/StellaOps.Vexer.Core.Tests/VexPolicyBinderTests.cs b/src/StellaOps.Excititor.Core.Tests/VexPolicyBinderTests.cs similarity index 50% rename from src/StellaOps.Vexer.Core.Tests/VexPolicyBinderTests.cs rename to src/StellaOps.Excititor.Core.Tests/VexPolicyBinderTests.cs index 9f48ffa5..700292b5 100644 --- a/src/StellaOps.Vexer.Core.Tests/VexPolicyBinderTests.cs +++ b/src/StellaOps.Excititor.Core.Tests/VexPolicyBinderTests.cs @@ -1,83 +1,130 @@ -using System; -using System.IO; -using System.Text; -using StellaOps.Vexer.Policy; - -namespace StellaOps.Vexer.Core.Tests; - -public sealed class VexPolicyBinderTests -{ - private const string JsonPolicy = """ - { - "version": "custom/v2", - "weights": { - "vendor": 0.95, - "distro": 0.85 - }, - "providerOverrides": { - "provider.example": 0.5 - } - } - """; - - private const string YamlPolicy = """ - version: custom/v3 - weights: - vendor: 0.8 - distro: 0.7 - platform: 0.6 - providerOverrides: - provider-a: 0.4 - provider-b: 0.3 - """; - - [Fact] - public void Bind_Json_ReturnsNormalizedOptions() - { - var result = VexPolicyBinder.Bind(JsonPolicy, VexPolicyDocumentFormat.Json); - - Assert.True(result.Success); - Assert.NotNull(result.Options); - Assert.NotNull(result.NormalizedOptions); - Assert.Equal("custom/v2", result.Options!.Version); - Assert.Equal("custom/v2", result.NormalizedOptions!.Version); - Assert.Empty(result.Issues); - } - - [Fact] - public void Bind_Yaml_ReturnsOverridesAndWarningsSorted() - { - var result = VexPolicyBinder.Bind(YamlPolicy, VexPolicyDocumentFormat.Yaml); - - Assert.True(result.Success); - Assert.NotNull(result.NormalizedOptions); - var overrides = result.NormalizedOptions!.ProviderOverrides; - Assert.Equal(2, overrides.Count); - Assert.Equal(0.4, overrides["provider-a"]); - Assert.Equal(0.3, overrides["provider-b"]); - Assert.Empty(result.Issues); - } - - [Fact] - public void Bind_InvalidJson_ReturnsError() - { - const string invalidJson = "{ \"weights\": { \"vendor\": \"not-a-number\" }"; - - var result = VexPolicyBinder.Bind(invalidJson, VexPolicyDocumentFormat.Json); - - Assert.False(result.Success); - var issue = Assert.Single(result.Issues); - Assert.Equal(VexPolicyIssueSeverity.Error, issue.Severity); - Assert.StartsWith("policy.parse.json", issue.Code, StringComparison.Ordinal); - } - - [Fact] - public void Bind_Stream_SupportsEncoding() - { - using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonPolicy)); - var result = VexPolicyBinder.Bind(stream, VexPolicyDocumentFormat.Json); - - Assert.True(result.Success); - Assert.NotNull(result.Options); - } -} +using System; +using System.IO; +using System.Text; +using StellaOps.Excititor.Policy; + +namespace StellaOps.Excititor.Core.Tests; + +public sealed class VexPolicyBinderTests +{ + private const string JsonPolicy = """ + { + "version": "custom/v2", + "weights": { + "vendor": 1.3, + "distro": 0.85, + "ceiling": 2.0 + }, + "scoring": { + "alpha": 0.35, + "beta": 0.75 + }, + "providerOverrides": { + "provider.example": 1.8 + } + } + """; + + private const string YamlPolicy = """ + version: custom/v3 + weights: + vendor: 0.8 + distro: 0.7 + platform: 0.6 + providerOverrides: + provider-a: 0.4 + provider-b: 0.3 + """; + + [Fact] + public void Bind_Json_ReturnsNormalizedOptions() + { + var result = VexPolicyBinder.Bind(JsonPolicy, VexPolicyDocumentFormat.Json); + + Assert.True(result.Success); + Assert.NotNull(result.Options); + Assert.NotNull(result.NormalizedOptions); + Assert.Equal("custom/v2", result.Options!.Version); + Assert.Equal("custom/v2", result.NormalizedOptions!.Version); + Assert.Equal(1.3, result.NormalizedOptions.VendorWeight); + Assert.Equal(0.85, result.NormalizedOptions.DistroWeight); + Assert.Equal(2.0, result.NormalizedOptions.WeightCeiling); + Assert.Equal(0.35, result.NormalizedOptions.Alpha); + Assert.Equal(0.75, result.NormalizedOptions.Beta); + Assert.Equal(1.8, result.NormalizedOptions.ProviderOverrides["provider.example"]); + Assert.Empty(result.Issues); + } + + [Fact] + public void Bind_Yaml_ReturnsOverridesAndWarningsSorted() + { + var result = VexPolicyBinder.Bind(YamlPolicy, VexPolicyDocumentFormat.Yaml); + + Assert.True(result.Success); + Assert.NotNull(result.NormalizedOptions); + var overrides = result.NormalizedOptions!.ProviderOverrides; + Assert.Equal(2, overrides.Count); + Assert.Equal(0.4, overrides["provider-a"]); + Assert.Equal(0.3, overrides["provider-b"]); + Assert.Empty(result.Issues); + } + + [Fact] + public void Bind_InvalidJson_ReturnsError() + { + const string invalidJson = "{ \"weights\": { \"vendor\": \"not-a-number\" }"; + + var result = VexPolicyBinder.Bind(invalidJson, VexPolicyDocumentFormat.Json); + + Assert.False(result.Success); + var issue = Assert.Single(result.Issues); + Assert.Equal(VexPolicyIssueSeverity.Error, issue.Severity); + Assert.StartsWith("policy.parse.json", issue.Code, StringComparison.Ordinal); + } + + [Fact] + public void Bind_Stream_SupportsEncoding() + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(JsonPolicy)); + var result = VexPolicyBinder.Bind(stream, VexPolicyDocumentFormat.Json); + + Assert.True(result.Success); + Assert.NotNull(result.Options); + } + + [Fact] + public void Bind_InvalidWeightsAndScoring_EmitsWarningsAndClamps() + { + const string policy = """ + { + "weights": { + "vendor": 3.5, + "ceiling": 0.8 + }, + "scoring": { + "alpha": -0.1, + "beta": 10.0 + }, + "providerOverrides": { + "bad": 4.0 + } + } + """; + + var result = VexPolicyBinder.Bind(policy, VexPolicyDocumentFormat.Json); + + Assert.True(result.Success); + Assert.NotNull(result.NormalizedOptions); + var consensus = result.NormalizedOptions!; + Assert.Equal(1.0, consensus.WeightCeiling); + Assert.Equal(1.0, consensus.VendorWeight); + Assert.Equal(1.0, consensus.ProviderOverrides["bad"]); + Assert.Equal(VexConsensusPolicyOptions.DefaultAlpha, consensus.Alpha); + Assert.Equal(VexConsensusPolicyOptions.MaxSupportedCoefficient, consensus.Beta); + Assert.Contains(result.Issues, issue => issue.Code == "weights.ceiling.minimum"); + Assert.Contains(result.Issues, issue => issue.Code == "weights.vendor.range"); + Assert.Contains(result.Issues, issue => issue.Code == "weights.overrides.bad.range"); + Assert.Contains(result.Issues, issue => issue.Code == "scoring.alpha.range"); + Assert.Contains(result.Issues, issue => issue.Code == "scoring.beta.maximum"); + } +} diff --git a/src/StellaOps.Vexer.Core.Tests/VexPolicyDiagnosticsTests.cs b/src/StellaOps.Excititor.Core.Tests/VexPolicyDiagnosticsTests.cs similarity index 93% rename from src/StellaOps.Vexer.Core.Tests/VexPolicyDiagnosticsTests.cs rename to src/StellaOps.Excititor.Core.Tests/VexPolicyDiagnosticsTests.cs index 06971d3d..0d86cb9d 100644 --- a/src/StellaOps.Vexer.Core.Tests/VexPolicyDiagnosticsTests.cs +++ b/src/StellaOps.Excititor.Core.Tests/VexPolicyDiagnosticsTests.cs @@ -1,169 +1,169 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using Microsoft.Extensions.Time.Testing; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Policy; -using System.Diagnostics.Metrics; - -namespace StellaOps.Vexer.Core.Tests; - -public class VexPolicyDiagnosticsTests -{ - [Fact] - public void GetDiagnostics_ReportsCountsRecommendationsAndOverrides() - { - var overrides = new[] - { - new KeyValuePair("provider-a", 0.8), - new KeyValuePair("provider-b", 0.6), - }; - - var snapshot = new VexPolicySnapshot( - "custom/v1", - new VexConsensusPolicyOptions( - version: "custom/v1", - providerOverrides: overrides), - new BaselineVexConsensusPolicy(), - ImmutableArray.Create( - new VexPolicyIssue("sample.error", "Blocking issue.", VexPolicyIssueSeverity.Error), - new VexPolicyIssue("sample.warning", "Non-blocking issue.", VexPolicyIssueSeverity.Warning)), - "rev-test", - "ABCDEF"); - - var fakeProvider = new FakePolicyProvider(snapshot); - var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 16, 17, 0, 0, TimeSpan.Zero)); - var diagnostics = new VexPolicyDiagnostics(fakeProvider, fakeTime); - - var report = diagnostics.GetDiagnostics(); - - Assert.Equal("custom/v1", report.Version); - Assert.Equal("rev-test", report.RevisionId); - Assert.Equal("ABCDEF", report.Digest); - Assert.Equal(1, report.ErrorCount); - Assert.Equal(1, report.WarningCount); - Assert.Equal(fakeTime.GetUtcNow(), report.GeneratedAt); - Assert.Collection(report.Issues, - issue => Assert.Equal("sample.error", issue.Code), - issue => Assert.Equal("sample.warning", issue.Code)); - Assert.Equal(new[] { "provider-a", "provider-b" }, report.ActiveOverrides.Keys.OrderBy(static key => key, StringComparer.Ordinal)); - Assert.Contains(report.Recommendations, message => message.Contains("Resolve policy errors", StringComparison.OrdinalIgnoreCase)); - Assert.Contains(report.Recommendations, message => message.Contains("provider-a", StringComparison.OrdinalIgnoreCase)); - Assert.Contains(report.Recommendations, message => message.Contains("docs/ARCHITECTURE_VEXER.md", StringComparison.OrdinalIgnoreCase)); - } - - [Fact] - public void GetDiagnostics_WhenNoIssues_StillReturnsDefaultRecommendation() - { - var fakeProvider = new FakePolicyProvider(VexPolicySnapshot.Default); - var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 16, 17, 0, 0, TimeSpan.Zero)); - var diagnostics = new VexPolicyDiagnostics(fakeProvider, fakeTime); - - var report = diagnostics.GetDiagnostics(); - - Assert.Equal(0, report.ErrorCount); - Assert.Equal(0, report.WarningCount); - Assert.Empty(report.ActiveOverrides); - Assert.Single(report.Recommendations); - } - - [Fact] - public void PolicyProvider_ComputesRevisionAndDigest_AndEmitsTelemetry() - { - using var listener = new MeterListener(); - var reloadMeasurements = 0; - string? lastRevision = null; - listener.InstrumentPublished += (instrument, _) => - { - if (instrument.Meter.Name == "StellaOps.Vexer.Policy" && - instrument.Name == "vex.policy.reloads") - { - listener.EnableMeasurementEvents(instrument); - } - }; - - listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => - { - reloadMeasurements++; - foreach (var tag in tags) - { - if (tag.Key is "revision" && tag.Value is string revision) - { - lastRevision = revision; - break; - } - } - }); - - listener.Start(); - - var optionsMonitor = new MutableOptionsMonitor(new VexPolicyOptions()); - var provider = new VexPolicyProvider(optionsMonitor, NullLogger.Instance); - - var snapshot1 = provider.GetSnapshot(); - Assert.Equal("rev-1", snapshot1.RevisionId); - Assert.False(string.IsNullOrWhiteSpace(snapshot1.Digest)); - - var snapshot2 = provider.GetSnapshot(); - Assert.Equal("rev-1", snapshot2.RevisionId); - Assert.Equal(snapshot1.Digest, snapshot2.Digest); - - optionsMonitor.Update(new VexPolicyOptions - { - ProviderOverrides = new Dictionary - { - ["provider-a"] = 0.4 - } - }); - - var snapshot3 = provider.GetSnapshot(); - Assert.Equal("rev-2", snapshot3.RevisionId); - Assert.NotEqual(snapshot1.Digest, snapshot3.Digest); - - listener.Dispose(); - - Assert.True(reloadMeasurements >= 2); - Assert.Equal("rev-2", lastRevision); - } - - private sealed class FakePolicyProvider : IVexPolicyProvider - { - private readonly VexPolicySnapshot _snapshot; - - public FakePolicyProvider(VexPolicySnapshot snapshot) - { - _snapshot = snapshot; - } - - public VexPolicySnapshot GetSnapshot() => _snapshot; - } - - private sealed class MutableOptionsMonitor : IOptionsMonitor - { - private T _value; - - public MutableOptionsMonitor(T value) - { - _value = value; - } - - public T CurrentValue => _value; - - public T Get(string? name) => _value; - - public void Update(T newValue) => _value = newValue; - - public IDisposable OnChange(Action listener) => NullDisposable.Instance; - - private sealed class NullDisposable : IDisposable - { - public static readonly NullDisposable Instance = new(); - public void Dispose() - { - } - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using System.Diagnostics.Metrics; + +namespace StellaOps.Excititor.Core.Tests; + +public class VexPolicyDiagnosticsTests +{ + [Fact] + public void GetDiagnostics_ReportsCountsRecommendationsAndOverrides() + { + var overrides = new[] + { + new KeyValuePair("provider-a", 0.8), + new KeyValuePair("provider-b", 0.6), + }; + + var snapshot = new VexPolicySnapshot( + "custom/v1", + new VexConsensusPolicyOptions( + version: "custom/v1", + providerOverrides: overrides), + new BaselineVexConsensusPolicy(), + ImmutableArray.Create( + new VexPolicyIssue("sample.error", "Blocking issue.", VexPolicyIssueSeverity.Error), + new VexPolicyIssue("sample.warning", "Non-blocking issue.", VexPolicyIssueSeverity.Warning)), + "rev-test", + "ABCDEF"); + + var fakeProvider = new FakePolicyProvider(snapshot); + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 16, 17, 0, 0, TimeSpan.Zero)); + var diagnostics = new VexPolicyDiagnostics(fakeProvider, fakeTime); + + var report = diagnostics.GetDiagnostics(); + + Assert.Equal("custom/v1", report.Version); + Assert.Equal("rev-test", report.RevisionId); + Assert.Equal("ABCDEF", report.Digest); + Assert.Equal(1, report.ErrorCount); + Assert.Equal(1, report.WarningCount); + Assert.Equal(fakeTime.GetUtcNow(), report.GeneratedAt); + Assert.Collection(report.Issues, + issue => Assert.Equal("sample.error", issue.Code), + issue => Assert.Equal("sample.warning", issue.Code)); + Assert.Equal(new[] { "provider-a", "provider-b" }, report.ActiveOverrides.Keys.OrderBy(static key => key, StringComparer.Ordinal)); + Assert.Contains(report.Recommendations, message => message.Contains("Resolve policy errors", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(report.Recommendations, message => message.Contains("provider-a", StringComparison.OrdinalIgnoreCase)); + Assert.Contains(report.Recommendations, message => message.Contains("docs/ARCHITECTURE_EXCITITOR.md", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void GetDiagnostics_WhenNoIssues_StillReturnsDefaultRecommendation() + { + var fakeProvider = new FakePolicyProvider(VexPolicySnapshot.Default); + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 16, 17, 0, 0, TimeSpan.Zero)); + var diagnostics = new VexPolicyDiagnostics(fakeProvider, fakeTime); + + var report = diagnostics.GetDiagnostics(); + + Assert.Equal(0, report.ErrorCount); + Assert.Equal(0, report.WarningCount); + Assert.Empty(report.ActiveOverrides); + Assert.Single(report.Recommendations); + } + + [Fact] + public void PolicyProvider_ComputesRevisionAndDigest_AndEmitsTelemetry() + { + using var listener = new MeterListener(); + var reloadMeasurements = 0; + string? lastRevision = null; + listener.InstrumentPublished += (instrument, _) => + { + if (instrument.Meter.Name == "StellaOps.Excititor.Policy" && + instrument.Name == "vex.policy.reloads") + { + listener.EnableMeasurementEvents(instrument); + } + }; + + listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + reloadMeasurements++; + foreach (var tag in tags) + { + if (tag.Key is "revision" && tag.Value is string revision) + { + lastRevision = revision; + break; + } + } + }); + + listener.Start(); + + var optionsMonitor = new MutableOptionsMonitor(new VexPolicyOptions()); + var provider = new VexPolicyProvider(optionsMonitor, NullLogger.Instance); + + var snapshot1 = provider.GetSnapshot(); + Assert.Equal("rev-1", snapshot1.RevisionId); + Assert.False(string.IsNullOrWhiteSpace(snapshot1.Digest)); + + var snapshot2 = provider.GetSnapshot(); + Assert.Equal("rev-1", snapshot2.RevisionId); + Assert.Equal(snapshot1.Digest, snapshot2.Digest); + + optionsMonitor.Update(new VexPolicyOptions + { + ProviderOverrides = new Dictionary + { + ["provider-a"] = 0.4 + } + }); + + var snapshot3 = provider.GetSnapshot(); + Assert.Equal("rev-2", snapshot3.RevisionId); + Assert.NotEqual(snapshot1.Digest, snapshot3.Digest); + + listener.Dispose(); + + Assert.True(reloadMeasurements >= 2); + Assert.Equal("rev-2", lastRevision); + } + + private sealed class FakePolicyProvider : IVexPolicyProvider + { + private readonly VexPolicySnapshot _snapshot; + + public FakePolicyProvider(VexPolicySnapshot snapshot) + { + _snapshot = snapshot; + } + + public VexPolicySnapshot GetSnapshot() => _snapshot; + } + + private sealed class MutableOptionsMonitor : IOptionsMonitor + { + private T _value; + + public MutableOptionsMonitor(T value) + { + _value = value; + } + + public T CurrentValue => _value; + + public T Get(string? name) => _value; + + public void Update(T newValue) => _value = newValue; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Vexer.Core.Tests/VexQuerySignatureTests.cs b/src/StellaOps.Excititor.Core.Tests/VexQuerySignatureTests.cs similarity index 92% rename from src/StellaOps.Vexer.Core.Tests/VexQuerySignatureTests.cs rename to src/StellaOps.Excititor.Core.Tests/VexQuerySignatureTests.cs index 688a71fc..32303f54 100644 --- a/src/StellaOps.Vexer.Core.Tests/VexQuerySignatureTests.cs +++ b/src/StellaOps.Excititor.Core.Tests/VexQuerySignatureTests.cs @@ -1,59 +1,59 @@ -using System.Collections.Generic; -using StellaOps.Vexer.Core; -using Xunit; - -namespace StellaOps.Vexer.Core.Tests; - -public sealed class VexQuerySignatureTests -{ - [Fact] - public void FromFilters_SortsAlphabetically() - { - var filters = new[] - { - new KeyValuePair("provider", "redhat"), - new KeyValuePair("vulnId", "CVE-2025-0001"), - new KeyValuePair("provider", "cisco"), - }; - - var signature = VexQuerySignature.FromFilters(filters); - - Assert.Equal("provider=cisco&provider=redhat&vulnId=CVE-2025-0001", signature.Value); - } - - [Fact] - public void FromQuery_NormalizesFiltersAndSort() - { - var query = VexQuery.Create( - filters: new[] - { - new VexQueryFilter(" provider ", " redhat "), - new VexQueryFilter("vulnId", "CVE-2025-0002"), - }, - sort: new[] - { - new VexQuerySort("published", true), - new VexQuerySort("severity", false), - }, - limit: 200, - offset: 10, - view: "consensus"); - - var signature = VexQuerySignature.FromQuery(query); - - Assert.Equal( - "provider=redhat&vulnId=CVE-2025-0002&sort=-published&sort=+severity&limit=200&offset=10&view=consensus", - signature.Value); - } - - [Fact] - public void ComputeHash_ReturnsStableSha256() - { - var signature = new VexQuerySignature("provider=redhat&vulnId=CVE-2025-0003"); - - var address = signature.ComputeHash(); - - Assert.Equal("sha256", address.Algorithm); - Assert.Equal("44c9881aaa79050ae943eaaf78afa697b1a4d3e38b03e20db332f2bd1e5b1029", address.Digest); - } -} +using System.Collections.Generic; +using StellaOps.Excititor.Core; +using Xunit; + +namespace StellaOps.Excititor.Core.Tests; + +public sealed class VexQuerySignatureTests +{ + [Fact] + public void FromFilters_SortsAlphabetically() + { + var filters = new[] + { + new KeyValuePair("provider", "redhat"), + new KeyValuePair("vulnId", "CVE-2025-0001"), + new KeyValuePair("provider", "cisco"), + }; + + var signature = VexQuerySignature.FromFilters(filters); + + Assert.Equal("provider=cisco&provider=redhat&vulnId=CVE-2025-0001", signature.Value); + } + + [Fact] + public void FromQuery_NormalizesFiltersAndSort() + { + var query = VexQuery.Create( + filters: new[] + { + new VexQueryFilter(" provider ", " redhat "), + new VexQueryFilter("vulnId", "CVE-2025-0002"), + }, + sort: new[] + { + new VexQuerySort("published", true), + new VexQuerySort("severity", false), + }, + limit: 200, + offset: 10, + view: "consensus"); + + var signature = VexQuerySignature.FromQuery(query); + + Assert.Equal( + "provider=redhat&vulnId=CVE-2025-0002&sort=-published&sort=+severity&limit=200&offset=10&view=consensus", + signature.Value); + } + + [Fact] + public void ComputeHash_ReturnsStableSha256() + { + var signature = new VexQuerySignature("provider=redhat&vulnId=CVE-2025-0003"); + + var address = signature.ComputeHash(); + + Assert.Equal("sha256", address.Algorithm); + Assert.Equal("44c9881aaa79050ae943eaaf78afa697b1a4d3e38b03e20db332f2bd1e5b1029", address.Digest); + } +} diff --git a/src/StellaOps.Excititor.Core.Tests/VexSignalSnapshotTests.cs b/src/StellaOps.Excititor.Core.Tests/VexSignalSnapshotTests.cs new file mode 100644 index 00000000..b635c727 --- /dev/null +++ b/src/StellaOps.Excititor.Core.Tests/VexSignalSnapshotTests.cs @@ -0,0 +1,35 @@ +using System; +using Xunit; + +namespace StellaOps.Excititor.Core.Tests; + +public sealed class VexSignalSnapshotTests +{ + [Theory] + [InlineData(-0.01)] + [InlineData(1.01)] + [InlineData(double.NaN)] + [InlineData(double.PositiveInfinity)] + public void Constructor_InvalidEpss_Throws(double value) + { + Assert.Throws(() => new VexSignalSnapshot(epss: value)); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData(null)] + public void VexSeveritySignal_InvalidScheme_Throws(string? scheme) + { + Assert.Throws(() => new VexSeveritySignal(scheme!)); + } + + [Theory] + [InlineData(-0.1)] + [InlineData(double.NaN)] + [InlineData(double.NegativeInfinity)] + public void VexSeveritySignal_InvalidScore_Throws(double value) + { + Assert.Throws(() => new VexSeveritySignal("cvss", value)); + } +} diff --git a/src/StellaOps.Vexer.Core/AGENTS.md b/src/StellaOps.Excititor.Core/AGENTS.md similarity index 77% rename from src/StellaOps.Vexer.Core/AGENTS.md rename to src/StellaOps.Excititor.Core/AGENTS.md index dbe5b507..a04ff935 100644 --- a/src/StellaOps.Vexer.Core/AGENTS.md +++ b/src/StellaOps.Excititor.Core/AGENTS.md @@ -1,26 +1,26 @@ -# AGENTS -## Role -Domain source of truth for VEX statements, consensus rollups, and trust policy orchestration across all Vexer services. -## Scope -- Records for raw document metadata, normalized claims, consensus projections, and export descriptors. -- Policy + weighting engine that projects provider trust tiers into consensus status outcomes. -- Connector, normalizer, export, and attestation contracts shared by WebService, Worker, and plug-ins. -- Deterministic hashing utilities (query signatures, artifact digests, attestation subjects). -## Participants -- Vexer WebService uses the models to persist ingress/egress payloads and to perform consensus mutations. -- Vexer Worker executes reconciliation and verification routines using policy helpers defined here. -- Export/Attestation modules depend on record definitions for envelopes and manifest payloads. -## Interfaces & contracts -- `IVexConnector`, `INormalizer`, `IExportEngine`, `ITransparencyLogClient`, `IArtifactStore`, and policy abstractions for consensus resolution. -- Value objects for provider metadata, VexClaim, VexConsensusEntry, ExportManifest, QuerySignature. -- Deterministic comparer utilities and stable JSON serialization helpers for tests and cache keys. -## In/Out of scope -In: domain invariants, policy evaluation helpers, deterministic serialization, shared abstractions. -Out: Mongo persistence implementations, HTTP endpoints, background scheduling, concrete connector logic. -## Observability & security expectations -- Avoid secret handling; provide structured logging extension methods for consensus decisions. -- Emit correlation identifiers and query signatures without embedding PII. -- Ensure deterministic logging order to keep reproducibility guarantees intact. -## Tests -- Unit coverage lives in `../StellaOps.Vexer.Core.Tests` (to be scaffolded) focusing on consensus, policy gates, and serialization determinism. -- Golden fixtures must rely on canonical JSON snapshots produced via stable serializers. +# AGENTS +## Role +Domain source of truth for VEX statements, consensus rollups, and trust policy orchestration across all Excititor services. +## Scope +- Records for raw document metadata, normalized claims, consensus projections, and export descriptors. +- Policy + weighting engine that projects provider trust tiers into consensus status outcomes. +- Connector, normalizer, export, and attestation contracts shared by WebService, Worker, and plug-ins. +- Deterministic hashing utilities (query signatures, artifact digests, attestation subjects). +## Participants +- Excititor WebService uses the models to persist ingress/egress payloads and to perform consensus mutations. +- Excititor Worker executes reconciliation and verification routines using policy helpers defined here. +- Export/Attestation modules depend on record definitions for envelopes and manifest payloads. +## Interfaces & contracts +- `IVexConnector`, `INormalizer`, `IExportEngine`, `ITransparencyLogClient`, `IArtifactStore`, and policy abstractions for consensus resolution. +- Value objects for provider metadata, VexClaim, VexConsensusEntry, ExportManifest, QuerySignature. +- Deterministic comparer utilities and stable JSON serialization helpers for tests and cache keys. +## In/Out of scope +In: domain invariants, policy evaluation helpers, deterministic serialization, shared abstractions. +Out: Mongo persistence implementations, HTTP endpoints, background scheduling, concrete connector logic. +## Observability & security expectations +- Avoid secret handling; provide structured logging extension methods for consensus decisions. +- Emit correlation identifiers and query signatures without embedding PII. +- Ensure deterministic logging order to keep reproducibility guarantees intact. +## Tests +- Unit coverage lives in `../StellaOps.Excititor.Core.Tests` (to be scaffolded) focusing on consensus, policy gates, and serialization determinism. +- Golden fixtures must rely on canonical JSON snapshots produced via stable serializers. diff --git a/src/StellaOps.Vexer.Core/BaselineVexConsensusPolicy.cs b/src/StellaOps.Excititor.Core/BaselineVexConsensusPolicy.cs similarity index 94% rename from src/StellaOps.Vexer.Core/BaselineVexConsensusPolicy.cs rename to src/StellaOps.Excititor.Core/BaselineVexConsensusPolicy.cs index 170fec6e..dee90ff6 100644 --- a/src/StellaOps.Vexer.Core/BaselineVexConsensusPolicy.cs +++ b/src/StellaOps.Excititor.Core/BaselineVexConsensusPolicy.cs @@ -1,61 +1,61 @@ -namespace StellaOps.Vexer.Core; - -/// -/// Baseline consensus policy applying tier-based weights and enforcing justification gates. -/// -public sealed class BaselineVexConsensusPolicy : IVexConsensusPolicy -{ - private readonly VexConsensusPolicyOptions _options; - - public BaselineVexConsensusPolicy(VexConsensusPolicyOptions? options = null) - { - _options = options ?? new VexConsensusPolicyOptions(); - } - - public string Version => _options.Version; - - public double GetProviderWeight(VexProvider provider) - { - if (provider is null) - { - throw new ArgumentNullException(nameof(provider)); - } - - if (_options.ProviderOverrides.TryGetValue(provider.Id, out var overrideWeight)) - { - return overrideWeight; - } - - return provider.Kind switch - { - VexProviderKind.Vendor => _options.VendorWeight, - VexProviderKind.Distro => _options.DistroWeight, - VexProviderKind.Platform => _options.PlatformWeight, - VexProviderKind.Hub => _options.HubWeight, - VexProviderKind.Attestation => _options.AttestationWeight, - _ => 0, - }; - } - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - if (claim is null) - { - throw new ArgumentNullException(nameof(claim)); - } - - if (provider is null) - { - throw new ArgumentNullException(nameof(provider)); - } - - if (claim.Status is VexClaimStatus.NotAffected && claim.Justification is null) - { - rejectionReason = "missing_justification"; - return false; - } - - rejectionReason = null; - return true; - } -} +namespace StellaOps.Excititor.Core; + +/// +/// Baseline consensus policy applying tier-based weights and enforcing justification gates. +/// +public sealed class BaselineVexConsensusPolicy : IVexConsensusPolicy +{ + private readonly VexConsensusPolicyOptions _options; + + public BaselineVexConsensusPolicy(VexConsensusPolicyOptions? options = null) + { + _options = options ?? new VexConsensusPolicyOptions(); + } + + public string Version => _options.Version; + + public double GetProviderWeight(VexProvider provider) + { + if (provider is null) + { + throw new ArgumentNullException(nameof(provider)); + } + + if (_options.ProviderOverrides.TryGetValue(provider.Id, out var overrideWeight)) + { + return overrideWeight; + } + + return provider.Kind switch + { + VexProviderKind.Vendor => _options.VendorWeight, + VexProviderKind.Distro => _options.DistroWeight, + VexProviderKind.Platform => _options.PlatformWeight, + VexProviderKind.Hub => _options.HubWeight, + VexProviderKind.Attestation => _options.AttestationWeight, + _ => 0, + }; + } + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + if (claim is null) + { + throw new ArgumentNullException(nameof(claim)); + } + + if (provider is null) + { + throw new ArgumentNullException(nameof(provider)); + } + + if (claim.Status is VexClaimStatus.NotAffected && claim.Justification is null) + { + rejectionReason = "missing_justification"; + return false; + } + + rejectionReason = null; + return true; + } +} diff --git a/src/StellaOps.Vexer.Core/IVexConsensusPolicy.cs b/src/StellaOps.Excititor.Core/IVexConsensusPolicy.cs similarity index 84% rename from src/StellaOps.Vexer.Core/IVexConsensusPolicy.cs rename to src/StellaOps.Excititor.Core/IVexConsensusPolicy.cs index 26444ebf..47b6e8a6 100644 --- a/src/StellaOps.Vexer.Core/IVexConsensusPolicy.cs +++ b/src/StellaOps.Excititor.Core/IVexConsensusPolicy.cs @@ -1,26 +1,26 @@ -namespace StellaOps.Vexer.Core; - -/// -/// Policy abstraction supplying trust weights and gating logic for consensus decisions. -/// -public interface IVexConsensusPolicy -{ - /// - /// Semantic version describing the active policy. - /// - string Version { get; } - - /// - /// Returns the effective weight (0-1) to apply for the provided VEX source. - /// - double GetProviderWeight(VexProvider provider); - - /// - /// Determines whether the claim is eligible to participate in consensus. - /// - /// Normalized claim to evaluate. - /// Provider metadata for the claim. - /// Textual reason when the claim is rejected. - /// true if the claim should participate; false otherwise. - bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason); -} +namespace StellaOps.Excititor.Core; + +/// +/// Policy abstraction supplying trust weights and gating logic for consensus decisions. +/// +public interface IVexConsensusPolicy +{ + /// + /// Semantic version describing the active policy. + /// + string Version { get; } + + /// + /// Returns the effective weight (bounded by the policy ceiling) to apply for the provided VEX source. + /// + double GetProviderWeight(VexProvider provider); + + /// + /// Determines whether the claim is eligible to participate in consensus. + /// + /// Normalized claim to evaluate. + /// Provider metadata for the claim. + /// Textual reason when the claim is rejected. + /// true if the claim should participate; false otherwise. + bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason); +} diff --git a/src/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj b/src/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj new file mode 100644 index 00000000..2b6aadf4 --- /dev/null +++ b/src/StellaOps.Excititor.Core/StellaOps.Excititor.Core.csproj @@ -0,0 +1,9 @@ + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Excititor.Core/TASKS.md b/src/StellaOps.Excititor.Core/TASKS.md new file mode 100644 index 00000000..3d06ea35 --- /dev/null +++ b/src/StellaOps.Excititor.Core/TASKS.md @@ -0,0 +1,9 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-CORE-01-001 – Canonical VEX domain records|Team Excititor Core & Policy|docs/ARCHITECTURE_EXCITITOR.md|DONE (2025-10-15) – Introduced `VexClaim`, `VexConsensus`, provider metadata, export manifest records, and deterministic JSON serialization with tests covering canonical ordering and query signatures.| +|EXCITITOR-CORE-01-002 – Trust-weighted consensus resolver|Team Excititor Core & Policy|EXCITITOR-CORE-01-001|DONE (2025-10-15) – Added consensus resolver, baseline policy (tier weights + justification gate), telemetry output, and tests covering acceptance, conflict ties, and determinism.| +|EXCITITOR-CORE-01-003 – Shared contracts & query signatures|Team Excititor Core & Policy|EXCITITOR-CORE-01-001|DONE (2025-10-15) – Published connector/normalizer/exporter/attestation abstractions and expanded deterministic `VexQuerySignature`/hash utilities with test coverage.| +|EXCITITOR-CORE-02-001 – Context signal schema prep|Team Excititor Core & Policy|EXCITITOR-POLICY-02-001|DONE (2025-10-19) – Added `VexSignalSnapshot` (severity/KEV/EPSS) to claims/consensus, updated canonical serializer + resolver plumbing, documented storage follow-up, and validated via `dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`.| +|EXCITITOR-CORE-02-002 – Deterministic risk scoring engine|Team Excititor Core & Policy|EXCITITOR-CORE-02-001, EXCITITOR-POLICY-02-001|BACKLOG – Introduce the scoring calculator invoked by consensus, persist score envelopes with audit trails, and add regression fixtures covering gate/boost behaviour before enabling exports.| diff --git a/src/StellaOps.Vexer.Core/VexAttestationAbstractions.cs b/src/StellaOps.Excititor.Core/VexAttestationAbstractions.cs similarity index 93% rename from src/StellaOps.Vexer.Core/VexAttestationAbstractions.cs rename to src/StellaOps.Excititor.Core/VexAttestationAbstractions.cs index 8c8a8fe0..7e4748f1 100644 --- a/src/StellaOps.Vexer.Core/VexAttestationAbstractions.cs +++ b/src/StellaOps.Excititor.Core/VexAttestationAbstractions.cs @@ -1,30 +1,30 @@ -using System; -using System.Collections.Immutable; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Vexer.Core; - -public interface IVexAttestationClient -{ - ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken); - - ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken); -} - -public sealed record VexAttestationRequest( - string ExportId, - VexQuerySignature QuerySignature, - VexContentAddress Artifact, - VexExportFormat Format, - DateTimeOffset CreatedAt, - ImmutableArray SourceProviders, - ImmutableDictionary Metadata); - -public sealed record VexAttestationResponse( - VexAttestationMetadata Attestation, - ImmutableDictionary Diagnostics); - -public sealed record VexAttestationVerification( - bool IsValid, - ImmutableDictionary Diagnostics); +using System; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Core; + +public interface IVexAttestationClient +{ + ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken); + + ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken); +} + +public sealed record VexAttestationRequest( + string ExportId, + VexQuerySignature QuerySignature, + VexContentAddress Artifact, + VexExportFormat Format, + DateTimeOffset CreatedAt, + ImmutableArray SourceProviders, + ImmutableDictionary Metadata); + +public sealed record VexAttestationResponse( + VexAttestationMetadata Attestation, + ImmutableDictionary Diagnostics); + +public sealed record VexAttestationVerification( + bool IsValid, + ImmutableDictionary Diagnostics); diff --git a/src/StellaOps.Vexer.Core/VexCacheEntry.cs b/src/StellaOps.Excititor.Core/VexCacheEntry.cs similarity index 94% rename from src/StellaOps.Vexer.Core/VexCacheEntry.cs rename to src/StellaOps.Excititor.Core/VexCacheEntry.cs index 3fec0666..457520f3 100644 --- a/src/StellaOps.Vexer.Core/VexCacheEntry.cs +++ b/src/StellaOps.Excititor.Core/VexCacheEntry.cs @@ -1,56 +1,56 @@ -using System; - -namespace StellaOps.Vexer.Core; - -/// -/// Cached export artifact metadata allowing reuse of previously generated manifests. -/// -public sealed class VexCacheEntry -{ - public VexCacheEntry( - VexQuerySignature querySignature, - VexExportFormat format, - VexContentAddress artifact, - DateTimeOffset createdAt, - long sizeBytes, - string? manifestId = null, - string? gridFsObjectId = null, - DateTimeOffset? expiresAt = null) - { - QuerySignature = querySignature ?? throw new ArgumentNullException(nameof(querySignature)); - Artifact = artifact ?? throw new ArgumentNullException(nameof(artifact)); - Format = format; - CreatedAt = createdAt; - SizeBytes = sizeBytes >= 0 - ? sizeBytes - : throw new ArgumentOutOfRangeException(nameof(sizeBytes), sizeBytes, "Size must be non-negative."); - ManifestId = Normalize(manifestId); - GridFsObjectId = Normalize(gridFsObjectId); - - if (expiresAt.HasValue && expiresAt.Value < createdAt) - { - throw new ArgumentOutOfRangeException(nameof(expiresAt), expiresAt, "Expiration cannot be before creation."); - } - - ExpiresAt = expiresAt; - } - - public VexQuerySignature QuerySignature { get; } - - public VexExportFormat Format { get; } - - public VexContentAddress Artifact { get; } - - public DateTimeOffset CreatedAt { get; } - - public long SizeBytes { get; } - - public string? ManifestId { get; } - - public string? GridFsObjectId { get; } - - public DateTimeOffset? ExpiresAt { get; } - - private static string? Normalize(string? value) - => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); -} +using System; + +namespace StellaOps.Excititor.Core; + +/// +/// Cached export artifact metadata allowing reuse of previously generated manifests. +/// +public sealed class VexCacheEntry +{ + public VexCacheEntry( + VexQuerySignature querySignature, + VexExportFormat format, + VexContentAddress artifact, + DateTimeOffset createdAt, + long sizeBytes, + string? manifestId = null, + string? gridFsObjectId = null, + DateTimeOffset? expiresAt = null) + { + QuerySignature = querySignature ?? throw new ArgumentNullException(nameof(querySignature)); + Artifact = artifact ?? throw new ArgumentNullException(nameof(artifact)); + Format = format; + CreatedAt = createdAt; + SizeBytes = sizeBytes >= 0 + ? sizeBytes + : throw new ArgumentOutOfRangeException(nameof(sizeBytes), sizeBytes, "Size must be non-negative."); + ManifestId = Normalize(manifestId); + GridFsObjectId = Normalize(gridFsObjectId); + + if (expiresAt.HasValue && expiresAt.Value < createdAt) + { + throw new ArgumentOutOfRangeException(nameof(expiresAt), expiresAt, "Expiration cannot be before creation."); + } + + ExpiresAt = expiresAt; + } + + public VexQuerySignature QuerySignature { get; } + + public VexExportFormat Format { get; } + + public VexContentAddress Artifact { get; } + + public DateTimeOffset CreatedAt { get; } + + public long SizeBytes { get; } + + public string? ManifestId { get; } + + public string? GridFsObjectId { get; } + + public DateTimeOffset? ExpiresAt { get; } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); +} diff --git a/src/StellaOps.Vexer.Core/VexCanonicalJsonSerializer.cs b/src/StellaOps.Excititor.Core/VexCanonicalJsonSerializer.cs similarity index 88% rename from src/StellaOps.Vexer.Core/VexCanonicalJsonSerializer.cs rename to src/StellaOps.Excititor.Core/VexCanonicalJsonSerializer.cs index 4c6dc055..7e040743 100644 --- a/src/StellaOps.Vexer.Core/VexCanonicalJsonSerializer.cs +++ b/src/StellaOps.Excititor.Core/VexCanonicalJsonSerializer.cs @@ -1,494 +1,544 @@ -using System.Collections.Generic; -using System.Reflection; -using System.Runtime.Serialization; -using System.Text.Encodings.Web; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Text.Json.Serialization.Metadata; - -namespace StellaOps.Vexer.Core; - -public static class VexCanonicalJsonSerializer -{ - private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); - private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); - - private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary - { - { - typeof(VexProvider), - new[] - { - "id", - "displayName", - "kind", - "baseUris", - "discovery", - "trust", - "enabled", - } - }, - { - typeof(VexProviderDiscovery), - new[] - { - "wellKnownMetadata", - "rolIeService", - } - }, - { - typeof(VexProviderTrust), - new[] - { - "weight", - "cosign", - "pgpFingerprints", - } - }, - { - typeof(VexCosignTrust), - new[] - { - "issuer", - "identityPattern", - } - }, - { - typeof(VexClaim), - new[] - { - "vulnerabilityId", - "providerId", - "product", - "status", - "justification", - "detail", - "document", - "firstSeen", - "lastSeen", - "confidence", - "additionalMetadata", - } - }, - { - typeof(VexProduct), - new[] - { - "key", - "name", - "version", - "purl", - "cpe", - "componentIdentifiers", - } - }, - { - typeof(VexClaimDocument), - new[] - { - "format", - "digest", - "sourceUri", - "revision", - "signature", - } - }, - { - typeof(VexSignatureMetadata), - new[] - { - "type", - "subject", - "issuer", - "keyId", - "verifiedAt", - "transparencyLogReference", - } - }, - { - typeof(VexConfidence), - new[] - { - "level", - "score", - "method", - } - }, - { - typeof(VexConsensus), - new[] - { - "vulnerabilityId", - "product", - "status", - "calculatedAt", - "sources", - "conflicts", - "policyVersion", - "summary", - } - }, - { - typeof(VexConsensusSource), - new[] - { - "providerId", - "status", - "documentDigest", - "weight", - "justification", - "detail", - "confidence", - } - }, - { - typeof(VexConsensusConflict), - new[] - { - "providerId", - "status", - "documentDigest", - "justification", - "detail", - "reason", - } - }, - { - typeof(VexConnectorSettings), - new[] - { - "values", - } - }, - { - typeof(VexConnectorContext), - new[] - { - "since", - "settings", - "rawSink", - "signatureVerifier", - "normalizers", - "services", - } - }, - { - typeof(VexRawDocument), - new[] - { - "documentId", - "providerId", - "format", - "sourceUri", - "retrievedAt", - "digest", - "content", - "metadata", - } - }, - { - typeof(VexClaimBatch), - new[] - { - "source", - "claims", - "diagnostics", - } - }, - { - typeof(VexExportManifest), - new[] - { - "exportId", - "querySignature", - "format", - "createdAt", - "artifact", - "claimCount", - "fromCache", - "sourceProviders", - "consensusRevision", - "attestation", - "sizeBytes", - } - }, - { - typeof(VexContentAddress), - new[] - { - "algorithm", - "digest", - } - }, - { - typeof(VexAttestationMetadata), - new[] - { - "predicateType", - "rekor", - "envelopeDigest", - "signedAt", - } - }, - { - typeof(VexRekorReference), - new[] - { - "apiVersion", - "location", - "logIndex", - "inclusionProofUri", - } - }, - { - typeof(VexQuerySignature), - new[] - { - "value", - } - }, - { - typeof(VexQuery), - new[] - { - "filters", - "sort", - "limit", - "offset", - "view", - } - }, - { - typeof(VexQueryFilter), - new[] - { - "key", - "value", - } - }, - { - typeof(VexQuerySort), - new[] - { - "field", - "descending", - } - }, - { - typeof(VexExportRequest), - new[] - { - "query", - "consensus", - "claims", - "generatedAt", - } - }, - { - typeof(VexExportResult), - new[] - { - "digest", - "bytesWritten", - "metadata", - } - }, - { - typeof(VexAttestationRequest), - new[] - { - "querySignature", - "artifact", - "format", - "createdAt", - "metadata", - } - }, - { - typeof(VexAttestationResponse), - new[] - { - "attestation", - "diagnostics", - } - }, - { - typeof(VexAttestationVerification), - new[] - { - "isValid", - "diagnostics", - } - }, - }; - - public static string Serialize(T value) - => JsonSerializer.Serialize(value, CompactOptions); - - public static string SerializeIndented(T value) - => JsonSerializer.Serialize(value, PrettyOptions); - - public static T Deserialize(string json) - => JsonSerializer.Deserialize(json, PrettyOptions) - ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); - - private static JsonSerializerOptions CreateOptions(bool writeIndented) - { - var options = new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, - DefaultIgnoreCondition = JsonIgnoreCondition.Never, - WriteIndented = writeIndented, - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, - }; - - var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); - options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); - options.Converters.Add(new EnumMemberJsonConverterFactory()); - return options; - } - - private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver - { - private readonly IJsonTypeInfoResolver _inner; - - public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) - { - _inner = inner ?? throw new ArgumentNullException(nameof(inner)); - } - - public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options) - { - var info = _inner.GetTypeInfo(type, options); - if (info is null) - { - return null; - } - - if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) - { - var ordered = info.Properties - .OrderBy(property => GetPropertyOrder(type, property.Name)) - .ThenBy(property => property.Name, StringComparer.Ordinal) - .ToArray(); - - info.Properties.Clear(); - foreach (var property in ordered) - { - info.Properties.Add(property); - } - } - - return info; - } - - private static int GetPropertyOrder(Type type, string propertyName) - { - if (PropertyOrderOverrides.TryGetValue(type, out var order) && - Array.IndexOf(order, propertyName) is var index && - index >= 0) - { - return index; - } - - return int.MaxValue; - } - } - - private sealed class EnumMemberJsonConverterFactory : JsonConverterFactory - { - public override bool CanConvert(Type typeToConvert) - { - var type = Nullable.GetUnderlyingType(typeToConvert) ?? typeToConvert; - return type.IsEnum; - } - - public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) - { - var underlying = Nullable.GetUnderlyingType(typeToConvert); - if (underlying is not null) - { - var nullableConverterType = typeof(NullableEnumMemberJsonConverter<>).MakeGenericType(underlying); - return (JsonConverter)Activator.CreateInstance(nullableConverterType)!; - } - - var converterType = typeof(EnumMemberJsonConverter<>).MakeGenericType(typeToConvert); - return (JsonConverter)Activator.CreateInstance(converterType)!; - } - - private sealed class EnumMemberJsonConverter : JsonConverter - where T : struct, Enum - { - private readonly Dictionary _nameToValue; - private readonly Dictionary _valueToName; - - public EnumMemberJsonConverter() - { - _nameToValue = new Dictionary(StringComparer.Ordinal); - _valueToName = new Dictionary(); - foreach (var value in Enum.GetValues()) - { - var name = value.ToString(); - var enumMember = typeof(T).GetField(name)!.GetCustomAttribute(); - var text = enumMember?.Value ?? name; - _nameToValue[text] = value; - _valueToName[value] = text; - } - } - - public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (reader.TokenType != JsonTokenType.String) - { - throw new JsonException($"Unexpected token '{reader.TokenType}' when parsing enum '{typeof(T).Name}'."); - } - - var text = reader.GetString(); - if (text is null || !_nameToValue.TryGetValue(text, out var value)) - { - throw new JsonException($"Value '{text}' is not defined for enum '{typeof(T).Name}'."); - } - - return value; - } - - public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) - { - if (!_valueToName.TryGetValue(value, out var text)) - { - throw new JsonException($"Value '{value}' is not defined for enum '{typeof(T).Name}'."); - } - - writer.WriteStringValue(text); - } - } - - private sealed class NullableEnumMemberJsonConverter : JsonConverter - where T : struct, Enum - { - private readonly EnumMemberJsonConverter _inner = new(); - - public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - if (reader.TokenType == JsonTokenType.Null) - { - return null; - } - - return _inner.Read(ref reader, typeof(T), options); - } - - public override void Write(Utf8JsonWriter writer, T? value, JsonSerializerOptions options) - { - if (value is null) - { - writer.WriteNullValue(); - return; - } - - _inner.Write(writer, value.Value, options); - } - } - } -} +using System.Collections.Generic; +using System.Reflection; +using System.Runtime.Serialization; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Excititor.Core; + +public static class VexCanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary + { + { + typeof(VexProvider), + new[] + { + "id", + "displayName", + "kind", + "baseUris", + "discovery", + "trust", + "enabled", + } + }, + { + typeof(VexProviderDiscovery), + new[] + { + "wellKnownMetadata", + "rolIeService", + } + }, + { + typeof(VexProviderTrust), + new[] + { + "weight", + "cosign", + "pgpFingerprints", + } + }, + { + typeof(VexCosignTrust), + new[] + { + "issuer", + "identityPattern", + } + }, + { + typeof(VexClaim), + new[] + { + "vulnerabilityId", + "providerId", + "product", + "status", + "justification", + "detail", + "signals", + "document", + "firstSeen", + "lastSeen", + "confidence", + "additionalMetadata", + } + }, + { + typeof(VexProduct), + new[] + { + "key", + "name", + "version", + "purl", + "cpe", + "componentIdentifiers", + } + }, + { + typeof(VexClaimDocument), + new[] + { + "format", + "digest", + "sourceUri", + "revision", + "signature", + } + }, + { + typeof(VexSignatureMetadata), + new[] + { + "type", + "subject", + "issuer", + "keyId", + "verifiedAt", + "transparencyLogReference", + } + }, + { + typeof(VexConfidence), + new[] + { + "level", + "score", + "method", + } + }, + { + typeof(VexConsensus), + new[] + { + "vulnerabilityId", + "product", + "status", + "calculatedAt", + "sources", + "conflicts", + "signals", + "policyVersion", + "summary", + } + }, + { + typeof(VexConsensusSource), + new[] + { + "providerId", + "status", + "documentDigest", + "weight", + "justification", + "detail", + "confidence", + } + }, + { + typeof(VexConsensusConflict), + new[] + { + "providerId", + "status", + "documentDigest", + "justification", + "detail", + "reason", + } + }, + { + typeof(VexConnectorSettings), + new[] + { + "values", + } + }, + { + typeof(VexConnectorContext), + new[] + { + "since", + "settings", + "rawSink", + "signatureVerifier", + "normalizers", + "services", + } + }, + { + typeof(VexRawDocument), + new[] + { + "documentId", + "providerId", + "format", + "sourceUri", + "retrievedAt", + "digest", + "content", + "metadata", + } + }, + { + typeof(VexClaimBatch), + new[] + { + "source", + "claims", + "diagnostics", + } + }, + { + typeof(VexSignalSnapshot), + new[] + { + "severity", + "kev", + "epss", + } + }, + { + typeof(VexSeveritySignal), + new[] + { + "scheme", + "score", + "label", + "vector", + } + }, + { + typeof(VexExportManifest), + new[] + { + "exportId", + "querySignature", + "format", + "createdAt", + "artifact", + "claimCount", + "fromCache", + "sourceProviders", + "consensusRevision", + "policyRevisionId", + "policyDigest", + "consensusDigest", + "scoreDigest", + "attestation", + "sizeBytes", + } + }, + { + typeof(VexScoreEnvelope), + new[] + { + "generatedAt", + "policyRevisionId", + "policyDigest", + "alpha", + "beta", + "weightCeiling", + "entries", + } + }, + { + typeof(VexScoreEntry), + new[] + { + "vulnerabilityId", + "productKey", + "status", + "calculatedAt", + "signals", + "score", + } + }, + { + typeof(VexContentAddress), + new[] + { + "algorithm", + "digest", + } + }, + { + typeof(VexAttestationMetadata), + new[] + { + "predicateType", + "rekor", + "envelopeDigest", + "signedAt", + } + }, + { + typeof(VexRekorReference), + new[] + { + "apiVersion", + "location", + "logIndex", + "inclusionProofUri", + } + }, + { + typeof(VexQuerySignature), + new[] + { + "value", + } + }, + { + typeof(VexQuery), + new[] + { + "filters", + "sort", + "limit", + "offset", + "view", + } + }, + { + typeof(VexQueryFilter), + new[] + { + "key", + "value", + } + }, + { + typeof(VexQuerySort), + new[] + { + "field", + "descending", + } + }, + { + typeof(VexExportRequest), + new[] + { + "query", + "consensus", + "claims", + "generatedAt", + } + }, + { + typeof(VexExportResult), + new[] + { + "digest", + "bytesWritten", + "metadata", + } + }, + { + typeof(VexAttestationRequest), + new[] + { + "querySignature", + "artifact", + "format", + "createdAt", + "metadata", + } + }, + { + typeof(VexAttestationResponse), + new[] + { + "attestation", + "diagnostics", + } + }, + { + typeof(VexAttestationVerification), + new[] + { + "isValid", + "diagnostics", + } + }, + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, PrettyOptions) + ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + options.Converters.Add(new EnumMemberJsonConverterFactory()); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options); + if (info is null) + { + return null; + } + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetPropertyOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetPropertyOrder(Type type, string propertyName) + { + if (PropertyOrderOverrides.TryGetValue(type, out var order) && + Array.IndexOf(order, propertyName) is var index && + index >= 0) + { + return index; + } + + return int.MaxValue; + } + } + + private sealed class EnumMemberJsonConverterFactory : JsonConverterFactory + { + public override bool CanConvert(Type typeToConvert) + { + var type = Nullable.GetUnderlyingType(typeToConvert) ?? typeToConvert; + return type.IsEnum; + } + + public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var underlying = Nullable.GetUnderlyingType(typeToConvert); + if (underlying is not null) + { + var nullableConverterType = typeof(NullableEnumMemberJsonConverter<>).MakeGenericType(underlying); + return (JsonConverter)Activator.CreateInstance(nullableConverterType)!; + } + + var converterType = typeof(EnumMemberJsonConverter<>).MakeGenericType(typeToConvert); + return (JsonConverter)Activator.CreateInstance(converterType)!; + } + + private sealed class EnumMemberJsonConverter : JsonConverter + where T : struct, Enum + { + private readonly Dictionary _nameToValue; + private readonly Dictionary _valueToName; + + public EnumMemberJsonConverter() + { + _nameToValue = new Dictionary(StringComparer.Ordinal); + _valueToName = new Dictionary(); + foreach (var value in Enum.GetValues()) + { + var name = value.ToString(); + var enumMember = typeof(T).GetField(name)!.GetCustomAttribute(); + var text = enumMember?.Value ?? name; + _nameToValue[text] = value; + _valueToName[value] = text; + } + } + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.String) + { + throw new JsonException($"Unexpected token '{reader.TokenType}' when parsing enum '{typeof(T).Name}'."); + } + + var text = reader.GetString(); + if (text is null || !_nameToValue.TryGetValue(text, out var value)) + { + throw new JsonException($"Value '{text}' is not defined for enum '{typeof(T).Name}'."); + } + + return value; + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (!_valueToName.TryGetValue(value, out var text)) + { + throw new JsonException($"Value '{value}' is not defined for enum '{typeof(T).Name}'."); + } + + writer.WriteStringValue(text); + } + } + + private sealed class NullableEnumMemberJsonConverter : JsonConverter + where T : struct, Enum + { + private readonly EnumMemberJsonConverter _inner = new(); + + public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + + return _inner.Read(ref reader, typeof(T), options); + } + + public override void Write(Utf8JsonWriter writer, T? value, JsonSerializerOptions options) + { + if (value is null) + { + writer.WriteNullValue(); + return; + } + + _inner.Write(writer, value.Value, options); + } + } + } +} diff --git a/src/StellaOps.Vexer.Core/VexClaim.cs b/src/StellaOps.Excititor.Core/VexClaim.cs similarity index 94% rename from src/StellaOps.Vexer.Core/VexClaim.cs rename to src/StellaOps.Excititor.Core/VexClaim.cs index 207ba09b..6ff42310 100644 --- a/src/StellaOps.Vexer.Core/VexClaim.cs +++ b/src/StellaOps.Excititor.Core/VexClaim.cs @@ -1,326 +1,330 @@ -using System.Collections.Immutable; -using System.Runtime.Serialization; - -namespace StellaOps.Vexer.Core; - -public sealed record VexClaim -{ - public VexClaim( - string vulnerabilityId, - string providerId, - VexProduct product, - VexClaimStatus status, - VexClaimDocument document, - DateTimeOffset firstSeen, - DateTimeOffset lastSeen, - VexJustification? justification = null, - string? detail = null, - VexConfidence? confidence = null, - ImmutableDictionary? additionalMetadata = null) - { - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - throw new ArgumentException("Vulnerability id must be provided.", nameof(vulnerabilityId)); - } - - if (string.IsNullOrWhiteSpace(providerId)) - { - throw new ArgumentException("Provider id must be provided.", nameof(providerId)); - } - - if (lastSeen < firstSeen) - { - throw new ArgumentOutOfRangeException(nameof(lastSeen), "Last seen timestamp cannot be earlier than first seen."); - } - - VulnerabilityId = vulnerabilityId.Trim(); - ProviderId = providerId.Trim(); - Product = product ?? throw new ArgumentNullException(nameof(product)); - Status = status; - Document = document ?? throw new ArgumentNullException(nameof(document)); - FirstSeen = firstSeen; - LastSeen = lastSeen; - Justification = justification; - Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); - Confidence = confidence; - AdditionalMetadata = NormalizeMetadata(additionalMetadata); - } - - public string VulnerabilityId { get; } - - public string ProviderId { get; } - - public VexProduct Product { get; } - - public VexClaimStatus Status { get; } - - public VexJustification? Justification { get; } - - public string? Detail { get; } - - public VexClaimDocument Document { get; } - - public DateTimeOffset FirstSeen { get; } - - public DateTimeOffset LastSeen { get; } - - public VexConfidence? Confidence { get; } - - public ImmutableSortedDictionary AdditionalMetadata { get; } - - private static ImmutableSortedDictionary NormalizeMetadata( - ImmutableDictionary? additionalMetadata) - { - if (additionalMetadata is null || additionalMetadata.Count == 0) - { - return ImmutableSortedDictionary.Empty; - } - - var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); - foreach (var (key, value) in additionalMetadata) - { - if (string.IsNullOrWhiteSpace(key)) - { - continue; - } - - builder[key.Trim()] = value?.Trim() ?? string.Empty; - } - - return builder.ToImmutable(); - } -} - -public sealed record VexProduct -{ - public VexProduct( - string key, - string? name, - string? version = null, - string? purl = null, - string? cpe = null, - IEnumerable? componentIdentifiers = null) - { - if (string.IsNullOrWhiteSpace(key)) - { - throw new ArgumentException("Product key must be provided.", nameof(key)); - } - - Key = key.Trim(); - Name = string.IsNullOrWhiteSpace(name) ? null : name.Trim(); - Version = string.IsNullOrWhiteSpace(version) ? null : version.Trim(); - Purl = string.IsNullOrWhiteSpace(purl) ? null : purl.Trim(); - Cpe = string.IsNullOrWhiteSpace(cpe) ? null : cpe.Trim(); - ComponentIdentifiers = NormalizeComponentIdentifiers(componentIdentifiers); - } - - public string Key { get; } - - public string? Name { get; } - - public string? Version { get; } - - public string? Purl { get; } - - public string? Cpe { get; } - - public ImmutableArray ComponentIdentifiers { get; } - - private static ImmutableArray NormalizeComponentIdentifiers(IEnumerable? identifiers) - { - if (identifiers is null) - { - return ImmutableArray.Empty; - } - - var set = new SortedSet(StringComparer.Ordinal); - foreach (var identifier in identifiers) - { - if (string.IsNullOrWhiteSpace(identifier)) - { - continue; - } - - set.Add(identifier.Trim()); - } - - return set.Count == 0 ? ImmutableArray.Empty : set.ToImmutableArray(); - } -} - -public sealed record VexClaimDocument -{ - public VexClaimDocument( - VexDocumentFormat format, - string digest, - Uri sourceUri, - string? revision = null, - VexSignatureMetadata? signature = null) - { - if (string.IsNullOrWhiteSpace(digest)) - { - throw new ArgumentException("Document digest must be provided.", nameof(digest)); - } - - Format = format; - Digest = digest.Trim(); - SourceUri = sourceUri ?? throw new ArgumentNullException(nameof(sourceUri)); - Revision = string.IsNullOrWhiteSpace(revision) ? null : revision.Trim(); - Signature = signature; - } - - public VexDocumentFormat Format { get; } - - public string Digest { get; } - - public Uri SourceUri { get; } - - public string? Revision { get; } - - public VexSignatureMetadata? Signature { get; } -} - -public sealed record VexSignatureMetadata -{ - public VexSignatureMetadata( - string type, - string? subject = null, - string? issuer = null, - string? keyId = null, - DateTimeOffset? verifiedAt = null, - string? transparencyLogReference = null) - { - if (string.IsNullOrWhiteSpace(type)) - { - throw new ArgumentException("Signature type must be provided.", nameof(type)); - } - - Type = type.Trim(); - Subject = string.IsNullOrWhiteSpace(subject) ? null : subject.Trim(); - Issuer = string.IsNullOrWhiteSpace(issuer) ? null : issuer.Trim(); - KeyId = string.IsNullOrWhiteSpace(keyId) ? null : keyId.Trim(); - VerifiedAt = verifiedAt; - TransparencyLogReference = string.IsNullOrWhiteSpace(transparencyLogReference) - ? null - : transparencyLogReference.Trim(); - } - - public string Type { get; } - - public string? Subject { get; } - - public string? Issuer { get; } - - public string? KeyId { get; } - - public DateTimeOffset? VerifiedAt { get; } - - public string? TransparencyLogReference { get; } -} - -public sealed record VexConfidence -{ - public VexConfidence(string level, double? score = null, string? method = null) - { - if (string.IsNullOrWhiteSpace(level)) - { - throw new ArgumentException("Confidence level must be provided.", nameof(level)); - } - - if (score is not null && (double.IsNaN(score.Value) || double.IsInfinity(score.Value))) - { - throw new ArgumentOutOfRangeException(nameof(score), "Confidence score must be a finite number."); - } - - Level = level.Trim(); - Score = score; - Method = string.IsNullOrWhiteSpace(method) ? null : method.Trim(); - } - - public string Level { get; } - - public double? Score { get; } - - public string? Method { get; } -} - -[DataContract] -public enum VexDocumentFormat -{ - [EnumMember(Value = "csaf")] - Csaf, - - [EnumMember(Value = "cyclonedx")] - CycloneDx, - - [EnumMember(Value = "openvex")] - OpenVex, - - [EnumMember(Value = "oci_attestation")] - OciAttestation, -} - -[DataContract] -public enum VexClaimStatus -{ - [EnumMember(Value = "affected")] - Affected, - - [EnumMember(Value = "not_affected")] - NotAffected, - - [EnumMember(Value = "fixed")] - Fixed, - - [EnumMember(Value = "under_investigation")] - UnderInvestigation, -} - -[DataContract] -public enum VexJustification -{ - [EnumMember(Value = "component_not_present")] - ComponentNotPresent, - - [EnumMember(Value = "component_not_configured")] - ComponentNotConfigured, - - [EnumMember(Value = "vulnerable_code_not_present")] - VulnerableCodeNotPresent, - - [EnumMember(Value = "vulnerable_code_not_in_execute_path")] - VulnerableCodeNotInExecutePath, - - [EnumMember(Value = "vulnerable_code_cannot_be_controlled_by_adversary")] - VulnerableCodeCannotBeControlledByAdversary, - - [EnumMember(Value = "inline_mitigations_already_exist")] - InlineMitigationsAlreadyExist, - - [EnumMember(Value = "protected_by_mitigating_control")] - ProtectedByMitigatingControl, - - [EnumMember(Value = "code_not_present")] - CodeNotPresent, - - [EnumMember(Value = "code_not_reachable")] - CodeNotReachable, - - [EnumMember(Value = "requires_configuration")] - RequiresConfiguration, - - [EnumMember(Value = "requires_dependency")] - RequiresDependency, - - [EnumMember(Value = "requires_environment")] - RequiresEnvironment, - - [EnumMember(Value = "protected_by_compensating_control")] - ProtectedByCompensatingControl, - - [EnumMember(Value = "protected_at_perimeter")] - ProtectedAtPerimeter, - - [EnumMember(Value = "protected_at_runtime")] - ProtectedAtRuntime, -} +using System.Collections.Immutable; +using System.Runtime.Serialization; + +namespace StellaOps.Excititor.Core; + +public sealed record VexClaim +{ + public VexClaim( + string vulnerabilityId, + string providerId, + VexProduct product, + VexClaimStatus status, + VexClaimDocument document, + DateTimeOffset firstSeen, + DateTimeOffset lastSeen, + VexJustification? justification = null, + string? detail = null, + VexConfidence? confidence = null, + VexSignalSnapshot? signals = null, + ImmutableDictionary? additionalMetadata = null) + { + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + throw new ArgumentException("Vulnerability id must be provided.", nameof(vulnerabilityId)); + } + + if (string.IsNullOrWhiteSpace(providerId)) + { + throw new ArgumentException("Provider id must be provided.", nameof(providerId)); + } + + if (lastSeen < firstSeen) + { + throw new ArgumentOutOfRangeException(nameof(lastSeen), "Last seen timestamp cannot be earlier than first seen."); + } + + VulnerabilityId = vulnerabilityId.Trim(); + ProviderId = providerId.Trim(); + Product = product ?? throw new ArgumentNullException(nameof(product)); + Status = status; + Document = document ?? throw new ArgumentNullException(nameof(document)); + FirstSeen = firstSeen; + LastSeen = lastSeen; + Justification = justification; + Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); + Confidence = confidence; + Signals = signals; + AdditionalMetadata = NormalizeMetadata(additionalMetadata); + } + + public string VulnerabilityId { get; } + + public string ProviderId { get; } + + public VexProduct Product { get; } + + public VexClaimStatus Status { get; } + + public VexJustification? Justification { get; } + + public string? Detail { get; } + + public VexClaimDocument Document { get; } + + public DateTimeOffset FirstSeen { get; } + + public DateTimeOffset LastSeen { get; } + + public VexConfidence? Confidence { get; } + + public VexSignalSnapshot? Signals { get; } + + public ImmutableSortedDictionary AdditionalMetadata { get; } + + private static ImmutableSortedDictionary NormalizeMetadata( + ImmutableDictionary? additionalMetadata) + { + if (additionalMetadata is null || additionalMetadata.Count == 0) + { + return ImmutableSortedDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in additionalMetadata) + { + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + builder[key.Trim()] = value?.Trim() ?? string.Empty; + } + + return builder.ToImmutable(); + } +} + +public sealed record VexProduct +{ + public VexProduct( + string key, + string? name, + string? version = null, + string? purl = null, + string? cpe = null, + IEnumerable? componentIdentifiers = null) + { + if (string.IsNullOrWhiteSpace(key)) + { + throw new ArgumentException("Product key must be provided.", nameof(key)); + } + + Key = key.Trim(); + Name = string.IsNullOrWhiteSpace(name) ? null : name.Trim(); + Version = string.IsNullOrWhiteSpace(version) ? null : version.Trim(); + Purl = string.IsNullOrWhiteSpace(purl) ? null : purl.Trim(); + Cpe = string.IsNullOrWhiteSpace(cpe) ? null : cpe.Trim(); + ComponentIdentifiers = NormalizeComponentIdentifiers(componentIdentifiers); + } + + public string Key { get; } + + public string? Name { get; } + + public string? Version { get; } + + public string? Purl { get; } + + public string? Cpe { get; } + + public ImmutableArray ComponentIdentifiers { get; } + + private static ImmutableArray NormalizeComponentIdentifiers(IEnumerable? identifiers) + { + if (identifiers is null) + { + return ImmutableArray.Empty; + } + + var set = new SortedSet(StringComparer.Ordinal); + foreach (var identifier in identifiers) + { + if (string.IsNullOrWhiteSpace(identifier)) + { + continue; + } + + set.Add(identifier.Trim()); + } + + return set.Count == 0 ? ImmutableArray.Empty : set.ToImmutableArray(); + } +} + +public sealed record VexClaimDocument +{ + public VexClaimDocument( + VexDocumentFormat format, + string digest, + Uri sourceUri, + string? revision = null, + VexSignatureMetadata? signature = null) + { + if (string.IsNullOrWhiteSpace(digest)) + { + throw new ArgumentException("Document digest must be provided.", nameof(digest)); + } + + Format = format; + Digest = digest.Trim(); + SourceUri = sourceUri ?? throw new ArgumentNullException(nameof(sourceUri)); + Revision = string.IsNullOrWhiteSpace(revision) ? null : revision.Trim(); + Signature = signature; + } + + public VexDocumentFormat Format { get; } + + public string Digest { get; } + + public Uri SourceUri { get; } + + public string? Revision { get; } + + public VexSignatureMetadata? Signature { get; } +} + +public sealed record VexSignatureMetadata +{ + public VexSignatureMetadata( + string type, + string? subject = null, + string? issuer = null, + string? keyId = null, + DateTimeOffset? verifiedAt = null, + string? transparencyLogReference = null) + { + if (string.IsNullOrWhiteSpace(type)) + { + throw new ArgumentException("Signature type must be provided.", nameof(type)); + } + + Type = type.Trim(); + Subject = string.IsNullOrWhiteSpace(subject) ? null : subject.Trim(); + Issuer = string.IsNullOrWhiteSpace(issuer) ? null : issuer.Trim(); + KeyId = string.IsNullOrWhiteSpace(keyId) ? null : keyId.Trim(); + VerifiedAt = verifiedAt; + TransparencyLogReference = string.IsNullOrWhiteSpace(transparencyLogReference) + ? null + : transparencyLogReference.Trim(); + } + + public string Type { get; } + + public string? Subject { get; } + + public string? Issuer { get; } + + public string? KeyId { get; } + + public DateTimeOffset? VerifiedAt { get; } + + public string? TransparencyLogReference { get; } +} + +public sealed record VexConfidence +{ + public VexConfidence(string level, double? score = null, string? method = null) + { + if (string.IsNullOrWhiteSpace(level)) + { + throw new ArgumentException("Confidence level must be provided.", nameof(level)); + } + + if (score is not null && (double.IsNaN(score.Value) || double.IsInfinity(score.Value))) + { + throw new ArgumentOutOfRangeException(nameof(score), "Confidence score must be a finite number."); + } + + Level = level.Trim(); + Score = score; + Method = string.IsNullOrWhiteSpace(method) ? null : method.Trim(); + } + + public string Level { get; } + + public double? Score { get; } + + public string? Method { get; } +} + +[DataContract] +public enum VexDocumentFormat +{ + [EnumMember(Value = "csaf")] + Csaf, + + [EnumMember(Value = "cyclonedx")] + CycloneDx, + + [EnumMember(Value = "openvex")] + OpenVex, + + [EnumMember(Value = "oci_attestation")] + OciAttestation, +} + +[DataContract] +public enum VexClaimStatus +{ + [EnumMember(Value = "affected")] + Affected, + + [EnumMember(Value = "not_affected")] + NotAffected, + + [EnumMember(Value = "fixed")] + Fixed, + + [EnumMember(Value = "under_investigation")] + UnderInvestigation, +} + +[DataContract] +public enum VexJustification +{ + [EnumMember(Value = "component_not_present")] + ComponentNotPresent, + + [EnumMember(Value = "component_not_configured")] + ComponentNotConfigured, + + [EnumMember(Value = "vulnerable_code_not_present")] + VulnerableCodeNotPresent, + + [EnumMember(Value = "vulnerable_code_not_in_execute_path")] + VulnerableCodeNotInExecutePath, + + [EnumMember(Value = "vulnerable_code_cannot_be_controlled_by_adversary")] + VulnerableCodeCannotBeControlledByAdversary, + + [EnumMember(Value = "inline_mitigations_already_exist")] + InlineMitigationsAlreadyExist, + + [EnumMember(Value = "protected_by_mitigating_control")] + ProtectedByMitigatingControl, + + [EnumMember(Value = "code_not_present")] + CodeNotPresent, + + [EnumMember(Value = "code_not_reachable")] + CodeNotReachable, + + [EnumMember(Value = "requires_configuration")] + RequiresConfiguration, + + [EnumMember(Value = "requires_dependency")] + RequiresDependency, + + [EnumMember(Value = "requires_environment")] + RequiresEnvironment, + + [EnumMember(Value = "protected_by_compensating_control")] + ProtectedByCompensatingControl, + + [EnumMember(Value = "protected_at_perimeter")] + ProtectedAtPerimeter, + + [EnumMember(Value = "protected_at_runtime")] + ProtectedAtRuntime, +} diff --git a/src/StellaOps.Vexer.Core/VexConnectorAbstractions.cs b/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs similarity index 95% rename from src/StellaOps.Vexer.Core/VexConnectorAbstractions.cs rename to src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs index 0f247aff..5a54673d 100644 --- a/src/StellaOps.Vexer.Core/VexConnectorAbstractions.cs +++ b/src/StellaOps.Excititor.Core/VexConnectorAbstractions.cs @@ -1,88 +1,88 @@ -using System; -using System.Collections.Immutable; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Vexer.Core; - -/// -/// Shared connector contract for fetching and normalizing provider-specific VEX data. -/// -public interface IVexConnector -{ - string Id { get; } - - VexProviderKind Kind { get; } - - ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken); - - IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken); - - ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); -} - -/// -/// Connector context populated by the orchestrator/worker. -/// -public sealed record VexConnectorContext( - DateTimeOffset? Since, - VexConnectorSettings Settings, - IVexRawDocumentSink RawSink, - IVexSignatureVerifier SignatureVerifier, - IVexNormalizerRouter Normalizers, - IServiceProvider Services); - -/// -/// Normalized connector configuration values. -/// -public sealed record VexConnectorSettings(ImmutableDictionary Values) -{ - public static VexConnectorSettings Empty { get; } = new(ImmutableDictionary.Empty); -} - -/// -/// Raw document retrieved from a connector pull. -/// -public sealed record VexRawDocument( - string ProviderId, - VexDocumentFormat Format, - Uri SourceUri, - DateTimeOffset RetrievedAt, - string Digest, - ReadOnlyMemory Content, - ImmutableDictionary Metadata) -{ - public Guid DocumentId { get; init; } = Guid.NewGuid(); -} - -/// -/// Batch of normalized claims derived from a raw document. -/// -public sealed record VexClaimBatch( - VexRawDocument Source, - ImmutableArray Claims, - ImmutableDictionary Diagnostics); - -/// -/// Sink abstraction allowing connectors to stream raw documents for persistence. -/// -public interface IVexRawDocumentSink -{ - ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken); -} - -/// -/// Signature/attestation verification service used while ingesting documents. -/// -public interface IVexSignatureVerifier -{ - ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken); -} - -/// -/// Normalizer router providing format-specific normalization helpers. -/// -public interface IVexNormalizerRouter -{ - ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); -} +using System; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Core; + +/// +/// Shared connector contract for fetching and normalizing provider-specific VEX data. +/// +public interface IVexConnector +{ + string Id { get; } + + VexProviderKind Kind { get; } + + ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken); + + IAsyncEnumerable FetchAsync(VexConnectorContext context, CancellationToken cancellationToken); + + ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); +} + +/// +/// Connector context populated by the orchestrator/worker. +/// +public sealed record VexConnectorContext( + DateTimeOffset? Since, + VexConnectorSettings Settings, + IVexRawDocumentSink RawSink, + IVexSignatureVerifier SignatureVerifier, + IVexNormalizerRouter Normalizers, + IServiceProvider Services); + +/// +/// Normalized connector configuration values. +/// +public sealed record VexConnectorSettings(ImmutableDictionary Values) +{ + public static VexConnectorSettings Empty { get; } = new(ImmutableDictionary.Empty); +} + +/// +/// Raw document retrieved from a connector pull. +/// +public sealed record VexRawDocument( + string ProviderId, + VexDocumentFormat Format, + Uri SourceUri, + DateTimeOffset RetrievedAt, + string Digest, + ReadOnlyMemory Content, + ImmutableDictionary Metadata) +{ + public Guid DocumentId { get; init; } = Guid.NewGuid(); +} + +/// +/// Batch of normalized claims derived from a raw document. +/// +public sealed record VexClaimBatch( + VexRawDocument Source, + ImmutableArray Claims, + ImmutableDictionary Diagnostics); + +/// +/// Sink abstraction allowing connectors to stream raw documents for persistence. +/// +public interface IVexRawDocumentSink +{ + ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken); +} + +/// +/// Signature/attestation verification service used while ingesting documents. +/// +public interface IVexSignatureVerifier +{ + ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken); +} + +/// +/// Normalizer router providing format-specific normalization helpers. +/// +public interface IVexNormalizerRouter +{ + ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Core/VexConsensus.cs b/src/StellaOps.Excititor.Core/VexConsensus.cs similarity index 94% rename from src/StellaOps.Vexer.Core/VexConsensus.cs rename to src/StellaOps.Excititor.Core/VexConsensus.cs index a6217ed1..12f342e8 100644 --- a/src/StellaOps.Vexer.Core/VexConsensus.cs +++ b/src/StellaOps.Excititor.Core/VexConsensus.cs @@ -1,202 +1,206 @@ -using System.Collections.Immutable; -using System.Runtime.Serialization; - -namespace StellaOps.Vexer.Core; - -public sealed record VexConsensus -{ - public VexConsensus( - string vulnerabilityId, - VexProduct product, - VexConsensusStatus status, - DateTimeOffset calculatedAt, - IEnumerable sources, - IEnumerable? conflicts = null, - string? policyVersion = null, - string? summary = null, - string? policyRevisionId = null, - string? policyDigest = null) - { - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - throw new ArgumentException("Vulnerability id must be provided.", nameof(vulnerabilityId)); - } - - VulnerabilityId = vulnerabilityId.Trim(); - Product = product ?? throw new ArgumentNullException(nameof(product)); - Status = status; - CalculatedAt = calculatedAt; - Sources = NormalizeSources(sources); - Conflicts = NormalizeConflicts(conflicts); - PolicyVersion = string.IsNullOrWhiteSpace(policyVersion) ? null : policyVersion.Trim(); - Summary = string.IsNullOrWhiteSpace(summary) ? null : summary.Trim(); - PolicyRevisionId = string.IsNullOrWhiteSpace(policyRevisionId) ? null : policyRevisionId.Trim(); - PolicyDigest = string.IsNullOrWhiteSpace(policyDigest) ? null : policyDigest.Trim(); - } - - public string VulnerabilityId { get; } - - public VexProduct Product { get; } - - public VexConsensusStatus Status { get; } - - public DateTimeOffset CalculatedAt { get; } - - public ImmutableArray Sources { get; } - - public ImmutableArray Conflicts { get; } - - public string? PolicyVersion { get; } - - public string? Summary { get; } - - public string? PolicyRevisionId { get; } - - public string? PolicyDigest { get; } - - private static ImmutableArray NormalizeSources(IEnumerable sources) - { - if (sources is null) - { - throw new ArgumentNullException(nameof(sources)); - } - - var builder = ImmutableArray.CreateBuilder(); - builder.AddRange(sources); - if (builder.Count == 0) - { - return ImmutableArray.Empty; - } - - return builder - .OrderBy(static x => x.ProviderId, StringComparer.Ordinal) - .ThenBy(static x => x.DocumentDigest, StringComparer.Ordinal) - .ToImmutableArray(); - } - - private static ImmutableArray NormalizeConflicts(IEnumerable? conflicts) - { - if (conflicts is null) - { - return ImmutableArray.Empty; - } - - var items = conflicts.ToArray(); - return items.Length == 0 - ? ImmutableArray.Empty - : items - .OrderBy(static x => x.ProviderId, StringComparer.Ordinal) - .ThenBy(static x => x.DocumentDigest, StringComparer.Ordinal) - .ToImmutableArray(); - } -} - -public sealed record VexConsensusSource -{ - public VexConsensusSource( - string providerId, - VexClaimStatus status, - string documentDigest, - double weight, - VexJustification? justification = null, - string? detail = null, - VexConfidence? confidence = null) - { - if (string.IsNullOrWhiteSpace(providerId)) - { - throw new ArgumentException("Provider id must be provided.", nameof(providerId)); - } - - if (string.IsNullOrWhiteSpace(documentDigest)) - { - throw new ArgumentException("Document digest must be provided.", nameof(documentDigest)); - } - - if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0) - { - throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite, non-negative number."); - } - - ProviderId = providerId.Trim(); - Status = status; - DocumentDigest = documentDigest.Trim(); - Weight = weight; - Justification = justification; - Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); - Confidence = confidence; - } - - public string ProviderId { get; } - - public VexClaimStatus Status { get; } - - public string DocumentDigest { get; } - - public double Weight { get; } - - public VexJustification? Justification { get; } - - public string? Detail { get; } - - public VexConfidence? Confidence { get; } -} - -public sealed record VexConsensusConflict -{ - public VexConsensusConflict( - string providerId, - VexClaimStatus status, - string documentDigest, - VexJustification? justification = null, - string? detail = null, - string? reason = null) - { - if (string.IsNullOrWhiteSpace(providerId)) - { - throw new ArgumentException("Provider id must be provided.", nameof(providerId)); - } - - if (string.IsNullOrWhiteSpace(documentDigest)) - { - throw new ArgumentException("Document digest must be provided.", nameof(documentDigest)); - } - - ProviderId = providerId.Trim(); - Status = status; - DocumentDigest = documentDigest.Trim(); - Justification = justification; - Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); - Reason = string.IsNullOrWhiteSpace(reason) ? null : reason.Trim(); - } - - public string ProviderId { get; } - - public VexClaimStatus Status { get; } - - public string DocumentDigest { get; } - - public VexJustification? Justification { get; } - - public string? Detail { get; } - - public string? Reason { get; } -} - -[DataContract] -public enum VexConsensusStatus -{ - [EnumMember(Value = "affected")] - Affected, - - [EnumMember(Value = "not_affected")] - NotAffected, - - [EnumMember(Value = "fixed")] - Fixed, - - [EnumMember(Value = "under_investigation")] - UnderInvestigation, - - [EnumMember(Value = "divergent")] - Divergent, -} +using System.Collections.Immutable; +using System.Runtime.Serialization; + +namespace StellaOps.Excititor.Core; + +public sealed record VexConsensus +{ + public VexConsensus( + string vulnerabilityId, + VexProduct product, + VexConsensusStatus status, + DateTimeOffset calculatedAt, + IEnumerable sources, + IEnumerable? conflicts = null, + VexSignalSnapshot? signals = null, + string? policyVersion = null, + string? summary = null, + string? policyRevisionId = null, + string? policyDigest = null) + { + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + throw new ArgumentException("Vulnerability id must be provided.", nameof(vulnerabilityId)); + } + + VulnerabilityId = vulnerabilityId.Trim(); + Product = product ?? throw new ArgumentNullException(nameof(product)); + Status = status; + CalculatedAt = calculatedAt; + Sources = NormalizeSources(sources); + Conflicts = NormalizeConflicts(conflicts); + Signals = signals; + PolicyVersion = string.IsNullOrWhiteSpace(policyVersion) ? null : policyVersion.Trim(); + Summary = string.IsNullOrWhiteSpace(summary) ? null : summary.Trim(); + PolicyRevisionId = string.IsNullOrWhiteSpace(policyRevisionId) ? null : policyRevisionId.Trim(); + PolicyDigest = string.IsNullOrWhiteSpace(policyDigest) ? null : policyDigest.Trim(); + } + + public string VulnerabilityId { get; } + + public VexProduct Product { get; } + + public VexConsensusStatus Status { get; } + + public DateTimeOffset CalculatedAt { get; } + + public ImmutableArray Sources { get; } + + public ImmutableArray Conflicts { get; } + + public VexSignalSnapshot? Signals { get; } + + public string? PolicyVersion { get; } + + public string? Summary { get; } + + public string? PolicyRevisionId { get; } + + public string? PolicyDigest { get; } + + private static ImmutableArray NormalizeSources(IEnumerable sources) + { + if (sources is null) + { + throw new ArgumentNullException(nameof(sources)); + } + + var builder = ImmutableArray.CreateBuilder(); + builder.AddRange(sources); + if (builder.Count == 0) + { + return ImmutableArray.Empty; + } + + return builder + .OrderBy(static x => x.ProviderId, StringComparer.Ordinal) + .ThenBy(static x => x.DocumentDigest, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeConflicts(IEnumerable? conflicts) + { + if (conflicts is null) + { + return ImmutableArray.Empty; + } + + var items = conflicts.ToArray(); + return items.Length == 0 + ? ImmutableArray.Empty + : items + .OrderBy(static x => x.ProviderId, StringComparer.Ordinal) + .ThenBy(static x => x.DocumentDigest, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +public sealed record VexConsensusSource +{ + public VexConsensusSource( + string providerId, + VexClaimStatus status, + string documentDigest, + double weight, + VexJustification? justification = null, + string? detail = null, + VexConfidence? confidence = null) + { + if (string.IsNullOrWhiteSpace(providerId)) + { + throw new ArgumentException("Provider id must be provided.", nameof(providerId)); + } + + if (string.IsNullOrWhiteSpace(documentDigest)) + { + throw new ArgumentException("Document digest must be provided.", nameof(documentDigest)); + } + + if (double.IsNaN(weight) || double.IsInfinity(weight) || weight < 0) + { + throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite, non-negative number."); + } + + ProviderId = providerId.Trim(); + Status = status; + DocumentDigest = documentDigest.Trim(); + Weight = weight; + Justification = justification; + Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); + Confidence = confidence; + } + + public string ProviderId { get; } + + public VexClaimStatus Status { get; } + + public string DocumentDigest { get; } + + public double Weight { get; } + + public VexJustification? Justification { get; } + + public string? Detail { get; } + + public VexConfidence? Confidence { get; } +} + +public sealed record VexConsensusConflict +{ + public VexConsensusConflict( + string providerId, + VexClaimStatus status, + string documentDigest, + VexJustification? justification = null, + string? detail = null, + string? reason = null) + { + if (string.IsNullOrWhiteSpace(providerId)) + { + throw new ArgumentException("Provider id must be provided.", nameof(providerId)); + } + + if (string.IsNullOrWhiteSpace(documentDigest)) + { + throw new ArgumentException("Document digest must be provided.", nameof(documentDigest)); + } + + ProviderId = providerId.Trim(); + Status = status; + DocumentDigest = documentDigest.Trim(); + Justification = justification; + Detail = string.IsNullOrWhiteSpace(detail) ? null : detail.Trim(); + Reason = string.IsNullOrWhiteSpace(reason) ? null : reason.Trim(); + } + + public string ProviderId { get; } + + public VexClaimStatus Status { get; } + + public string DocumentDigest { get; } + + public VexJustification? Justification { get; } + + public string? Detail { get; } + + public string? Reason { get; } +} + +[DataContract] +public enum VexConsensusStatus +{ + [EnumMember(Value = "affected")] + Affected, + + [EnumMember(Value = "not_affected")] + NotAffected, + + [EnumMember(Value = "fixed")] + Fixed, + + [EnumMember(Value = "under_investigation")] + UnderInvestigation, + + [EnumMember(Value = "divergent")] + Divergent, +} diff --git a/src/StellaOps.Excititor.Core/VexConsensusPolicyOptions.cs b/src/StellaOps.Excititor.Core/VexConsensusPolicyOptions.cs new file mode 100644 index 00000000..abc3ef41 --- /dev/null +++ b/src/StellaOps.Excititor.Core/VexConsensusPolicyOptions.cs @@ -0,0 +1,146 @@ +using System.Collections.Immutable; + +namespace StellaOps.Excititor.Core; + +public sealed record VexConsensusPolicyOptions +{ + public const string BaselineVersion = "baseline/v1"; + + public const double DefaultWeightCeiling = 1.0; + public const double DefaultAlpha = 0.25; + public const double DefaultBeta = 0.5; + public const double MaxSupportedCeiling = 5.0; + public const double MaxSupportedCoefficient = 5.0; + + public VexConsensusPolicyOptions( + string? version = null, + double vendorWeight = 1.0, + double distroWeight = 0.9, + double platformWeight = 0.7, + double hubWeight = 0.5, + double attestationWeight = 0.6, + IEnumerable>? providerOverrides = null, + double weightCeiling = DefaultWeightCeiling, + double alpha = DefaultAlpha, + double beta = DefaultBeta) + { + Version = string.IsNullOrWhiteSpace(version) ? BaselineVersion : version.Trim(); + WeightCeiling = NormalizeWeightCeiling(weightCeiling); + VendorWeight = NormalizeWeight(vendorWeight, WeightCeiling); + DistroWeight = NormalizeWeight(distroWeight, WeightCeiling); + PlatformWeight = NormalizeWeight(platformWeight, WeightCeiling); + HubWeight = NormalizeWeight(hubWeight, WeightCeiling); + AttestationWeight = NormalizeWeight(attestationWeight, WeightCeiling); + ProviderOverrides = NormalizeOverrides(providerOverrides, WeightCeiling); + Alpha = NormalizeCoefficient(alpha, nameof(alpha)); + Beta = NormalizeCoefficient(beta, nameof(beta)); + } + + public string Version { get; } + + public double VendorWeight { get; } + + public double DistroWeight { get; } + + public double PlatformWeight { get; } + + public double HubWeight { get; } + + public double AttestationWeight { get; } + + public double WeightCeiling { get; } + + public double Alpha { get; } + + public double Beta { get; } + + public ImmutableDictionary ProviderOverrides { get; } + + private static double NormalizeWeight(double weight, double ceiling) + { + if (double.IsNaN(weight) || double.IsInfinity(weight)) + { + throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite number."); + } + + if (weight <= 0) + { + return 0; + } + + if (weight >= ceiling) + { + return ceiling; + } + + return weight; + } + + private static ImmutableDictionary NormalizeOverrides( + IEnumerable>? overrides, + double ceiling) + { + if (overrides is null) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, weight) in overrides) + { + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + builder[key.Trim()] = NormalizeWeight(weight, ceiling); + } + + return builder.ToImmutable(); + } + + private static double NormalizeWeightCeiling(double value) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + { + throw new ArgumentOutOfRangeException(nameof(value), "Weight ceiling must be a finite number."); + } + + if (value <= 0) + { + throw new ArgumentOutOfRangeException(nameof(value), "Weight ceiling must be greater than zero."); + } + + if (value < 1) + { + return 1; + } + + if (value > MaxSupportedCeiling) + { + return MaxSupportedCeiling; + } + + return value; + } + + private static double NormalizeCoefficient(double value, string name) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + { + throw new ArgumentOutOfRangeException(name, "Coefficient must be a finite number."); + } + + if (value < 0) + { + throw new ArgumentOutOfRangeException(name, "Coefficient must be non-negative."); + } + + if (value > MaxSupportedCoefficient) + { + return MaxSupportedCoefficient; + } + + return value; + } +} diff --git a/src/StellaOps.Vexer.Core/VexConsensusResolver.cs b/src/StellaOps.Excititor.Core/VexConsensusResolver.cs similarity index 86% rename from src/StellaOps.Vexer.Core/VexConsensusResolver.cs rename to src/StellaOps.Excititor.Core/VexConsensusResolver.cs index 71e8f7ca..6f81233d 100644 --- a/src/StellaOps.Vexer.Core/VexConsensusResolver.cs +++ b/src/StellaOps.Excititor.Core/VexConsensusResolver.cs @@ -1,293 +1,299 @@ -using System.Collections.Immutable; -using System.Globalization; - -namespace StellaOps.Vexer.Core; - -public sealed class VexConsensusResolver -{ - private readonly IVexConsensusPolicy _policy; - - public VexConsensusResolver(IVexConsensusPolicy policy) - { - _policy = policy ?? throw new ArgumentNullException(nameof(policy)); - } - - public VexConsensusResolution Resolve(VexConsensusRequest request) - { - if (request is null) - { - throw new ArgumentNullException(nameof(request)); - } - - var orderedClaims = request.Claims - .OrderBy(static claim => claim.ProviderId, StringComparer.Ordinal) - .ThenBy(static claim => claim.Document.Digest, StringComparer.Ordinal) - .ThenBy(static claim => claim.Document.SourceUri.ToString(), StringComparer.Ordinal) - .ToArray(); - - var decisions = ImmutableArray.CreateBuilder(orderedClaims.Length); - var acceptedSources = new List(orderedClaims.Length); - var conflicts = new List(); - var conflictKeys = new HashSet(StringComparer.Ordinal); - var weightByStatus = new Dictionary(); - - foreach (var claim in orderedClaims) - { - request.Providers.TryGetValue(claim.ProviderId, out var provider); - - string? rejectionReason = null; - double weight = 0; - var included = false; - - if (provider is null) - { - rejectionReason = "provider_not_registered"; - } - else - { - weight = NormalizeWeight(_policy.GetProviderWeight(provider)); - if (weight <= 0) - { - rejectionReason = "weight_not_positive"; - } - else if (!_policy.IsClaimEligible(claim, provider, out rejectionReason)) - { - rejectionReason ??= "rejected_by_policy"; - } - else - { - included = true; - TrackStatusWeight(weightByStatus, claim.Status, weight); - acceptedSources.Add(new VexConsensusSource( - claim.ProviderId, - claim.Status, - claim.Document.Digest, - weight, - claim.Justification, - claim.Detail, - claim.Confidence)); - } - } - - if (!included) - { - var conflict = new VexConsensusConflict( - claim.ProviderId, - claim.Status, - claim.Document.Digest, - claim.Justification, - claim.Detail, - rejectionReason); - if (conflictKeys.Add(CreateConflictKey(conflict.ProviderId, conflict.DocumentDigest))) - { - conflicts.Add(conflict); - } - } - - decisions.Add(new VexConsensusDecisionTelemetry( - claim.ProviderId, - claim.Document.Digest, - claim.Status, - included, - weight, - rejectionReason, - claim.Justification, - claim.Detail)); - } - - var consensusStatus = DetermineConsensusStatus(weightByStatus); - var summary = BuildSummary(weightByStatus, consensusStatus); - - var consensus = new VexConsensus( - request.VulnerabilityId, - request.Product, - consensusStatus, - request.CalculatedAt, - acceptedSources, - AttachConflictDetails(conflicts, acceptedSources, consensusStatus, conflictKeys), - _policy.Version, - summary, - request.PolicyRevisionId, - request.PolicyDigest); - - return new VexConsensusResolution(consensus, decisions.ToImmutable()); - } - - private static Dictionary TrackStatusWeight( - Dictionary accumulator, - VexClaimStatus status, - double weight) - { - if (accumulator.TryGetValue(status, out var current)) - { - accumulator[status] = current + weight; - } - else - { - accumulator[status] = weight; - } - - return accumulator; - } - - private static double NormalizeWeight(double weight) - { - if (double.IsNaN(weight) || double.IsInfinity(weight) || weight <= 0) - { - return 0; - } - - if (weight >= 1) - { - return 1; - } - - return weight; - } - - private static VexConsensusStatus DetermineConsensusStatus( - IReadOnlyDictionary weights) - { - if (weights.Count == 0) - { - return VexConsensusStatus.UnderInvestigation; - } - - var ordered = weights - .OrderByDescending(static pair => pair.Value) - .ThenBy(static pair => pair.Key) - .ToArray(); - - var topStatus = ordered[0].Key; - var topWeight = ordered[0].Value; - var totalWeight = ordered.Sum(static pair => pair.Value); - var remainder = totalWeight - topWeight; - - if (topWeight <= 0) - { - return VexConsensusStatus.UnderInvestigation; - } - - if (topWeight > remainder) - { - return topStatus switch - { - VexClaimStatus.Affected => VexConsensusStatus.Affected, - VexClaimStatus.Fixed => VexConsensusStatus.Fixed, - VexClaimStatus.NotAffected => VexConsensusStatus.NotAffected, - _ => VexConsensusStatus.UnderInvestigation, - }; - } - - return VexConsensusStatus.UnderInvestigation; - } - - private static string BuildSummary( - IReadOnlyDictionary weights, - VexConsensusStatus status) - { - if (weights.Count == 0) - { - return "No eligible claims met policy requirements."; - } - - var breakdown = string.Join( - ", ", - weights - .OrderByDescending(static pair => pair.Value) - .ThenBy(static pair => pair.Key) - .Select(pair => $"{FormatStatus(pair.Key)}={pair.Value.ToString("0.###", CultureInfo.InvariantCulture)}")); - - if (status == VexConsensusStatus.UnderInvestigation) - { - return $"No majority consensus; weighted breakdown {breakdown}."; - } - - return $"{FormatStatus(status)} determined via weighted majority; breakdown {breakdown}."; - } - - private static List AttachConflictDetails( - List conflicts, - IEnumerable acceptedSources, - VexConsensusStatus status, - HashSet conflictKeys) - { - var consensusClaimStatus = status switch - { - VexConsensusStatus.Affected => VexClaimStatus.Affected, - VexConsensusStatus.NotAffected => VexClaimStatus.NotAffected, - VexConsensusStatus.Fixed => VexClaimStatus.Fixed, - VexConsensusStatus.UnderInvestigation => (VexClaimStatus?)null, - VexConsensusStatus.Divergent => (VexClaimStatus?)null, - _ => null, - }; - - foreach (var source in acceptedSources) - { - if (consensusClaimStatus is null || source.Status != consensusClaimStatus.Value) - { - var conflict = new VexConsensusConflict( - source.ProviderId, - source.Status, - source.DocumentDigest, - source.Justification, - source.Detail, - consensusClaimStatus is null ? "no_majority" : "status_conflict"); - - if (conflictKeys.Add(CreateConflictKey(conflict.ProviderId, conflict.DocumentDigest))) - { - conflicts.Add(conflict); - } - } - } - - return conflicts; - } - - private static string FormatStatus(VexClaimStatus status) - => status switch - { - VexClaimStatus.Affected => "affected", - VexClaimStatus.NotAffected => "not_affected", - VexClaimStatus.Fixed => "fixed", - VexClaimStatus.UnderInvestigation => "under_investigation", - _ => status.ToString().ToLowerInvariant(), - }; - - private static string CreateConflictKey(string providerId, string documentDigest) - => $"{providerId}|{documentDigest}"; - - private static string FormatStatus(VexConsensusStatus status) - => status switch - { - VexConsensusStatus.Affected => "affected", - VexConsensusStatus.NotAffected => "not_affected", - VexConsensusStatus.Fixed => "fixed", - VexConsensusStatus.UnderInvestigation => "under_investigation", - VexConsensusStatus.Divergent => "divergent", - _ => status.ToString().ToLowerInvariant(), - }; -} - -public sealed record VexConsensusRequest( - string VulnerabilityId, - VexProduct Product, - IReadOnlyList Claims, - IReadOnlyDictionary Providers, - DateTimeOffset CalculatedAt, - string? PolicyRevisionId = null, - string? PolicyDigest = null); - -public sealed record VexConsensusResolution( - VexConsensus Consensus, - ImmutableArray DecisionLog); - -public sealed record VexConsensusDecisionTelemetry( - string ProviderId, - string DocumentDigest, - VexClaimStatus Status, - bool Included, - double Weight, - string? Reason, - VexJustification? Justification, - string? Detail); +using System.Collections.Immutable; +using System.Globalization; + +namespace StellaOps.Excititor.Core; + +public sealed class VexConsensusResolver +{ + private readonly IVexConsensusPolicy _policy; + + public VexConsensusResolver(IVexConsensusPolicy policy) + { + _policy = policy ?? throw new ArgumentNullException(nameof(policy)); + } + + public VexConsensusResolution Resolve(VexConsensusRequest request) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + var orderedClaims = request.Claims + .OrderBy(static claim => claim.ProviderId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Document.Digest, StringComparer.Ordinal) + .ThenBy(static claim => claim.Document.SourceUri.ToString(), StringComparer.Ordinal) + .ToArray(); + + var decisions = ImmutableArray.CreateBuilder(orderedClaims.Length); + var acceptedSources = new List(orderedClaims.Length); + var conflicts = new List(); + var conflictKeys = new HashSet(StringComparer.Ordinal); + var weightByStatus = new Dictionary(); + + foreach (var claim in orderedClaims) + { + request.Providers.TryGetValue(claim.ProviderId, out var provider); + + string? rejectionReason = null; + double weight = 0; + var included = false; + + if (provider is null) + { + rejectionReason = "provider_not_registered"; + } + else + { + var ceiling = request.WeightCeiling <= 0 || double.IsNaN(request.WeightCeiling) || double.IsInfinity(request.WeightCeiling) + ? VexConsensusPolicyOptions.DefaultWeightCeiling + : Math.Clamp(request.WeightCeiling, 0.1, VexConsensusPolicyOptions.MaxSupportedCeiling); + weight = NormalizeWeight(_policy.GetProviderWeight(provider), ceiling); + if (weight <= 0) + { + rejectionReason = "weight_not_positive"; + } + else if (!_policy.IsClaimEligible(claim, provider, out rejectionReason)) + { + rejectionReason ??= "rejected_by_policy"; + } + else + { + included = true; + TrackStatusWeight(weightByStatus, claim.Status, weight); + acceptedSources.Add(new VexConsensusSource( + claim.ProviderId, + claim.Status, + claim.Document.Digest, + weight, + claim.Justification, + claim.Detail, + claim.Confidence)); + } + } + + if (!included) + { + var conflict = new VexConsensusConflict( + claim.ProviderId, + claim.Status, + claim.Document.Digest, + claim.Justification, + claim.Detail, + rejectionReason); + if (conflictKeys.Add(CreateConflictKey(conflict.ProviderId, conflict.DocumentDigest))) + { + conflicts.Add(conflict); + } + } + + decisions.Add(new VexConsensusDecisionTelemetry( + claim.ProviderId, + claim.Document.Digest, + claim.Status, + included, + weight, + rejectionReason, + claim.Justification, + claim.Detail)); + } + + var consensusStatus = DetermineConsensusStatus(weightByStatus); + var summary = BuildSummary(weightByStatus, consensusStatus); + + var consensus = new VexConsensus( + request.VulnerabilityId, + request.Product, + consensusStatus, + request.CalculatedAt, + acceptedSources, + AttachConflictDetails(conflicts, acceptedSources, consensusStatus, conflictKeys), + request.Signals, + _policy.Version, + summary, + request.PolicyRevisionId, + request.PolicyDigest); + + return new VexConsensusResolution(consensus, decisions.ToImmutable()); + } + + private static Dictionary TrackStatusWeight( + Dictionary accumulator, + VexClaimStatus status, + double weight) + { + if (accumulator.TryGetValue(status, out var current)) + { + accumulator[status] = current + weight; + } + else + { + accumulator[status] = weight; + } + + return accumulator; + } + + private static double NormalizeWeight(double weight, double ceiling) + { + if (double.IsNaN(weight) || double.IsInfinity(weight) || weight <= 0) + { + return 0; + } + + if (weight >= ceiling) + { + return ceiling; + } + + return weight; + } + + private static VexConsensusStatus DetermineConsensusStatus( + IReadOnlyDictionary weights) + { + if (weights.Count == 0) + { + return VexConsensusStatus.UnderInvestigation; + } + + var ordered = weights + .OrderByDescending(static pair => pair.Value) + .ThenBy(static pair => pair.Key) + .ToArray(); + + var topStatus = ordered[0].Key; + var topWeight = ordered[0].Value; + var totalWeight = ordered.Sum(static pair => pair.Value); + var remainder = totalWeight - topWeight; + + if (topWeight <= 0) + { + return VexConsensusStatus.UnderInvestigation; + } + + if (topWeight > remainder) + { + return topStatus switch + { + VexClaimStatus.Affected => VexConsensusStatus.Affected, + VexClaimStatus.Fixed => VexConsensusStatus.Fixed, + VexClaimStatus.NotAffected => VexConsensusStatus.NotAffected, + _ => VexConsensusStatus.UnderInvestigation, + }; + } + + return VexConsensusStatus.UnderInvestigation; + } + + private static string BuildSummary( + IReadOnlyDictionary weights, + VexConsensusStatus status) + { + if (weights.Count == 0) + { + return "No eligible claims met policy requirements."; + } + + var breakdown = string.Join( + ", ", + weights + .OrderByDescending(static pair => pair.Value) + .ThenBy(static pair => pair.Key) + .Select(pair => $"{FormatStatus(pair.Key)}={pair.Value.ToString("0.###", CultureInfo.InvariantCulture)}")); + + if (status == VexConsensusStatus.UnderInvestigation) + { + return $"No majority consensus; weighted breakdown {breakdown}."; + } + + return $"{FormatStatus(status)} determined via weighted majority; breakdown {breakdown}."; + } + + private static List AttachConflictDetails( + List conflicts, + IEnumerable acceptedSources, + VexConsensusStatus status, + HashSet conflictKeys) + { + var consensusClaimStatus = status switch + { + VexConsensusStatus.Affected => VexClaimStatus.Affected, + VexConsensusStatus.NotAffected => VexClaimStatus.NotAffected, + VexConsensusStatus.Fixed => VexClaimStatus.Fixed, + VexConsensusStatus.UnderInvestigation => (VexClaimStatus?)null, + VexConsensusStatus.Divergent => (VexClaimStatus?)null, + _ => null, + }; + + foreach (var source in acceptedSources) + { + if (consensusClaimStatus is null || source.Status != consensusClaimStatus.Value) + { + var conflict = new VexConsensusConflict( + source.ProviderId, + source.Status, + source.DocumentDigest, + source.Justification, + source.Detail, + consensusClaimStatus is null ? "no_majority" : "status_conflict"); + + if (conflictKeys.Add(CreateConflictKey(conflict.ProviderId, conflict.DocumentDigest))) + { + conflicts.Add(conflict); + } + } + } + + return conflicts; + } + + private static string FormatStatus(VexClaimStatus status) + => status switch + { + VexClaimStatus.Affected => "affected", + VexClaimStatus.NotAffected => "not_affected", + VexClaimStatus.Fixed => "fixed", + VexClaimStatus.UnderInvestigation => "under_investigation", + _ => status.ToString().ToLowerInvariant(), + }; + + private static string CreateConflictKey(string providerId, string documentDigest) + => $"{providerId}|{documentDigest}"; + + private static string FormatStatus(VexConsensusStatus status) + => status switch + { + VexConsensusStatus.Affected => "affected", + VexConsensusStatus.NotAffected => "not_affected", + VexConsensusStatus.Fixed => "fixed", + VexConsensusStatus.UnderInvestigation => "under_investigation", + VexConsensusStatus.Divergent => "divergent", + _ => status.ToString().ToLowerInvariant(), + }; +} + +public sealed record VexConsensusRequest( + string VulnerabilityId, + VexProduct Product, + IReadOnlyList Claims, + IReadOnlyDictionary Providers, + DateTimeOffset CalculatedAt, + double WeightCeiling = VexConsensusPolicyOptions.DefaultWeightCeiling, + VexSignalSnapshot? Signals = null, + string? PolicyRevisionId = null, + string? PolicyDigest = null); + +public sealed record VexConsensusResolution( + VexConsensus Consensus, + ImmutableArray DecisionLog); + +public sealed record VexConsensusDecisionTelemetry( + string ProviderId, + string DocumentDigest, + VexClaimStatus Status, + bool Included, + double Weight, + string? Reason, + VexJustification? Justification, + string? Detail); diff --git a/src/StellaOps.Vexer.Core/VexExportManifest.cs b/src/StellaOps.Excititor.Core/VexExportManifest.cs similarity index 88% rename from src/StellaOps.Vexer.Core/VexExportManifest.cs rename to src/StellaOps.Excititor.Core/VexExportManifest.cs index a75fed80..120dc6cb 100644 --- a/src/StellaOps.Vexer.Core/VexExportManifest.cs +++ b/src/StellaOps.Excititor.Core/VexExportManifest.cs @@ -1,257 +1,273 @@ -using System.Collections.Immutable; -using System.Runtime.Serialization; -using System.Text; - -namespace StellaOps.Vexer.Core; - -public sealed record VexExportManifest -{ - public VexExportManifest( - string exportId, - VexQuerySignature querySignature, - VexExportFormat format, - DateTimeOffset createdAt, - VexContentAddress artifact, - int claimCount, - IEnumerable sourceProviders, - bool fromCache = false, - string? consensusRevision = null, - VexAttestationMetadata? attestation = null, - long sizeBytes = 0) - { - if (string.IsNullOrWhiteSpace(exportId)) - { - throw new ArgumentException("Export id must be provided.", nameof(exportId)); - } - - if (claimCount < 0) - { - throw new ArgumentOutOfRangeException(nameof(claimCount), "Claim count cannot be negative."); - } - - if (sizeBytes < 0) - { - throw new ArgumentOutOfRangeException(nameof(sizeBytes), "Export size cannot be negative."); - } - - ExportId = exportId.Trim(); - QuerySignature = querySignature ?? throw new ArgumentNullException(nameof(querySignature)); - Format = format; - CreatedAt = createdAt; - Artifact = artifact ?? throw new ArgumentNullException(nameof(artifact)); - ClaimCount = claimCount; - FromCache = fromCache; - SourceProviders = NormalizeProviders(sourceProviders); - ConsensusRevision = string.IsNullOrWhiteSpace(consensusRevision) ? null : consensusRevision.Trim(); - Attestation = attestation; - SizeBytes = sizeBytes; - } - - public string ExportId { get; } - - public VexQuerySignature QuerySignature { get; } - - public VexExportFormat Format { get; } - - public DateTimeOffset CreatedAt { get; } - - public VexContentAddress Artifact { get; } - - public int ClaimCount { get; } - - public bool FromCache { get; } - - public ImmutableArray SourceProviders { get; } - - public string? ConsensusRevision { get; } - - public VexAttestationMetadata? Attestation { get; } - - public long SizeBytes { get; } - - private static ImmutableArray NormalizeProviders(IEnumerable providers) - { - if (providers is null) - { - throw new ArgumentNullException(nameof(providers)); - } - - var set = new SortedSet(StringComparer.Ordinal); - foreach (var provider in providers) - { - if (string.IsNullOrWhiteSpace(provider)) - { - continue; - } - - set.Add(provider.Trim()); - } - - return set.Count == 0 - ? ImmutableArray.Empty - : set.ToImmutableArray(); - } -} - -public sealed record VexContentAddress -{ - public VexContentAddress(string algorithm, string digest) - { - if (string.IsNullOrWhiteSpace(algorithm)) - { - throw new ArgumentException("Content algorithm must be provided.", nameof(algorithm)); - } - - if (string.IsNullOrWhiteSpace(digest)) - { - throw new ArgumentException("Content digest must be provided.", nameof(digest)); - } - - Algorithm = algorithm.Trim(); - Digest = digest.Trim(); - } - - public string Algorithm { get; } - - public string Digest { get; } - - public string ToUri() => $"{Algorithm}:{Digest}"; - - public override string ToString() => ToUri(); -} - -public sealed record VexAttestationMetadata -{ - public VexAttestationMetadata( - string predicateType, - VexRekorReference? rekor = null, - string? envelopeDigest = null, - DateTimeOffset? signedAt = null) - { - if (string.IsNullOrWhiteSpace(predicateType)) - { - throw new ArgumentException("Predicate type must be provided.", nameof(predicateType)); - } - - PredicateType = predicateType.Trim(); - Rekor = rekor; - EnvelopeDigest = string.IsNullOrWhiteSpace(envelopeDigest) ? null : envelopeDigest.Trim(); - SignedAt = signedAt; - } - - public string PredicateType { get; } - - public VexRekorReference? Rekor { get; } - - public string? EnvelopeDigest { get; } - - public DateTimeOffset? SignedAt { get; } -} - -public sealed record VexRekorReference -{ - public VexRekorReference(string apiVersion, string location, string? logIndex = null, Uri? inclusionProofUri = null) - { - if (string.IsNullOrWhiteSpace(apiVersion)) - { - throw new ArgumentException("Rekor API version must be provided.", nameof(apiVersion)); - } - - if (string.IsNullOrWhiteSpace(location)) - { - throw new ArgumentException("Rekor location must be provided.", nameof(location)); - } - - ApiVersion = apiVersion.Trim(); - Location = location.Trim(); - LogIndex = string.IsNullOrWhiteSpace(logIndex) ? null : logIndex.Trim(); - InclusionProofUri = inclusionProofUri; - } - - public string ApiVersion { get; } - - public string Location { get; } - - public string? LogIndex { get; } - - public Uri? InclusionProofUri { get; } -} - -public sealed partial record VexQuerySignature -{ - public VexQuerySignature(string value) - { - if (string.IsNullOrWhiteSpace(value)) - { - throw new ArgumentException("Query signature must be provided.", nameof(value)); - } - - Value = value.Trim(); - } - - public string Value { get; } - - public static VexQuerySignature FromFilters(IEnumerable> filters) - { - if (filters is null) - { - throw new ArgumentNullException(nameof(filters)); - } - - var builder = ImmutableArray.CreateBuilder>(); - foreach (var pair in filters) - { - if (string.IsNullOrWhiteSpace(pair.Key)) - { - continue; - } - - var key = pair.Key.Trim(); - var value = pair.Value?.Trim() ?? string.Empty; - builder.Add(new KeyValuePair(key, value)); - } - - if (builder.Count == 0) - { - throw new ArgumentException("At least one filter is required to build a query signature.", nameof(filters)); - } - - var ordered = builder - .OrderBy(static pair => pair.Key, StringComparer.Ordinal) - .ThenBy(static pair => pair.Value, StringComparer.Ordinal) - .ToImmutableArray(); - - var sb = new StringBuilder(); - for (var i = 0; i < ordered.Length; i++) - { - if (i > 0) - { - sb.Append('&'); - } - - sb.Append(ordered[i].Key); - sb.Append('='); - sb.Append(ordered[i].Value); - } - - return new VexQuerySignature(sb.ToString()); - } - - public override string ToString() => Value; -} - -[DataContract] -public enum VexExportFormat -{ - [EnumMember(Value = "json")] - Json, - - [EnumMember(Value = "jsonl")] - JsonLines, - - [EnumMember(Value = "openvex")] - OpenVex, - - [EnumMember(Value = "csaf")] - Csaf, -} +using System.Collections.Immutable; +using System.Runtime.Serialization; +using System.Text; + +namespace StellaOps.Excititor.Core; + +public sealed record VexExportManifest +{ + public VexExportManifest( + string exportId, + VexQuerySignature querySignature, + VexExportFormat format, + DateTimeOffset createdAt, + VexContentAddress artifact, + int claimCount, + IEnumerable sourceProviders, + bool fromCache = false, + string? consensusRevision = null, + string? policyRevisionId = null, + string? policyDigest = null, + VexContentAddress? consensusDigest = null, + VexContentAddress? scoreDigest = null, + VexAttestationMetadata? attestation = null, + long sizeBytes = 0) + { + if (string.IsNullOrWhiteSpace(exportId)) + { + throw new ArgumentException("Export id must be provided.", nameof(exportId)); + } + + if (claimCount < 0) + { + throw new ArgumentOutOfRangeException(nameof(claimCount), "Claim count cannot be negative."); + } + + if (sizeBytes < 0) + { + throw new ArgumentOutOfRangeException(nameof(sizeBytes), "Export size cannot be negative."); + } + + ExportId = exportId.Trim(); + QuerySignature = querySignature ?? throw new ArgumentNullException(nameof(querySignature)); + Format = format; + CreatedAt = createdAt; + Artifact = artifact ?? throw new ArgumentNullException(nameof(artifact)); + ClaimCount = claimCount; + FromCache = fromCache; + SourceProviders = NormalizeProviders(sourceProviders); + ConsensusRevision = string.IsNullOrWhiteSpace(consensusRevision) ? null : consensusRevision.Trim(); + PolicyRevisionId = string.IsNullOrWhiteSpace(policyRevisionId) ? null : policyRevisionId.Trim(); + PolicyDigest = string.IsNullOrWhiteSpace(policyDigest) ? null : policyDigest.Trim(); + ConsensusDigest = consensusDigest; + ScoreDigest = scoreDigest; + Attestation = attestation; + SizeBytes = sizeBytes; + } + + public string ExportId { get; } + + public VexQuerySignature QuerySignature { get; } + + public VexExportFormat Format { get; } + + public DateTimeOffset CreatedAt { get; } + + public VexContentAddress Artifact { get; } + + public int ClaimCount { get; } + + public bool FromCache { get; } + + public ImmutableArray SourceProviders { get; } + + public string? ConsensusRevision { get; } + + public string? PolicyRevisionId { get; } + + public string? PolicyDigest { get; } + + public VexContentAddress? ConsensusDigest { get; } + + public VexContentAddress? ScoreDigest { get; } + + public VexAttestationMetadata? Attestation { get; } + + public long SizeBytes { get; } + + private static ImmutableArray NormalizeProviders(IEnumerable providers) + { + if (providers is null) + { + throw new ArgumentNullException(nameof(providers)); + } + + var set = new SortedSet(StringComparer.Ordinal); + foreach (var provider in providers) + { + if (string.IsNullOrWhiteSpace(provider)) + { + continue; + } + + set.Add(provider.Trim()); + } + + return set.Count == 0 + ? ImmutableArray.Empty + : set.ToImmutableArray(); + } +} + +public sealed record VexContentAddress +{ + public VexContentAddress(string algorithm, string digest) + { + if (string.IsNullOrWhiteSpace(algorithm)) + { + throw new ArgumentException("Content algorithm must be provided.", nameof(algorithm)); + } + + if (string.IsNullOrWhiteSpace(digest)) + { + throw new ArgumentException("Content digest must be provided.", nameof(digest)); + } + + Algorithm = algorithm.Trim(); + Digest = digest.Trim(); + } + + public string Algorithm { get; } + + public string Digest { get; } + + public string ToUri() => $"{Algorithm}:{Digest}"; + + public override string ToString() => ToUri(); +} + +public sealed record VexAttestationMetadata +{ + public VexAttestationMetadata( + string predicateType, + VexRekorReference? rekor = null, + string? envelopeDigest = null, + DateTimeOffset? signedAt = null) + { + if (string.IsNullOrWhiteSpace(predicateType)) + { + throw new ArgumentException("Predicate type must be provided.", nameof(predicateType)); + } + + PredicateType = predicateType.Trim(); + Rekor = rekor; + EnvelopeDigest = string.IsNullOrWhiteSpace(envelopeDigest) ? null : envelopeDigest.Trim(); + SignedAt = signedAt; + } + + public string PredicateType { get; } + + public VexRekorReference? Rekor { get; } + + public string? EnvelopeDigest { get; } + + public DateTimeOffset? SignedAt { get; } +} + +public sealed record VexRekorReference +{ + public VexRekorReference(string apiVersion, string location, string? logIndex = null, Uri? inclusionProofUri = null) + { + if (string.IsNullOrWhiteSpace(apiVersion)) + { + throw new ArgumentException("Rekor API version must be provided.", nameof(apiVersion)); + } + + if (string.IsNullOrWhiteSpace(location)) + { + throw new ArgumentException("Rekor location must be provided.", nameof(location)); + } + + ApiVersion = apiVersion.Trim(); + Location = location.Trim(); + LogIndex = string.IsNullOrWhiteSpace(logIndex) ? null : logIndex.Trim(); + InclusionProofUri = inclusionProofUri; + } + + public string ApiVersion { get; } + + public string Location { get; } + + public string? LogIndex { get; } + + public Uri? InclusionProofUri { get; } +} + +public sealed partial record VexQuerySignature +{ + public VexQuerySignature(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Query signature must be provided.", nameof(value)); + } + + Value = value.Trim(); + } + + public string Value { get; } + + public static VexQuerySignature FromFilters(IEnumerable> filters) + { + if (filters is null) + { + throw new ArgumentNullException(nameof(filters)); + } + + var builder = ImmutableArray.CreateBuilder>(); + foreach (var pair in filters) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + var key = pair.Key.Trim(); + var value = pair.Value?.Trim() ?? string.Empty; + builder.Add(new KeyValuePair(key, value)); + } + + if (builder.Count == 0) + { + throw new ArgumentException("At least one filter is required to build a query signature.", nameof(filters)); + } + + var ordered = builder + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ThenBy(static pair => pair.Value, StringComparer.Ordinal) + .ToImmutableArray(); + + var sb = new StringBuilder(); + for (var i = 0; i < ordered.Length; i++) + { + if (i > 0) + { + sb.Append('&'); + } + + sb.Append(ordered[i].Key); + sb.Append('='); + sb.Append(ordered[i].Value); + } + + return new VexQuerySignature(sb.ToString()); + } + + public override string ToString() => Value; +} + +[DataContract] +public enum VexExportFormat +{ + [EnumMember(Value = "json")] + Json, + + [EnumMember(Value = "jsonl")] + JsonLines, + + [EnumMember(Value = "openvex")] + OpenVex, + + [EnumMember(Value = "csaf")] + Csaf, +} diff --git a/src/StellaOps.Vexer.Core/VexExporterAbstractions.cs b/src/StellaOps.Excititor.Core/VexExporterAbstractions.cs similarity index 91% rename from src/StellaOps.Vexer.Core/VexExporterAbstractions.cs rename to src/StellaOps.Excititor.Core/VexExporterAbstractions.cs index d61df0fe..ffdceb48 100644 --- a/src/StellaOps.Vexer.Core/VexExporterAbstractions.cs +++ b/src/StellaOps.Excititor.Core/VexExporterAbstractions.cs @@ -1,30 +1,30 @@ -using System; -using System.Collections.Immutable; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Vexer.Core; - -public interface IVexExporter -{ - VexExportFormat Format { get; } - - VexContentAddress Digest(VexExportRequest request); - - ValueTask SerializeAsync( - VexExportRequest request, - Stream output, - CancellationToken cancellationToken); -} - -public sealed record VexExportRequest( - VexQuery Query, - ImmutableArray Consensus, - ImmutableArray Claims, - DateTimeOffset GeneratedAt); - -public sealed record VexExportResult( - VexContentAddress Digest, - long BytesWritten, - ImmutableDictionary Metadata); +using System; +using System.Collections.Immutable; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Core; + +public interface IVexExporter +{ + VexExportFormat Format { get; } + + VexContentAddress Digest(VexExportRequest request); + + ValueTask SerializeAsync( + VexExportRequest request, + Stream output, + CancellationToken cancellationToken); +} + +public sealed record VexExportRequest( + VexQuery Query, + ImmutableArray Consensus, + ImmutableArray Claims, + DateTimeOffset GeneratedAt); + +public sealed record VexExportResult( + VexContentAddress Digest, + long BytesWritten, + ImmutableDictionary Metadata); diff --git a/src/StellaOps.Vexer.Core/VexNormalizerAbstractions.cs b/src/StellaOps.Excititor.Core/VexNormalizerAbstractions.cs similarity index 92% rename from src/StellaOps.Vexer.Core/VexNormalizerAbstractions.cs rename to src/StellaOps.Excititor.Core/VexNormalizerAbstractions.cs index 5a1b541f..c7e880bc 100644 --- a/src/StellaOps.Vexer.Core/VexNormalizerAbstractions.cs +++ b/src/StellaOps.Excititor.Core/VexNormalizerAbstractions.cs @@ -1,28 +1,28 @@ -using System; -using System.Collections.Immutable; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace StellaOps.Vexer.Core; - -/// -/// Normalizer contract for translating raw connector documents into canonical claims. -/// -public interface IVexNormalizer -{ - string Format { get; } - - bool CanHandle(VexRawDocument document); - - ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken); -} - -/// -/// Registry that maps formats to registered normalizers. -/// -public sealed record VexNormalizerRegistry(ImmutableArray Normalizers) -{ - public IVexNormalizer? Resolve(VexRawDocument document) - => Normalizers.FirstOrDefault(normalizer => normalizer.CanHandle(document)); -} +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Core; + +/// +/// Normalizer contract for translating raw connector documents into canonical claims. +/// +public interface IVexNormalizer +{ + string Format { get; } + + bool CanHandle(VexRawDocument document); + + ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken); +} + +/// +/// Registry that maps formats to registered normalizers. +/// +public sealed record VexNormalizerRegistry(ImmutableArray Normalizers) +{ + public IVexNormalizer? Resolve(VexRawDocument document) + => Normalizers.FirstOrDefault(normalizer => normalizer.CanHandle(document)); +} diff --git a/src/StellaOps.Vexer.Core/VexProvider.cs b/src/StellaOps.Excititor.Core/VexProvider.cs similarity index 95% rename from src/StellaOps.Vexer.Core/VexProvider.cs rename to src/StellaOps.Excititor.Core/VexProvider.cs index 4a825bab..037ecf6f 100644 --- a/src/StellaOps.Vexer.Core/VexProvider.cs +++ b/src/StellaOps.Excititor.Core/VexProvider.cs @@ -1,206 +1,206 @@ -using System.Collections.Immutable; -using System.Runtime.Serialization; - -namespace StellaOps.Vexer.Core; - -/// -/// Metadata describing a VEX provider (vendor, distro, hub, platform). -/// -public sealed record VexProvider -{ - public VexProvider( - string id, - string displayName, - VexProviderKind kind, - IEnumerable? baseUris = null, - VexProviderDiscovery? discovery = null, - VexProviderTrust? trust = null, - bool enabled = true) - { - if (string.IsNullOrWhiteSpace(id)) - { - throw new ArgumentException("Provider id must be non-empty.", nameof(id)); - } - - if (string.IsNullOrWhiteSpace(displayName)) - { - throw new ArgumentException("Provider display name must be non-empty.", nameof(displayName)); - } - - Id = id.Trim(); - DisplayName = displayName.Trim(); - Kind = kind; - BaseUris = NormalizeUris(baseUris); - Discovery = discovery ?? VexProviderDiscovery.Empty; - Trust = trust ?? VexProviderTrust.Default; - Enabled = enabled; - } - - public string Id { get; } - - public string DisplayName { get; } - - public VexProviderKind Kind { get; } - - public ImmutableArray BaseUris { get; } - - public VexProviderDiscovery Discovery { get; } - - public VexProviderTrust Trust { get; } - - public bool Enabled { get; } - - private static ImmutableArray NormalizeUris(IEnumerable? baseUris) - { - if (baseUris is null) - { - return ImmutableArray.Empty; - } - - var distinct = new HashSet(StringComparer.Ordinal); - var builder = ImmutableArray.CreateBuilder(); - foreach (var uri in baseUris) - { - if (uri is null) - { - continue; - } - - var canonical = uri.ToString(); - if (distinct.Add(canonical)) - { - builder.Add(uri); - } - } - - if (builder.Count == 0) - { - return ImmutableArray.Empty; - } - - return builder - .OrderBy(static x => x.ToString(), StringComparer.Ordinal) - .ToImmutableArray(); - } -} - -public sealed record VexProviderDiscovery -{ - public static readonly VexProviderDiscovery Empty = new(null, null); - - public VexProviderDiscovery(Uri? wellKnownMetadata, Uri? rolieService) - { - WellKnownMetadata = wellKnownMetadata; - RolIeService = rolieService; - } - - public Uri? WellKnownMetadata { get; } - - public Uri? RolIeService { get; } -} - -public sealed record VexProviderTrust -{ - public static readonly VexProviderTrust Default = new(1.0, null, ImmutableArray.Empty); - - public VexProviderTrust( - double weight, - VexCosignTrust? cosign, - IEnumerable? pgpFingerprints = null) - { - Weight = NormalizeWeight(weight); - Cosign = cosign; - PgpFingerprints = NormalizeFingerprints(pgpFingerprints); - } - - public double Weight { get; } - - public VexCosignTrust? Cosign { get; } - - public ImmutableArray PgpFingerprints { get; } - - private static double NormalizeWeight(double weight) - { - if (double.IsNaN(weight) || double.IsInfinity(weight)) - { - throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite number."); - } - - if (weight <= 0) - { - return 0.0; - } - - if (weight >= 1.0) - { - return 1.0; - } - - return weight; - } - - private static ImmutableArray NormalizeFingerprints(IEnumerable? values) - { - if (values is null) - { - return ImmutableArray.Empty; - } - - var set = new SortedSet(StringComparer.Ordinal); - foreach (var value in values) - { - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - set.Add(value.Trim()); - } - - return set.Count == 0 - ? ImmutableArray.Empty - : set.ToImmutableArray(); - } -} - -public sealed record VexCosignTrust -{ - public VexCosignTrust(string issuer, string identityPattern) - { - if (string.IsNullOrWhiteSpace(issuer)) - { - throw new ArgumentException("Issuer must be provided for cosign trust metadata.", nameof(issuer)); - } - - if (string.IsNullOrWhiteSpace(identityPattern)) - { - throw new ArgumentException("Identity pattern must be provided for cosign trust metadata.", nameof(identityPattern)); - } - - Issuer = issuer.Trim(); - IdentityPattern = identityPattern.Trim(); - } - - public string Issuer { get; } - - public string IdentityPattern { get; } -} - -[DataContract] -public enum VexProviderKind -{ - [EnumMember(Value = "vendor")] - Vendor, - - [EnumMember(Value = "distro")] - Distro, - - [EnumMember(Value = "hub")] - Hub, - - [EnumMember(Value = "platform")] - Platform, - - [EnumMember(Value = "attestation")] - Attestation, -} +using System.Collections.Immutable; +using System.Runtime.Serialization; + +namespace StellaOps.Excititor.Core; + +/// +/// Metadata describing a VEX provider (vendor, distro, hub, platform). +/// +public sealed record VexProvider +{ + public VexProvider( + string id, + string displayName, + VexProviderKind kind, + IEnumerable? baseUris = null, + VexProviderDiscovery? discovery = null, + VexProviderTrust? trust = null, + bool enabled = true) + { + if (string.IsNullOrWhiteSpace(id)) + { + throw new ArgumentException("Provider id must be non-empty.", nameof(id)); + } + + if (string.IsNullOrWhiteSpace(displayName)) + { + throw new ArgumentException("Provider display name must be non-empty.", nameof(displayName)); + } + + Id = id.Trim(); + DisplayName = displayName.Trim(); + Kind = kind; + BaseUris = NormalizeUris(baseUris); + Discovery = discovery ?? VexProviderDiscovery.Empty; + Trust = trust ?? VexProviderTrust.Default; + Enabled = enabled; + } + + public string Id { get; } + + public string DisplayName { get; } + + public VexProviderKind Kind { get; } + + public ImmutableArray BaseUris { get; } + + public VexProviderDiscovery Discovery { get; } + + public VexProviderTrust Trust { get; } + + public bool Enabled { get; } + + private static ImmutableArray NormalizeUris(IEnumerable? baseUris) + { + if (baseUris is null) + { + return ImmutableArray.Empty; + } + + var distinct = new HashSet(StringComparer.Ordinal); + var builder = ImmutableArray.CreateBuilder(); + foreach (var uri in baseUris) + { + if (uri is null) + { + continue; + } + + var canonical = uri.ToString(); + if (distinct.Add(canonical)) + { + builder.Add(uri); + } + } + + if (builder.Count == 0) + { + return ImmutableArray.Empty; + } + + return builder + .OrderBy(static x => x.ToString(), StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +public sealed record VexProviderDiscovery +{ + public static readonly VexProviderDiscovery Empty = new(null, null); + + public VexProviderDiscovery(Uri? wellKnownMetadata, Uri? rolieService) + { + WellKnownMetadata = wellKnownMetadata; + RolIeService = rolieService; + } + + public Uri? WellKnownMetadata { get; } + + public Uri? RolIeService { get; } +} + +public sealed record VexProviderTrust +{ + public static readonly VexProviderTrust Default = new(1.0, null, ImmutableArray.Empty); + + public VexProviderTrust( + double weight, + VexCosignTrust? cosign, + IEnumerable? pgpFingerprints = null) + { + Weight = NormalizeWeight(weight); + Cosign = cosign; + PgpFingerprints = NormalizeFingerprints(pgpFingerprints); + } + + public double Weight { get; } + + public VexCosignTrust? Cosign { get; } + + public ImmutableArray PgpFingerprints { get; } + + private static double NormalizeWeight(double weight) + { + if (double.IsNaN(weight) || double.IsInfinity(weight)) + { + throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite number."); + } + + if (weight <= 0) + { + return 0.0; + } + + if (weight >= 1.0) + { + return 1.0; + } + + return weight; + } + + private static ImmutableArray NormalizeFingerprints(IEnumerable? values) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + var set = new SortedSet(StringComparer.Ordinal); + foreach (var value in values) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + set.Add(value.Trim()); + } + + return set.Count == 0 + ? ImmutableArray.Empty + : set.ToImmutableArray(); + } +} + +public sealed record VexCosignTrust +{ + public VexCosignTrust(string issuer, string identityPattern) + { + if (string.IsNullOrWhiteSpace(issuer)) + { + throw new ArgumentException("Issuer must be provided for cosign trust metadata.", nameof(issuer)); + } + + if (string.IsNullOrWhiteSpace(identityPattern)) + { + throw new ArgumentException("Identity pattern must be provided for cosign trust metadata.", nameof(identityPattern)); + } + + Issuer = issuer.Trim(); + IdentityPattern = identityPattern.Trim(); + } + + public string Issuer { get; } + + public string IdentityPattern { get; } +} + +[DataContract] +public enum VexProviderKind +{ + [EnumMember(Value = "vendor")] + Vendor, + + [EnumMember(Value = "distro")] + Distro, + + [EnumMember(Value = "hub")] + Hub, + + [EnumMember(Value = "platform")] + Platform, + + [EnumMember(Value = "attestation")] + Attestation, +} diff --git a/src/StellaOps.Vexer.Core/VexQuery.cs b/src/StellaOps.Excititor.Core/VexQuery.cs similarity index 96% rename from src/StellaOps.Vexer.Core/VexQuery.cs rename to src/StellaOps.Excititor.Core/VexQuery.cs index 2b4c7bfb..df8d1d1f 100644 --- a/src/StellaOps.Vexer.Core/VexQuery.cs +++ b/src/StellaOps.Excititor.Core/VexQuery.cs @@ -1,143 +1,143 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using System.Security.Cryptography; -using System.Text; -using System.Linq; - -namespace StellaOps.Vexer.Core; - -public sealed record VexQuery( - ImmutableArray Filters, - ImmutableArray Sort, - int? Limit = null, - int? Offset = null, - string? View = null) -{ - public static VexQuery Empty { get; } = new( - ImmutableArray.Empty, - ImmutableArray.Empty); - - public static VexQuery Create( - IEnumerable? filters = null, - IEnumerable? sort = null, - int? limit = null, - int? offset = null, - string? view = null) - { - var normalizedFilters = NormalizeFilters(filters); - var normalizedSort = NormalizeSort(sort); - return new VexQuery(normalizedFilters, normalizedSort, NormalizeBound(limit), NormalizeBound(offset), NormalizeView(view)); - } - - public VexQuery WithFilters(IEnumerable filters) - => this with { Filters = NormalizeFilters(filters) }; - - public VexQuery WithSort(IEnumerable sort) - => this with { Sort = NormalizeSort(sort) }; - - public VexQuery WithBounds(int? limit = null, int? offset = null) - => this with { Limit = NormalizeBound(limit), Offset = NormalizeBound(offset) }; - - public VexQuery WithView(string? view) - => this with { View = NormalizeView(view) }; - - private static ImmutableArray NormalizeFilters(IEnumerable? filters) - { - if (filters is null) - { - return ImmutableArray.Empty; - } - - return filters - .Where(filter => !string.IsNullOrWhiteSpace(filter.Key)) - .Select(filter => new VexQueryFilter(filter.Key.Trim(), filter.Value?.Trim() ?? string.Empty)) - .OrderBy(filter => filter.Key, StringComparer.Ordinal) - .ThenBy(filter => filter.Value, StringComparer.Ordinal) - .ToImmutableArray(); - } - - private static ImmutableArray NormalizeSort(IEnumerable? sort) - { - if (sort is null) - { - return ImmutableArray.Empty; - } - - return sort - .Where(s => !string.IsNullOrWhiteSpace(s.Field)) - .Select(s => new VexQuerySort(s.Field.Trim(), s.Descending)) - .OrderBy(s => s.Field, StringComparer.Ordinal) - .ThenBy(s => s.Descending) - .ToImmutableArray(); - } - - private static int? NormalizeBound(int? value) - { - if (value is null) - { - return null; - } - - if (value.Value < 0) - { - return 0; - } - - return value.Value; - } - - private static string? NormalizeView(string? view) - => string.IsNullOrWhiteSpace(view) ? null : view.Trim(); -} - -public sealed record VexQueryFilter(string Key, string Value); - -public sealed record VexQuerySort(string Field, bool Descending); - -public sealed partial record VexQuerySignature -{ - public static VexQuerySignature FromQuery(VexQuery query) - { - if (query is null) - { - throw new ArgumentNullException(nameof(query)); - } - - var components = new List(query.Filters.Length + query.Sort.Length + 3); - components.AddRange(query.Filters.Select(filter => $"{filter.Key}={filter.Value}")); - components.AddRange(query.Sort.Select(sort => sort.Descending ? $"sort=-{sort.Field}" : $"sort=+{sort.Field}")); - - if (query.Limit is not null) - { - components.Add($"limit={query.Limit.Value.ToString(CultureInfo.InvariantCulture)}"); - } - - if (query.Offset is not null) - { - components.Add($"offset={query.Offset.Value.ToString(CultureInfo.InvariantCulture)}"); - } - - if (!string.IsNullOrWhiteSpace(query.View)) - { - components.Add($"view={query.View}"); - } - - return new VexQuerySignature(string.Join('&', components)); - } - - public VexContentAddress ComputeHash() - { - using var sha = SHA256.Create(); - var bytes = Encoding.UTF8.GetBytes(Value); - var digest = sha.ComputeHash(bytes); - var builder = new StringBuilder(digest.Length * 2); - foreach (var b in digest) - { - _ = builder.Append(b.ToString("x2", CultureInfo.InvariantCulture)); - } - - return new VexContentAddress("sha256", builder.ToString()); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Linq; + +namespace StellaOps.Excititor.Core; + +public sealed record VexQuery( + ImmutableArray Filters, + ImmutableArray Sort, + int? Limit = null, + int? Offset = null, + string? View = null) +{ + public static VexQuery Empty { get; } = new( + ImmutableArray.Empty, + ImmutableArray.Empty); + + public static VexQuery Create( + IEnumerable? filters = null, + IEnumerable? sort = null, + int? limit = null, + int? offset = null, + string? view = null) + { + var normalizedFilters = NormalizeFilters(filters); + var normalizedSort = NormalizeSort(sort); + return new VexQuery(normalizedFilters, normalizedSort, NormalizeBound(limit), NormalizeBound(offset), NormalizeView(view)); + } + + public VexQuery WithFilters(IEnumerable filters) + => this with { Filters = NormalizeFilters(filters) }; + + public VexQuery WithSort(IEnumerable sort) + => this with { Sort = NormalizeSort(sort) }; + + public VexQuery WithBounds(int? limit = null, int? offset = null) + => this with { Limit = NormalizeBound(limit), Offset = NormalizeBound(offset) }; + + public VexQuery WithView(string? view) + => this with { View = NormalizeView(view) }; + + private static ImmutableArray NormalizeFilters(IEnumerable? filters) + { + if (filters is null) + { + return ImmutableArray.Empty; + } + + return filters + .Where(filter => !string.IsNullOrWhiteSpace(filter.Key)) + .Select(filter => new VexQueryFilter(filter.Key.Trim(), filter.Value?.Trim() ?? string.Empty)) + .OrderBy(filter => filter.Key, StringComparer.Ordinal) + .ThenBy(filter => filter.Value, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeSort(IEnumerable? sort) + { + if (sort is null) + { + return ImmutableArray.Empty; + } + + return sort + .Where(s => !string.IsNullOrWhiteSpace(s.Field)) + .Select(s => new VexQuerySort(s.Field.Trim(), s.Descending)) + .OrderBy(s => s.Field, StringComparer.Ordinal) + .ThenBy(s => s.Descending) + .ToImmutableArray(); + } + + private static int? NormalizeBound(int? value) + { + if (value is null) + { + return null; + } + + if (value.Value < 0) + { + return 0; + } + + return value.Value; + } + + private static string? NormalizeView(string? view) + => string.IsNullOrWhiteSpace(view) ? null : view.Trim(); +} + +public sealed record VexQueryFilter(string Key, string Value); + +public sealed record VexQuerySort(string Field, bool Descending); + +public sealed partial record VexQuerySignature +{ + public static VexQuerySignature FromQuery(VexQuery query) + { + if (query is null) + { + throw new ArgumentNullException(nameof(query)); + } + + var components = new List(query.Filters.Length + query.Sort.Length + 3); + components.AddRange(query.Filters.Select(filter => $"{filter.Key}={filter.Value}")); + components.AddRange(query.Sort.Select(sort => sort.Descending ? $"sort=-{sort.Field}" : $"sort=+{sort.Field}")); + + if (query.Limit is not null) + { + components.Add($"limit={query.Limit.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + if (query.Offset is not null) + { + components.Add($"offset={query.Offset.Value.ToString(CultureInfo.InvariantCulture)}"); + } + + if (!string.IsNullOrWhiteSpace(query.View)) + { + components.Add($"view={query.View}"); + } + + return new VexQuerySignature(string.Join('&', components)); + } + + public VexContentAddress ComputeHash() + { + using var sha = SHA256.Create(); + var bytes = Encoding.UTF8.GetBytes(Value); + var digest = sha.ComputeHash(bytes); + var builder = new StringBuilder(digest.Length * 2); + foreach (var b in digest) + { + _ = builder.Append(b.ToString("x2", CultureInfo.InvariantCulture)); + } + + return new VexContentAddress("sha256", builder.ToString()); + } +} diff --git a/src/StellaOps.Excititor.Core/VexScoreEnvelope.cs b/src/StellaOps.Excititor.Core/VexScoreEnvelope.cs new file mode 100644 index 00000000..7372acec --- /dev/null +++ b/src/StellaOps.Excititor.Core/VexScoreEnvelope.cs @@ -0,0 +1,158 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Excititor.Core; + +public sealed record VexScoreEnvelope +{ + public VexScoreEnvelope( + DateTimeOffset generatedAt, + string policyRevisionId, + string? policyDigest, + double alpha, + double beta, + double weightCeiling, + ImmutableArray entries) + { + if (string.IsNullOrWhiteSpace(policyRevisionId)) + { + throw new ArgumentException("Policy revision id must be provided.", nameof(policyRevisionId)); + } + + if (double.IsNaN(alpha) || double.IsInfinity(alpha) || alpha < 0) + { + throw new ArgumentOutOfRangeException(nameof(alpha), "Alpha must be a finite, non-negative number."); + } + + if (double.IsNaN(beta) || double.IsInfinity(beta) || beta < 0) + { + throw new ArgumentOutOfRangeException(nameof(beta), "Beta must be a finite, non-negative number."); + } + + if (double.IsNaN(weightCeiling) || double.IsInfinity(weightCeiling) || weightCeiling <= 0) + { + throw new ArgumentOutOfRangeException(nameof(weightCeiling), "Weight ceiling must be a finite number greater than zero."); + } + + GeneratedAt = generatedAt; + PolicyRevisionId = policyRevisionId.Trim(); + PolicyDigest = string.IsNullOrWhiteSpace(policyDigest) ? null : policyDigest.Trim(); + Alpha = alpha; + Beta = beta; + WeightCeiling = weightCeiling; + Entries = entries; + } + + public VexScoreEnvelope( + DateTimeOffset generatedAt, + string policyRevisionId, + string? policyDigest, + double alpha, + double beta, + double weightCeiling, + IEnumerable entries) + : this( + generatedAt, + policyRevisionId, + policyDigest, + alpha, + beta, + weightCeiling, + NormalizeEntries(entries)) + { + } + + public DateTimeOffset GeneratedAt { get; } + + public string PolicyRevisionId { get; } + + public string? PolicyDigest { get; } + + public double Alpha { get; } + + public double Beta { get; } + + public double WeightCeiling { get; } + + public ImmutableArray Entries { get; } + + private static ImmutableArray NormalizeEntries(IEnumerable entries) + { + if (entries is null) + { + throw new ArgumentNullException(nameof(entries)); + } + + return entries + .OrderBy(static entry => entry.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static entry => entry.ProductKey, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +public sealed record VexScoreEntry +{ + public VexScoreEntry( + string vulnerabilityId, + string productKey, + VexConsensusStatus status, + DateTimeOffset calculatedAt, + VexSignalSnapshot? signals, + double? score) + { + VulnerabilityId = ValidateVulnerability(vulnerabilityId); + ProductKey = ValidateProduct(productKey); + Status = status; + CalculatedAt = calculatedAt; + Signals = signals; + Score = ValidateScore(score); + } + + public string VulnerabilityId { get; } + + public string ProductKey { get; } + + public VexConsensusStatus Status { get; } + + public DateTimeOffset CalculatedAt { get; } + + public VexSignalSnapshot? Signals { get; } + + public double? Score { get; } + + private static string ValidateVulnerability(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Vulnerability id must be provided.", nameof(value)); + } + + return value.Trim(); + } + + private static string ValidateProduct(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Product key must be provided.", nameof(value)); + } + + return value.Trim(); + } + + private static double? ValidateScore(double? score) + { + if (score is null) + { + return null; + } + + if (double.IsNaN(score.Value) || double.IsInfinity(score.Value) || score.Value < 0) + { + throw new ArgumentOutOfRangeException(nameof(score), "Score must be a finite, non-negative number."); + } + + return score; + } +} diff --git a/src/StellaOps.Excititor.Core/VexSignals.cs b/src/StellaOps.Excititor.Core/VexSignals.cs new file mode 100644 index 00000000..da1cc63c --- /dev/null +++ b/src/StellaOps.Excititor.Core/VexSignals.cs @@ -0,0 +1,64 @@ +namespace StellaOps.Excititor.Core; + +public sealed record VexSignalSnapshot +{ + public VexSignalSnapshot( + VexSeveritySignal? severity = null, + bool? kev = null, + double? epss = null) + { + if (epss is { } epssValue) + { + if (double.IsNaN(epssValue) || double.IsInfinity(epssValue) || epssValue < 0 || epssValue > 1) + { + throw new ArgumentOutOfRangeException(nameof(epss), "EPSS probability must be between 0 and 1."); + } + } + + Severity = severity; + Kev = kev; + Epss = epss; + } + + public VexSeveritySignal? Severity { get; } + + public bool? Kev { get; } + + public double? Epss { get; } +} + +public sealed record VexSeveritySignal +{ + public VexSeveritySignal( + string scheme, + double? score = null, + string? label = null, + string? vector = null) + { + if (string.IsNullOrWhiteSpace(scheme)) + { + throw new ArgumentException("Severity scheme must be provided.", nameof(scheme)); + } + + if (score is { } scoreValue) + { + if (double.IsNaN(scoreValue) || double.IsInfinity(scoreValue) || scoreValue < 0) + { + throw new ArgumentOutOfRangeException(nameof(score), "Severity score must be a finite, non-negative number."); + } + } + + Scheme = scheme.Trim(); + Score = score; + Label = string.IsNullOrWhiteSpace(label) ? null : label.Trim(); + Vector = string.IsNullOrWhiteSpace(vector) ? null : vector.Trim(); + } + + public string Scheme { get; } + + public double? Score { get; } + + public string? Label { get; } + + public string? Vector { get; } +} diff --git a/src/StellaOps.Excititor.Core/VexSignatureVerifiers.cs b/src/StellaOps.Excititor.Core/VexSignatureVerifiers.cs new file mode 100644 index 00000000..89b74da4 --- /dev/null +++ b/src/StellaOps.Excititor.Core/VexSignatureVerifiers.cs @@ -0,0 +1,17 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Excititor.Core; + +/// +/// Signature verifier implementation that trusts ingress sources without performing verification. +/// Useful for offline development flows and ingestion pipelines that perform verification upstream. +/// +public sealed class NoopVexSignatureVerifier : IVexSignatureVerifier +{ + public ValueTask VerifyAsync(VexRawDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + return ValueTask.FromResult(null); + } +} diff --git a/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs b/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs similarity index 94% rename from src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs rename to src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs index e7ecf44c..93ddadff 100644 --- a/src/StellaOps.Vexer.Export.Tests/ExportEngineTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/ExportEngineTests.cs @@ -1,276 +1,277 @@ -using System; -using System.Collections.Immutable; -using System.IO; -using System.Text; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; -using StellaOps.Vexer.Policy; -using StellaOps.Vexer.Storage.Mongo; -using Xunit; - -namespace StellaOps.Vexer.Export.Tests; - -public sealed class ExportEngineTests -{ - [Fact] - public async Task ExportAsync_GeneratesAndCachesManifest() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false); - - var manifest = await engine.ExportAsync(context, CancellationToken.None); - - Assert.False(manifest.FromCache); - Assert.Equal(VexExportFormat.Json, manifest.Format); - Assert.Equal("baseline/v1", manifest.ConsensusRevision); - Assert.Equal(1, manifest.ClaimCount); - - // second call hits cache - var cached = await engine.ExportAsync(context, CancellationToken.None); - Assert.True(cached.FromCache); - Assert.Equal(manifest.ExportId, cached.ExportId); - } - - [Fact] - public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var cacheIndex = new RecordingCacheIndex(); - var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance, cacheIndex); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); - _ = await engine.ExportAsync(initialContext, CancellationToken.None); - - var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true); - var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None); - - Assert.False(refreshed.FromCache); - var signature = VexQuerySignature.FromQuery(refreshContext.Query); - Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed)); - Assert.True(removed); - } - - [Fact] - public async Task ExportAsync_WritesArtifactsToAllStores() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var recorder1 = new RecordingArtifactStore(); - var recorder2 = new RecordingArtifactStore(); - var engine = new VexExportEngine( - store, - evaluator, - dataSource, - new[] { exporter }, - NullLogger.Instance, - cacheIndex: null, - artifactStores: new[] { recorder1, recorder2 }); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); - - await engine.ExportAsync(context, CancellationToken.None); - - Assert.Equal(1, recorder1.SaveCount); - Assert.Equal(1, recorder2.SaveCount); - } - - [Fact] - public async Task ExportAsync_AttachesAttestationMetadata() - { - var store = new InMemoryExportStore(); - var evaluator = new StaticPolicyEvaluator("baseline/v1"); - var dataSource = new InMemoryExportDataSource(); - var exporter = new DummyExporter(VexExportFormat.Json); - var attestation = new RecordingAttestationClient(); - var engine = new VexExportEngine( - store, - evaluator, - dataSource, - new[] { exporter }, - NullLogger.Instance, - cacheIndex: null, - artifactStores: null, - attestationClient: attestation); - - var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); - var requestedAt = DateTimeOffset.UtcNow; - var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); - - var manifest = await engine.ExportAsync(context, CancellationToken.None); - - Assert.NotNull(attestation.LastRequest); - Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); - Assert.NotNull(manifest.Attestation); - Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); - Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); - - Assert.NotNull(store.LastSavedManifest); - Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); - } - - private sealed class InMemoryExportStore : IVexExportStore - { - private readonly Dictionary _store = new(StringComparer.Ordinal); - - public VexExportManifest? LastSavedManifest { get; private set; } - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - var key = CreateKey(signature.Value, format); - _store.TryGetValue(key, out var manifest); - return ValueTask.FromResult(manifest); - } - - public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken) - { - var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); - _store[key] = manifest; - LastSavedManifest = manifest; - return ValueTask.CompletedTask; - } - - private static string CreateKey(string signature, VexExportFormat format) - => FormattableString.Invariant($"{signature}|{format}"); - } - - private sealed class RecordingAttestationClient : IVexAttestationClient - { - public VexAttestationRequest? LastRequest { get; private set; } - - public VexAttestationResponse Response { get; } = new VexAttestationResponse( - new VexAttestationMetadata( - predicateType: "https://stella-ops.org/attestations/vex-export", - rekor: new VexRekorReference("0.2", "rekor://entry", "123"), - envelopeDigest: "sha256:envelope", - signedAt: DateTimeOffset.UnixEpoch), - ImmutableDictionary.Empty); - - public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) - { - LastRequest = request; - return ValueTask.FromResult(Response); - } - - public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); - } - - private sealed class RecordingCacheIndex : IVexCacheIndex - { - public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - - public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - RemoveCalls[(signature.Value, format)] = true; - return ValueTask.CompletedTask; - } - } - - private sealed class RecordingArtifactStore : IVexArtifactStore - { - public int SaveCount { get; private set; } - - public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - SaveCount++; - return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata)); - } - - public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - } - - private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator - { - public StaticPolicyEvaluator(string version) - { - Version = version; - } - - public string Version { get; } - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class InMemoryExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var claim = new VexClaim( - "CVE-2025-0001", - "vendor", - new VexProduct("pkg:demo/app", "Demo"), - VexClaimStatus.Affected, - new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")), - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow); - - var consensus = new VexConsensus( - "CVE-2025-0001", - claim.Product, - VexConsensusStatus.Affected, - DateTimeOffset.UtcNow, - new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) }, - conflicts: null, - policyVersion: "baseline/v1", - summary: "affected"); - - return ValueTask.FromResult(new VexExportDataSet( - ImmutableArray.Create(consensus), - ImmutableArray.Create(claim), - ImmutableArray.Create("vendor"))); - } - } - - private sealed class DummyExporter : IVexExporter - { - public DummyExporter(VexExportFormat format) - { - Format = format; - } - - public VexExportFormat Format { get; } - - public VexContentAddress Digest(VexExportRequest request) - => new("sha256", "deadbeef"); - - public ValueTask SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken) - { - var bytes = System.Text.Encoding.UTF8.GetBytes("{}"); - output.Write(bytes); - return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary.Empty)); - } - } - -} +using System; +using System.Collections.Immutable; +using System.IO; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using Xunit; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class ExportEngineTests +{ + [Fact] + public async Task ExportAsync_GeneratesAndCachesManifest() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow, ForceRefresh: false); + + var manifest = await engine.ExportAsync(context, CancellationToken.None); + + Assert.False(manifest.FromCache); + Assert.Equal(VexExportFormat.Json, manifest.Format); + Assert.Equal("baseline/v1", manifest.ConsensusRevision); + Assert.Equal(1, manifest.ClaimCount); + + // second call hits cache + var cached = await engine.ExportAsync(context, CancellationToken.None); + Assert.True(cached.FromCache); + Assert.Equal(manifest.ExportId, cached.ExportId); + } + + [Fact] + public async Task ExportAsync_ForceRefreshInvalidatesCacheEntry() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var cacheIndex = new RecordingCacheIndex(); + var engine = new VexExportEngine(store, evaluator, dataSource, new[] { exporter }, NullLogger.Instance, cacheIndex); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var initialContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); + _ = await engine.ExportAsync(initialContext, CancellationToken.None); + + var refreshContext = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow.AddMinutes(1), ForceRefresh: true); + var refreshed = await engine.ExportAsync(refreshContext, CancellationToken.None); + + Assert.False(refreshed.FromCache); + var signature = VexQuerySignature.FromQuery(refreshContext.Query); + Assert.True(cacheIndex.RemoveCalls.TryGetValue((signature.Value, refreshContext.Format), out var removed)); + Assert.True(removed); + } + + [Fact] + public async Task ExportAsync_WritesArtifactsToAllStores() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var recorder1 = new RecordingArtifactStore(); + var recorder2 = new RecordingArtifactStore(); + var engine = new VexExportEngine( + store, + evaluator, + dataSource, + new[] { exporter }, + NullLogger.Instance, + cacheIndex: null, + artifactStores: new[] { recorder1, recorder2 }); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var context = new VexExportRequestContext(query, VexExportFormat.Json, DateTimeOffset.UtcNow); + + await engine.ExportAsync(context, CancellationToken.None); + + Assert.Equal(1, recorder1.SaveCount); + Assert.Equal(1, recorder2.SaveCount); + } + + [Fact] + public async Task ExportAsync_AttachesAttestationMetadata() + { + var store = new InMemoryExportStore(); + var evaluator = new StaticPolicyEvaluator("baseline/v1"); + var dataSource = new InMemoryExportDataSource(); + var exporter = new DummyExporter(VexExportFormat.Json); + var attestation = new RecordingAttestationClient(); + var engine = new VexExportEngine( + store, + evaluator, + dataSource, + new[] { exporter }, + NullLogger.Instance, + cacheIndex: null, + artifactStores: null, + attestationClient: attestation); + + var query = VexQuery.Create(new[] { new VexQueryFilter("vulnId", "CVE-2025-0001") }); + var requestedAt = DateTimeOffset.UtcNow; + var context = new VexExportRequestContext(query, VexExportFormat.Json, requestedAt); + + var manifest = await engine.ExportAsync(context, CancellationToken.None); + + Assert.NotNull(attestation.LastRequest); + Assert.Equal(manifest.ExportId, attestation.LastRequest!.ExportId); + Assert.NotNull(manifest.Attestation); + Assert.Equal(attestation.Response.Attestation.EnvelopeDigest, manifest.Attestation!.EnvelopeDigest); + Assert.Equal(attestation.Response.Attestation.PredicateType, manifest.Attestation.PredicateType); + + Assert.NotNull(store.LastSavedManifest); + Assert.Equal(manifest.Attestation, store.LastSavedManifest!.Attestation); + } + + private sealed class InMemoryExportStore : IVexExportStore + { + private readonly Dictionary _store = new(StringComparer.Ordinal); + + public VexExportManifest? LastSavedManifest { get; private set; } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var key = CreateKey(signature.Value, format); + _store.TryGetValue(key, out var manifest); + return ValueTask.FromResult(manifest); + } + + public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var key = CreateKey(manifest.QuerySignature.Value, manifest.Format); + _store[key] = manifest; + LastSavedManifest = manifest; + return ValueTask.CompletedTask; + } + + private static string CreateKey(string signature, VexExportFormat format) + => FormattableString.Invariant($"{signature}|{format}"); + } + + private sealed class RecordingAttestationClient : IVexAttestationClient + { + public VexAttestationRequest? LastRequest { get; private set; } + + public VexAttestationResponse Response { get; } = new VexAttestationResponse( + new VexAttestationMetadata( + predicateType: "https://stella-ops.org/attestations/vex-export", + rekor: new VexRekorReference("0.2", "rekor://entry", "123"), + envelopeDigest: "sha256:envelope", + signedAt: DateTimeOffset.UnixEpoch), + ImmutableDictionary.Empty); + + public ValueTask SignAsync(VexAttestationRequest request, CancellationToken cancellationToken) + { + LastRequest = request; + return ValueTask.FromResult(Response); + } + + public ValueTask VerifyAsync(VexAttestationRequest request, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexAttestationVerification(true, ImmutableDictionary.Empty)); + } + + private sealed class RecordingCacheIndex : IVexCacheIndex + { + public Dictionary<(string Signature, VexExportFormat Format), bool> RemoveCalls { get; } = new(); + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + + public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + { + RemoveCalls[(signature.Value, format)] = true; + return ValueTask.CompletedTask; + } + } + + private sealed class RecordingArtifactStore : IVexArtifactStore + { + public int SaveCount { get; private set; } + + public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + SaveCount++; + return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory", artifact.Content.Length, artifact.Metadata)); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + } + + private sealed class StaticPolicyEvaluator : IVexPolicyEvaluator + { + public StaticPolicyEvaluator(string version) + { + Version = version; + } + + public string Version { get; } + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + private sealed class InMemoryExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + var claim = new VexClaim( + "CVE-2025-0001", + "vendor", + new VexProduct("pkg:demo/app", "Demo"), + VexClaimStatus.Affected, + new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:demo", new Uri("https://example.org/demo")), + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow); + + var consensus = new VexConsensus( + "CVE-2025-0001", + claim.Product, + VexConsensusStatus.Affected, + DateTimeOffset.UtcNow, + new[] { new VexConsensusSource("vendor", VexClaimStatus.Affected, "sha256:demo", 1.0) }, + conflicts: null, + policyVersion: "baseline/v1", + summary: "affected"); + + return ValueTask.FromResult(new VexExportDataSet( + ImmutableArray.Create(consensus), + ImmutableArray.Create(claim), + ImmutableArray.Create("vendor"))); + } + } + + private sealed class DummyExporter : IVexExporter + { + public DummyExporter(VexExportFormat format) + { + Format = format; + } + + public VexExportFormat Format { get; } + + public VexContentAddress Digest(VexExportRequest request) + => new("sha256", "deadbeef"); + + public ValueTask SerializeAsync(VexExportRequest request, Stream output, CancellationToken cancellationToken) + { + var bytes = System.Text.Encoding.UTF8.GetBytes("{}"); + output.Write(bytes); + return ValueTask.FromResult(new VexExportResult(new VexContentAddress("sha256", "deadbeef"), bytes.Length, ImmutableDictionary.Empty)); + } + } + +} diff --git a/src/StellaOps.Vexer.Export.Tests/FileSystemArtifactStoreTests.cs b/src/StellaOps.Excititor.Export.Tests/FileSystemArtifactStoreTests.cs similarity index 88% rename from src/StellaOps.Vexer.Export.Tests/FileSystemArtifactStoreTests.cs rename to src/StellaOps.Excititor.Export.Tests/FileSystemArtifactStoreTests.cs index b7cc8522..12fb6cbe 100644 --- a/src/StellaOps.Vexer.Export.Tests/FileSystemArtifactStoreTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/FileSystemArtifactStoreTests.cs @@ -1,33 +1,33 @@ -using System.Collections.Immutable; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; -using System.IO.Abstractions.TestingHelpers; - -namespace StellaOps.Vexer.Export.Tests; - -public sealed class FileSystemArtifactStoreTests -{ - [Fact] - public async Task SaveAsync_WritesArtifactToDisk() - { - var fs = new MockFileSystem(); - var options = Options.Create(new FileSystemArtifactStoreOptions { RootPath = "/exports" }); - var store = new FileSystemArtifactStore(options, NullLogger.Instance, fs); - - var content = new byte[] { 1, 2, 3 }; - var artifact = new VexExportArtifact( - new VexContentAddress("sha256", "deadbeef"), - VexExportFormat.Json, - content, - ImmutableDictionary.Empty); - - var stored = await store.SaveAsync(artifact, CancellationToken.None); - - Assert.Equal(artifact.Content.Length, stored.SizeBytes); - var filePath = fs.Path.Combine(options.Value.RootPath, stored.Location); - Assert.True(fs.FileExists(filePath)); - Assert.Equal(content, fs.File.ReadAllBytes(filePath)); - } -} +using System.Collections.Immutable; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using System.IO.Abstractions.TestingHelpers; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class FileSystemArtifactStoreTests +{ + [Fact] + public async Task SaveAsync_WritesArtifactToDisk() + { + var fs = new MockFileSystem(); + var options = Options.Create(new FileSystemArtifactStoreOptions { RootPath = "/exports" }); + var store = new FileSystemArtifactStore(options, NullLogger.Instance, fs); + + var content = new byte[] { 1, 2, 3 }; + var artifact = new VexExportArtifact( + new VexContentAddress("sha256", "deadbeef"), + VexExportFormat.Json, + content, + ImmutableDictionary.Empty); + + var stored = await store.SaveAsync(artifact, CancellationToken.None); + + Assert.Equal(artifact.Content.Length, stored.SizeBytes); + var filePath = fs.Path.Combine(options.Value.RootPath, stored.Location); + Assert.True(fs.FileExists(filePath)); + Assert.Equal(content, fs.File.ReadAllBytes(filePath)); + } +} diff --git a/src/StellaOps.Vexer.Export.Tests/OfflineBundleArtifactStoreTests.cs b/src/StellaOps.Excititor.Export.Tests/OfflineBundleArtifactStoreTests.cs similarity index 93% rename from src/StellaOps.Vexer.Export.Tests/OfflineBundleArtifactStoreTests.cs rename to src/StellaOps.Excititor.Export.Tests/OfflineBundleArtifactStoreTests.cs index 7d3abf04..a3a50562 100644 --- a/src/StellaOps.Vexer.Export.Tests/OfflineBundleArtifactStoreTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/OfflineBundleArtifactStoreTests.cs @@ -1,59 +1,59 @@ -using System.Collections.Immutable; -using System.IO.Abstractions.TestingHelpers; -using System.Linq; -using System.Text.Json; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; - -namespace StellaOps.Vexer.Export.Tests; - -public sealed class OfflineBundleArtifactStoreTests -{ - [Fact] - public async Task SaveAsync_WritesArtifactAndManifest() - { - var fs = new MockFileSystem(); - var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" }); - var store = new OfflineBundleArtifactStore(options, NullLogger.Instance, fs); - - var content = new byte[] { 1, 2, 3 }; - var digest = "sha256:" + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(content)).ToLowerInvariant(); - var artifact = new VexExportArtifact( - new VexContentAddress("sha256", digest.Split(':')[1]), - VexExportFormat.Json, - content, - ImmutableDictionary.Empty); - - var stored = await store.SaveAsync(artifact, CancellationToken.None); - - var artifactPath = fs.Path.Combine(options.Value.RootPath, stored.Location); - Assert.True(fs.FileExists(artifactPath)); - - var manifestPath = fs.Path.Combine(options.Value.RootPath, options.Value.ManifestFileName); - Assert.True(fs.FileExists(manifestPath)); - await using var manifestStream = fs.File.OpenRead(manifestPath); - using var document = await JsonDocument.ParseAsync(manifestStream); - var artifacts = document.RootElement.GetProperty("artifacts"); - Assert.True(artifacts.GetArrayLength() >= 1); - var first = artifacts.EnumerateArray().First(); - Assert.Equal(digest, first.GetProperty("digest").GetString()); - } - - [Fact] - public async Task SaveAsync_ThrowsOnDigestMismatch() - { - var fs = new MockFileSystem(); - var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" }); - var store = new OfflineBundleArtifactStore(options, NullLogger.Instance, fs); - - var artifact = new VexExportArtifact( - new VexContentAddress("sha256", "deadbeef"), - VexExportFormat.Json, - new byte[] { 0x01, 0x02 }, - ImmutableDictionary.Empty); - - await Assert.ThrowsAsync(() => store.SaveAsync(artifact, CancellationToken.None).AsTask()); - } -} +using System.Collections.Immutable; +using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class OfflineBundleArtifactStoreTests +{ + [Fact] + public async Task SaveAsync_WritesArtifactAndManifest() + { + var fs = new MockFileSystem(); + var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" }); + var store = new OfflineBundleArtifactStore(options, NullLogger.Instance, fs); + + var content = new byte[] { 1, 2, 3 }; + var digest = "sha256:" + Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(content)).ToLowerInvariant(); + var artifact = new VexExportArtifact( + new VexContentAddress("sha256", digest.Split(':')[1]), + VexExportFormat.Json, + content, + ImmutableDictionary.Empty); + + var stored = await store.SaveAsync(artifact, CancellationToken.None); + + var artifactPath = fs.Path.Combine(options.Value.RootPath, stored.Location); + Assert.True(fs.FileExists(artifactPath)); + + var manifestPath = fs.Path.Combine(options.Value.RootPath, options.Value.ManifestFileName); + Assert.True(fs.FileExists(manifestPath)); + await using var manifestStream = fs.File.OpenRead(manifestPath); + using var document = await JsonDocument.ParseAsync(manifestStream); + var artifacts = document.RootElement.GetProperty("artifacts"); + Assert.True(artifacts.GetArrayLength() >= 1); + var first = artifacts.EnumerateArray().First(); + Assert.Equal(digest, first.GetProperty("digest").GetString()); + } + + [Fact] + public async Task SaveAsync_ThrowsOnDigestMismatch() + { + var fs = new MockFileSystem(); + var options = Options.Create(new OfflineBundleArtifactStoreOptions { RootPath = "/offline" }); + var store = new OfflineBundleArtifactStore(options, NullLogger.Instance, fs); + + var artifact = new VexExportArtifact( + new VexContentAddress("sha256", "deadbeef"), + VexExportFormat.Json, + new byte[] { 0x01, 0x02 }, + ImmutableDictionary.Empty); + + await Assert.ThrowsAsync(() => store.SaveAsync(artifact, CancellationToken.None).AsTask()); + } +} diff --git a/src/StellaOps.Vexer.Export.Tests/S3ArtifactStoreTests.cs b/src/StellaOps.Excititor.Export.Tests/S3ArtifactStoreTests.cs similarity index 95% rename from src/StellaOps.Vexer.Export.Tests/S3ArtifactStoreTests.cs rename to src/StellaOps.Excititor.Export.Tests/S3ArtifactStoreTests.cs index 64c3b482..6b90ed46 100644 --- a/src/StellaOps.Vexer.Export.Tests/S3ArtifactStoreTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/S3ArtifactStoreTests.cs @@ -1,95 +1,95 @@ -using System.Collections.Concurrent; -using System.Collections.Immutable; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; - -namespace StellaOps.Vexer.Export.Tests; - -public sealed class S3ArtifactStoreTests -{ - [Fact] - public async Task SaveAsync_UploadsContentWithMetadata() - { - var client = new FakeS3Client(); - var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" }); - var store = new S3ArtifactStore(client, options, NullLogger.Instance); - - var content = new byte[] { 1, 2, 3, 4 }; - var artifact = new VexExportArtifact( - new VexContentAddress("sha256", "deadbeef"), - VexExportFormat.Json, - content, - ImmutableDictionary.Empty); - - await store.SaveAsync(artifact, CancellationToken.None); - - Assert.True(client.PutCalls.TryGetValue("exports", out var bucketEntries)); - Assert.NotNull(bucketEntries); - var entry = bucketEntries!.Single(); - Assert.Equal("vex/json/deadbeef.json", entry.Key); - Assert.Equal(content, entry.Content); - Assert.Equal("sha256:deadbeef", entry.Metadata["vex-digest"]); - } - - [Fact] - public async Task OpenReadAsync_ReturnsStoredContent() - { - var client = new FakeS3Client(); - var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" }); - var store = new S3ArtifactStore(client, options, NullLogger.Instance); - - var address = new VexContentAddress("sha256", "cafebabe"); - client.SeedObject("exports", "vex/json/cafebabe.json", new byte[] { 9, 9, 9 }); - - var stream = await store.OpenReadAsync(address, CancellationToken.None); - Assert.NotNull(stream); - using var ms = new MemoryStream(); - await stream!.CopyToAsync(ms); - Assert.Equal(new byte[] { 9, 9, 9 }, ms.ToArray()); - } - - private sealed class FakeS3Client : IS3ArtifactClient - { - public ConcurrentDictionary> PutCalls { get; } = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary<(string Bucket, string Key), byte[]> _storage = new(); - - public void SeedObject(string bucket, string key, byte[] content) - { - PutCalls.GetOrAdd(bucket, _ => new List()).Add(new S3Entry(key, content, new Dictionary())); - _storage[(bucket, key)] = content; - } - - public Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken) - => Task.FromResult(_storage.ContainsKey((bucketName, key))); - - public Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken) - { - using var ms = new MemoryStream(); - content.CopyTo(ms); - var bytes = ms.ToArray(); - PutCalls.GetOrAdd(bucketName, _ => new List()).Add(new S3Entry(key, bytes, new Dictionary(metadata))); - _storage[(bucketName, key)] = bytes; - return Task.CompletedTask; - } - - public Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken) - { - if (_storage.TryGetValue((bucketName, key), out var bytes)) - { - return Task.FromResult(new MemoryStream(bytes, writable: false)); - } - - return Task.FromResult(null); - } - - public Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken) - { - _storage.TryRemove((bucketName, key), out _); - return Task.CompletedTask; - } - - public readonly record struct S3Entry(string Key, byte[] Content, IDictionary Metadata); - } -} +using System.Collections.Concurrent; +using System.Collections.Immutable; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class S3ArtifactStoreTests +{ + [Fact] + public async Task SaveAsync_UploadsContentWithMetadata() + { + var client = new FakeS3Client(); + var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" }); + var store = new S3ArtifactStore(client, options, NullLogger.Instance); + + var content = new byte[] { 1, 2, 3, 4 }; + var artifact = new VexExportArtifact( + new VexContentAddress("sha256", "deadbeef"), + VexExportFormat.Json, + content, + ImmutableDictionary.Empty); + + await store.SaveAsync(artifact, CancellationToken.None); + + Assert.True(client.PutCalls.TryGetValue("exports", out var bucketEntries)); + Assert.NotNull(bucketEntries); + var entry = bucketEntries!.Single(); + Assert.Equal("vex/json/deadbeef.json", entry.Key); + Assert.Equal(content, entry.Content); + Assert.Equal("sha256:deadbeef", entry.Metadata["vex-digest"]); + } + + [Fact] + public async Task OpenReadAsync_ReturnsStoredContent() + { + var client = new FakeS3Client(); + var options = Options.Create(new S3ArtifactStoreOptions { BucketName = "exports", Prefix = "vex" }); + var store = new S3ArtifactStore(client, options, NullLogger.Instance); + + var address = new VexContentAddress("sha256", "cafebabe"); + client.SeedObject("exports", "vex/json/cafebabe.json", new byte[] { 9, 9, 9 }); + + var stream = await store.OpenReadAsync(address, CancellationToken.None); + Assert.NotNull(stream); + using var ms = new MemoryStream(); + await stream!.CopyToAsync(ms); + Assert.Equal(new byte[] { 9, 9, 9 }, ms.ToArray()); + } + + private sealed class FakeS3Client : IS3ArtifactClient + { + public ConcurrentDictionary> PutCalls { get; } = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary<(string Bucket, string Key), byte[]> _storage = new(); + + public void SeedObject(string bucket, string key, byte[] content) + { + PutCalls.GetOrAdd(bucket, _ => new List()).Add(new S3Entry(key, content, new Dictionary())); + _storage[(bucket, key)] = content; + } + + public Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken) + => Task.FromResult(_storage.ContainsKey((bucketName, key))); + + public Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken) + { + using var ms = new MemoryStream(); + content.CopyTo(ms); + var bytes = ms.ToArray(); + PutCalls.GetOrAdd(bucketName, _ => new List()).Add(new S3Entry(key, bytes, new Dictionary(metadata))); + _storage[(bucketName, key)] = bytes; + return Task.CompletedTask; + } + + public Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken) + { + if (_storage.TryGetValue((bucketName, key), out var bytes)) + { + return Task.FromResult(new MemoryStream(bytes, writable: false)); + } + + return Task.FromResult(null); + } + + public Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken) + { + _storage.TryRemove((bucketName, key), out _); + return Task.CompletedTask; + } + + public readonly record struct S3Entry(string Key, byte[] Content, IDictionary Metadata); + } +} diff --git a/src/StellaOps.Vexer.Export.Tests/StellaOps.Vexer.Export.Tests.csproj b/src/StellaOps.Excititor.Export.Tests/StellaOps.Excititor.Export.Tests.csproj similarity index 79% rename from src/StellaOps.Vexer.Export.Tests/StellaOps.Vexer.Export.Tests.csproj rename to src/StellaOps.Excititor.Export.Tests/StellaOps.Excititor.Export.Tests.csproj index 3a38978f..81a8488a 100644 --- a/src/StellaOps.Vexer.Export.Tests/StellaOps.Vexer.Export.Tests.csproj +++ b/src/StellaOps.Excititor.Export.Tests/StellaOps.Excititor.Export.Tests.csproj @@ -1,15 +1,15 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Vexer.Export.Tests/VexExportCacheServiceTests.cs b/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs similarity index 87% rename from src/StellaOps.Vexer.Export.Tests/VexExportCacheServiceTests.cs rename to src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs index cbaa7efe..29c84f9f 100644 --- a/src/StellaOps.Vexer.Export.Tests/VexExportCacheServiceTests.cs +++ b/src/StellaOps.Excititor.Export.Tests/VexExportCacheServiceTests.cs @@ -1,81 +1,82 @@ -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Export.Tests; - -public sealed class VexExportCacheServiceTests -{ - [Fact] - public async Task InvalidateAsync_RemovesEntry() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance(); - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var signature = new VexQuerySignature("format=json|provider=vendor"); - await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None); - - Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value); - Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat); - Assert.Equal(1, cacheIndex.RemoveCalls); - } - - [Fact] - public async Task PruneExpiredAsync_ReturnsCount() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance { ExpiredCount = 3 }; - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); - - Assert.Equal(3, removed); - } - - [Fact] - public async Task PruneDanglingAsync_ReturnsCount() - { - var cacheIndex = new RecordingIndex(); - var maintenance = new StubMaintenance { DanglingCount = 2 }; - var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); - - var removed = await service.PruneDanglingAsync(CancellationToken.None); - - Assert.Equal(2, removed); - } - - private sealed class RecordingIndex : IVexCacheIndex - { - public VexQuerySignature? LastSignature { get; private set; } - public VexExportFormat LastFormat { get; private set; } - public int RemoveCalls { get; private set; } - - public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - => ValueTask.FromResult(null); - - public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) - => ValueTask.CompletedTask; - - public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - LastSignature = signature; - LastFormat = format; - RemoveCalls++; - return ValueTask.CompletedTask; - } - } - - private sealed class StubMaintenance : IVexCacheMaintenance - { - public int ExpiredCount { get; set; } - public int DanglingCount { get; set; } - - public ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) - => ValueTask.FromResult(ExpiredCount); - - public ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken) - => ValueTask.FromResult(DanglingCount); - } -} +using Microsoft.Extensions.Logging.Abstractions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Export.Tests; + +public sealed class VexExportCacheServiceTests +{ + [Fact] + public async Task InvalidateAsync_RemovesEntry() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance(); + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var signature = new VexQuerySignature("format=json|provider=vendor"); + await service.InvalidateAsync(signature, VexExportFormat.Json, CancellationToken.None); + + Assert.Equal(signature.Value, cacheIndex.LastSignature?.Value); + Assert.Equal(VexExportFormat.Json, cacheIndex.LastFormat); + Assert.Equal(1, cacheIndex.RemoveCalls); + } + + [Fact] + public async Task PruneExpiredAsync_ReturnsCount() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance { ExpiredCount = 3 }; + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var removed = await service.PruneExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); + + Assert.Equal(3, removed); + } + + [Fact] + public async Task PruneDanglingAsync_ReturnsCount() + { + var cacheIndex = new RecordingIndex(); + var maintenance = new StubMaintenance { DanglingCount = 2 }; + var service = new VexExportCacheService(cacheIndex, maintenance, NullLogger.Instance); + + var removed = await service.PruneDanglingAsync(CancellationToken.None); + + Assert.Equal(2, removed); + } + + private sealed class RecordingIndex : IVexCacheIndex + { + public VexQuerySignature? LastSignature { get; private set; } + public VexExportFormat LastFormat { get; private set; } + public int RemoveCalls { get; private set; } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + => ValueTask.FromResult(null); + + public ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) + => ValueTask.CompletedTask; + + public ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + { + LastSignature = signature; + LastFormat = format; + RemoveCalls++; + return ValueTask.CompletedTask; + } + } + + private sealed class StubMaintenance : IVexCacheMaintenance + { + public int ExpiredCount { get; set; } + public int DanglingCount { get; set; } + + public ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(ExpiredCount); + + public ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.FromResult(DanglingCount); + } +} diff --git a/src/StellaOps.Vexer.Export/AGENTS.md b/src/StellaOps.Excititor.Export/AGENTS.md similarity index 88% rename from src/StellaOps.Vexer.Export/AGENTS.md rename to src/StellaOps.Excititor.Export/AGENTS.md index b0166f35..0e34cfbc 100644 --- a/src/StellaOps.Vexer.Export/AGENTS.md +++ b/src/StellaOps.Excititor.Export/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Produces deterministic VEX export artifacts, coordinates cache lookups, and bridges artifact storage with attestation generation. -## Scope -- Export orchestration pipeline: query signature resolution, cache lookup, snapshot building, attestation handoff. -- Format-neutral builder interfaces consumed by format-specific plug-ins. -- Artifact store abstraction wiring (S3/MinIO/filesystem) with offline-friendly packaging. -- Export metrics/logging and deterministic manifest emission. -## Participants -- WebService invokes the export engine to service `/vexer/export` requests. -- Attestation module receives built artifacts through this layer for signing. -- Worker reuses caching and artifact utilities for scheduled exports and GC routines. -## Interfaces & contracts -- `IExportEngine`, `IExportSnapshotBuilder`, cache provider interfaces, and artifact store adapters. -- Hook points for format plug-ins (JSON, JSONL, OpenVEX, CSAF, ZIP bundle). -## In/Out of scope -In: orchestration, caching, artifact store interactions, manifest metadata. -Out: format-specific serialization (lives in Formats.*), policy evaluation (Policy), HTTP presentation (WebService). -## Observability & security expectations -- Emit cache hit/miss counters, export durations, artifact sizes, and attestation timing logs. -- Ensure no sensitive tokens/URIs are logged. -## Tests -- Engine orchestration tests, cache behavior, and artifact lifecycle coverage will live in `../StellaOps.Vexer.Export.Tests`. +# AGENTS +## Role +Produces deterministic VEX export artifacts, coordinates cache lookups, and bridges artifact storage with attestation generation. +## Scope +- Export orchestration pipeline: query signature resolution, cache lookup, snapshot building, attestation handoff. +- Format-neutral builder interfaces consumed by format-specific plug-ins. +- Artifact store abstraction wiring (S3/MinIO/filesystem) with offline-friendly packaging. +- Export metrics/logging and deterministic manifest emission. +## Participants +- WebService invokes the export engine to service `/excititor/export` requests. +- Attestation module receives built artifacts through this layer for signing. +- Worker reuses caching and artifact utilities for scheduled exports and GC routines. +## Interfaces & contracts +- `IExportEngine`, `IExportSnapshotBuilder`, cache provider interfaces, and artifact store adapters. +- Hook points for format plug-ins (JSON, JSONL, OpenVEX, CSAF, ZIP bundle). +## In/Out of scope +In: orchestration, caching, artifact store interactions, manifest metadata. +Out: format-specific serialization (lives in Formats.*), policy evaluation (Policy), HTTP presentation (WebService). +## Observability & security expectations +- Emit cache hit/miss counters, export durations, artifact sizes, and attestation timing logs. +- Ensure no sensitive tokens/URIs are logged. +## Tests +- Engine orchestration tests, cache behavior, and artifact lifecycle coverage will live in `../StellaOps.Excititor.Export.Tests`. diff --git a/src/StellaOps.Vexer.Export/ExportEngine.cs b/src/StellaOps.Excititor.Export/ExportEngine.cs similarity index 95% rename from src/StellaOps.Vexer.Export/ExportEngine.cs rename to src/StellaOps.Excititor.Export/ExportEngine.cs index a93ee805..0c4397dc 100644 --- a/src/StellaOps.Vexer.Export/ExportEngine.cs +++ b/src/StellaOps.Excititor.Export/ExportEngine.cs @@ -1,209 +1,209 @@ -using System.Collections.Immutable; -using System.IO; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Policy; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Export; - -public interface IExportEngine -{ - ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken); -} - -public sealed record VexExportRequestContext( - VexQuery Query, - VexExportFormat Format, - DateTimeOffset RequestedAt, - bool ForceRefresh = false); - -public interface IVexExportDataSource -{ - ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken); -} - -public sealed record VexExportDataSet( - ImmutableArray Consensus, - ImmutableArray Claims, - ImmutableArray SourceProviders); - -public sealed class VexExportEngine : IExportEngine -{ - private readonly IVexExportStore _exportStore; - private readonly IVexPolicyEvaluator _policyEvaluator; - private readonly IVexExportDataSource _dataSource; - private readonly IReadOnlyDictionary _exporters; - private readonly ILogger _logger; - private readonly IVexCacheIndex? _cacheIndex; - private readonly IReadOnlyList _artifactStores; - private readonly IVexAttestationClient? _attestationClient; - - public VexExportEngine( - IVexExportStore exportStore, - IVexPolicyEvaluator policyEvaluator, - IVexExportDataSource dataSource, - IEnumerable exporters, - ILogger logger, - IVexCacheIndex? cacheIndex = null, - IEnumerable? artifactStores = null, - IVexAttestationClient? attestationClient = null) - { - _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); - _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); - _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _cacheIndex = cacheIndex; - _artifactStores = artifactStores?.ToArray() ?? Array.Empty(); - _attestationClient = attestationClient; - - if (exporters is null) - { - throw new ArgumentNullException(nameof(exporters)); - } - - _exporters = exporters.ToDictionary(x => x.Format); - } - - public async ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - var signature = VexQuerySignature.FromQuery(context.Query); - - if (!context.ForceRefresh) - { - var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); - if (cached is not null) - { - _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); - return new VexExportManifest( - cached.ExportId, - cached.QuerySignature, - cached.Format, - cached.CreatedAt, - cached.Artifact, - cached.ClaimCount, - cached.SourceProviders, - fromCache: true, - cached.ConsensusRevision, - cached.Attestation, - cached.SizeBytes); - } - } - else if (_cacheIndex is not null) - { - await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); - _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); - } - - var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); - var exporter = ResolveExporter(context.Format); - - var exportRequest = new VexExportRequest( - context.Query, - dataset.Consensus, - dataset.Claims, - context.RequestedAt); - - var digest = exporter.Digest(exportRequest); - var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); - - await using var buffer = new MemoryStream(); - var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); - - if (_artifactStores.Count > 0) - { - var writtenBytes = buffer.ToArray(); - try - { - var artifact = new VexExportArtifact( - result.Digest, - context.Format, - writtenBytes, - result.Metadata); - - foreach (var store in _artifactStores) - { - await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); - } - - _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); - throw; - } - } - - VexAttestationMetadata? attestationMetadata = null; - if (_attestationClient is not null) - { - var attestationRequest = new VexAttestationRequest( - exportId, - signature, - digest, - context.Format, - context.RequestedAt, - dataset.SourceProviders, - result.Metadata); - - var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); - attestationMetadata = response.Attestation; - - if (!response.Diagnostics.IsEmpty) - { - foreach (var diagnostic in response.Diagnostics) - { - _logger.LogDebug( - "Attestation diagnostic {Key}={Value} for export {ExportId}", - diagnostic.Key, - diagnostic.Value, - exportId); - } - } - - _logger.LogInformation("Attestation generated for export {ExportId}", exportId); - } - - var manifest = new VexExportManifest( - exportId, - signature, - context.Format, - context.RequestedAt, - digest, - dataset.Claims.Length, - dataset.SourceProviders, - fromCache: false, - consensusRevision: _policyEvaluator.Version, - attestation: attestationMetadata, - sizeBytes: result.BytesWritten); - - await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation( - "Export generated for {Signature} ({Format}) size={SizeBytes} bytes", - signature.Value, - context.Format, - result.BytesWritten); - - return manifest; - } - - private IVexExporter ResolveExporter(VexExportFormat format) - => _exporters.TryGetValue(format, out var exporter) - ? exporter - : throw new InvalidOperationException($"No exporter registered for format '{format}'."); -} - -public static class VexExportServiceCollectionExtensions -{ - public static IServiceCollection AddVexExportEngine(this IServiceCollection services) - { - services.AddSingleton(); - services.AddVexExportCacheServices(); - return services; - } -} +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Export; + +public interface IExportEngine +{ + ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken); +} + +public sealed record VexExportRequestContext( + VexQuery Query, + VexExportFormat Format, + DateTimeOffset RequestedAt, + bool ForceRefresh = false); + +public interface IVexExportDataSource +{ + ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken); +} + +public sealed record VexExportDataSet( + ImmutableArray Consensus, + ImmutableArray Claims, + ImmutableArray SourceProviders); + +public sealed class VexExportEngine : IExportEngine +{ + private readonly IVexExportStore _exportStore; + private readonly IVexPolicyEvaluator _policyEvaluator; + private readonly IVexExportDataSource _dataSource; + private readonly IReadOnlyDictionary _exporters; + private readonly ILogger _logger; + private readonly IVexCacheIndex? _cacheIndex; + private readonly IReadOnlyList _artifactStores; + private readonly IVexAttestationClient? _attestationClient; + + public VexExportEngine( + IVexExportStore exportStore, + IVexPolicyEvaluator policyEvaluator, + IVexExportDataSource dataSource, + IEnumerable exporters, + ILogger logger, + IVexCacheIndex? cacheIndex = null, + IEnumerable? artifactStores = null, + IVexAttestationClient? attestationClient = null) + { + _exportStore = exportStore ?? throw new ArgumentNullException(nameof(exportStore)); + _policyEvaluator = policyEvaluator ?? throw new ArgumentNullException(nameof(policyEvaluator)); + _dataSource = dataSource ?? throw new ArgumentNullException(nameof(dataSource)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _cacheIndex = cacheIndex; + _artifactStores = artifactStores?.ToArray() ?? Array.Empty(); + _attestationClient = attestationClient; + + if (exporters is null) + { + throw new ArgumentNullException(nameof(exporters)); + } + + _exporters = exporters.ToDictionary(x => x.Format); + } + + public async ValueTask ExportAsync(VexExportRequestContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + var signature = VexQuerySignature.FromQuery(context.Query); + + if (!context.ForceRefresh) + { + var cached = await _exportStore.FindAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); + if (cached is not null) + { + _logger.LogInformation("Reusing cached export for {Signature} ({Format})", signature.Value, context.Format); + return new VexExportManifest( + cached.ExportId, + cached.QuerySignature, + cached.Format, + cached.CreatedAt, + cached.Artifact, + cached.ClaimCount, + cached.SourceProviders, + fromCache: true, + cached.ConsensusRevision, + cached.Attestation, + cached.SizeBytes); + } + } + else if (_cacheIndex is not null) + { + await _cacheIndex.RemoveAsync(signature, context.Format, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Force refresh requested; invalidated cache entry for {Signature} ({Format})", signature.Value, context.Format); + } + + var dataset = await _dataSource.FetchAsync(context.Query, cancellationToken).ConfigureAwait(false); + var exporter = ResolveExporter(context.Format); + + var exportRequest = new VexExportRequest( + context.Query, + dataset.Consensus, + dataset.Claims, + context.RequestedAt); + + var digest = exporter.Digest(exportRequest); + var exportId = FormattableString.Invariant($"exports/{context.RequestedAt:yyyyMMddTHHmmssfffZ}/{digest.Digest}"); + + await using var buffer = new MemoryStream(); + var result = await exporter.SerializeAsync(exportRequest, buffer, cancellationToken).ConfigureAwait(false); + + if (_artifactStores.Count > 0) + { + var writtenBytes = buffer.ToArray(); + try + { + var artifact = new VexExportArtifact( + result.Digest, + context.Format, + writtenBytes, + result.Metadata); + + foreach (var store in _artifactStores) + { + await store.SaveAsync(artifact, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation("Stored export artifact {Digest} via {StoreCount} store(s)", result.Digest.ToUri(), _artifactStores.Count); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to store export artifact {Digest}", result.Digest.ToUri()); + throw; + } + } + + VexAttestationMetadata? attestationMetadata = null; + if (_attestationClient is not null) + { + var attestationRequest = new VexAttestationRequest( + exportId, + signature, + digest, + context.Format, + context.RequestedAt, + dataset.SourceProviders, + result.Metadata); + + var response = await _attestationClient.SignAsync(attestationRequest, cancellationToken).ConfigureAwait(false); + attestationMetadata = response.Attestation; + + if (!response.Diagnostics.IsEmpty) + { + foreach (var diagnostic in response.Diagnostics) + { + _logger.LogDebug( + "Attestation diagnostic {Key}={Value} for export {ExportId}", + diagnostic.Key, + diagnostic.Value, + exportId); + } + } + + _logger.LogInformation("Attestation generated for export {ExportId}", exportId); + } + + var manifest = new VexExportManifest( + exportId, + signature, + context.Format, + context.RequestedAt, + digest, + dataset.Claims.Length, + dataset.SourceProviders, + fromCache: false, + consensusRevision: _policyEvaluator.Version, + attestation: attestationMetadata, + sizeBytes: result.BytesWritten); + + await _exportStore.SaveAsync(manifest, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Export generated for {Signature} ({Format}) size={SizeBytes} bytes", + signature.Value, + context.Format, + result.BytesWritten); + + return manifest; + } + + private IVexExporter ResolveExporter(VexExportFormat format) + => _exporters.TryGetValue(format, out var exporter) + ? exporter + : throw new InvalidOperationException($"No exporter registered for format '{format}'."); +} + +public static class VexExportServiceCollectionExtensions +{ + public static IServiceCollection AddVexExportEngine(this IServiceCollection services) + { + services.AddSingleton(); + services.AddVexExportCacheServices(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Export/FileSystemArtifactStore.cs b/src/StellaOps.Excititor.Export/FileSystemArtifactStore.cs similarity index 96% rename from src/StellaOps.Vexer.Export/FileSystemArtifactStore.cs rename to src/StellaOps.Excititor.Export/FileSystemArtifactStore.cs index 9aa2f2a8..a82f0226 100644 --- a/src/StellaOps.Vexer.Export/FileSystemArtifactStore.cs +++ b/src/StellaOps.Excititor.Export/FileSystemArtifactStore.cs @@ -1,159 +1,159 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Abstractions; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Export; - -public sealed class FileSystemArtifactStoreOptions -{ - public string RootPath { get; set; } = "."; - - public bool OverwriteExisting { get; set; } = false; -} - -public sealed class FileSystemArtifactStore : IVexArtifactStore -{ - private readonly IFileSystem _fileSystem; - private readonly FileSystemArtifactStoreOptions _options; - private readonly ILogger _logger; - - public FileSystemArtifactStore( - IOptions options, - ILogger logger, - IFileSystem? fileSystem = null) - { - ArgumentNullException.ThrowIfNull(options); - _options = options.Value; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _fileSystem = fileSystem ?? new FileSystem(); - - if (string.IsNullOrWhiteSpace(_options.RootPath)) - { - throw new ArgumentException("RootPath must be provided for FileSystemArtifactStore.", nameof(options)); - } - - var root = _fileSystem.Path.GetFullPath(_options.RootPath); - _fileSystem.Directory.CreateDirectory(root); - _options.RootPath = root; - } - - public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(artifact); - - var relativePath = BuildArtifactPath(artifact.ContentAddress, artifact.Format); - var destination = _fileSystem.Path.Combine(_options.RootPath, relativePath); - var directory = _fileSystem.Path.GetDirectoryName(destination); - if (!string.IsNullOrEmpty(directory)) - { - _fileSystem.Directory.CreateDirectory(directory); - } - - if (_fileSystem.File.Exists(destination) && !_options.OverwriteExisting) - { - _logger.LogInformation("Artifact {Digest} already exists at {Path}; skipping write.", artifact.ContentAddress.ToUri(), destination); - } - else - { - await using var stream = _fileSystem.File.Create(destination); - await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); - } - - var location = destination.Replace(_options.RootPath, string.Empty).TrimStart(_fileSystem.Path.DirectorySeparatorChar, _fileSystem.Path.AltDirectorySeparatorChar); - - return new VexStoredArtifact( - artifact.ContentAddress, - location, - artifact.Content.Length, - artifact.Metadata); - } - - public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - var path = MaterializePath(contentAddress); - if (path is not null && _fileSystem.File.Exists(path)) - { - _fileSystem.File.Delete(path); - } - - return ValueTask.CompletedTask; - } - - public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - var path = MaterializePath(contentAddress); - if (path is null || !_fileSystem.File.Exists(path)) - { - return ValueTask.FromResult(null); - } - - Stream stream = _fileSystem.File.OpenRead(path); - return ValueTask.FromResult(stream); - } - - private static string BuildArtifactPath(VexContentAddress address, VexExportFormat format) - { - var formatSegment = format.ToString().ToLowerInvariant(); - var safeDigest = address.Digest.Replace(':', '_'); - var extension = GetExtension(format); - return Path.Combine(formatSegment, safeDigest + extension); - } - - private string? MaterializePath(VexContentAddress address) - { - ArgumentNullException.ThrowIfNull(address); - var sanitized = address.Digest.Replace(':', '_'); - - foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) - { - var candidate = _fileSystem.Path.Combine(_options.RootPath, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); - if (_fileSystem.File.Exists(candidate)) - { - return candidate; - } - } - - // fallback: direct root search with common extensions - foreach (var extension in new[] { ".json", ".jsonl" }) - { - var candidate = _fileSystem.Path.Combine(_options.RootPath, sanitized + extension); - if (_fileSystem.File.Exists(candidate)) - { - return candidate; - } - } - - return null; - } - - private static string GetExtension(VexExportFormat format) - => format switch - { - VexExportFormat.Json => ".json", - VexExportFormat.JsonLines => ".jsonl", - VexExportFormat.OpenVex => ".json", - VexExportFormat.Csaf => ".json", - _ => ".bin", - }; -} - -public static class FileSystemArtifactStoreServiceCollectionExtensions -{ - public static IServiceCollection AddVexFileSystemArtifactStore(this IServiceCollection services, Action? configure = null) - { - if (configure is not null) - { - services.Configure(configure); - } - - services.AddSingleton(); - return services; - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Abstractions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Export; + +public sealed class FileSystemArtifactStoreOptions +{ + public string RootPath { get; set; } = "."; + + public bool OverwriteExisting { get; set; } = false; +} + +public sealed class FileSystemArtifactStore : IVexArtifactStore +{ + private readonly IFileSystem _fileSystem; + private readonly FileSystemArtifactStoreOptions _options; + private readonly ILogger _logger; + + public FileSystemArtifactStore( + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _fileSystem = fileSystem ?? new FileSystem(); + + if (string.IsNullOrWhiteSpace(_options.RootPath)) + { + throw new ArgumentException("RootPath must be provided for FileSystemArtifactStore.", nameof(options)); + } + + var root = _fileSystem.Path.GetFullPath(_options.RootPath); + _fileSystem.Directory.CreateDirectory(root); + _options.RootPath = root; + } + + public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(artifact); + + var relativePath = BuildArtifactPath(artifact.ContentAddress, artifact.Format); + var destination = _fileSystem.Path.Combine(_options.RootPath, relativePath); + var directory = _fileSystem.Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(directory)) + { + _fileSystem.Directory.CreateDirectory(directory); + } + + if (_fileSystem.File.Exists(destination) && !_options.OverwriteExisting) + { + _logger.LogInformation("Artifact {Digest} already exists at {Path}; skipping write.", artifact.ContentAddress.ToUri(), destination); + } + else + { + await using var stream = _fileSystem.File.Create(destination); + await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); + } + + var location = destination.Replace(_options.RootPath, string.Empty).TrimStart(_fileSystem.Path.DirectorySeparatorChar, _fileSystem.Path.AltDirectorySeparatorChar); + + return new VexStoredArtifact( + artifact.ContentAddress, + location, + artifact.Content.Length, + artifact.Metadata); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + var path = MaterializePath(contentAddress); + if (path is not null && _fileSystem.File.Exists(path)) + { + _fileSystem.File.Delete(path); + } + + return ValueTask.CompletedTask; + } + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + var path = MaterializePath(contentAddress); + if (path is null || !_fileSystem.File.Exists(path)) + { + return ValueTask.FromResult(null); + } + + Stream stream = _fileSystem.File.OpenRead(path); + return ValueTask.FromResult(stream); + } + + private static string BuildArtifactPath(VexContentAddress address, VexExportFormat format) + { + var formatSegment = format.ToString().ToLowerInvariant(); + var safeDigest = address.Digest.Replace(':', '_'); + var extension = GetExtension(format); + return Path.Combine(formatSegment, safeDigest + extension); + } + + private string? MaterializePath(VexContentAddress address) + { + ArgumentNullException.ThrowIfNull(address); + var sanitized = address.Digest.Replace(':', '_'); + + foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) + { + var candidate = _fileSystem.Path.Combine(_options.RootPath, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); + if (_fileSystem.File.Exists(candidate)) + { + return candidate; + } + } + + // fallback: direct root search with common extensions + foreach (var extension in new[] { ".json", ".jsonl" }) + { + var candidate = _fileSystem.Path.Combine(_options.RootPath, sanitized + extension); + if (_fileSystem.File.Exists(candidate)) + { + return candidate; + } + } + + return null; + } + + private static string GetExtension(VexExportFormat format) + => format switch + { + VexExportFormat.Json => ".json", + VexExportFormat.JsonLines => ".jsonl", + VexExportFormat.OpenVex => ".json", + VexExportFormat.Csaf => ".json", + _ => ".bin", + }; +} + +public static class FileSystemArtifactStoreServiceCollectionExtensions +{ + public static IServiceCollection AddVexFileSystemArtifactStore(this IServiceCollection services, Action? configure = null) + { + if (configure is not null) + { + services.Configure(configure); + } + + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Export/IVexArtifactStore.cs b/src/StellaOps.Excititor.Export/IVexArtifactStore.cs similarity index 89% rename from src/StellaOps.Vexer.Export/IVexArtifactStore.cs rename to src/StellaOps.Excititor.Export/IVexArtifactStore.cs index 5de445ac..3a6c416d 100644 --- a/src/StellaOps.Vexer.Export/IVexArtifactStore.cs +++ b/src/StellaOps.Excititor.Export/IVexArtifactStore.cs @@ -1,28 +1,28 @@ -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Export; - -public sealed record VexExportArtifact( - VexContentAddress ContentAddress, - VexExportFormat Format, - ReadOnlyMemory Content, - IReadOnlyDictionary Metadata); - -public sealed record VexStoredArtifact( - VexContentAddress ContentAddress, - string Location, - long SizeBytes, - IReadOnlyDictionary Metadata); - -public interface IVexArtifactStore -{ - ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken); - - ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); - - ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); -} +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Export; + +public sealed record VexExportArtifact( + VexContentAddress ContentAddress, + VexExportFormat Format, + ReadOnlyMemory Content, + IReadOnlyDictionary Metadata); + +public sealed record VexStoredArtifact( + VexContentAddress ContentAddress, + string Location, + long SizeBytes, + IReadOnlyDictionary Metadata); + +public interface IVexArtifactStore +{ + ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken); + + ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); + + ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Export/OfflineBundleArtifactStore.cs b/src/StellaOps.Excititor.Export/OfflineBundleArtifactStore.cs similarity index 96% rename from src/StellaOps.Vexer.Export/OfflineBundleArtifactStore.cs rename to src/StellaOps.Excititor.Export/OfflineBundleArtifactStore.cs index ba5e771e..088f85b5 100644 --- a/src/StellaOps.Vexer.Export/OfflineBundleArtifactStore.cs +++ b/src/StellaOps.Excititor.Export/OfflineBundleArtifactStore.cs @@ -1,243 +1,243 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.IO; -using System.IO.Abstractions; -using System.IO.Compression; -using System.Security.Cryptography; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Export; - -public sealed class OfflineBundleArtifactStoreOptions -{ - public string RootPath { get; set; } = "."; - - public string ArtifactsFolder { get; set; } = "artifacts"; - - public string BundlesFolder { get; set; } = "bundles"; - - public string ManifestFileName { get; set; } = "offline-manifest.json"; -} - -public sealed class OfflineBundleArtifactStore : IVexArtifactStore -{ - private readonly IFileSystem _fileSystem; - private readonly OfflineBundleArtifactStoreOptions _options; - private readonly ILogger _logger; - private readonly JsonSerializerOptions _serializerOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase, - WriteIndented = true, - }; - - public OfflineBundleArtifactStore( - IOptions options, - ILogger logger, - IFileSystem? fileSystem = null) - { - ArgumentNullException.ThrowIfNull(options); - _options = options.Value; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _fileSystem = fileSystem ?? new FileSystem(); - - if (string.IsNullOrWhiteSpace(_options.RootPath)) - { - throw new ArgumentException("RootPath must be provided for OfflineBundleArtifactStore.", nameof(options)); - } - - var root = _fileSystem.Path.GetFullPath(_options.RootPath); - _fileSystem.Directory.CreateDirectory(root); - _options.RootPath = root; - _fileSystem.Directory.CreateDirectory(GetArtifactsRoot()); - _fileSystem.Directory.CreateDirectory(GetBundlesRoot()); - } - - public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(artifact); - EnforceDigestMatch(artifact); - - var artifactRelativePath = BuildArtifactRelativePath(artifact); - var artifactFullPath = _fileSystem.Path.Combine(_options.RootPath, artifactRelativePath); - var artifactDirectory = _fileSystem.Path.GetDirectoryName(artifactFullPath); - if (!string.IsNullOrEmpty(artifactDirectory)) - { - _fileSystem.Directory.CreateDirectory(artifactDirectory); - } - - await using (var stream = _fileSystem.File.Create(artifactFullPath)) - { - await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); - } - - WriteOfflineBundle(artifactRelativePath, artifact, cancellationToken); - await UpdateManifestAsync(artifactRelativePath, artifact, cancellationToken).ConfigureAwait(false); - - _logger.LogInformation("Stored offline artifact {Digest} at {Path}", artifact.ContentAddress.ToUri(), artifactRelativePath); - - return new VexStoredArtifact( - artifact.ContentAddress, - artifactRelativePath, - artifact.Content.Length, - artifact.Metadata); - } - - public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(contentAddress); - var sanitized = contentAddress.Digest.Replace(':', '_'); - var artifactsRoot = GetArtifactsRoot(); - - foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) - { - var extension = GetExtension(format); - var path = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + extension); - if (_fileSystem.File.Exists(path)) - { - _fileSystem.File.Delete(path); - } - - var bundlePath = _fileSystem.Path.Combine(GetBundlesRoot(), sanitized + ".zip"); - if (_fileSystem.File.Exists(bundlePath)) - { - _fileSystem.File.Delete(bundlePath); - } - } - - return ValueTask.CompletedTask; - } - - public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(contentAddress); - var artifactsRoot = GetArtifactsRoot(); - var sanitized = contentAddress.Digest.Replace(':', '_'); - - foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) - { - var candidate = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); - if (_fileSystem.File.Exists(candidate)) - { - return ValueTask.FromResult(_fileSystem.File.OpenRead(candidate)); - } - } - - return ValueTask.FromResult(null); - } - - private void EnforceDigestMatch(VexExportArtifact artifact) - { - if (!artifact.ContentAddress.Algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) - { - return; - } - - using var sha = SHA256.Create(); - var computed = "sha256:" + Convert.ToHexString(sha.ComputeHash(artifact.Content.ToArray())).ToLowerInvariant(); - if (!string.Equals(computed, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)) - { - throw new InvalidOperationException($"Artifact content digest mismatch. Expected {artifact.ContentAddress.ToUri()} but computed {computed}."); - } - } - - private string BuildArtifactRelativePath(VexExportArtifact artifact) - { - var sanitized = artifact.ContentAddress.Digest.Replace(':', '_'); - var folder = _fileSystem.Path.Combine(_options.ArtifactsFolder, artifact.Format.ToString().ToLowerInvariant()); - return _fileSystem.Path.Combine(folder, sanitized + GetExtension(artifact.Format)); - } - - private void WriteOfflineBundle(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) - { - var zipPath = _fileSystem.Path.Combine(GetBundlesRoot(), artifact.ContentAddress.Digest.Replace(':', '_') + ".zip"); - using var zipStream = _fileSystem.File.Create(zipPath); - using var archive = new ZipArchive(zipStream, ZipArchiveMode.Create); - var entry = archive.CreateEntry(artifactRelativePath, CompressionLevel.Optimal); - using (var entryStream = entry.Open()) - { - entryStream.Write(artifact.Content.Span); - } - - // embed metadata file - var metadataEntry = archive.CreateEntry("metadata.json", CompressionLevel.Optimal); - using var metadataStream = new StreamWriter(metadataEntry.Open()); - var metadata = new Dictionary - { - ["digest"] = artifact.ContentAddress.ToUri(), - ["format"] = artifact.Format.ToString().ToLowerInvariant(), - ["sizeBytes"] = artifact.Content.Length, - ["metadata"] = artifact.Metadata, - }; - metadataStream.Write(JsonSerializer.Serialize(metadata, _serializerOptions)); - } - - private async Task UpdateManifestAsync(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) - { - var manifestPath = _fileSystem.Path.Combine(_options.RootPath, _options.ManifestFileName); - var records = new List(); - - if (_fileSystem.File.Exists(manifestPath)) - { - await using var existingStream = _fileSystem.File.OpenRead(manifestPath); - var existing = await JsonSerializer.DeserializeAsync(existingStream, _serializerOptions, cancellationToken).ConfigureAwait(false); - if (existing is not null) - { - records.AddRange(existing.Artifacts); - } - } - - records.RemoveAll(x => string.Equals(x.Digest, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)); - records.Add(new ManifestEntry( - artifact.ContentAddress.ToUri(), - artifact.Format.ToString().ToLowerInvariant(), - artifactRelativePath.Replace("\\", "/"), - artifact.Content.Length, - artifact.Metadata)); - - records.Sort(static (a, b) => string.CompareOrdinal(a.Digest, b.Digest)); - - var doc = new ManifestDocument(records.ToImmutableArray()); - - await using var stream = _fileSystem.File.Create(manifestPath); - await JsonSerializer.SerializeAsync(stream, doc, _serializerOptions, cancellationToken).ConfigureAwait(false); - } - - private string GetArtifactsRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.ArtifactsFolder); - - private string GetBundlesRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.BundlesFolder); - - private static string GetExtension(VexExportFormat format) - => format switch - { - VexExportFormat.Json => ".json", - VexExportFormat.JsonLines => ".jsonl", - VexExportFormat.OpenVex => ".json", - VexExportFormat.Csaf => ".json", - _ => ".bin", - }; - - private sealed record ManifestDocument(ImmutableArray Artifacts); - - private sealed record ManifestEntry(string Digest, string Format, string Path, long SizeBytes, IReadOnlyDictionary Metadata); -} - -public static class OfflineBundleArtifactStoreServiceCollectionExtensions -{ - public static IServiceCollection AddVexOfflineBundleArtifactStore(this IServiceCollection services, Action? configure = null) - { - if (configure is not null) - { - services.Configure(configure); - } - - services.AddSingleton(); - return services; - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.IO.Abstractions; +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Export; + +public sealed class OfflineBundleArtifactStoreOptions +{ + public string RootPath { get; set; } = "."; + + public string ArtifactsFolder { get; set; } = "artifacts"; + + public string BundlesFolder { get; set; } = "bundles"; + + public string ManifestFileName { get; set; } = "offline-manifest.json"; +} + +public sealed class OfflineBundleArtifactStore : IVexArtifactStore +{ + private readonly IFileSystem _fileSystem; + private readonly OfflineBundleArtifactStoreOptions _options; + private readonly ILogger _logger; + private readonly JsonSerializerOptions _serializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + }; + + public OfflineBundleArtifactStore( + IOptions options, + ILogger logger, + IFileSystem? fileSystem = null) + { + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _fileSystem = fileSystem ?? new FileSystem(); + + if (string.IsNullOrWhiteSpace(_options.RootPath)) + { + throw new ArgumentException("RootPath must be provided for OfflineBundleArtifactStore.", nameof(options)); + } + + var root = _fileSystem.Path.GetFullPath(_options.RootPath); + _fileSystem.Directory.CreateDirectory(root); + _options.RootPath = root; + _fileSystem.Directory.CreateDirectory(GetArtifactsRoot()); + _fileSystem.Directory.CreateDirectory(GetBundlesRoot()); + } + + public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(artifact); + EnforceDigestMatch(artifact); + + var artifactRelativePath = BuildArtifactRelativePath(artifact); + var artifactFullPath = _fileSystem.Path.Combine(_options.RootPath, artifactRelativePath); + var artifactDirectory = _fileSystem.Path.GetDirectoryName(artifactFullPath); + if (!string.IsNullOrEmpty(artifactDirectory)) + { + _fileSystem.Directory.CreateDirectory(artifactDirectory); + } + + await using (var stream = _fileSystem.File.Create(artifactFullPath)) + { + await stream.WriteAsync(artifact.Content, cancellationToken).ConfigureAwait(false); + } + + WriteOfflineBundle(artifactRelativePath, artifact, cancellationToken); + await UpdateManifestAsync(artifactRelativePath, artifact, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Stored offline artifact {Digest} at {Path}", artifact.ContentAddress.ToUri(), artifactRelativePath); + + return new VexStoredArtifact( + artifact.ContentAddress, + artifactRelativePath, + artifact.Content.Length, + artifact.Metadata); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(contentAddress); + var sanitized = contentAddress.Digest.Replace(':', '_'); + var artifactsRoot = GetArtifactsRoot(); + + foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) + { + var extension = GetExtension(format); + var path = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + extension); + if (_fileSystem.File.Exists(path)) + { + _fileSystem.File.Delete(path); + } + + var bundlePath = _fileSystem.Path.Combine(GetBundlesRoot(), sanitized + ".zip"); + if (_fileSystem.File.Exists(bundlePath)) + { + _fileSystem.File.Delete(bundlePath); + } + } + + return ValueTask.CompletedTask; + } + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(contentAddress); + var artifactsRoot = GetArtifactsRoot(); + var sanitized = contentAddress.Digest.Replace(':', '_'); + + foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) + { + var candidate = _fileSystem.Path.Combine(artifactsRoot, format.ToString().ToLowerInvariant(), sanitized + GetExtension(format)); + if (_fileSystem.File.Exists(candidate)) + { + return ValueTask.FromResult(_fileSystem.File.OpenRead(candidate)); + } + } + + return ValueTask.FromResult(null); + } + + private void EnforceDigestMatch(VexExportArtifact artifact) + { + if (!artifact.ContentAddress.Algorithm.Equals("sha256", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + using var sha = SHA256.Create(); + var computed = "sha256:" + Convert.ToHexString(sha.ComputeHash(artifact.Content.ToArray())).ToLowerInvariant(); + if (!string.Equals(computed, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Artifact content digest mismatch. Expected {artifact.ContentAddress.ToUri()} but computed {computed}."); + } + } + + private string BuildArtifactRelativePath(VexExportArtifact artifact) + { + var sanitized = artifact.ContentAddress.Digest.Replace(':', '_'); + var folder = _fileSystem.Path.Combine(_options.ArtifactsFolder, artifact.Format.ToString().ToLowerInvariant()); + return _fileSystem.Path.Combine(folder, sanitized + GetExtension(artifact.Format)); + } + + private void WriteOfflineBundle(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) + { + var zipPath = _fileSystem.Path.Combine(GetBundlesRoot(), artifact.ContentAddress.Digest.Replace(':', '_') + ".zip"); + using var zipStream = _fileSystem.File.Create(zipPath); + using var archive = new ZipArchive(zipStream, ZipArchiveMode.Create); + var entry = archive.CreateEntry(artifactRelativePath, CompressionLevel.Optimal); + using (var entryStream = entry.Open()) + { + entryStream.Write(artifact.Content.Span); + } + + // embed metadata file + var metadataEntry = archive.CreateEntry("metadata.json", CompressionLevel.Optimal); + using var metadataStream = new StreamWriter(metadataEntry.Open()); + var metadata = new Dictionary + { + ["digest"] = artifact.ContentAddress.ToUri(), + ["format"] = artifact.Format.ToString().ToLowerInvariant(), + ["sizeBytes"] = artifact.Content.Length, + ["metadata"] = artifact.Metadata, + }; + metadataStream.Write(JsonSerializer.Serialize(metadata, _serializerOptions)); + } + + private async Task UpdateManifestAsync(string artifactRelativePath, VexExportArtifact artifact, CancellationToken cancellationToken) + { + var manifestPath = _fileSystem.Path.Combine(_options.RootPath, _options.ManifestFileName); + var records = new List(); + + if (_fileSystem.File.Exists(manifestPath)) + { + await using var existingStream = _fileSystem.File.OpenRead(manifestPath); + var existing = await JsonSerializer.DeserializeAsync(existingStream, _serializerOptions, cancellationToken).ConfigureAwait(false); + if (existing is not null) + { + records.AddRange(existing.Artifacts); + } + } + + records.RemoveAll(x => string.Equals(x.Digest, artifact.ContentAddress.ToUri(), StringComparison.OrdinalIgnoreCase)); + records.Add(new ManifestEntry( + artifact.ContentAddress.ToUri(), + artifact.Format.ToString().ToLowerInvariant(), + artifactRelativePath.Replace("\\", "/"), + artifact.Content.Length, + artifact.Metadata)); + + records.Sort(static (a, b) => string.CompareOrdinal(a.Digest, b.Digest)); + + var doc = new ManifestDocument(records.ToImmutableArray()); + + await using var stream = _fileSystem.File.Create(manifestPath); + await JsonSerializer.SerializeAsync(stream, doc, _serializerOptions, cancellationToken).ConfigureAwait(false); + } + + private string GetArtifactsRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.ArtifactsFolder); + + private string GetBundlesRoot() => _fileSystem.Path.Combine(_options.RootPath, _options.BundlesFolder); + + private static string GetExtension(VexExportFormat format) + => format switch + { + VexExportFormat.Json => ".json", + VexExportFormat.JsonLines => ".jsonl", + VexExportFormat.OpenVex => ".json", + VexExportFormat.Csaf => ".json", + _ => ".bin", + }; + + private sealed record ManifestDocument(ImmutableArray Artifacts); + + private sealed record ManifestEntry(string Digest, string Format, string Path, long SizeBytes, IReadOnlyDictionary Metadata); +} + +public static class OfflineBundleArtifactStoreServiceCollectionExtensions +{ + public static IServiceCollection AddVexOfflineBundleArtifactStore(this IServiceCollection services, Action? configure = null) + { + if (configure is not null) + { + services.Configure(configure); + } + + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Excititor.Export/Properties/AssemblyInfo.cs b/src/StellaOps.Excititor.Export/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..9a35fb0e --- /dev/null +++ b/src/StellaOps.Excititor.Export/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Excititor.Export.Tests")] diff --git a/src/StellaOps.Vexer.Export/S3ArtifactStore.cs b/src/StellaOps.Excititor.Export/S3ArtifactStore.cs similarity index 96% rename from src/StellaOps.Vexer.Export/S3ArtifactStore.cs rename to src/StellaOps.Excititor.Export/S3ArtifactStore.cs index 9cd99f89..5cc84280 100644 --- a/src/StellaOps.Vexer.Export/S3ArtifactStore.cs +++ b/src/StellaOps.Excititor.Export/S3ArtifactStore.cs @@ -1,181 +1,181 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Export; - -public sealed class S3ArtifactStoreOptions -{ - public string BucketName { get; set; } = string.Empty; - - public string? Prefix { get; set; } - = null; - - public bool OverwriteExisting { get; set; } - = true; -} - -public interface IS3ArtifactClient -{ - Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken); - - Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken); - - Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken); - - Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken); -} - -public sealed class S3ArtifactStore : IVexArtifactStore -{ - private readonly IS3ArtifactClient _client; - private readonly S3ArtifactStoreOptions _options; - private readonly ILogger _logger; - - public S3ArtifactStore( - IS3ArtifactClient client, - IOptions options, - ILogger logger) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - ArgumentNullException.ThrowIfNull(options); - _options = options.Value; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - - if (string.IsNullOrWhiteSpace(_options.BucketName)) - { - throw new ArgumentException("BucketName must be provided for S3ArtifactStore.", nameof(options)); - } - } - - public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(artifact); - var key = BuildObjectKey(artifact.ContentAddress, artifact.Format); - - if (!_options.OverwriteExisting) - { - var exists = await _client.ObjectExistsAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); - if (exists) - { - _logger.LogInformation("S3 object {Bucket}/{Key} already exists; skipping upload.", _options.BucketName, key); - return new VexStoredArtifact(artifact.ContentAddress, key, artifact.Content.Length, artifact.Metadata); - } - } - - using var contentStream = new MemoryStream(artifact.Content.ToArray()); - await _client.PutObjectAsync( - _options.BucketName, - key, - contentStream, - BuildObjectMetadata(artifact), - cancellationToken).ConfigureAwait(false); - - _logger.LogInformation("Uploaded export artifact {Digest} to {Bucket}/{Key}", artifact.ContentAddress.ToUri(), _options.BucketName, key); - - return new VexStoredArtifact( - artifact.ContentAddress, - key, - artifact.Content.Length, - artifact.Metadata); - } - - public async ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(contentAddress); - foreach (var key in BuildCandidateKeys(contentAddress)) - { - await _client.DeleteObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); - } - _logger.LogInformation("Deleted export artifact {Digest} from {Bucket}", contentAddress.ToUri(), _options.BucketName); - } - - public async ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(contentAddress); - foreach (var key in BuildCandidateKeys(contentAddress)) - { - var stream = await _client.GetObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); - if (stream is not null) - { - return stream; - } - } - - return null; - } - - private string BuildObjectKey(VexContentAddress address, VexExportFormat format) - { - var sanitizedDigest = address.Digest.Replace(':', '_'); - var prefix = string.IsNullOrWhiteSpace(_options.Prefix) ? string.Empty : _options.Prefix.TrimEnd('/') + "/"; - var formatSegment = format.ToString().ToLowerInvariant(); - return $"{prefix}{formatSegment}/{sanitizedDigest}{GetExtension(format)}"; - } - - private IEnumerable BuildCandidateKeys(VexContentAddress address) - { - foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) - { - yield return BuildObjectKey(address, format); - } - - if (!string.IsNullOrWhiteSpace(_options.Prefix)) - { - yield return $"{_options.Prefix.TrimEnd('/')}/{address.Digest.Replace(':', '_')}"; - } - - yield return address.Digest.Replace(':', '_'); - } - - private static IDictionary BuildObjectMetadata(VexExportArtifact artifact) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["vex-format"] = artifact.Format.ToString().ToLowerInvariant(), - ["vex-digest"] = artifact.ContentAddress.ToUri(), - ["content-type"] = artifact.Format switch - { - VexExportFormat.Json => "application/json", - VexExportFormat.JsonLines => "application/json", - VexExportFormat.OpenVex => "application/vnd.openvex+json", - VexExportFormat.Csaf => "application/json", - _ => "application/octet-stream", - }, - }; - - foreach (var kvp in artifact.Metadata) - { - metadata[$"meta-{kvp.Key}"] = kvp.Value; - } - - return metadata; - } - - private static string GetExtension(VexExportFormat format) - => format switch - { - VexExportFormat.Json => ".json", - VexExportFormat.JsonLines => ".jsonl", - VexExportFormat.OpenVex => ".json", - VexExportFormat.Csaf => ".json", - _ => ".bin", - }; -} - -public static class S3ArtifactStoreServiceCollectionExtensions -{ - public static IServiceCollection AddVexS3ArtifactStore(this IServiceCollection services, Action configure) - { - ArgumentNullException.ThrowIfNull(configure); - services.Configure(configure); - services.AddSingleton(); - return services; - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Export; + +public sealed class S3ArtifactStoreOptions +{ + public string BucketName { get; set; } = string.Empty; + + public string? Prefix { get; set; } + = null; + + public bool OverwriteExisting { get; set; } + = true; +} + +public interface IS3ArtifactClient +{ + Task ObjectExistsAsync(string bucketName, string key, CancellationToken cancellationToken); + + Task PutObjectAsync(string bucketName, string key, Stream content, IDictionary metadata, CancellationToken cancellationToken); + + Task GetObjectAsync(string bucketName, string key, CancellationToken cancellationToken); + + Task DeleteObjectAsync(string bucketName, string key, CancellationToken cancellationToken); +} + +public sealed class S3ArtifactStore : IVexArtifactStore +{ + private readonly IS3ArtifactClient _client; + private readonly S3ArtifactStoreOptions _options; + private readonly ILogger _logger; + + public S3ArtifactStore( + IS3ArtifactClient client, + IOptions options, + ILogger logger) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + ArgumentNullException.ThrowIfNull(options); + _options = options.Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (string.IsNullOrWhiteSpace(_options.BucketName)) + { + throw new ArgumentException("BucketName must be provided for S3ArtifactStore.", nameof(options)); + } + } + + public async ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(artifact); + var key = BuildObjectKey(artifact.ContentAddress, artifact.Format); + + if (!_options.OverwriteExisting) + { + var exists = await _client.ObjectExistsAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); + if (exists) + { + _logger.LogInformation("S3 object {Bucket}/{Key} already exists; skipping upload.", _options.BucketName, key); + return new VexStoredArtifact(artifact.ContentAddress, key, artifact.Content.Length, artifact.Metadata); + } + } + + using var contentStream = new MemoryStream(artifact.Content.ToArray()); + await _client.PutObjectAsync( + _options.BucketName, + key, + contentStream, + BuildObjectMetadata(artifact), + cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Uploaded export artifact {Digest} to {Bucket}/{Key}", artifact.ContentAddress.ToUri(), _options.BucketName, key); + + return new VexStoredArtifact( + artifact.ContentAddress, + key, + artifact.Content.Length, + artifact.Metadata); + } + + public async ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(contentAddress); + foreach (var key in BuildCandidateKeys(contentAddress)) + { + await _client.DeleteObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); + } + _logger.LogInformation("Deleted export artifact {Digest} from {Bucket}", contentAddress.ToUri(), _options.BucketName); + } + + public async ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(contentAddress); + foreach (var key in BuildCandidateKeys(contentAddress)) + { + var stream = await _client.GetObjectAsync(_options.BucketName, key, cancellationToken).ConfigureAwait(false); + if (stream is not null) + { + return stream; + } + } + + return null; + } + + private string BuildObjectKey(VexContentAddress address, VexExportFormat format) + { + var sanitizedDigest = address.Digest.Replace(':', '_'); + var prefix = string.IsNullOrWhiteSpace(_options.Prefix) ? string.Empty : _options.Prefix.TrimEnd('/') + "/"; + var formatSegment = format.ToString().ToLowerInvariant(); + return $"{prefix}{formatSegment}/{sanitizedDigest}{GetExtension(format)}"; + } + + private IEnumerable BuildCandidateKeys(VexContentAddress address) + { + foreach (VexExportFormat format in Enum.GetValues(typeof(VexExportFormat))) + { + yield return BuildObjectKey(address, format); + } + + if (!string.IsNullOrWhiteSpace(_options.Prefix)) + { + yield return $"{_options.Prefix.TrimEnd('/')}/{address.Digest.Replace(':', '_')}"; + } + + yield return address.Digest.Replace(':', '_'); + } + + private static IDictionary BuildObjectMetadata(VexExportArtifact artifact) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["vex-format"] = artifact.Format.ToString().ToLowerInvariant(), + ["vex-digest"] = artifact.ContentAddress.ToUri(), + ["content-type"] = artifact.Format switch + { + VexExportFormat.Json => "application/json", + VexExportFormat.JsonLines => "application/json", + VexExportFormat.OpenVex => "application/vnd.openvex+json", + VexExportFormat.Csaf => "application/json", + _ => "application/octet-stream", + }, + }; + + foreach (var kvp in artifact.Metadata) + { + metadata[$"meta-{kvp.Key}"] = kvp.Value; + } + + return metadata; + } + + private static string GetExtension(VexExportFormat format) + => format switch + { + VexExportFormat.Json => ".json", + VexExportFormat.JsonLines => ".jsonl", + VexExportFormat.OpenVex => ".json", + VexExportFormat.Csaf => ".json", + _ => ".bin", + }; +} + +public static class S3ArtifactStoreServiceCollectionExtensions +{ + public static IServiceCollection AddVexS3ArtifactStore(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + services.Configure(configure); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Export/StellaOps.Vexer.Export.csproj b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj similarity index 60% rename from src/StellaOps.Vexer.Export/StellaOps.Vexer.Export.csproj rename to src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj index dc199446..66bdedf8 100644 --- a/src/StellaOps.Vexer.Export/StellaOps.Vexer.Export.csproj +++ b/src/StellaOps.Excititor.Export/StellaOps.Excititor.Export.csproj @@ -1,19 +1,19 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Export/TASKS.md b/src/StellaOps.Excititor.Export/TASKS.md new file mode 100644 index 00000000..1cbe4965 --- /dev/null +++ b/src/StellaOps.Excititor.Export/TASKS.md @@ -0,0 +1,11 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-EXPORT-01-001 – Export engine orchestration|Team Excititor Export|EXCITITOR-CORE-01-003|DONE (2025-10-15) – Export engine scaffolding with cache lookup, data source hooks, and deterministic manifest emission.| +|EXCITITOR-EXPORT-01-002 – Cache index & eviction hooks|Team Excititor Export|EXCITITOR-EXPORT-01-001, EXCITITOR-STORAGE-01-003|**DONE (2025-10-16)** – Export engine now invalidates cache entries on force refresh, cache services expose prune/invalidate APIs, and storage maintenance trims expired/dangling records with Mongo2Go coverage.| +|EXCITITOR-EXPORT-01-003 – Artifact store adapters|Team Excititor Export|EXCITITOR-EXPORT-01-001|**DONE (2025-10-16)** – Implemented multi-store pipeline with filesystem, S3-compatible, and offline bundle adapters (hash verification + manifest/zip output) plus unit coverage and DI hooks.| +|EXCITITOR-EXPORT-01-004 – Attestation handoff integration|Team Excititor Export|EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001|**DONE (2025-10-17)** – Export engine now invokes attestation client, logs diagnostics, and persists Rekor/envelope metadata on manifests; regression coverage added in `ExportEngineTests.ExportAsync_AttachesAttestationMetadata`.| +|EXCITITOR-EXPORT-01-005 – Score & resolve envelope surfaces|Team Excititor Export|EXCITITOR-EXPORT-01-004, EXCITITOR-CORE-02-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-EXPORT-01-004 and EXCITITOR-CORE-02-001 confirmed DONE; planning export updates to emit consensus+score envelopes, include policy/scoring digests, and extend offline bundle/ORAS layouts for signed VEX responses.| +|EXCITITOR-EXPORT-01-006 – Quiet provenance packaging|Team Excititor Export|EXCITITOR-EXPORT-01-005, POLICY-CORE-09-005|TODO – Attach `quietedBy` statement IDs, signers, and justification codes to exports/offline bundles, mirror metadata into attested manifest, and add regression fixtures.| +|EXCITITOR-EXPORT-01-007 – Mirror bundle + domain manifest|Team Excititor Export|EXCITITOR-EXPORT-01-006|TODO – Create per-domain mirror bundles with consensus/score artifacts, publish signed index for downstream Excititor sync, and ensure deterministic digests + fixtures.| diff --git a/src/StellaOps.Vexer.Export/VexExportCacheService.cs b/src/StellaOps.Excititor.Export/VexExportCacheService.cs similarity index 92% rename from src/StellaOps.Vexer.Export/VexExportCacheService.cs rename to src/StellaOps.Excititor.Export/VexExportCacheService.cs index 6cdda3f9..001aa375 100644 --- a/src/StellaOps.Vexer.Export/VexExportCacheService.cs +++ b/src/StellaOps.Excititor.Export/VexExportCacheService.cs @@ -1,54 +1,54 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Storage.Mongo; - -namespace StellaOps.Vexer.Export; - -public interface IVexExportCacheService -{ - ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); - - ValueTask PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken); - - ValueTask PruneDanglingAsync(CancellationToken cancellationToken); -} - -internal sealed class VexExportCacheService : IVexExportCacheService -{ - private readonly IVexCacheIndex _cacheIndex; - private readonly IVexCacheMaintenance _maintenance; - private readonly ILogger _logger; - - public VexExportCacheService( - IVexCacheIndex cacheIndex, - IVexCacheMaintenance maintenance, - ILogger logger) - { - _cacheIndex = cacheIndex ?? throw new ArgumentNullException(nameof(cacheIndex)); - _maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(signature); - await _cacheIndex.RemoveAsync(signature, format, cancellationToken).ConfigureAwait(false); - _logger.LogInformation("Invalidated export cache entry {Signature} ({Format})", signature.Value, format); - } - - public ValueTask PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) - => _maintenance.RemoveExpiredAsync(asOf, cancellationToken); - - public ValueTask PruneDanglingAsync(CancellationToken cancellationToken) - => _maintenance.RemoveMissingManifestReferencesAsync(cancellationToken); -} - -public static class VexExportCacheServiceCollectionExtensions -{ - public static IServiceCollection AddVexExportCacheServices(this IServiceCollection services) - { - services.AddSingleton(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Export; + +public interface IVexExportCacheService +{ + ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); + + ValueTask PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken); + + ValueTask PruneDanglingAsync(CancellationToken cancellationToken); +} + +internal sealed class VexExportCacheService : IVexExportCacheService +{ + private readonly IVexCacheIndex _cacheIndex; + private readonly IVexCacheMaintenance _maintenance; + private readonly ILogger _logger; + + public VexExportCacheService( + IVexCacheIndex cacheIndex, + IVexCacheMaintenance maintenance, + ILogger logger) + { + _cacheIndex = cacheIndex ?? throw new ArgumentNullException(nameof(cacheIndex)); + _maintenance = maintenance ?? throw new ArgumentNullException(nameof(maintenance)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask InvalidateAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(signature); + await _cacheIndex.RemoveAsync(signature, format, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Invalidated export cache entry {Signature} ({Format})", signature.Value, format); + } + + public ValueTask PruneExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) + => _maintenance.RemoveExpiredAsync(asOf, cancellationToken); + + public ValueTask PruneDanglingAsync(CancellationToken cancellationToken) + => _maintenance.RemoveMissingManifestReferencesAsync(cancellationToken); +} + +public static class VexExportCacheServiceCollectionExtensions +{ + public static IServiceCollection AddVexExportCacheServices(this IServiceCollection services) + { + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs b/src/StellaOps.Excititor.Formats.CSAF.Tests/CsafNormalizerTests.cs similarity index 90% rename from src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs rename to src/StellaOps.Excititor.Formats.CSAF.Tests/CsafNormalizerTests.cs index f14a9d61..3f722c8f 100644 --- a/src/StellaOps.Vexer.Formats.CSAF.Tests/CsafNormalizerTests.cs +++ b/src/StellaOps.Excititor.Formats.CSAF.Tests/CsafNormalizerTests.cs @@ -1,131 +1,131 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using System.IO; -using FluentAssertions; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Formats.CSAF; - -namespace StellaOps.Vexer.Formats.CSAF.Tests; - -public sealed class CsafNormalizerTests -{ - [Fact] - public async Task NormalizeAsync_ProducesClaimsPerProductStatus() - { - var json = """ - { - "document": { - "tracking": { - "id": "RHSA-2025:0001", - "version": "3", - "revision": "3", - "status": "final", - "initial_release_date": "2025-10-01T00:00:00Z", - "current_release_date": "2025-10-10T00:00:00Z" - }, - "publisher": { - "name": "Red Hat Product Security", - "category": "vendor" - } - }, - "product_tree": { - "full_product_names": [ - { - "product_id": "CSAFPID-0001", - "name": "Red Hat Enterprise Linux 9", - "product_identification_helper": { - "cpe": "cpe:/o:redhat:enterprise_linux:9", - "purl": "pkg:rpm/redhat/enterprise-linux@9" - } - } - ] - }, - "vulnerabilities": [ - { - "cve": "CVE-2025-0001", - "title": "Kernel vulnerability", - "product_status": { - "known_affected": [ "CSAFPID-0001" ] - } - }, - { - "id": "VEX-0002", - "title": "Library issue", - "product_status": { - "known_not_affected": [ "CSAFPID-0001" ] - } - } - ] - } - """; - - var rawDocument = new VexRawDocument( - ProviderId: "vexer:redhat", - VexDocumentFormat.Csaf, - new Uri("https://example.com/csaf/rhsa-2025-0001.json"), - new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero), - "sha256:dummydigest", - Encoding.UTF8.GetBytes(json), - ImmutableDictionary.Empty); - - var provider = new VexProvider("vexer:redhat", "Red Hat CSAF", VexProviderKind.Distro); - var normalizer = new CsafNormalizer(NullLogger.Instance); - - var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); - - batch.Claims.Should().HaveCount(2); - - var affectedClaim = batch.Claims.First(c => c.VulnerabilityId == "CVE-2025-0001"); - affectedClaim.Status.Should().Be(VexClaimStatus.Affected); - affectedClaim.Product.Key.Should().Be("CSAFPID-0001"); - affectedClaim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); - affectedClaim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); - affectedClaim.Document.Revision.Should().Be("3"); - affectedClaim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero)); - affectedClaim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); - affectedClaim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); - affectedClaim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); - - var notAffectedClaim = batch.Claims.First(c => c.VulnerabilityId == "VEX-0002"); - notAffectedClaim.Status.Should().Be(VexClaimStatus.NotAffected); - } - - [Fact] - public async Task NormalizeAsync_PreservesRedHatSpecificMetadata() - { - var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "rhsa-sample.json"); - var json = await File.ReadAllTextAsync(path); - - var rawDocument = new VexRawDocument( - ProviderId: "vexer:redhat", - VexDocumentFormat.Csaf, - new Uri("https://security.example.com/rhsa-2025-1001.json"), - new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), - "sha256:rhdadigest", - Encoding.UTF8.GetBytes(json), - ImmutableDictionary.Empty); - - var provider = new VexProvider("vexer:redhat", "Red Hat CSAF", VexProviderKind.Distro); - var normalizer = new CsafNormalizer(NullLogger.Instance); - - var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); - batch.Claims.Should().ContainSingle(); - - var claim = batch.Claims[0]; - claim.VulnerabilityId.Should().Be("CVE-2025-1234"); - claim.Status.Should().Be(VexClaimStatus.Affected); - claim.Product.Key.Should().Be("rh-enterprise-linux-9"); - claim.Product.Name.Should().Be("Red Hat Enterprise Linux 9"); - claim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); - claim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); - claim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 12, 0, 0, TimeSpan.Zero)); - claim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 5, 10, 0, 0, TimeSpan.Zero)); - - claim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); - claim.AdditionalMetadata["csaf.tracking.id"].Should().Be("RHSA-2025:1001"); - claim.AdditionalMetadata["csaf.tracking.status"].Should().Be("final"); - claim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); - } -} +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.IO; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Formats.CSAF; + +namespace StellaOps.Excititor.Formats.CSAF.Tests; + +public sealed class CsafNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_ProducesClaimsPerProductStatus() + { + var json = """ + { + "document": { + "tracking": { + "id": "RHSA-2025:0001", + "version": "3", + "revision": "3", + "status": "final", + "initial_release_date": "2025-10-01T00:00:00Z", + "current_release_date": "2025-10-10T00:00:00Z" + }, + "publisher": { + "name": "Red Hat Product Security", + "category": "vendor" + } + }, + "product_tree": { + "full_product_names": [ + { + "product_id": "CSAFPID-0001", + "name": "Red Hat Enterprise Linux 9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9", + "purl": "pkg:rpm/redhat/enterprise-linux@9" + } + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-0001", + "title": "Kernel vulnerability", + "product_status": { + "known_affected": [ "CSAFPID-0001" ] + } + }, + { + "id": "VEX-0002", + "title": "Library issue", + "product_status": { + "known_not_affected": [ "CSAFPID-0001" ] + } + } + ] + } + """; + + var rawDocument = new VexRawDocument( + ProviderId: "excititor:redhat", + VexDocumentFormat.Csaf, + new Uri("https://example.com/csaf/rhsa-2025-0001.json"), + new DateTimeOffset(2025, 10, 11, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("excititor:redhat", "Red Hat CSAF", VexProviderKind.Distro); + var normalizer = new CsafNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var affectedClaim = batch.Claims.First(c => c.VulnerabilityId == "CVE-2025-0001"); + affectedClaim.Status.Should().Be(VexClaimStatus.Affected); + affectedClaim.Product.Key.Should().Be("CSAFPID-0001"); + affectedClaim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); + affectedClaim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); + affectedClaim.Document.Revision.Should().Be("3"); + affectedClaim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero)); + affectedClaim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 10, 0, 0, 0, TimeSpan.Zero)); + affectedClaim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); + affectedClaim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); + + var notAffectedClaim = batch.Claims.First(c => c.VulnerabilityId == "VEX-0002"); + notAffectedClaim.Status.Should().Be(VexClaimStatus.NotAffected); + } + + [Fact] + public async Task NormalizeAsync_PreservesRedHatSpecificMetadata() + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", "rhsa-sample.json"); + var json = await File.ReadAllTextAsync(path); + + var rawDocument = new VexRawDocument( + ProviderId: "excititor:redhat", + VexDocumentFormat.Csaf, + new Uri("https://security.example.com/rhsa-2025-1001.json"), + new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), + "sha256:rhdadigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("excititor:redhat", "Red Hat CSAF", VexProviderKind.Distro); + var normalizer = new CsafNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + batch.Claims.Should().ContainSingle(); + + var claim = batch.Claims[0]; + claim.VulnerabilityId.Should().Be("CVE-2025-1234"); + claim.Status.Should().Be(VexClaimStatus.Affected); + claim.Product.Key.Should().Be("rh-enterprise-linux-9"); + claim.Product.Name.Should().Be("Red Hat Enterprise Linux 9"); + claim.Product.Purl.Should().Be("pkg:rpm/redhat/enterprise-linux@9"); + claim.Product.Cpe.Should().Be("cpe:/o:redhat:enterprise_linux:9"); + claim.FirstSeen.Should().Be(new DateTimeOffset(2025, 10, 1, 12, 0, 0, TimeSpan.Zero)); + claim.LastSeen.Should().Be(new DateTimeOffset(2025, 10, 5, 10, 0, 0, TimeSpan.Zero)); + + claim.AdditionalMetadata.Should().ContainKey("csaf.tracking.id"); + claim.AdditionalMetadata["csaf.tracking.id"].Should().Be("RHSA-2025:1001"); + claim.AdditionalMetadata["csaf.tracking.status"].Should().Be("final"); + claim.AdditionalMetadata["csaf.publisher.name"].Should().Be("Red Hat Product Security"); + } +} diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json b/src/StellaOps.Excititor.Formats.CSAF.Tests/Fixtures/rhsa-sample.json similarity index 95% rename from src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json rename to src/StellaOps.Excititor.Formats.CSAF.Tests/Fixtures/rhsa-sample.json index 65be9b12..60047452 100644 --- a/src/StellaOps.Vexer.Formats.CSAF.Tests/Fixtures/rhsa-sample.json +++ b/src/StellaOps.Excititor.Formats.CSAF.Tests/Fixtures/rhsa-sample.json @@ -1,47 +1,47 @@ -{ - "document": { - "publisher": { - "name": "Red Hat Product Security", - "category": "vendor" - }, - "tracking": { - "id": "RHSA-2025:1001", - "status": "final", - "version": "3", - "initial_release_date": "2025-10-01T12:00:00Z", - "current_release_date": "2025-10-05T10:00:00Z" - } - }, - "product_tree": { - "full_product_names": [ - { - "product_id": "rh-enterprise-linux-9", - "name": "Red Hat Enterprise Linux 9", - "product_identification_helper": { - "cpe": "cpe:/o:redhat:enterprise_linux:9", - "purl": "pkg:rpm/redhat/enterprise-linux@9" - } - } - ], - "branches": [ - { - "name": "Red Hat Enterprise Linux", - "product": { - "product_id": "rh-enterprise-linux-9", - "name": "Red Hat Enterprise Linux 9" - } - } - ] - }, - "vulnerabilities": [ - { - "cve": "CVE-2025-1234", - "title": "Kernel privilege escalation", - "product_status": { - "known_affected": [ - "rh-enterprise-linux-9" - ] - } - } - ] -} +{ + "document": { + "publisher": { + "name": "Red Hat Product Security", + "category": "vendor" + }, + "tracking": { + "id": "RHSA-2025:1001", + "status": "final", + "version": "3", + "initial_release_date": "2025-10-01T12:00:00Z", + "current_release_date": "2025-10-05T10:00:00Z" + } + }, + "product_tree": { + "full_product_names": [ + { + "product_id": "rh-enterprise-linux-9", + "name": "Red Hat Enterprise Linux 9", + "product_identification_helper": { + "cpe": "cpe:/o:redhat:enterprise_linux:9", + "purl": "pkg:rpm/redhat/enterprise-linux@9" + } + } + ], + "branches": [ + { + "name": "Red Hat Enterprise Linux", + "product": { + "product_id": "rh-enterprise-linux-9", + "name": "Red Hat Enterprise Linux 9" + } + } + ] + }, + "vulnerabilities": [ + { + "cve": "CVE-2025-1234", + "title": "Kernel privilege escalation", + "product_status": { + "known_affected": [ + "rh-enterprise-linux-9" + ] + } + } + ] +} diff --git a/src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj b/src/StellaOps.Excititor.Formats.CSAF.Tests/StellaOps.Excititor.Formats.CSAF.Tests.csproj similarity index 68% rename from src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj rename to src/StellaOps.Excititor.Formats.CSAF.Tests/StellaOps.Excititor.Formats.CSAF.Tests.csproj index 013f6a75..7654565d 100644 --- a/src/StellaOps.Vexer.Formats.CSAF.Tests/StellaOps.Vexer.Formats.CSAF.Tests.csproj +++ b/src/StellaOps.Excititor.Formats.CSAF.Tests/StellaOps.Excititor.Formats.CSAF.Tests.csproj @@ -1,20 +1,20 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CSAF/AGENTS.md b/src/StellaOps.Excititor.Formats.CSAF/AGENTS.md similarity index 88% rename from src/StellaOps.Vexer.Formats.CSAF/AGENTS.md rename to src/StellaOps.Excititor.Formats.CSAF/AGENTS.md index 67a3913c..94174806 100644 --- a/src/StellaOps.Vexer.Formats.CSAF/AGENTS.md +++ b/src/StellaOps.Excititor.Formats.CSAF/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Normalize CSAF VEX profile documents into Vexer claims and provide CSAF export adapters. -## Scope -- CSAF ingestion helpers: provider metadata parsing, document revision handling, vulnerability/action mappings. -- Normalizer implementation fulfilling `INormalizer` for CSAF sources (Red Hat, Cisco, SUSE, MSRC, Oracle, Ubuntu). -- Export adapters producing CSAF-compliant output slices from consensus data. -- Schema/version compatibility checks (CSAF 2.0 profile validation). -## Participants -- Connectors deliver raw CSAF documents to this module for normalization. -- Export module leverages adapters when producing CSAF exports. -- Policy engine consumes normalized justification/status fields for gating. -## Interfaces & contracts -- Parser/normalizer classes, helper utilities for `product_tree`, `vulnerabilities`, and `notes`. -- Export writer interfaces for per-provider/per-product CSAF packaging. -## In/Out of scope -In: CSAF parsing/normalization/export, schema validation, mapping to canonical claims. -Out: HTTP fetching (connectors), storage persistence, attestation logic. -## Observability & security expectations -- Emit structured diagnostics when CSAF documents fail schema validation, including source URI and revision. -- Provide counters for normalization outcomes (status distribution, justification coverage). -## Tests -- Fixture-driven parsing/export tests will live in `../StellaOps.Vexer.Formats.CSAF.Tests` using real CSAF samples. +# AGENTS +## Role +Normalize CSAF VEX profile documents into Excititor claims and provide CSAF export adapters. +## Scope +- CSAF ingestion helpers: provider metadata parsing, document revision handling, vulnerability/action mappings. +- Normalizer implementation fulfilling `INormalizer` for CSAF sources (Red Hat, Cisco, SUSE, MSRC, Oracle, Ubuntu). +- Export adapters producing CSAF-compliant output slices from consensus data. +- Schema/version compatibility checks (CSAF 2.0 profile validation). +## Participants +- Connectors deliver raw CSAF documents to this module for normalization. +- Export module leverages adapters when producing CSAF exports. +- Policy engine consumes normalized justification/status fields for gating. +## Interfaces & contracts +- Parser/normalizer classes, helper utilities for `product_tree`, `vulnerabilities`, and `notes`. +- Export writer interfaces for per-provider/per-product CSAF packaging. +## In/Out of scope +In: CSAF parsing/normalization/export, schema validation, mapping to canonical claims. +Out: HTTP fetching (connectors), storage persistence, attestation logic. +## Observability & security expectations +- Emit structured diagnostics when CSAF documents fail schema validation, including source URI and revision. +- Provide counters for normalization outcomes (status distribution, justification coverage). +## Tests +- Fixture-driven parsing/export tests will live in `../StellaOps.Excititor.Formats.CSAF.Tests` using real CSAF samples. diff --git a/src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs b/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs similarity index 51% rename from src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs rename to src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs index 342ed295..11cb0978 100644 --- a/src/StellaOps.Vexer.Formats.CSAF/CsafNormalizer.cs +++ b/src/StellaOps.Excititor.Formats.CSAF/CsafNormalizer.cs @@ -1,532 +1,875 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.CSAF; - -public sealed class CsafNormalizer : IVexNormalizer -{ - private static readonly ImmutableDictionary StatusPrecedence = new Dictionary - { - [VexClaimStatus.UnderInvestigation] = 0, - [VexClaimStatus.Affected] = 1, - [VexClaimStatus.NotAffected] = 2, - [VexClaimStatus.Fixed] = 3, - }.ToImmutableDictionary(); - - private readonly ILogger _logger; - - public CsafNormalizer(ILogger logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); - - public bool CanHandle(VexRawDocument document) - => document is not null && document.Format == VexDocumentFormat.Csaf; - - public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(provider); - - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var result = CsafParser.Parse(document); - var claims = ImmutableArray.CreateBuilder(result.Claims.Length); - foreach (var entry in result.Claims) - { - var product = new VexProduct( - entry.Product.ProductId, - entry.Product.Name, - entry.Product.Version, - entry.Product.Purl, - entry.Product.Cpe); - - var claimDocument = new VexClaimDocument( - VexDocumentFormat.Csaf, - document.Digest, - document.SourceUri, - result.Revision, - signature: null); - - var claim = new VexClaim( - entry.VulnerabilityId, - provider.Id, - product, - entry.Status, - claimDocument, - result.FirstRelease, - result.LastRelease, - justification: null, - detail: entry.Detail, - confidence: null, - additionalMetadata: result.Metadata); - - claims.Add(claim); - } - - var orderedClaims = claims - .ToImmutable() - .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) - .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) - .ToImmutableArray(); - - _logger.LogInformation( - "Normalized CSAF document {Source} into {ClaimCount} claim(s).", - document.SourceUri, - orderedClaims.Length); - - return ValueTask.FromResult(new VexClaimBatch( - document, - orderedClaims, - ImmutableDictionary.Empty)); - } - catch (JsonException ex) - { - _logger.LogError(ex, "Failed to parse CSAF document {SourceUri}", document.SourceUri); - throw; - } - } - - private static class CsafParser - { - public static CsafParseResult Parse(VexRawDocument document) - { - using var json = JsonDocument.Parse(document.Content.ToArray()); - var root = json.RootElement; - - var tracking = root.TryGetProperty("document", out var documentElement) && - documentElement.ValueKind == JsonValueKind.Object && - documentElement.TryGetProperty("tracking", out var trackingElement) - ? trackingElement - : default; - - var firstRelease = ParseDate(tracking, "initial_release_date") ?? document.RetrievedAt; - var lastRelease = ParseDate(tracking, "current_release_date") ?? firstRelease; - - if (lastRelease < firstRelease) - { - lastRelease = firstRelease; - } - - var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - AddIfPresent(metadataBuilder, tracking, "id", "csaf.tracking.id"); - AddIfPresent(metadataBuilder, tracking, "version", "csaf.tracking.version"); - AddIfPresent(metadataBuilder, tracking, "status", "csaf.tracking.status"); - AddPublisherMetadata(metadataBuilder, documentElement); - - var revision = TryGetString(tracking, "revision"); - - var productCatalog = CollectProducts(root); - var claimsBuilder = ImmutableArray.CreateBuilder(); - - if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && - vulnerabilitiesElement.ValueKind == JsonValueKind.Array) - { - foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) - { - var vulnerabilityId = ResolveVulnerabilityId(vulnerability); - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - continue; - } - - var detail = ResolveDetail(vulnerability); - var productClaims = BuildClaimsForVulnerability( - vulnerabilityId, - vulnerability, - productCatalog, - detail); - - claimsBuilder.AddRange(productClaims); - } - } - - return new CsafParseResult( - firstRelease, - lastRelease, - revision, - metadataBuilder.ToImmutable(), - claimsBuilder.ToImmutable()); - } - - private static IReadOnlyList BuildClaimsForVulnerability( - string vulnerabilityId, - JsonElement vulnerability, - IReadOnlyDictionary productCatalog, - string? detail) - { - if (!vulnerability.TryGetProperty("product_status", out var statusElement) || - statusElement.ValueKind != JsonValueKind.Object) - { - return Array.Empty(); - } - - var claims = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var statusProperty in statusElement.EnumerateObject()) - { - var status = MapStatus(statusProperty.Name); - if (status is null) - { - continue; - } - - if (statusProperty.Value.ValueKind != JsonValueKind.Array) - { - continue; - } - - foreach (var productIdElement in statusProperty.Value.EnumerateArray()) - { - var productId = productIdElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - continue; - } - - var product = ResolveProduct(productCatalog, productId); - UpdateClaim(claims, product, status.Value, detail); - } - } - - if (claims.Count == 0) - { - return Array.Empty(); - } - - return claims.Values - .Select(builder => new CsafClaimEntry( - vulnerabilityId, - builder.Product, - builder.Status, - builder.Detail)) - .ToArray(); - } - - private static void UpdateClaim( - IDictionary claims, - CsafProductInfo product, - VexClaimStatus status, - string? detail) - { - if (!claims.TryGetValue(product.ProductId, out var existing) || - StatusPrecedence[status] > StatusPrecedence[existing.Status]) - { - claims[product.ProductId] = new CsafClaimEntryBuilder(product, status, detail); - } - } - - private static CsafProductInfo ResolveProduct( - IReadOnlyDictionary catalog, - string productId) - { - if (catalog.TryGetValue(productId, out var product)) - { - return product; - } - - return new CsafProductInfo(productId, productId, null, null, null); - } - - private static string ResolveVulnerabilityId(JsonElement vulnerability) - { - var id = TryGetString(vulnerability, "cve") - ?? TryGetString(vulnerability, "id") - ?? TryGetString(vulnerability, "vuln_id"); - - return string.IsNullOrWhiteSpace(id) ? string.Empty : id.Trim(); - } - - private static string? ResolveDetail(JsonElement vulnerability) - { - var title = TryGetString(vulnerability, "title"); - if (!string.IsNullOrWhiteSpace(title)) - { - return title.Trim(); - } - - if (vulnerability.TryGetProperty("notes", out var notesElement) && - notesElement.ValueKind == JsonValueKind.Array) - { - foreach (var note in notesElement.EnumerateArray()) - { - if (note.ValueKind != JsonValueKind.Object) - { - continue; - } - - var category = TryGetString(note, "category"); - if (!string.IsNullOrWhiteSpace(category) && - !string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var text = TryGetString(note, "text"); - if (!string.IsNullOrWhiteSpace(text)) - { - return text.Trim(); - } - } - } - - return null; - } - - private static Dictionary CollectProducts(JsonElement root) - { - var products = new Dictionary(StringComparer.OrdinalIgnoreCase); - - if (!root.TryGetProperty("product_tree", out var productTree) || - productTree.ValueKind != JsonValueKind.Object) - { - return products; - } - - if (productTree.TryGetProperty("full_product_names", out var fullNames) && - fullNames.ValueKind == JsonValueKind.Array) - { - foreach (var productEntry in fullNames.EnumerateArray()) - { - var product = ParseProduct(productEntry, parentBranchName: null); - if (product is not null) - { - AddOrUpdate(product); - } - } - } - - if (productTree.TryGetProperty("branches", out var branches) && - branches.ValueKind == JsonValueKind.Array) - { - foreach (var branch in branches.EnumerateArray()) - { - VisitBranch(branch, parentBranchName: null); - } - } - - return products; - - void VisitBranch(JsonElement branch, string? parentBranchName) - { - if (branch.ValueKind != JsonValueKind.Object) - { - return; - } - - var branchName = TryGetString(branch, "name") ?? parentBranchName; - - if (branch.TryGetProperty("product", out var productElement)) - { - var product = ParseProduct(productElement, branchName); - if (product is not null) - { - AddOrUpdate(product); - } - } - - if (branch.TryGetProperty("branches", out var childBranches) && - childBranches.ValueKind == JsonValueKind.Array) - { - foreach (var childBranch in childBranches.EnumerateArray()) - { - VisitBranch(childBranch, branchName); - } - } - } - - void AddOrUpdate(CsafProductInfo product) - { - if (products.TryGetValue(product.ProductId, out var existing)) - { - products[product.ProductId] = MergeProducts(existing, product); - } - else - { - products[product.ProductId] = product; - } - } - - static CsafProductInfo MergeProducts(CsafProductInfo existing, CsafProductInfo incoming) - { - static string ChooseName(string incoming, string fallback) - => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; - - static string? ChooseOptional(string? incoming, string? fallback) - => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; - - return new CsafProductInfo( - existing.ProductId, - ChooseName(incoming.Name, existing.Name), - ChooseOptional(incoming.Version, existing.Version), - ChooseOptional(incoming.Purl, existing.Purl), - ChooseOptional(incoming.Cpe, existing.Cpe)); - } - } - - private static CsafProductInfo? ParseProduct(JsonElement element, string? parentBranchName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - JsonElement productElement = element; - if (!element.TryGetProperty("product_id", out var idElement) && - element.TryGetProperty("product", out var nestedProduct) && - nestedProduct.ValueKind == JsonValueKind.Object && - nestedProduct.TryGetProperty("product_id", out idElement)) - { - productElement = nestedProduct; - } - - var productId = idElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - return null; - } - - var name = TryGetString(productElement, "name") - ?? TryGetString(element, "name") - ?? parentBranchName - ?? productId; - - var version = TryGetString(productElement, "product_version") - ?? TryGetString(productElement, "version") - ?? TryGetString(element, "product_version"); - - string? cpe = null; - string? purl = null; - if (productElement.TryGetProperty("product_identification_helper", out var helper) && - helper.ValueKind == JsonValueKind.Object) - { - cpe = TryGetString(helper, "cpe"); - purl = TryGetString(helper, "purl"); - } - - return new CsafProductInfo(productId.Trim(), name.Trim(), version?.Trim(), purl?.Trim(), cpe?.Trim()); - } - - private static VexClaimStatus? MapStatus(string statusName) - { - if (string.IsNullOrWhiteSpace(statusName)) - { - return null; - } - - return statusName switch - { - "known_affected" or "fixed_after_release" or "first_affected" or "last_affected" => VexClaimStatus.Affected, - "known_not_affected" or "last_not_affected" or "first_not_affected" => VexClaimStatus.NotAffected, - "fixed" or "first_fixed" or "last_fixed" => VexClaimStatus.Fixed, - "under_investigation" or "investigating" => VexClaimStatus.UnderInvestigation, - _ => null, - }; - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var dateElement)) - { - return null; - } - - var value = dateElement.GetString(); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, out var parsed)) - { - return parsed; - } - - return null; - } - - private static string? TryGetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var property)) - { - return null; - } - - return property.ValueKind == JsonValueKind.String ? property.GetString() : null; - } - - private static void AddIfPresent( - ImmutableDictionary.Builder builder, - JsonElement element, - string propertyName, - string metadataKey) - { - var value = TryGetString(element, propertyName); - if (!string.IsNullOrWhiteSpace(value)) - { - builder[metadataKey] = value.Trim(); - } - } - - private static void AddPublisherMetadata( - ImmutableDictionary.Builder builder, - JsonElement documentElement) - { - if (documentElement.ValueKind != JsonValueKind.Object || - !documentElement.TryGetProperty("publisher", out var publisher) || - publisher.ValueKind != JsonValueKind.Object) - { - return; - } - - AddIfPresent(builder, publisher, "name", "csaf.publisher.name"); - AddIfPresent(builder, publisher, "category", "csaf.publisher.category"); - } - - private readonly record struct CsafClaimEntryBuilder( - CsafProductInfo Product, - VexClaimStatus Status, - string? Detail); - } - - private sealed record CsafParseResult( - DateTimeOffset FirstRelease, - DateTimeOffset LastRelease, - string? Revision, - ImmutableDictionary Metadata, - ImmutableArray Claims); - - private sealed record CsafClaimEntry( - string VulnerabilityId, - CsafProductInfo Product, - VexClaimStatus Status, - string? Detail); - - private sealed record CsafProductInfo( - string ProductId, - string Name, - string? Version, - string? Purl, - string? Cpe); -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.CSAF; + +public sealed class CsafNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StatusPrecedence = new Dictionary + { + [VexClaimStatus.UnderInvestigation] = 0, + [VexClaimStatus.Affected] = 1, + [VexClaimStatus.NotAffected] = 2, + [VexClaimStatus.Fixed] = 3, + }.ToImmutableDictionary(); + + private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["known_affected"] = VexClaimStatus.Affected, + ["first_affected"] = VexClaimStatus.Affected, + ["last_affected"] = VexClaimStatus.Affected, + ["affected"] = VexClaimStatus.Affected, + ["fixed_after_release"] = VexClaimStatus.Fixed, + ["fixed"] = VexClaimStatus.Fixed, + ["first_fixed"] = VexClaimStatus.Fixed, + ["last_fixed"] = VexClaimStatus.Fixed, + ["recommended"] = VexClaimStatus.Fixed, + ["known_not_affected"] = VexClaimStatus.NotAffected, + ["first_not_affected"] = VexClaimStatus.NotAffected, + ["last_not_affected"] = VexClaimStatus.NotAffected, + ["not_affected"] = VexClaimStatus.NotAffected, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + ["investigating"] = VexClaimStatus.UnderInvestigation, + ["in_investigation"] = VexClaimStatus.UnderInvestigation, + ["in_triage"] = VexClaimStatus.UnderInvestigation, + ["unknown"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["protected_at_runtime"] = VexJustification.ProtectedAtRuntime, + ["protected_at_perimeter"] = VexJustification.ProtectedAtPerimeter, + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public CsafNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.Csaf.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.Csaf; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = CsafParser.Parse(document); + var claims = ImmutableArray.CreateBuilder(result.Claims.Length); + foreach (var entry in result.Claims) + { + var product = new VexProduct( + entry.Product.ProductId, + entry.Product.Name, + entry.Product.Version, + entry.Product.Purl, + entry.Product.Cpe); + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.Csaf, + document.Digest, + document.SourceUri, + result.Revision, + signature: null); + + var metadata = result.Metadata; + if (!string.IsNullOrWhiteSpace(entry.RawStatus)) + { + metadata = metadata.SetItem("csaf.product_status.raw", entry.RawStatus); + } + + if (!string.IsNullOrWhiteSpace(entry.RawJustification)) + { + metadata = metadata.SetItem("csaf.justification.label", entry.RawJustification); + } + + var claim = new VexClaim( + entry.VulnerabilityId, + provider.Id, + product, + entry.Status, + claimDocument, + result.FirstRelease, + result.LastRelease, + entry.Justification, + detail: entry.Detail, + confidence: null, + additionalMetadata: metadata); + + claims.Add(claim); + } + + var orderedClaims = claims + .ToImmutable() + .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized CSAF document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + var diagnosticsBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + if (!result.UnsupportedStatuses.IsDefaultOrEmpty && result.UnsupportedStatuses.Length > 0) + { + diagnosticsBuilder["csaf.unsupported_statuses"] = string.Join(",", result.UnsupportedStatuses); + } + + if (!result.UnsupportedJustifications.IsDefaultOrEmpty && result.UnsupportedJustifications.Length > 0) + { + diagnosticsBuilder["csaf.unsupported_justifications"] = string.Join(",", result.UnsupportedJustifications); + } + + if (!result.ConflictingJustifications.IsDefaultOrEmpty && result.ConflictingJustifications.Length > 0) + { + diagnosticsBuilder["csaf.justification_conflicts"] = string.Join(",", result.ConflictingJustifications); + } + + var diagnostics = diagnosticsBuilder.Count == 0 + ? ImmutableDictionary.Empty + : diagnosticsBuilder.ToImmutable(); + + return ValueTask.FromResult(new VexClaimBatch(document, orderedClaims, diagnostics)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse CSAF document {SourceUri}", document.SourceUri); + throw; + } + } + + private static class CsafParser + { + public static CsafParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var tracking = root.TryGetProperty("document", out var documentElement) && + documentElement.ValueKind == JsonValueKind.Object && + documentElement.TryGetProperty("tracking", out var trackingElement) + ? trackingElement + : default; + + var firstRelease = ParseDate(tracking, "initial_release_date") ?? document.RetrievedAt; + var lastRelease = ParseDate(tracking, "current_release_date") ?? firstRelease; + + if (lastRelease < firstRelease) + { + lastRelease = firstRelease; + } + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + AddIfPresent(metadataBuilder, tracking, "id", "csaf.tracking.id"); + AddIfPresent(metadataBuilder, tracking, "version", "csaf.tracking.version"); + AddIfPresent(metadataBuilder, tracking, "status", "csaf.tracking.status"); + AddPublisherMetadata(metadataBuilder, documentElement); + + var revision = TryGetString(tracking, "revision"); + + var productCatalog = CollectProducts(root); + var productGroups = CollectProductGroups(root); + + var unsupportedStatuses = new HashSet(StringComparer.OrdinalIgnoreCase); + var unsupportedJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); + var conflictingJustifications = new HashSet(StringComparer.OrdinalIgnoreCase); + + var claimsBuilder = ImmutableArray.CreateBuilder(); + + if (root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) && + vulnerabilitiesElement.ValueKind == JsonValueKind.Array) + { + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + var vulnerabilityId = ResolveVulnerabilityId(vulnerability); + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + continue; + } + + var detail = ResolveDetail(vulnerability); + var justifications = CollectJustifications( + vulnerability, + productCatalog, + productGroups, + unsupportedJustifications, + conflictingJustifications); + + var productClaims = BuildClaimsForVulnerability( + vulnerabilityId, + vulnerability, + productCatalog, + justifications, + detail, + unsupportedStatuses); + + claimsBuilder.AddRange(productClaims); + } + } + + return new CsafParseResult( + firstRelease, + lastRelease, + revision, + metadataBuilder.ToImmutable(), + claimsBuilder.ToImmutable(), + unsupportedStatuses.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), + unsupportedJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray(), + conflictingJustifications.OrderBy(static s => s, StringComparer.OrdinalIgnoreCase).ToImmutableArray()); + } + + private static IReadOnlyList BuildClaimsForVulnerability( + string vulnerabilityId, + JsonElement vulnerability, + IReadOnlyDictionary productCatalog, + ImmutableDictionary justifications, + string? detail, + ISet unsupportedStatuses) + { + if (!vulnerability.TryGetProperty("product_status", out var statusElement) || + statusElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var claims = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var statusProperty in statusElement.EnumerateObject()) + { + var status = MapStatus(statusProperty.Name, unsupportedStatuses); + if (status is null) + { + continue; + } + + if (statusProperty.Value.ValueKind != JsonValueKind.Array) + { + continue; + } + + foreach (var productIdElement in statusProperty.Value.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + var trimmedProductId = productId.Trim(); + var product = ResolveProduct(productCatalog, trimmedProductId); + justifications.TryGetValue(trimmedProductId, out var justificationInfo); + + UpdateClaim(claims, product, status.Value, statusProperty.Name, detail, justificationInfo); + } + } + + if (claims.Count == 0) + { + return Array.Empty(); + } + + return claims.Values + .Select(builder => new CsafClaimEntry( + vulnerabilityId, + builder.Product, + builder.Status, + builder.RawStatus, + builder.Detail, + builder.Justification, + builder.RawJustification)) + .ToArray(); + } + + private static void UpdateClaim( + IDictionary claims, + CsafProductInfo product, + VexClaimStatus status, + string rawStatus, + string? detail, + CsafJustificationInfo? justification) + { + if (!claims.TryGetValue(product.ProductId, out var existing) || + StatusPrecedence[status] > StatusPrecedence[existing.Status]) + { + claims[product.ProductId] = new CsafClaimEntryBuilder( + product, + status, + NormalizeRaw(rawStatus), + detail, + justification?.Normalized, + justification?.RawValue); + return; + } + + if (StatusPrecedence[status] < StatusPrecedence[existing.Status]) + { + return; + } + + var updated = existing; + + if (string.IsNullOrWhiteSpace(existing.RawStatus)) + { + updated = updated with { RawStatus = NormalizeRaw(rawStatus) }; + } + + if (existing.Detail is null && detail is not null) + { + updated = updated with { Detail = detail }; + } + + if (justification is not null) + { + if (existing.Justification is null && justification.Normalized is not null) + { + updated = updated with + { + Justification = justification.Normalized, + RawJustification = justification.RawValue + }; + } + else if (existing.Justification is null && + justification.Normalized is null && + string.IsNullOrWhiteSpace(existing.RawJustification) && + !string.IsNullOrWhiteSpace(justification.RawValue)) + { + updated = updated with { RawJustification = justification.RawValue }; + } + } + + claims[product.ProductId] = updated; + } + + private static CsafProductInfo ResolveProduct( + IReadOnlyDictionary catalog, + string productId) + { + if (catalog.TryGetValue(productId, out var product)) + { + return product; + } + + return new CsafProductInfo(productId, productId, null, null, null); + } + + private static string ResolveVulnerabilityId(JsonElement vulnerability) + { + var id = TryGetString(vulnerability, "cve") + ?? TryGetString(vulnerability, "id") + ?? TryGetString(vulnerability, "vuln_id"); + + return string.IsNullOrWhiteSpace(id) ? string.Empty : id.Trim(); + } + + private static string? ResolveDetail(JsonElement vulnerability) + { + var title = TryGetString(vulnerability, "title"); + if (!string.IsNullOrWhiteSpace(title)) + { + return title.Trim(); + } + + if (vulnerability.TryGetProperty("notes", out var notesElement) && + notesElement.ValueKind == JsonValueKind.Array) + { + foreach (var note in notesElement.EnumerateArray()) + { + if (note.ValueKind != JsonValueKind.Object) + { + continue; + } + + var category = TryGetString(note, "category"); + if (!string.IsNullOrWhiteSpace(category) && + !string.Equals(category, "description", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var text = TryGetString(note, "text"); + if (!string.IsNullOrWhiteSpace(text)) + { + return text.Trim(); + } + } + } + + return null; + } + + private static Dictionary CollectProducts(JsonElement root) + { + var products = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (!root.TryGetProperty("product_tree", out var productTree) || + productTree.ValueKind != JsonValueKind.Object) + { + return products; + } + + if (productTree.TryGetProperty("full_product_names", out var fullNames) && + fullNames.ValueKind == JsonValueKind.Array) + { + foreach (var productEntry in fullNames.EnumerateArray()) + { + var product = ParseProduct(productEntry, parentBranchName: null); + if (product is not null) + { + AddOrUpdate(product); + } + } + } + + if (productTree.TryGetProperty("branches", out var branches) && + branches.ValueKind == JsonValueKind.Array) + { + foreach (var branch in branches.EnumerateArray()) + { + VisitBranch(branch, parentBranchName: null); + } + } + + return products; + + void VisitBranch(JsonElement branch, string? parentBranchName) + { + if (branch.ValueKind != JsonValueKind.Object) + { + return; + } + + var branchName = TryGetString(branch, "name") ?? parentBranchName; + + if (branch.TryGetProperty("product", out var productElement)) + { + var product = ParseProduct(productElement, branchName); + if (product is not null) + { + AddOrUpdate(product); + } + } + + if (branch.TryGetProperty("branches", out var childBranches) && + childBranches.ValueKind == JsonValueKind.Array) + { + foreach (var childBranch in childBranches.EnumerateArray()) + { + VisitBranch(childBranch, branchName); + } + } + } + + void AddOrUpdate(CsafProductInfo product) + { + if (products.TryGetValue(product.ProductId, out var existing)) + { + products[product.ProductId] = MergeProducts(existing, product); + } + else + { + products[product.ProductId] = product; + } + } + + static CsafProductInfo MergeProducts(CsafProductInfo existing, CsafProductInfo incoming) + { + static string ChooseName(string incoming, string fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + static string? ChooseOptional(string? incoming, string? fallback) + => string.IsNullOrWhiteSpace(incoming) ? fallback : incoming; + + return new CsafProductInfo( + existing.ProductId, + ChooseName(incoming.Name, existing.Name), + ChooseOptional(incoming.Version, existing.Version), + ChooseOptional(incoming.Purl, existing.Purl), + ChooseOptional(incoming.Cpe, existing.Cpe)); + } + } + + private static ImmutableDictionary> CollectProductGroups(JsonElement root) + { + if (!root.TryGetProperty("product_tree", out var productTree) || + productTree.ValueKind != JsonValueKind.Object || + !productTree.TryGetProperty("product_groups", out var groupsElement) || + groupsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableDictionary>.Empty; + } + + var groups = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var group in groupsElement.EnumerateArray()) + { + if (group.ValueKind != JsonValueKind.Object) + { + continue; + } + + var groupId = TryGetString(group, "group_id"); + if (string.IsNullOrWhiteSpace(groupId)) + { + continue; + } + + if (!group.TryGetProperty("product_ids", out var productIdsElement) || + productIdsElement.ValueKind != JsonValueKind.Array) + { + continue; + } + + var members = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var productIdElement in productIdsElement.EnumerateArray()) + { + var productId = productIdElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + continue; + } + + members.Add(productId.Trim()); + } + + if (members.Count == 0) + { + continue; + } + + groups[groupId.Trim()] = members + .OrderBy(static id => id, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + return groups.Count == 0 + ? ImmutableDictionary>.Empty + : groups.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + } + + private static ImmutableDictionary CollectJustifications( + JsonElement vulnerability, + IReadOnlyDictionary productCatalog, + ImmutableDictionary> productGroups, + ISet unsupportedJustifications, + ISet conflictingJustifications) + { + if (!vulnerability.TryGetProperty("flags", out var flagsElement) || + flagsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableDictionary.Empty; + } + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var flag in flagsElement.EnumerateArray()) + { + if (flag.ValueKind != JsonValueKind.Object) + { + continue; + } + + var label = TryGetString(flag, "label"); + if (string.IsNullOrWhiteSpace(label)) + { + continue; + } + + var rawLabel = NormalizeRaw(label); + var normalized = MapJustification(rawLabel, unsupportedJustifications); + + var targetIds = ExpandFlagProducts(flag, productGroups); + foreach (var productId in targetIds) + { + if (!productCatalog.ContainsKey(productId)) + { + continue; + } + + var info = new CsafJustificationInfo(rawLabel, normalized); + if (map.TryGetValue(productId, out var existing)) + { + if (existing.Normalized is null && normalized is not null) + { + map[productId] = info; + } + else if (existing.Normalized is not null && normalized is not null && existing.Normalized != normalized) + { + conflictingJustifications.Add(productId); + } + else if (existing.Normalized is null && + normalized is null && + string.IsNullOrWhiteSpace(existing.RawValue) && + !string.IsNullOrWhiteSpace(rawLabel)) + { + map[productId] = info; + } + } + else + { + map[productId] = info; + } + } + } + + return map.Count == 0 + ? ImmutableDictionary.Empty + : map.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + } + + private static IEnumerable ExpandFlagProducts( + JsonElement flag, + ImmutableDictionary> productGroups) + { + var productIds = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (flag.TryGetProperty("product_ids", out var productIdsElement) && + productIdsElement.ValueKind == JsonValueKind.Array) + { + foreach (var idElement in productIdsElement.EnumerateArray()) + { + var id = idElement.GetString(); + if (string.IsNullOrWhiteSpace(id)) + { + continue; + } + + productIds.Add(id.Trim()); + } + } + + if (flag.TryGetProperty("group_ids", out var groupIdsElement) && + groupIdsElement.ValueKind == JsonValueKind.Array) + { + foreach (var groupIdElement in groupIdsElement.EnumerateArray()) + { + var groupId = groupIdElement.GetString(); + if (string.IsNullOrWhiteSpace(groupId)) + { + continue; + } + + if (productGroups.TryGetValue(groupId.Trim(), out var members)) + { + foreach (var member in members) + { + productIds.Add(member); + } + } + } + } + + return productIds; + } + + private static VexJustification? MapJustification(string justification, ISet unsupportedJustifications) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + if (JustificationMap.TryGetValue(justification, out var mapped)) + { + return mapped; + } + + unsupportedJustifications.Add(justification); + return null; + } + + private static string NormalizeRaw(string value) + => string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim(); + + private static CsafProductInfo? ParseProduct(JsonElement element, string? parentBranchName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + JsonElement productElement = element; + if (!element.TryGetProperty("product_id", out var idElement) && + element.TryGetProperty("product", out var nestedProduct) && + nestedProduct.ValueKind == JsonValueKind.Object && + nestedProduct.TryGetProperty("product_id", out idElement)) + { + productElement = nestedProduct; + } + + var productId = idElement.GetString(); + if (string.IsNullOrWhiteSpace(productId)) + { + return null; + } + + var name = TryGetString(productElement, "name") + ?? TryGetString(element, "name") + ?? parentBranchName + ?? productId; + + var version = TryGetString(productElement, "product_version") + ?? TryGetString(productElement, "version") + ?? TryGetString(element, "product_version"); + + string? cpe = null; + string? purl = null; + if (productElement.TryGetProperty("product_identification_helper", out var helper) && + helper.ValueKind == JsonValueKind.Object) + { + cpe = TryGetString(helper, "cpe"); + purl = TryGetString(helper, "purl"); + } + + return new CsafProductInfo(productId.Trim(), name.Trim(), version?.Trim(), purl?.Trim(), cpe?.Trim()); + } + + private static VexClaimStatus? MapStatus(string statusName, ISet unsupportedStatuses) + { + if (string.IsNullOrWhiteSpace(statusName)) + { + return null; + } + + var normalized = statusName.Trim(); + if (StatusMap.TryGetValue(normalized, out var mapped)) + { + return mapped; + } + + unsupportedStatuses.Add(normalized); + return null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var dateElement)) + { + return null; + } + + var value = dateElement.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + if (DateTimeOffset.TryParse(value, out var parsed)) + { + return parsed; + } + + return null; + } + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var property)) + { + return null; + } + + return property.ValueKind == JsonValueKind.String ? property.GetString() : null; + } + + private static void AddIfPresent( + ImmutableDictionary.Builder builder, + JsonElement element, + string propertyName, + string metadataKey) + { + var value = TryGetString(element, propertyName); + if (!string.IsNullOrWhiteSpace(value)) + { + builder[metadataKey] = value.Trim(); + } + } + + private static void AddPublisherMetadata( + ImmutableDictionary.Builder builder, + JsonElement documentElement) + { + if (documentElement.ValueKind != JsonValueKind.Object || + !documentElement.TryGetProperty("publisher", out var publisher) || + publisher.ValueKind != JsonValueKind.Object) + { + return; + } + + AddIfPresent(builder, publisher, "name", "csaf.publisher.name"); + AddIfPresent(builder, publisher, "category", "csaf.publisher.category"); + } + + private readonly record struct CsafClaimEntryBuilder( + CsafProductInfo Product, + VexClaimStatus Status, + string RawStatus, + string? Detail, + VexJustification? Justification, + string? RawJustification); + } + + private sealed record CsafParseResult( + DateTimeOffset FirstRelease, + DateTimeOffset LastRelease, + string? Revision, + ImmutableDictionary Metadata, + ImmutableArray Claims, + ImmutableArray UnsupportedStatuses, + ImmutableArray UnsupportedJustifications, + ImmutableArray ConflictingJustifications); + + private sealed record CsafClaimEntry( + string VulnerabilityId, + CsafProductInfo Product, + VexClaimStatus Status, + string RawStatus, + string? Detail, + VexJustification? Justification, + string? RawJustification); + + private sealed record CsafProductInfo( + string ProductId, + string Name, + string? Version, + string? Purl, + string? Cpe); +} diff --git a/src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Formats.CSAF/ServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Formats.CSAF/ServiceCollectionExtensions.cs index 8e282c08..11cf214d 100644 --- a/src/StellaOps.Vexer.Formats.CSAF/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Formats.CSAF/ServiceCollectionExtensions.cs @@ -1,14 +1,14 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.CSAF; - -public static class CsafFormatsServiceCollectionExtensions -{ - public static IServiceCollection AddCsafNormalizer(this IServiceCollection services) - { - ArgumentNullException.ThrowIfNull(services); - services.AddSingleton(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.CSAF; + +public static class CsafFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddCsafNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj similarity index 83% rename from src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj rename to src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj index 54abf95e..a94d7788 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX/StellaOps.Vexer.Formats.OpenVEX.csproj +++ b/src/StellaOps.Excititor.Formats.CSAF/StellaOps.Excititor.Formats.CSAF.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.CSAF/TASKS.md b/src/StellaOps.Excititor.Formats.CSAF/TASKS.md new file mode 100644 index 00000000..2651c956 --- /dev/null +++ b/src/StellaOps.Excititor.Formats.CSAF/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-FMT-CSAF-01-001 – CSAF normalizer foundation|Team Excititor Formats|EXCITITOR-CORE-01-001|**DONE (2025-10-17)** – Implemented CSAF normalizer + DI hook, parsing tracking metadata, product tree branches/full names, and mapping product statuses into canonical `VexClaim`s with baseline precedence. Regression added in `CsafNormalizerTests`.| +|EXCITITOR-FMT-CSAF-01-002 – Status/justification mapping|Team Excititor Formats|EXCITITOR-FMT-CSAF-01-001, EXCITITOR-POLICY-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-FMT-CSAF-01-001 & EXCITITOR-POLICY-01-001 verified DONE; starting normalization of `product_status`/`justification` values with policy-aligned diagnostics.| +|EXCITITOR-FMT-CSAF-01-003 – CSAF export adapter|Team Excititor Formats|EXCITITOR-EXPORT-01-001, EXCITITOR-FMT-CSAF-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-EXPORT-01-001 & EXCITITOR-FMT-CSAF-01-001 confirmed DONE; drafting deterministic CSAF exporter and manifest metadata flow.| diff --git a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs b/src/StellaOps.Excititor.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs similarity index 90% rename from src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs rename to src/StellaOps.Excititor.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs index 5145b830..03850d93 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs +++ b/src/StellaOps.Excititor.Formats.CycloneDX.Tests/CycloneDxNormalizerTests.cs @@ -1,93 +1,93 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Formats.CycloneDX; - -namespace StellaOps.Vexer.Formats.CycloneDX.Tests; - -public sealed class CycloneDxNormalizerTests -{ - [Fact] - public async Task NormalizeAsync_MapsAnalysisStateAndJustification() - { - var json = """ - { - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "serialNumber": "urn:uuid:1234", - "version": "7", - "metadata": { - "timestamp": "2025-10-15T12:00:00Z" - }, - "components": [ - { - "bom-ref": "pkg:npm/acme/lib@1.0.0", - "name": "acme-lib", - "version": "1.0.0", - "purl": "pkg:npm/acme/lib@1.0.0" - } - ], - "vulnerabilities": [ - { - "id": "CVE-2025-1000", - "detail": "Library issue", - "analysis": { - "state": "not_affected", - "justification": "code_not_present", - "response": [ "can_not_fix", "will_not_fix" ] - }, - "affects": [ - { "ref": "pkg:npm/acme/lib@1.0.0" } - ] - }, - { - "id": "CVE-2025-1001", - "description": "Investigating impact", - "analysis": { - "state": "in_triage" - }, - "affects": [ - { "ref": "pkg:npm/missing/component@2.0.0" } - ] - } - ] - } - """; - - var rawDocument = new VexRawDocument( - "vexer:cyclonedx", - VexDocumentFormat.CycloneDx, - new Uri("https://example.org/vex.json"), - new DateTimeOffset(2025, 10, 16, 0, 0, 0, TimeSpan.Zero), - "sha256:dummydigest", - Encoding.UTF8.GetBytes(json), - ImmutableDictionary.Empty); - - var provider = new VexProvider("vexer:cyclonedx", "CycloneDX Provider", VexProviderKind.Vendor); - var normalizer = new CycloneDxNormalizer(NullLogger.Instance); - - var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); - - batch.Claims.Should().HaveCount(2); - - var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1000"); - notAffected.Status.Should().Be(VexClaimStatus.NotAffected); - notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); - notAffected.Product.Key.Should().Be("pkg:npm/acme/lib@1.0.0"); - notAffected.Product.Purl.Should().Be("pkg:npm/acme/lib@1.0.0"); - notAffected.Document.Revision.Should().Be("7"); - notAffected.AdditionalMetadata["cyclonedx.specVersion"].Should().Be("1.4"); - notAffected.AdditionalMetadata["cyclonedx.analysis.state"].Should().Be("not_affected"); - notAffected.AdditionalMetadata["cyclonedx.analysis.response"].Should().Be("can_not_fix,will_not_fix"); - - var investigating = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1001"); - investigating.Status.Should().Be(VexClaimStatus.UnderInvestigation); - investigating.Justification.Should().BeNull(); - investigating.Product.Key.Should().Be("pkg:npm/missing/component@2.0.0"); - investigating.Product.Name.Should().Be("pkg:npm/missing/component@2.0.0"); - investigating.AdditionalMetadata.Should().ContainKey("cyclonedx.specVersion"); - } -} +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Formats.CycloneDX; + +namespace StellaOps.Excititor.Formats.CycloneDX.Tests; + +public sealed class CycloneDxNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_MapsAnalysisStateAndJustification() + { + var json = """ + { + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "serialNumber": "urn:uuid:1234", + "version": "7", + "metadata": { + "timestamp": "2025-10-15T12:00:00Z" + }, + "components": [ + { + "bom-ref": "pkg:npm/acme/lib@1.0.0", + "name": "acme-lib", + "version": "1.0.0", + "purl": "pkg:npm/acme/lib@1.0.0" + } + ], + "vulnerabilities": [ + { + "id": "CVE-2025-1000", + "detail": "Library issue", + "analysis": { + "state": "not_affected", + "justification": "code_not_present", + "response": [ "can_not_fix", "will_not_fix" ] + }, + "affects": [ + { "ref": "pkg:npm/acme/lib@1.0.0" } + ] + }, + { + "id": "CVE-2025-1001", + "description": "Investigating impact", + "analysis": { + "state": "in_triage" + }, + "affects": [ + { "ref": "pkg:npm/missing/component@2.0.0" } + ] + } + ] + } + """; + + var rawDocument = new VexRawDocument( + "excititor:cyclonedx", + VexDocumentFormat.CycloneDx, + new Uri("https://example.org/vex.json"), + new DateTimeOffset(2025, 10, 16, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("excititor:cyclonedx", "CycloneDX Provider", VexProviderKind.Vendor); + var normalizer = new CycloneDxNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1000"); + notAffected.Status.Should().Be(VexClaimStatus.NotAffected); + notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); + notAffected.Product.Key.Should().Be("pkg:npm/acme/lib@1.0.0"); + notAffected.Product.Purl.Should().Be("pkg:npm/acme/lib@1.0.0"); + notAffected.Document.Revision.Should().Be("7"); + notAffected.AdditionalMetadata["cyclonedx.specVersion"].Should().Be("1.4"); + notAffected.AdditionalMetadata["cyclonedx.analysis.state"].Should().Be("not_affected"); + notAffected.AdditionalMetadata["cyclonedx.analysis.response"].Should().Be("can_not_fix,will_not_fix"); + + var investigating = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-1001"); + investigating.Status.Should().Be(VexClaimStatus.UnderInvestigation); + investigating.Justification.Should().BeNull(); + investigating.Product.Key.Should().Be("pkg:npm/missing/component@2.0.0"); + investigating.Product.Name.Should().Be("pkg:npm/missing/component@2.0.0"); + investigating.AdditionalMetadata.Should().ContainKey("cyclonedx.specVersion"); + } +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj b/src/StellaOps.Excititor.Formats.CycloneDX.Tests/StellaOps.Excititor.Formats.CycloneDX.Tests.csproj similarity index 62% rename from src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj rename to src/StellaOps.Excititor.Formats.CycloneDX.Tests/StellaOps.Excititor.Formats.CycloneDX.Tests.csproj index 4a6b0f69..ccc7d16c 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX.Tests/StellaOps.Vexer.Formats.CycloneDX.Tests.csproj +++ b/src/StellaOps.Excititor.Formats.CycloneDX.Tests/StellaOps.Excititor.Formats.CycloneDX.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/AGENTS.md b/src/StellaOps.Excititor.Formats.CycloneDX/AGENTS.md similarity index 88% rename from src/StellaOps.Vexer.Formats.CycloneDX/AGENTS.md rename to src/StellaOps.Excititor.Formats.CycloneDX/AGENTS.md index 4357e8b8..a7b7b25a 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX/AGENTS.md +++ b/src/StellaOps.Excititor.Formats.CycloneDX/AGENTS.md @@ -1,22 +1,22 @@ -# AGENTS -## Role -Normalize CycloneDX VEX documents and expose serialization utilities for CycloneDX-based exports. -## Scope -- Parsing of CycloneDX VEX statements (`analysis.state`, `justification`, `impact`) into canonical claims. -- Utilities to align SBOM references (components, services) with policy expectations. -- Export builders that emit CycloneDX-compliant VEX bundles or augment existing SBOMs. -- Validation against CycloneDX schema versions and namespace compatibility. -## Participants -- Connectors ingesting CycloneDX VEX or SBOM attestations send documents here for normalization. -- Export module uses serializers to produce CycloneDX JSON/JSONL as requested. -- Policy/consensus logic depends on status/justification mapping provided here. -## Interfaces & contracts -- Normalizer implementations, component reference mapping helpers, export serializers. -- Schema validation adaptor for offline mode and fixture-driven testing. -## In/Out of scope -In: CycloneDX parsing, normalization, export writing, schema validation. -Out: Connector transport, storage, attestation; these rely on other modules. -## Observability & security expectations -- Log schema mismatches with document digest and component references; avoid logging proprietary component details where possible. -## Tests -- Unit and fixture tests will live in `../StellaOps.Vexer.Formats.CycloneDX.Tests`, covering normalization and serialization determinism. +# AGENTS +## Role +Normalize CycloneDX VEX documents and expose serialization utilities for CycloneDX-based exports. +## Scope +- Parsing of CycloneDX VEX statements (`analysis.state`, `justification`, `impact`) into canonical claims. +- Utilities to align SBOM references (components, services) with policy expectations. +- Export builders that emit CycloneDX-compliant VEX bundles or augment existing SBOMs. +- Validation against CycloneDX schema versions and namespace compatibility. +## Participants +- Connectors ingesting CycloneDX VEX or SBOM attestations send documents here for normalization. +- Export module uses serializers to produce CycloneDX JSON/JSONL as requested. +- Policy/consensus logic depends on status/justification mapping provided here. +## Interfaces & contracts +- Normalizer implementations, component reference mapping helpers, export serializers. +- Schema validation adaptor for offline mode and fixture-driven testing. +## In/Out of scope +In: CycloneDX parsing, normalization, export writing, schema validation. +Out: Connector transport, storage, attestation; these rely on other modules. +## Observability & security expectations +- Log schema mismatches with document digest and component references; avoid logging proprietary component details where possible. +## Tests +- Unit and fixture tests will live in `../StellaOps.Excititor.Formats.CycloneDX.Tests`, covering normalization and serialization determinism. diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs b/src/StellaOps.Excititor.Formats.CycloneDX/CycloneDxNormalizer.cs similarity index 97% rename from src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs rename to src/StellaOps.Excititor.Formats.CycloneDX/CycloneDxNormalizer.cs index c12c0d94..8e2c041a 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX/CycloneDxNormalizer.cs +++ b/src/StellaOps.Excititor.Formats.CycloneDX/CycloneDxNormalizer.cs @@ -1,459 +1,459 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.CycloneDX; - -public sealed class CycloneDxNormalizer : IVexNormalizer -{ - private static readonly ImmutableDictionary StateMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["not_affected"] = VexClaimStatus.NotAffected, - ["resolved"] = VexClaimStatus.Fixed, - ["resolved_with_patches"] = VexClaimStatus.Fixed, - ["resolved_no_fix"] = VexClaimStatus.Fixed, - ["fixed"] = VexClaimStatus.Fixed, - ["affected"] = VexClaimStatus.Affected, - ["known_affected"] = VexClaimStatus.Affected, - ["exploitable"] = VexClaimStatus.Affected, - ["in_triage"] = VexClaimStatus.UnderInvestigation, - ["under_investigation"] = VexClaimStatus.UnderInvestigation, - ["unknown"] = VexClaimStatus.UnderInvestigation, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["code_not_present"] = VexJustification.CodeNotPresent, - ["code_not_reachable"] = VexJustification.CodeNotReachable, - ["component_not_present"] = VexJustification.ComponentNotPresent, - ["component_not_configured"] = VexJustification.ComponentNotConfigured, - ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, - ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, - ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, - ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, - ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, - ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, - ["requires_configuration"] = VexJustification.RequiresConfiguration, - ["requires_dependency"] = VexJustification.RequiresDependency, - ["requires_environment"] = VexJustification.RequiresEnvironment, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private readonly ILogger _logger; - - public CycloneDxNormalizer(ILogger logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string Format => VexDocumentFormat.CycloneDx.ToString().ToLowerInvariant(); - - public bool CanHandle(VexRawDocument document) - => document is not null && document.Format == VexDocumentFormat.CycloneDx; - - public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(provider); - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var parseResult = CycloneDxParser.Parse(document); - var baseMetadata = parseResult.Metadata; - var claimsBuilder = ImmutableArray.CreateBuilder(); - - foreach (var vulnerability in parseResult.Vulnerabilities) - { - cancellationToken.ThrowIfCancellationRequested(); - - var state = MapState(vulnerability.AnalysisState, out var stateRaw); - var justification = MapJustification(vulnerability.AnalysisJustification); - var responses = vulnerability.AnalysisResponses; - - foreach (var affect in vulnerability.Affects) - { - var productInfo = parseResult.ResolveProduct(affect.ComponentRef); - var product = new VexProduct( - productInfo.Key, - productInfo.Name, - productInfo.Version, - productInfo.Purl, - productInfo.Cpe); - - var metadata = baseMetadata; - if (!string.IsNullOrWhiteSpace(stateRaw)) - { - metadata = metadata.SetItem("cyclonedx.analysis.state", stateRaw); - } - - if (!string.IsNullOrWhiteSpace(vulnerability.AnalysisJustification)) - { - metadata = metadata.SetItem("cyclonedx.analysis.justification", vulnerability.AnalysisJustification); - } - - if (responses.Length > 0) - { - metadata = metadata.SetItem("cyclonedx.analysis.response", string.Join(",", responses)); - } - - if (!string.IsNullOrWhiteSpace(affect.ComponentRef)) - { - metadata = metadata.SetItem("cyclonedx.affects.ref", affect.ComponentRef); - } - - var claimDocument = new VexClaimDocument( - VexDocumentFormat.CycloneDx, - document.Digest, - document.SourceUri, - parseResult.BomVersion, - signature: null); - - var claim = new VexClaim( - vulnerability.VulnerabilityId, - provider.Id, - product, - state, - claimDocument, - parseResult.FirstObserved, - parseResult.LastObserved, - justification, - vulnerability.Detail, - confidence: null, - additionalMetadata: metadata); - - claimsBuilder.Add(claim); - } - } - - var orderedClaims = claimsBuilder - .ToImmutable() - .OrderBy(static c => c.VulnerabilityId, StringComparer.Ordinal) - .ThenBy(static c => c.Product.Key, StringComparer.Ordinal) - .ToImmutableArray(); - - _logger.LogInformation( - "Normalized CycloneDX document {Source} into {ClaimCount} claim(s).", - document.SourceUri, - orderedClaims.Length); - - return ValueTask.FromResult(new VexClaimBatch( - document, - orderedClaims, - ImmutableDictionary.Empty)); - } - catch (JsonException ex) - { - _logger.LogError(ex, "Failed to parse CycloneDX VEX document {SourceUri}", document.SourceUri); - throw; - } - } - - private static VexClaimStatus MapState(string? state, out string? raw) - { - raw = state?.Trim(); - - if (!string.IsNullOrWhiteSpace(state) && StateMap.TryGetValue(state.Trim(), out var mapped)) - { - return mapped; - } - - return VexClaimStatus.UnderInvestigation; - } - - private static VexJustification? MapJustification(string? justification) - { - if (string.IsNullOrWhiteSpace(justification)) - { - return null; - } - - return JustificationMap.TryGetValue(justification.Trim(), out var mapped) - ? mapped - : null; - } - - private sealed class CycloneDxParser - { - public static CycloneDxParseResult Parse(VexRawDocument document) - { - using var json = JsonDocument.Parse(document.Content.ToArray()); - var root = json.RootElement; - - var specVersion = TryGetString(root, "specVersion"); - var bomVersion = TryGetString(root, "version"); - var serialNumber = TryGetString(root, "serialNumber"); - - var metadataTimestamp = ParseDate(TryGetProperty(root, "metadata"), "timestamp"); - var observedTimestamp = metadataTimestamp ?? document.RetrievedAt; - - var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - if (!string.IsNullOrWhiteSpace(specVersion)) - { - metadataBuilder["cyclonedx.specVersion"] = specVersion!; - } - - if (!string.IsNullOrWhiteSpace(bomVersion)) - { - metadataBuilder["cyclonedx.version"] = bomVersion!; - } - - if (!string.IsNullOrWhiteSpace(serialNumber)) - { - metadataBuilder["cyclonedx.serialNumber"] = serialNumber!; - } - - var components = CollectComponents(root); - var vulnerabilities = CollectVulnerabilities(root); - - return new CycloneDxParseResult( - metadataBuilder.ToImmutable(), - bomVersion, - observedTimestamp, - observedTimestamp, - components, - vulnerabilities); - } - - private static ImmutableDictionary CollectComponents(JsonElement root) - { - var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - - if (root.TryGetProperty("components", out var components) && components.ValueKind == JsonValueKind.Array) - { - foreach (var component in components.EnumerateArray()) - { - if (component.ValueKind != JsonValueKind.Object) - { - continue; - } - - var reference = TryGetString(component, "bom-ref") ?? TryGetString(component, "bomRef"); - if (string.IsNullOrWhiteSpace(reference)) - { - continue; - } - - var name = TryGetString(component, "name") ?? reference; - var version = TryGetString(component, "version"); - var purl = TryGetString(component, "purl"); - - string? cpe = null; - if (component.TryGetProperty("externalReferences", out var externalRefs) && externalRefs.ValueKind == JsonValueKind.Array) - { - foreach (var referenceEntry in externalRefs.EnumerateArray()) - { - if (referenceEntry.ValueKind != JsonValueKind.Object) - { - continue; - } - - var type = TryGetString(referenceEntry, "type"); - if (!string.Equals(type, "cpe", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (referenceEntry.TryGetProperty("url", out var url) && url.ValueKind == JsonValueKind.String) - { - cpe = url.GetString(); - break; - } - } - } - - builder[reference!] = new CycloneDxComponent(reference!, name ?? reference!, version, purl, cpe); - } - } - - return builder.ToImmutable(); - } - - private static ImmutableArray CollectVulnerabilities(JsonElement root) - { - if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) || - vulnerabilitiesElement.ValueKind != JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - - foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) - { - if (vulnerability.ValueKind != JsonValueKind.Object) - { - continue; - } - - var vulnerabilityId = - TryGetString(vulnerability, "id") ?? - TryGetString(vulnerability, "bom-ref") ?? - TryGetString(vulnerability, "bomRef") ?? - TryGetString(vulnerability, "cve"); - - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - continue; - } - - var detail = TryGetString(vulnerability, "detail") ?? TryGetString(vulnerability, "description"); - - var analysis = TryGetProperty(vulnerability, "analysis"); - var analysisState = TryGetString(analysis, "state"); - var analysisJustification = TryGetString(analysis, "justification"); - var analysisResponses = CollectResponses(analysis); - - var affects = CollectAffects(vulnerability); - if (affects.Length == 0) - { - continue; - } - - builder.Add(new CycloneDxVulnerability( - vulnerabilityId.Trim(), - detail?.Trim(), - analysisState, - analysisJustification, - analysisResponses, - affects)); - } - - return builder.ToImmutable(); - } - - private static ImmutableArray CollectResponses(JsonElement analysis) - { - if (analysis.ValueKind != JsonValueKind.Object || - !analysis.TryGetProperty("response", out var responseElement) || - responseElement.ValueKind != JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var responses = new SortedSet(StringComparer.OrdinalIgnoreCase); - foreach (var response in responseElement.EnumerateArray()) - { - if (response.ValueKind == JsonValueKind.String) - { - var value = response.GetString(); - if (!string.IsNullOrWhiteSpace(value)) - { - responses.Add(value.Trim()); - } - } - } - - return responses.Count == 0 ? ImmutableArray.Empty : responses.ToImmutableArray(); - } - - private static ImmutableArray CollectAffects(JsonElement vulnerability) - { - if (!vulnerability.TryGetProperty("affects", out var affectsElement) || - affectsElement.ValueKind != JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var affect in affectsElement.EnumerateArray()) - { - if (affect.ValueKind != JsonValueKind.Object) - { - continue; - } - - var reference = TryGetString(affect, "ref"); - if (string.IsNullOrWhiteSpace(reference)) - { - continue; - } - - builder.Add(new CycloneDxAffect(reference.Trim())); - } - - return builder.ToImmutable(); - } - - private static JsonElement TryGetProperty(JsonElement element, string propertyName) - => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) - ? value - : default; - - private static string? TryGetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var value)) - { - return null; - } - - return value.ValueKind == JsonValueKind.String ? value.GetString() : null; - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - var value = TryGetString(element, propertyName); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; - } - } - - private sealed record CycloneDxParseResult( - ImmutableDictionary Metadata, - string? BomVersion, - DateTimeOffset FirstObserved, - DateTimeOffset LastObserved, - ImmutableDictionary Components, - ImmutableArray Vulnerabilities) - { - public CycloneDxProductInfo ResolveProduct(string? componentRef) - { - if (!string.IsNullOrWhiteSpace(componentRef) && - Components.TryGetValue(componentRef.Trim(), out var component)) - { - return new CycloneDxProductInfo(component.Reference, component.Name, component.Version, component.Purl, component.Cpe); - } - - var key = string.IsNullOrWhiteSpace(componentRef) ? "unknown-component" : componentRef.Trim(); - return new CycloneDxProductInfo(key, key, null, null, null); - } - } - - private sealed record CycloneDxComponent( - string Reference, - string Name, - string? Version, - string? Purl, - string? Cpe); - - private sealed record CycloneDxVulnerability( - string VulnerabilityId, - string? Detail, - string? AnalysisState, - string? AnalysisJustification, - ImmutableArray AnalysisResponses, - ImmutableArray Affects); - - private sealed record CycloneDxAffect(string ComponentRef); - - private sealed record CycloneDxProductInfo( - string Key, - string Name, - string? Version, - string? Purl, - string? Cpe); -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.CycloneDX; + +public sealed class CycloneDxNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StateMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["not_affected"] = VexClaimStatus.NotAffected, + ["resolved"] = VexClaimStatus.Fixed, + ["resolved_with_patches"] = VexClaimStatus.Fixed, + ["resolved_no_fix"] = VexClaimStatus.Fixed, + ["fixed"] = VexClaimStatus.Fixed, + ["affected"] = VexClaimStatus.Affected, + ["known_affected"] = VexClaimStatus.Affected, + ["exploitable"] = VexClaimStatus.Affected, + ["in_triage"] = VexClaimStatus.UnderInvestigation, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + ["unknown"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public CycloneDxNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.CycloneDx.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.CycloneDx; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var parseResult = CycloneDxParser.Parse(document); + var baseMetadata = parseResult.Metadata; + var claimsBuilder = ImmutableArray.CreateBuilder(); + + foreach (var vulnerability in parseResult.Vulnerabilities) + { + cancellationToken.ThrowIfCancellationRequested(); + + var state = MapState(vulnerability.AnalysisState, out var stateRaw); + var justification = MapJustification(vulnerability.AnalysisJustification); + var responses = vulnerability.AnalysisResponses; + + foreach (var affect in vulnerability.Affects) + { + var productInfo = parseResult.ResolveProduct(affect.ComponentRef); + var product = new VexProduct( + productInfo.Key, + productInfo.Name, + productInfo.Version, + productInfo.Purl, + productInfo.Cpe); + + var metadata = baseMetadata; + if (!string.IsNullOrWhiteSpace(stateRaw)) + { + metadata = metadata.SetItem("cyclonedx.analysis.state", stateRaw); + } + + if (!string.IsNullOrWhiteSpace(vulnerability.AnalysisJustification)) + { + metadata = metadata.SetItem("cyclonedx.analysis.justification", vulnerability.AnalysisJustification); + } + + if (responses.Length > 0) + { + metadata = metadata.SetItem("cyclonedx.analysis.response", string.Join(",", responses)); + } + + if (!string.IsNullOrWhiteSpace(affect.ComponentRef)) + { + metadata = metadata.SetItem("cyclonedx.affects.ref", affect.ComponentRef); + } + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.CycloneDx, + document.Digest, + document.SourceUri, + parseResult.BomVersion, + signature: null); + + var claim = new VexClaim( + vulnerability.VulnerabilityId, + provider.Id, + product, + state, + claimDocument, + parseResult.FirstObserved, + parseResult.LastObserved, + justification, + vulnerability.Detail, + confidence: null, + additionalMetadata: metadata); + + claimsBuilder.Add(claim); + } + } + + var orderedClaims = claimsBuilder + .ToImmutable() + .OrderBy(static c => c.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static c => c.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized CycloneDX document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + return ValueTask.FromResult(new VexClaimBatch( + document, + orderedClaims, + ImmutableDictionary.Empty)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse CycloneDX VEX document {SourceUri}", document.SourceUri); + throw; + } + } + + private static VexClaimStatus MapState(string? state, out string? raw) + { + raw = state?.Trim(); + + if (!string.IsNullOrWhiteSpace(state) && StateMap.TryGetValue(state.Trim(), out var mapped)) + { + return mapped; + } + + return VexClaimStatus.UnderInvestigation; + } + + private static VexJustification? MapJustification(string? justification) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + return JustificationMap.TryGetValue(justification.Trim(), out var mapped) + ? mapped + : null; + } + + private sealed class CycloneDxParser + { + public static CycloneDxParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var specVersion = TryGetString(root, "specVersion"); + var bomVersion = TryGetString(root, "version"); + var serialNumber = TryGetString(root, "serialNumber"); + + var metadataTimestamp = ParseDate(TryGetProperty(root, "metadata"), "timestamp"); + var observedTimestamp = metadataTimestamp ?? document.RetrievedAt; + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + if (!string.IsNullOrWhiteSpace(specVersion)) + { + metadataBuilder["cyclonedx.specVersion"] = specVersion!; + } + + if (!string.IsNullOrWhiteSpace(bomVersion)) + { + metadataBuilder["cyclonedx.version"] = bomVersion!; + } + + if (!string.IsNullOrWhiteSpace(serialNumber)) + { + metadataBuilder["cyclonedx.serialNumber"] = serialNumber!; + } + + var components = CollectComponents(root); + var vulnerabilities = CollectVulnerabilities(root); + + return new CycloneDxParseResult( + metadataBuilder.ToImmutable(), + bomVersion, + observedTimestamp, + observedTimestamp, + components, + vulnerabilities); + } + + private static ImmutableDictionary CollectComponents(JsonElement root) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + + if (root.TryGetProperty("components", out var components) && components.ValueKind == JsonValueKind.Array) + { + foreach (var component in components.EnumerateArray()) + { + if (component.ValueKind != JsonValueKind.Object) + { + continue; + } + + var reference = TryGetString(component, "bom-ref") ?? TryGetString(component, "bomRef"); + if (string.IsNullOrWhiteSpace(reference)) + { + continue; + } + + var name = TryGetString(component, "name") ?? reference; + var version = TryGetString(component, "version"); + var purl = TryGetString(component, "purl"); + + string? cpe = null; + if (component.TryGetProperty("externalReferences", out var externalRefs) && externalRefs.ValueKind == JsonValueKind.Array) + { + foreach (var referenceEntry in externalRefs.EnumerateArray()) + { + if (referenceEntry.ValueKind != JsonValueKind.Object) + { + continue; + } + + var type = TryGetString(referenceEntry, "type"); + if (!string.Equals(type, "cpe", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (referenceEntry.TryGetProperty("url", out var url) && url.ValueKind == JsonValueKind.String) + { + cpe = url.GetString(); + break; + } + } + } + + builder[reference!] = new CycloneDxComponent(reference!, name ?? reference!, version, purl, cpe); + } + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectVulnerabilities(JsonElement root) + { + if (!root.TryGetProperty("vulnerabilities", out var vulnerabilitiesElement) || + vulnerabilitiesElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + + foreach (var vulnerability in vulnerabilitiesElement.EnumerateArray()) + { + if (vulnerability.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vulnerabilityId = + TryGetString(vulnerability, "id") ?? + TryGetString(vulnerability, "bom-ref") ?? + TryGetString(vulnerability, "bomRef") ?? + TryGetString(vulnerability, "cve"); + + if (string.IsNullOrWhiteSpace(vulnerabilityId)) + { + continue; + } + + var detail = TryGetString(vulnerability, "detail") ?? TryGetString(vulnerability, "description"); + + var analysis = TryGetProperty(vulnerability, "analysis"); + var analysisState = TryGetString(analysis, "state"); + var analysisJustification = TryGetString(analysis, "justification"); + var analysisResponses = CollectResponses(analysis); + + var affects = CollectAffects(vulnerability); + if (affects.Length == 0) + { + continue; + } + + builder.Add(new CycloneDxVulnerability( + vulnerabilityId.Trim(), + detail?.Trim(), + analysisState, + analysisJustification, + analysisResponses, + affects)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectResponses(JsonElement analysis) + { + if (analysis.ValueKind != JsonValueKind.Object || + !analysis.TryGetProperty("response", out var responseElement) || + responseElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var responses = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var response in responseElement.EnumerateArray()) + { + if (response.ValueKind == JsonValueKind.String) + { + var value = response.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + responses.Add(value.Trim()); + } + } + } + + return responses.Count == 0 ? ImmutableArray.Empty : responses.ToImmutableArray(); + } + + private static ImmutableArray CollectAffects(JsonElement vulnerability) + { + if (!vulnerability.TryGetProperty("affects", out var affectsElement) || + affectsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var affect in affectsElement.EnumerateArray()) + { + if (affect.ValueKind != JsonValueKind.Object) + { + continue; + } + + var reference = TryGetString(affect, "ref"); + if (string.IsNullOrWhiteSpace(reference)) + { + continue; + } + + builder.Add(new CycloneDxAffect(reference.Trim())); + } + + return builder.ToImmutable(); + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) + ? value + : default; + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var value = TryGetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + } + } + + private sealed record CycloneDxParseResult( + ImmutableDictionary Metadata, + string? BomVersion, + DateTimeOffset FirstObserved, + DateTimeOffset LastObserved, + ImmutableDictionary Components, + ImmutableArray Vulnerabilities) + { + public CycloneDxProductInfo ResolveProduct(string? componentRef) + { + if (!string.IsNullOrWhiteSpace(componentRef) && + Components.TryGetValue(componentRef.Trim(), out var component)) + { + return new CycloneDxProductInfo(component.Reference, component.Name, component.Version, component.Purl, component.Cpe); + } + + var key = string.IsNullOrWhiteSpace(componentRef) ? "unknown-component" : componentRef.Trim(); + return new CycloneDxProductInfo(key, key, null, null, null); + } + } + + private sealed record CycloneDxComponent( + string Reference, + string Name, + string? Version, + string? Purl, + string? Cpe); + + private sealed record CycloneDxVulnerability( + string VulnerabilityId, + string? Detail, + string? AnalysisState, + string? AnalysisJustification, + ImmutableArray AnalysisResponses, + ImmutableArray Affects); + + private sealed record CycloneDxAffect(string ComponentRef); + + private sealed record CycloneDxProductInfo( + string Key, + string Name, + string? Version, + string? Purl, + string? Cpe); +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Formats.CycloneDX/ServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Formats.CycloneDX/ServiceCollectionExtensions.cs index d39c3c78..518d7223 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Formats.CycloneDX/ServiceCollectionExtensions.cs @@ -1,14 +1,14 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.CycloneDX; - -public static class CycloneDxFormatsServiceCollectionExtensions -{ - public static IServiceCollection AddCycloneDxNormalizer(this IServiceCollection services) - { - ArgumentNullException.ThrowIfNull(services); - services.AddSingleton(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.CycloneDX; + +public static class CycloneDxFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddCycloneDxNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj similarity index 83% rename from src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj rename to src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj index 54abf95e..a94d7788 100644 --- a/src/StellaOps.Vexer.Formats.CycloneDX/StellaOps.Vexer.Formats.CycloneDX.csproj +++ b/src/StellaOps.Excititor.Formats.CycloneDX/StellaOps.Excititor.Formats.CycloneDX.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md b/src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md new file mode 100644 index 00000000..6b4af5b8 --- /dev/null +++ b/src/StellaOps.Excititor.Formats.CycloneDX/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-FMT-CYCLONE-01-001 – CycloneDX VEX normalizer|Team Excititor Formats|EXCITITOR-CORE-01-001|**DONE (2025-10-17)** – CycloneDX normalizer parses `analysis` data, resolves component references, and emits canonical `VexClaim`s; regression lives in `CycloneDxNormalizerTests`.| +|EXCITITOR-FMT-CYCLONE-01-002 – Component reference reconciliation|Team Excititor Formats|EXCITITOR-FMT-CYCLONE-01-001|**DOING (2025-10-19)** – Prereq EXCITITOR-FMT-CYCLONE-01-001 confirmed DONE; proceeding with reference reconciliation helpers and diagnostics for missing SBOM links.| +|EXCITITOR-FMT-CYCLONE-01-003 – CycloneDX export serializer|Team Excititor Formats|EXCITITOR-EXPORT-01-001, EXCITITOR-FMT-CYCLONE-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-EXPORT-01-001 & EXCITITOR-FMT-CYCLONE-01-001 verified DONE; initiating deterministic CycloneDX VEX exporter work.| diff --git a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs b/src/StellaOps.Excititor.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs similarity index 89% rename from src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs rename to src/StellaOps.Excititor.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs index 37989d9d..3f456660 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs +++ b/src/StellaOps.Excititor.Formats.OpenVEX.Tests/OpenVexNormalizerTests.cs @@ -1,87 +1,87 @@ -using System.Collections.Immutable; -using System.Linq; -using System.Text; -using FluentAssertions; -using Microsoft.Extensions.Logging.Abstractions; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Formats.OpenVEX; - -namespace StellaOps.Vexer.Formats.OpenVEX.Tests; - -public sealed class OpenVexNormalizerTests -{ - [Fact] - public async Task NormalizeAsync_ProducesClaimsForStatements() - { - var json = """ - { - "document": { - "author": "Acme Security", - "version": "1", - "issued": "2025-10-01T00:00:00Z", - "last_updated": "2025-10-05T00:00:00Z" - }, - "statements": [ - { - "id": "statement-1", - "vulnerability": "CVE-2025-2000", - "status": "not_affected", - "justification": "code_not_present", - "products": [ - { - "id": "acme-widget@1.2.3", - "name": "Acme Widget", - "version": "1.2.3", - "purl": "pkg:acme/widget@1.2.3", - "cpe": "cpe:/a:acme:widget:1.2.3" - } - ], - "statement": "The vulnerable code was never shipped." - }, - { - "id": "statement-2", - "vulnerability": "CVE-2025-2001", - "status": "affected", - "products": [ - "pkg:acme/widget@2.0.0" - ], - "remediation": "Upgrade to 2.1.0" - } - ] - } - """; - - var rawDocument = new VexRawDocument( - "vexer:openvex", - VexDocumentFormat.OpenVex, - new Uri("https://example.com/openvex.json"), - new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), - "sha256:dummydigest", - Encoding.UTF8.GetBytes(json), - ImmutableDictionary.Empty); - - var provider = new VexProvider("vexer:openvex", "OpenVEX Provider", VexProviderKind.Vendor); - var normalizer = new OpenVexNormalizer(NullLogger.Instance); - - var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); - - batch.Claims.Should().HaveCount(2); - - var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2000"); - notAffected.Status.Should().Be(VexClaimStatus.NotAffected); - notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); - notAffected.Product.Key.Should().Be("acme-widget@1.2.3"); - notAffected.Product.Purl.Should().Be("pkg:acme/widget@1.2.3"); - notAffected.Document.Revision.Should().Be("1"); - notAffected.AdditionalMetadata["openvex.document.author"].Should().Be("Acme Security"); - notAffected.AdditionalMetadata["openvex.statement.status"].Should().Be("not_affected"); - notAffected.Detail.Should().Be("The vulnerable code was never shipped."); - - var affected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2001"); - affected.Status.Should().Be(VexClaimStatus.Affected); - affected.Justification.Should().BeNull(); - affected.Product.Key.Should().Be("pkg:acme/widget@2.0.0"); - affected.Product.Name.Should().Be("pkg:acme/widget@2.0.0"); - affected.Detail.Should().Be("Upgrade to 2.1.0"); - } -} +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Formats.OpenVEX; + +namespace StellaOps.Excititor.Formats.OpenVEX.Tests; + +public sealed class OpenVexNormalizerTests +{ + [Fact] + public async Task NormalizeAsync_ProducesClaimsForStatements() + { + var json = """ + { + "document": { + "author": "Acme Security", + "version": "1", + "issued": "2025-10-01T00:00:00Z", + "last_updated": "2025-10-05T00:00:00Z" + }, + "statements": [ + { + "id": "statement-1", + "vulnerability": "CVE-2025-2000", + "status": "not_affected", + "justification": "code_not_present", + "products": [ + { + "id": "acme-widget@1.2.3", + "name": "Acme Widget", + "version": "1.2.3", + "purl": "pkg:acme/widget@1.2.3", + "cpe": "cpe:/a:acme:widget:1.2.3" + } + ], + "statement": "The vulnerable code was never shipped." + }, + { + "id": "statement-2", + "vulnerability": "CVE-2025-2001", + "status": "affected", + "products": [ + "pkg:acme/widget@2.0.0" + ], + "remediation": "Upgrade to 2.1.0" + } + ] + } + """; + + var rawDocument = new VexRawDocument( + "excititor:openvex", + VexDocumentFormat.OpenVex, + new Uri("https://example.com/openvex.json"), + new DateTimeOffset(2025, 10, 6, 0, 0, 0, TimeSpan.Zero), + "sha256:dummydigest", + Encoding.UTF8.GetBytes(json), + ImmutableDictionary.Empty); + + var provider = new VexProvider("excititor:openvex", "OpenVEX Provider", VexProviderKind.Vendor); + var normalizer = new OpenVexNormalizer(NullLogger.Instance); + + var batch = await normalizer.NormalizeAsync(rawDocument, provider, CancellationToken.None); + + batch.Claims.Should().HaveCount(2); + + var notAffected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2000"); + notAffected.Status.Should().Be(VexClaimStatus.NotAffected); + notAffected.Justification.Should().Be(VexJustification.CodeNotPresent); + notAffected.Product.Key.Should().Be("acme-widget@1.2.3"); + notAffected.Product.Purl.Should().Be("pkg:acme/widget@1.2.3"); + notAffected.Document.Revision.Should().Be("1"); + notAffected.AdditionalMetadata["openvex.document.author"].Should().Be("Acme Security"); + notAffected.AdditionalMetadata["openvex.statement.status"].Should().Be("not_affected"); + notAffected.Detail.Should().Be("The vulnerable code was never shipped."); + + var affected = batch.Claims.Single(c => c.VulnerabilityId == "CVE-2025-2001"); + affected.Status.Should().Be(VexClaimStatus.Affected); + affected.Justification.Should().BeNull(); + affected.Product.Key.Should().Be("pkg:acme/widget@2.0.0"); + affected.Product.Name.Should().Be("pkg:acme/widget@2.0.0"); + affected.Detail.Should().Be("Upgrade to 2.1.0"); + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj b/src/StellaOps.Excititor.Formats.OpenVEX.Tests/StellaOps.Excititor.Formats.OpenVEX.Tests.csproj similarity index 63% rename from src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj rename to src/StellaOps.Excititor.Formats.OpenVEX.Tests/StellaOps.Excititor.Formats.OpenVEX.Tests.csproj index bdd68e59..8aa2f2c4 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX.Tests/StellaOps.Vexer.Formats.OpenVEX.Tests.csproj +++ b/src/StellaOps.Excititor.Formats.OpenVEX.Tests/StellaOps.Excititor.Formats.OpenVEX.Tests.csproj @@ -1,17 +1,17 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/AGENTS.md b/src/StellaOps.Excititor.Formats.OpenVEX/AGENTS.md similarity index 95% rename from src/StellaOps.Vexer.Formats.OpenVEX/AGENTS.md rename to src/StellaOps.Excititor.Formats.OpenVEX/AGENTS.md index d6040f70..ca7796ee 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX/AGENTS.md +++ b/src/StellaOps.Excititor.Formats.OpenVEX/AGENTS.md @@ -1,21 +1,21 @@ -# AGENTS -## Role -Provides OpenVEX statement normalization and export writers for lightweight attestation-oriented outputs. -## Scope -- Parse OpenVEX documents/attestations into canonical claims with provenance metadata. -- Utilities to merge multiple OpenVEX statements and resolve conflicts for consensus ingestion. -- Export writer emitting OpenVEX envelopes from consensus data with deterministic ordering. -- Optional SBOM linkage helpers referencing component digests or PURLs. -## Participants -- OCI/OpenVEX connector and other attest-based sources depend on this module for normalization. -- Export module uses writers for `--format openvex` requests. -- Attestation layer references emitted statements to populate predicate subjects. -## Interfaces & contracts -- Normalizer classes implementing `INormalizer`, reducer utilities to consolidate OpenVEX events, export serializer. -## In/Out of scope -In: OpenVEX parsing, normalization, export serialization, helper utilities. -Out: OCI registry access, policy evaluation, attestation signing (handled by other modules). -## Observability & security expectations -- Log normalization anomalies with subject digest and justification mapping while respecting offline constraints. -## Tests -- Snapshot-driven normalization/export tests will be placed in `../StellaOps.Vexer.Formats.OpenVEX.Tests`. +# AGENTS +## Role +Provides OpenVEX statement normalization and export writers for lightweight attestation-oriented outputs. +## Scope +- Parse OpenVEX documents/attestations into canonical claims with provenance metadata. +- Utilities to merge multiple OpenVEX statements and resolve conflicts for consensus ingestion. +- Export writer emitting OpenVEX envelopes from consensus data with deterministic ordering. +- Optional SBOM linkage helpers referencing component digests or PURLs. +## Participants +- OCI/OpenVEX connector and other attest-based sources depend on this module for normalization. +- Export module uses writers for `--format openvex` requests. +- Attestation layer references emitted statements to populate predicate subjects. +## Interfaces & contracts +- Normalizer classes implementing `INormalizer`, reducer utilities to consolidate OpenVEX events, export serializer. +## In/Out of scope +In: OpenVEX parsing, normalization, export serialization, helper utilities. +Out: OCI registry access, policy evaluation, attestation signing (handled by other modules). +## Observability & security expectations +- Log normalization anomalies with subject digest and justification mapping while respecting offline constraints. +## Tests +- Snapshot-driven normalization/export tests will be placed in `../StellaOps.Excititor.Formats.OpenVEX.Tests`. diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs b/src/StellaOps.Excititor.Formats.OpenVEX/OpenVexNormalizer.cs similarity index 96% rename from src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs rename to src/StellaOps.Excititor.Formats.OpenVEX/OpenVexNormalizer.cs index ac748af9..9a284346 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX/OpenVexNormalizer.cs +++ b/src/StellaOps.Excititor.Formats.OpenVEX/OpenVexNormalizer.cs @@ -1,367 +1,367 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Linq; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.OpenVEX; - -public sealed class OpenVexNormalizer : IVexNormalizer -{ - private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["affected"] = VexClaimStatus.Affected, - ["not_affected"] = VexClaimStatus.NotAffected, - ["fixed"] = VexClaimStatus.Fixed, - ["under_investigation"] = VexClaimStatus.UnderInvestigation, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["component_not_present"] = VexJustification.ComponentNotPresent, - ["component_not_configured"] = VexJustification.ComponentNotConfigured, - ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, - ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, - ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, - ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, - ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, - ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, - ["code_not_present"] = VexJustification.CodeNotPresent, - ["code_not_reachable"] = VexJustification.CodeNotReachable, - ["requires_configuration"] = VexJustification.RequiresConfiguration, - ["requires_dependency"] = VexJustification.RequiresDependency, - ["requires_environment"] = VexJustification.RequiresEnvironment, - }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); - - private readonly ILogger _logger; - - public OpenVexNormalizer(ILogger logger) - { - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public string Format => VexDocumentFormat.OpenVex.ToString().ToLowerInvariant(); - - public bool CanHandle(VexRawDocument document) - => document is not null && document.Format == VexDocumentFormat.OpenVex; - - public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(provider); - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var result = OpenVexParser.Parse(document); - var claims = ImmutableArray.CreateBuilder(result.Statements.Length); - - foreach (var statement in result.Statements) - { - cancellationToken.ThrowIfCancellationRequested(); - - var status = MapStatus(statement.Status); - var justification = MapJustification(statement.Justification); - - foreach (var product in statement.Products) - { - var vexProduct = new VexProduct( - product.Key, - product.Name, - product.Version, - product.Purl, - product.Cpe); - - var metadata = result.Metadata; - - metadata = metadata.SetItem("openvex.statement.id", statement.Id); - if (!string.IsNullOrWhiteSpace(statement.Status)) - { - metadata = metadata.SetItem("openvex.statement.status", statement.Status!); - } - - if (!string.IsNullOrWhiteSpace(statement.Justification)) - { - metadata = metadata.SetItem("openvex.statement.justification", statement.Justification!); - } - - if (!string.IsNullOrWhiteSpace(product.OriginalId)) - { - metadata = metadata.SetItem("openvex.product.source", product.OriginalId!); - } - - var claimDocument = new VexClaimDocument( - VexDocumentFormat.OpenVex, - document.Digest, - document.SourceUri, - result.DocumentVersion, - signature: null); - - var claim = new VexClaim( - statement.Vulnerability, - provider.Id, - vexProduct, - status, - claimDocument, - result.FirstObserved, - result.LastObserved, - justification, - statement.Remarks, - confidence: null, - additionalMetadata: metadata); - - claims.Add(claim); - } - } - - var orderedClaims = claims - .ToImmutable() - .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) - .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) - .ToImmutableArray(); - - _logger.LogInformation( - "Normalized OpenVEX document {Source} into {ClaimCount} claim(s).", - document.SourceUri, - orderedClaims.Length); - - return ValueTask.FromResult(new VexClaimBatch( - document, - orderedClaims, - ImmutableDictionary.Empty)); - } - catch (JsonException ex) - { - _logger.LogError(ex, "Failed to parse OpenVEX document {SourceUri}", document.SourceUri); - throw; - } - } - - private static VexClaimStatus MapStatus(string? status) - { - if (!string.IsNullOrWhiteSpace(status) && StatusMap.TryGetValue(status.Trim(), out var mapped)) - { - return mapped; - } - - return VexClaimStatus.UnderInvestigation; - } - - private static VexJustification? MapJustification(string? justification) - { - if (string.IsNullOrWhiteSpace(justification)) - { - return null; - } - - return JustificationMap.TryGetValue(justification.Trim(), out var mapped) - ? mapped - : null; - } - - private static class OpenVexParser - { - public static OpenVexParseResult Parse(VexRawDocument document) - { - using var json = JsonDocument.Parse(document.Content.ToArray()); - var root = json.RootElement; - - var metadata = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - - var documentElement = TryGetProperty(root, "document"); - var version = TryGetString(documentElement, "version"); - var author = TryGetString(documentElement, "author"); - - if (!string.IsNullOrWhiteSpace(version)) - { - metadata["openvex.document.version"] = version!; - } - - if (!string.IsNullOrWhiteSpace(author)) - { - metadata["openvex.document.author"] = author!; - } - - var issued = ParseDate(documentElement, "issued"); - var lastUpdated = ParseDate(documentElement, "last_updated") ?? issued ?? document.RetrievedAt; - var effectiveDate = ParseDate(documentElement, "effective_date") ?? issued ?? document.RetrievedAt; - - var statements = CollectStatements(root); - - return new OpenVexParseResult( - metadata.ToImmutable(), - version, - effectiveDate, - lastUpdated, - statements); - } - - private static ImmutableArray CollectStatements(JsonElement root) - { - if (!root.TryGetProperty("statements", out var statementsElement) || - statementsElement.ValueKind != JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var statement in statementsElement.EnumerateArray()) - { - if (statement.ValueKind != JsonValueKind.Object) - { - continue; - } - - var vulnerability = TryGetString(statement, "vulnerability") ?? TryGetString(statement, "vuln") ?? string.Empty; - if (string.IsNullOrWhiteSpace(vulnerability)) - { - continue; - } - - var id = TryGetString(statement, "id") ?? Guid.NewGuid().ToString(); - var status = TryGetString(statement, "status"); - var justification = TryGetString(statement, "justification"); - var remarks = TryGetString(statement, "remediation") ?? TryGetString(statement, "statement"); - var products = CollectProducts(statement); - if (products.Length == 0) - { - continue; - } - - builder.Add(new OpenVexStatement( - id, - vulnerability.Trim(), - status, - justification, - remarks, - products)); - } - - return builder.ToImmutable(); - } - - private static ImmutableArray CollectProducts(JsonElement statement) - { - if (!statement.TryGetProperty("products", out var productsElement) || - productsElement.ValueKind != JsonValueKind.Array) - { - return ImmutableArray.Empty; - } - - var builder = ImmutableArray.CreateBuilder(); - foreach (var product in productsElement.EnumerateArray()) - { - if (product.ValueKind != JsonValueKind.String && product.ValueKind != JsonValueKind.Object) - { - continue; - } - - if (product.ValueKind == JsonValueKind.String) - { - var value = product.GetString(); - if (string.IsNullOrWhiteSpace(value)) - { - continue; - } - - builder.Add(OpenVexProduct.FromString(value.Trim())); - continue; - } - - var id = TryGetString(product, "id") ?? TryGetString(product, "product_id"); - var name = TryGetString(product, "name"); - var version = TryGetString(product, "version"); - var purl = TryGetString(product, "purl"); - var cpe = TryGetString(product, "cpe"); - - if (string.IsNullOrWhiteSpace(id) && string.IsNullOrWhiteSpace(purl)) - { - continue; - } - - builder.Add(new OpenVexProduct( - id ?? purl!, - name ?? id ?? purl!, - version, - purl, - cpe, - OriginalId: id)); - } - - return builder.ToImmutable(); - } - - private static JsonElement TryGetProperty(JsonElement element, string propertyName) - => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) - ? value - : default; - - private static string? TryGetString(JsonElement element, string propertyName) - { - if (element.ValueKind != JsonValueKind.Object) - { - return null; - } - - if (!element.TryGetProperty(propertyName, out var value)) - { - return null; - } - - return value.ValueKind == JsonValueKind.String ? value.GetString() : null; - } - - private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) - { - var value = TryGetString(element, propertyName); - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; - } - } - - private sealed record OpenVexParseResult( - ImmutableDictionary Metadata, - string? DocumentVersion, - DateTimeOffset FirstObserved, - DateTimeOffset LastObserved, - ImmutableArray Statements); - - private sealed record OpenVexStatement( - string Id, - string Vulnerability, - string? Status, - string? Justification, - string? Remarks, - ImmutableArray Products); - - private sealed record OpenVexProduct( - string Key, - string Name, - string? Version, - string? Purl, - string? Cpe, - string? OriginalId) - { - public static OpenVexProduct FromString(string value) - { - var key = value; - string? purl = null; - string? name = value; - - if (value.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase)) - { - purl = value; - } - - return new OpenVexProduct(key, name, null, purl, null, OriginalId: value); - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.OpenVEX; + +public sealed class OpenVexNormalizer : IVexNormalizer +{ + private static readonly ImmutableDictionary StatusMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["affected"] = VexClaimStatus.Affected, + ["not_affected"] = VexClaimStatus.NotAffected, + ["fixed"] = VexClaimStatus.Fixed, + ["under_investigation"] = VexClaimStatus.UnderInvestigation, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private static readonly ImmutableDictionary JustificationMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["component_not_present"] = VexJustification.ComponentNotPresent, + ["component_not_configured"] = VexJustification.ComponentNotConfigured, + ["vulnerable_code_not_present"] = VexJustification.VulnerableCodeNotPresent, + ["vulnerable_code_not_in_execute_path"] = VexJustification.VulnerableCodeNotInExecutePath, + ["vulnerable_code_cannot_be_controlled_by_adversary"] = VexJustification.VulnerableCodeCannotBeControlledByAdversary, + ["inline_mitigations_already_exist"] = VexJustification.InlineMitigationsAlreadyExist, + ["protected_by_mitigating_control"] = VexJustification.ProtectedByMitigatingControl, + ["protected_by_compensating_control"] = VexJustification.ProtectedByCompensatingControl, + ["code_not_present"] = VexJustification.CodeNotPresent, + ["code_not_reachable"] = VexJustification.CodeNotReachable, + ["requires_configuration"] = VexJustification.RequiresConfiguration, + ["requires_dependency"] = VexJustification.RequiresDependency, + ["requires_environment"] = VexJustification.RequiresEnvironment, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ILogger _logger; + + public OpenVexNormalizer(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public string Format => VexDocumentFormat.OpenVex.ToString().ToLowerInvariant(); + + public bool CanHandle(VexRawDocument document) + => document is not null && document.Format == VexDocumentFormat.OpenVex; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(provider); + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = OpenVexParser.Parse(document); + var claims = ImmutableArray.CreateBuilder(result.Statements.Length); + + foreach (var statement in result.Statements) + { + cancellationToken.ThrowIfCancellationRequested(); + + var status = MapStatus(statement.Status); + var justification = MapJustification(statement.Justification); + + foreach (var product in statement.Products) + { + var vexProduct = new VexProduct( + product.Key, + product.Name, + product.Version, + product.Purl, + product.Cpe); + + var metadata = result.Metadata; + + metadata = metadata.SetItem("openvex.statement.id", statement.Id); + if (!string.IsNullOrWhiteSpace(statement.Status)) + { + metadata = metadata.SetItem("openvex.statement.status", statement.Status!); + } + + if (!string.IsNullOrWhiteSpace(statement.Justification)) + { + metadata = metadata.SetItem("openvex.statement.justification", statement.Justification!); + } + + if (!string.IsNullOrWhiteSpace(product.OriginalId)) + { + metadata = metadata.SetItem("openvex.product.source", product.OriginalId!); + } + + var claimDocument = new VexClaimDocument( + VexDocumentFormat.OpenVex, + document.Digest, + document.SourceUri, + result.DocumentVersion, + signature: null); + + var claim = new VexClaim( + statement.Vulnerability, + provider.Id, + vexProduct, + status, + claimDocument, + result.FirstObserved, + result.LastObserved, + justification, + statement.Remarks, + confidence: null, + additionalMetadata: metadata); + + claims.Add(claim); + } + } + + var orderedClaims = claims + .ToImmutable() + .OrderBy(static claim => claim.VulnerabilityId, StringComparer.Ordinal) + .ThenBy(static claim => claim.Product.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + _logger.LogInformation( + "Normalized OpenVEX document {Source} into {ClaimCount} claim(s).", + document.SourceUri, + orderedClaims.Length); + + return ValueTask.FromResult(new VexClaimBatch( + document, + orderedClaims, + ImmutableDictionary.Empty)); + } + catch (JsonException ex) + { + _logger.LogError(ex, "Failed to parse OpenVEX document {SourceUri}", document.SourceUri); + throw; + } + } + + private static VexClaimStatus MapStatus(string? status) + { + if (!string.IsNullOrWhiteSpace(status) && StatusMap.TryGetValue(status.Trim(), out var mapped)) + { + return mapped; + } + + return VexClaimStatus.UnderInvestigation; + } + + private static VexJustification? MapJustification(string? justification) + { + if (string.IsNullOrWhiteSpace(justification)) + { + return null; + } + + return JustificationMap.TryGetValue(justification.Trim(), out var mapped) + ? mapped + : null; + } + + private static class OpenVexParser + { + public static OpenVexParseResult Parse(VexRawDocument document) + { + using var json = JsonDocument.Parse(document.Content.ToArray()); + var root = json.RootElement; + + var metadata = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + + var documentElement = TryGetProperty(root, "document"); + var version = TryGetString(documentElement, "version"); + var author = TryGetString(documentElement, "author"); + + if (!string.IsNullOrWhiteSpace(version)) + { + metadata["openvex.document.version"] = version!; + } + + if (!string.IsNullOrWhiteSpace(author)) + { + metadata["openvex.document.author"] = author!; + } + + var issued = ParseDate(documentElement, "issued"); + var lastUpdated = ParseDate(documentElement, "last_updated") ?? issued ?? document.RetrievedAt; + var effectiveDate = ParseDate(documentElement, "effective_date") ?? issued ?? document.RetrievedAt; + + var statements = CollectStatements(root); + + return new OpenVexParseResult( + metadata.ToImmutable(), + version, + effectiveDate, + lastUpdated, + statements); + } + + private static ImmutableArray CollectStatements(JsonElement root) + { + if (!root.TryGetProperty("statements", out var statementsElement) || + statementsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var statement in statementsElement.EnumerateArray()) + { + if (statement.ValueKind != JsonValueKind.Object) + { + continue; + } + + var vulnerability = TryGetString(statement, "vulnerability") ?? TryGetString(statement, "vuln") ?? string.Empty; + if (string.IsNullOrWhiteSpace(vulnerability)) + { + continue; + } + + var id = TryGetString(statement, "id") ?? Guid.NewGuid().ToString(); + var status = TryGetString(statement, "status"); + var justification = TryGetString(statement, "justification"); + var remarks = TryGetString(statement, "remediation") ?? TryGetString(statement, "statement"); + var products = CollectProducts(statement); + if (products.Length == 0) + { + continue; + } + + builder.Add(new OpenVexStatement( + id, + vulnerability.Trim(), + status, + justification, + remarks, + products)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray CollectProducts(JsonElement statement) + { + if (!statement.TryGetProperty("products", out var productsElement) || + productsElement.ValueKind != JsonValueKind.Array) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var product in productsElement.EnumerateArray()) + { + if (product.ValueKind != JsonValueKind.String && product.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (product.ValueKind == JsonValueKind.String) + { + var value = product.GetString(); + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + builder.Add(OpenVexProduct.FromString(value.Trim())); + continue; + } + + var id = TryGetString(product, "id") ?? TryGetString(product, "product_id"); + var name = TryGetString(product, "name"); + var version = TryGetString(product, "version"); + var purl = TryGetString(product, "purl"); + var cpe = TryGetString(product, "cpe"); + + if (string.IsNullOrWhiteSpace(id) && string.IsNullOrWhiteSpace(purl)) + { + continue; + } + + builder.Add(new OpenVexProduct( + id ?? purl!, + name ?? id ?? purl!, + version, + purl, + cpe, + OriginalId: id)); + } + + return builder.ToImmutable(); + } + + private static JsonElement TryGetProperty(JsonElement element, string propertyName) + => element.ValueKind == JsonValueKind.Object && element.TryGetProperty(propertyName, out var value) + ? value + : default; + + private static string? TryGetString(JsonElement element, string propertyName) + { + if (element.ValueKind != JsonValueKind.Object) + { + return null; + } + + if (!element.TryGetProperty(propertyName, out var value)) + { + return null; + } + + return value.ValueKind == JsonValueKind.String ? value.GetString() : null; + } + + private static DateTimeOffset? ParseDate(JsonElement element, string propertyName) + { + var value = TryGetString(element, propertyName); + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.TryParse(value, out var parsed) ? parsed : null; + } + } + + private sealed record OpenVexParseResult( + ImmutableDictionary Metadata, + string? DocumentVersion, + DateTimeOffset FirstObserved, + DateTimeOffset LastObserved, + ImmutableArray Statements); + + private sealed record OpenVexStatement( + string Id, + string Vulnerability, + string? Status, + string? Justification, + string? Remarks, + ImmutableArray Products); + + private sealed record OpenVexProduct( + string Key, + string Name, + string? Version, + string? Purl, + string? Cpe, + string? OriginalId) + { + public static OpenVexProduct FromString(string value) + { + var key = value; + string? purl = null; + string? name = value; + + if (value.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase)) + { + purl = value; + } + + return new OpenVexProduct(key, name, null, purl, null, OriginalId: value); + } + } +} diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Formats.OpenVEX/ServiceCollectionExtensions.cs similarity index 79% rename from src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs rename to src/StellaOps.Excititor.Formats.OpenVEX/ServiceCollectionExtensions.cs index 5866009e..392392fb 100644 --- a/src/StellaOps.Vexer.Formats.OpenVEX/ServiceCollectionExtensions.cs +++ b/src/StellaOps.Excititor.Formats.OpenVEX/ServiceCollectionExtensions.cs @@ -1,14 +1,14 @@ -using Microsoft.Extensions.DependencyInjection; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Formats.OpenVEX; - -public static class OpenVexFormatsServiceCollectionExtensions -{ - public static IServiceCollection AddOpenVexNormalizer(this IServiceCollection services) - { - ArgumentNullException.ThrowIfNull(services); - services.AddSingleton(); - return services; - } -} +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Formats.OpenVEX; + +public static class OpenVexFormatsServiceCollectionExtensions +{ + public static IServiceCollection AddOpenVexNormalizer(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj similarity index 83% rename from src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj rename to src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj index 54abf95e..a94d7788 100644 --- a/src/StellaOps.Vexer.Formats.CSAF/StellaOps.Vexer.Formats.CSAF.csproj +++ b/src/StellaOps.Excititor.Formats.OpenVEX/StellaOps.Excititor.Formats.OpenVEX.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md b/src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md new file mode 100644 index 00000000..8502c87f --- /dev/null +++ b/src/StellaOps.Excititor.Formats.OpenVEX/TASKS.md @@ -0,0 +1,7 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-FMT-OPENVEX-01-001 – OpenVEX normalizer|Team Excititor Formats|EXCITITOR-CORE-01-001|**DONE (2025-10-17)** – OpenVEX normalizer parses statements/products, maps status/justification, and surfaces provenance metadata; coverage in `OpenVexNormalizerTests`.| +|EXCITITOR-FMT-OPENVEX-01-002 – Statement merge utilities|Team Excititor Formats|EXCITITOR-FMT-OPENVEX-01-001|**DOING (2025-10-19)** – Prereq EXCITITOR-FMT-OPENVEX-01-001 confirmed DONE; building deterministic merge reducers with policy diagnostics.| +|EXCITITOR-FMT-OPENVEX-01-003 – OpenVEX export writer|Team Excititor Formats|EXCITITOR-EXPORT-01-001, EXCITITOR-FMT-OPENVEX-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-EXPORT-01-001 & EXCITITOR-FMT-OPENVEX-01-001 verified DONE; starting canonical OpenVEX exporter with stable ordering/SBOM references.| diff --git a/src/StellaOps.Vexer.Core.Tests/StellaOps.Vexer.Core.Tests.csproj b/src/StellaOps.Excititor.Policy.Tests/StellaOps.Excititor.Policy.Tests.csproj similarity index 64% rename from src/StellaOps.Vexer.Core.Tests/StellaOps.Vexer.Core.Tests.csproj rename to src/StellaOps.Excititor.Policy.Tests/StellaOps.Excititor.Policy.Tests.csproj index 1d4ebbf1..acd42b0a 100644 --- a/src/StellaOps.Vexer.Core.Tests/StellaOps.Vexer.Core.Tests.csproj +++ b/src/StellaOps.Excititor.Policy.Tests/StellaOps.Excititor.Policy.Tests.csproj @@ -1,13 +1,12 @@ - - - net10.0 - preview - enable - enable - true - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + diff --git a/src/StellaOps.Vexer.Policy.Tests/VexPolicyProviderTests.cs b/src/StellaOps.Excititor.Policy.Tests/VexPolicyProviderTests.cs similarity index 79% rename from src/StellaOps.Vexer.Policy.Tests/VexPolicyProviderTests.cs rename to src/StellaOps.Excititor.Policy.Tests/VexPolicyProviderTests.cs index 8015332e..71015463 100644 --- a/src/StellaOps.Vexer.Policy.Tests/VexPolicyProviderTests.cs +++ b/src/StellaOps.Excititor.Policy.Tests/VexPolicyProviderTests.cs @@ -1,96 +1,102 @@ -using System; -using System.Collections.Generic; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Policy; -using Xunit; - -namespace StellaOps.Vexer.Policy.Tests; - -public sealed class VexPolicyProviderTests -{ - [Fact] - public void GetSnapshot_UsesDefaultsWhenOptionsMissing() - { - var provider = new VexPolicyProvider( - new OptionsMonitorStub(new VexPolicyOptions()), - NullLogger.Instance); - - var snapshot = provider.GetSnapshot(); - - Assert.Equal(VexConsensusPolicyOptions.BaselineVersion, snapshot.Version); - Assert.Empty(snapshot.Issues); - - var evaluator = new VexPolicyEvaluator(provider); - var consensusProvider = new VexProvider("vendor", "Vendor", VexProviderKind.Vendor); - var claim = new VexClaim( - "CVE-2025-0001", - "vendor", - new VexProduct("pkg:vendor/app", "app"), - VexClaimStatus.NotAffected, - new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:test", new Uri("https://example.com")), - DateTimeOffset.UtcNow, - DateTimeOffset.UtcNow); - - Assert.Equal(1.0, evaluator.GetProviderWeight(consensusProvider)); - Assert.False(evaluator.IsClaimEligible(claim, consensusProvider, out var reason)); - Assert.Equal("missing_justification", reason); - } - - [Fact] - public void GetSnapshot_AppliesOverridesAndClampsInvalidValues() - { - var options = new VexPolicyOptions - { - Version = "custom/v1", - Weights = new VexPolicyWeightOptions - { - Vendor = 1.2, - Distro = 0.8, - }, - ProviderOverrides = new Dictionary - { - ["vendor"] = 0.95, - [" "] = 0.5, - }, - }; - - var provider = new VexPolicyProvider(new OptionsMonitorStub(options), NullLogger.Instance); - - var snapshot = provider.GetSnapshot(); - - Assert.Equal("custom/v1", snapshot.Version); - Assert.NotEmpty(snapshot.Issues); - Assert.Equal(0.95, snapshot.ConsensusOptions.ProviderOverrides["vendor"]); - - var evaluator = new VexPolicyEvaluator(provider); - var vendor = new VexProvider("vendor", "Vendor", VexProviderKind.Vendor); - Assert.Equal(0.95, evaluator.GetProviderWeight(vendor)); - } - - private sealed class OptionsMonitorStub : IOptionsMonitor - { - private readonly VexPolicyOptions _value; - - public OptionsMonitorStub(VexPolicyOptions value) - { - _value = value; - } - - public VexPolicyOptions CurrentValue => _value; - - public VexPolicyOptions Get(string? name) => _value; - - public IDisposable OnChange(Action listener) => DisposableAction.Instance; - - private sealed class DisposableAction : IDisposable - { - public static readonly DisposableAction Instance = new(); - - public void Dispose() - { - } - } - } -} +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using Xunit; + +namespace StellaOps.Excititor.Policy.Tests; + +public sealed class VexPolicyProviderTests +{ + [Fact] + public void GetSnapshot_UsesDefaultsWhenOptionsMissing() + { + var provider = new VexPolicyProvider( + new OptionsMonitorStub(new VexPolicyOptions()), + NullLogger.Instance); + + var snapshot = provider.GetSnapshot(); + + Assert.Equal(VexConsensusPolicyOptions.BaselineVersion, snapshot.Version); + Assert.Empty(snapshot.Issues); + Assert.Equal(VexConsensusPolicyOptions.DefaultWeightCeiling, snapshot.ConsensusOptions.WeightCeiling); + Assert.Equal(VexConsensusPolicyOptions.DefaultAlpha, snapshot.ConsensusOptions.Alpha); + Assert.Equal(VexConsensusPolicyOptions.DefaultBeta, snapshot.ConsensusOptions.Beta); + + var evaluator = new VexPolicyEvaluator(provider); + var consensusProvider = new VexProvider("vendor", "Vendor", VexProviderKind.Vendor); + var claim = new VexClaim( + "CVE-2025-0001", + "vendor", + new VexProduct("pkg:vendor/app", "app"), + VexClaimStatus.NotAffected, + new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:test", new Uri("https://example.com")), + DateTimeOffset.UtcNow, + DateTimeOffset.UtcNow); + + Assert.Equal(1.0, evaluator.GetProviderWeight(consensusProvider)); + Assert.False(evaluator.IsClaimEligible(claim, consensusProvider, out var reason)); + Assert.Equal("missing_justification", reason); + } + + [Fact] + public void GetSnapshot_AppliesOverridesAndClampsInvalidValues() + { + var options = new VexPolicyOptions + { + Version = "custom/v1", + Weights = new VexPolicyWeightOptions + { + Vendor = 1.2, + Distro = 0.8, + }, + ProviderOverrides = new Dictionary + { + ["vendor"] = 0.95, + [" "] = 0.5, + }, + }; + + var provider = new VexPolicyProvider(new OptionsMonitorStub(options), NullLogger.Instance); + + var snapshot = provider.GetSnapshot(); + + Assert.Equal("custom/v1", snapshot.Version); + Assert.NotEmpty(snapshot.Issues); + Assert.Equal(0.95, snapshot.ConsensusOptions.ProviderOverrides["vendor"]); + Assert.Contains(snapshot.Issues, issue => issue.Code == "weights.vendor.range"); + Assert.Equal(VexConsensusPolicyOptions.DefaultWeightCeiling, snapshot.ConsensusOptions.WeightCeiling); + Assert.Equal(1.0, snapshot.ConsensusOptions.VendorWeight); + + var evaluator = new VexPolicyEvaluator(provider); + var vendor = new VexProvider("vendor", "Vendor", VexProviderKind.Vendor); + Assert.Equal(0.95, evaluator.GetProviderWeight(vendor)); + } + + private sealed class OptionsMonitorStub : IOptionsMonitor + { + private readonly VexPolicyOptions _value; + + public OptionsMonitorStub(VexPolicyOptions value) + { + _value = value; + } + + public VexPolicyOptions CurrentValue => _value; + + public VexPolicyOptions Get(string? name) => _value; + + public IDisposable OnChange(Action listener) => DisposableAction.Instance; + + private sealed class DisposableAction : IDisposable + { + public static readonly DisposableAction Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Vexer.Policy/AGENTS.md b/src/StellaOps.Excititor.Policy/AGENTS.md similarity index 88% rename from src/StellaOps.Vexer.Policy/AGENTS.md rename to src/StellaOps.Excititor.Policy/AGENTS.md index 87c91367..056b5193 100644 --- a/src/StellaOps.Vexer.Policy/AGENTS.md +++ b/src/StellaOps.Excititor.Policy/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Centralizes policy configuration, provider trust weights, and justification guardrails applied to Vexer consensus decisions. -## Scope -- Policy models for tier weighting, provider overrides, justification allowlists, and conflict escalation. -- Configuration binding helpers (YAML/JSON) and validation of operator-supplied policy bundles. -- Evaluation services that expose policy revisions and change tracking to WebService/Worker. -- Documentation anchors for policy schema and upgrade guidance. -## Participants -- WebService consumes policy bindings to authorize ingest/export operations and to recompute consensus. -- Worker schedules reconciliation runs using policy revisions from this module. -- CLI exposes policy inspection commands based on exported descriptors. -## Interfaces & contracts -- `IVexPolicyProvider`, `IVexPolicyEvaluator`, and immutable policy snapshot value objects. -- Validation diagnostics APIs surfacing structured errors and warnings for operators. -## In/Out of scope -In: policy schema definition, binding/validation, evaluation utilities, audit logging helpers. -Out: persistence/migrations, HTTP exposure, connector-specific trust logic (lives in Core/Connectors). -## Observability & security expectations -- Emit structured events on policy load/update with revision IDs, but do not log full sensitive policy documents. -- Maintain deterministic error ordering for reproducible diagnostics. -## Tests -- Policy fixtures and regression coverage will live in `../StellaOps.Vexer.Policy.Tests` once scaffolded; leverage snapshot comparisons for YAML bindings. +# AGENTS +## Role +Centralizes policy configuration, provider trust weights, and justification guardrails applied to Excititor consensus decisions. +## Scope +- Policy models for tier weighting, provider overrides, justification allowlists, and conflict escalation. +- Configuration binding helpers (YAML/JSON) and validation of operator-supplied policy bundles. +- Evaluation services that expose policy revisions and change tracking to WebService/Worker. +- Documentation anchors for policy schema and upgrade guidance. +## Participants +- WebService consumes policy bindings to authorize ingest/export operations and to recompute consensus. +- Worker schedules reconciliation runs using policy revisions from this module. +- CLI exposes policy inspection commands based on exported descriptors. +## Interfaces & contracts +- `IVexPolicyProvider`, `IVexPolicyEvaluator`, and immutable policy snapshot value objects. +- Validation diagnostics APIs surfacing structured errors and warnings for operators. +## In/Out of scope +In: policy schema definition, binding/validation, evaluation utilities, audit logging helpers. +Out: persistence/migrations, HTTP exposure, connector-specific trust logic (lives in Core/Connectors). +## Observability & security expectations +- Emit structured events on policy load/update with revision IDs, but do not log full sensitive policy documents. +- Maintain deterministic error ordering for reproducible diagnostics. +## Tests +- Policy fixtures and regression coverage will live in `../StellaOps.Excititor.Policy.Tests` once scaffolded; leverage snapshot comparisons for YAML bindings. diff --git a/src/StellaOps.Vexer.Policy/IVexPolicyProvider.cs b/src/StellaOps.Excititor.Policy/IVexPolicyProvider.cs similarity index 95% rename from src/StellaOps.Vexer.Policy/IVexPolicyProvider.cs rename to src/StellaOps.Excititor.Policy/IVexPolicyProvider.cs index b0a675f8..30d35d03 100644 --- a/src/StellaOps.Vexer.Policy/IVexPolicyProvider.cs +++ b/src/StellaOps.Excititor.Policy/IVexPolicyProvider.cs @@ -1,161 +1,161 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Policy; - -public interface IVexPolicyProvider -{ - VexPolicySnapshot GetSnapshot(); -} - -public interface IVexPolicyEvaluator -{ - string Version { get; } - - VexPolicySnapshot Snapshot { get; } - - double GetProviderWeight(VexProvider provider); - - bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason); -} - -public sealed record VexPolicySnapshot( - string Version, - VexConsensusPolicyOptions ConsensusOptions, - IVexConsensusPolicy ConsensusPolicy, - ImmutableArray Issues, - string RevisionId, - string Digest) -{ - public static readonly VexPolicySnapshot Default = new( - VexConsensusPolicyOptions.BaselineVersion, - new VexConsensusPolicyOptions(), - new BaselineVexConsensusPolicy(), - ImmutableArray.Empty, - "rev-0", - string.Empty); -} - -public sealed record VexPolicyIssue( - string Code, - string Message, - VexPolicyIssueSeverity Severity); - -public enum VexPolicyIssueSeverity -{ - Warning, - Error, -} - -public sealed class VexPolicyProvider : IVexPolicyProvider -{ - private readonly IOptionsMonitor _options; - private readonly ILogger _logger; - private readonly object _sync = new(); - private long _revisionCounter; - private string? _currentRevisionId; - private string? _currentDigest; - - public VexPolicyProvider( - IOptionsMonitor options, - ILogger logger) - { - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public VexPolicySnapshot GetSnapshot() - { - var options = _options.CurrentValue ?? new VexPolicyOptions(); - return BuildSnapshot(options); - } - - private VexPolicySnapshot BuildSnapshot(VexPolicyOptions options) - { - var normalization = VexPolicyProcessing.Normalize(options); - var digest = VexPolicyDigest.Compute(normalization.ConsensusOptions); - string revisionId; - bool isNewRevision; - - lock (_sync) - { - if (!string.Equals(_currentDigest, digest, StringComparison.Ordinal)) - { - _revisionCounter++; - revisionId = $"rev-{_revisionCounter}"; - _currentDigest = digest; - _currentRevisionId = revisionId; - isNewRevision = true; - } - else - { - revisionId = _currentRevisionId ?? "rev-0"; - isNewRevision = false; - } - } - - var policy = new BaselineVexConsensusPolicy(normalization.ConsensusOptions); - var snapshot = new VexPolicySnapshot( - normalization.ConsensusOptions.Version, - normalization.ConsensusOptions, - policy, - normalization.Issues, - revisionId, - digest); - - if (isNewRevision) - { - _logger.LogInformation( - "Policy snapshot updated: revision {RevisionId}, version {Version}, digest {Digest}, issues {IssueCount}", - snapshot.RevisionId, - snapshot.Version, - snapshot.Digest, - snapshot.Issues.Length); - VexPolicyTelemetry.RecordReload(snapshot.RevisionId, snapshot.Version, snapshot.Issues.Length); - } - else if (snapshot.Issues.Length > 0) - { - foreach (var issue in snapshot.Issues) - { - _logger.LogWarning("Policy issue {Code}: {Message}", issue.Code, issue.Message); - } - } - - return snapshot; - } -} - -public sealed class VexPolicyEvaluator : IVexPolicyEvaluator -{ - private readonly IVexPolicyProvider _provider; - - public VexPolicyEvaluator(IVexPolicyProvider provider) - { - _provider = provider ?? throw new ArgumentNullException(nameof(provider)); - } - - public string Version => Snapshot.Version; - - public VexPolicySnapshot Snapshot => _provider.GetSnapshot(); - - public double GetProviderWeight(VexProvider provider) - => Snapshot.ConsensusPolicy.GetProviderWeight(provider); - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - => Snapshot.ConsensusPolicy.IsClaimEligible(claim, provider, out rejectionReason); -} - -public static class VexPolicyServiceCollectionExtensions -{ - public static IServiceCollection AddVexPolicy(this IServiceCollection services) - { - services.AddSingleton(); - services.AddSingleton(); - return services; - } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Policy; + +public interface IVexPolicyProvider +{ + VexPolicySnapshot GetSnapshot(); +} + +public interface IVexPolicyEvaluator +{ + string Version { get; } + + VexPolicySnapshot Snapshot { get; } + + double GetProviderWeight(VexProvider provider); + + bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason); +} + +public sealed record VexPolicySnapshot( + string Version, + VexConsensusPolicyOptions ConsensusOptions, + IVexConsensusPolicy ConsensusPolicy, + ImmutableArray Issues, + string RevisionId, + string Digest) +{ + public static readonly VexPolicySnapshot Default = new( + VexConsensusPolicyOptions.BaselineVersion, + new VexConsensusPolicyOptions(), + new BaselineVexConsensusPolicy(), + ImmutableArray.Empty, + "rev-0", + string.Empty); +} + +public sealed record VexPolicyIssue( + string Code, + string Message, + VexPolicyIssueSeverity Severity); + +public enum VexPolicyIssueSeverity +{ + Warning, + Error, +} + +public sealed class VexPolicyProvider : IVexPolicyProvider +{ + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + private readonly object _sync = new(); + private long _revisionCounter; + private string? _currentRevisionId; + private string? _currentDigest; + + public VexPolicyProvider( + IOptionsMonitor options, + ILogger logger) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public VexPolicySnapshot GetSnapshot() + { + var options = _options.CurrentValue ?? new VexPolicyOptions(); + return BuildSnapshot(options); + } + + private VexPolicySnapshot BuildSnapshot(VexPolicyOptions options) + { + var normalization = VexPolicyProcessing.Normalize(options); + var digest = VexPolicyDigest.Compute(normalization.ConsensusOptions); + string revisionId; + bool isNewRevision; + + lock (_sync) + { + if (!string.Equals(_currentDigest, digest, StringComparison.Ordinal)) + { + _revisionCounter++; + revisionId = $"rev-{_revisionCounter}"; + _currentDigest = digest; + _currentRevisionId = revisionId; + isNewRevision = true; + } + else + { + revisionId = _currentRevisionId ?? "rev-0"; + isNewRevision = false; + } + } + + var policy = new BaselineVexConsensusPolicy(normalization.ConsensusOptions); + var snapshot = new VexPolicySnapshot( + normalization.ConsensusOptions.Version, + normalization.ConsensusOptions, + policy, + normalization.Issues, + revisionId, + digest); + + if (isNewRevision) + { + _logger.LogInformation( + "Policy snapshot updated: revision {RevisionId}, version {Version}, digest {Digest}, issues {IssueCount}", + snapshot.RevisionId, + snapshot.Version, + snapshot.Digest, + snapshot.Issues.Length); + VexPolicyTelemetry.RecordReload(snapshot.RevisionId, snapshot.Version, snapshot.Issues.Length); + } + else if (snapshot.Issues.Length > 0) + { + foreach (var issue in snapshot.Issues) + { + _logger.LogWarning("Policy issue {Code}: {Message}", issue.Code, issue.Message); + } + } + + return snapshot; + } +} + +public sealed class VexPolicyEvaluator : IVexPolicyEvaluator +{ + private readonly IVexPolicyProvider _provider; + + public VexPolicyEvaluator(IVexPolicyProvider provider) + { + _provider = provider ?? throw new ArgumentNullException(nameof(provider)); + } + + public string Version => Snapshot.Version; + + public VexPolicySnapshot Snapshot => _provider.GetSnapshot(); + + public double GetProviderWeight(VexProvider provider) + => Snapshot.ConsensusPolicy.GetProviderWeight(provider); + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + => Snapshot.ConsensusPolicy.IsClaimEligible(claim, provider, out rejectionReason); +} + +public static class VexPolicyServiceCollectionExtensions +{ + public static IServiceCollection AddVexPolicy(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Policy/StellaOps.Vexer.Policy.csproj b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj similarity index 77% rename from src/StellaOps.Vexer.Policy/StellaOps.Vexer.Policy.csproj rename to src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj index 95980268..b4fef087 100644 --- a/src/StellaOps.Vexer.Policy/StellaOps.Vexer.Policy.csproj +++ b/src/StellaOps.Excititor.Policy/StellaOps.Excititor.Policy.csproj @@ -1,17 +1,17 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Policy/TASKS.md b/src/StellaOps.Excititor.Policy/TASKS.md new file mode 100644 index 00000000..423e7f0a --- /dev/null +++ b/src/StellaOps.Excititor.Policy/TASKS.md @@ -0,0 +1,11 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-POLICY-01-001 – Policy schema & binding|Team Excititor Policy|EXCITITOR-CORE-01-001|DONE (2025-10-15) – Established `VexPolicyOptions`, options binding, and snapshot provider covering baseline weights/overrides.| +|EXCITITOR-POLICY-01-002 – Policy evaluator service|Team Excititor Policy|EXCITITOR-POLICY-01-001|DONE (2025-10-15) – `VexPolicyEvaluator` exposes immutable snapshots to consensus and normalizes rejection reasons.| +|EXCITITOR-POLICY-01-003 – Operator diagnostics & docs|Team Excititor Policy|EXCITITOR-POLICY-01-001|**DONE (2025-10-16)** – Surface structured diagnostics (CLI/WebService) and author policy upgrade guidance in docs/ARCHITECTURE_EXCITITOR.md appendix.
          2025-10-16: Added `IVexPolicyDiagnostics`/`VexPolicyDiagnosticsReport`, sorted issue ordering, recommendations, and appendix guidance. Tests: `dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`.| +|EXCITITOR-POLICY-01-004 – Policy schema validation & YAML binding|Team Excititor Policy|EXCITITOR-POLICY-01-001|**DONE (2025-10-16)** – Added strongly-typed YAML/JSON binding, schema validation, and deterministic diagnostics for operator-supplied policy bundles.| +|EXCITITOR-POLICY-01-005 – Policy change tracking & telemetry|Team Excititor Policy|EXCITITOR-POLICY-01-002|**DONE (2025-10-16)** – Emit revision history, expose snapshot digests via CLI/WebService, and add structured logging/metrics for policy reloads.
          2025-10-16: `VexPolicySnapshot` now carries revision/digest, provider logs reloads, `vex.policy.reloads` metric emitted, binder/diagnostics expose digest metadata. Tests: `dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj`.| +|EXCITITOR-POLICY-02-001 – Scoring coefficients & weight ceilings|Team Excititor Policy|EXCITITOR-POLICY-01-004|DONE (2025-10-19) – Added `weights.ceiling` + `scoring.{alpha,beta}` options with normalization warnings, extended consensus policy/digest, refreshed docs (`docs/ARCHITECTURE_EXCITITOR.md`, `docs/EXCITITOR_SCORRING.md`), and validated via `dotnet test` for core/policy suites.| +|EXCITITOR-POLICY-02-002 – Diagnostics for scoring signals|Team Excititor Policy|EXCITITOR-POLICY-02-001|BACKLOG – Update diagnostics reports to surface missing severity/KEV/EPSS mappings, coefficient overrides, and provide actionable recommendations for policy tuning.| diff --git a/src/StellaOps.Vexer.Policy/VexPolicyBinder.cs b/src/StellaOps.Excititor.Policy/VexPolicyBinder.cs similarity index 95% rename from src/StellaOps.Vexer.Policy/VexPolicyBinder.cs rename to src/StellaOps.Excititor.Policy/VexPolicyBinder.cs index 5e09de5b..c6d7cef0 100644 --- a/src/StellaOps.Vexer.Policy/VexPolicyBinder.cs +++ b/src/StellaOps.Excititor.Policy/VexPolicyBinder.cs @@ -1,94 +1,94 @@ -using System.Collections.Immutable; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.Json; -using StellaOps.Vexer.Core; -using YamlDotNet.Serialization; -using YamlDotNet.Serialization.NamingConventions; - -namespace StellaOps.Vexer.Policy; - -public enum VexPolicyDocumentFormat -{ - Json, - Yaml, -} - -public sealed record VexPolicyBindingResult( - bool Success, - VexPolicyOptions? Options, - VexConsensusPolicyOptions? NormalizedOptions, - ImmutableArray Issues); - -public static class VexPolicyBinder -{ - public static VexPolicyBindingResult Bind(string content, VexPolicyDocumentFormat format) - { - if (string.IsNullOrWhiteSpace(content)) - { - return Failure("policy.empty", "Policy document is empty."); - } - - try - { - var options = Parse(content, format); - return Normalize(options); - } - catch (JsonException ex) - { - return Failure("policy.parse.json", $"Failed to parse JSON policy document: {ex.Message}"); - } - catch (YamlDotNet.Core.YamlException ex) - { - return Failure("policy.parse.yaml", $"Failed to parse YAML policy document: {ex.Message}"); - } - } - - public static VexPolicyBindingResult Bind(Stream stream, VexPolicyDocumentFormat format, Encoding? encoding = null) - { - if (stream is null) - { - throw new ArgumentNullException(nameof(stream)); - } - - encoding ??= Encoding.UTF8; - using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, leaveOpen: true); - var content = reader.ReadToEnd(); - return Bind(content, format); - } - - private static VexPolicyBindingResult Normalize(VexPolicyOptions options) - { - var normalization = VexPolicyProcessing.Normalize(options); - var hasErrors = normalization.Issues.Any(static issue => issue.Severity == VexPolicyIssueSeverity.Error); - return new VexPolicyBindingResult(!hasErrors, options, normalization.ConsensusOptions, normalization.Issues); - } - - private static VexPolicyBindingResult Failure(string code, string message) - { - var issue = new VexPolicyIssue(code, message, VexPolicyIssueSeverity.Error); - return new VexPolicyBindingResult(false, null, null, ImmutableArray.Create(issue)); - } - - private static VexPolicyOptions Parse(string content, VexPolicyDocumentFormat format) - { - return format switch - { - VexPolicyDocumentFormat.Json => JsonSerializer.Deserialize(content, new JsonSerializerOptions - { - PropertyNameCaseInsensitive = true, - ReadCommentHandling = JsonCommentHandling.Skip, - AllowTrailingCommas = true, - }) ?? new VexPolicyOptions(), - VexPolicyDocumentFormat.Yaml => BuildYamlDeserializer().Deserialize(content) ?? new VexPolicyOptions(), - _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported policy document format."), - }; - } - - private static IDeserializer BuildYamlDeserializer() - => new DeserializerBuilder() - .WithNamingConvention(CamelCaseNamingConvention.Instance) - .IgnoreUnmatchedProperties() - .Build(); -} +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using StellaOps.Excititor.Core; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Excititor.Policy; + +public enum VexPolicyDocumentFormat +{ + Json, + Yaml, +} + +public sealed record VexPolicyBindingResult( + bool Success, + VexPolicyOptions? Options, + VexConsensusPolicyOptions? NormalizedOptions, + ImmutableArray Issues); + +public static class VexPolicyBinder +{ + public static VexPolicyBindingResult Bind(string content, VexPolicyDocumentFormat format) + { + if (string.IsNullOrWhiteSpace(content)) + { + return Failure("policy.empty", "Policy document is empty."); + } + + try + { + var options = Parse(content, format); + return Normalize(options); + } + catch (JsonException ex) + { + return Failure("policy.parse.json", $"Failed to parse JSON policy document: {ex.Message}"); + } + catch (YamlDotNet.Core.YamlException ex) + { + return Failure("policy.parse.yaml", $"Failed to parse YAML policy document: {ex.Message}"); + } + } + + public static VexPolicyBindingResult Bind(Stream stream, VexPolicyDocumentFormat format, Encoding? encoding = null) + { + if (stream is null) + { + throw new ArgumentNullException(nameof(stream)); + } + + encoding ??= Encoding.UTF8; + using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var content = reader.ReadToEnd(); + return Bind(content, format); + } + + private static VexPolicyBindingResult Normalize(VexPolicyOptions options) + { + var normalization = VexPolicyProcessing.Normalize(options); + var hasErrors = normalization.Issues.Any(static issue => issue.Severity == VexPolicyIssueSeverity.Error); + return new VexPolicyBindingResult(!hasErrors, options, normalization.ConsensusOptions, normalization.Issues); + } + + private static VexPolicyBindingResult Failure(string code, string message) + { + var issue = new VexPolicyIssue(code, message, VexPolicyIssueSeverity.Error); + return new VexPolicyBindingResult(false, null, null, ImmutableArray.Create(issue)); + } + + private static VexPolicyOptions Parse(string content, VexPolicyDocumentFormat format) + { + return format switch + { + VexPolicyDocumentFormat.Json => JsonSerializer.Deserialize(content, new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + }) ?? new VexPolicyOptions(), + VexPolicyDocumentFormat.Yaml => BuildYamlDeserializer().Deserialize(content) ?? new VexPolicyOptions(), + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported policy document format."), + }; + } + + private static IDeserializer BuildYamlDeserializer() + => new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); +} diff --git a/src/StellaOps.Vexer.Policy/VexPolicyDiagnostics.cs b/src/StellaOps.Excititor.Policy/VexPolicyDiagnostics.cs similarity index 91% rename from src/StellaOps.Vexer.Policy/VexPolicyDiagnostics.cs rename to src/StellaOps.Excititor.Policy/VexPolicyDiagnostics.cs index da4ce4b8..2140c5ed 100644 --- a/src/StellaOps.Vexer.Policy/VexPolicyDiagnostics.cs +++ b/src/StellaOps.Excititor.Policy/VexPolicyDiagnostics.cs @@ -1,87 +1,87 @@ -using System; -using System.Collections.Immutable; -using System.Linq; - -namespace StellaOps.Vexer.Policy; - -public interface IVexPolicyDiagnostics -{ - VexPolicyDiagnosticsReport GetDiagnostics(); -} - -public sealed record VexPolicyDiagnosticsReport( - string Version, - string RevisionId, - string Digest, - int ErrorCount, - int WarningCount, - DateTimeOffset GeneratedAt, - ImmutableArray Issues, - ImmutableArray Recommendations, - ImmutableDictionary ActiveOverrides); - -public sealed class VexPolicyDiagnostics : IVexPolicyDiagnostics -{ - private readonly IVexPolicyProvider _policyProvider; - private readonly TimeProvider _timeProvider; - - public VexPolicyDiagnostics( - IVexPolicyProvider policyProvider, - TimeProvider? timeProvider = null) - { - _policyProvider = policyProvider ?? throw new ArgumentNullException(nameof(policyProvider)); - _timeProvider = timeProvider ?? TimeProvider.System; - } - - public VexPolicyDiagnosticsReport GetDiagnostics() - { - var snapshot = _policyProvider.GetSnapshot(); - var issues = snapshot.Issues; - - var errorCount = issues.Count(static issue => issue.Severity == VexPolicyIssueSeverity.Error); - var warningCount = issues.Count(static issue => issue.Severity == VexPolicyIssueSeverity.Warning); - var overrides = snapshot.ConsensusOptions.ProviderOverrides - .OrderBy(static pair => pair.Key, StringComparer.Ordinal) - .ToImmutableDictionary(); - - var recommendations = BuildRecommendations(errorCount, warningCount, overrides); - - return new VexPolicyDiagnosticsReport( - snapshot.Version, - snapshot.RevisionId, - snapshot.Digest, - errorCount, - warningCount, - _timeProvider.GetUtcNow(), - issues, - recommendations, - overrides); - } - - private static ImmutableArray BuildRecommendations( - int errorCount, - int warningCount, - ImmutableDictionary overrides) - { - var messages = ImmutableArray.CreateBuilder(); - - if (errorCount > 0) - { - messages.Add("Resolve policy errors before running consensus; defaults are used while errors persist."); - } - - if (warningCount > 0) - { - messages.Add("Review policy warnings via CLI/Web diagnostics and adjust configuration as needed."); - } - - if (overrides.Count > 0) - { - messages.Add($"Provider overrides active for: {string.Join(", ", overrides.Keys)}."); - } - - messages.Add("Refer to docs/ARCHITECTURE_VEXER.md for policy upgrade and diagnostics guidance."); - - return messages.ToImmutable(); - } -} +using System; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Excititor.Policy; + +public interface IVexPolicyDiagnostics +{ + VexPolicyDiagnosticsReport GetDiagnostics(); +} + +public sealed record VexPolicyDiagnosticsReport( + string Version, + string RevisionId, + string Digest, + int ErrorCount, + int WarningCount, + DateTimeOffset GeneratedAt, + ImmutableArray Issues, + ImmutableArray Recommendations, + ImmutableDictionary ActiveOverrides); + +public sealed class VexPolicyDiagnostics : IVexPolicyDiagnostics +{ + private readonly IVexPolicyProvider _policyProvider; + private readonly TimeProvider _timeProvider; + + public VexPolicyDiagnostics( + IVexPolicyProvider policyProvider, + TimeProvider? timeProvider = null) + { + _policyProvider = policyProvider ?? throw new ArgumentNullException(nameof(policyProvider)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public VexPolicyDiagnosticsReport GetDiagnostics() + { + var snapshot = _policyProvider.GetSnapshot(); + var issues = snapshot.Issues; + + var errorCount = issues.Count(static issue => issue.Severity == VexPolicyIssueSeverity.Error); + var warningCount = issues.Count(static issue => issue.Severity == VexPolicyIssueSeverity.Warning); + var overrides = snapshot.ConsensusOptions.ProviderOverrides + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToImmutableDictionary(); + + var recommendations = BuildRecommendations(errorCount, warningCount, overrides); + + return new VexPolicyDiagnosticsReport( + snapshot.Version, + snapshot.RevisionId, + snapshot.Digest, + errorCount, + warningCount, + _timeProvider.GetUtcNow(), + issues, + recommendations, + overrides); + } + + private static ImmutableArray BuildRecommendations( + int errorCount, + int warningCount, + ImmutableDictionary overrides) + { + var messages = ImmutableArray.CreateBuilder(); + + if (errorCount > 0) + { + messages.Add("Resolve policy errors before running consensus; defaults are used while errors persist."); + } + + if (warningCount > 0) + { + messages.Add("Review policy warnings via CLI/Web diagnostics and adjust configuration as needed."); + } + + if (overrides.Count > 0) + { + messages.Add($"Provider overrides active for: {string.Join(", ", overrides.Keys)}."); + } + + messages.Add("Refer to docs/ARCHITECTURE_EXCITITOR.md for policy upgrade and diagnostics guidance."); + + return messages.ToImmutable(); + } +} diff --git a/src/StellaOps.Vexer.Policy/VexPolicyDigest.cs b/src/StellaOps.Excititor.Policy/VexPolicyDigest.cs similarity index 75% rename from src/StellaOps.Vexer.Policy/VexPolicyDigest.cs rename to src/StellaOps.Excititor.Policy/VexPolicyDigest.cs index fa1e0480..857de709 100644 --- a/src/StellaOps.Vexer.Policy/VexPolicyDigest.cs +++ b/src/StellaOps.Excititor.Policy/VexPolicyDigest.cs @@ -1,35 +1,38 @@ -using System.Globalization; -using System.Security.Cryptography; -using System.Text; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Policy; - -internal static class VexPolicyDigest -{ - public static string Compute(VexConsensusPolicyOptions options) - { - ArgumentNullException.ThrowIfNull(options); - - var builder = new StringBuilder(); - builder.Append(options.Version).Append('|') - .Append(options.VendorWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') - .Append(options.DistroWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') - .Append(options.PlatformWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') - .Append(options.HubWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') - .Append(options.AttestationWeight.ToString("F6", CultureInfo.InvariantCulture)); - - foreach (var kvp in options.ProviderOverrides - .OrderBy(static pair => pair.Key, StringComparer.Ordinal)) - { - builder.Append('|') - .Append(kvp.Key) - .Append('=') - .Append(kvp.Value.ToString("F6", CultureInfo.InvariantCulture)); - } - - var input = builder.ToString(); - var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); - return Convert.ToHexString(hash); - } -} +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Policy; + +internal static class VexPolicyDigest +{ + public static string Compute(VexConsensusPolicyOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var builder = new StringBuilder(); + builder.Append(options.Version).Append('|') + .Append(options.VendorWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.DistroWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.PlatformWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.HubWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.AttestationWeight.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.WeightCeiling.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.Alpha.ToString("F6", CultureInfo.InvariantCulture)).Append('|') + .Append(options.Beta.ToString("F6", CultureInfo.InvariantCulture)); + + foreach (var kvp in options.ProviderOverrides + .OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + builder.Append('|') + .Append(kvp.Key) + .Append('=') + .Append(kvp.Value.ToString("F6", CultureInfo.InvariantCulture)); + } + + var input = builder.ToString(); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexString(hash); + } +} diff --git a/src/StellaOps.Vexer.Policy/VexPolicyOptions.cs b/src/StellaOps.Excititor.Policy/VexPolicyOptions.cs similarity index 62% rename from src/StellaOps.Vexer.Policy/VexPolicyOptions.cs rename to src/StellaOps.Excititor.Policy/VexPolicyOptions.cs index 527661da..33b480fe 100644 --- a/src/StellaOps.Vexer.Policy/VexPolicyOptions.cs +++ b/src/StellaOps.Excititor.Policy/VexPolicyOptions.cs @@ -1,25 +1,36 @@ -using System.Collections.Generic; - -namespace StellaOps.Vexer.Policy; - -public sealed class VexPolicyOptions -{ - public string? Version { get; set; } - - public VexPolicyWeightOptions Weights { get; set; } = new(); - - public IDictionary? ProviderOverrides { get; set; } -} - -public sealed class VexPolicyWeightOptions -{ - public double? Vendor { get; set; } - - public double? Distro { get; set; } - - public double? Platform { get; set; } - - public double? Hub { get; set; } - - public double? Attestation { get; set; } -} +using System.Collections.Generic; + +namespace StellaOps.Excititor.Policy; + +public sealed class VexPolicyOptions +{ + public string? Version { get; set; } + + public VexPolicyWeightOptions Weights { get; set; } = new(); + + public VexPolicyScoringOptions Scoring { get; set; } = new(); + + public IDictionary? ProviderOverrides { get; set; } +} + +public sealed class VexPolicyWeightOptions +{ + public double? Vendor { get; set; } + + public double? Distro { get; set; } + + public double? Platform { get; set; } + + public double? Hub { get; set; } + + public double? Attestation { get; set; } + + public double? Ceiling { get; set; } +} + +public sealed class VexPolicyScoringOptions +{ + public double? Alpha { get; set; } + + public double? Beta { get; set; } +} diff --git a/src/StellaOps.Excititor.Policy/VexPolicyProcessing.cs b/src/StellaOps.Excititor.Policy/VexPolicyProcessing.cs new file mode 100644 index 00000000..c6061e72 --- /dev/null +++ b/src/StellaOps.Excititor.Policy/VexPolicyProcessing.cs @@ -0,0 +1,282 @@ +using System.Collections.Immutable; +using System.Globalization; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Policy; + +internal static class VexPolicyProcessing +{ + private const double DefaultVendorWeight = 1.0; + private const double DefaultDistroWeight = 0.9; + private const double DefaultPlatformWeight = 0.7; + private const double DefaultHubWeight = 0.5; + private const double DefaultAttestationWeight = 0.6; + + public static VexPolicyNormalizationResult Normalize(VexPolicyOptions? options) + { + var issues = ImmutableArray.CreateBuilder(); + + var policyOptions = options ?? new VexPolicyOptions(); + + var normalizedWeights = NormalizeWeights(policyOptions.Weights, issues); + var overrides = NormalizeOverrides(policyOptions.ProviderOverrides, normalizedWeights.Ceiling, issues); + var scoring = NormalizeScoring(policyOptions.Scoring, issues); + + var consensusOptions = new VexConsensusPolicyOptions( + policyOptions.Version ?? VexConsensusPolicyOptions.BaselineVersion, + normalizedWeights.Vendor, + normalizedWeights.Distro, + normalizedWeights.Platform, + normalizedWeights.Hub, + normalizedWeights.Attestation, + overrides, + normalizedWeights.Ceiling, + scoring.Alpha, + scoring.Beta); + + var orderedIssues = issues.ToImmutable().Sort(IssueComparer); + + return new VexPolicyNormalizationResult(consensusOptions, orderedIssues); + } + + public static ImmutableArray SortIssues(IEnumerable issues) + => issues.ToImmutableArray().Sort(IssueComparer); + + private static WeightNormalizationResult NormalizeWeights( + VexPolicyWeightOptions? options, + ImmutableArray.Builder issues) + { + var ceiling = NormalizeWeightCeiling(options?.Ceiling, issues); + + var vendor = NormalizeWeightValue( + options?.Vendor, + "vendor", + DefaultVendorWeight, + ceiling, + issues); + var distro = NormalizeWeightValue( + options?.Distro, + "distro", + DefaultDistroWeight, + ceiling, + issues); + var platform = NormalizeWeightValue( + options?.Platform, + "platform", + DefaultPlatformWeight, + ceiling, + issues); + var hub = NormalizeWeightValue( + options?.Hub, + "hub", + DefaultHubWeight, + ceiling, + issues); + var attestation = NormalizeWeightValue( + options?.Attestation, + "attestation", + DefaultAttestationWeight, + ceiling, + issues); + + return new WeightNormalizationResult(vendor, distro, platform, hub, attestation, ceiling); + } + + private static double NormalizeWeightValue( + double? value, + string fieldName, + double defaultValue, + double ceiling, + ImmutableArray.Builder issues) + { + if (value is null) + { + return defaultValue; + } + + if (double.IsNaN(value.Value) || double.IsInfinity(value.Value)) + { + issues.Add(new VexPolicyIssue( + $"weights.{fieldName}.invalid", + $"{fieldName} must be a finite number; default {defaultValue.ToString(CultureInfo.InvariantCulture)} applied.", + VexPolicyIssueSeverity.Warning)); + return defaultValue; + } + + if (value.Value < 0 || value.Value > ceiling) + { + issues.Add(new VexPolicyIssue( + $"weights.{fieldName}.range", + $"{fieldName} must be between 0 and {ceiling.ToString(CultureInfo.InvariantCulture)}; value {value.Value.ToString(CultureInfo.InvariantCulture)} was clamped.", + VexPolicyIssueSeverity.Warning)); + return Math.Clamp(value.Value, 0, ceiling); + } + + return value.Value; + } + + private static double NormalizeWeightCeiling(double? ceiling, ImmutableArray.Builder issues) + { + if (ceiling is null) + { + return VexConsensusPolicyOptions.DefaultWeightCeiling; + } + + if (double.IsNaN(ceiling.Value) || double.IsInfinity(ceiling.Value) || ceiling.Value <= 0) + { + issues.Add(new VexPolicyIssue( + "weights.ceiling.invalid", + "weights.ceiling must be a positive, finite number; default ceiling applied.", + VexPolicyIssueSeverity.Warning)); + return VexConsensusPolicyOptions.DefaultWeightCeiling; + } + + if (ceiling.Value < 1) + { + issues.Add(new VexPolicyIssue( + "weights.ceiling.minimum", + "weights.ceiling below 1 falls back to 1 to preserve baseline behaviour.", + VexPolicyIssueSeverity.Warning)); + return 1; + } + + if (ceiling.Value > VexConsensusPolicyOptions.MaxSupportedCeiling) + { + issues.Add(new VexPolicyIssue( + "weights.ceiling.maximum", + $"weights.ceiling exceeded supported range; value {ceiling.Value.ToString(CultureInfo.InvariantCulture)} was clamped to {VexConsensusPolicyOptions.MaxSupportedCeiling.ToString(CultureInfo.InvariantCulture)}.", + VexPolicyIssueSeverity.Warning)); + return VexConsensusPolicyOptions.MaxSupportedCeiling; + } + + return ceiling.Value; + } + + private static ImmutableDictionary NormalizeOverrides( + IDictionary? overrides, + double ceiling, + ImmutableArray.Builder issues) + { + if (overrides is null || overrides.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var kvp in overrides) + { + if (string.IsNullOrWhiteSpace(kvp.Key)) + { + issues.Add(new VexPolicyIssue( + "overrides.key.missing", + "Encountered provider override with empty key; ignoring entry.", + VexPolicyIssueSeverity.Warning)); + continue; + } + + var key = kvp.Key.Trim(); + var weight = NormalizeWeightValue( + kvp.Value, + $"overrides.{key}", + DefaultVendorWeight, + ceiling, + issues); + builder[key] = weight; + } + + return builder.ToImmutable(); + } + + private static ScoringNormalizationResult NormalizeScoring( + VexPolicyScoringOptions? options, + ImmutableArray.Builder issues) + { + var alpha = NormalizeCoefficient( + options?.Alpha, + "alpha", + VexConsensusPolicyOptions.DefaultAlpha, + issues); + var beta = NormalizeCoefficient( + options?.Beta, + "beta", + VexConsensusPolicyOptions.DefaultBeta, + issues); + return new ScoringNormalizationResult(alpha, beta); + } + + private static double NormalizeCoefficient( + double? value, + string fieldName, + double defaultValue, + ImmutableArray.Builder issues) + { + if (value is null) + { + return defaultValue; + } + + if (double.IsNaN(value.Value) || double.IsInfinity(value.Value)) + { + issues.Add(new VexPolicyIssue( + $"scoring.{fieldName}.invalid", + $"{fieldName} coefficient must be a finite number; default {defaultValue.ToString(CultureInfo.InvariantCulture)} applied.", + VexPolicyIssueSeverity.Warning)); + return defaultValue; + } + + if (value.Value < 0) + { + issues.Add(new VexPolicyIssue( + $"scoring.{fieldName}.range", + $"{fieldName} cannot be negative; default {defaultValue.ToString(CultureInfo.InvariantCulture)} applied.", + VexPolicyIssueSeverity.Warning)); + return defaultValue; + } + + if (value.Value > VexConsensusPolicyOptions.MaxSupportedCoefficient) + { + issues.Add(new VexPolicyIssue( + $"scoring.{fieldName}.maximum", + $"{fieldName} exceeded supported range; value {value.Value.ToString(CultureInfo.InvariantCulture)} was clamped to {VexConsensusPolicyOptions.MaxSupportedCoefficient.ToString(CultureInfo.InvariantCulture)}.", + VexPolicyIssueSeverity.Warning)); + return VexConsensusPolicyOptions.MaxSupportedCoefficient; + } + + return value.Value; + } + + private static int CompareIssues(VexPolicyIssue left, VexPolicyIssue right) + { + var severityCompare = GetSeverityRank(left.Severity).CompareTo(GetSeverityRank(right.Severity)); + if (severityCompare != 0) + { + return severityCompare; + } + + return string.Compare(left.Code, right.Code, StringComparison.Ordinal); + } + + private static int GetSeverityRank(VexPolicyIssueSeverity severity) + => severity switch + { + VexPolicyIssueSeverity.Error => 0, + VexPolicyIssueSeverity.Warning => 1, + _ => 2, + }; + + private static readonly Comparer IssueComparer = Comparer.Create(CompareIssues); + + internal sealed record VexPolicyNormalizationResult( + VexConsensusPolicyOptions ConsensusOptions, + ImmutableArray Issues); + + private sealed record WeightNormalizationResult( + double Vendor, + double Distro, + double Platform, + double Hub, + double Attestation, + double Ceiling); + + private sealed record ScoringNormalizationResult(double Alpha, double Beta); +} diff --git a/src/StellaOps.Vexer.Policy/VexPolicyTelemetry.cs b/src/StellaOps.Excititor.Policy/VexPolicyTelemetry.cs similarity index 84% rename from src/StellaOps.Vexer.Policy/VexPolicyTelemetry.cs rename to src/StellaOps.Excititor.Policy/VexPolicyTelemetry.cs index 4acb48d2..0a7ebb44 100644 --- a/src/StellaOps.Vexer.Policy/VexPolicyTelemetry.cs +++ b/src/StellaOps.Excititor.Policy/VexPolicyTelemetry.cs @@ -1,24 +1,24 @@ -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Vexer.Policy; - -internal static class VexPolicyTelemetry -{ - private const string MeterName = "StellaOps.Vexer.Policy"; - private const string MeterVersion = "1.0.0"; - - private static readonly Meter Meter = new(MeterName, MeterVersion); - private static readonly Counter PolicyReloads = Meter.CreateCounter("vex.policy.reloads", unit: "events"); - - public static void RecordReload(string revisionId, string version, int issueCount) - { - var tags = new KeyValuePair[] - { - new("revision", revisionId), - new("version", version), - new("issues", issueCount), - }; - PolicyReloads.Add(1, tags); - } -} +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Excititor.Policy; + +internal static class VexPolicyTelemetry +{ + private const string MeterName = "StellaOps.Excititor.Policy"; + private const string MeterVersion = "1.0.0"; + + private static readonly Meter Meter = new(MeterName, MeterVersion); + private static readonly Counter PolicyReloads = Meter.CreateCounter("vex.policy.reloads", unit: "events"); + + public static void RecordReload(string revisionId, string version, int issueCount) + { + var tags = new KeyValuePair[] + { + new("revision", revisionId), + new("version", version), + new("issues", issueCount), + }; + PolicyReloads.Add(1, tags); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs similarity index 95% rename from src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs rename to src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs index 2fcde859..9da04a4d 100644 --- a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexCacheMaintenanceTests.cs @@ -1,122 +1,122 @@ -using System.Collections.Generic; -using Microsoft.Extensions.Logging.Abstractions; -using Mongo2Go; -using MongoDB.Bson; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo.Tests; - -public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime -{ - private readonly MongoDbRunner _runner; - private readonly IMongoDatabase _database; - - public MongoVexCacheMaintenanceTests() - { - _runner = MongoDbRunner.Start(); - var client = new MongoClient(_runner.ConnectionString); - _database = client.GetDatabase("vex-cache-maintenance-tests"); - VexMongoMappingRegistry.Register(); - } - - [Fact] - public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff() - { - var collection = _database.GetCollection(VexMongoCollectionNames.Cache); - var now = DateTime.UtcNow; - - await collection.InsertManyAsync(new[] - { - new VexCacheEntryRecord - { - Id = "sig-1|json", - QuerySignature = "sig-1", - Format = "json", - ArtifactAlgorithm = "sha256", - ArtifactDigest = "deadbeef", - CreatedAt = now.AddHours(-2), - ExpiresAt = now.AddHours(-1), - }, - new VexCacheEntryRecord - { - Id = "sig-2|json", - QuerySignature = "sig-2", - Format = "json", - ArtifactAlgorithm = "sha256", - ArtifactDigest = "cafebabe", - CreatedAt = now, - ExpiresAt = now.AddHours(1), - }, - }); - - var maintenance = new MongoVexCacheMaintenance(_database, NullLogger.Instance); - var removed = await maintenance.RemoveExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); - - Assert.Equal(1, removed); - - var remaining = await collection.CountDocumentsAsync(FilterDefinition.Empty); - Assert.Equal(1, remaining); - } - - [Fact] - public async Task RemoveMissingManifestReferencesAsync_DropsDanglingEntries() - { - var cache = _database.GetCollection(VexMongoCollectionNames.Cache); - var exports = _database.GetCollection(VexMongoCollectionNames.Exports); - - await exports.InsertOneAsync(new VexExportManifestRecord - { - Id = "manifest-existing", - QuerySignature = "sig-keep", - Format = "json", - CreatedAt = DateTime.UtcNow, - ArtifactAlgorithm = "sha256", - ArtifactDigest = "keep", - ClaimCount = 1, - SourceProviders = new List { "vendor" }, - }); - - await cache.InsertManyAsync(new[] - { - new VexCacheEntryRecord - { - Id = "sig-remove|json", - QuerySignature = "sig-remove", - Format = "json", - ArtifactAlgorithm = "sha256", - ArtifactDigest = "drop", - CreatedAt = DateTime.UtcNow, - ManifestId = "manifest-missing", - }, - new VexCacheEntryRecord - { - Id = "sig-keep|json", - QuerySignature = "sig-keep", - Format = "json", - ArtifactAlgorithm = "sha256", - ArtifactDigest = "keep", - CreatedAt = DateTime.UtcNow, - ManifestId = "manifest-existing", - }, - }); - - var maintenance = new MongoVexCacheMaintenance(_database, NullLogger.Instance); - var removed = await maintenance.RemoveMissingManifestReferencesAsync(CancellationToken.None); - - Assert.Equal(1, removed); - - var remainingIds = await cache.Find(Builders.Filter.Empty) - .Project(x => x.Id) - .ToListAsync(); - Assert.Single(remainingIds); - Assert.Contains("sig-keep|json", remainingIds); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - _runner.Dispose(); - return Task.CompletedTask; - } -} +using System.Collections.Generic; +using Microsoft.Extensions.Logging.Abstractions; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class MongoVexCacheMaintenanceTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + private readonly IMongoDatabase _database; + + public MongoVexCacheMaintenanceTests() + { + _runner = MongoDbRunner.Start(); + var client = new MongoClient(_runner.ConnectionString); + _database = client.GetDatabase("vex-cache-maintenance-tests"); + VexMongoMappingRegistry.Register(); + } + + [Fact] + public async Task RemoveExpiredAsync_DeletesEntriesBeforeCutoff() + { + var collection = _database.GetCollection(VexMongoCollectionNames.Cache); + var now = DateTime.UtcNow; + + await collection.InsertManyAsync(new[] + { + new VexCacheEntryRecord + { + Id = "sig-1|json", + QuerySignature = "sig-1", + Format = "json", + ArtifactAlgorithm = "sha256", + ArtifactDigest = "deadbeef", + CreatedAt = now.AddHours(-2), + ExpiresAt = now.AddHours(-1), + }, + new VexCacheEntryRecord + { + Id = "sig-2|json", + QuerySignature = "sig-2", + Format = "json", + ArtifactAlgorithm = "sha256", + ArtifactDigest = "cafebabe", + CreatedAt = now, + ExpiresAt = now.AddHours(1), + }, + }); + + var maintenance = new MongoVexCacheMaintenance(_database, NullLogger.Instance); + var removed = await maintenance.RemoveExpiredAsync(DateTimeOffset.UtcNow, CancellationToken.None); + + Assert.Equal(1, removed); + + var remaining = await collection.CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(1, remaining); + } + + [Fact] + public async Task RemoveMissingManifestReferencesAsync_DropsDanglingEntries() + { + var cache = _database.GetCollection(VexMongoCollectionNames.Cache); + var exports = _database.GetCollection(VexMongoCollectionNames.Exports); + + await exports.InsertOneAsync(new VexExportManifestRecord + { + Id = "manifest-existing", + QuerySignature = "sig-keep", + Format = "json", + CreatedAt = DateTime.UtcNow, + ArtifactAlgorithm = "sha256", + ArtifactDigest = "keep", + ClaimCount = 1, + SourceProviders = new List { "vendor" }, + }); + + await cache.InsertManyAsync(new[] + { + new VexCacheEntryRecord + { + Id = "sig-remove|json", + QuerySignature = "sig-remove", + Format = "json", + ArtifactAlgorithm = "sha256", + ArtifactDigest = "drop", + CreatedAt = DateTime.UtcNow, + ManifestId = "manifest-missing", + }, + new VexCacheEntryRecord + { + Id = "sig-keep|json", + QuerySignature = "sig-keep", + Format = "json", + ArtifactAlgorithm = "sha256", + ArtifactDigest = "keep", + CreatedAt = DateTime.UtcNow, + ManifestId = "manifest-existing", + }, + }); + + var maintenance = new MongoVexCacheMaintenance(_database, NullLogger.Instance); + var removed = await maintenance.RemoveMissingManifestReferencesAsync(CancellationToken.None); + + Assert.Equal(1, removed); + + var remainingIds = await cache.Find(Builders.Filter.Empty) + .Project(x => x.Id) + .ToListAsync(); + Assert.Single(remainingIds); + Assert.Contains("sig-keep|json", remainingIds); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexRepositoryTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexRepositoryTests.cs similarity index 65% rename from src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexRepositoryTests.cs rename to src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexRepositoryTests.cs index fd773f01..c158a212 100644 --- a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexRepositoryTests.cs +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexRepositoryTests.cs @@ -1,206 +1,282 @@ -using System.Collections.Immutable; -using System.Globalization; -using System.Text; -using Microsoft.Extensions.Options; -using Mongo2Go; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo.Tests; - -public sealed class MongoVexRepositoryTests : IAsyncLifetime -{ - private readonly MongoDbRunner _runner; - private readonly MongoClient _client; - - public MongoVexRepositoryTests() - { - _runner = MongoDbRunner.Start(); - _client = new MongoClient(_runner.ConnectionString); - } - - [Fact] - public async Task RawStore_UsesGridFsForLargePayloads() - { - var database = _client.GetDatabase($"vex-raw-gridfs-{Guid.NewGuid():N}"); - var store = CreateRawStore(database, thresholdBytes: 32); - - var payload = Encoding.UTF8.GetBytes(new string('A', 256)); - var document = new VexRawDocument( - "red-hat", - VexDocumentFormat.Csaf, - new Uri("https://example.com/redhat/csaf.json"), - DateTimeOffset.UtcNow, - "sha256:large", - payload, - ImmutableDictionary.Empty); - - await store.StoreAsync(document, CancellationToken.None); - - var rawCollection = database.GetCollection(VexMongoCollectionNames.Raw); - var stored = await rawCollection.Find(Builders.Filter.Eq("_id", document.Digest)) - .FirstOrDefaultAsync(); - - Assert.NotNull(stored); - Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId)); - Assert.False(gridId.IsBsonNull); - Assert.Empty(stored["Content"].AsBsonBinaryData.Bytes); - - var filesCollection = database.GetCollection("vex.raw.files"); - var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition.Empty); - Assert.Equal(1, fileCount); - - var fetched = await store.FindByDigestAsync(document.Digest, CancellationToken.None); - Assert.NotNull(fetched); - Assert.Equal(payload, fetched!.Content.ToArray()); - } - - [Fact] - public async Task RawStore_ReplacesGridFsWithInlinePayload() - { - var database = _client.GetDatabase($"vex-raw-inline-{Guid.NewGuid():N}"); - var store = CreateRawStore(database, thresholdBytes: 16); - - var largePayload = Encoding.UTF8.GetBytes(new string('B', 128)); - var digest = "sha256:inline"; - var largeDocument = new VexRawDocument( - "cisco", - VexDocumentFormat.CycloneDx, - new Uri("https://example.com/cyclonedx.json"), - DateTimeOffset.UtcNow, - digest, - largePayload, - ImmutableDictionary.Empty); - - await store.StoreAsync(largeDocument, CancellationToken.None); - - var smallDocument = largeDocument with - { - RetrievedAt = DateTimeOffset.UtcNow.AddMinutes(1), - Content = Encoding.UTF8.GetBytes("small"), - }; - - await store.StoreAsync(smallDocument, CancellationToken.None); - - var rawCollection = database.GetCollection(VexMongoCollectionNames.Raw); - var stored = await rawCollection.Find(Builders.Filter.Eq("_id", digest)) - .FirstOrDefaultAsync(); - - Assert.NotNull(stored); - Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId)); - Assert.True(gridId.IsBsonNull); - Assert.Equal("small", Encoding.UTF8.GetString(stored["Content"].AsBsonBinaryData.Bytes)); - - var filesCollection = database.GetCollection("vex.raw.files"); - var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition.Empty); - Assert.Equal(0, fileCount); - } - - [Fact] - public async Task ExportStore_SavesManifestAndCacheTransactionally() - { - var database = _client.GetDatabase($"vex-export-save-{Guid.NewGuid():N}"); - var options = Options.Create(new VexMongoStorageOptions - { - ExportCacheTtl = TimeSpan.FromHours(6), - GridFsInlineThresholdBytes = 64, - }); - - var store = new MongoVexExportStore(_client, database, options); - var signature = new VexQuerySignature("format=csaf|provider=redhat"); - var manifest = new VexExportManifest( - "exports/20251016/redhat", - signature, - VexExportFormat.Csaf, - DateTimeOffset.UtcNow, - new VexContentAddress("sha256", "abcdef123456"), - claimCount: 5, - sourceProviders: new[] { "red-hat" }, - fromCache: false, - consensusRevision: "rev-1", - attestation: null, - sizeBytes: 1024); - - await store.SaveAsync(manifest, CancellationToken.None); - - var exportsCollection = database.GetCollection(VexMongoCollectionNames.Exports); - var exportKey = BuildExportKey(signature, VexExportFormat.Csaf); - var exportDoc = await exportsCollection.Find(Builders.Filter.Eq("_id", exportKey)) - .FirstOrDefaultAsync(); - Assert.NotNull(exportDoc); - - var cacheCollection = database.GetCollection(VexMongoCollectionNames.Cache); - var cacheKey = BuildExportKey(signature, VexExportFormat.Csaf); - var cacheDoc = await cacheCollection.Find(Builders.Filter.Eq("_id", cacheKey)) - .FirstOrDefaultAsync(); - - Assert.NotNull(cacheDoc); - Assert.Equal(manifest.ExportId, cacheDoc!["ManifestId"].AsString); - Assert.True(cacheDoc.TryGetValue("ExpiresAt", out var expiresValue)); - Assert.False(expiresValue.IsBsonNull); - } - - [Fact] - public async Task ExportStore_FindAsync_ExpiresCacheEntries() - { - var database = _client.GetDatabase($"vex-export-expire-{Guid.NewGuid():N}"); - var options = Options.Create(new VexMongoStorageOptions - { - ExportCacheTtl = TimeSpan.FromMinutes(5), - GridFsInlineThresholdBytes = 64, - }); - - var store = new MongoVexExportStore(_client, database, options); - var signature = new VexQuerySignature("format=json|provider=cisco"); - var manifest = new VexExportManifest( - "exports/20251016/cisco", - signature, - VexExportFormat.Json, - DateTimeOffset.UtcNow, - new VexContentAddress("sha256", "deadbeef"), - claimCount: 3, - sourceProviders: new[] { "cisco" }, - fromCache: false, - consensusRevision: "rev-2", - attestation: null, - sizeBytes: 2048); - - await store.SaveAsync(manifest, CancellationToken.None); - - var cacheCollection = database.GetCollection(VexMongoCollectionNames.Cache); - var cacheId = BuildExportKey(signature, VexExportFormat.Json); - var update = Builders.Update.Set("ExpiresAt", DateTime.UtcNow.AddMinutes(-10)); - await cacheCollection.UpdateOneAsync(Builders.Filter.Eq("_id", cacheId), update); - - var cached = await store.FindAsync(signature, VexExportFormat.Json, CancellationToken.None); - Assert.Null(cached); - - var remaining = await cacheCollection.Find(Builders.Filter.Eq("_id", cacheId)) - .FirstOrDefaultAsync(); - Assert.Null(remaining); - } - - private MongoVexRawStore CreateRawStore(IMongoDatabase database, int thresholdBytes) - { - var options = Options.Create(new VexMongoStorageOptions - { - RawBucketName = "vex.raw", - GridFsInlineThresholdBytes = thresholdBytes, - ExportCacheTtl = TimeSpan.FromHours(1), - }); - - return new MongoVexRawStore(_client, database, options); - } - - private static string BuildExportKey(VexQuerySignature signature, VexExportFormat format) - => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - _runner.Dispose(); - return Task.CompletedTask; - } -} +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text; +using Microsoft.Extensions.Options; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class MongoVexRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + private readonly MongoClient _client; + + public MongoVexRepositoryTests() + { + _runner = MongoDbRunner.Start(); + _client = new MongoClient(_runner.ConnectionString); + } + + [Fact] + public async Task RawStore_UsesGridFsForLargePayloads() + { + var database = _client.GetDatabase($"vex-raw-gridfs-{Guid.NewGuid():N}"); + var store = CreateRawStore(database, thresholdBytes: 32); + + var payload = Encoding.UTF8.GetBytes(new string('A', 256)); + var document = new VexRawDocument( + "red-hat", + VexDocumentFormat.Csaf, + new Uri("https://example.com/redhat/csaf.json"), + DateTimeOffset.UtcNow, + "sha256:large", + payload, + ImmutableDictionary.Empty); + + await store.StoreAsync(document, CancellationToken.None); + + var rawCollection = database.GetCollection(VexMongoCollectionNames.Raw); + var stored = await rawCollection.Find(Builders.Filter.Eq("_id", document.Digest)) + .FirstOrDefaultAsync(); + + Assert.NotNull(stored); + Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId)); + Assert.False(gridId.IsBsonNull); + Assert.Empty(stored["Content"].AsBsonBinaryData.Bytes); + + var filesCollection = database.GetCollection("vex.raw.files"); + var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(1, fileCount); + + var fetched = await store.FindByDigestAsync(document.Digest, CancellationToken.None); + Assert.NotNull(fetched); + Assert.Equal(payload, fetched!.Content.ToArray()); + } + + [Fact] + public async Task RawStore_ReplacesGridFsWithInlinePayload() + { + var database = _client.GetDatabase($"vex-raw-inline-{Guid.NewGuid():N}"); + var store = CreateRawStore(database, thresholdBytes: 16); + + var largePayload = Encoding.UTF8.GetBytes(new string('B', 128)); + var digest = "sha256:inline"; + var largeDocument = new VexRawDocument( + "cisco", + VexDocumentFormat.CycloneDx, + new Uri("https://example.com/cyclonedx.json"), + DateTimeOffset.UtcNow, + digest, + largePayload, + ImmutableDictionary.Empty); + + await store.StoreAsync(largeDocument, CancellationToken.None); + + var smallDocument = largeDocument with + { + RetrievedAt = DateTimeOffset.UtcNow.AddMinutes(1), + Content = Encoding.UTF8.GetBytes("small"), + }; + + await store.StoreAsync(smallDocument, CancellationToken.None); + + var rawCollection = database.GetCollection(VexMongoCollectionNames.Raw); + var stored = await rawCollection.Find(Builders.Filter.Eq("_id", digest)) + .FirstOrDefaultAsync(); + + Assert.NotNull(stored); + Assert.True(stored!.TryGetValue("GridFsObjectId", out var gridId)); + Assert.True(gridId.IsBsonNull); + Assert.Equal("small", Encoding.UTF8.GetString(stored["Content"].AsBsonBinaryData.Bytes)); + + var filesCollection = database.GetCollection("vex.raw.files"); + var fileCount = await filesCollection.CountDocumentsAsync(FilterDefinition.Empty); + Assert.Equal(0, fileCount); + } + + [Fact] + public async Task ExportStore_SavesManifestAndCacheTransactionally() + { + var database = _client.GetDatabase($"vex-export-save-{Guid.NewGuid():N}"); + var options = Options.Create(new VexMongoStorageOptions + { + ExportCacheTtl = TimeSpan.FromHours(6), + GridFsInlineThresholdBytes = 64, + }); + + var sessionProvider = new VexMongoSessionProvider(_client, options); + var store = new MongoVexExportStore(_client, database, options, sessionProvider); + var signature = new VexQuerySignature("format=csaf|provider=redhat"); + var manifest = new VexExportManifest( + "exports/20251016/redhat", + signature, + VexExportFormat.Csaf, + DateTimeOffset.UtcNow, + new VexContentAddress("sha256", "abcdef123456"), + claimCount: 5, + sourceProviders: new[] { "red-hat" }, + fromCache: false, + consensusRevision: "rev-1", + attestation: null, + sizeBytes: 1024); + + await store.SaveAsync(manifest, CancellationToken.None); + + var exportsCollection = database.GetCollection(VexMongoCollectionNames.Exports); + var exportKey = BuildExportKey(signature, VexExportFormat.Csaf); + var exportDoc = await exportsCollection.Find(Builders.Filter.Eq("_id", exportKey)) + .FirstOrDefaultAsync(); + Assert.NotNull(exportDoc); + + var cacheCollection = database.GetCollection(VexMongoCollectionNames.Cache); + var cacheKey = BuildExportKey(signature, VexExportFormat.Csaf); + var cacheDoc = await cacheCollection.Find(Builders.Filter.Eq("_id", cacheKey)) + .FirstOrDefaultAsync(); + + Assert.NotNull(cacheDoc); + Assert.Equal(manifest.ExportId, cacheDoc!["ManifestId"].AsString); + Assert.True(cacheDoc.TryGetValue("ExpiresAt", out var expiresValue)); + Assert.False(expiresValue.IsBsonNull); + } + + [Fact] + public async Task ExportStore_FindAsync_ExpiresCacheEntries() + { + var database = _client.GetDatabase($"vex-export-expire-{Guid.NewGuid():N}"); + var options = Options.Create(new VexMongoStorageOptions + { + ExportCacheTtl = TimeSpan.FromMinutes(5), + GridFsInlineThresholdBytes = 64, + }); + + var sessionProvider = new VexMongoSessionProvider(_client, options); + var store = new MongoVexExportStore(_client, database, options, sessionProvider); + var signature = new VexQuerySignature("format=json|provider=cisco"); + var manifest = new VexExportManifest( + "exports/20251016/cisco", + signature, + VexExportFormat.Json, + DateTimeOffset.UtcNow, + new VexContentAddress("sha256", "deadbeef"), + claimCount: 3, + sourceProviders: new[] { "cisco" }, + fromCache: false, + consensusRevision: "rev-2", + attestation: null, + sizeBytes: 2048); + + await store.SaveAsync(manifest, CancellationToken.None); + + var cacheCollection = database.GetCollection(VexMongoCollectionNames.Cache); + var cacheId = BuildExportKey(signature, VexExportFormat.Json); + var update = Builders.Update.Set("ExpiresAt", DateTime.UtcNow.AddMinutes(-10)); + await cacheCollection.UpdateOneAsync(Builders.Filter.Eq("_id", cacheId), update); + + var cached = await store.FindAsync(signature, VexExportFormat.Json, CancellationToken.None); + Assert.Null(cached); + + var remaining = await cacheCollection.Find(Builders.Filter.Eq("_id", cacheId)) + .FirstOrDefaultAsync(); + Assert.Null(remaining); + } + + [Fact] + public async Task ClaimStore_AppendsAndQueriesStatements() + { + var database = _client.GetDatabase($"vex-claims-{Guid.NewGuid():N}"); + var store = new MongoVexClaimStore(database); + + var product = new VexProduct("pkg:demo/app", "Demo App", version: "1.0.0", purl: "pkg:demo/app@1.0.0"); + var document = new VexClaimDocument( + VexDocumentFormat.Csaf, + "sha256:claim-1", + new Uri("https://example.org/vex/claim-1.json"), + revision: "2025-10-19"); + + var initialClaim = new VexClaim( + vulnerabilityId: "CVE-2025-0101", + providerId: "redhat", + product: product, + status: VexClaimStatus.NotAffected, + document: document, + firstSeen: DateTimeOffset.UtcNow.AddMinutes(-30), + lastSeen: DateTimeOffset.UtcNow.AddMinutes(-10), + justification: VexJustification.ComponentNotPresent, + detail: "Package not shipped in this channel.", + confidence: new VexConfidence("high", 0.9, "policy/default"), + signals: new VexSignalSnapshot( + new VexSeveritySignal("CVSS:3.1", 5.8, "medium", "CVSS:3.1/..."), + kev: false, + epss: 0.21), + additionalMetadata: ImmutableDictionary.Empty.Add("source", "csaf")); + + await store.AppendAsync(new[] { initialClaim }, DateTimeOffset.UtcNow.AddMinutes(-5), CancellationToken.None); + + var secondDocument = new VexClaimDocument( + VexDocumentFormat.Csaf, + "sha256:claim-2", + new Uri("https://example.org/vex/claim-2.json"), + revision: "2025-10-19.1"); + + var secondClaim = new VexClaim( + vulnerabilityId: initialClaim.VulnerabilityId, + providerId: initialClaim.ProviderId, + product: initialClaim.Product, + status: initialClaim.Status, + document: secondDocument, + firstSeen: initialClaim.FirstSeen, + lastSeen: DateTimeOffset.UtcNow, + justification: initialClaim.Justification, + detail: initialClaim.Detail, + confidence: initialClaim.Confidence, + signals: new VexSignalSnapshot( + new VexSeveritySignal("CVSS:3.1", 7.2, "high"), + kev: true, + epss: 0.43), + additionalMetadata: initialClaim.AdditionalMetadata.ToImmutableDictionary(kvp => kvp.Key, kvp => kvp.Value)); + + await store.AppendAsync(new[] { secondClaim }, DateTimeOffset.UtcNow, CancellationToken.None); + + var all = await store.FindAsync("CVE-2025-0101", product.Key, since: null, CancellationToken.None); + var allList = all.ToList(); + Assert.Equal(2, allList.Count); + Assert.Equal("sha256:claim-2", allList[0].Document.Digest); + Assert.True(allList[0].Signals?.Kev); + Assert.Equal(0.43, allList[0].Signals?.Epss); + Assert.Equal("sha256:claim-1", allList[1].Document.Digest); + Assert.Equal("csaf", allList[1].AdditionalMetadata["source"]); + + var recentOnly = await store.FindAsync("CVE-2025-0101", product.Key, DateTimeOffset.UtcNow.AddMinutes(-2), CancellationToken.None); + var recentList = recentOnly.ToList(); + Assert.Single(recentList); + Assert.Equal("sha256:claim-2", recentList[0].Document.Digest); + } + + private MongoVexRawStore CreateRawStore(IMongoDatabase database, int thresholdBytes) + { + var options = Options.Create(new VexMongoStorageOptions + { + RawBucketName = "vex.raw", + GridFsInlineThresholdBytes = thresholdBytes, + ExportCacheTtl = TimeSpan.FromHours(1), + }); + + var sessionProvider = new VexMongoSessionProvider(_client, options); + return new MongoVexRawStore(_client, database, options, sessionProvider); + } + + private static string BuildExportKey(VexQuerySignature signature, VexExportFormat format) + => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexSessionConsistencyTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexSessionConsistencyTests.cs new file mode 100644 index 00000000..019f9e5f --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexSessionConsistencyTests.cs @@ -0,0 +1,184 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class MongoVexSessionConsistencyTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + + public MongoVexSessionConsistencyTests() + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + } + + [Fact] + public async Task SessionProvidesReadYourWrites() + { + await using var provider = BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var sessionProvider = scope.ServiceProvider.GetRequiredService(); + var providerStore = scope.ServiceProvider.GetRequiredService(); + + var session = await sessionProvider.StartSessionAsync(); + var descriptor = new VexProvider("red-hat", "Red Hat", VexProviderKind.Vendor); + + await providerStore.SaveAsync(descriptor, CancellationToken.None, session); + var fetched = await providerStore.FindAsync(descriptor.Id, CancellationToken.None, session); + + Assert.NotNull(fetched); + Assert.Equal(descriptor.DisplayName, fetched!.DisplayName); + } + + [Fact] + public async Task SessionMaintainsMonotonicReadsAcrossStepDown() + { + await using var provider = BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var client = scope.ServiceProvider.GetRequiredService(); + var sessionProvider = scope.ServiceProvider.GetRequiredService(); + var providerStore = scope.ServiceProvider.GetRequiredService(); + + var session = await sessionProvider.StartSessionAsync(); + var initial = new VexProvider("cisco", "Cisco", VexProviderKind.Vendor); + + await providerStore.SaveAsync(initial, CancellationToken.None, session); + var baseline = await providerStore.FindAsync(initial.Id, CancellationToken.None, session); + Assert.Equal("Cisco", baseline!.DisplayName); + + await ForcePrimaryStepDownAsync(client, CancellationToken.None); + await WaitForPrimaryAsync(client, CancellationToken.None); + + await ExecuteWithRetryAsync(async () => + { + var updated = new VexProvider(initial.Id, "Cisco Systems", initial.Kind); + await providerStore.SaveAsync(updated, CancellationToken.None, session); + }, CancellationToken.None); + + var afterFailover = await providerStore.FindAsync(initial.Id, CancellationToken.None, session); + Assert.Equal("Cisco Systems", afterFailover!.DisplayName); + + var subsequent = await providerStore.FindAsync(initial.Id, CancellationToken.None, session); + Assert.Equal("Cisco Systems", subsequent!.DisplayName); + } + + private ServiceProvider BuildServiceProvider() + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddDebug()); + services.Configure(options => + { + options.ConnectionString = _runner.ConnectionString; + options.DatabaseName = $"excititor-session-tests-{Guid.NewGuid():N}"; + options.CommandTimeout = TimeSpan.FromSeconds(5); + options.RawBucketName = "vex.raw"; + }); + services.AddExcititorMongoStorage(); + return services.BuildServiceProvider(); + } + + private static async Task ExecuteWithRetryAsync(Func action, CancellationToken cancellationToken) + { + const int maxAttempts = 10; + var attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await action(); + return; + } + catch (MongoException ex) when (IsStepDownTransient(ex) && attempt++ < maxAttempts) + { + await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken); + } + } + } + + private static bool IsStepDownTransient(MongoException ex) + { + if (ex is MongoConnectionException) + { + return true; + } + + if (ex is MongoCommandException command) + { + return command.Code is 7 or 89 or 91 or 10107 or 11600 + || string.Equals(command.CodeName, "NotPrimaryNoSecondaryOk", StringComparison.OrdinalIgnoreCase) + || string.Equals(command.CodeName, "NotWritablePrimary", StringComparison.OrdinalIgnoreCase) + || string.Equals(command.CodeName, "PrimarySteppedDown", StringComparison.OrdinalIgnoreCase) + || string.Equals(command.CodeName, "NotPrimary", StringComparison.OrdinalIgnoreCase); + } + + return false; + } + + private static async Task ForcePrimaryStepDownAsync(IMongoClient client, CancellationToken cancellationToken) + { + var admin = client.GetDatabase("admin"); + var command = new BsonDocument + { + { "replSetStepDown", 1 }, + { "force", true }, + }; + + try + { + await admin.RunCommandAsync(command, cancellationToken: cancellationToken); + } + catch (MongoException ex) when (IsStepDownTransient(ex)) + { + // Expected when the primary closes connections during the step-down sequence. + } + } + + private static async Task WaitForPrimaryAsync(IMongoClient client, CancellationToken cancellationToken) + { + var admin = client.GetDatabase("admin"); + var helloCommand = new BsonDocument("hello", 1); + + for (var attempt = 0; attempt < 40; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + var result = await admin.RunCommandAsync(helloCommand, cancellationToken: cancellationToken); + if (result.TryGetValue("isWritablePrimary", out var value) && value.IsBoolean && value.AsBoolean) + { + return; + } + } + catch (MongoException ex) when (IsStepDownTransient(ex)) + { + // Primary still recovering, retry. + } + + await Task.Delay(TimeSpan.FromMilliseconds(200), cancellationToken); + } + + throw new TimeoutException("Replica set primary did not recover in time."); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStatementBackfillServiceTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStatementBackfillServiceTests.cs new file mode 100644 index 00000000..ef3596df --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStatementBackfillServiceTests.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Mongo2Go; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class MongoVexStatementBackfillServiceTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + + public MongoVexStatementBackfillServiceTests() + { + _runner = MongoDbRunner.Start(singleNodeReplSet: true); + } + + [Fact] + public async Task RunAsync_BackfillsStatementsFromRawDocuments() + { + await using var provider = BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var rawStore = scope.ServiceProvider.GetRequiredService(); + var claimStore = scope.ServiceProvider.GetRequiredService(); + var backfill = scope.ServiceProvider.GetRequiredService(); + + var retrievedAt = DateTimeOffset.UtcNow.AddMinutes(-15); + var metadata = ImmutableDictionary.Empty + .Add("vulnId", "CVE-2025-0001") + .Add("productKey", "pkg:test/app"); + + var document = new VexRawDocument( + "test-provider", + VexDocumentFormat.Csaf, + new Uri("https://example.test/vex.json"), + retrievedAt, + "sha256:test-doc", + ReadOnlyMemory.Empty, + metadata); + + await rawStore.StoreAsync(document, CancellationToken.None); + + var result = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None); + + Assert.Equal(1, result.DocumentsEvaluated); + Assert.Equal(1, result.DocumentsBackfilled); + Assert.Equal(1, result.ClaimsWritten); + Assert.Equal(0, result.NormalizationFailures); + + var claims = await claimStore.FindAsync("CVE-2025-0001", "pkg:test/app", since: null, CancellationToken.None); + var claim = Assert.Single(claims); + Assert.Equal(VexClaimStatus.NotAffected, claim.Status); + Assert.Equal("test-provider", claim.ProviderId); + Assert.Equal(retrievedAt.ToUnixTimeSeconds(), claim.FirstSeen.ToUnixTimeSeconds()); + Assert.NotNull(claim.Signals); + Assert.Equal(0.2, claim.Signals!.Epss); + Assert.Equal("cvss", claim.Signals!.Severity?.Scheme); + } + + [Fact] + public async Task RunAsync_SkipsExistingDocumentsUnlessForced() + { + await using var provider = BuildServiceProvider(); + await using var scope = provider.CreateAsyncScope(); + + var rawStore = scope.ServiceProvider.GetRequiredService(); + var claimStore = scope.ServiceProvider.GetRequiredService(); + var backfill = scope.ServiceProvider.GetRequiredService(); + + var metadata = ImmutableDictionary.Empty + .Add("vulnId", "CVE-2025-0002") + .Add("productKey", "pkg:test/api"); + + var document = new VexRawDocument( + "test-provider", + VexDocumentFormat.Csaf, + new Uri("https://example.test/vex-2.json"), + DateTimeOffset.UtcNow.AddMinutes(-10), + "sha256:test-doc-2", + ReadOnlyMemory.Empty, + metadata); + + await rawStore.StoreAsync(document, CancellationToken.None); + + var first = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None); + Assert.Equal(1, first.DocumentsBackfilled); + + var second = await backfill.RunAsync(new VexStatementBackfillRequest(), CancellationToken.None); + Assert.Equal(1, second.DocumentsEvaluated); + Assert.Equal(0, second.DocumentsBackfilled); + Assert.Equal(1, second.SkippedExisting); + + var forced = await backfill.RunAsync(new VexStatementBackfillRequest(Force: true), CancellationToken.None); + Assert.Equal(1, forced.DocumentsBackfilled); + + var claims = await claimStore.FindAsync("CVE-2025-0002", "pkg:test/api", since: null, CancellationToken.None); + Assert.Equal(2, claims.Count); + } + + private ServiceProvider BuildServiceProvider() + { + var services = new ServiceCollection(); + services.AddLogging(builder => builder.AddDebug()); + services.AddSingleton(TimeProvider.System); + services.Configure(options => + { + options.ConnectionString = _runner.ConnectionString; + options.DatabaseName = $"excititor-backfill-tests-{Guid.NewGuid():N}"; + options.CommandTimeout = TimeSpan.FromSeconds(5); + options.RawBucketName = "vex.raw"; + options.GridFsInlineThresholdBytes = 1024; + options.ExportCacheTtl = TimeSpan.FromHours(1); + }); + services.AddExcititorMongoStorage(); + services.AddSingleton(); + return services.BuildServiceProvider(); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + private sealed class TestNormalizer : IVexNormalizer + { + public string Format => "csaf"; + + public bool CanHandle(VexRawDocument document) => true; + + public ValueTask NormalizeAsync(VexRawDocument document, VexProvider provider, CancellationToken cancellationToken) + { + var productKey = document.Metadata.TryGetValue("productKey", out var value) ? value : "pkg:test/default"; + var vulnId = document.Metadata.TryGetValue("vulnId", out var vuln) ? vuln : "CVE-TEST-0000"; + + var product = new VexProduct(productKey, "Test Product"); + var claimDocument = new VexClaimDocument( + document.Format, + document.Digest, + document.SourceUri); + + var timestamp = document.RetrievedAt == default ? DateTimeOffset.UtcNow : document.RetrievedAt; + + var claim = new VexClaim( + vulnId, + provider.Id, + product, + VexClaimStatus.NotAffected, + claimDocument, + timestamp, + timestamp, + VexJustification.ComponentNotPresent, + detail: "backfill-test", + confidence: new VexConfidence("high", 0.95, "unit-test"), + signals: new VexSignalSnapshot( + new VexSeveritySignal("cvss", 5.4, "medium"), + kev: false, + epss: 0.2)); + + var claims = ImmutableArray.Create(claim); + return ValueTask.FromResult(new VexClaimBatch(document, claims, ImmutableDictionary.Empty)); + } + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs similarity index 87% rename from src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs rename to src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs index ee085bd4..9c46bd5e 100644 --- a/src/StellaOps.Vexer.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/MongoVexStoreMappingTests.cs @@ -1,242 +1,267 @@ -using System.Globalization; -using Mongo2Go; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo.Tests; - -public sealed class MongoVexStoreMappingTests : IAsyncLifetime -{ - private readonly MongoDbRunner _runner; - private readonly IMongoDatabase _database; - - public MongoVexStoreMappingTests() - { - _runner = MongoDbRunner.Start(); - var client = new MongoClient(_runner.ConnectionString); - _database = client.GetDatabase("vexer-storage-mapping-tests"); - VexMongoMappingRegistry.Register(); - } - - [Fact] - public async Task ProviderStore_RoundTrips_WithExtraFields() - { - var providers = _database.GetCollection(VexMongoCollectionNames.Providers); - var providerId = "red-hat"; - - var document = new BsonDocument - { - { "_id", providerId }, - { "DisplayName", "Red Hat CSAF" }, - { "Kind", "vendor" }, - { "BaseUris", new BsonArray { "https://example.com/csaf" } }, - { - "Discovery", - new BsonDocument - { - { "WellKnownMetadata", "https://example.com/.well-known/csaf" }, - { "RolIeService", "https://example.com/service/rolie" }, - { "UnsupportedField", "ignored" }, - } - }, - { - "Trust", - new BsonDocument - { - { "Weight", 0.75 }, - { - "Cosign", - new BsonDocument - { - { "Issuer", "issuer@example.com" }, - { "IdentityPattern", "spiffe://example/*" }, - { "Unexpected", true }, - } - }, - { "PgpFingerprints", new BsonArray { "ABCDEF1234567890" } }, - { "AnotherIgnoredField", 123 }, - } - }, - { "Enabled", true }, - { "UnexpectedRoot", new BsonDocument { { "flag", true } } }, - }; - - await providers.InsertOneAsync(document); - - var store = new MongoVexProviderStore(_database); - var result = await store.FindAsync(providerId, CancellationToken.None); - - Assert.NotNull(result); - Assert.Equal(providerId, result!.Id); - Assert.Equal("Red Hat CSAF", result.DisplayName); - Assert.Equal(VexProviderKind.Vendor, result.Kind); - Assert.Single(result.BaseUris); - Assert.Equal("https://example.com/csaf", result.BaseUris[0].ToString()); - Assert.Equal("https://example.com/.well-known/csaf", result.Discovery.WellKnownMetadata?.ToString()); - Assert.Equal("https://example.com/service/rolie", result.Discovery.RolIeService?.ToString()); - Assert.Equal(0.75, result.Trust.Weight); - Assert.NotNull(result.Trust.Cosign); - Assert.Equal("issuer@example.com", result.Trust.Cosign!.Issuer); - Assert.Equal("spiffe://example/*", result.Trust.Cosign!.IdentityPattern); - Assert.Contains("ABCDEF1234567890", result.Trust.PgpFingerprints); - Assert.True(result.Enabled); - } - - [Fact] - public async Task ConsensusStore_IgnoresUnknownFields() - { - var consensus = _database.GetCollection(VexMongoCollectionNames.Consensus); - var vulnerabilityId = "CVE-2025-12345"; - var productKey = "pkg:maven/org.example/app@1.2.3"; - var consensusId = string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim()); - - var document = new BsonDocument - { - { "_id", consensusId }, - { "VulnerabilityId", vulnerabilityId }, - { - "Product", - new BsonDocument - { - { "Key", productKey }, - { "Name", "Example App" }, - { "Version", "1.2.3" }, - { "Purl", productKey }, - { "Extra", "ignored" }, - } - }, - { "Status", "notaffected" }, - { "CalculatedAt", DateTime.UtcNow }, - { - "Sources", - new BsonArray - { - new BsonDocument - { - { "ProviderId", "red-hat" }, - { "Status", "notaffected" }, - { "DocumentDigest", "sha256:123" }, - { "Weight", 0.9 }, - { "Justification", "componentnotpresent" }, - { "Detail", "Vendor statement" }, - { - "Confidence", - new BsonDocument - { - { "Level", "high" }, - { "Score", 0.7 }, - { "Method", "review" }, - { "Unexpected", "ignored" }, - } - }, - { "UnknownField", true }, - }, - } - }, - { - "Conflicts", - new BsonArray - { - new BsonDocument - { - { "ProviderId", "cisco" }, - { "Status", "affected" }, - { "DocumentDigest", "sha256:999" }, - { "Justification", "requiresconfiguration" }, - { "Detail", "Different guidance" }, - { "Reason", "policy_override" }, - { "Other", 1 }, - }, - } - }, - { "PolicyVersion", "2025.10" }, - { "PolicyRevisionId", "rev-1" }, - { "PolicyDigest", "sha256:abc" }, - { "Summary", "Vendor confirms not affected." }, - { "Unexpected", new BsonDocument { { "foo", "bar" } } }, - }; - - await consensus.InsertOneAsync(document); - - var store = new MongoVexConsensusStore(_database); - var result = await store.FindAsync(vulnerabilityId, productKey, CancellationToken.None); - - Assert.NotNull(result); - Assert.Equal(vulnerabilityId, result!.VulnerabilityId); - Assert.Equal(productKey, result.Product.Key); - Assert.Equal("Example App", result.Product.Name); - Assert.Equal(VexConsensusStatus.NotAffected, result.Status); - Assert.Single(result.Sources); - var source = result.Sources[0]; - Assert.Equal("red-hat", source.ProviderId); - Assert.Equal(VexClaimStatus.NotAffected, source.Status); - Assert.Equal("sha256:123", source.DocumentDigest); - Assert.Equal(0.9, source.Weight); - Assert.Equal(VexJustification.ComponentNotPresent, source.Justification); - Assert.NotNull(source.Confidence); - Assert.Equal("high", source.Confidence!.Level); - Assert.Equal(0.7, source.Confidence!.Score); - Assert.Equal("review", source.Confidence!.Method); - Assert.Single(result.Conflicts); - var conflict = result.Conflicts[0]; - Assert.Equal("cisco", conflict.ProviderId); - Assert.Equal(VexClaimStatus.Affected, conflict.Status); - Assert.Equal(VexJustification.RequiresConfiguration, conflict.Justification); - Assert.Equal("policy_override", conflict.Reason); - Assert.Equal("Vendor confirms not affected.", result.Summary); - Assert.Equal("2025.10", result.PolicyVersion); - } - - [Fact] - public async Task CacheIndex_RoundTripsGridFsMetadata() - { - var gridObjectId = ObjectId.GenerateNewId().ToString(); - var index = new MongoVexCacheIndex(_database); - var signature = new VexQuerySignature("format=csaf|vendor=redhat"); - var now = DateTimeOffset.UtcNow; - var expires = now.AddHours(12); - var entry = new VexCacheEntry( - signature, - VexExportFormat.Csaf, - new VexContentAddress("sha256", "abcdef123456"), - now, - sizeBytes: 1024, - manifestId: "manifest-001", - gridFsObjectId: gridObjectId, - expiresAt: expires); - - await index.SaveAsync(entry, CancellationToken.None); - - var cacheId = string.Format( - CultureInfo.InvariantCulture, - "{0}|{1}", - signature.Value, - entry.Format.ToString().ToLowerInvariant()); - - var cache = _database.GetCollection(VexMongoCollectionNames.Cache); - var filter = Builders.Filter.Eq("_id", cacheId); - var update = Builders.Update.Set("UnexpectedField", true); - await cache.UpdateOneAsync(filter, update); - - var roundTrip = await index.FindAsync(signature, VexExportFormat.Csaf, CancellationToken.None); - - Assert.NotNull(roundTrip); - Assert.Equal(entry.QuerySignature.Value, roundTrip!.QuerySignature.Value); - Assert.Equal(entry.Format, roundTrip.Format); - Assert.Equal(entry.Artifact.Digest, roundTrip.Artifact.Digest); - Assert.Equal(entry.ManifestId, roundTrip.ManifestId); - Assert.Equal(entry.GridFsObjectId, roundTrip.GridFsObjectId); - Assert.Equal(entry.SizeBytes, roundTrip.SizeBytes); - Assert.NotNull(roundTrip.ExpiresAt); - Assert.Equal(expires.ToUnixTimeMilliseconds(), roundTrip.ExpiresAt!.Value.ToUnixTimeMilliseconds()); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - _runner.Dispose(); - return Task.CompletedTask; - } -} +using System.Globalization; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class MongoVexStoreMappingTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + private readonly IMongoDatabase _database; + + public MongoVexStoreMappingTests() + { + _runner = MongoDbRunner.Start(); + var client = new MongoClient(_runner.ConnectionString); + _database = client.GetDatabase("excititor-storage-mapping-tests"); + VexMongoMappingRegistry.Register(); + } + + [Fact] + public async Task ProviderStore_RoundTrips_WithExtraFields() + { + var providers = _database.GetCollection(VexMongoCollectionNames.Providers); + var providerId = "red-hat"; + + var document = new BsonDocument + { + { "_id", providerId }, + { "DisplayName", "Red Hat CSAF" }, + { "Kind", "vendor" }, + { "BaseUris", new BsonArray { "https://example.com/csaf" } }, + { + "Discovery", + new BsonDocument + { + { "WellKnownMetadata", "https://example.com/.well-known/csaf" }, + { "RolIeService", "https://example.com/service/rolie" }, + { "UnsupportedField", "ignored" }, + } + }, + { + "Trust", + new BsonDocument + { + { "Weight", 0.75 }, + { + "Cosign", + new BsonDocument + { + { "Issuer", "issuer@example.com" }, + { "IdentityPattern", "spiffe://example/*" }, + { "Unexpected", true }, + } + }, + { "PgpFingerprints", new BsonArray { "ABCDEF1234567890" } }, + { "AnotherIgnoredField", 123 }, + } + }, + { "Enabled", true }, + { "UnexpectedRoot", new BsonDocument { { "flag", true } } }, + }; + + await providers.InsertOneAsync(document); + + var store = new MongoVexProviderStore(_database); + var result = await store.FindAsync(providerId, CancellationToken.None); + + Assert.NotNull(result); + Assert.Equal(providerId, result!.Id); + Assert.Equal("Red Hat CSAF", result.DisplayName); + Assert.Equal(VexProviderKind.Vendor, result.Kind); + Assert.Single(result.BaseUris); + Assert.Equal("https://example.com/csaf", result.BaseUris[0].ToString()); + Assert.Equal("https://example.com/.well-known/csaf", result.Discovery.WellKnownMetadata?.ToString()); + Assert.Equal("https://example.com/service/rolie", result.Discovery.RolIeService?.ToString()); + Assert.Equal(0.75, result.Trust.Weight); + Assert.NotNull(result.Trust.Cosign); + Assert.Equal("issuer@example.com", result.Trust.Cosign!.Issuer); + Assert.Equal("spiffe://example/*", result.Trust.Cosign!.IdentityPattern); + Assert.Contains("ABCDEF1234567890", result.Trust.PgpFingerprints); + Assert.True(result.Enabled); + } + + [Fact] + public async Task ConsensusStore_IgnoresUnknownFields() + { + var consensus = _database.GetCollection(VexMongoCollectionNames.Consensus); + var vulnerabilityId = "CVE-2025-12345"; + var productKey = "pkg:maven/org.example/app@1.2.3"; + var consensusId = string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim()); + + var document = new BsonDocument + { + { "_id", consensusId }, + { "VulnerabilityId", vulnerabilityId }, + { + "Product", + new BsonDocument + { + { "Key", productKey }, + { "Name", "Example App" }, + { "Version", "1.2.3" }, + { "Purl", productKey }, + { "Extra", "ignored" }, + } + }, + { "Status", "notaffected" }, + { "CalculatedAt", DateTime.UtcNow }, + { + "Sources", + new BsonArray + { + new BsonDocument + { + { "ProviderId", "red-hat" }, + { "Status", "notaffected" }, + { "DocumentDigest", "sha256:123" }, + { "Weight", 0.9 }, + { "Justification", "componentnotpresent" }, + { "Detail", "Vendor statement" }, + { + "Confidence", + new BsonDocument + { + { "Level", "high" }, + { "Score", 0.7 }, + { "Method", "review" }, + { "Unexpected", "ignored" }, + } + }, + { "UnknownField", true }, + }, + } + }, + { + "Conflicts", + new BsonArray + { + new BsonDocument + { + { "ProviderId", "cisco" }, + { "Status", "affected" }, + { "DocumentDigest", "sha256:999" }, + { "Justification", "requiresconfiguration" }, + { "Detail", "Different guidance" }, + { "Reason", "policy_override" }, + { "Other", 1 }, + }, + } + }, + { + "Signals", + new BsonDocument + { + { + "Severity", + new BsonDocument + { + { "Scheme", "CVSS:3.1" }, + { "Score", 7.5 }, + { "Label", "high" }, + { "Vector", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + } + }, + { "Kev", true }, + { "Epss", 0.42 }, + } + }, + { "PolicyVersion", "2025.10" }, + { "PolicyRevisionId", "rev-1" }, + { "PolicyDigest", "sha256:abc" }, + { "Summary", "Vendor confirms not affected." }, + { "GeneratedAt", DateTime.UtcNow }, + { "Unexpected", new BsonDocument { { "foo", "bar" } } }, + }; + + await consensus.InsertOneAsync(document); + + var store = new MongoVexConsensusStore(_database); + var result = await store.FindAsync(vulnerabilityId, productKey, CancellationToken.None); + + Assert.NotNull(result); + Assert.Equal(vulnerabilityId, result!.VulnerabilityId); + Assert.Equal(productKey, result.Product.Key); + Assert.Equal("Example App", result.Product.Name); + Assert.Equal(VexConsensusStatus.NotAffected, result.Status); + Assert.Single(result.Sources); + var source = result.Sources[0]; + Assert.Equal("red-hat", source.ProviderId); + Assert.Equal(VexClaimStatus.NotAffected, source.Status); + Assert.Equal("sha256:123", source.DocumentDigest); + Assert.Equal(0.9, source.Weight); + Assert.Equal(VexJustification.ComponentNotPresent, source.Justification); + Assert.NotNull(source.Confidence); + Assert.Equal("high", source.Confidence!.Level); + Assert.Equal(0.7, source.Confidence!.Score); + Assert.Equal("review", source.Confidence!.Method); + Assert.Single(result.Conflicts); + var conflict = result.Conflicts[0]; + Assert.Equal("cisco", conflict.ProviderId); + Assert.Equal(VexClaimStatus.Affected, conflict.Status); + Assert.Equal(VexJustification.RequiresConfiguration, conflict.Justification); + Assert.Equal("policy_override", conflict.Reason); + Assert.Equal("Vendor confirms not affected.", result.Summary); + Assert.Equal("2025.10", result.PolicyVersion); + Assert.NotNull(result.Signals); + Assert.True(result.Signals!.Kev); + Assert.Equal(0.42, result.Signals.Epss); + Assert.NotNull(result.Signals.Severity); + Assert.Equal("CVSS:3.1", result.Signals.Severity!.Scheme); + Assert.Equal(7.5, result.Signals.Severity.Score); + } + + [Fact] + public async Task CacheIndex_RoundTripsGridFsMetadata() + { + var gridObjectId = ObjectId.GenerateNewId().ToString(); + var index = new MongoVexCacheIndex(_database); + var signature = new VexQuerySignature("format=csaf|vendor=redhat"); + var now = DateTimeOffset.UtcNow; + var expires = now.AddHours(12); + var entry = new VexCacheEntry( + signature, + VexExportFormat.Csaf, + new VexContentAddress("sha256", "abcdef123456"), + now, + sizeBytes: 1024, + manifestId: "manifest-001", + gridFsObjectId: gridObjectId, + expiresAt: expires); + + await index.SaveAsync(entry, CancellationToken.None); + + var cacheId = string.Format( + CultureInfo.InvariantCulture, + "{0}|{1}", + signature.Value, + entry.Format.ToString().ToLowerInvariant()); + + var cache = _database.GetCollection(VexMongoCollectionNames.Cache); + var filter = Builders.Filter.Eq("_id", cacheId); + var update = Builders.Update.Set("UnexpectedField", true); + await cache.UpdateOneAsync(filter, update); + + var roundTrip = await index.FindAsync(signature, VexExportFormat.Csaf, CancellationToken.None); + + Assert.NotNull(roundTrip); + Assert.Equal(entry.QuerySignature.Value, roundTrip!.QuerySignature.Value); + Assert.Equal(entry.Format, roundTrip.Format); + Assert.Equal(entry.Artifact.Digest, roundTrip.Artifact.Digest); + Assert.Equal(entry.ManifestId, roundTrip.ManifestId); + Assert.Equal(entry.GridFsObjectId, roundTrip.GridFsObjectId); + Assert.Equal(entry.SizeBytes, roundTrip.SizeBytes); + Assert.NotNull(roundTrip.ExpiresAt); + Assert.Equal(expires.ToUnixTimeMilliseconds(), roundTrip.ExpiresAt!.Value.ToUnixTimeMilliseconds()); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj b/src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..847c8d77 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj @@ -0,0 +1,15 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Vexer.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs b/src/StellaOps.Excititor.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs similarity index 63% rename from src/StellaOps.Vexer.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs rename to src/StellaOps.Excititor.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs index cb6ca681..495633ad 100644 --- a/src/StellaOps.Vexer.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs +++ b/src/StellaOps.Excititor.Storage.Mongo.Tests/VexMongoMigrationRunnerTests.cs @@ -1,59 +1,68 @@ -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging.Abstractions; -using Mongo2Go; -using MongoDB.Driver; -using StellaOps.Vexer.Storage.Mongo.Migrations; - -namespace StellaOps.Vexer.Storage.Mongo.Tests; - -public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime -{ - private readonly MongoDbRunner _runner; - private readonly IMongoDatabase _database; - - public VexMongoMigrationRunnerTests() - { - _runner = MongoDbRunner.Start(); - var client = new MongoClient(_runner.ConnectionString); - _database = client.GetDatabase("vexer-migrations-tests"); - } - - [Fact] - public async Task RunAsync_AppliesInitialIndexesOnce() - { - var migration = new VexInitialIndexMigration(); - var runner = new VexMongoMigrationRunner(_database, new[] { migration }, NullLogger.Instance); - - await runner.RunAsync(CancellationToken.None); - await runner.RunAsync(CancellationToken.None); - - var appliedCollection = _database.GetCollection(VexMongoCollectionNames.Migrations); - var applied = await appliedCollection.Find(FilterDefinition.Empty).ToListAsync(); - Assert.Single(applied); - Assert.Equal(migration.Id, applied[0].Id); - - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Raw), "ProviderId_1_Format_1_RetrievedAt_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Providers), "Kind_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Consensus), "VulnerabilityId_1_Product.Key_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_PolicyDigest_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Exports), "QuerySignature_1_Format_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Cache), "QuerySignature_1_Format_1")); - Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Cache), "ExpiresAt_1")); - } - - private static bool HasIndex(IMongoCollection collection, string name) - { - var indexes = collection.Indexes.List().ToList(); - return indexes.Any(index => index["name"].AsString == name); - } - - public Task InitializeAsync() => Task.CompletedTask; - - public Task DisposeAsync() - { - _runner.Dispose(); - return Task.CompletedTask; - } -} +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Excititor.Storage.Mongo.Migrations; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Storage.Mongo.Tests; + +public sealed class VexMongoMigrationRunnerTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner; + private readonly IMongoDatabase _database; + + public VexMongoMigrationRunnerTests() + { + _runner = MongoDbRunner.Start(); + var client = new MongoClient(_runner.ConnectionString); + _database = client.GetDatabase("excititor-migrations-tests"); + } + + [Fact] + public async Task RunAsync_AppliesInitialIndexesOnce() + { + var migrations = new IVexMongoMigration[] + { + new VexInitialIndexMigration(), + new VexConsensusSignalsMigration(), + }; + var runner = new VexMongoMigrationRunner(_database, migrations, NullLogger.Instance); + + await runner.RunAsync(CancellationToken.None); + await runner.RunAsync(CancellationToken.None); + + var appliedCollection = _database.GetCollection(VexMongoCollectionNames.Migrations); + var applied = await appliedCollection.Find(FilterDefinition.Empty).ToListAsync(); + Assert.Equal(2, applied.Count); + Assert.Equal(migrations.Select(m => m.Id).OrderBy(id => id, StringComparer.Ordinal), applied.Select(record => record.Id).OrderBy(id => id, StringComparer.Ordinal)); + + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Raw), "ProviderId_1_Format_1_RetrievedAt_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Providers), "Kind_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Consensus), "VulnerabilityId_1_Product.Key_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_PolicyDigest_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Consensus), "PolicyRevisionId_1_CalculatedAt_-1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Exports), "QuerySignature_1_Format_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Cache), "QuerySignature_1_Format_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Cache), "ExpiresAt_1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Statements), "VulnerabilityId_1_Product.Key_1_InsertedAt_-1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Statements), "ProviderId_1_InsertedAt_-1")); + Assert.True(HasIndex(_database.GetCollection(VexMongoCollectionNames.Statements), "Document.Digest_1")); + } + + private static bool HasIndex(IMongoCollection collection, string name) + { + var indexes = collection.Indexes.List().ToList(); + return indexes.Any(index => index["name"].AsString == name); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/AGENTS.md b/src/StellaOps.Excititor.Storage.Mongo/AGENTS.md similarity index 88% rename from src/StellaOps.Vexer.Storage.Mongo/AGENTS.md rename to src/StellaOps.Excititor.Storage.Mongo/AGENTS.md index 45b4988b..c81334f3 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/AGENTS.md +++ b/src/StellaOps.Excititor.Storage.Mongo/AGENTS.md @@ -1,24 +1,24 @@ -# AGENTS -## Role -MongoDB persistence layer for Vexer raw documents, claims, consensus snapshots, exports, and cache metadata. -## Scope -- Collection schemas, Bson class maps, repositories, and transactional write patterns for ingest/export flows. -- GridFS integration for raw source documents and artifact metadata persistence. -- Migrations, index builders, and bootstrap routines aligned with offline-first deployments. -- Deterministic query helpers used by WebService, Worker, and Export modules. -## Participants -- WebService invokes repositories to store ingest runs, recompute consensus, and register exports. -- Worker relies on repositories for resume markers, retry queues, and cache GC flows. -- Export/Attestation modules pull stored claims/consensus data for snapshot building. -## Interfaces & contracts -- Repository abstractions (`IVexRawStore`, `IVexClaimStore`, `IVexConsensusStore`, `IVexExportStore`, `IVexCacheIndex`) and migration host interfaces. -- Diagnostics hooks providing collection health metrics and schema validation results. -## In/Out of scope -In: MongoDB data access, migrations, transactional semantics, schema documentation. -Out: domain modeling (Core), policy evaluation (Policy), HTTP surfaces (WebService). -## Observability & security expectations -- Emit structured logs for collection/migration events including revision ids and elapsed timings. -- Expose health metrics (counts, queue backlog) and publish to OpenTelemetry when enabled. -- Ensure no raw secret material is logged; mask tokens/URLs in diagnostics. -## Tests -- Integration fixtures (Mongo runner) and schema regression tests will reside in `../StellaOps.Vexer.Storage.Mongo.Tests`. +# AGENTS +## Role +MongoDB persistence layer for Excititor raw documents, claims, consensus snapshots, exports, and cache metadata. +## Scope +- Collection schemas, Bson class maps, repositories, and transactional write patterns for ingest/export flows. +- GridFS integration for raw source documents and artifact metadata persistence. +- Migrations, index builders, and bootstrap routines aligned with offline-first deployments. +- Deterministic query helpers used by WebService, Worker, and Export modules. +## Participants +- WebService invokes repositories to store ingest runs, recompute consensus, and register exports. +- Worker relies on repositories for resume markers, retry queues, and cache GC flows. +- Export/Attestation modules pull stored claims/consensus data for snapshot building. +## Interfaces & contracts +- Repository abstractions (`IVexRawStore`, `IVexClaimStore`, `IVexConsensusStore`, `IVexExportStore`, `IVexCacheIndex`) and migration host interfaces. +- Diagnostics hooks providing collection health metrics and schema validation results. +## In/Out of scope +In: MongoDB data access, migrations, transactional semantics, schema documentation. +Out: domain modeling (Core), policy evaluation (Policy), HTTP surfaces (WebService). +## Observability & security expectations +- Emit structured logs for collection/migration events including revision ids and elapsed timings. +- Expose health metrics (counts, queue backlog) and publish to OpenTelemetry when enabled. +- Ensure no raw secret material is logged; mask tokens/URLs in diagnostics. +## Tests +- Integration fixtures (Mongo runner) and schema regression tests will reside in `../StellaOps.Excititor.Storage.Mongo.Tests`. diff --git a/src/StellaOps.Vexer.Storage.Mongo/IVexRawStore.cs b/src/StellaOps.Excititor.Storage.Mongo/IVexExportStore.cs similarity index 52% rename from src/StellaOps.Vexer.Storage.Mongo/IVexRawStore.cs rename to src/StellaOps.Excititor.Storage.Mongo/IVexExportStore.cs index ecf3da8e..509c25ea 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/IVexRawStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/IVexExportStore.cs @@ -1,17 +1,13 @@ -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public interface IVexRawStore : IVexRawDocumentSink -{ - ValueTask FindByDigestAsync(string digest, CancellationToken cancellationToken); -} - -public interface IVexExportStore -{ - ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); - - ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken); -} +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public interface IVexExportStore +{ + ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/IVexRawStore.cs b/src/StellaOps.Excititor.Storage.Mongo/IVexRawStore.cs new file mode 100644 index 00000000..dc883cff --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/IVexRawStore.cs @@ -0,0 +1,13 @@ +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public interface IVexRawStore : IVexRawDocumentSink +{ + ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs b/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs new file mode 100644 index 00000000..35e00099 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/IVexStorageContracts.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public interface IVexProviderStore +{ + ValueTask FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public interface IVexConsensusStore +{ + ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public interface IVexClaimStore +{ + ValueTask AppendAsync(IEnumerable claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public sealed record VexConnectorState( + string ConnectorId, + DateTimeOffset? LastUpdated, + ImmutableArray DocumentDigests, + ImmutableDictionary ResumeTokens, + DateTimeOffset? LastSuccessAt, + int FailureCount, + DateTimeOffset? NextEligibleRun, + string? LastFailureReason) +{ + public VexConnectorState( + string connectorId, + DateTimeOffset? lastUpdated, + ImmutableArray documentDigests) + : this( + connectorId, + lastUpdated, + documentDigests, + ImmutableDictionary.Empty, + LastSuccessAt: null, + FailureCount: 0, + NextEligibleRun: null, + LastFailureReason: null) + { + } +} + +public interface IVexConnectorStateRepository +{ + ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public interface IVexCacheIndex +{ + ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null); +} + +public interface IVexCacheMaintenance +{ + ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null); + + ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null); +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Migrations/IVexMongoMigration.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/IVexMongoMigration.cs similarity index 77% rename from src/StellaOps.Vexer.Storage.Mongo/Migrations/IVexMongoMigration.cs rename to src/StellaOps.Excititor.Storage.Mongo/Migrations/IVexMongoMigration.cs index 612442f6..6e10985e 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/Migrations/IVexMongoMigration.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/IVexMongoMigration.cs @@ -1,12 +1,12 @@ -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo.Migrations; - -internal interface IVexMongoMigration -{ - string Id { get; } - - ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken); -} +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal interface IVexMongoMigration +{ + string Id { get; } + + ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusSignalsMigration.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusSignalsMigration.cs new file mode 100644 index 00000000..f00c3e93 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexConsensusSignalsMigration.cs @@ -0,0 +1,52 @@ +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexConsensusSignalsMigration : IVexMongoMigration +{ + public string Id => "20251019-consensus-signals-statements"; + + public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + await EnsureConsensusIndexesAsync(database, cancellationToken).ConfigureAwait(false); + await EnsureStatementIndexesAsync(database, cancellationToken).ConfigureAwait(false); + } + + private static Task EnsureConsensusIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Consensus); + var revisionGeneratedIndex = Builders.IndexKeys + .Ascending(x => x.PolicyRevisionId) + .Descending(x => x.CalculatedAt); + + return collection.Indexes.CreateOneAsync( + new CreateIndexModel(revisionGeneratedIndex), + cancellationToken: cancellationToken); + } + + private static Task EnsureStatementIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Statements); + + var vulnProductInsertedIndex = Builders.IndexKeys + .Ascending(x => x.VulnerabilityId) + .Ascending(x => x.Product.Key) + .Descending(x => x.InsertedAt); + + var providerInsertedIndex = Builders.IndexKeys + .Ascending(x => x.ProviderId) + .Descending(x => x.InsertedAt); + + var digestIndex = Builders.IndexKeys + .Ascending(x => x.Document.Digest); + + return Task.WhenAll( + collection.Indexes.CreateOneAsync(new CreateIndexModel(vulnProductInsertedIndex), cancellationToken: cancellationToken), + collection.Indexes.CreateOneAsync(new CreateIndexModel(providerInsertedIndex), cancellationToken: cancellationToken), + collection.Indexes.CreateOneAsync(new CreateIndexModel(digestIndex), cancellationToken: cancellationToken)); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexInitialIndexMigration.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexInitialIndexMigration.cs similarity index 97% rename from src/StellaOps.Vexer.Storage.Mongo/Migrations/VexInitialIndexMigration.cs rename to src/StellaOps.Excititor.Storage.Mongo/Migrations/VexInitialIndexMigration.cs index 34c916cf..1e26a23e 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexInitialIndexMigration.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexInitialIndexMigration.cs @@ -1,75 +1,75 @@ -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo.Migrations; - -internal sealed class VexInitialIndexMigration : IVexMongoMigration -{ - public string Id => "20251016-initial-indexes"; - - public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(database); - - await EnsureRawIndexesAsync(database, cancellationToken).ConfigureAwait(false); - await EnsureProviderIndexesAsync(database, cancellationToken).ConfigureAwait(false); - await EnsureConsensusIndexesAsync(database, cancellationToken).ConfigureAwait(false); - await EnsureExportIndexesAsync(database, cancellationToken).ConfigureAwait(false); - await EnsureCacheIndexesAsync(database, cancellationToken).ConfigureAwait(false); - } - - private static Task EnsureRawIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(VexMongoCollectionNames.Raw); - var providerFormatIndex = Builders.IndexKeys - .Ascending(x => x.ProviderId) - .Ascending(x => x.Format) - .Ascending(x => x.RetrievedAt); - return collection.Indexes.CreateOneAsync(new CreateIndexModel(providerFormatIndex), cancellationToken: cancellationToken); - } - - private static Task EnsureProviderIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(VexMongoCollectionNames.Providers); - var kindIndex = Builders.IndexKeys.Ascending(x => x.Kind); - return collection.Indexes.CreateOneAsync(new CreateIndexModel(kindIndex), cancellationToken: cancellationToken); - } - - private static Task EnsureConsensusIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(VexMongoCollectionNames.Consensus); - var vulnProductIndex = Builders.IndexKeys - .Ascending(x => x.VulnerabilityId) - .Ascending(x => x.Product.Key); - var policyIndex = Builders.IndexKeys - .Ascending(x => x.PolicyRevisionId) - .Ascending(x => x.PolicyDigest); - - return Task.WhenAll( - collection.Indexes.CreateOneAsync(new CreateIndexModel(vulnProductIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken), - collection.Indexes.CreateOneAsync(new CreateIndexModel(policyIndex), cancellationToken: cancellationToken)); - } - - private static Task EnsureExportIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(VexMongoCollectionNames.Exports); - var signatureIndex = Builders.IndexKeys - .Ascending(x => x.QuerySignature) - .Ascending(x => x.Format); - return collection.Indexes.CreateOneAsync(new CreateIndexModel(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken); - } - - private static Task EnsureCacheIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) - { - var collection = database.GetCollection(VexMongoCollectionNames.Cache); - var signatureIndex = Builders.IndexKeys - .Ascending(x => x.QuerySignature) - .Ascending(x => x.Format); - var expirationIndex = Builders.IndexKeys.Ascending(x => x.ExpiresAt); - - return Task.WhenAll( - collection.Indexes.CreateOneAsync(new CreateIndexModel(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken), - collection.Indexes.CreateOneAsync(new CreateIndexModel(expirationIndex, new CreateIndexOptions { ExpireAfter = TimeSpan.FromSeconds(0) }), cancellationToken: cancellationToken)); - } -} +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexInitialIndexMigration : IVexMongoMigration +{ + public string Id => "20251016-initial-indexes"; + + public async ValueTask ExecuteAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(database); + + await EnsureRawIndexesAsync(database, cancellationToken).ConfigureAwait(false); + await EnsureProviderIndexesAsync(database, cancellationToken).ConfigureAwait(false); + await EnsureConsensusIndexesAsync(database, cancellationToken).ConfigureAwait(false); + await EnsureExportIndexesAsync(database, cancellationToken).ConfigureAwait(false); + await EnsureCacheIndexesAsync(database, cancellationToken).ConfigureAwait(false); + } + + private static Task EnsureRawIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Raw); + var providerFormatIndex = Builders.IndexKeys + .Ascending(x => x.ProviderId) + .Ascending(x => x.Format) + .Ascending(x => x.RetrievedAt); + return collection.Indexes.CreateOneAsync(new CreateIndexModel(providerFormatIndex), cancellationToken: cancellationToken); + } + + private static Task EnsureProviderIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Providers); + var kindIndex = Builders.IndexKeys.Ascending(x => x.Kind); + return collection.Indexes.CreateOneAsync(new CreateIndexModel(kindIndex), cancellationToken: cancellationToken); + } + + private static Task EnsureConsensusIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Consensus); + var vulnProductIndex = Builders.IndexKeys + .Ascending(x => x.VulnerabilityId) + .Ascending(x => x.Product.Key); + var policyIndex = Builders.IndexKeys + .Ascending(x => x.PolicyRevisionId) + .Ascending(x => x.PolicyDigest); + + return Task.WhenAll( + collection.Indexes.CreateOneAsync(new CreateIndexModel(vulnProductIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken), + collection.Indexes.CreateOneAsync(new CreateIndexModel(policyIndex), cancellationToken: cancellationToken)); + } + + private static Task EnsureExportIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Exports); + var signatureIndex = Builders.IndexKeys + .Ascending(x => x.QuerySignature) + .Ascending(x => x.Format); + return collection.Indexes.CreateOneAsync(new CreateIndexModel(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken); + } + + private static Task EnsureCacheIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(VexMongoCollectionNames.Cache); + var signatureIndex = Builders.IndexKeys + .Ascending(x => x.QuerySignature) + .Ascending(x => x.Format); + var expirationIndex = Builders.IndexKeys.Ascending(x => x.ExpiresAt); + + return Task.WhenAll( + collection.Indexes.CreateOneAsync(new CreateIndexModel(signatureIndex, new CreateIndexOptions { Unique = true }), cancellationToken: cancellationToken), + collection.Indexes.CreateOneAsync(new CreateIndexModel(expirationIndex, new CreateIndexOptions { ExpireAfter = TimeSpan.FromSeconds(0) }), cancellationToken: cancellationToken)); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMigrationRecord.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMigrationRecord.cs similarity index 85% rename from src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMigrationRecord.cs rename to src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMigrationRecord.cs index 9c392bdb..8ce25398 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMigrationRecord.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMigrationRecord.cs @@ -1,18 +1,18 @@ -using System; -using MongoDB.Bson.Serialization.Attributes; - -namespace StellaOps.Vexer.Storage.Mongo.Migrations; - -internal sealed class VexMigrationRecord -{ - public VexMigrationRecord(string id, DateTimeOffset executedAt) - { - Id = string.IsNullOrWhiteSpace(id) ? throw new ArgumentException("Migration id must be provided.", nameof(id)) : id.Trim(); - ExecutedAt = executedAt; - } - - [BsonId] - public string Id { get; } - - public DateTimeOffset ExecutedAt { get; } -} +using System; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexMigrationRecord +{ + public VexMigrationRecord(string id, DateTimeOffset executedAt) + { + Id = string.IsNullOrWhiteSpace(id) ? throw new ArgumentException("Migration id must be provided.", nameof(id)) : id.Trim(); + ExecutedAt = executedAt; + } + + [BsonId] + public string Id { get; } + + public DateTimeOffset ExecutedAt { get; } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs similarity index 88% rename from src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs rename to src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs index 1274bb80..9a415235 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationHostedService.cs @@ -1,22 +1,22 @@ -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; - -namespace StellaOps.Vexer.Storage.Mongo.Migrations; - -internal sealed class VexMongoMigrationHostedService : IHostedService -{ - private readonly VexMongoMigrationRunner _runner; - - public VexMongoMigrationHostedService(VexMongoMigrationRunner runner) - { - _runner = runner ?? throw new ArgumentNullException(nameof(runner)); - } - - public async Task StartAsync(CancellationToken cancellationToken) - { - await _runner.RunAsync(cancellationToken).ConfigureAwait(false); - } - - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; -} +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexMongoMigrationHostedService : IHostedService +{ + private readonly VexMongoMigrationRunner _runner; + + public VexMongoMigrationHostedService(VexMongoMigrationRunner runner) + { + _runner = runner ?? throw new ArgumentNullException(nameof(runner)); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + await _runner.RunAsync(cancellationToken).ConfigureAwait(false); + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs similarity index 88% rename from src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs rename to src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs index 2cff832d..663a941d 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/Migrations/VexMongoMigrationRunner.cs @@ -1,74 +1,74 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo.Migrations; - -internal sealed class VexMongoMigrationRunner -{ - private readonly IMongoDatabase _database; - private readonly IReadOnlyList _migrations; - private readonly ILogger _logger; - - public VexMongoMigrationRunner( - IMongoDatabase database, - IEnumerable migrations, - ILogger logger) - { - _database = database ?? throw new ArgumentNullException(nameof(database)); - _migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations))) - .OrderBy(migration => migration.Id, StringComparer.Ordinal) - .ToArray(); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async ValueTask RunAsync(CancellationToken cancellationToken) - { - if (_migrations.Count == 0) - { - return; - } - - var migrationsCollection = _database.GetCollection(VexMongoCollectionNames.Migrations); - await EnsureMigrationsIndexAsync(migrationsCollection, cancellationToken).ConfigureAwait(false); - - var applied = await LoadAppliedMigrationsAsync(migrationsCollection, cancellationToken).ConfigureAwait(false); - - foreach (var migration in _migrations) - { - if (applied.Contains(migration.Id)) - { - continue; - } - - _logger.LogInformation("Applying Vexer Mongo migration {MigrationId}", migration.Id); - await migration.ExecuteAsync(_database, cancellationToken).ConfigureAwait(false); - - var record = new VexMigrationRecord(migration.Id, DateTimeOffset.UtcNow); - await migrationsCollection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false); - _logger.LogInformation("Completed Vexer Mongo migration {MigrationId}", migration.Id); - } - } - - private static ValueTask EnsureMigrationsIndexAsync( - IMongoCollection collection, - CancellationToken cancellationToken) - { - // default _id index already enforces uniqueness - return ValueTask.CompletedTask; - } - - private static async ValueTask> LoadAppliedMigrationsAsync( - IMongoCollection collection, - CancellationToken cancellationToken) - { - var records = await collection.Find(FilterDefinition.Empty) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - return records.Select(static record => record.Id) - .ToHashSet(StringComparer.Ordinal); - } -} +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo.Migrations; + +internal sealed class VexMongoMigrationRunner +{ + private readonly IMongoDatabase _database; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + + public VexMongoMigrationRunner( + IMongoDatabase database, + IEnumerable migrations, + ILogger logger) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations))) + .OrderBy(migration => migration.Id, StringComparer.Ordinal) + .ToArray(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + var migrationsCollection = _database.GetCollection(VexMongoCollectionNames.Migrations); + await EnsureMigrationsIndexAsync(migrationsCollection, cancellationToken).ConfigureAwait(false); + + var applied = await LoadAppliedMigrationsAsync(migrationsCollection, cancellationToken).ConfigureAwait(false); + + foreach (var migration in _migrations) + { + if (applied.Contains(migration.Id)) + { + continue; + } + + _logger.LogInformation("Applying Excititor Mongo migration {MigrationId}", migration.Id); + await migration.ExecuteAsync(_database, cancellationToken).ConfigureAwait(false); + + var record = new VexMigrationRecord(migration.Id, DateTimeOffset.UtcNow); + await migrationsCollection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed Excititor Mongo migration {MigrationId}", migration.Id); + } + } + + private static ValueTask EnsureMigrationsIndexAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + // default _id index already enforces uniqueness + return ValueTask.CompletedTask; + } + + private static async ValueTask> LoadAppliedMigrationsAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + var records = await collection.Find(FilterDefinition.Empty) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return records.Select(static record => record.Id) + .ToHashSet(StringComparer.Ordinal); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheIndex.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheIndex.cs similarity index 51% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheIndex.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheIndex.cs index 8ea4fa96..64346e00 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheIndex.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheIndex.cs @@ -1,43 +1,59 @@ -using System; -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Bson; -using MongoDB.Driver; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexCacheIndex : IVexCacheIndex -{ - private readonly IMongoCollection _collection; - - public MongoVexCacheIndex(IMongoDatabase database) - { - ArgumentNullException.ThrowIfNull(database); - VexMongoMappingRegistry.Register(); - _collection = database.GetCollection(VexMongoCollectionNames.Cache); - } - - public async ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(signature); - var filter = Builders.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format)); - var record = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return record?.ToDomain(); - } - - public async ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(entry); - var record = VexCacheEntryRecord.FromDomain(entry); - var filter = Builders.Filter.Eq(x => x.Id, record.Id); - await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } - - public async ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(signature); - var filter = Builders.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format)); - await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); - } -} +using System; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexCacheIndex : IVexCacheIndex +{ + private readonly IMongoCollection _collection; + + public MongoVexCacheIndex(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.Cache); + } + + public async ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(signature); + var filter = Builders.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format)); + var record = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return record?.ToDomain(); + } + + public async ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(entry); + var record = VexCacheEntryRecord.FromDomain(entry); + var filter = Builders.Filter.Eq(x => x.Id, record.Id); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(signature); + var filter = Builders.Filter.Eq(x => x.Id, VexCacheEntryRecord.CreateId(signature, format)); + if (session is null) + { + await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.DeleteOneAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheMaintenance.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheMaintenance.cs similarity index 62% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheMaintenance.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheMaintenance.cs index 77545f37..b6771aad 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexCacheMaintenance.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexCacheMaintenance.cs @@ -1,85 +1,106 @@ -using System.Collections.Generic; -using Microsoft.Extensions.Logging; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo; - -internal sealed class MongoVexCacheMaintenance : IVexCacheMaintenance -{ - private readonly IMongoCollection _cache; - private readonly IMongoCollection _exports; - private readonly ILogger _logger; - - public MongoVexCacheMaintenance( - IMongoDatabase database, - ILogger logger) - { - ArgumentNullException.ThrowIfNull(database); - ArgumentNullException.ThrowIfNull(logger); - - VexMongoMappingRegistry.Register(); - - _cache = database.GetCollection(VexMongoCollectionNames.Cache); - _exports = database.GetCollection(VexMongoCollectionNames.Exports); - _logger = logger; - } - - public async ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken) - { - var cutoff = asOf.UtcDateTime; - var filter = Builders.Filter.Lt(x => x.ExpiresAt, cutoff); - var result = await _cache.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false); - - var removed = (int)result.DeletedCount; - if (removed > 0) - { - _logger.LogInformation("Pruned {Count} expired VEX export cache entries (cutoff {Cutoff})", removed, cutoff); - } - - return removed; - } - - public async ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken) - { - var filter = Builders.Filter.Ne(x => x.ManifestId, null); - var cursor = await _cache.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); - - if (cursor.Count == 0) - { - return 0; - } - - var danglingIds = new List(cursor.Count); - foreach (var entry in cursor) - { - if (string.IsNullOrWhiteSpace(entry.ManifestId)) - { - continue; - } - - var manifestExists = await _exports - .Find(Builders.Filter.Eq(x => x.Id, entry.ManifestId)) - .Limit(1) - .AnyAsync(cancellationToken) - .ConfigureAwait(false); - - if (!manifestExists) - { - danglingIds.Add(entry.Id); - } - } - - if (danglingIds.Count == 0) - { - return 0; - } - - var danglingFilter = Builders.Filter.In(x => x.Id, danglingIds); - var result = await _cache.DeleteManyAsync(danglingFilter, cancellationToken).ConfigureAwait(false); - - var removed = (int)result.DeletedCount; - _logger.LogWarning("Removed {Count} cache entries referencing missing export manifests.", removed); - - return removed; - } -} +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo; + +internal sealed class MongoVexCacheMaintenance : IVexCacheMaintenance +{ + private readonly IMongoCollection _cache; + private readonly IMongoCollection _exports; + private readonly ILogger _logger; + + public MongoVexCacheMaintenance( + IMongoDatabase database, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(logger); + + VexMongoMappingRegistry.Register(); + + _cache = database.GetCollection(VexMongoCollectionNames.Cache); + _exports = database.GetCollection(VexMongoCollectionNames.Exports); + _logger = logger; + } + + public async ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var cutoff = asOf.UtcDateTime; + var filter = Builders.Filter.Lt(x => x.ExpiresAt, cutoff); + DeleteResult result; + if (session is null) + { + result = await _cache.DeleteManyAsync(filter, cancellationToken).ConfigureAwait(false); + } + else + { + result = await _cache.DeleteManyAsync(session, filter, options: null, cancellationToken).ConfigureAwait(false); + } + + var removed = (int)result.DeletedCount; + if (removed > 0) + { + _logger.LogInformation("Pruned {Count} expired VEX export cache entries (cutoff {Cutoff})", removed, cutoff); + } + + return removed; + } + + public async ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var filter = Builders.Filter.Ne(x => x.ManifestId, null); + var cursor = session is null + ? await _cache.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false) + : await _cache.Find(session, filter).ToListAsync(cancellationToken).ConfigureAwait(false); + + if (cursor.Count == 0) + { + return 0; + } + + var danglingIds = new List(cursor.Count); + foreach (var entry in cursor) + { + if (string.IsNullOrWhiteSpace(entry.ManifestId)) + { + continue; + } + + var manifestFilter = Builders.Filter.Eq(x => x.Id, entry.ManifestId); + var manifestQuery = session is null + ? _exports.Find(manifestFilter) + : _exports.Find(session, manifestFilter); + var manifestExists = await manifestQuery + .Limit(1) + .AnyAsync(cancellationToken) + .ConfigureAwait(false); + + if (!manifestExists) + { + danglingIds.Add(entry.Id); + } + } + + if (danglingIds.Count == 0) + { + return 0; + } + + var danglingFilter = Builders.Filter.In(x => x.Id, danglingIds); + DeleteResult result; + if (session is null) + { + result = await _cache.DeleteManyAsync(danglingFilter, cancellationToken).ConfigureAwait(false); + } + else + { + result = await _cache.DeleteManyAsync(session, danglingFilter, options: null, cancellationToken).ConfigureAwait(false); + } + + var removed = (int)result.DeletedCount; + _logger.LogWarning("Removed {Count} cache entries referencing missing export manifests.", removed); + + return removed; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/MongoVexClaimStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexClaimStore.cs new file mode 100644 index 00000000..24e7faec --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexClaimStore.cs @@ -0,0 +1,67 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexClaimStore : IVexClaimStore +{ + private readonly IMongoCollection _collection; + + public MongoVexClaimStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.Statements); + } + + public async ValueTask AppendAsync(IEnumerable claims, DateTimeOffset observedAt, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(claims); + var records = claims + .Select(claim => VexStatementRecord.FromDomain(claim, observedAt)) + .ToList(); + + if (records.Count == 0) + { + return; + } + + if (session is null) + { + await _collection.InsertManyAsync(records, new InsertManyOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.InsertManyAsync(session, records, new InsertManyOptions { IsOrdered = false }, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask> FindAsync(string vulnerabilityId, string productKey, DateTimeOffset? since, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); + ArgumentException.ThrowIfNullOrWhiteSpace(productKey); + + var filter = Builders.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim()) & + Builders.Filter.Eq(x => x.Product.Key, productKey.Trim()); + + if (since is { } sinceValue) + { + filter &= Builders.Filter.Gte(x => x.InsertedAt, sinceValue.UtcDateTime); + } + + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + + var records = await find + .SortByDescending(x => x.InsertedAt) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return records.ConvertAll(static record => record.ToDomain()); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConnectorStateRepository.cs similarity index 67% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexConnectorStateRepository.cs index b3dda181..e8f7de4b 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexConnectorStateRepository.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConnectorStateRepository.cs @@ -1,55 +1,64 @@ -using System; -using System.Collections.Immutable; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Driver; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexConnectorStateRepository : IVexConnectorStateRepository -{ - private readonly IMongoCollection _collection; - - public MongoVexConnectorStateRepository(IMongoDatabase database) - { - ArgumentNullException.ThrowIfNull(database); - VexMongoMappingRegistry.Register(); - _collection = database.GetCollection(VexMongoCollectionNames.ConnectorState); - } - - public async ValueTask GetAsync(string connectorId, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(connectorId); - - var filter = Builders.Filter.Eq(x => x.ConnectorId, connectorId.Trim()); - var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return document?.ToRecord(); - } - - public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(state); - - var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests()); - var filter = Builders.Filter.Eq(x => x.ConnectorId, document.ConnectorId); - await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } -} - -internal static class VexConnectorStateExtensions -{ - private const int MaxDigestHistory = 200; - - public static VexConnectorState WithNormalizedDigests(this VexConnectorState state) - { - var digests = state.DocumentDigests; - if (digests.Length <= MaxDigestHistory) - { - return state; - } - - var trimmed = digests.Skip(digests.Length - MaxDigestHistory).ToImmutableArray(); - return state with { DocumentDigests = trimmed }; - } -} +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexConnectorStateRepository : IVexConnectorStateRepository +{ + private readonly IMongoCollection _collection; + + public MongoVexConnectorStateRepository(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.ConnectorState); + } + + public async ValueTask GetAsync(string connectorId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectorId); + + var filter = Builders.Filter.Eq(x => x.ConnectorId, connectorId.Trim()); + var document = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document?.ToRecord(); + } + + public async ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(state); + + var document = VexConnectorStateDocument.FromRecord(state.WithNormalizedDigests()); + var filter = Builders.Filter.Eq(x => x.ConnectorId, document.ConnectorId); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } +} + +internal static class VexConnectorStateExtensions +{ + private const int MaxDigestHistory = 200; + + public static VexConnectorState WithNormalizedDigests(this VexConnectorState state) + { + var digests = state.DocumentDigests; + if (digests.Length <= MaxDigestHistory) + { + return state; + } + + var trimmed = digests.Skip(digests.Length - MaxDigestHistory).ToImmutableArray(); + return state with { DocumentDigests = trimmed }; + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexConsensusStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs similarity index 58% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexConsensusStore.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs index 684de965..472fba3b 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexConsensusStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexConsensusStore.cs @@ -1,46 +1,58 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Driver; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexConsensusStore : IVexConsensusStore -{ - private readonly IMongoCollection _collection; - - public MongoVexConsensusStore(IMongoDatabase database) - { - ArgumentNullException.ThrowIfNull(database); - VexMongoMappingRegistry.Register(); - _collection = database.GetCollection(VexMongoCollectionNames.Consensus); - } - - public async ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); - ArgumentException.ThrowIfNullOrWhiteSpace(productKey); - var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey); - var filter = Builders.Filter.Eq(x => x.Id, id); - var record = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return record?.ToDomain(); - } - - public async ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); - var filter = Builders.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim()); - var records = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); - return records.ConvertAll(static record => record.ToDomain()); - } - - public async ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(consensus); - var record = VexConsensusRecord.FromDomain(consensus); - var filter = Builders.Filter.Eq(x => x.Id, record.Id); - await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } - -} +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexConsensusStore : IVexConsensusStore +{ + private readonly IMongoCollection _collection; + + public MongoVexConsensusStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.Consensus); + } + + public async ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); + ArgumentException.ThrowIfNullOrWhiteSpace(productKey); + var id = VexConsensusRecord.CreateId(vulnerabilityId, productKey); + var filter = Builders.Filter.Eq(x => x.Id, id); + var record = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return record?.ToDomain(); + } + + public async ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(vulnerabilityId); + var filter = Builders.Filter.Eq(x => x.VulnerabilityId, vulnerabilityId.Trim()); + var find = session is null + ? _collection.Find(filter) + : _collection.Find(session, filter); + var records = await find.ToListAsync(cancellationToken).ConfigureAwait(false); + return records.ConvertAll(static record => record.ToDomain()); + } + + public async ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(consensus); + var record = VexConsensusRecord.FromDomain(consensus); + var filter = Builders.Filter.Eq(x => x.Id, record.Id); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } + +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexExportStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexExportStore.cs similarity index 64% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexExportStore.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexExportStore.cs index 1be1b47c..4957b78c 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexExportStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexExportStore.cs @@ -1,150 +1,172 @@ -using System; -using System.ComponentModel.DataAnnotations; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Options; -using MongoDB.Driver; -using MongoDB.Driver.Core.Clusters; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexExportStore : IVexExportStore -{ - private readonly IMongoClient _client; - private readonly IMongoCollection _exports; - private readonly IMongoCollection _cache; - private readonly VexMongoStorageOptions _options; - - public MongoVexExportStore( - IMongoClient client, - IMongoDatabase database, - IOptions options) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - ArgumentNullException.ThrowIfNull(database); - ArgumentNullException.ThrowIfNull(options); - - _options = options.Value; - Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true); - - VexMongoMappingRegistry.Register(); - _exports = database.GetCollection(VexMongoCollectionNames.Exports); - _cache = database.GetCollection(VexMongoCollectionNames.Cache); - } - - public async ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(signature); - var cacheId = VexCacheEntryRecord.CreateId(signature, format); - var cacheFilter = Builders.Filter.Eq(x => x.Id, cacheId); - var cacheRecord = await _cache.Find(cacheFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - - if (cacheRecord is null) - { - return null; - } - - if (cacheRecord.ExpiresAt is DateTime expiresAt && expiresAt <= DateTime.UtcNow) - { - await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); - return null; - } - - var manifestId = VexExportManifestRecord.CreateId(signature, format); - var manifestFilter = Builders.Filter.Eq(x => x.Id, manifestId); - var manifest = await _exports.Find(manifestFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - - if (manifest is null) - { - await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); - return null; - } - - if (!string.IsNullOrWhiteSpace(cacheRecord.ManifestId) && - !string.Equals(cacheRecord.ManifestId, manifest.Id, StringComparison.Ordinal)) - { - await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); - return null; - } - - return manifest.ToDomain(); - } - - public async ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(manifest); - - using var session = await _client.StartSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - var supportsTransactions = session.Client.Cluster.Description.Type != ClusterType.Standalone; - - var startedTransaction = false; - if (supportsTransactions) - { - try - { - session.StartTransaction(); - startedTransaction = true; - } - catch (NotSupportedException) - { - supportsTransactions = false; - } - } - - try - { - var manifestRecord = VexExportManifestRecord.FromDomain(manifest); - var manifestFilter = Builders.Filter.Eq(x => x.Id, manifestRecord.Id); - - await _exports - .ReplaceOneAsync( - session, - manifestFilter, - manifestRecord, - new ReplaceOptions { IsUpsert = true }, - cancellationToken) - .ConfigureAwait(false); - - var cacheEntry = CreateCacheEntry(manifest); - var cacheRecord = VexCacheEntryRecord.FromDomain(cacheEntry); - var cacheFilter = Builders.Filter.Eq(x => x.Id, cacheRecord.Id); - - await _cache - .ReplaceOneAsync( - session, - cacheFilter, - cacheRecord, - new ReplaceOptions { IsUpsert = true }, - cancellationToken) - .ConfigureAwait(false); - - if (startedTransaction) - { - await session.CommitTransactionAsync(cancellationToken).ConfigureAwait(false); - } - } - catch - { - if (startedTransaction && session.IsInTransaction) - { - await session.AbortTransactionAsync(cancellationToken).ConfigureAwait(false); - } - throw; - } - } - - private VexCacheEntry CreateCacheEntry(VexExportManifest manifest) - { - var expiresAt = manifest.CreatedAt + _options.ExportCacheTtl; - return new VexCacheEntry( - manifest.QuerySignature, - manifest.Format, - manifest.Artifact, - manifest.CreatedAt, - manifest.SizeBytes, - manifestId: manifest.ExportId, - gridFsObjectId: null, - expiresAt: expiresAt); - } -} +using System; +using System.ComponentModel.DataAnnotations; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using MongoDB.Driver.Core.Clusters; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexExportStore : IVexExportStore +{ + private readonly IMongoClient _client; + private readonly IMongoCollection _exports; + private readonly IMongoCollection _cache; + private readonly VexMongoStorageOptions _options; + private readonly IVexMongoSessionProvider _sessionProvider; + + public MongoVexExportStore( + IMongoClient client, + IMongoDatabase database, + IOptions options, + IVexMongoSessionProvider sessionProvider) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(options); + _sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider)); + + _options = options.Value; + Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true); + + VexMongoMappingRegistry.Register(); + _exports = database.GetCollection(VexMongoCollectionNames.Exports); + _cache = database.GetCollection(VexMongoCollectionNames.Cache); + } + + public async ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(signature); + var cacheId = VexCacheEntryRecord.CreateId(signature, format); + var cacheFilter = Builders.Filter.Eq(x => x.Id, cacheId); + var cacheRecord = session is null + ? await _cache.Find(cacheFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _cache.Find(session, cacheFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + if (cacheRecord is null) + { + return null; + } + + if (cacheRecord.ExpiresAt is DateTime expiresAt && expiresAt <= DateTime.UtcNow) + { + await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); + return null; + } + + var manifestId = VexExportManifestRecord.CreateId(signature, format); + var manifestFilter = Builders.Filter.Eq(x => x.Id, manifestId); + var manifest = session is null + ? await _exports.Find(manifestFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _exports.Find(session, manifestFilter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + + if (manifest is null) + { + if (session is null) + { + await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); + } + else + { + await _cache.DeleteOneAsync(session, cacheFilter, options: null, cancellationToken).ConfigureAwait(false); + } + return null; + } + + if (!string.IsNullOrWhiteSpace(cacheRecord.ManifestId) && + !string.Equals(cacheRecord.ManifestId, manifest.Id, StringComparison.Ordinal)) + { + if (session is null) + { + await _cache.DeleteOneAsync(cacheFilter, cancellationToken).ConfigureAwait(false); + } + else + { + await _cache.DeleteOneAsync(session, cacheFilter, options: null, cancellationToken).ConfigureAwait(false); + } + return null; + } + + return manifest.ToDomain(); + } + + public async ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(manifest); + + var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone + && !sessionHandle.IsInTransaction; + + var startedTransaction = false; + if (supportsTransactions) + { + try + { + sessionHandle.StartTransaction(); + startedTransaction = true; + } + catch (NotSupportedException) + { + supportsTransactions = false; + } + } + + try + { + var manifestRecord = VexExportManifestRecord.FromDomain(manifest); + var manifestFilter = Builders.Filter.Eq(x => x.Id, manifestRecord.Id); + + await _exports + .ReplaceOneAsync( + sessionHandle, + manifestFilter, + manifestRecord, + new ReplaceOptions { IsUpsert = true }, + cancellationToken) + .ConfigureAwait(false); + + var cacheEntry = CreateCacheEntry(manifest); + var cacheRecord = VexCacheEntryRecord.FromDomain(cacheEntry); + var cacheFilter = Builders.Filter.Eq(x => x.Id, cacheRecord.Id); + + await _cache + .ReplaceOneAsync( + sessionHandle, + cacheFilter, + cacheRecord, + new ReplaceOptions { IsUpsert = true }, + cancellationToken) + .ConfigureAwait(false); + + if (startedTransaction) + { + await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false); + } + } + catch + { + if (startedTransaction && sessionHandle.IsInTransaction) + { + await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false); + } + throw; + } + } + + private VexCacheEntry CreateCacheEntry(VexExportManifest manifest) + { + var expiresAt = manifest.CreatedAt + _options.ExportCacheTtl; + return new VexCacheEntry( + manifest.QuerySignature, + manifest.Format, + manifest.Artifact, + manifest.CreatedAt, + manifest.SizeBytes, + manifestId: manifest.ExportId, + gridFsObjectId: null, + expiresAt: expiresAt); + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexProviderStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexProviderStore.cs similarity index 52% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexProviderStore.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexProviderStore.cs index b551b820..37dc9531 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexProviderStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexProviderStore.cs @@ -1,45 +1,57 @@ -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using MongoDB.Driver; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexProviderStore : IVexProviderStore -{ - private readonly IMongoCollection _collection; - - public MongoVexProviderStore(IMongoDatabase database) - { - ArgumentNullException.ThrowIfNull(database); - VexMongoMappingRegistry.Register(); - _collection = database.GetCollection(VexMongoCollectionNames.Providers); - } - - public async ValueTask FindAsync(string id, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(id); - var filter = Builders.Filter.Eq(x => x.Id, id.Trim()); - var record = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - return record?.ToDomain(); - } - - public async ValueTask> ListAsync(CancellationToken cancellationToken) - { - var records = await _collection.Find(FilterDefinition.Empty) - .Sort(Builders.Sort.Ascending(x => x.Id)) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); - - return records.ConvertAll(static record => record.ToDomain()); - } - - public async ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(provider); - var record = VexProviderRecord.FromDomain(provider); - var filter = Builders.Filter.Eq(x => x.Id, record.Id); - await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); - } -} +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexProviderStore : IVexProviderStore +{ + private readonly IMongoCollection _collection; + + public MongoVexProviderStore(IMongoDatabase database) + { + ArgumentNullException.ThrowIfNull(database); + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.Providers); + } + + public async ValueTask FindAsync(string id, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(id); + var filter = Builders.Filter.Eq(x => x.Id, id.Trim()); + var record = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return record?.ToDomain(); + } + + public async ValueTask> ListAsync(CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + var find = session is null + ? _collection.Find(FilterDefinition.Empty) + : _collection.Find(session, FilterDefinition.Empty); + var records = await find + .Sort(Builders.Sort.Ascending(x => x.Id)) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + return records.ConvertAll(static record => record.ToDomain()); + } + + public async ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(provider); + var record = VexProviderRecord.FromDomain(provider); + var filter = Builders.Filter.Eq(x => x.Id, record.Id); + if (session is null) + { + await _collection.ReplaceOneAsync(filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + else + { + await _collection.ReplaceOneAsync(session, filter, record, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/MongoVexRawStore.cs b/src/StellaOps.Excititor.Storage.Mongo/MongoVexRawStore.cs similarity index 66% rename from src/StellaOps.Vexer.Storage.Mongo/MongoVexRawStore.cs rename to src/StellaOps.Excititor.Storage.Mongo/MongoVexRawStore.cs index 92a3de3f..a28988bc 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/MongoVexRawStore.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/MongoVexRawStore.cs @@ -1,199 +1,210 @@ -using System; -using System.ComponentModel.DataAnnotations; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Options; -using MongoDB.Bson; -using MongoDB.Driver; -using MongoDB.Driver.Core.Clusters; -using MongoDB.Driver.GridFS; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public sealed class MongoVexRawStore : IVexRawStore -{ - private readonly IMongoClient _client; - private readonly IMongoCollection _collection; - private readonly GridFSBucket _bucket; - private readonly VexMongoStorageOptions _options; - - public MongoVexRawStore( - IMongoClient client, - IMongoDatabase database, - IOptions options) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - ArgumentNullException.ThrowIfNull(database); - ArgumentNullException.ThrowIfNull(options); - - _options = options.Value; - Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true); - - VexMongoMappingRegistry.Register(); - _collection = database.GetCollection(VexMongoCollectionNames.Raw); - _bucket = new GridFSBucket(database, new GridFSBucketOptions - { - BucketName = _options.RawBucketName, - ReadConcern = database.Settings.ReadConcern, - ReadPreference = database.Settings.ReadPreference, - WriteConcern = database.Settings.WriteConcern, - }); - } - - public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(document); - - var threshold = _options.GridFsInlineThresholdBytes; - var useInline = threshold == 0 || document.Content.Length <= threshold; - string? newGridId = null; - string? oldGridIdToDelete = null; - - if (!useInline) - { - newGridId = await UploadToGridFsAsync(document, cancellationToken).ConfigureAwait(false); - } - - using var session = await _client.StartSessionAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - var supportsTransactions = session.Client.Cluster.Description.Type != ClusterType.Standalone; - - var startedTransaction = false; - if (supportsTransactions) - { - try - { - session.StartTransaction(); - startedTransaction = true; - } - catch (NotSupportedException) - { - supportsTransactions = false; - } - } - - try - { - var filter = Builders.Filter.Eq(x => x.Id, document.Digest); - var existing = await _collection - .Find(session, filter) - .FirstOrDefaultAsync(cancellationToken) - .ConfigureAwait(false); - - var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline); - record.GridFsObjectId = useInline ? null : newGridId; - - await _collection - .ReplaceOneAsync( - session, - filter, - record, - new ReplaceOptions { IsUpsert = true }, - cancellationToken) - .ConfigureAwait(false); - - if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId)) - { - if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal)) - { - oldGridIdToDelete = oldGridId; - } - } - - if (startedTransaction) - { - await session.CommitTransactionAsync(cancellationToken).ConfigureAwait(false); - } - } - catch - { - if (startedTransaction && session.IsInTransaction) - { - await session.AbortTransactionAsync(cancellationToken).ConfigureAwait(false); - } - - if (!useInline && !string.IsNullOrWhiteSpace(newGridId)) - { - await DeleteFromGridFsAsync(newGridId, cancellationToken).ConfigureAwait(false); - } - - throw; - } - - if (!string.IsNullOrWhiteSpace(oldGridIdToDelete)) - { - await DeleteFromGridFsAsync(oldGridIdToDelete!, cancellationToken).ConfigureAwait(false); - } - } - - public async ValueTask FindByDigestAsync(string digest, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(digest)) - { - throw new ArgumentException("Digest must be provided.", nameof(digest)); - } - - var trimmed = digest.Trim(); - var filter = Builders.Filter.Eq(x => x.Id, trimmed); - var record = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); - if (record is null) - { - return null; - } - - if (!string.IsNullOrWhiteSpace(record.GridFsObjectId)) - { - var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, cancellationToken).ConfigureAwait(false); - return record.ToDomain(new ReadOnlyMemory(bytes)); - } - - return record.ToDomain(); - } - - private async Task UploadToGridFsAsync(VexRawDocument document, CancellationToken cancellationToken) - { - using var stream = new MemoryStream(document.Content.ToArray(), writable: false); - var metadata = new BsonDocument - { - { "providerId", document.ProviderId }, - { "format", document.Format.ToString().ToLowerInvariant() }, - { "sourceUri", document.SourceUri.ToString() }, - { "retrievedAt", document.RetrievedAt.UtcDateTime }, - }; - - var options = new GridFSUploadOptions { Metadata = metadata }; - var objectId = await _bucket - .UploadFromStreamAsync(document.Digest, stream, options, cancellationToken) - .ConfigureAwait(false); - - return objectId.ToString(); - } - - private async Task DeleteFromGridFsAsync(string gridFsObjectId, CancellationToken cancellationToken) - { - if (!ObjectId.TryParse(gridFsObjectId, out var objectId)) - { - return; - } - - try - { - await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false); - } - catch (GridFSFileNotFoundException) - { - // file already removed by TTL or manual cleanup - } - } - - private async Task DownloadFromGridFsAsync(string gridFsObjectId, CancellationToken cancellationToken) - { - if (!ObjectId.TryParse(gridFsObjectId, out var objectId)) - { - return Array.Empty(); - } - - return await _bucket.DownloadAsBytesAsync(objectId, cancellationToken: cancellationToken).ConfigureAwait(false); - } -} +using System; +using System.ComponentModel.DataAnnotations; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.Core.Clusters; +using MongoDB.Driver.GridFS; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed class MongoVexRawStore : IVexRawStore +{ + private readonly IMongoClient _client; + private readonly IMongoCollection _collection; + private readonly GridFSBucket _bucket; + private readonly VexMongoStorageOptions _options; + private readonly IVexMongoSessionProvider _sessionProvider; + + public MongoVexRawStore( + IMongoClient client, + IMongoDatabase database, + IOptions options, + IVexMongoSessionProvider sessionProvider) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + ArgumentNullException.ThrowIfNull(database); + ArgumentNullException.ThrowIfNull(options); + _sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider)); + + _options = options.Value; + Validator.ValidateObject(_options, new ValidationContext(_options), validateAllProperties: true); + + VexMongoMappingRegistry.Register(); + _collection = database.GetCollection(VexMongoCollectionNames.Raw); + _bucket = new GridFSBucket(database, new GridFSBucketOptions + { + BucketName = _options.RawBucketName, + ReadConcern = database.Settings.ReadConcern, + ReadPreference = database.Settings.ReadPreference, + WriteConcern = database.Settings.WriteConcern, + }); + } + + public async ValueTask StoreAsync(VexRawDocument document, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + ArgumentNullException.ThrowIfNull(document); + + var threshold = _options.GridFsInlineThresholdBytes; + var useInline = threshold == 0 || document.Content.Length <= threshold; + string? newGridId = null; + string? oldGridIdToDelete = null; + + var sessionHandle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + if (!useInline) + { + newGridId = await UploadToGridFsAsync(document, sessionHandle, cancellationToken).ConfigureAwait(false); + } + + var supportsTransactions = sessionHandle.Client.Cluster.Description.Type != ClusterType.Standalone + && !sessionHandle.IsInTransaction; + + var startedTransaction = false; + if (supportsTransactions) + { + try + { + sessionHandle.StartTransaction(); + startedTransaction = true; + } + catch (NotSupportedException) + { + supportsTransactions = false; + } + } + + try + { + var filter = Builders.Filter.Eq(x => x.Id, document.Digest); + var existing = await _collection + .Find(sessionHandle, filter) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + + var record = VexRawDocumentRecord.FromDomain(document, includeContent: useInline); + record.GridFsObjectId = useInline ? null : newGridId; + + await _collection + .ReplaceOneAsync( + sessionHandle, + filter, + record, + new ReplaceOptions { IsUpsert = true }, + cancellationToken) + .ConfigureAwait(false); + + if (existing?.GridFsObjectId is string oldGridId && !string.IsNullOrWhiteSpace(oldGridId)) + { + if (useInline || !string.Equals(newGridId, oldGridId, StringComparison.Ordinal)) + { + oldGridIdToDelete = oldGridId; + } + } + + if (startedTransaction) + { + await sessionHandle.CommitTransactionAsync(cancellationToken).ConfigureAwait(false); + } + } + catch + { + if (startedTransaction && sessionHandle.IsInTransaction) + { + await sessionHandle.AbortTransactionAsync(cancellationToken).ConfigureAwait(false); + } + + if (!useInline && !string.IsNullOrWhiteSpace(newGridId)) + { + await DeleteFromGridFsAsync(newGridId, sessionHandle, cancellationToken).ConfigureAwait(false); + } + + throw; + } + + if (!string.IsNullOrWhiteSpace(oldGridIdToDelete)) + { + await DeleteFromGridFsAsync(oldGridIdToDelete!, sessionHandle, cancellationToken).ConfigureAwait(false); + } + } + + public async ValueTask FindByDigestAsync(string digest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + if (string.IsNullOrWhiteSpace(digest)) + { + throw new ArgumentException("Digest must be provided.", nameof(digest)); + } + + var trimmed = digest.Trim(); + var filter = Builders.Filter.Eq(x => x.Id, trimmed); + var record = session is null + ? await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false) + : await _collection.Find(session, filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + if (record is null) + { + return null; + } + + if (!string.IsNullOrWhiteSpace(record.GridFsObjectId)) + { + var handle = session ?? await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + var bytes = await DownloadFromGridFsAsync(record.GridFsObjectId, handle, cancellationToken).ConfigureAwait(false); + return record.ToDomain(new ReadOnlyMemory(bytes)); + } + + return record.ToDomain(); + } + + private async Task UploadToGridFsAsync(VexRawDocument document, IClientSessionHandle? session, CancellationToken cancellationToken) + { + using var stream = new MemoryStream(document.Content.ToArray(), writable: false); + var metadata = new BsonDocument + { + { "providerId", document.ProviderId }, + { "format", document.Format.ToString().ToLowerInvariant() }, + { "sourceUri", document.SourceUri.ToString() }, + { "retrievedAt", document.RetrievedAt.UtcDateTime }, + }; + + var options = new GridFSUploadOptions { Metadata = metadata }; + var objectId = await _bucket + .UploadFromStreamAsync(document.Digest, stream, options, cancellationToken) + .ConfigureAwait(false); + + return objectId.ToString(); + } + + private async Task DeleteFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken) + { + if (!ObjectId.TryParse(gridFsObjectId, out var objectId)) + { + return; + } + + try + { + await _bucket.DeleteAsync(objectId, cancellationToken).ConfigureAwait(false); + } + catch (GridFSFileNotFoundException) + { + // file already removed by TTL or manual cleanup + } + } + + private async Task DownloadFromGridFsAsync(string gridFsObjectId, IClientSessionHandle? session, CancellationToken cancellationToken) + { + if (!ObjectId.TryParse(gridFsObjectId, out var objectId)) + { + return Array.Empty(); + } + + return await _bucket.DownloadAsBytesAsync(objectId, null, cancellationToken).ConfigureAwait(false); + } + + async ValueTask IVexRawDocumentSink.StoreAsync(VexRawDocument document, CancellationToken cancellationToken) + => await StoreAsync(document, cancellationToken, session: null).ConfigureAwait(false); +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Excititor.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..c6256462 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Excititor.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..aebb4409 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/ServiceCollectionExtensions.cs @@ -0,0 +1,65 @@ +using System.ComponentModel.DataAnnotations; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo.Migrations; + +namespace StellaOps.Excititor.Storage.Mongo; + +public static class VexMongoServiceCollectionExtensions +{ + public static IServiceCollection AddExcititorMongoStorage(this IServiceCollection services) + { + services.AddOptions(); + + services.TryAddSingleton(static provider => + { + var options = provider.GetRequiredService>().Value; + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + + var mongoUrl = MongoUrl.Create(options.ConnectionString); + var settings = MongoClientSettings.FromUrl(mongoUrl); + settings.ReadConcern = ReadConcern.Majority; + settings.ReadPreference = ReadPreference.Primary; + settings.WriteConcern = WriteConcern.WMajority.With(wTimeout: options.CommandTimeout); + settings.RetryReads = true; + settings.RetryWrites = true; + return new MongoClient(settings); + }); + + services.TryAddScoped(static provider => + { + var options = provider.GetRequiredService>().Value; + var client = provider.GetRequiredService(); + + var settings = new MongoDatabaseSettings + { + ReadConcern = ReadConcern.Majority, + ReadPreference = ReadPreference.PrimaryPreferred, + WriteConcern = WriteConcern.WMajority.With(wTimeout: options.CommandTimeout), + }; + + return client.GetDatabase(options.GetDatabaseName(), settings); + }); + + services.AddScoped(); + + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + return services; + } +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/StellaOps.Vexer.Storage.Mongo.csproj b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj similarity index 64% rename from src/StellaOps.Vexer.Storage.Mongo/StellaOps.Vexer.Storage.Mongo.csproj rename to src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj index 3a50e0d9..246185f0 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/StellaOps.Vexer.Storage.Mongo.csproj +++ b/src/StellaOps.Excititor.Storage.Mongo/StellaOps.Excititor.Storage.Mongo.csproj @@ -1,19 +1,18 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Storage.Mongo/StorageBackedVexNormalizerRouter.cs b/src/StellaOps.Excititor.Storage.Mongo/StorageBackedVexNormalizerRouter.cs new file mode 100644 index 00000000..0bcce84c --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/StorageBackedVexNormalizerRouter.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +/// +/// Normalizer router that resolves providers from Mongo storage before invoking the format-specific normalizer. +/// +public sealed class StorageBackedVexNormalizerRouter : IVexNormalizerRouter +{ + private readonly VexNormalizerRegistry _registry; + private readonly IVexProviderStore _providerStore; + private readonly IVexMongoSessionProvider _sessionProvider; + private readonly ILogger _logger; + + public StorageBackedVexNormalizerRouter( + IEnumerable normalizers, + IVexProviderStore providerStore, + IVexMongoSessionProvider sessionProvider, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(normalizers); + _providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore)); + _sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + _registry = new VexNormalizerRegistry(normalizers.ToImmutableArray()); + } + + public async ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + + var normalizer = _registry.Resolve(document); + if (normalizer is null) + { + _logger.LogWarning("No normalizer registered for VEX document format {Format}. Skipping normalization for {Digest}.", document.Format, document.Digest); + return new VexClaimBatch( + document, + ImmutableArray.Empty, + ImmutableDictionary.Empty); + } + + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + var provider = await _providerStore.FindAsync(document.ProviderId, cancellationToken, session).ConfigureAwait(false) + ?? new VexProvider(document.ProviderId, document.ProviderId, VexProviderKind.Vendor); + + return await normalizer.NormalizeAsync(document, provider, cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/TASKS.md b/src/StellaOps.Excititor.Storage.Mongo/TASKS.md new file mode 100644 index 00000000..cb2175a7 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/TASKS.md @@ -0,0 +1,11 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-STORAGE-01-001 – Collection schemas & class maps|Team Excititor Storage|EXCITITOR-CORE-01-001|DONE (2025-10-15) – Added Mongo mapping registry with raw/export entities and service registration groundwork.| +|EXCITITOR-STORAGE-01-002 – Migrations & indices bootstrap|Team Excititor Storage|EXCITITOR-STORAGE-01-001|**DONE (2025-10-16)** – Add bootstrapper creating indices (claims by vulnId/product, exports by querySignature, etc.) and migrations for existing deployments.
          2025-10-16: Introduced migration runner + hosted service, initial index migration covers raw/providers/consensus/exports/cache, and tests use Mongo2Go to verify execution.| +|EXCITITOR-STORAGE-01-003 – Repository layer & transactional flows|Team Excititor Storage|EXCITITOR-STORAGE-01-001|**DONE (2025-10-16)** – Added GridFS-backed raw store with transactional upserts (including fallback for non-replicaset Mongo), export/cache repository coordination, and coverage verifying cache TTL + GridFS round-trips.| +|EXCITITOR-STORAGE-01-004 – Provider/consensus/cache mappings|Team Excititor Storage|EXCITITOR-STORAGE-01-001|**DONE (2025-10-16)** – Registered MongoDB class maps for provider/consensus/cache records with forward-compatible field handling and added coverage ensuring GridFS-linked cache entries round-trip cleanly.| +|EXCITITOR-STORAGE-02-001 – Statement events & scoring signals|Team Excititor Storage|EXCITITOR-CORE-02-001|DONE (2025-10-19) – Added immutable `vex.statements` collection + claim store, extended consensus persistence with severity/KEV/EPSS signals, shipped migration `20251019-consensus-signals-statements`, and updated docs. Tests: `dotnet test src/StellaOps.Excititor.Core.Tests/StellaOps.Excititor.Core.Tests.csproj` & `dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`; worker/web suites pending due to NU1903 (`Microsoft.Extensions.Caching.Memory`) advisory.| +|EXCITITOR-STORAGE-03-001 – Statement backfill tooling|Team Excititor Storage|EXCITITOR-STORAGE-02-001|**DONE (2025-10-19)** – Shipped Mongo-backed statement replay service + `/excititor/admin/backfill-statements`, wired CLI command `stellaops excititor backfill-statements`, added integration tests, and documented the runbook in `docs/dev/EXCITITOR_STATEMENT_BACKFILL.md`.| +|EXCITITOR-STORAGE-MONGO-08-001 – Session + causal consistency hardening|Team Excititor Storage|EXCITITOR-STORAGE-01-003|**DONE (2025-10-19)** – Completed session-aware overloads across all repositories, persisted claims/signals/connector state with new Mongo records, updated orchestrators/workers to reuse scoped sessions, and added replica-set consistency tests (`dotnet test src/StellaOps.Excititor.Storage.Mongo.Tests/StellaOps.Excititor.Storage.Mongo.Tests.csproj`). GridFS operations fall back to majority semantics due to driver limits; transactions cover metadata writes to preserve determinism.| diff --git a/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs similarity index 80% rename from src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs rename to src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs index a36abdc5..14e57bdd 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/VexMongoMappingRegistry.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoMappingRegistry.cs @@ -1,71 +1,77 @@ -using System.Threading; -using MongoDB.Bson.Serialization; -using MongoDB.Bson.Serialization.Serializers; - -namespace StellaOps.Vexer.Storage.Mongo; - -public static class VexMongoMappingRegistry -{ - private static int _initialized; - - public static void Register() - { - if (Interlocked.Exchange(ref _initialized, 1) == 1) - { - return; - } - - try - { - BsonSerializer.RegisterSerializer(typeof(byte[]), new ByteArraySerializer()); - } - catch - { - // serializer already registered – safe to ignore - } - - RegisterClassMaps(); - } - - private static void RegisterClassMaps() - { - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - RegisterClassMap(); - } - - private static void RegisterClassMap() - where TDocument : class - { - if (BsonClassMap.IsClassMapRegistered(typeof(TDocument))) - { - return; - } - - BsonClassMap.RegisterClassMap(classMap => - { - classMap.AutoMap(); - classMap.SetIgnoreExtraElements(true); - }); - } -} - -public static class VexMongoCollectionNames -{ - public const string Migrations = "vex.migrations"; - public const string Providers = "vex.providers"; - public const string Raw = "vex.raw"; - public const string Claims = "vex.claims"; - public const string Consensus = "vex.consensus"; - public const string Exports = "vex.exports"; - public const string Cache = "vex.cache"; - public const string ConnectorState = "vex.connector_state"; -} +using System.Threading; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; + +namespace StellaOps.Excititor.Storage.Mongo; + +public static class VexMongoMappingRegistry +{ + private static int _initialized; + + public static void Register() + { + if (Interlocked.Exchange(ref _initialized, 1) == 1) + { + return; + } + + try + { + BsonSerializer.RegisterSerializer(typeof(byte[]), new ByteArraySerializer()); + } + catch + { + // serializer already registered – safe to ignore + } + + RegisterClassMaps(); + } + + private static void RegisterClassMaps() + { + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + RegisterClassMap(); + } + + private static void RegisterClassMap() + where TDocument : class + { + if (BsonClassMap.IsClassMapRegistered(typeof(TDocument))) + { + return; + } + + BsonClassMap.RegisterClassMap(classMap => + { + classMap.AutoMap(); + classMap.SetIgnoreExtraElements(true); + }); + } +} + +public static class VexMongoCollectionNames +{ + public const string Migrations = "vex.migrations"; + public const string Providers = "vex.providers"; + public const string Raw = "vex.raw"; + public const string Statements = "vex.statements"; + public const string Claims = Statements; + public const string Consensus = "vex.consensus"; + public const string Exports = "vex.exports"; + public const string Cache = "vex.cache"; + public const string ConnectorState = "vex.connector_state"; +} diff --git a/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs similarity index 63% rename from src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs rename to src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs index c9f61f48..a07c95f4 100644 --- a/src/StellaOps.Vexer.Storage.Mongo/VexMongoModels.cs +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoModels.cs @@ -1,604 +1,900 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; -using System.Linq; -using MongoDB.Bson; -using MongoDB.Bson.Serialization.Attributes; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -[BsonIgnoreExtraElements] -internal sealed class VexRawDocumentRecord -{ - [BsonId] - public string Id { get; set; } = default!; - - public string ProviderId { get; set; } = default!; - - public string Format { get; set; } = default!; - - public string SourceUri { get; set; } = default!; - - public DateTime RetrievedAt { get; set; } - = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); - - public string Digest { get; set; } = default!; - - public byte[] Content { get; set; } = Array.Empty(); - - [BsonRepresentation(BsonType.ObjectId)] - public string? GridFsObjectId { get; set; } - = null; - - public Dictionary Metadata { get; set; } = new(StringComparer.Ordinal); - - public static VexRawDocumentRecord FromDomain(VexRawDocument document, bool includeContent = true) - => new() - { - Id = document.Digest, - ProviderId = document.ProviderId, - Format = document.Format.ToString().ToLowerInvariant(), - SourceUri = document.SourceUri.ToString(), - RetrievedAt = document.RetrievedAt.UtcDateTime, - Digest = document.Digest, - Content = includeContent ? document.Content.ToArray() : Array.Empty(), - Metadata = document.Metadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal), - }; - - public VexRawDocument ToDomain() - => ToDomain(new ReadOnlyMemory(Content ?? Array.Empty())); - - public VexRawDocument ToDomain(ReadOnlyMemory content) - => new( - ProviderId, - Enum.Parse(Format, ignoreCase: true), - new Uri(SourceUri), - RetrievedAt, - Digest, - content, - (Metadata ?? new Dictionary(StringComparer.Ordinal)) - .ToImmutableDictionary(StringComparer.Ordinal)); -} - -[BsonIgnoreExtraElements] -internal sealed class VexExportManifestRecord -{ - [BsonId] - public string Id { get; set; } = default!; - - public string QuerySignature { get; set; } = default!; - - public string Format { get; set; } = default!; - - public DateTime CreatedAt { get; set; } - = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); - - public string ArtifactAlgorithm { get; set; } = default!; - - public string ArtifactDigest { get; set; } = default!; - - public int ClaimCount { get; set; } - = 0; - - public bool FromCache { get; set; } - = false; - - public List SourceProviders { get; set; } = new(); - - public string? ConsensusRevision { get; set; } - = null; - - public string? PredicateType { get; set; } - = null; - - public string? RekorApiVersion { get; set; } - = null; - - public string? RekorLocation { get; set; } - = null; - - public string? RekorLogIndex { get; set; } - = null; - - public string? RekorInclusionProofUri { get; set; } - = null; - - public string? EnvelopeDigest { get; set; } - = null; - - public DateTime? SignedAt { get; set; } - = null; - - public long SizeBytes { get; set; } - = 0; - - public static VexExportManifestRecord FromDomain(VexExportManifest manifest) - => new() - { - Id = CreateId(manifest.QuerySignature, manifest.Format), - QuerySignature = manifest.QuerySignature.Value, - Format = manifest.Format.ToString().ToLowerInvariant(), - CreatedAt = manifest.CreatedAt.UtcDateTime, - ArtifactAlgorithm = manifest.Artifact.Algorithm, - ArtifactDigest = manifest.Artifact.Digest, - ClaimCount = manifest.ClaimCount, - FromCache = manifest.FromCache, - SourceProviders = manifest.SourceProviders.ToList(), - ConsensusRevision = manifest.ConsensusRevision, - PredicateType = manifest.Attestation?.PredicateType, - RekorApiVersion = manifest.Attestation?.Rekor?.ApiVersion, - RekorLocation = manifest.Attestation?.Rekor?.Location, - RekorLogIndex = manifest.Attestation?.Rekor?.LogIndex, - RekorInclusionProofUri = manifest.Attestation?.Rekor?.InclusionProofUri?.ToString(), - EnvelopeDigest = manifest.Attestation?.EnvelopeDigest, - SignedAt = manifest.Attestation?.SignedAt?.UtcDateTime, - SizeBytes = manifest.SizeBytes, - }; - - public VexExportManifest ToDomain() - { - var signedAt = SignedAt.HasValue - ? new DateTimeOffset(DateTime.SpecifyKind(SignedAt.Value, DateTimeKind.Utc)) - : (DateTimeOffset?)null; - - var attestation = PredicateType is null - ? null - : new VexAttestationMetadata( - PredicateType, - RekorApiVersion is null || RekorLocation is null - ? null - : new VexRekorReference( - RekorApiVersion, - RekorLocation, - RekorLogIndex, - RekorInclusionProofUri is null ? null : new Uri(RekorInclusionProofUri)), - EnvelopeDigest, - signedAt); - - return new VexExportManifest( - Id, - new VexQuerySignature(QuerySignature), - Enum.Parse(Format, ignoreCase: true), - CreatedAt, - new VexContentAddress(ArtifactAlgorithm, ArtifactDigest), - ClaimCount, - SourceProviders, - FromCache, - ConsensusRevision, - attestation, - SizeBytes); - } - - public static string CreateId(VexQuerySignature signature, VexExportFormat format) - => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); -} - -[BsonIgnoreExtraElements] -internal sealed class VexProviderRecord -{ - [BsonId] - public string Id { get; set; } = default!; - - public string DisplayName { get; set; } = default!; - - public string Kind { get; set; } = default!; - - public List BaseUris { get; set; } = new(); - - public VexProviderDiscoveryDocument? Discovery { get; set; } - = null; - - public VexProviderTrustDocument? Trust { get; set; } - = null; - - public bool Enabled { get; set; } - = true; - - public static VexProviderRecord FromDomain(VexProvider provider) - => new() - { - Id = provider.Id, - DisplayName = provider.DisplayName, - Kind = provider.Kind.ToString().ToLowerInvariant(), - BaseUris = provider.BaseUris.Select(uri => uri.ToString()).ToList(), - Discovery = VexProviderDiscoveryDocument.FromDomain(provider.Discovery), - Trust = VexProviderTrustDocument.FromDomain(provider.Trust), - Enabled = provider.Enabled, - }; - - public VexProvider ToDomain() - { - var uris = BaseUris?.Select(uri => new Uri(uri)) ?? Enumerable.Empty(); - return new VexProvider( - Id, - DisplayName, - Enum.Parse(Kind, ignoreCase: true), - uris, - Discovery?.ToDomain(), - Trust?.ToDomain(), - Enabled); - } -} - -[BsonIgnoreExtraElements] -internal sealed class VexProviderDiscoveryDocument -{ - public string? WellKnownMetadata { get; set; } - = null; - - public string? RolIeService { get; set; } - = null; - - public static VexProviderDiscoveryDocument? FromDomain(VexProviderDiscovery? discovery) - => discovery is null - ? null - : new VexProviderDiscoveryDocument - { - WellKnownMetadata = discovery.WellKnownMetadata?.ToString(), - RolIeService = discovery.RolIeService?.ToString(), - }; - - public VexProviderDiscovery ToDomain() - => new( - WellKnownMetadata is null ? null : new Uri(WellKnownMetadata), - RolIeService is null ? null : new Uri(RolIeService)); -} - -[BsonIgnoreExtraElements] -internal sealed class VexProviderTrustDocument -{ - public double Weight { get; set; } - = 1.0; - - public VexCosignTrustDocument? Cosign { get; set; } - = null; - - public List PgpFingerprints { get; set; } = new(); - - public static VexProviderTrustDocument? FromDomain(VexProviderTrust? trust) - => trust is null - ? null - : new VexProviderTrustDocument - { - Weight = trust.Weight, - Cosign = trust.Cosign is null ? null : VexCosignTrustDocument.FromDomain(trust.Cosign), - PgpFingerprints = trust.PgpFingerprints.ToList(), - }; - - public VexProviderTrust ToDomain() - => new( - Weight, - Cosign?.ToDomain(), - PgpFingerprints); -} - -[BsonIgnoreExtraElements] -internal sealed class VexCosignTrustDocument -{ - public string Issuer { get; set; } = default!; - - public string IdentityPattern { get; set; } = default!; - - public static VexCosignTrustDocument FromDomain(VexCosignTrust trust) - => new() - { - Issuer = trust.Issuer, - IdentityPattern = trust.IdentityPattern, - }; - - public VexCosignTrust ToDomain() - => new(Issuer, IdentityPattern); -} - -[BsonIgnoreExtraElements] -internal sealed class VexConsensusRecord -{ - [BsonId] - public string Id { get; set; } = default!; - - public string VulnerabilityId { get; set; } = default!; - - public VexProductDocument Product { get; set; } = default!; - - public string Status { get; set; } = default!; - - public DateTime CalculatedAt { get; set; } - = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); - - public List Sources { get; set; } = new(); - - public List Conflicts { get; set; } = new(); - - public string? PolicyVersion { get; set; } - = null; - - public string? PolicyRevisionId { get; set; } - = null; - - public string? PolicyDigest { get; set; } - = null; - - public string? Summary { get; set; } - = null; - - public static string CreateId(string vulnerabilityId, string productKey) - => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim()); - - public static VexConsensusRecord FromDomain(VexConsensus consensus) - => new() - { - Id = CreateId(consensus.VulnerabilityId, consensus.Product.Key), - VulnerabilityId = consensus.VulnerabilityId, - Product = VexProductDocument.FromDomain(consensus.Product), - Status = consensus.Status.ToString().ToLowerInvariant(), - CalculatedAt = consensus.CalculatedAt.UtcDateTime, - Sources = consensus.Sources.Select(VexConsensusSourceDocument.FromDomain).ToList(), - Conflicts = consensus.Conflicts.Select(VexConsensusConflictDocument.FromDomain).ToList(), - PolicyVersion = consensus.PolicyVersion, - PolicyRevisionId = consensus.PolicyRevisionId, - PolicyDigest = consensus.PolicyDigest, - Summary = consensus.Summary, - }; - - public VexConsensus ToDomain() - => new( - VulnerabilityId, - Product.ToDomain(), - Enum.Parse(Status, ignoreCase: true), - new DateTimeOffset(CalculatedAt, TimeSpan.Zero), - Sources.Select(static source => source.ToDomain()), - Conflicts.Select(static conflict => conflict.ToDomain()), - PolicyVersion, - Summary, - PolicyRevisionId, - PolicyDigest); -} - -[BsonIgnoreExtraElements] -internal sealed class VexProductDocument -{ - public string Key { get; set; } = default!; - - public string? Name { get; set; } - = null; - - public string? Version { get; set; } - = null; - - public string? Purl { get; set; } - = null; - - public string? Cpe { get; set; } - = null; - - public List ComponentIdentifiers { get; set; } = new(); - - public static VexProductDocument FromDomain(VexProduct product) - => new() - { - Key = product.Key, - Name = product.Name, - Version = product.Version, - Purl = product.Purl, - Cpe = product.Cpe, - ComponentIdentifiers = product.ComponentIdentifiers.ToList(), - }; - - public VexProduct ToDomain() - => new( - Key, - Name, - Version, - Purl, - Cpe, - ComponentIdentifiers); -} - -[BsonIgnoreExtraElements] -internal sealed class VexConsensusSourceDocument -{ - public string ProviderId { get; set; } = default!; - - public string Status { get; set; } = default!; - - public string DocumentDigest { get; set; } = default!; - - public double Weight { get; set; } - = 0; - - public string? Justification { get; set; } - = null; - - public string? Detail { get; set; } - = null; - - public VexConfidenceDocument? Confidence { get; set; } - = null; - - public static VexConsensusSourceDocument FromDomain(VexConsensusSource source) - => new() - { - ProviderId = source.ProviderId, - Status = source.Status.ToString().ToLowerInvariant(), - DocumentDigest = source.DocumentDigest, - Weight = source.Weight, - Justification = source.Justification?.ToString().ToLowerInvariant(), - Detail = source.Detail, - Confidence = source.Confidence is null ? null : VexConfidenceDocument.FromDomain(source.Confidence), - }; - - public VexConsensusSource ToDomain() - => new( - ProviderId, - Enum.Parse(Status, ignoreCase: true), - DocumentDigest, - Weight, - string.IsNullOrWhiteSpace(Justification) ? null : Enum.Parse(Justification, ignoreCase: true), - Detail, - Confidence?.ToDomain()); -} - -[BsonIgnoreExtraElements] -internal sealed class VexConsensusConflictDocument -{ - public string ProviderId { get; set; } = default!; - - public string Status { get; set; } = default!; - - public string DocumentDigest { get; set; } = default!; - - public string? Justification { get; set; } - = null; - - public string? Detail { get; set; } - = null; - - public string? Reason { get; set; } - = null; - - public static VexConsensusConflictDocument FromDomain(VexConsensusConflict conflict) - => new() - { - ProviderId = conflict.ProviderId, - Status = conflict.Status.ToString().ToLowerInvariant(), - DocumentDigest = conflict.DocumentDigest, - Justification = conflict.Justification?.ToString().ToLowerInvariant(), - Detail = conflict.Detail, - Reason = conflict.Reason, - }; - - public VexConsensusConflict ToDomain() - => new( - ProviderId, - Enum.Parse(Status, ignoreCase: true), - DocumentDigest, - string.IsNullOrWhiteSpace(Justification) ? null : Enum.Parse(Justification, ignoreCase: true), - Detail, - Reason); -} - -[BsonIgnoreExtraElements] -internal sealed class VexConfidenceDocument -{ - public string Level { get; set; } = default!; - - public double? Score { get; set; } - = null; - - public string? Method { get; set; } - = null; - - public static VexConfidenceDocument FromDomain(VexConfidence confidence) - => new() - { - Level = confidence.Level, - Score = confidence.Score, - Method = confidence.Method, - }; - - public VexConfidence ToDomain() - => new(Level, Score, Method); -} - -[BsonIgnoreExtraElements] -internal sealed class VexCacheEntryRecord -{ - [BsonId] - public string Id { get; set; } = default!; - - public string QuerySignature { get; set; } = default!; - - public string Format { get; set; } = default!; - - public string ArtifactAlgorithm { get; set; } = default!; - - public string ArtifactDigest { get; set; } = default!; - - public DateTime CreatedAt { get; set; } - = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); - - public long SizeBytes { get; set; } - = 0; - - public string? ManifestId { get; set; } - = null; - - [BsonRepresentation(BsonType.ObjectId)] - public string? GridFsObjectId { get; set; } - = null; - - public DateTime? ExpiresAt { get; set; } - = null; - - public static string CreateId(VexQuerySignature signature, VexExportFormat format) - => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); - - public static VexCacheEntryRecord FromDomain(VexCacheEntry entry) - => new() - { - Id = CreateId(entry.QuerySignature, entry.Format), - QuerySignature = entry.QuerySignature.Value, - Format = entry.Format.ToString().ToLowerInvariant(), - ArtifactAlgorithm = entry.Artifact.Algorithm, - ArtifactDigest = entry.Artifact.Digest, - CreatedAt = entry.CreatedAt.UtcDateTime, - SizeBytes = entry.SizeBytes, - ManifestId = entry.ManifestId, - GridFsObjectId = entry.GridFsObjectId, - ExpiresAt = entry.ExpiresAt?.UtcDateTime, - }; - - public VexCacheEntry ToDomain() - { - var signature = new VexQuerySignature(QuerySignature); - var artifact = new VexContentAddress(ArtifactAlgorithm, ArtifactDigest); - var createdAt = new DateTimeOffset(DateTime.SpecifyKind(CreatedAt, DateTimeKind.Utc)); - var expires = ExpiresAt.HasValue - ? new DateTimeOffset(DateTime.SpecifyKind(ExpiresAt.Value, DateTimeKind.Utc)) - : (DateTimeOffset?)null; - - return new VexCacheEntry( - signature, - Enum.Parse(Format, ignoreCase: true), - artifact, - createdAt, - SizeBytes, - ManifestId, - GridFsObjectId, - expires); - } -} - -[BsonIgnoreExtraElements] -internal sealed class VexConnectorStateDocument -{ - [BsonId] - public string ConnectorId { get; set; } = default!; - - public DateTime? LastUpdated { get; set; } - = null; - - public List DocumentDigests { get; set; } = new(); - - public static VexConnectorStateDocument FromRecord(VexConnectorState state) - => new() - { - ConnectorId = state.ConnectorId, - LastUpdated = state.LastUpdated?.UtcDateTime, - DocumentDigests = state.DocumentDigests.ToList(), - }; - - public VexConnectorState ToRecord() - { - var lastUpdated = LastUpdated.HasValue - ? new DateTimeOffset(DateTime.SpecifyKind(LastUpdated.Value, DateTimeKind.Utc)) - : (DateTimeOffset?)null; - - return new VexConnectorState( - ConnectorId, - lastUpdated, - DocumentDigests.ToImmutableArray()); - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +[BsonIgnoreExtraElements] +internal sealed class VexRawDocumentRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string ProviderId { get; set; } = default!; + + public string Format { get; set; } = default!; + + public string SourceUri { get; set; } = default!; + + public DateTime RetrievedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public string Digest { get; set; } = default!; + + public byte[] Content { get; set; } = Array.Empty(); + + [BsonRepresentation(BsonType.ObjectId)] + public string? GridFsObjectId { get; set; } + = null; + + public Dictionary Metadata { get; set; } = new(StringComparer.Ordinal); + + public static VexRawDocumentRecord FromDomain(VexRawDocument document, bool includeContent = true) + => new() + { + Id = document.Digest, + ProviderId = document.ProviderId, + Format = document.Format.ToString().ToLowerInvariant(), + SourceUri = document.SourceUri.ToString(), + RetrievedAt = document.RetrievedAt.UtcDateTime, + Digest = document.Digest, + Content = includeContent ? document.Content.ToArray() : Array.Empty(), + Metadata = document.Metadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal), + }; + + public VexRawDocument ToDomain() + => ToDomain(new ReadOnlyMemory(Content ?? Array.Empty())); + + public VexRawDocument ToDomain(ReadOnlyMemory content) + => new( + ProviderId, + Enum.Parse(Format, ignoreCase: true), + new Uri(SourceUri), + RetrievedAt, + Digest, + content, + (Metadata ?? new Dictionary(StringComparer.Ordinal)) + .ToImmutableDictionary(StringComparer.Ordinal)); +} + +[BsonIgnoreExtraElements] +internal sealed class VexExportManifestRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string QuerySignature { get; set; } = default!; + + public string Format { get; set; } = default!; + + public DateTime CreatedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public string ArtifactAlgorithm { get; set; } = default!; + + public string ArtifactDigest { get; set; } = default!; + + public int ClaimCount { get; set; } + = 0; + + public bool FromCache { get; set; } + = false; + + public List SourceProviders { get; set; } = new(); + + public string? ConsensusRevision { get; set; } + = null; + + public string? PolicyRevisionId { get; set; } + = null; + + public string? PolicyDigest { get; set; } + = null; + + public string? ConsensusDigestAlgorithm { get; set; } + = null; + + public string? ConsensusDigestValue { get; set; } + = null; + + public string? ScoreDigestAlgorithm { get; set; } + = null; + + public string? ScoreDigestValue { get; set; } + = null; + + public string? PredicateType { get; set; } + = null; + + public string? RekorApiVersion { get; set; } + = null; + + public string? RekorLocation { get; set; } + = null; + + public string? RekorLogIndex { get; set; } + = null; + + public string? RekorInclusionProofUri { get; set; } + = null; + + public string? EnvelopeDigest { get; set; } + = null; + + public DateTime? SignedAt { get; set; } + = null; + + public long SizeBytes { get; set; } + = 0; + + public static VexExportManifestRecord FromDomain(VexExportManifest manifest) + => new() + { + Id = CreateId(manifest.QuerySignature, manifest.Format), + QuerySignature = manifest.QuerySignature.Value, + Format = manifest.Format.ToString().ToLowerInvariant(), + CreatedAt = manifest.CreatedAt.UtcDateTime, + ArtifactAlgorithm = manifest.Artifact.Algorithm, + ArtifactDigest = manifest.Artifact.Digest, + ClaimCount = manifest.ClaimCount, + FromCache = manifest.FromCache, + SourceProviders = manifest.SourceProviders.ToList(), + ConsensusRevision = manifest.ConsensusRevision, + PolicyRevisionId = manifest.PolicyRevisionId, + PolicyDigest = manifest.PolicyDigest, + ConsensusDigestAlgorithm = manifest.ConsensusDigest?.Algorithm, + ConsensusDigestValue = manifest.ConsensusDigest?.Digest, + ScoreDigestAlgorithm = manifest.ScoreDigest?.Algorithm, + ScoreDigestValue = manifest.ScoreDigest?.Digest, + PredicateType = manifest.Attestation?.PredicateType, + RekorApiVersion = manifest.Attestation?.Rekor?.ApiVersion, + RekorLocation = manifest.Attestation?.Rekor?.Location, + RekorLogIndex = manifest.Attestation?.Rekor?.LogIndex, + RekorInclusionProofUri = manifest.Attestation?.Rekor?.InclusionProofUri?.ToString(), + EnvelopeDigest = manifest.Attestation?.EnvelopeDigest, + SignedAt = manifest.Attestation?.SignedAt?.UtcDateTime, + SizeBytes = manifest.SizeBytes, + }; + + public VexExportManifest ToDomain() + { + var signedAt = SignedAt.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(SignedAt.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + var attestation = PredicateType is null + ? null + : new VexAttestationMetadata( + PredicateType, + RekorApiVersion is null || RekorLocation is null + ? null + : new VexRekorReference( + RekorApiVersion, + RekorLocation, + RekorLogIndex, + RekorInclusionProofUri is null ? null : new Uri(RekorInclusionProofUri)), + EnvelopeDigest, + signedAt); + + var consensusDigest = ConsensusDigestAlgorithm is null || ConsensusDigestValue is null + ? null + : new VexContentAddress(ConsensusDigestAlgorithm, ConsensusDigestValue); + + var scoreDigest = ScoreDigestAlgorithm is null || ScoreDigestValue is null + ? null + : new VexContentAddress(ScoreDigestAlgorithm, ScoreDigestValue); + + return new VexExportManifest( + Id, + new VexQuerySignature(QuerySignature), + Enum.Parse(Format, ignoreCase: true), + new DateTimeOffset(DateTime.SpecifyKind(CreatedAt, DateTimeKind.Utc)), + new VexContentAddress(ArtifactAlgorithm, ArtifactDigest), + ClaimCount, + SourceProviders, + FromCache, + ConsensusRevision, + PolicyRevisionId, + PolicyDigest, + consensusDigest, + scoreDigest, + attestation, + SizeBytes); + } + + public static string CreateId(VexQuerySignature signature, VexExportFormat format) + => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); +} + +[BsonIgnoreExtraElements] +internal sealed class VexProviderRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string DisplayName { get; set; } = default!; + + public string Kind { get; set; } = default!; + + public List BaseUris { get; set; } = new(); + + public VexProviderDiscoveryDocument? Discovery { get; set; } + = null; + + public VexProviderTrustDocument? Trust { get; set; } + = null; + + public bool Enabled { get; set; } + = true; + + public static VexProviderRecord FromDomain(VexProvider provider) + => new() + { + Id = provider.Id, + DisplayName = provider.DisplayName, + Kind = provider.Kind.ToString().ToLowerInvariant(), + BaseUris = provider.BaseUris.Select(uri => uri.ToString()).ToList(), + Discovery = VexProviderDiscoveryDocument.FromDomain(provider.Discovery), + Trust = VexProviderTrustDocument.FromDomain(provider.Trust), + Enabled = provider.Enabled, + }; + + public VexProvider ToDomain() + { + var uris = BaseUris?.Select(uri => new Uri(uri)) ?? Enumerable.Empty(); + return new VexProvider( + Id, + DisplayName, + Enum.Parse(Kind, ignoreCase: true), + uris, + Discovery?.ToDomain(), + Trust?.ToDomain(), + Enabled); + } +} + +[BsonIgnoreExtraElements] +internal sealed class VexProviderDiscoveryDocument +{ + public string? WellKnownMetadata { get; set; } + = null; + + public string? RolIeService { get; set; } + = null; + + public static VexProviderDiscoveryDocument? FromDomain(VexProviderDiscovery? discovery) + => discovery is null + ? null + : new VexProviderDiscoveryDocument + { + WellKnownMetadata = discovery.WellKnownMetadata?.ToString(), + RolIeService = discovery.RolIeService?.ToString(), + }; + + public VexProviderDiscovery ToDomain() + => new( + WellKnownMetadata is null ? null : new Uri(WellKnownMetadata), + RolIeService is null ? null : new Uri(RolIeService)); +} + +[BsonIgnoreExtraElements] +internal sealed class VexProviderTrustDocument +{ + public double Weight { get; set; } + = 1.0; + + public VexCosignTrustDocument? Cosign { get; set; } + = null; + + public List PgpFingerprints { get; set; } = new(); + + public static VexProviderTrustDocument? FromDomain(VexProviderTrust? trust) + => trust is null + ? null + : new VexProviderTrustDocument + { + Weight = trust.Weight, + Cosign = trust.Cosign is null ? null : VexCosignTrustDocument.FromDomain(trust.Cosign), + PgpFingerprints = trust.PgpFingerprints.ToList(), + }; + + public VexProviderTrust ToDomain() + => new( + Weight, + Cosign?.ToDomain(), + PgpFingerprints); +} + +[BsonIgnoreExtraElements] +internal sealed class VexCosignTrustDocument +{ + public string Issuer { get; set; } = default!; + + public string IdentityPattern { get; set; } = default!; + + public static VexCosignTrustDocument FromDomain(VexCosignTrust trust) + => new() + { + Issuer = trust.Issuer, + IdentityPattern = trust.IdentityPattern, + }; + + public VexCosignTrust ToDomain() + => new(Issuer, IdentityPattern); +} + +[BsonIgnoreExtraElements] +internal sealed class VexConsensusRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string VulnerabilityId { get; set; } = default!; + + public VexProductDocument Product { get; set; } = default!; + + public string Status { get; set; } = default!; + + public DateTime CalculatedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public List Sources { get; set; } = new(); + + public List Conflicts { get; set; } = new(); + + public VexSignalDocument? Signals { get; set; } + = null; + + public string? PolicyVersion { get; set; } + = null; + + public string? PolicyRevisionId { get; set; } + = null; + + public string? PolicyDigest { get; set; } + = null; + + public string? Summary { get; set; } + = null; + + public static string CreateId(string vulnerabilityId, string productKey) + => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", vulnerabilityId.Trim(), productKey.Trim()); + + public static VexConsensusRecord FromDomain(VexConsensus consensus) + => new() + { + Id = CreateId(consensus.VulnerabilityId, consensus.Product.Key), + VulnerabilityId = consensus.VulnerabilityId, + Product = VexProductDocument.FromDomain(consensus.Product), + Status = consensus.Status.ToString().ToLowerInvariant(), + CalculatedAt = consensus.CalculatedAt.UtcDateTime, + Sources = consensus.Sources.Select(VexConsensusSourceDocument.FromDomain).ToList(), + Conflicts = consensus.Conflicts.Select(VexConsensusConflictDocument.FromDomain).ToList(), + Signals = VexSignalDocument.FromDomain(consensus.Signals), + PolicyVersion = consensus.PolicyVersion, + PolicyRevisionId = consensus.PolicyRevisionId, + PolicyDigest = consensus.PolicyDigest, + Summary = consensus.Summary, + }; + + public VexConsensus ToDomain() + => new( + VulnerabilityId, + Product.ToDomain(), + Enum.Parse(Status, ignoreCase: true), + new DateTimeOffset(CalculatedAt, TimeSpan.Zero), + Sources.Select(static source => source.ToDomain()), + Conflicts.Select(static conflict => conflict.ToDomain()), + Signals?.ToDomain(), + PolicyVersion, + Summary, + PolicyRevisionId, + PolicyDigest); +} + +[BsonIgnoreExtraElements] +internal sealed class VexProductDocument +{ + public string Key { get; set; } = default!; + + public string? Name { get; set; } + = null; + + public string? Version { get; set; } + = null; + + public string? Purl { get; set; } + = null; + + public string? Cpe { get; set; } + = null; + + public List ComponentIdentifiers { get; set; } = new(); + + public static VexProductDocument FromDomain(VexProduct product) + => new() + { + Key = product.Key, + Name = product.Name, + Version = product.Version, + Purl = product.Purl, + Cpe = product.Cpe, + ComponentIdentifiers = product.ComponentIdentifiers.ToList(), + }; + + public VexProduct ToDomain() + => new( + Key, + Name, + Version, + Purl, + Cpe, + ComponentIdentifiers); +} + +[BsonIgnoreExtraElements] +internal sealed class VexConsensusSourceDocument +{ + public string ProviderId { get; set; } = default!; + + public string Status { get; set; } = default!; + + public string DocumentDigest { get; set; } = default!; + + public double Weight { get; set; } + = 0; + + public string? Justification { get; set; } + = null; + + public string? Detail { get; set; } + = null; + + public VexConfidenceDocument? Confidence { get; set; } + = null; + + public static VexConsensusSourceDocument FromDomain(VexConsensusSource source) + => new() + { + ProviderId = source.ProviderId, + Status = source.Status.ToString().ToLowerInvariant(), + DocumentDigest = source.DocumentDigest, + Weight = source.Weight, + Justification = source.Justification?.ToString().ToLowerInvariant(), + Detail = source.Detail, + Confidence = source.Confidence is null ? null : VexConfidenceDocument.FromDomain(source.Confidence), + }; + + public VexConsensusSource ToDomain() + => new( + ProviderId, + Enum.Parse(Status, ignoreCase: true), + DocumentDigest, + Weight, + string.IsNullOrWhiteSpace(Justification) ? null : Enum.Parse(Justification, ignoreCase: true), + Detail, + Confidence?.ToDomain()); +} + +[BsonIgnoreExtraElements] +internal sealed class VexConsensusConflictDocument +{ + public string ProviderId { get; set; } = default!; + + public string Status { get; set; } = default!; + + public string DocumentDigest { get; set; } = default!; + + public string? Justification { get; set; } + = null; + + public string? Detail { get; set; } + = null; + + public string? Reason { get; set; } + = null; + + public static VexConsensusConflictDocument FromDomain(VexConsensusConflict conflict) + => new() + { + ProviderId = conflict.ProviderId, + Status = conflict.Status.ToString().ToLowerInvariant(), + DocumentDigest = conflict.DocumentDigest, + Justification = conflict.Justification?.ToString().ToLowerInvariant(), + Detail = conflict.Detail, + Reason = conflict.Reason, + }; + + public VexConsensusConflict ToDomain() + => new( + ProviderId, + Enum.Parse(Status, ignoreCase: true), + DocumentDigest, + string.IsNullOrWhiteSpace(Justification) ? null : Enum.Parse(Justification, ignoreCase: true), + Detail, + Reason); +} + +[BsonIgnoreExtraElements] +internal sealed class VexConfidenceDocument +{ + public string Level { get; set; } = default!; + + public double? Score { get; set; } + = null; + + public string? Method { get; set; } + = null; + + public static VexConfidenceDocument FromDomain(VexConfidence confidence) + => new() + { + Level = confidence.Level, + Score = confidence.Score, + Method = confidence.Method, + }; + + public VexConfidence ToDomain() + => new(Level, Score, Method); +} + +[BsonIgnoreExtraElements] +internal sealed class VexSeveritySignalDocument +{ + public string Scheme { get; set; } = default!; + + public double? Score { get; set; } + = null; + + public string? Label { get; set; } + = null; + + public string? Vector { get; set; } + = null; + + public static VexSeveritySignalDocument FromDomain(VexSeveritySignal severity) + => new() + { + Scheme = severity.Scheme, + Score = severity.Score, + Label = severity.Label, + Vector = severity.Vector, + }; + + public VexSeveritySignal ToDomain() + => new(Scheme, Score, Label, Vector); +} + +[BsonIgnoreExtraElements] +internal sealed class VexSignalDocument +{ + public VexSeveritySignalDocument? Severity { get; set; } + = null; + + public bool? Kev { get; set; } + = null; + + public double? Epss { get; set; } + = null; + + public static VexSignalDocument? FromDomain(VexSignalSnapshot? snapshot) + => snapshot is null + ? null + : new VexSignalDocument + { + Severity = snapshot.Severity is null ? null : VexSeveritySignalDocument.FromDomain(snapshot.Severity), + Kev = snapshot.Kev, + Epss = snapshot.Epss, + }; + + public VexSignalSnapshot ToDomain() + => new( + Severity?.ToDomain(), + Kev, + Epss); +} + +[BsonIgnoreExtraElements] +internal sealed class VexSignatureMetadataDocument +{ + public string Type { get; set; } = default!; + + public string? Subject { get; set; } + = null; + + public string? Issuer { get; set; } + = null; + + public string? KeyId { get; set; } + = null; + + public DateTime? VerifiedAt { get; set; } + = null; + + public string? TransparencyLogReference { get; set; } + = null; + + public static VexSignatureMetadataDocument? FromDomain(VexSignatureMetadata? signature) + => signature is null + ? null + : new VexSignatureMetadataDocument + { + Type = signature.Type, + Subject = signature.Subject, + Issuer = signature.Issuer, + KeyId = signature.KeyId, + VerifiedAt = signature.VerifiedAt?.UtcDateTime, + TransparencyLogReference = signature.TransparencyLogReference, + }; + + public VexSignatureMetadata ToDomain() + { + var verifiedAt = VerifiedAt.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(VerifiedAt.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + return new VexSignatureMetadata( + Type, + Subject, + Issuer, + KeyId, + verifiedAt, + TransparencyLogReference); + } +} + +[BsonIgnoreExtraElements] +internal sealed class VexClaimDocumentRecord +{ + public string Format { get; set; } = default!; + + public string Digest { get; set; } = default!; + + public string SourceUri { get; set; } = default!; + + public string? Revision { get; set; } + = null; + + public VexSignatureMetadataDocument? Signature { get; set; } + = null; + + public static VexClaimDocumentRecord FromDomain(VexClaimDocument document) + => new() + { + Format = document.Format.ToString().ToLowerInvariant(), + Digest = document.Digest, + SourceUri = document.SourceUri.ToString(), + Revision = document.Revision, + Signature = VexSignatureMetadataDocument.FromDomain(document.Signature), + }; + + public VexClaimDocument ToDomain() + => new( + Enum.Parse(Format, ignoreCase: true), + Digest, + new Uri(SourceUri), + Revision, + Signature?.ToDomain()); +} + +[BsonIgnoreExtraElements] +internal sealed class VexStatementRecord +{ + [BsonId] + public ObjectId Id { get; set; } + = ObjectId.GenerateNewId(); + + public string VulnerabilityId { get; set; } = default!; + + public string ProviderId { get; set; } = default!; + + public VexProductDocument Product { get; set; } = default!; + + public string Status { get; set; } = default!; + + public string? Justification { get; set; } + = null; + + public string? Detail { get; set; } + = null; + + public VexClaimDocumentRecord Document { get; set; } = default!; + + public DateTime FirstSeen { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public DateTime LastSeen { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public DateTime InsertedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public VexConfidenceDocument? Confidence { get; set; } + = null; + + public VexSignalDocument? Signals { get; set; } + = null; + + public Dictionary AdditionalMetadata { get; set; } = new(StringComparer.Ordinal); + + public static VexStatementRecord FromDomain(VexClaim claim, DateTimeOffset observedAt) + => new() + { + VulnerabilityId = claim.VulnerabilityId, + ProviderId = claim.ProviderId, + Product = VexProductDocument.FromDomain(claim.Product), + Status = claim.Status.ToString().ToLowerInvariant(), + Justification = claim.Justification?.ToString().ToLowerInvariant(), + Detail = claim.Detail, + Document = VexClaimDocumentRecord.FromDomain(claim.Document), + FirstSeen = claim.FirstSeen.UtcDateTime, + LastSeen = claim.LastSeen.UtcDateTime, + InsertedAt = observedAt.UtcDateTime, + Confidence = claim.Confidence is null ? null : VexConfidenceDocument.FromDomain(claim.Confidence), + Signals = VexSignalDocument.FromDomain(claim.Signals), + AdditionalMetadata = claim.AdditionalMetadata.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal), + }; + + public VexClaim ToDomain() + { + var firstSeen = new DateTimeOffset(DateTime.SpecifyKind(FirstSeen, DateTimeKind.Utc)); + var lastSeen = new DateTimeOffset(DateTime.SpecifyKind(LastSeen, DateTimeKind.Utc)); + var justification = string.IsNullOrWhiteSpace(Justification) + ? (VexJustification?)null + : Enum.Parse(Justification, ignoreCase: true); + var metadata = (AdditionalMetadata ?? new Dictionary(StringComparer.Ordinal)) + .ToImmutableDictionary(StringComparer.Ordinal); + + return new VexClaim( + VulnerabilityId, + ProviderId, + Product.ToDomain(), + Enum.Parse(Status, ignoreCase: true), + Document.ToDomain(), + firstSeen, + lastSeen, + justification, + Detail, + Confidence?.ToDomain(), + Signals?.ToDomain(), + metadata); + } +} + +[BsonIgnoreExtraElements] +internal sealed class VexCacheEntryRecord +{ + [BsonId] + public string Id { get; set; } = default!; + + public string QuerySignature { get; set; } = default!; + + public string Format { get; set; } = default!; + + public string ArtifactAlgorithm { get; set; } = default!; + + public string ArtifactDigest { get; set; } = default!; + + public DateTime CreatedAt { get; set; } + = DateTime.SpecifyKind(DateTime.UtcNow, DateTimeKind.Utc); + + public long SizeBytes { get; set; } + = 0; + + public string? ManifestId { get; set; } + = null; + + [BsonRepresentation(BsonType.ObjectId)] + public string? GridFsObjectId { get; set; } + = null; + + public DateTime? ExpiresAt { get; set; } + = null; + + public static string CreateId(VexQuerySignature signature, VexExportFormat format) + => string.Format(CultureInfo.InvariantCulture, "{0}|{1}", signature.Value, format.ToString().ToLowerInvariant()); + + public static VexCacheEntryRecord FromDomain(VexCacheEntry entry) + => new() + { + Id = CreateId(entry.QuerySignature, entry.Format), + QuerySignature = entry.QuerySignature.Value, + Format = entry.Format.ToString().ToLowerInvariant(), + ArtifactAlgorithm = entry.Artifact.Algorithm, + ArtifactDigest = entry.Artifact.Digest, + CreatedAt = entry.CreatedAt.UtcDateTime, + SizeBytes = entry.SizeBytes, + ManifestId = entry.ManifestId, + GridFsObjectId = entry.GridFsObjectId, + ExpiresAt = entry.ExpiresAt?.UtcDateTime, + }; + + public VexCacheEntry ToDomain() + { + var signature = new VexQuerySignature(QuerySignature); + var artifact = new VexContentAddress(ArtifactAlgorithm, ArtifactDigest); + var createdAt = new DateTimeOffset(DateTime.SpecifyKind(CreatedAt, DateTimeKind.Utc)); + var expires = ExpiresAt.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(ExpiresAt.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + return new VexCacheEntry( + signature, + Enum.Parse(Format, ignoreCase: true), + artifact, + createdAt, + SizeBytes, + ManifestId, + GridFsObjectId, + expires); + } +} + +[BsonIgnoreExtraElements] +internal sealed class VexConnectorStateDocument +{ + [BsonId] + public string ConnectorId { get; set; } = default!; + + public DateTime? LastUpdated { get; set; } + = null; + + public List DocumentDigests { get; set; } = new(); + + public Dictionary ResumeTokens { get; set; } = new(StringComparer.Ordinal); + + public DateTime? LastSuccessAt { get; set; } + = null; + + public int FailureCount { get; set; } + = 0; + + public DateTime? NextEligibleRun { get; set; } + = null; + + public string? LastFailureReason { get; set; } + = null; + + public static VexConnectorStateDocument FromRecord(VexConnectorState state) + => new() + { + ConnectorId = state.ConnectorId, + LastUpdated = state.LastUpdated?.UtcDateTime, + DocumentDigests = state.DocumentDigests.IsDefault ? new List() : state.DocumentDigests.ToList(), + ResumeTokens = state.ResumeTokens.Count == 0 + ? new Dictionary(StringComparer.Ordinal) + : state.ResumeTokens.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal), + LastSuccessAt = state.LastSuccessAt?.UtcDateTime, + FailureCount = state.FailureCount, + NextEligibleRun = state.NextEligibleRun?.UtcDateTime, + LastFailureReason = state.LastFailureReason, + }; + + public VexConnectorState ToRecord() + { + var lastUpdated = LastUpdated.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(LastUpdated.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + var lastSuccessAt = LastSuccessAt.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(LastSuccessAt.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + var nextEligibleRun = NextEligibleRun.HasValue + ? new DateTimeOffset(DateTime.SpecifyKind(NextEligibleRun.Value, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + return new VexConnectorState( + ConnectorId, + lastUpdated, + DocumentDigests?.ToImmutableArray() ?? ImmutableArray.Empty, + (ResumeTokens ?? new Dictionary(StringComparer.Ordinal)).ToImmutableDictionary(StringComparer.Ordinal), + lastSuccessAt, + FailureCount, + nextEligibleRun, + string.IsNullOrWhiteSpace(LastFailureReason) ? null : LastFailureReason); + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/VexMongoSessionProvider.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoSessionProvider.cs new file mode 100644 index 00000000..b24b19b5 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoSessionProvider.cs @@ -0,0 +1,120 @@ +using Microsoft.Extensions.Options; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo; + +public interface IVexMongoSessionProvider : IAsyncDisposable +{ + ValueTask StartSessionAsync(CancellationToken cancellationToken = default); +} + +internal sealed class VexMongoSessionProvider : IVexMongoSessionProvider +{ + private readonly IMongoClient _client; + private readonly VexMongoStorageOptions _options; + private readonly object _gate = new(); + private Task? _sessionTask; + private IClientSessionHandle? _session; + private bool _disposed; + + public VexMongoSessionProvider(IMongoClient client, IOptions options) + { + _client = client ?? throw new ArgumentNullException(nameof(client)); + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + _options = options.Value; + } + + public async ValueTask StartSessionAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + + var existing = Volatile.Read(ref _session); + if (existing is not null) + { + return existing; + } + + Task startTask; + + lock (_gate) + { + if (_session is { } current) + { + return current; + } + + _sessionTask ??= StartSessionInternalAsync(cancellationToken); + startTask = _sessionTask; + } + + try + { + var handle = await startTask.WaitAsync(cancellationToken).ConfigureAwait(false); + if (_session is null) + { + lock (_gate) + { + if (_session is null) + { + _session = handle; + _sessionTask = Task.FromResult(handle); + } + } + } + + return handle; + } + catch + { + lock (_gate) + { + if (ReferenceEquals(_sessionTask, startTask)) + { + _sessionTask = null; + } + } + + throw; + } + } + + private Task StartSessionInternalAsync(CancellationToken cancellationToken) + { + var sessionOptions = new ClientSessionOptions + { + CausalConsistency = true, + DefaultTransactionOptions = new TransactionOptions( + readPreference: ReadPreference.Primary, + readConcern: ReadConcern.Majority, + writeConcern: WriteConcern.WMajority.With(wTimeout: _options.CommandTimeout)) + }; + + return _client.StartSessionAsync(sessionOptions, cancellationToken); + } + + public ValueTask DisposeAsync() + { + if (_disposed) + { + return ValueTask.CompletedTask; + } + + _disposed = true; + + IClientSessionHandle? handle; + lock (_gate) + { + handle = _session; + _session = null; + _sessionTask = null; + } + + handle?.Dispose(); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/VexMongoStorageOptions.cs b/src/StellaOps.Excititor.Storage.Mongo/VexMongoStorageOptions.cs new file mode 100644 index 00000000..d3399f35 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/VexMongoStorageOptions.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using MongoDB.Driver; + +namespace StellaOps.Excititor.Storage.Mongo; + +/// +/// Configuration controlling Mongo-backed storage for Excititor repositories. +/// +public sealed class VexMongoStorageOptions : IValidatableObject +{ + private const int DefaultInlineThreshold = 256 * 1024; + private static readonly TimeSpan DefaultCacheTtl = TimeSpan.FromHours(12); + private static readonly TimeSpan DefaultCommandTimeout = TimeSpan.FromSeconds(30); + + /// + /// MongoDB connection string for Excititor storage. + /// + public string ConnectionString { get; set; } = "mongodb://localhost:27017"; + + /// + /// Overrides the database name extracted from . + /// + public string? DatabaseName { get; set; } + + /// + /// Timeout applied to write operations to ensure majority acknowledgement completes promptly. + /// + public TimeSpan CommandTimeout { get; set; } = DefaultCommandTimeout; + + /// + /// Name of the GridFS bucket used for raw VEX payloads that exceed . + /// + public string RawBucketName { get; set; } = "vex.raw"; + + /// + /// Inline raw document payloads smaller than this threshold; larger payloads are stored in GridFS. + /// + public int GridFsInlineThresholdBytes { get; set; } = DefaultInlineThreshold; + + /// + /// Default TTL applied to export cache entries (absolute expiration). + /// + public TimeSpan ExportCacheTtl { get; set; } = DefaultCacheTtl; + + /// + /// Resolve the Mongo database name using the explicit override or connection string. + /// + public string GetDatabaseName() + { + if (!string.IsNullOrWhiteSpace(DatabaseName)) + { + return DatabaseName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(ConnectionString)) + { + var url = MongoUrl.Create(ConnectionString); + if (!string.IsNullOrWhiteSpace(url.DatabaseName)) + { + return url.DatabaseName; + } + } + + return "excititor"; + } + + public IEnumerable Validate(ValidationContext validationContext) + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + yield return new ValidationResult("Mongo connection string must be provided.", new[] { nameof(ConnectionString) }); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + yield return new ValidationResult("Command timeout must be greater than zero.", new[] { nameof(CommandTimeout) }); + } + + if (string.IsNullOrWhiteSpace(RawBucketName)) + { + yield return new ValidationResult("Raw bucket name must be provided.", new[] { nameof(RawBucketName) }); + } + + if (GridFsInlineThresholdBytes < 0) + { + yield return new ValidationResult("GridFS inline threshold must be non-negative.", new[] { nameof(GridFsInlineThresholdBytes) }); + } + + if (ExportCacheTtl <= TimeSpan.Zero) + { + yield return new ValidationResult("Export cache TTL must be greater than zero.", new[] { nameof(ExportCacheTtl) }); + } + + _ = GetDatabaseName(); + } +} diff --git a/src/StellaOps.Excititor.Storage.Mongo/VexStatementBackfillService.cs b/src/StellaOps.Excititor.Storage.Mongo/VexStatementBackfillService.cs new file mode 100644 index 00000000..a876e233 --- /dev/null +++ b/src/StellaOps.Excititor.Storage.Mongo/VexStatementBackfillService.cs @@ -0,0 +1,170 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Storage.Mongo; + +public sealed record VexStatementBackfillRequest( + DateTimeOffset? RetrievedSince = null, + bool Force = false, + int BatchSize = 100, + int? MaxDocuments = null); + +public sealed record VexStatementBackfillResult( + int DocumentsEvaluated, + int DocumentsBackfilled, + int ClaimsWritten, + int SkippedExisting, + int NormalizationFailures); + +public sealed class VexStatementBackfillService +{ + private readonly IVexRawStore _rawStore; + private readonly IVexNormalizerRouter _normalizerRouter; + private readonly IVexClaimStore _claimStore; + private readonly IVexMongoSessionProvider _sessionProvider; + private readonly IMongoCollection _rawCollection; + private readonly IMongoCollection _statementCollection; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public VexStatementBackfillService( + IMongoDatabase database, + IVexRawStore rawStore, + IVexNormalizerRouter normalizerRouter, + IVexClaimStore claimStore, + IVexMongoSessionProvider sessionProvider, + TimeProvider? timeProvider, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(database); + _rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore)); + _normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter)); + _claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore)); + _sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + VexMongoMappingRegistry.Register(); + _rawCollection = database.GetCollection(VexMongoCollectionNames.Raw); + _statementCollection = database.GetCollection(VexMongoCollectionNames.Statements); + } + + public async Task RunAsync(VexStatementBackfillRequest request, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + + if (request.BatchSize < 1) + { + throw new ArgumentOutOfRangeException(nameof(request.BatchSize), "Batch size must be at least 1."); + } + + if (request.MaxDocuments is { } max && max <= 0) + { + throw new ArgumentOutOfRangeException(nameof(request.MaxDocuments), "Max documents must be positive when specified."); + } + + var evaluated = 0; + var backfilled = 0; + var claimsWritten = 0; + var skipped = 0; + var failures = 0; + + var filter = request.RetrievedSince is { } since + ? Builders.Filter.Gte(x => x.RetrievedAt, since.UtcDateTime) + : FilterDefinition.Empty; + + var findOptions = new FindOptions + { + Sort = Builders.Sort.Ascending(x => x.RetrievedAt), + BatchSize = request.BatchSize, + }; + + if (request.MaxDocuments is { } limit) + { + findOptions.Limit = limit; + } + + using var cursor = await _rawCollection.FindAsync(filter, findOptions, cancellationToken).ConfigureAwait(false); + + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) + { + foreach (var record in cursor.Current) + { + cancellationToken.ThrowIfCancellationRequested(); + + evaluated++; + + if (!request.Force && await StatementExistsAsync(record.Digest, session, cancellationToken).ConfigureAwait(false)) + { + skipped++; + continue; + } + + var rawDocument = await _rawStore.FindByDigestAsync(record.Digest, cancellationToken, session).ConfigureAwait(false); + if (rawDocument is null) + { + failures++; + _logger.LogWarning("Backfill skipped missing raw document {Digest}.", record.Digest); + continue; + } + + VexClaimBatch batch; + try + { + batch = await _normalizerRouter.NormalizeAsync(rawDocument, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + failures++; + _logger.LogError(ex, "Failed to normalize raw document {Digest} during statement backfill.", record.Digest); + continue; + } + + if (batch.Claims.IsDefaultOrEmpty || batch.Claims.Length == 0) + { + failures++; + _logger.LogWarning("Backfill produced no claims for {Digest}; skipping.", record.Digest); + continue; + } + + var claims = batch.Claims.AsEnumerable(); + var observedAt = rawDocument.RetrievedAt == default + ? _timeProvider.GetUtcNow() + : rawDocument.RetrievedAt; + + await _claimStore.AppendAsync(claims, observedAt, cancellationToken, session).ConfigureAwait(false); + + backfilled++; + claimsWritten += batch.Claims.Length; + } + } + + var result = new VexStatementBackfillResult(evaluated, backfilled, claimsWritten, skipped, failures); + _logger.LogInformation( + "Statement backfill completed: evaluated {Evaluated} documents, backfilled {Backfilled}, wrote {Claims} claims, skipped {Skipped}, failures {Failures}.", + result.DocumentsEvaluated, + result.DocumentsBackfilled, + result.ClaimsWritten, + result.SkippedExisting, + result.NormalizationFailures); + + return result; + } + + private Task StatementExistsAsync(string digest, IClientSessionHandle session, CancellationToken cancellationToken) + { + var filter = Builders.Filter.Eq(x => x.Document.Digest, digest); + var find = session is null + ? _statementCollection.Find(filter) + : _statementCollection.Find(session, filter); + return find.Limit(1).AnyAsync(cancellationToken); + } +} diff --git a/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs b/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs new file mode 100644 index 00000000..ca91090f --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/MirrorEndpointsTests.cs @@ -0,0 +1,225 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using MongoDB.Driver; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Options; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class MirrorEndpointsTests : IClassFixture>, IDisposable +{ + private readonly WebApplicationFactory _factory; + private readonly Mongo2Go.MongoDbRunner _runner; + + public MirrorEndpointsTests(WebApplicationFactory factory) + { + _runner = Mongo2Go.MongoDbRunner.Start(); + _factory = factory.WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, configuration) => + { + var data = new Dictionary + { + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Id"] = "primary", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:DisplayName"] = "Primary Mirror", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxIndexRequestsPerHour"] = "1000", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:MaxDownloadRequestsPerHour"] = "1000", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Key"] = "consensus", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Format"] = "json", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:vulnId"] = "CVE-2025-0001", + [$"{MirrorDistributionOptions.SectionName}:Domains:0:Exports:0:Filters:productKey"] = "pkg:test/demo", + }; + + configuration.AddInMemoryCollection(data!); + }); + + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); + services.RemoveAll(); + services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("mirror-tests")); + + services.RemoveAll(); + services.AddSingleton(provider => + { + var timeProvider = provider.GetRequiredService(); + return new FakeExportStore(timeProvider); + }); + + services.RemoveAll(); + services.AddSingleton(_ => new FakeArtifactStore()); + services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + }); + }); + } + + [Fact] + public async Task ListDomains_ReturnsConfiguredDomain() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var domains = document.RootElement.GetProperty("domains"); + Assert.Equal(1, domains.GetArrayLength()); + Assert.Equal("primary", domains[0].GetProperty("id").GetString()); + } + + [Fact] + public async Task DomainIndex_ReturnsManifestMetadata() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains/primary/index"); + response.EnsureSuccessStatusCode(); + + using var document = JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + var exports = document.RootElement.GetProperty("exports"); + Assert.Equal(1, exports.GetArrayLength()); + var entry = exports[0]; + Assert.Equal("consensus", entry.GetProperty("exportKey").GetString()); + Assert.Equal("exports/20251019T000000000Z/abcdef", entry.GetProperty("exportId").GetString()); + var artifact = entry.GetProperty("artifact"); + Assert.Equal("sha256", artifact.GetProperty("algorithm").GetString()); + Assert.Equal("deadbeef", artifact.GetProperty("digest").GetString()); + } + + [Fact] + public async Task Download_ReturnsArtifactContent() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/mirror/domains/primary/exports/consensus/download"); + response.EnsureSuccessStatusCode(); + Assert.Equal("application/json", response.Content.Headers.ContentType?.MediaType); + var payload = await response.Content.ReadAsStringAsync(); + Assert.Equal("{\"status\":\"ok\"}", payload); + } + + public void Dispose() + { + _runner.Dispose(); + } + + private sealed class FakeExportStore : IVexExportStore + { + private readonly ConcurrentDictionary<(string Signature, VexExportFormat Format), VexExportManifest> _manifests = new(); + + public FakeExportStore(TimeProvider timeProvider) + { + var filters = new[] + { + new VexQueryFilter("vulnId", "CVE-2025-0001"), + new VexQueryFilter("productKey", "pkg:test/demo"), + }; + + var query = VexQuery.Create(filters, Enumerable.Empty()); + var signature = VexQuerySignature.FromQuery(query); + var createdAt = new DateTimeOffset(2025, 10, 19, 0, 0, 0, TimeSpan.Zero); + + var manifest = new VexExportManifest( + "exports/20251019T000000000Z/abcdef", + signature, + VexExportFormat.Json, + createdAt, + new VexContentAddress("sha256", "deadbeef"), + 1, + new[] { "primary" }, + fromCache: false, + consensusRevision: "rev-1", + attestation: new VexAttestationMetadata("https://stella-ops.org/attestations/vex-export"), + sizeBytes: 16); + + _manifests.TryAdd((signature.Value, VexExportFormat.Json), manifest); + + // Seed artifact content for download test. + FakeArtifactStore.Seed(manifest.Artifact, "{\"status\":\"ok\"}"); + } + + public ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken, IClientSessionHandle? session = null) + { + _manifests.TryGetValue((signature.Value, format), out var manifest); + return ValueTask.FromResult(manifest); + } + + public ValueTask SaveAsync(VexExportManifest manifest, CancellationToken cancellationToken, IClientSessionHandle? session = null) + => ValueTask.CompletedTask; + } + + private sealed class FakeArtifactStore : IVexArtifactStore + { + private static readonly ConcurrentDictionary Content = new(); + + public static void Seed(VexContentAddress contentAddress, string payload) + { + var bytes = System.Text.Encoding.UTF8.GetBytes(payload); + Content[contentAddress] = bytes; + } + + public ValueTask SaveAsync(VexExportArtifact artifact, CancellationToken cancellationToken) + { + Content[artifact.ContentAddress] = artifact.Content.ToArray(); + return ValueTask.FromResult(new VexStoredArtifact(artifact.ContentAddress, "memory://artifact", artifact.Content.Length, artifact.Metadata)); + } + + public ValueTask DeleteAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + Content.TryRemove(contentAddress, out _); + return ValueTask.CompletedTask; + } + + public ValueTask OpenReadAsync(VexContentAddress contentAddress, CancellationToken cancellationToken) + { + if (!Content.TryGetValue(contentAddress, out var bytes)) + { + return ValueTask.FromResult(null); + } + + return ValueTask.FromResult(new MemoryStream(bytes, writable: false)); + } + } + + private sealed class FakeSigner : StellaOps.Excititor.Attestation.Signing.IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new StellaOps.Excititor.Attestation.Signing.VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : StellaOps.Excititor.Policy.IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + private sealed class FakeExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); + return ValueTask.FromResult(dataset); + } + } +} diff --git a/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs b/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs new file mode 100644 index 00000000..ca0e44d9 --- /dev/null +++ b/src/StellaOps.Excititor.WebService.Tests/ResolveEndpointTests.cs @@ -0,0 +1,342 @@ +using System.Collections.Immutable; +using System.Net; +using System.Net.Http.Json; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class ResolveEndpointTests : IClassFixture>, IDisposable +{ + private readonly WebApplicationFactory _factory; + private readonly MongoDbRunner _runner; + + public ResolveEndpointTests(WebApplicationFactory factory) + { + _runner = MongoDbRunner.Start(); + _factory = factory.WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, config) => + { + var rootPath = Path.Combine(Path.GetTempPath(), "excititor-resolve-tests"); + Directory.CreateDirectory(rootPath); + var settings = new Dictionary + { + ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", + ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", + ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, + }; + config.AddInMemoryCollection(settings!); + }); + + builder.ConfigureServices(services => + { + services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); + services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("excititor-resolve-tests")); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); + }); + }); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsBadRequest_WhenInputsMissing() + { + var client = _factory.CreateClient(); + var response = await client.PostAsJsonAsync("/excititor/resolve", new { vulnerabilityIds = new[] { "CVE-2025-0001" } }); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task ResolveEndpoint_ComputesConsensusAndAttestation() + { + const string vulnerabilityId = "CVE-2025-2222"; + const string productKey = "pkg:nuget/StellaOps.Demo@1.0.0"; + const string providerId = "redhat"; + + await SeedProviderAsync(providerId); + await SeedClaimAsync(vulnerabilityId, productKey, providerId); + + var client = _factory.CreateClient(); + var request = new ResolveRequest( + new[] { productKey }, + null, + new[] { vulnerabilityId }, + null); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + response.EnsureSuccessStatusCode(); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.NotNull(payload!.Policy); + + var result = Assert.Single(payload.Results); + Assert.Equal(vulnerabilityId, result.VulnerabilityId); + Assert.Equal(productKey, result.ProductKey); + Assert.Equal("not_affected", result.Status); + Assert.NotNull(result.Envelope); + Assert.Equal("signature", result.Envelope!.ContentSignature!.Value); + Assert.Equal("key", result.Envelope.ContentSignature.KeyId); + Assert.NotEqual(default, result.CalculatedAt); + + Assert.NotNull(result.Signals); + Assert.True(result.Signals!.Kev); + Assert.NotNull(result.Envelope.AttestationSignature); + Assert.False(string.IsNullOrWhiteSpace(result.Envelope.AttestationEnvelope)); + Assert.Equal(payload.Policy.ActiveRevisionId, result.PolicyRevisionId); + Assert.Equal(payload.Policy.Version, result.PolicyVersion); + Assert.Equal(payload.Policy.Digest, result.PolicyDigest); + + var decision = Assert.Single(result.Decisions); + Assert.True(decision.Included); + Assert.Equal(providerId, decision.ProviderId); + } + + [Fact] + public async Task ResolveEndpoint_ReturnsConflict_WhenPolicyRevisionMismatch() + { + const string vulnerabilityId = "CVE-2025-3333"; + const string productKey = "pkg:docker/demo@sha256:abcd"; + + var client = _factory.CreateClient(); + var request = new ResolveRequest( + new[] { productKey }, + null, + new[] { vulnerabilityId }, + "rev-0"); + + var response = await client.PostAsJsonAsync("/excititor/resolve", request); + Assert.Equal(HttpStatusCode.Conflict, response.StatusCode); + } + + private async Task SeedProviderAsync(string providerId) + { + await using var scope = _factory.Services.CreateAsyncScope(); + var store = scope.ServiceProvider.GetRequiredService(); + var provider = new VexProvider(providerId, "Red Hat", VexProviderKind.Distro); + await store.SaveAsync(provider, CancellationToken.None); + } + + private async Task SeedClaimAsync(string vulnerabilityId, string productKey, string providerId) + { + await using var scope = _factory.Services.CreateAsyncScope(); + var store = scope.ServiceProvider.GetRequiredService(); + var timeProvider = scope.ServiceProvider.GetRequiredService(); + var observedAt = timeProvider.GetUtcNow(); + + var claim = new VexClaim( + vulnerabilityId, + providerId, + new VexProduct(productKey, "Demo Component", version: "1.0.0", purl: productKey), + VexClaimStatus.NotAffected, + new VexClaimDocument(VexDocumentFormat.Csaf, "sha256:deadbeef", new Uri("https://example.org/vex/csaf.json")), + observedAt.AddDays(-1), + observedAt, + VexJustification.ProtectedByMitigatingControl, + detail: "Test justification", + confidence: new VexConfidence("high", 0.9, "unit-test"), + signals: new VexSignalSnapshot( + new VexSeveritySignal("cvss:v3.1", 5.5, "medium"), + kev: true, + epss: 0.25)); + + await store.AppendAsync(new[] { claim }, observedAt, CancellationToken.None); + } + + public void Dispose() + { + _runner.Dispose(); + } + + private sealed class ResolveRequest + { + public ResolveRequest( + IReadOnlyList? productKeys, + IReadOnlyList? purls, + IReadOnlyList? vulnerabilityIds, + string? policyRevisionId) + { + ProductKeys = productKeys; + Purls = purls; + VulnerabilityIds = vulnerabilityIds; + PolicyRevisionId = policyRevisionId; + } + + public IReadOnlyList? ProductKeys { get; } + + public IReadOnlyList? Purls { get; } + + public IReadOnlyList? VulnerabilityIds { get; } + + public string? PolicyRevisionId { get; } + } + + private sealed class ResolveResponse + { + public required DateTimeOffset ResolvedAt { get; init; } + + public required ResolvePolicy Policy { get; init; } + + public required List Results { get; init; } + } + + private sealed class ResolvePolicy + { + public required string ActiveRevisionId { get; init; } + + public required string Version { get; init; } + + public required string Digest { get; init; } + + public string? RequestedRevisionId { get; init; } + } + + private sealed class ResolveResult + { + public required string VulnerabilityId { get; init; } + + public required string ProductKey { get; init; } + + public required string Status { get; init; } + + public required DateTimeOffset CalculatedAt { get; init; } + + public required List Sources { get; init; } + + public required List Conflicts { get; init; } + + public ResolveSignals? Signals { get; init; } + + public string? Summary { get; init; } + + public required string PolicyRevisionId { get; init; } + + public required string PolicyVersion { get; init; } + + public required string PolicyDigest { get; init; } + + public required List Decisions { get; init; } + + public ResolveEnvelope? Envelope { get; init; } + } + + private sealed class ResolveSource + { + public required string ProviderId { get; init; } + } + + private sealed class ResolveConflict + { + public string? ProviderId { get; init; } + } + + private sealed class ResolveSignals + { + public ResolveSeverity? Severity { get; init; } + + public bool? Kev { get; init; } + + public double? Epss { get; init; } + } + + private sealed class ResolveSeverity + { + public string? Scheme { get; init; } + + public double? Score { get; init; } + } + + private sealed class ResolveDecision + { + public required string ProviderId { get; init; } + + public required bool Included { get; init; } + + public string? Reason { get; init; } + } + + private sealed class ResolveEnvelope + { + public required ResolveArtifact Artifact { get; init; } + + public ResolveSignature? ContentSignature { get; init; } + + public ResolveAttestationMetadata? Attestation { get; init; } + + public string? AttestationEnvelope { get; init; } + + public ResolveSignature? AttestationSignature { get; init; } + } + + private sealed class ResolveArtifact + { + public required string Algorithm { get; init; } + + public required string Digest { get; init; } + } + + private sealed class ResolveSignature + { + public required string Value { get; init; } + + public string? KeyId { get; init; } + } + + private sealed class ResolveAttestationMetadata + { + public required string PredicateType { get; init; } + + public ResolveRekorReference? Rekor { get; init; } + + public string? EnvelopeDigest { get; init; } + + public DateTimeOffset? SignedAt { get; init; } + } + + private sealed class ResolveRekorReference + { + public string? Location { get; init; } + } + + private sealed class FakeSigner : IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + private sealed class FakeExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); + return ValueTask.FromResult(dataset); + } + } +} diff --git a/src/StellaOps.Vexer.WebService.Tests/StatusEndpointTests.cs b/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs similarity index 78% rename from src/StellaOps.Vexer.WebService.Tests/StatusEndpointTests.cs rename to src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs index 2463dfe6..42845ef8 100644 --- a/src/StellaOps.Vexer.WebService.Tests/StatusEndpointTests.cs +++ b/src/StellaOps.Excititor.WebService.Tests/StatusEndpointTests.cs @@ -1,105 +1,107 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Net.Http.Json; -using System.IO; -using Microsoft.AspNetCore.Mvc.Testing; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Mongo2Go; -using MongoDB.Driver; -using StellaOps.Vexer.Attestation.Signing; -using StellaOps.Vexer.Policy; -using StellaOps.Vexer.Core; -using StellaOps.Vexer.Export; -using StellaOps.Vexer.WebService; - -namespace StellaOps.Vexer.WebService.Tests; - -public sealed class StatusEndpointTests : IClassFixture>, IDisposable -{ - private readonly WebApplicationFactory _factory; - private readonly MongoDbRunner _runner; - - public StatusEndpointTests(WebApplicationFactory factory) - { - _runner = MongoDbRunner.Start(); - _factory = factory.WithWebHostBuilder(builder => - { - builder.ConfigureAppConfiguration((_, config) => - { - var rootPath = Path.Combine(Path.GetTempPath(), "vexer-offline-tests"); - Directory.CreateDirectory(rootPath); - var settings = new Dictionary - { - ["Vexer:Storage:Mongo:RawBucketName"] = "vex.raw", - ["Vexer:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", - ["Vexer:Artifacts:FileSystem:RootPath"] = rootPath, - }; - config.AddInMemoryCollection(settings!); - }); - - builder.ConfigureServices(services => - { - services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); - services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("vexer-web-tests")); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - }); - }); - } - - [Fact] - public async Task StatusEndpoint_ReturnsArtifactStores() - { - var client = _factory.CreateClient(); - var response = await client.GetAsync("/vexer/status"); - var raw = await response.Content.ReadAsStringAsync(); - Assert.True(response.IsSuccessStatusCode, raw); - - var payload = System.Text.Json.JsonSerializer.Deserialize(raw); - Assert.NotNull(payload); - Assert.NotEmpty(payload!.ArtifactStores); - } - - public void Dispose() - { - _runner.Dispose(); - } - - private sealed class StatusResponse - { - public string[] ArtifactStores { get; set; } = Array.Empty(); - } - - private sealed class FakeSigner : IVexSigner - { - public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) - => ValueTask.FromResult(new VexSignedPayload("signature", "key")); - } - - private sealed class FakePolicyEvaluator : IVexPolicyEvaluator - { - public string Version => "test"; - - public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; - - public double GetProviderWeight(VexProvider provider) => 1.0; - - public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) - { - rejectionReason = null; - return true; - } - } - - private sealed class FakeExportDataSource : IVexExportDataSource - { - public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) - { - var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); - return ValueTask.FromResult(dataset); - } - } -} +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Net.Http.Json; +using System.IO; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Mongo2Go; +using MongoDB.Driver; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.WebService; + +namespace StellaOps.Excititor.WebService.Tests; + +public sealed class StatusEndpointTests : IClassFixture>, IDisposable +{ + private readonly WebApplicationFactory _factory; + private readonly MongoDbRunner _runner; + + public StatusEndpointTests(WebApplicationFactory factory) + { + _runner = MongoDbRunner.Start(); + _factory = factory.WithWebHostBuilder(builder => + { + builder.ConfigureAppConfiguration((_, config) => + { + var rootPath = Path.Combine(Path.GetTempPath(), "excititor-offline-tests"); + Directory.CreateDirectory(rootPath); + var settings = new Dictionary + { + ["Excititor:Storage:Mongo:RawBucketName"] = "vex.raw", + ["Excititor:Storage:Mongo:GridFsInlineThresholdBytes"] = "256", + ["Excititor:Artifacts:FileSystem:RootPath"] = rootPath, + }; + config.AddInMemoryCollection(settings!); + }); + + builder.ConfigureServices(services => + { + services.AddSingleton(_ => new MongoClient(_runner.ConnectionString)); + services.AddSingleton(provider => provider.GetRequiredService().GetDatabase("excititor-web-tests")); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(new VexConnectorDescriptor("excititor:redhat", VexProviderKind.Distro, "Red Hat CSAF")); + }); + }); + } + + [Fact] + public async Task StatusEndpoint_ReturnsArtifactStores() + { + var client = _factory.CreateClient(); + var response = await client.GetAsync("/excititor/status"); + var raw = await response.Content.ReadAsStringAsync(); + Assert.True(response.IsSuccessStatusCode, raw); + + var payload = System.Text.Json.JsonSerializer.Deserialize(raw); + Assert.NotNull(payload); + Assert.NotEmpty(payload!.ArtifactStores); + } + + public void Dispose() + { + _runner.Dispose(); + } + + private sealed class StatusResponse + { + public string[] ArtifactStores { get; set; } = Array.Empty(); + } + + private sealed class FakeSigner : IVexSigner + { + public ValueTask SignAsync(ReadOnlyMemory payload, CancellationToken cancellationToken) + => ValueTask.FromResult(new VexSignedPayload("signature", "key")); + } + + private sealed class FakePolicyEvaluator : IVexPolicyEvaluator + { + public string Version => "test"; + + public VexPolicySnapshot Snapshot => VexPolicySnapshot.Default; + + public double GetProviderWeight(VexProvider provider) => 1.0; + + public bool IsClaimEligible(VexClaim claim, VexProvider provider, out string? rejectionReason) + { + rejectionReason = null; + return true; + } + } + + private sealed class FakeExportDataSource : IVexExportDataSource + { + public ValueTask FetchAsync(VexQuery query, CancellationToken cancellationToken) + { + var dataset = new VexExportDataSet(ImmutableArray.Empty, ImmutableArray.Empty, ImmutableArray.Empty); + return ValueTask.FromResult(dataset); + } + } +} diff --git a/src/StellaOps.Vexer.WebService.Tests/StellaOps.Vexer.WebService.Tests.csproj b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj similarity index 80% rename from src/StellaOps.Vexer.WebService.Tests/StellaOps.Vexer.WebService.Tests.csproj rename to src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj index 4ad5ef01..097a85d0 100644 --- a/src/StellaOps.Vexer.WebService.Tests/StellaOps.Vexer.WebService.Tests.csproj +++ b/src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj @@ -1,16 +1,16 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Vexer.WebService/AGENTS.md b/src/StellaOps.Excititor.WebService/AGENTS.md similarity index 76% rename from src/StellaOps.Vexer.WebService/AGENTS.md rename to src/StellaOps.Excititor.WebService/AGENTS.md index d5f12802..790102c2 100644 --- a/src/StellaOps.Vexer.WebService/AGENTS.md +++ b/src/StellaOps.Excititor.WebService/AGENTS.md @@ -1,25 +1,25 @@ -# AGENTS -## Role -ASP.NET Minimal API surface for Vexer ingest, provider administration, reconciliation, export, and verification flows. -## Scope -- Program bootstrap, DI wiring for connectors/normalizers/export/attestation/policy/storage. -- HTTP endpoints `/vexer/*` with authentication, authorization scopes, request validation, and deterministic responses. -- Job orchestration bridges for Worker hand-off (when co-hosted) and offline-friendly configuration. -- Observability (structured logs, metrics, tracing) aligned with StellaOps conventions. -## Participants -- StellaOps.Cli sends `vexer` verbs to this service via token-authenticated HTTPS. -- Worker receives scheduled jobs and uses shared infrastructure via common DI extensions. -- Authority service provides tokens; WebService enforces scopes before executing operations. -## Interfaces & contracts -- DTOs for ingest/export requests, run metadata, provider management. -- Background job interfaces for ingest/resume/reconcile triggering. -- Health/status endpoints exposing pull/export history and current policy revision. -## In/Out of scope -In: HTTP hosting, request orchestration, DI composition, auth/authorization, logging. -Out: long-running ingestion loops (Worker), export rendering (Export module), connector implementations. -## Observability & security expectations -- Enforce bearer token scopes, enforce audit logging (request/response correlation IDs, provider IDs). -- Emit structured events for ingest runs, export invocations, attestation references. -- Provide built-in counters/histograms for latency and throughput. -## Tests -- Minimal API contract/unit tests and integration harness will live in `../StellaOps.Vexer.WebService.Tests`. +# AGENTS +## Role +ASP.NET Minimal API surface for Excititor ingest, provider administration, reconciliation, export, and verification flows. +## Scope +- Program bootstrap, DI wiring for connectors/normalizers/export/attestation/policy/storage. +- HTTP endpoints `/excititor/*` with authentication, authorization scopes, request validation, and deterministic responses. +- Job orchestration bridges for Worker hand-off (when co-hosted) and offline-friendly configuration. +- Observability (structured logs, metrics, tracing) aligned with StellaOps conventions. +## Participants +- StellaOps.Cli sends `excititor` verbs to this service via token-authenticated HTTPS. +- Worker receives scheduled jobs and uses shared infrastructure via common DI extensions. +- Authority service provides tokens; WebService enforces scopes before executing operations. +## Interfaces & contracts +- DTOs for ingest/export requests, run metadata, provider management. +- Background job interfaces for ingest/resume/reconcile triggering. +- Health/status endpoints exposing pull/export history and current policy revision. +## In/Out of scope +In: HTTP hosting, request orchestration, DI composition, auth/authorization, logging. +Out: long-running ingestion loops (Worker), export rendering (Export module), connector implementations. +## Observability & security expectations +- Enforce bearer token scopes, enforce audit logging (request/response correlation IDs, provider IDs). +- Emit structured events for ingest runs, export invocations, attestation references. +- Provide built-in counters/histograms for latency and throughput. +## Tests +- Minimal API contract/unit tests and integration harness will live in `../StellaOps.Excititor.WebService.Tests`. diff --git a/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs b/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs new file mode 100644 index 00000000..5566e137 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Endpoints/IngestEndpoints.cs @@ -0,0 +1,284 @@ +using System.Collections.Immutable; +using System.Globalization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using StellaOps.Excititor.WebService.Services; + +namespace StellaOps.Excititor.WebService.Endpoints; + +internal static class IngestEndpoints +{ + private const string AdminScope = "vex.admin"; + + public static void MapIngestEndpoints(IEndpointRouteBuilder app) + { + var group = app.MapGroup("/excititor"); + + group.MapPost("/init", HandleInitAsync); + group.MapPost("/ingest/run", HandleRunAsync); + group.MapPost("/ingest/resume", HandleResumeAsync); + group.MapPost("/reconcile", HandleReconcileAsync); + } + + private static async Task HandleInitAsync( + HttpContext httpContext, + ExcititorInitRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); + if (scopeResult is not null) + { + return scopeResult; + } + + var providerIds = NormalizeProviders(request.Providers); + var options = new IngestInitOptions(providerIds, request.Resume ?? false, timeProvider); + + var summary = await orchestrator.InitializeAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Initialized {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; + + return Results.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + providers = summary.Providers.Select(static provider => new + { + provider.providerId, + provider.displayName, + provider.status, + provider.durationMs, + provider.error + }) + }); + } + + private static async Task HandleRunAsync( + HttpContext httpContext, + ExcititorIngestRunRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); + if (scopeResult is not null) + { + return scopeResult; + } + + if (!TryParseDateTimeOffset(request.Since, out var since, out var sinceError)) + { + return Results.BadRequest(new { message = sinceError }); + } + + if (!TryParseTimeSpan(request.Window, out var window, out var windowError)) + { + return Results.BadRequest(new { message = windowError }); + } + + var providerIds = NormalizeProviders(request.Providers); + var options = new IngestRunOptions( + providerIds, + since, + window, + request.Force ?? false, + timeProvider); + + var summary = await orchestrator.RunAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Ingest run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; + + return Results.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + provider.providerId, + provider.status, + provider.documents, + provider.claims, + provider.startedAt, + provider.completedAt, + provider.durationMs, + provider.lastDigest, + provider.lastUpdated, + provider.checkpoint, + provider.error + }) + }); + } + + private static async Task HandleResumeAsync( + HttpContext httpContext, + ExcititorIngestResumeRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); + if (scopeResult is not null) + { + return scopeResult; + } + + var providerIds = NormalizeProviders(request.Providers); + var options = new IngestResumeOptions(providerIds, request.Checkpoint, timeProvider); + + var summary = await orchestrator.ResumeAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Resume run completed for {summary.ProviderCount} provider(s); {summary.SuccessCount} succeeded, {summary.FailureCount} failed."; + + return Results.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + provider.providerId, + provider.status, + provider.documents, + provider.claims, + provider.startedAt, + provider.completedAt, + provider.durationMs, + provider.since, + provider.checkpoint, + provider.error + }) + }); + } + + private static async Task HandleReconcileAsync( + HttpContext httpContext, + ExcititorReconcileRequest request, + IVexIngestOrchestrator orchestrator, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + var scopeResult = ScopeAuthorization.RequireScope(httpContext, AdminScope); + if (scopeResult is not null) + { + return scopeResult; + } + + if (!TryParseTimeSpan(request.MaxAge, out var maxAge, out var error)) + { + return Results.BadRequest(new { message = error }); + } + + var providerIds = NormalizeProviders(request.Providers); + var options = new ReconcileOptions(providerIds, maxAge, timeProvider); + + var summary = await orchestrator.ReconcileAsync(options, cancellationToken).ConfigureAwait(false); + var message = $"Reconcile completed for {summary.ProviderCount} provider(s); {summary.ReconciledCount} reconciled, {summary.SkippedCount} skipped, {summary.FailureCount} failed."; + + return Results.Ok(new + { + message, + runId = summary.RunId, + startedAt = summary.StartedAt, + completedAt = summary.CompletedAt, + durationMs = summary.Duration.TotalMilliseconds, + providers = summary.Providers.Select(static provider => new + { + provider.providerId, + provider.status, + provider.action, + provider.lastUpdated, + provider.threshold, + provider.documents, + provider.claims, + provider.error + }) + }); + } + + private static ImmutableArray NormalizeProviders(IReadOnlyCollection? providers) + { + if (providers is null || providers.Count == 0) + { + return ImmutableArray.Empty; + } + + var set = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var provider in providers) + { + if (string.IsNullOrWhiteSpace(provider)) + { + continue; + } + + set.Add(provider.Trim()); + } + + return set.ToImmutableArray(); + } + + private static bool TryParseDateTimeOffset(string? value, out DateTimeOffset? result, out string? error) + { + result = null; + error = null; + + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + if (DateTimeOffset.TryParse( + value.Trim(), + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + result = parsed; + return true; + } + + error = "Invalid 'since' value. Use ISO-8601 format (e.g. 2025-10-19T12:30:00Z)."; + return false; + } + + private static bool TryParseTimeSpan(string? value, out TimeSpan? result, out string? error) + { + result = null; + error = null; + + if (string.IsNullOrWhiteSpace(value)) + { + return true; + } + + if (TimeSpan.TryParse(value.Trim(), CultureInfo.InvariantCulture, out var parsed) && parsed >= TimeSpan.Zero) + { + result = parsed; + return true; + } + + error = "Invalid duration value. Use TimeSpan format (e.g. 1.00:00:00)."; + return false; + } + + private sealed record ExcititorInitRequest(IReadOnlyList? Providers, bool? Resume); + + private sealed record ExcititorIngestRunRequest( + IReadOnlyList? Providers, + string? Since, + string? Window, + bool? Force); + + private sealed record ExcititorIngestResumeRequest( + IReadOnlyList? Providers, + string? Checkpoint); + + private sealed record ExcititorReconcileRequest( + IReadOnlyList? Providers, + string? MaxAge); +} diff --git a/src/StellaOps.Excititor.WebService/Endpoints/MirrorEndpoints.cs b/src/StellaOps.Excititor.WebService/Endpoints/MirrorEndpoints.cs new file mode 100644 index 00000000..4d0dc1e3 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Endpoints/MirrorEndpoints.cs @@ -0,0 +1,419 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Text; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Options; +using StellaOps.Excititor.WebService.Services; + +namespace StellaOps.Excititor.WebService.Endpoints; + +internal static class MirrorEndpoints +{ + public static void MapMirrorEndpoints(WebApplication app) + { + var group = app.MapGroup("/excititor/mirror"); + + group.MapGet("/domains", HandleListDomainsAsync); + group.MapGet("/domains/{domainId}", HandleDomainDetailAsync); + group.MapGet("/domains/{domainId}/index", HandleDomainIndexAsync); + group.MapGet("/domains/{domainId}/exports/{exportKey}", HandleExportMetadataAsync); + group.MapGet("/domains/{domainId}/exports/{exportKey}/download", HandleExportDownloadAsync); + } + + private static async Task HandleListDomainsAsync( + HttpContext httpContext, + IOptions options, + CancellationToken cancellationToken) + { + var domains = options.Value.Domains + .Select(static domain => new MirrorDomainSummary( + domain.Id, + string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName, + domain.RequireAuthentication, + Math.Max(domain.MaxIndexRequestsPerHour, 0), + Math.Max(domain.MaxDownloadRequestsPerHour, 0))) + .ToArray(); + + await WriteJsonAsync(httpContext, new MirrorDomainListResponse(domains), StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + private static async Task HandleDomainDetailAsync( + string domainId, + HttpContext httpContext, + IOptions options, + CancellationToken cancellationToken) + { + if (!TryFindDomain(options.Value, domainId, out var domain)) + { + return Results.NotFound(); + } + + var response = new MirrorDomainDetail( + domain.Id, + string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName, + domain.RequireAuthentication, + Math.Max(domain.MaxIndexRequestsPerHour, 0), + Math.Max(domain.MaxDownloadRequestsPerHour, 0), + domain.Exports.Select(static export => export.Key).OrderBy(static key => key, StringComparer.Ordinal).ToImmutableArray()); + + await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + private static async Task HandleDomainIndexAsync( + string domainId, + HttpContext httpContext, + IOptions options, + MirrorRateLimiter rateLimiter, + IVexExportStore exportStore, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + if (!TryFindDomain(options.Value, domainId, out var domain)) + { + return Results.NotFound(); + } + + if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true)) + { + return Results.Unauthorized(); + } + + if (!rateLimiter.TryAcquire(domain.Id, "index", Math.Max(domain.MaxIndexRequestsPerHour, 0), out var retryAfter)) + { + if (retryAfter is { } retry) + { + httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture); + } + + await WritePlainTextAsync(httpContext, "mirror index quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + var resolvedExports = new List(); + foreach (var exportOption in domain.Exports) + { + if (!TryBuildExportPlan(exportOption, out var plan, out var error)) + { + resolvedExports.Add(new MirrorExportIndexEntry( + exportOption.Key, + null, + null, + exportOption.Format, + null, + null, + 0, + null, + null, + error ?? "invalid_export_configuration")); + continue; + } + + var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false); + + if (manifest is null) + { + resolvedExports.Add(new MirrorExportIndexEntry( + exportOption.Key, + null, + plan.Signature.Value, + plan.Format.ToString().ToLowerInvariant(), + null, + null, + 0, + null, + null, + "manifest_not_found")); + continue; + } + + resolvedExports.Add(new MirrorExportIndexEntry( + exportOption.Key, + manifest.ExportId, + manifest.QuerySignature.Value, + manifest.Format.ToString().ToLowerInvariant(), + manifest.CreatedAt, + manifest.Artifact, + manifest.SizeBytes, + manifest.ConsensusRevision, + manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt), + null)); + } + + var indexResponse = new MirrorDomainIndex( + domain.Id, + string.IsNullOrWhiteSpace(domain.DisplayName) ? domain.Id : domain.DisplayName, + timeProvider.GetUtcNow(), + resolvedExports.ToImmutableArray()); + + await WriteJsonAsync(httpContext, indexResponse, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + private static async Task HandleExportMetadataAsync( + string domainId, + string exportKey, + HttpContext httpContext, + IOptions options, + MirrorRateLimiter rateLimiter, + IVexExportStore exportStore, + TimeProvider timeProvider, + CancellationToken cancellationToken) + { + if (!TryFindDomain(options.Value, domainId, out var domain)) + { + return Results.NotFound(); + } + + if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true)) + { + return Results.Unauthorized(); + } + + if (!TryFindExport(domain, exportKey, out var exportOptions)) + { + return Results.NotFound(); + } + + if (!TryBuildExportPlan(exportOptions, out var plan, out var error)) + { + await WritePlainTextAsync(httpContext, error ?? "invalid_export_configuration", StatusCodes.Status503ServiceUnavailable, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false); + if (manifest is null) + { + return Results.NotFound(); + } + + var payload = new MirrorExportMetadata( + domain.Id, + exportOptions.Key, + manifest.ExportId, + manifest.QuerySignature.Value, + manifest.Format.ToString().ToLowerInvariant(), + manifest.CreatedAt, + manifest.Artifact, + manifest.SizeBytes, + manifest.SourceProviders, + manifest.Attestation is null ? null : new MirrorExportAttestation(manifest.Attestation.PredicateType, manifest.Attestation.Rekor?.Location, manifest.Attestation.EnvelopeDigest, manifest.Attestation.SignedAt)); + + await WriteJsonAsync(httpContext, payload, StatusCodes.Status200OK, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + private static async Task HandleExportDownloadAsync( + string domainId, + string exportKey, + HttpContext httpContext, + IOptions options, + MirrorRateLimiter rateLimiter, + IVexExportStore exportStore, + IEnumerable artifactStores, + CancellationToken cancellationToken) + { + if (!TryFindDomain(options.Value, domainId, out var domain)) + { + return Results.NotFound(); + } + + if (domain.RequireAuthentication && (httpContext.User?.Identity?.IsAuthenticated is not true)) + { + return Results.Unauthorized(); + } + + if (!rateLimiter.TryAcquire(domain.Id, "download", Math.Max(domain.MaxDownloadRequestsPerHour, 0), out var retryAfter)) + { + if (retryAfter is { } retry) + { + httpContext.Response.Headers.RetryAfter = ((int)Math.Ceiling(retry.TotalSeconds)).ToString(CultureInfo.InvariantCulture); + } + + await WritePlainTextAsync(httpContext, "mirror download quota exceeded", StatusCodes.Status429TooManyRequests, cancellationToken).ConfigureAwait(false); + return Results.Empty; + } + + if (!TryFindExport(domain, exportKey, out var exportOptions) || !TryBuildExportPlan(exportOptions, out var plan, out _)) + { + return Results.NotFound(); + } + + var manifest = await exportStore.FindAsync(plan.Signature, plan.Format, cancellationToken).ConfigureAwait(false); + if (manifest is null) + { + return Results.NotFound(); + } + + Stream? contentStream = null; + foreach (var store in artifactStores) + { + contentStream = await store.OpenReadAsync(manifest.Artifact, cancellationToken).ConfigureAwait(false); + if (contentStream is not null) + { + break; + } + } + + if (contentStream is null) + { + return Results.NotFound(); + } + + await using (contentStream.ConfigureAwait(false)) + { + var contentType = ResolveContentType(manifest.Format); + httpContext.Response.StatusCode = StatusCodes.Status200OK; + httpContext.Response.ContentType = contentType; + httpContext.Response.Headers.ContentDisposition = FormattableString.Invariant($"attachment; filename=\"{BuildDownloadFileName(domain.Id, exportOptions.Key, manifest.Format)}\""); + + await contentStream.CopyToAsync(httpContext.Response.Body, cancellationToken).ConfigureAwait(false); + } + + return Results.Empty; + } + + private static bool TryFindDomain(MirrorDistributionOptions options, string domainId, out MirrorDomainOptions domain) + { + domain = options.Domains.FirstOrDefault(d => string.Equals(d.Id, domainId, StringComparison.OrdinalIgnoreCase))!; + return domain is not null; + } + + private static bool TryFindExport(MirrorDomainOptions domain, string exportKey, out MirrorExportOptions export) + { + export = domain.Exports.FirstOrDefault(e => string.Equals(e.Key, exportKey, StringComparison.OrdinalIgnoreCase))!; + return export is not null; + } + + private static bool TryBuildExportPlan(MirrorExportOptions exportOptions, out MirrorExportPlan plan, out string? error) + { + plan = null!; + error = null; + + if (string.IsNullOrWhiteSpace(exportOptions.Key)) + { + error = "missing_export_key"; + return false; + } + + if (string.IsNullOrWhiteSpace(exportOptions.Format) || !Enum.TryParse(exportOptions.Format, ignoreCase: true, out var format)) + { + error = "unsupported_export_format"; + return false; + } + + var filters = exportOptions.Filters.Select(pair => new KeyValuePair(pair.Key, pair.Value)).ToArray(); + var sorts = exportOptions.Sort.Select(pair => new VexQuerySort(pair.Key, pair.Value)).ToArray(); + var query = VexQuery.Create(filters.Select(kv => new VexQueryFilter(kv.Key, kv.Value)), sorts, exportOptions.Limit, exportOptions.Offset, exportOptions.View); + var signature = VexQuerySignature.FromQuery(query); + + plan = new MirrorExportPlan(format, query, signature); + return true; + } + + private static string ResolveContentType(VexExportFormat format) + => format switch + { + VexExportFormat.Json => "application/json", + VexExportFormat.JsonLines => "application/jsonl", + VexExportFormat.OpenVex => "application/json", + VexExportFormat.Csaf => "application/json", + _ => "application/octet-stream", + }; + + private static string BuildDownloadFileName(string domainId, string exportKey, VexExportFormat format) + { + var builder = new StringBuilder(domainId.Length + exportKey.Length + 8); + builder.Append(domainId).Append('-').Append(exportKey); + builder.Append(format switch + { + VexExportFormat.Json => ".json", + VexExportFormat.JsonLines => ".jsonl", + VexExportFormat.OpenVex => ".openvex.json", + VexExportFormat.Csaf => ".csaf.json", + _ => ".bin", + }); + return builder.ToString(); + } + + private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken) + { + context.Response.StatusCode = statusCode; + context.Response.ContentType = "text/plain"; + await context.Response.WriteAsync(message, cancellationToken); + } + + private static async Task WriteJsonAsync(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken) + { + context.Response.StatusCode = statusCode; + context.Response.ContentType = "application/json"; + var json = VexCanonicalJsonSerializer.Serialize(payload); + await context.Response.WriteAsync(json, cancellationToken); + } + + private sealed record MirrorExportPlan( + VexExportFormat Format, + VexQuery Query, + VexQuerySignature Signature); +} + +internal sealed record MirrorDomainListResponse(IReadOnlyList Domains); + +internal sealed record MirrorDomainSummary( + string Id, + string DisplayName, + bool RequireAuthentication, + int MaxIndexRequestsPerHour, + int MaxDownloadRequestsPerHour); + +internal sealed record MirrorDomainDetail( + string Id, + string DisplayName, + bool RequireAuthentication, + int MaxIndexRequestsPerHour, + int MaxDownloadRequestsPerHour, + IReadOnlyList Exports); + +internal sealed record MirrorDomainIndex( + string Id, + string DisplayName, + DateTimeOffset GeneratedAt, + IReadOnlyList Exports); + +internal sealed record MirrorExportIndexEntry( + string ExportKey, + string? ExportId, + string? QuerySignature, + string Format, + DateTimeOffset? CreatedAt, + VexContentAddress? Artifact, + long SizeBytes, + string? ConsensusRevision, + MirrorExportAttestation? Attestation, + string? Status); + +internal sealed record MirrorExportAttestation( + string PredicateType, + string? RekorLocation, + string? EnvelopeDigest, + DateTimeOffset? SignedAt); + +internal sealed record MirrorExportMetadata( + string DomainId, + string ExportKey, + string ExportId, + string QuerySignature, + string Format, + DateTimeOffset CreatedAt, + VexContentAddress Artifact, + long SizeBytes, + IReadOnlyList SourceProviders, + MirrorExportAttestation? Attestation); diff --git a/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs b/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs new file mode 100644 index 00000000..d64811f4 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Endpoints/ResolveEndpoint.cs @@ -0,0 +1,504 @@ +namespace StellaOps.Excititor.WebService.Endpoints; + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using StellaOps.Excititor.Attestation; +using StellaOps.Excititor.Attestation.Dsse; +using StellaOps.Excititor.Attestation.Signing; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; + +internal static class ResolveEndpoint +{ + private const int MaxSubjectPairs = 256; + + public static void MapResolveEndpoint(WebApplication app) + { + app.MapPost("/excititor/resolve", HandleResolveAsync); + } + + private static async Task HandleResolveAsync( + VexResolveRequest request, + HttpContext httpContext, + IVexClaimStore claimStore, + IVexConsensusStore consensusStore, + IVexProviderStore providerStore, + IVexPolicyProvider policyProvider, + TimeProvider timeProvider, + ILoggerFactory loggerFactory, + IVexAttestationClient? attestationClient, + IVexSigner? signer, + CancellationToken cancellationToken) + { + if (request is null) + { + return Results.BadRequest("Request payload is required."); + } + + var logger = loggerFactory.CreateLogger("ResolveEndpoint"); + + var productKeys = NormalizeValues(request.ProductKeys, request.Purls); + var vulnerabilityIds = NormalizeValues(request.VulnerabilityIds); + + if (productKeys.Count == 0) + { + await WritePlainTextAsync(httpContext, "At least one productKey or purl must be provided.", StatusCodes.Status400BadRequest, cancellationToken); + return Results.Empty; + } + + if (vulnerabilityIds.Count == 0) + { + await WritePlainTextAsync(httpContext, "At least one vulnerabilityId must be provided.", StatusCodes.Status400BadRequest, cancellationToken); + return Results.Empty; + } + + var pairCount = (long)productKeys.Count * vulnerabilityIds.Count; + if (pairCount > MaxSubjectPairs) + { + await WritePlainTextAsync(httpContext, FormattableString.Invariant($"A maximum of {MaxSubjectPairs} subject pairs are allowed per request."), StatusCodes.Status400BadRequest, cancellationToken); + return Results.Empty; + } + + var snapshot = policyProvider.GetSnapshot(); + + if (!string.IsNullOrWhiteSpace(request.PolicyRevisionId) && + !string.Equals(request.PolicyRevisionId.Trim(), snapshot.RevisionId, StringComparison.Ordinal)) + { + var conflictPayload = new + { + message = $"Requested policy revision '{request.PolicyRevisionId}' does not match active revision '{snapshot.RevisionId}'.", + activeRevision = snapshot.RevisionId, + requestedRevision = request.PolicyRevisionId, + }; + await WriteJsonAsync(httpContext, conflictPayload, StatusCodes.Status409Conflict, cancellationToken); + return Results.Empty; + } + + var resolver = new VexConsensusResolver(snapshot.ConsensusPolicy); + var resolvedAt = timeProvider.GetUtcNow(); + var providerCache = new Dictionary(StringComparer.Ordinal); + var results = new List((int)pairCount); + + foreach (var productKey in productKeys) + { + foreach (var vulnerabilityId in vulnerabilityIds) + { + var claims = await claimStore.FindAsync(vulnerabilityId, productKey, since: null, cancellationToken) + .ConfigureAwait(false); + + var claimArray = claims.Count == 0 ? Array.Empty() : claims.ToArray(); + var signals = AggregateSignals(claimArray); + var providers = await LoadProvidersAsync(claimArray, providerStore, providerCache, cancellationToken) + .ConfigureAwait(false); + var product = ResolveProduct(claimArray, productKey); + var calculatedAt = timeProvider.GetUtcNow(); + + var resolution = resolver.Resolve(new VexConsensusRequest( + vulnerabilityId, + product, + claimArray, + providers, + calculatedAt, + snapshot.ConsensusOptions.WeightCeiling, + signals, + snapshot.RevisionId, + snapshot.Digest)); + + var consensus = resolution.Consensus; + + if (!string.Equals(consensus.PolicyVersion, snapshot.Version, StringComparison.Ordinal) || + !string.Equals(consensus.PolicyRevisionId, snapshot.RevisionId, StringComparison.Ordinal) || + !string.Equals(consensus.PolicyDigest, snapshot.Digest, StringComparison.Ordinal)) + { + consensus = new VexConsensus( + consensus.VulnerabilityId, + consensus.Product, + consensus.Status, + consensus.CalculatedAt, + consensus.Sources, + consensus.Conflicts, + consensus.Signals, + snapshot.Version, + consensus.Summary, + snapshot.RevisionId, + snapshot.Digest); + } + + await consensusStore.SaveAsync(consensus, cancellationToken).ConfigureAwait(false); + + var payload = PreparePayload(consensus); + var contentSignature = await TrySignAsync(signer, payload, logger, cancellationToken).ConfigureAwait(false); + var attestation = await BuildAttestationAsync( + attestationClient, + consensus, + snapshot, + payload, + logger, + cancellationToken).ConfigureAwait(false); + + var decisions = resolution.DecisionLog.IsDefault + ? Array.Empty() + : resolution.DecisionLog.ToArray(); + + results.Add(new VexResolveResult( + consensus.VulnerabilityId, + consensus.Product.Key, + consensus.Status, + consensus.CalculatedAt, + consensus.Sources, + consensus.Conflicts, + consensus.Signals, + consensus.Summary, + consensus.PolicyRevisionId ?? snapshot.RevisionId, + consensus.PolicyVersion ?? snapshot.Version, + consensus.PolicyDigest ?? snapshot.Digest, + decisions, + new VexResolveEnvelope( + payload.Artifact, + contentSignature, + attestation.Metadata, + attestation.Envelope, + attestation.Signature))); + } + } + + var policy = new VexResolvePolicy( + snapshot.RevisionId, + snapshot.Version, + snapshot.Digest, + request.PolicyRevisionId?.Trim()); + + var response = new VexResolveResponse(resolvedAt, policy, results); + await WriteJsonAsync(httpContext, response, StatusCodes.Status200OK, cancellationToken); + return Results.Empty; + } + + private static List NormalizeValues(params IReadOnlyList?[] sources) + { + var result = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var source in sources) + { + if (source is null) + { + continue; + } + + foreach (var value in source) + { + if (string.IsNullOrWhiteSpace(value)) + { + continue; + } + + var normalized = value.Trim(); + if (seen.Add(normalized)) + { + result.Add(normalized); + } + } + } + + return result; + } + + private static VexSignalSnapshot? AggregateSignals(IReadOnlyList claims) + { + if (claims.Count == 0) + { + return null; + } + + VexSeveritySignal? bestSeverity = null; + double? bestScore = null; + bool kevPresent = false; + bool kevTrue = false; + double? bestEpss = null; + + foreach (var claim in claims) + { + if (claim.Signals is null) + { + continue; + } + + var severity = claim.Signals.Severity; + if (severity is not null) + { + var score = severity.Score; + if (bestSeverity is null || + (score is not null && (bestScore is null || score.Value > bestScore.Value)) || + (score is null && bestScore is null && !string.IsNullOrWhiteSpace(severity.Label) && string.IsNullOrWhiteSpace(bestSeverity.Label))) + { + bestSeverity = severity; + bestScore = severity.Score; + } + } + + if (claim.Signals.Kev is { } kevValue) + { + kevPresent = true; + if (kevValue) + { + kevTrue = true; + } + } + + if (claim.Signals.Epss is { } epss) + { + if (bestEpss is null || epss > bestEpss.Value) + { + bestEpss = epss; + } + } + } + + if (bestSeverity is null && !kevPresent && bestEpss is null) + { + return null; + } + + bool? kev = kevTrue ? true : (kevPresent ? false : null); + return new VexSignalSnapshot(bestSeverity, kev, bestEpss); + } + + private static async Task> LoadProvidersAsync( + IReadOnlyList claims, + IVexProviderStore providerStore, + IDictionary cache, + CancellationToken cancellationToken) + { + if (claims.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + var seen = new HashSet(StringComparer.Ordinal); + + foreach (var providerId in claims.Select(claim => claim.ProviderId)) + { + if (!seen.Add(providerId)) + { + continue; + } + + if (cache.TryGetValue(providerId, out var cached)) + { + builder[providerId] = cached; + continue; + } + + var provider = await providerStore.FindAsync(providerId, cancellationToken).ConfigureAwait(false); + if (provider is not null) + { + cache[providerId] = provider; + builder[providerId] = provider; + } + } + + return builder.ToImmutable(); + } + + private static VexProduct ResolveProduct(IReadOnlyList claims, string productKey) + { + if (claims.Count > 0) + { + return claims[0].Product; + } + + var inferredPurl = productKey.StartsWith("pkg:", StringComparison.OrdinalIgnoreCase) ? productKey : null; + return new VexProduct(productKey, name: null, version: null, purl: inferredPurl); + } + + private static ConsensusPayload PreparePayload(VexConsensus consensus) + { + var canonicalJson = VexCanonicalJsonSerializer.Serialize(consensus); + var bytes = Encoding.UTF8.GetBytes(canonicalJson); + var digest = SHA256.HashData(bytes); + var digestHex = Convert.ToHexString(digest).ToLowerInvariant(); + var address = new VexContentAddress("sha256", digestHex); + return new ConsensusPayload(address, bytes, canonicalJson); + } + + private static async ValueTask TrySignAsync( + IVexSigner? signer, + ConsensusPayload payload, + ILogger logger, + CancellationToken cancellationToken) + { + if (signer is null) + { + return null; + } + + try + { + var signature = await signer.SignAsync(payload.Bytes, cancellationToken).ConfigureAwait(false); + return new ResolveSignature(signature.Signature, signature.KeyId); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to sign resolve payload {Digest}", payload.Artifact.ToUri()); + return null; + } + } + + private static async ValueTask BuildAttestationAsync( + IVexAttestationClient? attestationClient, + VexConsensus consensus, + VexPolicySnapshot snapshot, + ConsensusPayload payload, + ILogger logger, + CancellationToken cancellationToken) + { + if (attestationClient is null) + { + return new ResolveAttestation(null, null, null); + } + + try + { + var exportId = BuildAttestationExportId(consensus.VulnerabilityId, consensus.Product.Key); + var filters = new[] + { + new KeyValuePair("vulnerabilityId", consensus.VulnerabilityId), + new KeyValuePair("productKey", consensus.Product.Key), + new KeyValuePair("policyRevisionId", snapshot.RevisionId), + }; + + var querySignature = VexQuerySignature.FromFilters(filters); + var providerIds = consensus.Sources + .Select(source => source.ProviderId) + .Distinct(StringComparer.Ordinal) + .ToImmutableArray(); + + var metadataBuilder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + metadataBuilder["consensusDigest"] = payload.Artifact.ToUri(); + metadataBuilder["policyRevisionId"] = snapshot.RevisionId; + metadataBuilder["policyVersion"] = snapshot.Version; + if (!string.IsNullOrWhiteSpace(snapshot.Digest)) + { + metadataBuilder["policyDigest"] = snapshot.Digest; + } + + var response = await attestationClient.SignAsync(new VexAttestationRequest( + exportId, + querySignature, + payload.Artifact, + VexExportFormat.Json, + consensus.CalculatedAt, + providerIds, + metadataBuilder.ToImmutable()), cancellationToken).ConfigureAwait(false); + + var envelopeJson = response.Diagnostics.TryGetValue("envelope", out var envelopeValue) + ? envelopeValue + : null; + + ResolveSignature? signature = null; + if (!string.IsNullOrWhiteSpace(envelopeJson)) + { + try + { + var envelope = JsonSerializer.Deserialize(envelopeJson); + var dsseSignature = envelope?.Signatures?.FirstOrDefault(); + if (dsseSignature is not null) + { + signature = new ResolveSignature(dsseSignature.Signature, dsseSignature.KeyId); + } + } + catch (Exception ex) + { + logger.LogDebug(ex, "Failed to deserialize DSSE envelope for resolve export {ExportId}", exportId); + } + } + + return new ResolveAttestation(response.Attestation, envelopeJson, signature); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Unable to produce attestation for {VulnerabilityId}/{ProductKey}", consensus.VulnerabilityId, consensus.Product.Key); + return new ResolveAttestation(null, null, null); + } + } + + private static string BuildAttestationExportId(string vulnerabilityId, string productKey) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(productKey)); + var digest = Convert.ToHexString(hash).ToLowerInvariant(); + return FormattableString.Invariant($"resolve/{vulnerabilityId}/{digest}"); + } + +private sealed record ConsensusPayload(VexContentAddress Artifact, byte[] Bytes, string CanonicalJson); + +private static async Task WritePlainTextAsync(HttpContext context, string message, int statusCode, CancellationToken cancellationToken) +{ + context.Response.StatusCode = statusCode; + context.Response.ContentType = "text/plain"; + await context.Response.WriteAsync(message, cancellationToken); +} + +private static async Task WriteJsonAsync(HttpContext context, T payload, int statusCode, CancellationToken cancellationToken) +{ + context.Response.StatusCode = statusCode; + context.Response.ContentType = "application/json"; + var json = VexCanonicalJsonSerializer.Serialize(payload); + await context.Response.WriteAsync(json, cancellationToken); +} +} + +public sealed record VexResolveRequest( + IReadOnlyList? ProductKeys, + IReadOnlyList? Purls, + IReadOnlyList? VulnerabilityIds, + string? PolicyRevisionId); + +internal sealed record VexResolvePolicy( + string ActiveRevisionId, + string Version, + string Digest, + string? RequestedRevisionId); + +internal sealed record VexResolveResponse( + DateTimeOffset ResolvedAt, + VexResolvePolicy Policy, + IReadOnlyList Results); + +internal sealed record VexResolveResult( + string VulnerabilityId, + string ProductKey, + VexConsensusStatus Status, + DateTimeOffset CalculatedAt, + IReadOnlyList Sources, + IReadOnlyList Conflicts, + VexSignalSnapshot? Signals, + string? Summary, + string PolicyRevisionId, + string PolicyVersion, + string PolicyDigest, + IReadOnlyList Decisions, + VexResolveEnvelope Envelope); + +internal sealed record VexResolveEnvelope( + VexContentAddress Artifact, + ResolveSignature? ContentSignature, + VexAttestationMetadata? Attestation, + string? AttestationEnvelope, + ResolveSignature? AttestationSignature); + +internal sealed record ResolveSignature(string Value, string? KeyId); + +internal sealed record ResolveAttestation( + VexAttestationMetadata? Metadata, + string? Envelope, + ResolveSignature? Signature); diff --git a/src/StellaOps.Excititor.WebService/Options/MirrorDistributionOptions.cs b/src/StellaOps.Excititor.WebService/Options/MirrorDistributionOptions.cs new file mode 100644 index 00000000..b583d222 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Options/MirrorDistributionOptions.cs @@ -0,0 +1,52 @@ +using System.Collections.Generic; + +namespace StellaOps.Excititor.WebService.Options; + +public sealed class MirrorDistributionOptions +{ + public const string SectionName = "Excititor:Mirror"; + + public List Domains { get; } = new(); +} + +public sealed class MirrorDomainOptions +{ + public string Id { get; set; } = string.Empty; + + public string DisplayName { get; set; } = string.Empty; + + public bool RequireAuthentication { get; set; } + = false; + + /// + /// Maximum index requests allowed per rolling window. + /// + public int MaxIndexRequestsPerHour { get; set; } = 120; + + /// + /// Maximum export downloads allowed per rolling window. + /// + public int MaxDownloadRequestsPerHour { get; set; } = 600; + + public List Exports { get; } = new(); +} + +public sealed class MirrorExportOptions +{ + public string Key { get; set; } = string.Empty; + + public string Format { get; set; } = string.Empty; + + public Dictionary Filters { get; } = new(); + + public Dictionary Sort { get; } = new(); + + public int? Limit { get; set; } + = null; + + public int? Offset { get; set; } + = null; + + public string? View { get; set; } + = null; +} diff --git a/src/StellaOps.Excititor.WebService/Program.cs b/src/StellaOps.Excititor.WebService/Program.cs new file mode 100644 index 00000000..39d86834 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Program.cs @@ -0,0 +1,263 @@ +using System.Collections.Generic; +using System.Linq; +using System.Collections.Immutable; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Attestation.Extensions; +using StellaOps.Excititor.Attestation; +using StellaOps.Excititor.Attestation.Transparency; +using StellaOps.Excititor.ArtifactStores.S3.Extensions; +using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Export; +using StellaOps.Excititor.Formats.CSAF; +using StellaOps.Excititor.Formats.CycloneDX; +using StellaOps.Excititor.Formats.OpenVEX; +using StellaOps.Excititor.Policy; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.WebService.Endpoints; +using StellaOps.Excititor.WebService.Options; +using StellaOps.Excititor.WebService.Services; + +var builder = WebApplication.CreateBuilder(args); +var configuration = builder.Configuration; +var services = builder.Services; + +services.AddOptions() + .Bind(configuration.GetSection("Excititor:Storage:Mongo")) + .ValidateOnStart(); + +services.AddExcititorMongoStorage(); +services.AddCsafNormalizer(); +services.AddCycloneDxNormalizer(); +services.AddOpenVexNormalizer(); +services.AddSingleton(); +services.AddScoped(); +services.AddVexExportEngine(); +services.AddVexExportCacheServices(); +services.AddVexAttestation(); +services.Configure(configuration.GetSection("Excititor:Attestation:Client")); +services.AddVexPolicy(); +services.AddRedHatCsafConnector(); +services.Configure(configuration.GetSection(MirrorDistributionOptions.SectionName)); +services.AddSingleton(); + +var rekorSection = configuration.GetSection("Excititor:Attestation:Rekor"); +if (rekorSection.Exists()) +{ + services.AddVexRekorClient(opts => rekorSection.Bind(opts)); +} + +var fileSystemSection = configuration.GetSection("Excititor:Artifacts:FileSystem"); +if (fileSystemSection.Exists()) +{ + services.AddVexFileSystemArtifactStore(opts => fileSystemSection.Bind(opts)); +} +else +{ + services.AddVexFileSystemArtifactStore(_ => { }); +} + +var s3Section = configuration.GetSection("Excititor:Artifacts:S3"); +if (s3Section.Exists()) +{ + services.AddVexS3ArtifactClient(opts => s3Section.GetSection("Client").Bind(opts)); + services.AddSingleton(provider => + { + var options = new S3ArtifactStoreOptions(); + s3Section.GetSection("Store").Bind(options); + return new S3ArtifactStore( + provider.GetRequiredService(), + Microsoft.Extensions.Options.Options.Create(options), + provider.GetRequiredService>()); + }); +} + +var offlineSection = configuration.GetSection("Excititor:Artifacts:OfflineBundle"); +if (offlineSection.Exists()) +{ + services.AddVexOfflineBundleArtifactStore(opts => offlineSection.Bind(opts)); +} + +services.AddEndpointsApiExplorer(); +services.AddHealthChecks(); +services.AddSingleton(TimeProvider.System); +services.AddMemoryCache(); +services.AddAuthentication(); +services.AddAuthorization(); + +var app = builder.Build(); + +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapGet("/excititor/status", async (HttpContext context, + IEnumerable artifactStores, + IOptions mongoOptions, + TimeProvider timeProvider) => +{ + var payload = new StatusResponse( + timeProvider.GetUtcNow(), + mongoOptions.Value.RawBucketName, + mongoOptions.Value.GridFsInlineThresholdBytes, + artifactStores.Select(store => store.GetType().Name).ToArray()); + + context.Response.ContentType = "application/json"; + await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, payload); +}); + +app.MapHealthChecks("/excititor/health"); + +app.MapPost("/excititor/statements", async ( + VexStatementIngestRequest request, + IVexClaimStore claimStore, + TimeProvider timeProvider, + CancellationToken cancellationToken) => +{ + if (request?.Statements is null || request.Statements.Count == 0) + { + return Results.BadRequest("At least one statement must be provided."); + } + + var claims = request.Statements.Select(statement => statement.ToDomainClaim()); + await claimStore.AppendAsync(claims, timeProvider.GetUtcNow(), cancellationToken).ConfigureAwait(false); + return Results.Accepted(); +}); + +app.MapGet("/excititor/statements/{vulnerabilityId}/{productKey}", async ( + string vulnerabilityId, + string productKey, + DateTimeOffset? since, + IVexClaimStore claimStore, + CancellationToken cancellationToken) => +{ + if (string.IsNullOrWhiteSpace(vulnerabilityId) || string.IsNullOrWhiteSpace(productKey)) + { + return Results.BadRequest("vulnerabilityId and productKey are required."); + } + + var claims = await claimStore.FindAsync(vulnerabilityId.Trim(), productKey.Trim(), since, cancellationToken).ConfigureAwait(false); + return Results.Ok(claims); +}); + +app.MapPost("/excititor/admin/backfill-statements", async ( + VexStatementBackfillRequest? request, + VexStatementBackfillService backfillService, + CancellationToken cancellationToken) => +{ + request ??= new VexStatementBackfillRequest(); + var result = await backfillService.RunAsync(request, cancellationToken).ConfigureAwait(false); + var message = FormattableString.Invariant( + $"Backfill completed: evaluated {result.DocumentsEvaluated}, backfilled {result.DocumentsBackfilled}, claims written {result.ClaimsWritten}, skipped {result.SkippedExisting}, failures {result.NormalizationFailures}."); + + return Results.Ok(new + { + message, + summary = result + }); +}); + +IngestEndpoints.MapIngestEndpoints(app); +ResolveEndpoint.MapResolveEndpoint(app); +MirrorEndpoints.MapMirrorEndpoints(app); + +app.Run(); + +public partial class Program; + +internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores); + +internal sealed record VexStatementIngestRequest(IReadOnlyList Statements); + +internal sealed record VexStatementEntry( + string VulnerabilityId, + string ProviderId, + string ProductKey, + string? ProductName, + string? ProductVersion, + string? ProductPurl, + string? ProductCpe, + IReadOnlyList? ComponentIdentifiers, + VexClaimStatus Status, + VexJustification? Justification, + string? Detail, + DateTimeOffset FirstSeen, + DateTimeOffset LastSeen, + VexDocumentFormat DocumentFormat, + string DocumentDigest, + string DocumentUri, + string? DocumentRevision, + VexSignatureMetadataRequest? Signature, + VexConfidenceRequest? Confidence, + VexSignalRequest? Signals, + IReadOnlyDictionary? Metadata) +{ + public VexClaim ToDomainClaim() + { + var product = new VexProduct( + ProductKey, + ProductName, + ProductVersion, + ProductPurl, + ProductCpe, + ComponentIdentifiers ?? Array.Empty()); + + if (!Uri.TryCreate(DocumentUri, UriKind.Absolute, out var uri)) + { + throw new InvalidOperationException($"DocumentUri '{DocumentUri}' is not a valid absolute URI."); + } + + var document = new VexClaimDocument( + DocumentFormat, + DocumentDigest, + uri, + DocumentRevision, + Signature?.ToDomain()); + + var additionalMetadata = Metadata is null + ? ImmutableDictionary.Empty + : Metadata.ToImmutableDictionary(StringComparer.Ordinal); + + return new VexClaim( + VulnerabilityId, + ProviderId, + product, + Status, + document, + FirstSeen, + LastSeen, + Justification, + Detail, + Confidence?.ToDomain(), + Signals?.ToDomain(), + additionalMetadata); + } +} + +internal sealed record VexSignatureMetadataRequest( + string Type, + string? Subject, + string? Issuer, + string? KeyId, + DateTimeOffset? VerifiedAt, + string? TransparencyLogReference) +{ + public VexSignatureMetadata ToDomain() + => new(Type, Subject, Issuer, KeyId, VerifiedAt, TransparencyLogReference); +} + +internal sealed record VexConfidenceRequest(string Level, double? Score, string? Method) +{ + public VexConfidence ToDomain() => new(Level, Score, Method); +} + +internal sealed record VexSignalRequest(VexSeveritySignalRequest? Severity, bool? Kev, double? Epss) +{ + public VexSignalSnapshot ToDomain() + => new(Severity?.ToDomain(), Kev, Epss); +} + +internal sealed record VexSeveritySignalRequest(string Scheme, double? Score, string? Label, string? Vector) +{ + public VexSeveritySignal ToDomain() => new(Scheme, Score, Label, Vector); +} diff --git a/src/StellaOps.Excititor.WebService/Services/MirrorRateLimiter.cs b/src/StellaOps.Excititor.WebService/Services/MirrorRateLimiter.cs new file mode 100644 index 00000000..b2d61a89 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Services/MirrorRateLimiter.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Caching.Memory; + +namespace StellaOps.Excititor.WebService.Services; + +internal sealed class MirrorRateLimiter +{ + private readonly IMemoryCache _cache; + private readonly TimeProvider _timeProvider; + private static readonly TimeSpan Window = TimeSpan.FromHours(1); + + public MirrorRateLimiter(IMemoryCache cache, TimeProvider timeProvider) + { + _cache = cache ?? throw new ArgumentNullException(nameof(cache)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public bool TryAcquire(string domainId, string scope, int limit, out TimeSpan? retryAfter) + { + retryAfter = null; + + if (limit <= 0 || limit == int.MaxValue) + { + return true; + } + + var key = CreateKey(domainId, scope); + var now = _timeProvider.GetUtcNow(); + + var counter = _cache.Get(key); + if (counter is null || now - counter.WindowStart >= Window) + { + counter = new Counter(now, 0); + } + + if (counter.Count >= limit) + { + var windowEnd = counter.WindowStart + Window; + retryAfter = windowEnd > now ? windowEnd - now : TimeSpan.Zero; + return false; + } + + counter = counter with { Count = counter.Count + 1 }; + var absoluteExpiration = counter.WindowStart + Window; + _cache.Set(key, counter, absoluteExpiration); + return true; + } + + private static string CreateKey(string domainId, string scope) + => string.Create(domainId.Length + scope.Length + 1, (domainId, scope), static (span, state) => + { + state.domainId.AsSpan().CopyTo(span); + span[state.domainId.Length] = '|'; + state.scope.AsSpan().CopyTo(span[(state.domainId.Length + 1)..]); + }); + + private sealed record Counter(DateTimeOffset WindowStart, int Count); +} diff --git a/src/StellaOps.Excititor.WebService/Services/ScopeAuthorization.cs b/src/StellaOps.Excititor.WebService/Services/ScopeAuthorization.cs new file mode 100644 index 00000000..e11ae561 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Services/ScopeAuthorization.cs @@ -0,0 +1,54 @@ +using System.Linq; +using System.Security.Claims; +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Excititor.WebService.Services; + +internal static class ScopeAuthorization +{ + public static IResult? RequireScope(HttpContext context, string scope) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + if (string.IsNullOrWhiteSpace(scope)) + { + throw new ArgumentException("Scope must be provided.", nameof(scope)); + } + + var user = context.User; + if (user?.Identity?.IsAuthenticated is not true) + { + return Results.Unauthorized(); + } + + if (!HasScope(user, scope)) + { + return Results.Forbid(); + } + + return null; + } + + private static bool HasScope(ClaimsPrincipal user, string requiredScope) + { + var comparison = StringComparer.OrdinalIgnoreCase; + foreach (var claim in user.FindAll("scope").Concat(user.FindAll("scp"))) + { + if (string.IsNullOrWhiteSpace(claim.Value)) + { + continue; + } + + var scopes = claim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (scopes.Any(scope => comparison.Equals(scope, requiredScope))) + { + return true; + } + } + + return false; + } +} diff --git a/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs b/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs new file mode 100644 index 00000000..465ddef6 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/Services/VexIngestOrchestrator.cs @@ -0,0 +1,580 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.Linq; +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Excititor.Connectors.Abstractions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.WebService.Services; + +internal interface IVexIngestOrchestrator +{ + Task InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken); + + Task RunAsync(IngestRunOptions options, CancellationToken cancellationToken); + + Task ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken); + + Task ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken); +} + +internal sealed class VexIngestOrchestrator : IVexIngestOrchestrator +{ + private readonly IServiceProvider _serviceProvider; + private readonly IReadOnlyDictionary _connectors; + private readonly IVexRawStore _rawStore; + private readonly IVexClaimStore _claimStore; + private readonly IVexProviderStore _providerStore; + private readonly IVexConnectorStateRepository _stateRepository; + private readonly IVexNormalizerRouter _normalizerRouter; + private readonly IVexSignatureVerifier _signatureVerifier; + private readonly IVexMongoSessionProvider _sessionProvider; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public VexIngestOrchestrator( + IServiceProvider serviceProvider, + IEnumerable connectors, + IVexRawStore rawStore, + IVexClaimStore claimStore, + IVexProviderStore providerStore, + IVexConnectorStateRepository stateRepository, + IVexNormalizerRouter normalizerRouter, + IVexSignatureVerifier signatureVerifier, + IVexMongoSessionProvider sessionProvider, + TimeProvider timeProvider, + ILogger logger) + { + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _rawStore = rawStore ?? throw new ArgumentNullException(nameof(rawStore)); + _claimStore = claimStore ?? throw new ArgumentNullException(nameof(claimStore)); + _providerStore = providerStore ?? throw new ArgumentNullException(nameof(providerStore)); + _stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository)); + _normalizerRouter = normalizerRouter ?? throw new ArgumentNullException(nameof(normalizerRouter)); + _signatureVerifier = signatureVerifier ?? throw new ArgumentNullException(nameof(signatureVerifier)); + _sessionProvider = sessionProvider ?? throw new ArgumentNullException(nameof(sessionProvider)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (connectors is null) + { + throw new ArgumentNullException(nameof(connectors)); + } + + _connectors = connectors + .GroupBy(connector => connector.Id, StringComparer.OrdinalIgnoreCase) + .ToDictionary(group => group.Key, group => group.First(), StringComparer.OrdinalIgnoreCase); + } + + public async Task InitializeAsync(IngestInitOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var runId = Guid.NewGuid(); + var startedAt = _timeProvider.GetUtcNow(); + var results = ImmutableArray.CreateBuilder(); + + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var (handles, missing) = ResolveConnectors(options.Providers); + foreach (var providerId in missing) + { + results.Add(new InitProviderResult(providerId, providerId, "missing", TimeSpan.Zero, "Provider connector is not registered.")); + } + + foreach (var handle in handles) + { + var stopwatch = Stopwatch.StartNew(); + try + { + await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false); + await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + + results.Add(new InitProviderResult( + handle.Descriptor.Id, + handle.Descriptor.DisplayName, + "succeeded", + stopwatch.Elapsed, + error: null)); + + _logger.LogInformation("Excititor init validated provider {ProviderId} in {Duration}ms.", handle.Descriptor.Id, stopwatch.Elapsed.TotalMilliseconds); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + stopwatch.Stop(); + results.Add(new InitProviderResult( + handle.Descriptor.Id, + handle.Descriptor.DisplayName, + "cancelled", + stopwatch.Elapsed, + "Operation cancelled.")); + _logger.LogWarning("Excititor init cancelled for provider {ProviderId}.", handle.Descriptor.Id); + } + catch (Exception ex) + { + stopwatch.Stop(); + results.Add(new InitProviderResult( + handle.Descriptor.Id, + handle.Descriptor.DisplayName, + "failed", + stopwatch.Elapsed, + ex.Message)); + _logger.LogError(ex, "Excititor init failed for provider {ProviderId}: {Message}", handle.Descriptor.Id, ex.Message); + } + } + + var completedAt = _timeProvider.GetUtcNow(); + return new InitSummary(runId, startedAt, completedAt, results.ToImmutable()); + } + + public async Task RunAsync(IngestRunOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var runId = Guid.NewGuid(); + var startedAt = _timeProvider.GetUtcNow(); + var since = ResolveSince(options.Since, options.Window, startedAt); + var results = ImmutableArray.CreateBuilder(); + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var (handles, missing) = ResolveConnectors(options.Providers); + foreach (var providerId in missing) + { + results.Add(ProviderRunResult.Missing(providerId, since)); + } + + foreach (var handle in handles) + { + var result = await ExecuteRunAsync(handle, since, options.Force, session, cancellationToken).ConfigureAwait(false); + results.Add(result); + } + + var completedAt = _timeProvider.GetUtcNow(); + return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable()); + } + + public async Task ResumeAsync(IngestResumeOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var runId = Guid.NewGuid(); + var startedAt = _timeProvider.GetUtcNow(); + var results = ImmutableArray.CreateBuilder(); + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var (handles, missing) = ResolveConnectors(options.Providers); + foreach (var providerId in missing) + { + results.Add(ProviderRunResult.Missing(providerId, since: null)); + } + + foreach (var handle in handles) + { + var since = await ResolveResumeSinceAsync(handle.Descriptor.Id, options.Checkpoint, session, cancellationToken).ConfigureAwait(false); + var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false); + results.Add(result); + } + + var completedAt = _timeProvider.GetUtcNow(); + return new IngestRunSummary(runId, startedAt, completedAt, results.ToImmutable()); + } + + public async Task ReconcileAsync(ReconcileOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + + var runId = Guid.NewGuid(); + var startedAt = _timeProvider.GetUtcNow(); + var threshold = options.MaxAge is null ? (DateTimeOffset?)null : startedAt - options.MaxAge.Value; + var results = ImmutableArray.CreateBuilder(); + var session = await _sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var (handles, missing) = ResolveConnectors(options.Providers); + foreach (var providerId in missing) + { + results.Add(new ReconcileProviderResult(providerId, "missing", "missing", null, threshold, 0, 0, "Provider connector is not registered.")); + } + + foreach (var handle in handles) + { + try + { + var state = await _stateRepository.GetAsync(handle.Descriptor.Id, cancellationToken, session).ConfigureAwait(false); + var lastUpdated = state?.LastUpdated; + var stale = threshold.HasValue && (lastUpdated is null || lastUpdated < threshold.Value); + + if (stale || state is null) + { + var since = stale ? threshold : lastUpdated; + var result = await ExecuteRunAsync(handle, since, force: false, session, cancellationToken).ConfigureAwait(false); + results.Add(new ReconcileProviderResult( + handle.Descriptor.Id, + result.Status, + "reconciled", + result.LastUpdated ?? result.CompletedAt, + threshold, + result.Documents, + result.Claims, + result.Error)); + } + else + { + results.Add(new ReconcileProviderResult( + handle.Descriptor.Id, + "succeeded", + "skipped", + lastUpdated, + threshold, + documents: 0, + claims: 0, + error: null)); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + results.Add(new ReconcileProviderResult( + handle.Descriptor.Id, + "cancelled", + "cancelled", + null, + threshold, + 0, + 0, + "Operation cancelled.")); + _logger.LogWarning("Excititor reconcile cancelled for provider {ProviderId}.", handle.Descriptor.Id); + } + catch (Exception ex) + { + results.Add(new ReconcileProviderResult( + handle.Descriptor.Id, + "failed", + "failed", + null, + threshold, + 0, + 0, + ex.Message)); + _logger.LogError(ex, "Excititor reconcile failed for provider {ProviderId}: {Message}", handle.Descriptor.Id, ex.Message); + } + } + + var completedAt = _timeProvider.GetUtcNow(); + return new ReconcileSummary(runId, startedAt, completedAt, results.ToImmutable()); + } + + private async Task ValidateConnectorAsync(ConnectorHandle handle, CancellationToken cancellationToken) + { + await handle.Connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false); + } + + private async Task EnsureProviderRegistrationAsync(VexConnectorDescriptor descriptor, IClientSessionHandle session, CancellationToken cancellationToken) + { + var existing = await _providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false); + if (existing is not null) + { + return; + } + + var provider = new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind); + await _providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); + } + + private async Task ExecuteRunAsync( + ConnectorHandle handle, + DateTimeOffset? since, + bool force, + IClientSessionHandle session, + CancellationToken cancellationToken) + { + var providerId = handle.Descriptor.Id; + var startedAt = _timeProvider.GetUtcNow(); + var stopwatch = Stopwatch.StartNew(); + + try + { + await ValidateConnectorAsync(handle, cancellationToken).ConfigureAwait(false); + await EnsureProviderRegistrationAsync(handle.Descriptor, session, cancellationToken).ConfigureAwait(false); + + if (force) + { + var resetState = new VexConnectorState(providerId, null, ImmutableArray.Empty); + await _stateRepository.SaveAsync(resetState, cancellationToken, session).ConfigureAwait(false); + } + + var context = new VexConnectorContext( + since, + VexConnectorSettings.Empty, + _rawStore, + _signatureVerifier, + _normalizerRouter, + _serviceProvider); + + var documents = 0; + var claims = 0; + string? lastDigest = null; + + await foreach (var document in handle.Connector.FetchAsync(context, cancellationToken).ConfigureAwait(false)) + { + documents++; + lastDigest = document.Digest; + + var batch = await _normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); + if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) + { + claims += batch.Claims.Length; + await _claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); + } + } + + stopwatch.Stop(); + var completedAt = _timeProvider.GetUtcNow(); + var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false); + + var checkpoint = state?.DocumentDigests.IsDefaultOrEmpty == false + ? state.DocumentDigests[^1] + : lastDigest; + + var result = new ProviderRunResult( + providerId, + "succeeded", + documents, + claims, + startedAt, + completedAt, + stopwatch.Elapsed, + lastDigest, + state?.LastUpdated, + checkpoint, + null, + since); + + _logger.LogInformation( + "Excititor ingest provider {ProviderId} completed: documents={Documents} claims={Claims} since={Since} duration={Duration}ms", + providerId, + documents, + claims, + since?.ToString("O", CultureInfo.InvariantCulture), + result.Duration.TotalMilliseconds); + + return result; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + stopwatch.Stop(); + var cancelledAt = _timeProvider.GetUtcNow(); + _logger.LogWarning("Excititor ingest provider {ProviderId} cancelled.", providerId); + return new ProviderRunResult( + providerId, + "cancelled", + 0, + 0, + startedAt, + cancelledAt, + stopwatch.Elapsed, + null, + null, + null, + "Operation cancelled.", + since); + } + catch (Exception ex) + { + stopwatch.Stop(); + var failedAt = _timeProvider.GetUtcNow(); + _logger.LogError(ex, "Excititor ingest provider {ProviderId} failed: {Message}", providerId, ex.Message); + return new ProviderRunResult( + providerId, + "failed", + 0, + 0, + startedAt, + failedAt, + stopwatch.Elapsed, + null, + null, + null, + ex.Message, + since); + } + } + + private async Task ResolveResumeSinceAsync(string providerId, string? checkpoint, IClientSessionHandle session, CancellationToken cancellationToken) + { + if (!string.IsNullOrWhiteSpace(checkpoint)) + { + if (DateTimeOffset.TryParse( + checkpoint.Trim(), + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + return parsed; + } + + var digest = checkpoint.Trim(); + var document = await _rawStore.FindByDigestAsync(digest, cancellationToken, session).ConfigureAwait(false); + if (document is not null) + { + return document.RetrievedAt; + } + } + + var state = await _stateRepository.GetAsync(providerId, cancellationToken, session).ConfigureAwait(false); + return state?.LastUpdated; + } + + private static DateTimeOffset? ResolveSince(DateTimeOffset? since, TimeSpan? window, DateTimeOffset reference) + { + if (since.HasValue) + { + return since.Value; + } + + if (window is { } duration && duration > TimeSpan.Zero) + { + var candidate = reference - duration; + return candidate < DateTimeOffset.MinValue ? DateTimeOffset.MinValue : candidate; + } + + return null; + } + + private (IReadOnlyList Handles, ImmutableArray Missing) ResolveConnectors(ImmutableArray requestedProviders) + { + var handles = new List(); + var missing = ImmutableArray.CreateBuilder(); + + if (requestedProviders.IsDefaultOrEmpty || requestedProviders.Length == 0) + { + foreach (var connector in _connectors.Values.OrderBy(static x => x.Id, StringComparer.OrdinalIgnoreCase)) + { + handles.Add(new ConnectorHandle(connector, CreateDescriptor(connector))); + } + + return (handles, missing.ToImmutable()); + } + + foreach (var providerId in requestedProviders) + { + if (_connectors.TryGetValue(providerId, out var connector)) + { + handles.Add(new ConnectorHandle(connector, CreateDescriptor(connector))); + } + else + { + missing.Add(providerId); + } + } + + return (handles, missing.ToImmutable()); + } + + private static VexConnectorDescriptor CreateDescriptor(IVexConnector connector) + => connector switch + { + VexConnectorBase baseConnector => baseConnector.Descriptor, + _ => new VexConnectorDescriptor(connector.Id, connector.Kind, connector.Id) + }; + + private sealed record ConnectorHandle(IVexConnector Connector, VexConnectorDescriptor Descriptor); +} + +internal sealed record IngestInitOptions( + ImmutableArray Providers, + bool Resume); + +internal sealed record IngestRunOptions( + ImmutableArray Providers, + DateTimeOffset? Since, + TimeSpan? Window, + bool Force); + +internal sealed record IngestResumeOptions( + ImmutableArray Providers, + string? Checkpoint); + +internal sealed record ReconcileOptions( + ImmutableArray Providers, + TimeSpan? MaxAge); + +internal sealed record InitSummary( + Guid RunId, + DateTimeOffset StartedAt, + DateTimeOffset CompletedAt, + ImmutableArray Providers) +{ + public int ProviderCount => Providers.Length; + public int SuccessCount => Providers.Count(result => string.Equals(result.Status, "succeeded", StringComparison.OrdinalIgnoreCase)); + public int FailureCount => Providers.Count(result => string.Equals(result.Status, "failed", StringComparison.OrdinalIgnoreCase)); +} + +internal sealed record InitProviderResult( + string ProviderId, + string DisplayName, + string Status, + TimeSpan Duration, + string? Error); + +internal sealed record IngestRunSummary( + Guid RunId, + DateTimeOffset StartedAt, + DateTimeOffset CompletedAt, + ImmutableArray Providers) +{ + public int ProviderCount => Providers.Length; + + public int SuccessCount => Providers.Count(provider => string.Equals(provider.Status, "succeeded", StringComparison.OrdinalIgnoreCase)); + + public int FailureCount => Providers.Count(provider => string.Equals(provider.Status, "failed", StringComparison.OrdinalIgnoreCase)); + + public TimeSpan Duration => CompletedAt - StartedAt; +} + +internal sealed record ProviderRunResult( + string ProviderId, + string Status, + int Documents, + int Claims, + DateTimeOffset StartedAt, + DateTimeOffset CompletedAt, + TimeSpan Duration, + string? LastDigest, + DateTimeOffset? LastUpdated, + string? Checkpoint, + string? Error, + DateTimeOffset? Since) +{ + public static ProviderRunResult Missing(string providerId, DateTimeOffset? since) + => new(providerId, "missing", 0, 0, DateTimeOffset.MinValue, DateTimeOffset.MinValue, TimeSpan.Zero, null, null, null, "Provider connector is not registered.", since); +} + +internal sealed record ReconcileSummary( + Guid RunId, + DateTimeOffset StartedAt, + DateTimeOffset CompletedAt, + ImmutableArray Providers) +{ + public int ProviderCount => Providers.Length; + + public int ReconciledCount => Providers.Count(result => string.Equals(result.Action, "reconciled", StringComparison.OrdinalIgnoreCase)); + + public int SkippedCount => Providers.Count(result => string.Equals(result.Action, "skipped", StringComparison.OrdinalIgnoreCase)); + + public int FailureCount => Providers.Count(result => string.Equals(result.Status, "failed", StringComparison.OrdinalIgnoreCase)); + + public TimeSpan Duration => CompletedAt - StartedAt; +} + +internal sealed record ReconcileProviderResult( + string ProviderId, + string Status, + string Action, + DateTimeOffset? LastUpdated, + DateTimeOffset? Threshold, + int Documents, + int Claims, + string? Error); diff --git a/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj b/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj new file mode 100644 index 00000000..17812bc5 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/StellaOps.Excititor.WebService.csproj @@ -0,0 +1,21 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.WebService/TASKS.md b/src/StellaOps.Excititor.WebService/TASKS.md new file mode 100644 index 00000000..01778400 --- /dev/null +++ b/src/StellaOps.Excititor.WebService/TASKS.md @@ -0,0 +1,9 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-WEB-01-001 – Minimal API bootstrap & DI|Team Excititor WebService|EXCITITOR-CORE-01-003, EXCITITOR-STORAGE-01-003|**DONE (2025-10-17)** – Minimal API host composes storage/export/attestation/artifact stores, binds Mongo/attestation options, and exposes `/excititor/status` + health endpoints with regression coverage in `StatusEndpointTests`.| +|EXCITITOR-WEB-01-002 – Ingest & reconcile endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001|**DOING (2025-10-19)** – Prereqs EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, and EXCITITOR-ATTEST-01-001 verified DONE; drafting `/excititor/init`, `/excititor/ingest/run`, `/excititor/ingest/resume`, `/excititor/reconcile` with scope enforcement & structured telemetry plan.| +|EXCITITOR-WEB-01-003 – Export & verify endpoints|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001|**DOING (2025-10-19)** – Prereqs confirmed (EXCITITOR-WEB-01-001, EXCITITOR-EXPORT-01-001, EXCITITOR-ATTEST-01-001); preparing `/excititor/export*` surfaces and `/excititor/verify` with artifact/attestation metadata caching strategy.| +|EXCITITOR-WEB-01-004 – Resolve API & signed responses|Team Excititor WebService|EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-002|**DOING (2025-10-19)** – Prereqs EXCITITOR-WEB-01-001, EXCITITOR-ATTEST-01-001, and EXCITITOR-ATTEST-01-002 verified DONE; planning `/excititor/resolve` signed response flow with consensus envelope + attestation metadata wiring.| +|EXCITITOR-WEB-01-005 – Mirror distribution endpoints|Team Excititor WebService|EXCITITOR-EXPORT-01-007, DEVOPS-MIRROR-08-001|**DONE (2025-10-19)** – `/excititor/mirror` surfaces domain listings, indices, metadata, and downloads with quota/auth checks; tests cover Happy-path listing/download (`dotnet test src/StellaOps.Excititor.WebService.Tests/StellaOps.Excititor.WebService.Tests.csproj`).| diff --git a/src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj b/src/StellaOps.Excititor.Worker.Tests/StellaOps.Excititor.Worker.Tests.csproj similarity index 77% rename from src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj rename to src/StellaOps.Excititor.Worker.Tests/StellaOps.Excititor.Worker.Tests.csproj index 9e3c17ca..1e8a8ab2 100644 --- a/src/StellaOps.Vexer.Worker.Tests/StellaOps.Vexer.Worker.Tests.csproj +++ b/src/StellaOps.Excititor.Worker.Tests/StellaOps.Excititor.Worker.Tests.csproj @@ -1,16 +1,16 @@ - - - net10.0 - enable - enable - - - - - - - - - - - + + + net10.0 + enable + enable + + + + + + + + + + + diff --git a/src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs b/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs similarity index 78% rename from src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs rename to src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs index 8860638c..d4f357f9 100644 --- a/src/StellaOps.Vexer.Worker.Tests/VexWorkerOptionsTests.cs +++ b/src/StellaOps.Excititor.Worker.Tests/VexWorkerOptionsTests.cs @@ -1,77 +1,79 @@ -using FluentAssertions; -using StellaOps.Vexer.Worker.Options; -using StellaOps.Vexer.Worker.Scheduling; -using Xunit; - -namespace StellaOps.Vexer.Worker.Tests; - -public sealed class VexWorkerOptionsTests -{ - [Fact] - public void ResolveSchedules_UsesDefaultIntervalWhenNotSpecified() - { - var options = new VexWorkerOptions - { - DefaultInterval = TimeSpan.FromMinutes(30), - OfflineInterval = TimeSpan.FromHours(6), - OfflineMode = false, - }; - options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:redhat" }); - - var schedules = options.ResolveSchedules(); - - schedules.Should().ContainSingle(); - schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(30)); - } - - [Fact] - public void ResolveSchedules_HonorsOfflineInterval() - { - var options = new VexWorkerOptions - { - DefaultInterval = TimeSpan.FromMinutes(30), - OfflineInterval = TimeSpan.FromHours(8), - OfflineMode = true, - }; - options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:offline" }); - - var schedules = options.ResolveSchedules(); - - schedules.Should().ContainSingle(); - schedules[0].Interval.Should().Be(TimeSpan.FromHours(8)); - } - - [Fact] - public void ResolveSchedules_SkipsDisabledProviders() - { - var options = new VexWorkerOptions(); - options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:enabled" }); - options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "vexer:disabled", Enabled = false }); - - var schedules = options.ResolveSchedules(); - - schedules.Should().HaveCount(1); - schedules[0].ProviderId.Should().Be("vexer:enabled"); - } - - [Fact] - public void ResolveSchedules_UsesProviderIntervalOverride() - { - var options = new VexWorkerOptions - { - DefaultInterval = TimeSpan.FromMinutes(15), - }; - options.Providers.Add(new VexWorkerProviderOptions - { - ProviderId = "vexer:custom", - Interval = TimeSpan.FromMinutes(5), - InitialDelay = TimeSpan.FromSeconds(10), - }); - - var schedules = options.ResolveSchedules(); - - schedules.Should().ContainSingle(); - schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(5)); - schedules[0].InitialDelay.Should().Be(TimeSpan.FromSeconds(10)); - } -} +using FluentAssertions; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Worker.Options; +using StellaOps.Excititor.Worker.Scheduling; +using Xunit; + +namespace StellaOps.Excititor.Worker.Tests; + +public sealed class VexWorkerOptionsTests +{ + [Fact] + public void ResolveSchedules_UsesDefaultIntervalWhenNotSpecified() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(30), + OfflineInterval = TimeSpan.FromHours(6), + OfflineMode = false, + }; + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "excititor:redhat" }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(30)); + schedules[0].Settings.Should().Be(VexConnectorSettings.Empty); + } + + [Fact] + public void ResolveSchedules_HonorsOfflineInterval() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(30), + OfflineInterval = TimeSpan.FromHours(8), + OfflineMode = true, + }; + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "excititor:offline" }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromHours(8)); + } + + [Fact] + public void ResolveSchedules_SkipsDisabledProviders() + { + var options = new VexWorkerOptions(); + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "excititor:enabled" }); + options.Providers.Add(new VexWorkerProviderOptions { ProviderId = "excititor:disabled", Enabled = false }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().HaveCount(1); + schedules[0].ProviderId.Should().Be("excititor:enabled"); + } + + [Fact] + public void ResolveSchedules_UsesProviderIntervalOverride() + { + var options = new VexWorkerOptions + { + DefaultInterval = TimeSpan.FromMinutes(15), + }; + options.Providers.Add(new VexWorkerProviderOptions + { + ProviderId = "excititor:custom", + Interval = TimeSpan.FromMinutes(5), + InitialDelay = TimeSpan.FromSeconds(10), + }); + + var schedules = options.ResolveSchedules(); + + schedules.Should().ContainSingle(); + schedules[0].Interval.Should().Be(TimeSpan.FromMinutes(5)); + schedules[0].InitialDelay.Should().Be(TimeSpan.FromSeconds(10)); + } +} diff --git a/src/StellaOps.Vexer.Worker/AGENTS.md b/src/StellaOps.Excititor.Worker/AGENTS.md similarity index 90% rename from src/StellaOps.Vexer.Worker/AGENTS.md rename to src/StellaOps.Excititor.Worker/AGENTS.md index d49907cc..bb042579 100644 --- a/src/StellaOps.Vexer.Worker/AGENTS.md +++ b/src/StellaOps.Excititor.Worker/AGENTS.md @@ -1,23 +1,23 @@ -# AGENTS -## Role -Background processing host coordinating scheduled pulls, retries, reconciliation, verification, and cache maintenance for Vexer. -## Scope -- Hosted service (Worker Service) wiring timers/queues for provider pulls and reconciliation cycles. -- Resume token management, retry policies, and failure quarantines for connectors. -- Re-verification of stored attestations and cache garbage collection routines. -- Operational metrics and structured logging for offline-friendly monitoring. -## Participants -- Triggered by WebService job requests or internal schedules to run connector pulls. -- Collaborates with Storage.Mongo repositories and Attestation verification utilities. -- Emits telemetry consumed by observability stack and CLI status queries. -## Interfaces & contracts -- Scheduler abstractions, provider run controllers, retry/backoff strategies, and queue processors. -- Hooks for policy revision changes and cache GC thresholds. -## In/Out of scope -In: background orchestration, job lifecycle management, observability for worker operations. -Out: HTTP endpoint definitions, domain modeling, connector-specific parsing logic. -## Observability & security expectations -- Publish metrics for pull latency, failure counts, retry depth, cache size, and verification outcomes. -- Log correlation IDs & provider IDs; avoid leaking secret config values. -## Tests -- Worker orchestration tests, timer controls, and retry behavior will live in `../StellaOps.Vexer.Worker.Tests`. +# AGENTS +## Role +Background processing host coordinating scheduled pulls, retries, reconciliation, verification, and cache maintenance for Excititor. +## Scope +- Hosted service (Worker Service) wiring timers/queues for provider pulls and reconciliation cycles. +- Resume token management, retry policies, and failure quarantines for connectors. +- Re-verification of stored attestations and cache garbage collection routines. +- Operational metrics and structured logging for offline-friendly monitoring. +## Participants +- Triggered by WebService job requests or internal schedules to run connector pulls. +- Collaborates with Storage.Mongo repositories and Attestation verification utilities. +- Emits telemetry consumed by observability stack and CLI status queries. +## Interfaces & contracts +- Scheduler abstractions, provider run controllers, retry/backoff strategies, and queue processors. +- Hooks for policy revision changes and cache GC thresholds. +## In/Out of scope +In: background orchestration, job lifecycle management, observability for worker operations. +Out: HTTP endpoint definitions, domain modeling, connector-specific parsing logic. +## Observability & security expectations +- Publish metrics for pull latency, failure counts, retry depth, cache size, and verification outcomes. +- Log correlation IDs & provider IDs; avoid leaking secret config values. +## Tests +- Worker orchestration tests, timer controls, and retry behavior will live in `../StellaOps.Excititor.Worker.Tests`. diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs similarity index 71% rename from src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs rename to src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs index e8a79079..13fe3d83 100644 --- a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptions.cs +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptions.cs @@ -1,62 +1,72 @@ -using System.Collections.Generic; -using StellaOps.Vexer.Worker.Scheduling; - -namespace StellaOps.Vexer.Worker.Options; - -public sealed class VexWorkerOptions -{ - public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1); - - public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6); - - public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5); - - public bool OfflineMode { get; set; } - - public IList Providers { get; } = new List(); - - internal IReadOnlyList ResolveSchedules() - { - var schedules = new List(); - foreach (var provider in Providers) - { - if (!provider.Enabled) - { - continue; - } - - var providerId = provider.ProviderId?.Trim(); - if (string.IsNullOrWhiteSpace(providerId)) - { - continue; - } - - var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval); - if (interval <= TimeSpan.Zero) - { - continue; - } - - var initialDelay = provider.InitialDelay ?? DefaultInitialDelay; - if (initialDelay < TimeSpan.Zero) - { - initialDelay = TimeSpan.Zero; - } - - schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay)); - } - - return schedules; - } -} - -public sealed class VexWorkerProviderOptions -{ - public string ProviderId { get; set; } = string.Empty; - - public bool Enabled { get; set; } = true; - - public TimeSpan? Interval { get; set; } - - public TimeSpan? InitialDelay { get; set; } -} +using System.Collections.Generic; +using System.Collections.Immutable; +using StellaOps.Excititor.Worker.Scheduling; +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Worker.Options; + +public sealed class VexWorkerOptions +{ + public TimeSpan DefaultInterval { get; set; } = TimeSpan.FromHours(1); + + public TimeSpan OfflineInterval { get; set; } = TimeSpan.FromHours(6); + + public TimeSpan DefaultInitialDelay { get; set; } = TimeSpan.FromMinutes(5); + + public bool OfflineMode { get; set; } + + public IList Providers { get; } = new List(); + + public VexWorkerRetryOptions Retry { get; } = new(); + + internal IReadOnlyList ResolveSchedules() + { + var schedules = new List(); + foreach (var provider in Providers) + { + if (!provider.Enabled) + { + continue; + } + + var providerId = provider.ProviderId?.Trim(); + if (string.IsNullOrWhiteSpace(providerId)) + { + continue; + } + + var interval = provider.Interval ?? (OfflineMode ? OfflineInterval : DefaultInterval); + if (interval <= TimeSpan.Zero) + { + continue; + } + + var initialDelay = provider.InitialDelay ?? DefaultInitialDelay; + if (initialDelay < TimeSpan.Zero) + { + initialDelay = TimeSpan.Zero; + } + + var connectorSettings = provider.Settings.Count == 0 + ? VexConnectorSettings.Empty + : new VexConnectorSettings(provider.Settings.ToImmutableDictionary(StringComparer.Ordinal)); + + schedules.Add(new VexWorkerSchedule(providerId, interval, initialDelay, connectorSettings)); + } + + return schedules; + } +} + +public sealed class VexWorkerProviderOptions +{ + public string ProviderId { get; set; } = string.Empty; + + public bool Enabled { get; set; } = true; + + public TimeSpan? Interval { get; set; } + + public TimeSpan? InitialDelay { get; set; } + + public IDictionary Settings { get; } = new Dictionary(StringComparer.Ordinal); +} diff --git a/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs new file mode 100644 index 00000000..b2749e52 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerOptionsValidator.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using Microsoft.Extensions.Options; + +namespace StellaOps.Excititor.Worker.Options; + +internal sealed class VexWorkerOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, VexWorkerOptions options) + { + var failures = new List(); + + if (options.DefaultInterval <= TimeSpan.Zero) + { + failures.Add("Excititor.Worker.DefaultInterval must be greater than zero."); + } + + if (options.OfflineInterval <= TimeSpan.Zero) + { + failures.Add("Excititor.Worker.OfflineInterval must be greater than zero."); + } + + if (options.DefaultInitialDelay < TimeSpan.Zero) + { + failures.Add("Excititor.Worker.DefaultInitialDelay cannot be negative."); + } + + if (options.Retry.BaseDelay <= TimeSpan.Zero) + { + failures.Add("Excititor.Worker.Retry.BaseDelay must be greater than zero."); + } + + if (options.Retry.MaxDelay < options.Retry.BaseDelay) + { + failures.Add("Excititor.Worker.Retry.MaxDelay must be greater than or equal to BaseDelay."); + } + + if (options.Retry.QuarantineDuration <= TimeSpan.Zero) + { + failures.Add("Excititor.Worker.Retry.QuarantineDuration must be greater than zero."); + } + + if (options.Retry.FailureThreshold < 1) + { + failures.Add("Excititor.Worker.Retry.FailureThreshold must be at least 1."); + } + + if (options.Retry.JitterRatio < 0 || options.Retry.JitterRatio > 1) + { + failures.Add("Excititor.Worker.Retry.JitterRatio must be between 0 and 1."); + } + + if (options.Retry.RetryCap < options.Retry.BaseDelay) + { + failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to BaseDelay."); + } + + if (options.Retry.RetryCap < options.Retry.MaxDelay) + { + failures.Add("Excititor.Worker.Retry.RetryCap must be greater than or equal to MaxDelay."); + } + + for (var i = 0; i < options.Providers.Count; i++) + { + var provider = options.Providers[i]; + if (string.IsNullOrWhiteSpace(provider.ProviderId)) + { + failures.Add($"Excititor.Worker.Providers[{i}].ProviderId must be set."); + } + + if (provider.Interval is { } interval && interval <= TimeSpan.Zero) + { + failures.Add($"Excititor.Worker.Providers[{i}].Interval must be greater than zero when specified."); + } + + if (provider.InitialDelay is { } delay && delay < TimeSpan.Zero) + { + failures.Add($"Excititor.Worker.Providers[{i}].InitialDelay cannot be negative."); + } + } + + return failures.Count > 0 + ? ValidateOptionsResult.Fail(failures) + : ValidateOptionsResult.Success; + } +} diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerPluginOptions.cs similarity index 80% rename from src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs rename to src/StellaOps.Excititor.Worker/Options/VexWorkerPluginOptions.cs index 487cd448..e4a25991 100644 --- a/src/StellaOps.Vexer.Worker/Options/VexWorkerPluginOptions.cs +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerPluginOptions.cs @@ -1,21 +1,21 @@ -using System; -using System.IO; - -namespace StellaOps.Vexer.Worker.Options; - -public sealed class VexWorkerPluginOptions -{ - public string? Directory { get; set; } - - public string? SearchPattern { get; set; } - - internal string ResolveDirectory() - => string.IsNullOrWhiteSpace(Directory) - ? Path.Combine(AppContext.BaseDirectory, "plugins") - : Path.GetFullPath(Directory); - - internal string ResolveSearchPattern() - => string.IsNullOrWhiteSpace(SearchPattern) - ? "StellaOps.Vexer.Connectors.*.dll" - : SearchPattern!; -} +using System; +using System.IO; + +namespace StellaOps.Excititor.Worker.Options; + +public sealed class VexWorkerPluginOptions +{ + public string? Directory { get; set; } + + public string? SearchPattern { get; set; } + + internal string ResolveDirectory() + => string.IsNullOrWhiteSpace(Directory) + ? Path.Combine(AppContext.BaseDirectory, "plugins") + : Path.GetFullPath(Directory); + + internal string ResolveSearchPattern() + => string.IsNullOrWhiteSpace(SearchPattern) + ? "StellaOps.Excititor.Connectors.*.dll" + : SearchPattern!; +} diff --git a/src/StellaOps.Excititor.Worker/Options/VexWorkerRetryOptions.cs b/src/StellaOps.Excititor.Worker/Options/VexWorkerRetryOptions.cs new file mode 100644 index 00000000..140030f2 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Options/VexWorkerRetryOptions.cs @@ -0,0 +1,20 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Excititor.Worker.Options; + +public sealed class VexWorkerRetryOptions +{ + [Range(1, int.MaxValue)] + public int FailureThreshold { get; set; } = 3; + + [Range(typeof(double), "0.0", "1.0")] + public double JitterRatio { get; set; } = 0.2; + + public TimeSpan BaseDelay { get; set; } = TimeSpan.FromMinutes(5); + + public TimeSpan MaxDelay { get; set; } = TimeSpan.FromHours(6); + + public TimeSpan QuarantineDuration { get; set; } = TimeSpan.FromHours(12); + + public TimeSpan RetryCap { get; set; } = TimeSpan.FromHours(24); +} diff --git a/src/StellaOps.Vexer.Worker/Program.cs b/src/StellaOps.Excititor.Worker/Program.cs similarity index 57% rename from src/StellaOps.Vexer.Worker/Program.cs rename to src/StellaOps.Excititor.Worker/Program.cs index 247c4f39..8ca6adac 100644 --- a/src/StellaOps.Vexer.Worker/Program.cs +++ b/src/StellaOps.Excititor.Worker/Program.cs @@ -1,59 +1,74 @@ -using System.IO; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Plugin; -using StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; -using StellaOps.Vexer.Worker.Options; -using StellaOps.Vexer.Worker.Scheduling; - -var builder = Host.CreateApplicationBuilder(args); -var services = builder.Services; -var configuration = builder.Configuration; -services.AddOptions() - .Bind(configuration.GetSection("Vexer:Worker")) - .ValidateOnStart(); - -services.Configure(configuration.GetSection("Vexer:Worker:Plugins")); -services.AddRedHatCsafConnector(); - -services.AddSingleton, VexWorkerOptionsValidator>(); -services.AddSingleton(TimeProvider.System); -services.PostConfigure(options => -{ - if (!options.Providers.Any(provider => string.Equals(provider.ProviderId, "vexer:redhat", StringComparison.OrdinalIgnoreCase))) - { - options.Providers.Add(new VexWorkerProviderOptions - { - ProviderId = "vexer:redhat", - }); - } -}); -services.AddSingleton(provider => -{ - var pluginOptions = provider.GetRequiredService>().Value; - var catalog = new PluginCatalog(); - - var directory = pluginOptions.ResolveDirectory(); - if (Directory.Exists(directory)) - { - catalog.AddFromDirectory(directory, pluginOptions.ResolveSearchPattern()); - } - else - { - var logger = provider.GetRequiredService>(); - logger.LogWarning("Vexer worker plugin directory '{Directory}' does not exist; proceeding without external connectors.", directory); - } - - return catalog; -}); - -services.AddSingleton(); -services.AddHostedService(); - -var host = builder.Build(); -await host.RunAsync(); - -public partial class Program; +using System.IO; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Plugin; +using StellaOps.Excititor.Connectors.RedHat.CSAF.DependencyInjection; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Formats.CSAF; +using StellaOps.Excititor.Formats.CycloneDX; +using StellaOps.Excititor.Formats.OpenVEX; +using StellaOps.Excititor.Storage.Mongo; +using StellaOps.Excititor.Worker.Options; +using StellaOps.Excititor.Worker.Scheduling; + +var builder = Host.CreateApplicationBuilder(args); +var services = builder.Services; +var configuration = builder.Configuration; +services.AddOptions() + .Bind(configuration.GetSection("Excititor:Worker")) + .ValidateOnStart(); + +services.Configure(configuration.GetSection("Excititor:Worker:Plugins")); +services.AddRedHatCsafConnector(); + +services.AddOptions() + .Bind(configuration.GetSection("Excititor:Storage:Mongo")) + .ValidateOnStart(); + +services.AddExcititorMongoStorage(); +services.AddCsafNormalizer(); +services.AddCycloneDxNormalizer(); +services.AddOpenVexNormalizer(); +services.AddSingleton(); + +services.AddSingleton, VexWorkerOptionsValidator>(); +services.AddSingleton(TimeProvider.System); +services.PostConfigure(options => +{ + if (!options.Providers.Any(provider => string.Equals(provider.ProviderId, "excititor:redhat", StringComparison.OrdinalIgnoreCase))) + { + options.Providers.Add(new VexWorkerProviderOptions + { + ProviderId = "excititor:redhat", + }); + } +}); +services.AddSingleton(provider => +{ + var pluginOptions = provider.GetRequiredService>().Value; + var catalog = new PluginCatalog(); + + var directory = pluginOptions.ResolveDirectory(); + if (Directory.Exists(directory)) + { + catalog.AddFromDirectory(directory, pluginOptions.ResolveSearchPattern()); + } + else + { + var logger = provider.GetRequiredService>(); + logger.LogWarning("Excititor worker plugin directory '{Directory}' does not exist; proceeding without external connectors.", directory); + } + + return catalog; +}); + +services.AddSingleton(); +services.AddHostedService(); + +var host = builder.Build(); +await host.RunAsync(); + +public partial class Program; diff --git a/src/StellaOps.Excititor.Worker/Properties/AssemblyInfo.cs b/src/StellaOps.Excititor.Worker/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..209b9185 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Excititor.Worker.Tests")] diff --git a/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs new file mode 100644 index 00000000..907f85f8 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Scheduling/DefaultVexProviderRunner.cs @@ -0,0 +1,116 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin; +using StellaOps.Excititor.Core; +using StellaOps.Excititor.Storage.Mongo; + +namespace StellaOps.Excititor.Worker.Scheduling; + +internal sealed class DefaultVexProviderRunner : IVexProviderRunner +{ + private readonly IServiceProvider _serviceProvider; + private readonly PluginCatalog _pluginCatalog; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public DefaultVexProviderRunner( + IServiceProvider serviceProvider, + PluginCatalog pluginCatalog, + ILogger logger, + TimeProvider timeProvider) + { + _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); + _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public async ValueTask RunAsync(string providerId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(providerId); + + using var scope = _serviceProvider.CreateScope(); + var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider); + var matched = availablePlugins.FirstOrDefault(plugin => + string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase)); + + if (matched is not null) + { + _logger.LogInformation( + "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", + matched.Name, + providerId); + } + else + { + _logger.LogInformation("No legacy connector plugin registered for provider {ProviderId}; falling back to DI-managed connectors.", providerId); + } + + var connectors = scope.ServiceProvider.GetServices(); + var connector = connectors.FirstOrDefault(c => string.Equals(c.Id, providerId, StringComparison.OrdinalIgnoreCase)); + + if (connector is null) + { + _logger.LogWarning("No IVexConnector implementation registered for provider {ProviderId}; skipping run.", providerId); + return; + } + + await ExecuteConnectorAsync(scope.ServiceProvider, connector, cancellationToken).ConfigureAwait(false); + } + + private async Task ExecuteConnectorAsync(IServiceProvider scopeProvider, IVexConnector connector, CancellationToken cancellationToken) + { + var rawStore = scopeProvider.GetRequiredService(); + var claimStore = scopeProvider.GetRequiredService(); + var providerStore = scopeProvider.GetRequiredService(); + var normalizerRouter = scopeProvider.GetRequiredService(); + var signatureVerifier = scopeProvider.GetRequiredService(); + var sessionProvider = scopeProvider.GetRequiredService(); + var session = await sessionProvider.StartSessionAsync(cancellationToken).ConfigureAwait(false); + + var descriptor = connector switch + { + VexConnectorBase baseConnector => baseConnector.Descriptor, + _ => new VexConnectorDescriptor(connector.Id, VexProviderKind.Vendor, connector.Id) + }; + + var provider = await providerStore.FindAsync(descriptor.Id, cancellationToken, session).ConfigureAwait(false) + ?? new VexProvider(descriptor.Id, descriptor.DisplayName, descriptor.Kind); + + await providerStore.SaveAsync(provider, cancellationToken, session).ConfigureAwait(false); + + await connector.ValidateAsync(VexConnectorSettings.Empty, cancellationToken).ConfigureAwait(false); + + var context = new VexConnectorContext( + Since: null, + Settings: VexConnectorSettings.Empty, + RawSink: rawStore, + SignatureVerifier: signatureVerifier, + Normalizers: normalizerRouter, + Services: scopeProvider); + + var documentCount = 0; + var claimCount = 0; + + await foreach (var document in connector.FetchAsync(context, cancellationToken)) + { + documentCount++; + + var batch = await normalizerRouter.NormalizeAsync(document, cancellationToken).ConfigureAwait(false); + if (!batch.Claims.IsDefaultOrEmpty && batch.Claims.Length > 0) + { + claimCount += batch.Claims.Length; + await claimStore.AppendAsync(batch.Claims, _timeProvider.GetUtcNow(), cancellationToken, session).ConfigureAwait(false); + } + } + + _logger.LogInformation( + "Connector {ConnectorId} persisted {DocumentCount} document(s) and {ClaimCount} claim(s) this run.", + connector.Id, + documentCount, + claimCount); + } +} diff --git a/src/StellaOps.Excititor.Worker/Scheduling/IVexProviderRunner.cs b/src/StellaOps.Excititor.Worker/Scheduling/IVexProviderRunner.cs new file mode 100644 index 00000000..ddadbfd3 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Scheduling/IVexProviderRunner.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Excititor.Worker.Scheduling; + +internal interface IVexProviderRunner +{ + ValueTask RunAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs b/src/StellaOps.Excititor.Worker/Scheduling/VexWorkerHostedService.cs similarity index 87% rename from src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs rename to src/StellaOps.Excititor.Worker/Scheduling/VexWorkerHostedService.cs index b37bbe38..4884d3b6 100644 --- a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerHostedService.cs +++ b/src/StellaOps.Excititor.Worker/Scheduling/VexWorkerHostedService.cs @@ -1,110 +1,110 @@ -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Worker.Options; - -namespace StellaOps.Vexer.Worker.Scheduling; - -internal sealed class VexWorkerHostedService : BackgroundService -{ - private readonly IOptions _options; - private readonly IVexProviderRunner _runner; - private readonly ILogger _logger; - private readonly TimeProvider _timeProvider; - - public VexWorkerHostedService( - IOptions options, - IVexProviderRunner runner, - ILogger logger, - TimeProvider timeProvider) - { - _options = options ?? throw new ArgumentNullException(nameof(options)); - _runner = runner ?? throw new ArgumentNullException(nameof(runner)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - } - - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - var schedules = _options.Value.ResolveSchedules(); - if (schedules.Count == 0) - { - _logger.LogWarning("Vexer worker has no configured provider schedules; the service will remain idle."); - await Task.CompletedTask; - return; - } - - _logger.LogInformation("Vexer worker starting with {ProviderCount} provider schedule(s).", schedules.Count); - - var tasks = new List(schedules.Count); - foreach (var schedule in schedules) - { - tasks.Add(RunScheduleAsync(schedule, stoppingToken)); - } - - await Task.WhenAll(tasks); - } - - private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken) - { - try - { - if (schedule.InitialDelay > TimeSpan.Zero) - { - _logger.LogInformation( - "Provider {ProviderId} initial delay of {InitialDelay} before first execution.", - schedule.ProviderId, - schedule.InitialDelay); - - await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false); - } - - using var timer = new PeriodicTimer(schedule.Interval); - do - { - var startedAt = _timeProvider.GetUtcNow(); - _logger.LogInformation( - "Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.", - schedule.ProviderId, - startedAt, - schedule.Interval); - - try - { - await _runner.RunAsync(schedule.ProviderId, cancellationToken).ConfigureAwait(false); - - var completedAt = _timeProvider.GetUtcNow(); - var elapsed = completedAt - startedAt; - - _logger.LogInformation( - "Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).", - schedule.ProviderId, - completedAt, - elapsed); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - _logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId); - break; - } - catch (Exception ex) - { - _logger.LogError( - ex, - "Provider {ProviderId} run failed: {Message}", - schedule.ProviderId, - ex.Message); - } - } - while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - _logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId); - } - } -} +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Excititor.Worker.Options; + +namespace StellaOps.Excititor.Worker.Scheduling; + +internal sealed class VexWorkerHostedService : BackgroundService +{ + private readonly IOptions _options; + private readonly IVexProviderRunner _runner; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public VexWorkerHostedService( + IOptions options, + IVexProviderRunner runner, + ILogger logger, + TimeProvider timeProvider) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _runner = runner ?? throw new ArgumentNullException(nameof(runner)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var schedules = _options.Value.ResolveSchedules(); + if (schedules.Count == 0) + { + _logger.LogWarning("Excititor worker has no configured provider schedules; the service will remain idle."); + await Task.CompletedTask; + return; + } + + _logger.LogInformation("Excititor worker starting with {ProviderCount} provider schedule(s).", schedules.Count); + + var tasks = new List(schedules.Count); + foreach (var schedule in schedules) + { + tasks.Add(RunScheduleAsync(schedule, stoppingToken)); + } + + await Task.WhenAll(tasks); + } + + private async Task RunScheduleAsync(VexWorkerSchedule schedule, CancellationToken cancellationToken) + { + try + { + if (schedule.InitialDelay > TimeSpan.Zero) + { + _logger.LogInformation( + "Provider {ProviderId} initial delay of {InitialDelay} before first execution.", + schedule.ProviderId, + schedule.InitialDelay); + + await Task.Delay(schedule.InitialDelay, cancellationToken).ConfigureAwait(false); + } + + using var timer = new PeriodicTimer(schedule.Interval); + do + { + var startedAt = _timeProvider.GetUtcNow(); + _logger.LogInformation( + "Provider {ProviderId} run started at {StartedAt}. Interval={Interval}.", + schedule.ProviderId, + startedAt, + schedule.Interval); + + try + { + await _runner.RunAsync(schedule, cancellationToken).ConfigureAwait(false); + + var completedAt = _timeProvider.GetUtcNow(); + var elapsed = completedAt - startedAt; + + _logger.LogInformation( + "Provider {ProviderId} run completed at {CompletedAt} (duration {Duration}).", + schedule.ProviderId, + completedAt, + elapsed); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Provider {ProviderId} run cancelled.", schedule.ProviderId); + break; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Provider {ProviderId} run failed: {Message}", + schedule.ProviderId, + ex.Message); + } + } + while (await timer.WaitForNextTickAsync(cancellationToken).ConfigureAwait(false)); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + _logger.LogInformation("Provider {ProviderId} schedule cancelled.", schedule.ProviderId); + } + } +} diff --git a/src/StellaOps.Excititor.Worker/Scheduling/VexWorkerSchedule.cs b/src/StellaOps.Excititor.Worker/Scheduling/VexWorkerSchedule.cs new file mode 100644 index 00000000..12bd1561 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/Scheduling/VexWorkerSchedule.cs @@ -0,0 +1,5 @@ +using StellaOps.Excititor.Core; + +namespace StellaOps.Excititor.Worker.Scheduling; + +internal sealed record VexWorkerSchedule(string ProviderId, TimeSpan Interval, TimeSpan InitialDelay, VexConnectorSettings Settings); diff --git a/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj new file mode 100644 index 00000000..2c694928 --- /dev/null +++ b/src/StellaOps.Excititor.Worker/StellaOps.Excititor.Worker.csproj @@ -0,0 +1,22 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Excititor.Worker/TASKS.md b/src/StellaOps.Excititor.Worker/TASKS.md new file mode 100644 index 00000000..871dde4f --- /dev/null +++ b/src/StellaOps.Excititor.Worker/TASKS.md @@ -0,0 +1,9 @@ +If you are working on this file you need to read docs/ARCHITECTURE_EXCITITOR.md and ./AGENTS.md). +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|EXCITITOR-WORKER-01-001 – Worker host & scheduling|Team Excititor Worker|EXCITITOR-STORAGE-01-003, EXCITITOR-WEB-01-001|**DONE (2025-10-17)** – Worker project bootstraps provider schedules from configuration, integrates plugin catalog discovery, and emits structured logs/metrics-ready events via `VexWorkerHostedService`; scheduling logic covered by `VexWorkerOptionsTests`.| +|EXCITITOR-WORKER-01-002 – Resume tokens & retry policy|Team Excititor Worker|EXCITITOR-WORKER-01-001|DOING (2025-10-19) – Prereq EXCITITOR-WORKER-01-001 closed 2025-10-17; implementing durable resume markers, jittered backoff, and failure quarantine flow.| +|EXCITITOR-WORKER-01-003 – Verification & cache GC loops|Team Excititor Worker|EXCITITOR-WORKER-01-001, EXCITITOR-ATTEST-01-003, EXCITITOR-EXPORT-01-002|TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios.| +|EXCITITOR-WORKER-01-004 – TTL refresh & stability damper|Team Excititor Worker|EXCITITOR-WORKER-01-001, EXCITITOR-CORE-02-001|DOING (2025-10-19) – Prereqs EXCITITOR-WORKER-01-001 (closed 2025-10-17) and EXCITITOR-CORE-02-001 (closed 2025-10-19) verified; building TTL monitor with dampers and re-resolve triggers.| +|EXCITITOR-WORKER-02-001 – Resolve Microsoft.Extensions.Caching.Memory advisory|Team Excititor Worker|EXCITITOR-WORKER-01-001|DOING (2025-10-19) – Prereq EXCITITOR-WORKER-01-001 closed 2025-10-17; upgrading `Microsoft.Extensions.Caching.Memory` stack and refreshing lockfiles/tests to clear NU1903.| diff --git a/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs b/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs deleted file mode 100644 index ee3914d7..00000000 --- a/src/StellaOps.Feedser.Core/Jobs/IJobStore.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace StellaOps.Feedser.Core.Jobs; - -public interface IJobStore -{ - Task CreateAsync(JobRunCreateRequest request, CancellationToken cancellationToken); - - Task TryStartAsync(Guid runId, DateTimeOffset startedAt, CancellationToken cancellationToken); - - Task TryCompleteAsync(Guid runId, JobRunCompletion completion, CancellationToken cancellationToken); - - Task FindAsync(Guid runId, CancellationToken cancellationToken); - - Task> GetRecentRunsAsync(string? kind, int limit, CancellationToken cancellationToken); - - Task> GetActiveRunsAsync(CancellationToken cancellationToken); - - Task GetLastRunAsync(string kind, CancellationToken cancellationToken); - - Task> GetLastRunsAsync(IEnumerable kinds, CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs b/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs deleted file mode 100644 index 1a60c0f3..00000000 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/JsonFeedExporterTests.cs +++ /dev/null @@ -1,311 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Exporter.Json; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Advisories; -using StellaOps.Feedser.Storage.Mongo.Exporting; - -namespace StellaOps.Feedser.Exporter.Json.Tests; - -public sealed class JsonFeedExporterTests : IDisposable -{ - private readonly string _root; - - public JsonFeedExporterTests() - { - _root = Directory.CreateTempSubdirectory("feedser-json-exporter-tests").FullName; - } - - [Fact] - public async Task ExportAsync_SkipsWhenDigestUnchanged() - { - var advisory = new Advisory( - advisoryKey: "CVE-2024-1234", - title: "Test Advisory", - summary: null, - language: "en", - published: DateTimeOffset.Parse("2024-01-01T00:00:00Z", CultureInfo.InvariantCulture), - modified: DateTimeOffset.Parse("2024-01-02T00:00:00Z", CultureInfo.InvariantCulture), - severity: "high", - exploitKnown: false, - aliases: new[] { "CVE-2024-1234" }, - references: Array.Empty(), - affectedPackages: Array.Empty(), - cvssMetrics: Array.Empty(), - provenance: Array.Empty()); - - var advisoryStore = new StubAdvisoryStore(advisory); - var options = Options.Create(new JsonExportOptions - { - OutputRoot = _root, - MaintainLatestSymlink = false, - }); - - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(DateTimeOffset.Parse("2024-07-15T12:00:00Z", CultureInfo.InvariantCulture)); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var exporter = new JsonFeedExporter( - advisoryStore, - options, - new VulnListJsonExportPathResolver(), - stateManager, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(record); - var firstUpdated = record!.UpdatedAt; - Assert.Equal("20240715T120000Z", record.BaseExportId); - Assert.Equal(record.LastFullDigest, record.ExportCursor); - - var firstExportPath = Path.Combine(_root, "20240715T120000Z"); - Assert.True(Directory.Exists(firstExportPath)); - - timeProvider.Advance(TimeSpan.FromMinutes(5)); - await exporter.ExportAsync(provider, CancellationToken.None); - - record = await stateStore.FindAsync(JsonFeedExporter.ExporterId, CancellationToken.None); - Assert.NotNull(record); - Assert.Equal(firstUpdated, record!.UpdatedAt); - - var secondExportPath = Path.Combine(_root, "20240715T120500Z"); - Assert.False(Directory.Exists(secondExportPath)); - } - - [Fact] - public async Task ExportAsync_WritesManifestMetadata() - { - var exportedAt = DateTimeOffset.Parse("2024-08-10T00:00:00Z", CultureInfo.InvariantCulture); - var recordedAt = DateTimeOffset.Parse("2024-07-02T00:00:00Z", CultureInfo.InvariantCulture); - var reference = new AdvisoryReference( - "http://Example.com/path/resource?b=2&a=1", - kind: "advisory", - sourceTag: "REF-001", - summary: "Primary vendor advisory", - provenance: new AdvisoryProvenance("ghsa", "map", "REF-001", recordedAt, new[] { ProvenanceFieldMasks.References })); - var weakness = new AdvisoryWeakness( - taxonomy: "cwe", - identifier: "CWE-79", - name: "Cross-site Scripting", - uri: "https://cwe.mitre.org/data/definitions/79.html", - provenance: new[] - { - new AdvisoryProvenance("nvd", "map", "CWE-79", recordedAt, new[] { ProvenanceFieldMasks.Weaknesses }) - }); - var cvssMetric = new CvssMetric( - "3.1", - "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", - 9.8, - "critical", - new AdvisoryProvenance("nvd", "map", "CVE-2024-4321", recordedAt, new[] { ProvenanceFieldMasks.CvssMetrics })); - - var advisory = new Advisory( - advisoryKey: "CVE-2024-4321", - title: "Manifest Test", - summary: "Short summary", - language: "en", - published: DateTimeOffset.Parse("2024-07-01T00:00:00Z", CultureInfo.InvariantCulture), - modified: recordedAt, - severity: "medium", - exploitKnown: false, - aliases: new[] { "CVE-2024-4321", "GHSA-xxxx-yyyy-zzzz" }, - credits: Array.Empty(), - references: new[] { reference }, - affectedPackages: Array.Empty(), - cvssMetrics: new[] { cvssMetric }, - provenance: new[] - { - new AdvisoryProvenance("ghsa", "map", "GHSA-xxxx-yyyy-zzzz", recordedAt, new[] { ProvenanceFieldMasks.Advisory }), - new AdvisoryProvenance("nvd", "map", "CVE-2024-4321", recordedAt, new[] { ProvenanceFieldMasks.Advisory }) - }, - description: "Detailed description capturing remediation steps.", - cwes: new[] { weakness }, - canonicalMetricId: "3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H"); - - var advisoryStore = new StubAdvisoryStore(advisory); - var optionsValue = new JsonExportOptions - { - OutputRoot = _root, - MaintainLatestSymlink = false, - }; - - var options = Options.Create(optionsValue); - var stateStore = new InMemoryExportStateStore(); - var timeProvider = new TestTimeProvider(exportedAt); - var stateManager = new ExportStateManager(stateStore, timeProvider); - var exporter = new JsonFeedExporter( - advisoryStore, - options, - new VulnListJsonExportPathResolver(), - stateManager, - NullLogger.Instance, - timeProvider); - - using var provider = new ServiceCollection().BuildServiceProvider(); - await exporter.ExportAsync(provider, CancellationToken.None); - - var exportId = exportedAt.ToString(optionsValue.DirectoryNameFormat, CultureInfo.InvariantCulture); - var exportDirectory = Path.Combine(_root, exportId); - var manifestPath = Path.Combine(exportDirectory, "manifest.json"); - - Assert.True(File.Exists(manifestPath)); - - using var document = JsonDocument.Parse(await File.ReadAllBytesAsync(manifestPath, CancellationToken.None)); - var root = document.RootElement; - - Assert.Equal(exportId, root.GetProperty("exportId").GetString()); - Assert.Equal(exportedAt.UtcDateTime, root.GetProperty("generatedAt").GetDateTime()); - Assert.Equal(1, root.GetProperty("advisoryCount").GetInt32()); - - var exportedFiles = Directory.EnumerateFiles(exportDirectory, "*.json", SearchOption.AllDirectories) - .Select(path => new - { - Absolute = path, - Relative = Path.GetRelativePath(exportDirectory, path).Replace("\\", "/", StringComparison.Ordinal), - }) - .Where(file => !string.Equals(file.Relative, "manifest.json", StringComparison.OrdinalIgnoreCase)) - .OrderBy(file => file.Relative, StringComparer.Ordinal) - .ToArray(); - - var filesElement = root.GetProperty("files") - .EnumerateArray() - .Select(element => new - { - Path = element.GetProperty("path").GetString(), - Bytes = element.GetProperty("bytes").GetInt64(), - Digest = element.GetProperty("digest").GetString(), - }) - .OrderBy(file => file.Path, StringComparer.Ordinal) - .ToArray(); - - var dataFile = Assert.Single(exportedFiles); - using (var advisoryDocument = JsonDocument.Parse(await File.ReadAllBytesAsync(dataFile.Absolute, CancellationToken.None))) - { - var advisoryRoot = advisoryDocument.RootElement; - Assert.Equal("Detailed description capturing remediation steps.", advisoryRoot.GetProperty("description").GetString()); - Assert.Equal("3.1|CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:H", advisoryRoot.GetProperty("canonicalMetricId").GetString()); - - var referenceElement = advisoryRoot.GetProperty("references").EnumerateArray().Single(); - Assert.Equal(reference.Url, referenceElement.GetProperty("url").GetString(), StringComparer.OrdinalIgnoreCase); - - var weaknessElement = advisoryRoot.GetProperty("cwes").EnumerateArray().Single(); - Assert.Equal("cwe", weaknessElement.GetProperty("taxonomy").GetString()); - Assert.Equal("CWE-79", weaknessElement.GetProperty("identifier").GetString()); - } - - Assert.Equal(exportedFiles.Select(file => file.Relative).ToArray(), filesElement.Select(file => file.Path).ToArray()); - - long totalBytes = exportedFiles.Select(file => new FileInfo(file.Absolute).Length).Sum(); - Assert.Equal(totalBytes, root.GetProperty("totalBytes").GetInt64()); - Assert.Equal(exportedFiles.Length, root.GetProperty("fileCount").GetInt32()); - - var digest = root.GetProperty("digest").GetString(); - var digestResult = new JsonExportResult( - exportDirectory, - exportedAt, - exportedFiles.Select(file => - { - var manifestEntry = filesElement.First(f => f.Path == file.Relative); - if (manifestEntry.Digest is null) - { - throw new InvalidOperationException($"Manifest entry for {file.Relative} missing digest."); - } - - return new JsonExportFile(file.Relative, new FileInfo(file.Absolute).Length, manifestEntry.Digest); - }), - exportedFiles.Length, - totalBytes); - var expectedDigest = ExportDigestCalculator.ComputeTreeDigest(digestResult); - Assert.Equal(expectedDigest, digest); - - var exporterVersion = root.GetProperty("exporterVersion").GetString(); - Assert.Equal(ExporterVersion.GetVersion(typeof(JsonFeedExporter)), exporterVersion); - } - - public void Dispose() - { - try - { - if (Directory.Exists(_root)) - { - Directory.Delete(_root, recursive: true); - } - } - catch - { - // best effort cleanup - } - } - - private sealed class StubAdvisoryStore : IAdvisoryStore - { - private readonly IReadOnlyList _advisories; - - public StubAdvisoryStore(params Advisory[] advisories) - { - _advisories = advisories; - } - - public Task> GetRecentAsync(int limit, CancellationToken cancellationToken) - => Task.FromResult(_advisories); - - public Task FindAsync(string advisoryKey, CancellationToken cancellationToken) - => Task.FromResult(_advisories.FirstOrDefault(a => a.AdvisoryKey == advisoryKey)); - - public Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken) - => Task.CompletedTask; - - public IAsyncEnumerable StreamAsync(CancellationToken cancellationToken) - { - return EnumerateAsync(cancellationToken); - - async IAsyncEnumerable EnumerateAsync([EnumeratorCancellation] CancellationToken ct) - { - foreach (var advisory in _advisories) - { - ct.ThrowIfCancellationRequested(); - yield return advisory; - await Task.Yield(); - } - } - } - } - - private sealed class InMemoryExportStateStore : IExportStateStore - { - private ExportStateRecord? _record; - - public Task FindAsync(string id, CancellationToken cancellationToken) - => Task.FromResult(_record); - - public Task UpsertAsync(ExportStateRecord record, CancellationToken cancellationToken) - { - _record = record; - return Task.FromResult(record); - } - } - - private sealed class TestTimeProvider : TimeProvider - { - private DateTimeOffset _now; - - public TestTimeProvider(DateTimeOffset start) => _now = start; - - public override DateTimeOffset GetUtcNow() => _now; - - public void Advance(TimeSpan delta) => _now = _now.Add(delta); - } -} diff --git a/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj b/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj deleted file mode 100644 index e579ef8c..00000000 --- a/src/StellaOps.Feedser.Exporter.Json.Tests/StellaOps.Feedser.Exporter.Json.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs b/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs deleted file mode 100644 index a09cad61..00000000 --- a/src/StellaOps.Feedser.Exporter.Json/JsonExportOptions.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.IO; - -namespace StellaOps.Feedser.Exporter.Json; - -/// -/// Configuration for JSON exporter output paths and determinism controls. -/// -public sealed class JsonExportOptions -{ - /// - /// Root directory where exports are written. Default "exports/json". - /// - public string OutputRoot { get; set; } = Path.Combine("exports", "json"); - - /// - /// Format string applied to the export timestamp to produce the directory name. - /// - public string DirectoryNameFormat { get; set; } = "yyyyMMdd'T'HHmmss'Z'"; - - /// - /// Optional static name for the symlink (or directory junction) pointing at the most recent export. - /// - public string LatestSymlinkName { get; set; } = "latest"; - - /// - /// When true, attempts to re-point after a successful export. - /// - public bool MaintainLatestSymlink { get; set; } = true; - - /// - /// Optional repository identifier recorded alongside export state metadata. - /// - public string? TargetRepository { get; set; } -} diff --git a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj b/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj deleted file mode 100644 index 6ac1cb21..00000000 --- a/src/StellaOps.Feedser.Exporter.TrivyDb.Tests/StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj b/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj deleted file mode 100644 index 756df80f..00000000 --- a/src/StellaOps.Feedser.Merge.Tests/StellaOps.Feedser.Merge.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj b/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj deleted file mode 100644 index d2bb7cd2..00000000 --- a/src/StellaOps.Feedser.Merge/StellaOps.Feedser.Merge.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Acsc.Tests/StellaOps.Feedser.Source.Acsc.Tests.csproj b/src/StellaOps.Feedser.Source.Acsc.Tests/StellaOps.Feedser.Source.Acsc.Tests.csproj deleted file mode 100644 index 2a28db6b..00000000 --- a/src/StellaOps.Feedser.Source.Acsc.Tests/StellaOps.Feedser.Source.Acsc.Tests.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj b/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj deleted file mode 100644 index c2d52780..00000000 --- a/src/StellaOps.Feedser.Source.Acsc/StellaOps.Feedser.Source.Acsc.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Cccs/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Cccs/Properties/AssemblyInfo.cs deleted file mode 100644 index 5f71f856..00000000 --- a/src/StellaOps.Feedser.Source.Cccs/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Cccs.Tests")] diff --git a/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj b/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj deleted file mode 100644 index 04f9158c..00000000 --- a/src/StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.CertBund/Configuration/CertBundOptions.cs b/src/StellaOps.Feedser.Source.CertBund/Configuration/CertBundOptions.cs deleted file mode 100644 index 5a8c6fc8..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Configuration/CertBundOptions.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System.Net; - -namespace StellaOps.Feedser.Source.CertBund.Configuration; - -public sealed class CertBundOptions -{ - public const string HttpClientName = "feedser.source.certbund"; - - /// - /// RSS feed providing the latest CERT-Bund advisories. - /// - public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss"); - - /// - /// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API). - /// - public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/"); - - /// - /// Detail API endpoint template; advisory identifier is appended as the name query parameter. - /// - public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory"); - - /// - /// Optional timeout override for feed/detail requests. - /// - public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); - - /// - /// Delay applied between successive detail fetches to respect upstream politeness. - /// - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - /// - /// Backoff recorded in source state when a fetch attempt fails. - /// - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Maximum number of advisories to enqueue per fetch iteration. - /// - public int MaxAdvisoriesPerFetch { get; set; } = 50; - - /// - /// Maximum number of advisory identifiers remembered to prevent re-processing. - /// - public int MaxKnownAdvisories { get; set; } = 512; - - public void Validate() - { - if (FeedUri is null || !FeedUri.IsAbsoluteUri) - { - throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI."); - } - - if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri) - { - throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI."); - } - - if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri) - { - throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); - } - - if (MaxAdvisoriesPerFetch <= 0) - { - throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); - } - - if (MaxKnownAdvisories <= 0) - { - throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero."); - } - } - - public Uri BuildDetailUri(string advisoryId) - { - if (string.IsNullOrWhiteSpace(advisoryId)) - { - throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId)); - } - - var builder = new UriBuilder(DetailApiUri); - var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&"; - builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}"; - return builder.Uri; - } -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundAdvisoryDto.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundAdvisoryDto.cs deleted file mode 100644 index 0c178bf7..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundAdvisoryDto.cs +++ /dev/null @@ -1,68 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -public sealed record CertBundAdvisoryDto -{ - [JsonPropertyName("advisoryId")] - public string AdvisoryId { get; init; } = string.Empty; - - [JsonPropertyName("title")] - public string Title { get; init; } = string.Empty; - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("contentHtml")] - public string ContentHtml { get; init; } = string.Empty; - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("language")] - public string Language { get; init; } = "de"; - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("modified")] - public DateTimeOffset? Modified { get; init; } - - [JsonPropertyName("portalUri")] - public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/"); - - [JsonPropertyName("detailUri")] - public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/"); - - [JsonPropertyName("cveIds")] - public IReadOnlyList CveIds { get; init; } = Array.Empty(); - - [JsonPropertyName("products")] - public IReadOnlyList Products { get; init; } = Array.Empty(); - - [JsonPropertyName("references")] - public IReadOnlyList References { get; init; } = Array.Empty(); -} - -public sealed record CertBundProductDto -{ - [JsonPropertyName("vendor")] - public string? Vendor { get; init; } - - [JsonPropertyName("name")] - public string? Name { get; init; } - - [JsonPropertyName("versions")] - public string? Versions { get; init; } -} - -public sealed record CertBundReferenceDto -{ - [JsonPropertyName("url")] - public string Url { get; init; } = string.Empty; - - [JsonPropertyName("label")] - public string? Label { get; init; } -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs deleted file mode 100644 index 32326bb4..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs +++ /dev/null @@ -1,118 +0,0 @@ -using System; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -internal sealed record CertBundCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - IReadOnlyCollection KnownAdvisories, - DateTimeOffset? LastPublished, - DateTimeOffset? LastFetchAt) -{ - private static readonly IReadOnlyCollection EmptyGuids = Array.Empty(); - private static readonly IReadOnlyCollection EmptyStrings = Array.Empty(); - - public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null); - - public CertBundCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = Distinct(documents) }; - - public CertBundCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = Distinct(mappings) }; - - public CertBundCursor WithKnownAdvisories(IEnumerable advisories) - => this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings }; - - public CertBundCursor WithLastPublished(DateTimeOffset? published) - => this with { LastPublished = published }; - - public CertBundCursor WithLastFetch(DateTimeOffset? timestamp) - => this with { LastFetchAt = timestamp }; - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - ["knownAdvisories"] = new BsonArray(KnownAdvisories), - }; - - if (LastPublished.HasValue) - { - document["lastPublished"] = LastPublished.Value.UtcDateTime; - } - - if (LastFetchAt.HasValue) - { - document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime; - } - - return document; - } - - public static CertBundCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var knownAdvisories = ReadStringArray(document, "knownAdvisories"); - var lastPublished = document.TryGetValue("lastPublished", out var publishedValue) - ? ParseDate(publishedValue) - : null; - var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue) - ? ParseDate(fetchValue) - : null; - - return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch); - } - - private static IReadOnlyCollection Distinct(IEnumerable? values) - => values?.Distinct().ToArray() ?? EmptyGuids; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuids; - } - - var items = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var id)) - { - items.Add(id); - } - } - - return items; - } - - private static IReadOnlyCollection ReadStringArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyStrings; - } - - return array.Select(element => element?.ToString() ?? string.Empty) - .Where(static s => !string.IsNullOrWhiteSpace(s)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailParser.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailParser.cs deleted file mode 100644 index 946925f3..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailParser.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; -using System.Text.Json.Serialization; -using StellaOps.Feedser.Source.Common.Html; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -public sealed class CertBundDetailParser -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - }; - - private readonly HtmlContentSanitizer _sanitizer; - - public CertBundDetailParser(HtmlContentSanitizer sanitizer) - => _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer)); - - public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload) - { - var detail = JsonSerializer.Deserialize(payload, SerializerOptions) - ?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null."); - - var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name."); - var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri); - - return new CertBundAdvisoryDto - { - AdvisoryId = advisoryId, - Title = detail.Title ?? advisoryId, - Summary = detail.Summary, - ContentHtml = contentHtml, - Severity = detail.Severity, - Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!, - Published = detail.Published, - Modified = detail.Updated ?? detail.Published, - PortalUri = portalUri, - DetailUri = detailUri, - CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id)) - .Select(static id => id!.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray() ?? Array.Empty(), - References = MapReferences(detail.References), - Products = MapProducts(detail.Products), - }; - } - - private static IReadOnlyList MapReferences(CertBundDetailReference[]? references) - { - if (references is null || references.Length == 0) - { - return Array.Empty(); - } - - return references - .Where(static reference => !string.IsNullOrWhiteSpace(reference.Url)) - .Select(reference => new CertBundReferenceDto - { - Url = reference.Url!, - Label = reference.Label, - }) - .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList MapProducts(CertBundDetailProduct[]? products) - { - if (products is null || products.Length == 0) - { - return Array.Empty(); - } - - return products - .Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name)) - .Select(product => new CertBundProductDto - { - Vendor = product.Vendor, - Name = product.Name, - Versions = product.Versions, - }) - .ToArray(); - } -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailResponse.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailResponse.cs deleted file mode 100644 index 6b48697f..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDetailResponse.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -internal sealed record CertBundDetailResponse -{ - [JsonPropertyName("name")] - public string? Name { get; init; } - - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("summary")] - public string? Summary { get; init; } - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("language")] - public string? Language { get; init; } - - [JsonPropertyName("published")] - public DateTimeOffset? Published { get; init; } - - [JsonPropertyName("updated")] - public DateTimeOffset? Updated { get; init; } - - [JsonPropertyName("cveIds")] - public string[]? CveIds { get; init; } - - [JsonPropertyName("references")] - public CertBundDetailReference[]? References { get; init; } - - [JsonPropertyName("products")] - public CertBundDetailProduct[]? Products { get; init; } -} - -internal sealed record CertBundDetailReference -{ - [JsonPropertyName("url")] - public string? Url { get; init; } - - [JsonPropertyName("label")] - public string? Label { get; init; } -} - -internal sealed record CertBundDetailProduct -{ - [JsonPropertyName("vendor")] - public string? Vendor { get; init; } - - [JsonPropertyName("name")] - public string? Name { get; init; } - - [JsonPropertyName("versions")] - public string? Versions { get; init; } -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDiagnostics.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDiagnostics.cs deleted file mode 100644 index dfc98bb3..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDiagnostics.cs +++ /dev/null @@ -1,191 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -/// -/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector. -/// -public sealed class CertBundDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.CertBund"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _feedFetchAttempts; - private readonly Counter _feedFetchSuccess; - private readonly Counter _feedFetchFailures; - private readonly Histogram _feedItemCount; - private readonly Histogram _feedEnqueuedCount; - private readonly Histogram _feedCoverageDays; - private readonly Counter _detailFetchAttempts; - private readonly Counter _detailFetchSuccess; - private readonly Counter _detailFetchNotModified; - private readonly Counter _detailFetchFailures; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Histogram _parseProductCount; - private readonly Histogram _parseCveCount; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Histogram _mapPackageCount; - private readonly Histogram _mapAliasCount; - - public CertBundDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _feedFetchAttempts = _meter.CreateCounter( - name: "certbund.feed.fetch.attempts", - unit: "operations", - description: "Number of RSS feed load attempts."); - _feedFetchSuccess = _meter.CreateCounter( - name: "certbund.feed.fetch.success", - unit: "operations", - description: "Number of successful RSS feed loads."); - _feedFetchFailures = _meter.CreateCounter( - name: "certbund.feed.fetch.failures", - unit: "operations", - description: "Number of RSS feed load failures."); - _feedItemCount = _meter.CreateHistogram( - name: "certbund.feed.items.count", - unit: "items", - description: "Distribution of RSS item counts per fetch."); - _feedEnqueuedCount = _meter.CreateHistogram( - name: "certbund.feed.enqueued.count", - unit: "documents", - description: "Distribution of advisory documents enqueued per fetch."); - _feedCoverageDays = _meter.CreateHistogram( - name: "certbund.feed.coverage.days", - unit: "days", - description: "Coverage window in days between fetch time and the oldest published advisory in the feed."); - _detailFetchAttempts = _meter.CreateCounter( - name: "certbund.detail.fetch.attempts", - unit: "operations", - description: "Number of detail fetch attempts."); - _detailFetchSuccess = _meter.CreateCounter( - name: "certbund.detail.fetch.success", - unit: "operations", - description: "Number of detail fetches that persisted a document."); - _detailFetchNotModified = _meter.CreateCounter( - name: "certbund.detail.fetch.not_modified", - unit: "operations", - description: "Number of detail fetches returning HTTP 304."); - _detailFetchFailures = _meter.CreateCounter( - name: "certbund.detail.fetch.failures", - unit: "operations", - description: "Number of detail fetches that failed."); - _parseSuccess = _meter.CreateCounter( - name: "certbund.parse.success", - unit: "documents", - description: "Number of documents parsed into CERT-Bund DTOs."); - _parseFailures = _meter.CreateCounter( - name: "certbund.parse.failures", - unit: "documents", - description: "Number of documents that failed to parse."); - _parseProductCount = _meter.CreateHistogram( - name: "certbund.parse.products.count", - unit: "products", - description: "Distribution of product entries captured per advisory."); - _parseCveCount = _meter.CreateHistogram( - name: "certbund.parse.cve.count", - unit: "aliases", - description: "Distribution of CVE identifiers captured per advisory."); - _mapSuccess = _meter.CreateCounter( - name: "certbund.map.success", - unit: "advisories", - description: "Number of canonical advisories emitted by the mapper."); - _mapFailures = _meter.CreateCounter( - name: "certbund.map.failures", - unit: "advisories", - description: "Number of mapping failures."); - _mapPackageCount = _meter.CreateHistogram( - name: "certbund.map.affected.count", - unit: "packages", - description: "Distribution of affected packages emitted per advisory."); - _mapAliasCount = _meter.CreateHistogram( - name: "certbund.map.aliases.count", - unit: "aliases", - description: "Distribution of alias counts per advisory."); - } - - public void FeedFetchAttempt() => _feedFetchAttempts.Add(1); - - public void FeedFetchSuccess(int itemCount) - { - _feedFetchSuccess.Add(1); - if (itemCount >= 0) - { - _feedItemCount.Record(itemCount); - } - } - - public void FeedFetchFailure(string reason = "error") - => _feedFetchFailures.Add(1, ReasonTag(reason)); - - public void RecordFeedCoverage(double? coverageDays) - { - if (coverageDays is { } days && days >= 0) - { - _feedCoverageDays.Record(days); - } - } - - public void DetailFetchAttempt() => _detailFetchAttempts.Add(1); - - public void DetailFetchSuccess() => _detailFetchSuccess.Add(1); - - public void DetailFetchNotModified() => _detailFetchNotModified.Add(1); - - public void DetailFetchFailure(string reason = "error") - => _detailFetchFailures.Add(1, ReasonTag(reason)); - - public void DetailFetchEnqueued(int count) - { - if (count >= 0) - { - _feedEnqueuedCount.Record(count); - } - } - - public void ParseSuccess(int productCount, int cveCount) - { - _parseSuccess.Add(1); - - if (productCount >= 0) - { - _parseProductCount.Record(productCount); - } - - if (cveCount >= 0) - { - _parseCveCount.Record(cveCount); - } - } - - public void ParseFailure(string reason = "error") - => _parseFailures.Add(1, ReasonTag(reason)); - - public void MapSuccess(int affectedPackages, int aliasCount) - { - _mapSuccess.Add(1); - - if (affectedPackages >= 0) - { - _mapPackageCount.Record(affectedPackages); - } - - if (aliasCount >= 0) - { - _mapAliasCount.Record(aliasCount); - } - } - - public void MapFailure(string reason = "error") - => _mapFailures.Add(1, ReasonTag(reason)); - - private static KeyValuePair ReasonTag(string reason) - => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant()); - - public void Dispose() => _meter.Dispose(); -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDocumentMetadata.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDocumentMetadata.cs deleted file mode 100644 index 818e1237..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundDocumentMetadata.cs +++ /dev/null @@ -1,29 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -internal static class CertBundDocumentMetadata -{ - public static Dictionary CreateMetadata(CertBundFeedItem item) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["certbund.advisoryId"] = item.AdvisoryId, - ["certbund.portalUri"] = item.PortalUri.ToString(), - ["certbund.published"] = item.Published.ToString("O"), - }; - - if (!string.IsNullOrWhiteSpace(item.Category)) - { - metadata["certbund.category"] = item.Category!; - } - - if (!string.IsNullOrWhiteSpace(item.Title)) - { - metadata["certbund.title"] = item.Title!; - } - - return metadata; - } -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedClient.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedClient.cs deleted file mode 100644 index cbd1dbd5..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedClient.cs +++ /dev/null @@ -1,143 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; -using System.Xml.Linq; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.CertBund.Configuration; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -public sealed class CertBundFeedClient -{ - private readonly IHttpClientFactory _httpClientFactory; - private readonly CertBundOptions _options; - private readonly ILogger _logger; - private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1); - private volatile bool _bootstrapped; - - public CertBundFeedClient( - IHttpClientFactory httpClientFactory, - IOptions options, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task> LoadAsync(CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName); - await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false); - - using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri); - request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8"); - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - var document = XDocument.Load(stream); - - var items = new List(); - foreach (var element in document.Descendants("item")) - { - cancellationToken.ThrowIfCancellationRequested(); - - var linkValue = element.Element("link")?.Value?.Trim(); - if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri)) - { - continue; - } - - var advisoryId = TryExtractNameParameter(portalUri); - if (string.IsNullOrWhiteSpace(advisoryId)) - { - continue; - } - - var detailUri = _options.BuildDetailUri(advisoryId); - var pubDateText = element.Element("pubDate")?.Value; - var published = ParseDate(pubDateText); - var title = element.Element("title")?.Value?.Trim(); - var category = element.Element("category")?.Value?.Trim(); - - items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category)); - } - - return items; - } - - private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken) - { - if (_bootstrapped) - { - return; - } - - await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (_bootstrapped) - { - return; - } - - using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri); - request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); - using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - _bootstrapped = true; - } - finally - { - _bootstrapSemaphore.Release(); - } - } - - private static string? TryExtractNameParameter(Uri portalUri) - { - if (portalUri is null) - { - return null; - } - - var query = portalUri.Query; - if (string.IsNullOrEmpty(query)) - { - return null; - } - - var trimmed = query.TrimStart('?'); - foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries)) - { - var separatorIndex = pair.IndexOf('='); - if (separatorIndex <= 0) - { - continue; - } - - var key = pair[..separatorIndex].Trim(); - if (!key.Equals("name", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - var value = pair[(separatorIndex + 1)..]; - return Uri.UnescapeDataString(value); - } - - return null; - } - - private static DateTimeOffset ParseDate(string? value) - => DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed) - ? parsed - : DateTimeOffset.UtcNow; -} diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedItem.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedItem.cs deleted file mode 100644 index a92374c9..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundFeedItem.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace StellaOps.Feedser.Source.CertBund.Internal; - -using System; - -public sealed record CertBundFeedItem( - string AdvisoryId, - Uri DetailUri, - Uri PortalUri, - DateTimeOffset Published, - string? Title, - string? Category); diff --git a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs b/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs deleted file mode 100644 index ff168ce2..00000000 --- a/src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs +++ /dev/null @@ -1,168 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.CertBund.Internal; - -internal static class CertBundMapper -{ - public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var packages = BuildPackages(dto, recordedAt); - var provenance = new AdvisoryProvenance( - CertBundConnectorPlugin.SourceName, - "advisory", - dto.AdvisoryId, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - advisoryKey: dto.AdvisoryId, - title: dto.Title, - summary: dto.Summary, - language: dto.Language?.ToLowerInvariant() ?? "de", - published: dto.Published, - modified: dto.Modified, - severity: MapSeverity(dto.Severity), - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: packages, - cvssMetrics: Array.Empty(), - provenance: new[] { provenance }); - } - - private static IReadOnlyList BuildAliases(CertBundAdvisoryDto dto) - { - var aliases = new List(capacity: 4) { dto.AdvisoryId }; - foreach (var cve in dto.CveIds) - { - if (!string.IsNullOrWhiteSpace(cve)) - { - aliases.Add(cve); - } - } - - return aliases - .Where(static alias => !string.IsNullOrWhiteSpace(alias)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt) - { - var references = new List - { - new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance( - CertBundConnectorPlugin.SourceName, - "reference", - dto.DetailUri.ToString(), - recordedAt, - new[] { ProvenanceFieldMasks.References })) - }; - - foreach (var reference in dto.References) - { - if (string.IsNullOrWhiteSpace(reference.Url)) - { - continue; - } - - references.Add(new AdvisoryReference( - reference.Url, - kind: "reference", - sourceTag: "cert-bund", - summary: reference.Label, - provenance: new AdvisoryProvenance( - CertBundConnectorPlugin.SourceName, - "reference", - reference.Url, - recordedAt, - new[] { ProvenanceFieldMasks.References }))); - } - - return references - .DistinctBy(static reference => reference.Url, StringComparer.Ordinal) - .OrderBy(static reference => reference.Url, StringComparer.Ordinal) - .ToArray(); - } - - private static IReadOnlyList BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt) - { - if (dto.Products.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Products.Count); - foreach (var product in dto.Products) - { - var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified"; - var name = Validation.TrimToNull(product.Name); - var identifier = name is null ? vendor : $"{vendor} {name}"; - - var provenance = new AdvisoryProvenance( - CertBundConnectorPlugin.SourceName, - "package", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.AffectedPackages }); - - var ranges = string.IsNullOrWhiteSpace(product.Versions) - ? Array.Empty() - : new[] - { - new AffectedVersionRange( - rangeKind: "string", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: product.Versions, - provenance: new AdvisoryProvenance( - CertBundConnectorPlugin.SourceName, - "package-range", - product.Versions, - recordedAt, - new[] { ProvenanceFieldMasks.VersionRanges })) - }; - - packages.Add(new AffectedPackage( - AffectedPackageTypes.Vendor, - identifier, - platform: null, - versionRanges: ranges, - statuses: Array.Empty(), - provenance: new[] { provenance }, - normalizedVersions: Array.Empty())); - } - - return packages - .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static string? MapSeverity(string? severity) - { - if (string.IsNullOrWhiteSpace(severity)) - { - return null; - } - - return severity.ToLowerInvariant() switch - { - "hoch" or "high" => "high", - "mittel" or "medium" => "medium", - "gering" or "low" => "low", - _ => severity.ToLowerInvariant(), - }; - } -} diff --git a/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md b/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md deleted file mode 100644 index 1325e84d..00000000 --- a/src/StellaOps.Feedser.Source.CertCc/FEEDCONN-CERTCC-02-012_HANDOFF.md +++ /dev/null @@ -1,20 +0,0 @@ -# FEEDCONN-CERTCC-02-012 – Schema Sync & Snapshot Regeneration - -## Summary -- Re-ran `StellaOps.Feedser.Source.CertCc.Tests` with `UPDATE_CERTCC_FIXTURES=1`; fixtures now capture SemVer-style normalized versions (`scheme=certcc.vendor`) and `provenance.decisionReason` values emitted by the mapper. -- Recorded HTTP request ordering is persisted in `certcc-requests.snapshot.json` to keep Merge aware of the deterministic fetch plan. -- Advisories snapshot (`certcc-advisories.snapshot.json`) reflects the dual-write storage changes (normalized versions + provenance) introduced by FEEDMODELS-SCHEMA-* and FEEDSTORAGE-DATA-*. - -## Artifacts -- `src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-advisories.snapshot.json` -- `src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-documents.snapshot.json` -- `src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-requests.snapshot.json` -- `src/StellaOps.Feedser.Source.CertCc.Tests/Fixtures/certcc-state.snapshot.json` - -## Validation steps -```bash -dotnet test src/StellaOps.Feedser.Source.CertCc.Tests -UPDATE_CERTCC_FIXTURES=1 dotnet test src/StellaOps.Feedser.Source.CertCc.Tests -``` - -The first command verifies deterministic behavior; the second regenerates fixtures if a future schema change occurs. Share the four snapshot files above with Merge for their backfill diff. diff --git a/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs deleted file mode 100644 index 96da4f07..00000000 --- a/src/StellaOps.Feedser.Source.CertCc/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.CertCc.Tests")] diff --git a/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj b/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj deleted file mode 100644 index 6cf96d45..00000000 --- a/src/StellaOps.Feedser.Source.CertFr.Tests/StellaOps.Feedser.Source.CertFr.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj b/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj deleted file mode 100644 index a01e6075..00000000 --- a/src/StellaOps.Feedser.Source.CertFr/StellaOps.Feedser.Source.CertFr.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj b/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj deleted file mode 100644 index c7000b4f..00000000 --- a/src/StellaOps.Feedser.Source.CertIn.Tests/StellaOps.Feedser.Source.CertIn.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj b/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj deleted file mode 100644 index 7e54853b..00000000 --- a/src/StellaOps.Feedser.Source.CertIn/StellaOps.Feedser.Source.CertIn.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs deleted file mode 100644 index 2d379827..00000000 --- a/src/StellaOps.Feedser.Source.Common/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Common.Tests")] diff --git a/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj b/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj deleted file mode 100644 index c6a19365..00000000 --- a/src/StellaOps.Feedser.Source.Cve.Tests/StellaOps.Feedser.Source.Cve.Tests.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj deleted file mode 100644 index d75a27bb..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Debian.Tests/StellaOps.Feedser.Source.Distro.Debian.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs deleted file mode 100644 index e2c83a72..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Debian/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Debian.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj b/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj deleted file mode 100644 index 96165c66..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Debian/StellaOps.Feedser.Source.Distro.Debian.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj deleted file mode 100644 index 654d8c5f..00000000 --- a/src/StellaOps.Feedser.Source.Distro.RedHat.Tests/StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs deleted file mode 100644 index 84fa6ffc..00000000 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.RedHat.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj b/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj deleted file mode 100644 index bedbc3b9..00000000 --- a/src/StellaOps.Feedser.Source.Distro.RedHat/StellaOps.Feedser.Source.Distro.RedHat.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj deleted file mode 100644 index f56d2dff..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Suse.Tests/StellaOps.Feedser.Source.Distro.Suse.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - PreserveNewest - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs deleted file mode 100644 index 0a90994d..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Suse/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Distro.Suse.Tests")] diff --git a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj b/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj deleted file mode 100644 index 96165c66..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Suse/StellaOps.Feedser.Source.Distro.Suse.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj deleted file mode 100644 index 65ac759c..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu.Tests/StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - PreserveNewest - - - diff --git a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj b/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj deleted file mode 100644 index 96165c66..00000000 --- a/src/StellaOps.Feedser.Source.Distro.Ubuntu/StellaOps.Feedser.Source.Distro.Ubuntu.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj b/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj deleted file mode 100644 index c476b167..00000000 --- a/src/StellaOps.Feedser.Source.Ghsa.Tests/StellaOps.Feedser.Source.Ghsa.Tests.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj b/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj deleted file mode 100644 index 2b65e902..00000000 --- a/src/StellaOps.Feedser.Source.Ghsa/StellaOps.Feedser.Source.Ghsa.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/StellaOps.Feedser.Source.Ics.Cisa.Tests.csproj b/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/StellaOps.Feedser.Source.Ics.Cisa.Tests.csproj deleted file mode 100644 index 48c95541..00000000 --- a/src/StellaOps.Feedser.Source.Ics.Cisa.Tests/StellaOps.Feedser.Source.Ics.Cisa.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj deleted file mode 100644 index c3a057e2..00000000 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky.Tests/StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj b/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj deleted file mode 100644 index 7e54853b..00000000 --- a/src/StellaOps.Feedser.Source.Ics.Kaspersky/StellaOps.Feedser.Source.Ics.Kaspersky.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj b/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj deleted file mode 100644 index 351f26a7..00000000 --- a/src/StellaOps.Feedser.Source.Jvn.Tests/StellaOps.Feedser.Source.Jvn.Tests.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj b/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj deleted file mode 100644 index c6e627a2..00000000 --- a/src/StellaOps.Feedser.Source.Jvn/StellaOps.Feedser.Source.Jvn.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj b/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj deleted file mode 100644 index 57e5921a..00000000 --- a/src/StellaOps.Feedser.Source.Kev.Tests/StellaOps.Feedser.Source.Kev.Tests.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj b/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj deleted file mode 100644 index 2f48c62c..00000000 --- a/src/StellaOps.Feedser.Source.Kisa/StellaOps.Feedser.Source.Kisa.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj b/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj deleted file mode 100644 index 0b170a63..00000000 --- a/src/StellaOps.Feedser.Source.Nvd.Tests/StellaOps.Feedser.Source.Nvd.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj b/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj deleted file mode 100644 index 74d98ea9..00000000 --- a/src/StellaOps.Feedser.Source.Nvd/StellaOps.Feedser.Source.Nvd.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj b/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj deleted file mode 100644 index e501ae0d..00000000 --- a/src/StellaOps.Feedser.Source.Osv.Tests/StellaOps.Feedser.Source.Osv.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - PreserveNewest - - - diff --git a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj b/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj deleted file mode 100644 index a3256c0b..00000000 --- a/src/StellaOps.Feedser.Source.Osv/StellaOps.Feedser.Source.Osv.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - <_Parameter1>StellaOps.Feedser.Tests - - - <_Parameter1>StellaOps.Feedser.Source.Osv.Tests - - - diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/StellaOps.Feedser.Source.Ru.Bdu.Tests.csproj b/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/StellaOps.Feedser.Source.Ru.Bdu.Tests.csproj deleted file mode 100644 index 148f9dc5..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Bdu.Tests/StellaOps.Feedser.Source.Ru.Bdu.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Ru.Bdu/Properties/AssemblyInfo.cs deleted file mode 100644 index b658730c..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Ru.Bdu.Tests")] diff --git a/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj b/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj deleted file mode 100644 index 77fb187a..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Bdu/StellaOps.Feedser.Source.Ru.Bdu.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj b/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj deleted file mode 100644 index 584d4b62..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki.Tests/StellaOps.Feedser.Source.Ru.Nkcki.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Ru.Nkcki/Properties/AssemblyInfo.cs deleted file mode 100644 index a4a4ea6c..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Ru.Nkcki.Tests")] diff --git a/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj b/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj deleted file mode 100644 index 947a0b1c..00000000 --- a/src/StellaOps.Feedser.Source.Ru.Nkcki/StellaOps.Feedser.Source.Ru.Nkcki.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj deleted file mode 100644 index dbd05cbe..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Apple.Tests/StellaOps.Feedser.Source.Vndr.Apple.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs deleted file mode 100644 index ff1c1ce0..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Apple.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj b/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj deleted file mode 100644 index c8aaf11c..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Apple/StellaOps.Feedser.Source.Vndr.Apple.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs deleted file mode 100644 index af25849d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Chromium/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Chromium.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/StellaOps.Feedser.Source.Vndr.Cisco.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/StellaOps.Feedser.Source.Vndr.Cisco.Tests.csproj deleted file mode 100644 index eaaa91d9..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco.Tests/StellaOps.Feedser.Source.Vndr.Cisco.Tests.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Configuration/CiscoOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Configuration/CiscoOptions.cs deleted file mode 100644 index 9b28458b..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Configuration/CiscoOptions.cs +++ /dev/null @@ -1,124 +0,0 @@ -using System.Globalization; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Configuration; - -public sealed class CiscoOptions -{ - public const string HttpClientName = "feedser.source.vndr.cisco"; - public const string AuthHttpClientName = "feedser.source.vndr.cisco.auth"; - - public Uri BaseUri { get; set; } = new("https://api.cisco.com/security/advisories/v2/", UriKind.Absolute); - - public Uri TokenEndpoint { get; set; } = new("https://id.cisco.com/oauth2/default/v1/token", UriKind.Absolute); - - public string ClientId { get; set; } = string.Empty; - - public string ClientSecret { get; set; } = string.Empty; - - public int PageSize { get; set; } = 100; - - public int MaxPagesPerFetch { get; set; } = 5; - - public int MaxAdvisoriesPerFetch { get; set; } = 200; - - public TimeSpan InitialBackfillWindow { get; set; } = TimeSpan.FromDays(30); - - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30); - - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - public TimeSpan TokenRefreshSkew { get; set; } = TimeSpan.FromMinutes(1); - - public string LastModifiedPathTemplate { get; set; } = "advisories/lastmodified/{0}"; - - public void Validate() - { - if (BaseUri is null || !BaseUri.IsAbsoluteUri) - { - throw new InvalidOperationException("Cisco BaseUri must be an absolute URI."); - } - - if (TokenEndpoint is null || !TokenEndpoint.IsAbsoluteUri) - { - throw new InvalidOperationException("Cisco TokenEndpoint must be an absolute URI."); - } - - if (string.IsNullOrWhiteSpace(ClientId)) - { - throw new InvalidOperationException("Cisco clientId must be configured."); - } - - if (string.IsNullOrWhiteSpace(ClientSecret)) - { - throw new InvalidOperationException("Cisco clientSecret must be configured."); - } - - if (PageSize is < 1 or > 100) - { - throw new InvalidOperationException("Cisco PageSize must be between 1 and 100."); - } - - if (MaxPagesPerFetch <= 0) - { - throw new InvalidOperationException("Cisco MaxPagesPerFetch must be greater than zero."); - } - - if (MaxAdvisoriesPerFetch <= 0) - { - throw new InvalidOperationException("Cisco MaxAdvisoriesPerFetch must be greater than zero."); - } - - if (InitialBackfillWindow <= TimeSpan.Zero) - { - throw new InvalidOperationException("Cisco InitialBackfillWindow must be positive."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException("Cisco RequestDelay cannot be negative."); - } - - if (RequestTimeout <= TimeSpan.Zero) - { - throw new InvalidOperationException("Cisco RequestTimeout must be positive."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException("Cisco FailureBackoff must be positive."); - } - - if (TokenRefreshSkew < TimeSpan.FromSeconds(5)) - { - throw new InvalidOperationException("Cisco TokenRefreshSkew must be at least 5 seconds."); - } - - if (string.IsNullOrWhiteSpace(LastModifiedPathTemplate)) - { - throw new InvalidOperationException("Cisco LastModifiedPathTemplate must be configured."); - } - } - - public Uri BuildLastModifiedUri(DateOnly date, int pageIndex, int pageSize) - { - if (pageIndex < 1) - { - throw new ArgumentOutOfRangeException(nameof(pageIndex), pageIndex, "Page index must be >= 1."); - } - - if (pageSize is < 1 or > 100) - { - throw new ArgumentOutOfRangeException(nameof(pageSize), pageSize, "Page size must be between 1 and 100."); - } - - var path = string.Format(CultureInfo.InvariantCulture, LastModifiedPathTemplate, date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)); - var builder = new UriBuilder(BaseUri); - var basePath = builder.Path.TrimEnd('/'); - builder.Path = $"{basePath}/{path}".Replace("//", "/", StringComparison.Ordinal); - var query = $"pageIndex={pageIndex.ToString(CultureInfo.InvariantCulture)}&pageSize={pageSize.ToString(CultureInfo.InvariantCulture)}"; - builder.Query = string.IsNullOrEmpty(builder.Query) ? query : builder.Query.TrimStart('?') + "&" + query; - return builder.Uri; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs deleted file mode 100644 index dbf670e2..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAccessTokenProvider.cs +++ /dev/null @@ -1,145 +0,0 @@ -using System.Net.Http.Headers; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -internal sealed class CiscoAccessTokenProvider : IDisposable -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - }; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IOptionsMonitor _options; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly SemaphoreSlim _refreshLock = new(1, 1); - - private volatile AccessToken? _cached; - private bool _disposed; - - public CiscoAccessTokenProvider( - IHttpClientFactory httpClientFactory, - IOptionsMonitor options, - TimeProvider? timeProvider, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task GetTokenAsync(CancellationToken cancellationToken) - => await GetTokenInternalAsync(forceRefresh: false, cancellationToken).ConfigureAwait(false); - - public void Invalidate() - => _cached = null; - - private async Task GetTokenInternalAsync(bool forceRefresh, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - - var options = _options.CurrentValue; - var now = _timeProvider.GetUtcNow(); - var cached = _cached; - if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) - { - return cached.Value; - } - - await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - cached = _cached; - now = _timeProvider.GetUtcNow(); - if (!forceRefresh && cached is not null && now < cached.ExpiresAt - options.TokenRefreshSkew) - { - return cached.Value; - } - - var fresh = await RequestTokenAsync(options, cancellationToken).ConfigureAwait(false); - _cached = fresh; - return fresh.Value; - } - finally - { - _refreshLock.Release(); - } - } - - private async Task RequestTokenAsync(CiscoOptions options, CancellationToken cancellationToken) - { - var client = _httpClientFactory.CreateClient(CiscoOptions.AuthHttpClientName); - client.Timeout = options.RequestTimeout; - - using var request = new HttpRequestMessage(HttpMethod.Post, options.TokenEndpoint); - request.Headers.Accept.Clear(); - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - - var content = new FormUrlEncodedContent(new Dictionary - { - ["grant_type"] = "client_credentials", - ["client_id"] = options.ClientId, - ["client_secret"] = options.ClientSecret, - }); - - request.Content = content; - - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - { - var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - var message = $"Cisco OAuth token request failed with status {(int)response.StatusCode} {response.StatusCode}."; - _logger.LogError("Cisco openVuln token request failed: {Message}; response={Preview}", message, preview); - throw new HttpRequestException(message); - } - - await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); - var payload = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken).ConfigureAwait(false); - if (payload is null || string.IsNullOrWhiteSpace(payload.AccessToken)) - { - throw new InvalidOperationException("Cisco OAuth token response did not include an access token."); - } - - var expiresIn = payload.ExpiresIn > 0 ? TimeSpan.FromSeconds(payload.ExpiresIn) : TimeSpan.FromHours(1); - var now = _timeProvider.GetUtcNow(); - var expiresAt = now + expiresIn; - _logger.LogInformation("Cisco openVuln token issued; expires in {ExpiresIn}", expiresIn); - return new AccessToken(payload.AccessToken, expiresAt); - } - - public async Task RefreshAsync(CancellationToken cancellationToken) - => await GetTokenInternalAsync(forceRefresh: true, cancellationToken).ConfigureAwait(false); - - private void ThrowIfDisposed() - { - if (_disposed) - { - throw new ObjectDisposedException(nameof(CiscoAccessTokenProvider)); - } - } - - public void Dispose() - { - if (_disposed) - { - return; - } - - _refreshLock.Dispose(); - _disposed = true; - } - - private sealed record AccessToken(string Value, DateTimeOffset ExpiresAt); - - private sealed record TokenResponse( - [property: JsonPropertyName("access_token")] string AccessToken, - [property: JsonPropertyName("expires_in")] int ExpiresIn, - [property: JsonPropertyName("token_type")] string? TokenType); -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs deleted file mode 100644 index 6f99b138..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoAdvisoryDto.cs +++ /dev/null @@ -1,25 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public sealed record CiscoAdvisoryDto( - string AdvisoryId, - string Title, - string? Summary, - string? Severity, - DateTimeOffset? Published, - DateTimeOffset? Updated, - string? PublicationUrl, - string? CsafUrl, - string? CvrfUrl, - double? CvssBaseScore, - IReadOnlyList Cves, - IReadOnlyList BugIds, - IReadOnlyList Products); - -public sealed record CiscoAffectedProductDto( - string Name, - string? ProductId, - string? Version, - IReadOnlyCollection Statuses); diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafClient.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafClient.cs deleted file mode 100644 index 623bbb9d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafClient.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System; -using System.IO; -using System.Net.Http; -using System.Text; -using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public interface ICiscoCsafClient -{ - Task TryFetchAsync(string? url, CancellationToken cancellationToken); -} - -public class CiscoCsafClient : ICiscoCsafClient -{ - private static readonly string[] AcceptHeaders = { "application/json", "application/csaf+json", "application/vnd.cisco.csaf+json" }; - - private readonly SourceFetchService _fetchService; - private readonly ILogger _logger; - - public CiscoCsafClient(SourceFetchService fetchService, ILogger logger) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public virtual async Task TryFetchAsync(string? url, CancellationToken cancellationToken) - { - if (string.IsNullOrWhiteSpace(url)) - { - return null; - } - - if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) - { - _logger.LogWarning("Cisco CSAF URL '{Url}' is not a valid absolute URI.", url); - return null; - } - - try - { - var request = new SourceFetchRequest(CiscoOptions.HttpClientName, VndrCiscoConnectorPlugin.SourceName, uri) - { - AcceptHeaders = AcceptHeaders, - }; - - var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - if (!result.IsSuccess || result.Content is null) - { - _logger.LogWarning("Cisco CSAF download returned status {Status} for {Url}", result.StatusCode, url); - return null; - } - - return System.Text.Encoding.UTF8.GetString(result.Content); - } - catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or IOException or InvalidOperationException) - { - _logger.LogWarning(ex, "Cisco CSAF download failed for {Url}", url); - return null; - } - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafData.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafData.cs deleted file mode 100644 index 5c57caa5..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafData.cs +++ /dev/null @@ -1,9 +0,0 @@ -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -internal sealed record CiscoCsafData( - IReadOnlyDictionary Products, - IReadOnlyDictionary> ProductStatuses); - -internal sealed record CiscoCsafProduct(string ProductId, string Name); diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafParser.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafParser.cs deleted file mode 100644 index 9f0b5636..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCsafParser.cs +++ /dev/null @@ -1,123 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using System.Text.Json; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -internal static class CiscoCsafParser -{ - public static CiscoCsafData Parse(string content) - { - if (string.IsNullOrWhiteSpace(content)) - { - return new CiscoCsafData( - Products: new Dictionary(0, StringComparer.OrdinalIgnoreCase), - ProductStatuses: new Dictionary>(0, StringComparer.OrdinalIgnoreCase)); - } - - using var document = JsonDocument.Parse(content); - var root = document.RootElement; - - var products = ParseProducts(root); - var statuses = ParseStatuses(root); - - return new CiscoCsafData(products, statuses); - } - - private static IReadOnlyDictionary ParseProducts(JsonElement root) - { - var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); - if (!root.TryGetProperty("product_tree", out var productTree)) - { - return dictionary; - } - - if (productTree.TryGetProperty("full_product_names", out var fullProductNames) - && fullProductNames.ValueKind == JsonValueKind.Array) - { - foreach (var entry in fullProductNames.EnumerateArray()) - { - var productId = entry.TryGetProperty("product_id", out var idElement) && idElement.ValueKind == JsonValueKind.String - ? idElement.GetString() - : null; - - if (string.IsNullOrWhiteSpace(productId)) - { - continue; - } - - var name = entry.TryGetProperty("name", out var nameElement) && nameElement.ValueKind == JsonValueKind.String - ? nameElement.GetString() - : null; - - if (string.IsNullOrWhiteSpace(name)) - { - name = productId; - } - - dictionary[productId] = new CiscoCsafProduct(productId, name); - } - } - - return dictionary; - } - - private static IReadOnlyDictionary> ParseStatuses(JsonElement root) - { - var map = new Dictionary>(StringComparer.OrdinalIgnoreCase); - - if (!root.TryGetProperty("vulnerabilities", out var vulnerabilities) - || vulnerabilities.ValueKind != JsonValueKind.Array) - { - return map.ToDictionary( - static kvp => kvp.Key, - static kvp => (IReadOnlyCollection)kvp.Value.ToArray(), - StringComparer.OrdinalIgnoreCase); - } - - foreach (var vulnerability in vulnerabilities.EnumerateArray()) - { - if (!vulnerability.TryGetProperty("product_status", out var productStatus) - || productStatus.ValueKind != JsonValueKind.Object) - { - continue; - } - - foreach (var property in productStatus.EnumerateObject()) - { - var statusLabel = property.Name; - if (property.Value.ValueKind != JsonValueKind.Array) - { - continue; - } - - foreach (var productIdElement in property.Value.EnumerateArray()) - { - if (productIdElement.ValueKind != JsonValueKind.String) - { - continue; - } - - var productId = productIdElement.GetString(); - if (string.IsNullOrWhiteSpace(productId)) - { - continue; - } - - if (!map.TryGetValue(productId, out var set)) - { - set = new HashSet(StringComparer.OrdinalIgnoreCase); - map[productId] = set; - } - - set.Add(statusLabel); - } - } - } - - return map.ToDictionary( - static kvp => kvp.Key, - static kvp => (IReadOnlyCollection)kvp.Value.ToArray(), - StringComparer.OrdinalIgnoreCase); - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCursor.cs deleted file mode 100644 index 58801d17..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoCursor.cs +++ /dev/null @@ -1,101 +0,0 @@ -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -internal sealed record CiscoCursor( - DateTimeOffset? LastModified, - string? LastAdvisoryId, - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings) -{ - private static readonly IReadOnlyCollection EmptyGuidCollection = Array.Empty(); - - public static CiscoCursor Empty { get; } = new(null, null, EmptyGuidCollection, EmptyGuidCollection); - - public BsonDocument ToBson() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastModified.HasValue) - { - document["lastModified"] = LastModified.Value.UtcDateTime; - } - - if (!string.IsNullOrWhiteSpace(LastAdvisoryId)) - { - document["lastAdvisoryId"] = LastAdvisoryId; - } - - return document; - } - - public static CiscoCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - DateTimeOffset? lastModified = null; - if (document.TryGetValue("lastModified", out var lastModifiedValue)) - { - lastModified = lastModifiedValue.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(lastModifiedValue.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(lastModifiedValue.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; - } - - string? lastAdvisoryId = null; - if (document.TryGetValue("lastAdvisoryId", out var idValue) && idValue.BsonType == BsonType.String) - { - var value = idValue.AsString.Trim(); - if (value.Length > 0) - { - lastAdvisoryId = value; - } - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - - return new CiscoCursor(lastModified, lastAdvisoryId, pendingDocuments, pendingMappings); - } - - public CiscoCursor WithCheckpoint(DateTimeOffset lastModified, string advisoryId) - => this with - { - LastModified = lastModified.ToUniversalTime(), - LastAdvisoryId = string.IsNullOrWhiteSpace(advisoryId) ? null : advisoryId.Trim(), - }; - - public CiscoCursor WithPendingDocuments(IEnumerable? documents) - => this with { PendingDocuments = documents?.Distinct().ToArray() ?? EmptyGuidCollection }; - - public CiscoCursor WithPendingMappings(IEnumerable? mappings) - => this with { PendingMappings = mappings?.Distinct().ToArray() ?? EmptyGuidCollection }; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string key) - { - if (!document.TryGetValue(key, out var value) || value is not BsonArray array) - { - return EmptyGuidCollection; - } - - var results = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element.ToString(), out var guid)) - { - results.Add(guid); - } - } - - return results; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDiagnostics.cs deleted file mode 100644 index 7941f2e0..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDiagnostics.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public sealed class CiscoDiagnostics : IDisposable -{ - public const string MeterName = "StellaOps.Feedser.Source.Vndr.Cisco"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _fetchDocuments; - private readonly Counter _fetchFailures; - private readonly Counter _fetchUnchanged; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Histogram _mapAffected; - - public CiscoDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _fetchDocuments = _meter.CreateCounter( - name: "cisco.fetch.documents", - unit: "documents", - description: "Number of Cisco advisories fetched."); - _fetchFailures = _meter.CreateCounter( - name: "cisco.fetch.failures", - unit: "operations", - description: "Number of Cisco fetch failures."); - _fetchUnchanged = _meter.CreateCounter( - name: "cisco.fetch.unchanged", - unit: "documents", - description: "Number of Cisco advisories skipped because they were unchanged."); - _parseSuccess = _meter.CreateCounter( - name: "cisco.parse.success", - unit: "documents", - description: "Number of Cisco documents parsed successfully."); - _parseFailures = _meter.CreateCounter( - name: "cisco.parse.failures", - unit: "documents", - description: "Number of Cisco documents that failed to parse."); - _mapSuccess = _meter.CreateCounter( - name: "cisco.map.success", - unit: "documents", - description: "Number of Cisco advisories mapped successfully."); - _mapFailures = _meter.CreateCounter( - name: "cisco.map.failures", - unit: "documents", - description: "Number of Cisco advisories that failed to map to canonical form."); - _mapAffected = _meter.CreateHistogram( - name: "cisco.map.affected.packages", - unit: "packages", - description: "Distribution of affected package counts emitted per Cisco advisory."); - } - - public Meter Meter => _meter; - - public void FetchDocument() => _fetchDocuments.Add(1); - - public void FetchFailure() => _fetchFailures.Add(1); - - public void FetchUnchanged() => _fetchUnchanged.Add(1); - - public void ParseSuccess() => _parseSuccess.Add(1); - - public void ParseFailure() => _parseFailures.Add(1); - - public void MapSuccess() => _mapSuccess.Add(1); - - public void MapFailure() => _mapFailures.Add(1); - - public void MapAffected(int count) - { - if (count >= 0) - { - _mapAffected.Record(count); - } - } - - public void Dispose() => _meter.Dispose(); -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDtoFactory.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDtoFactory.cs deleted file mode 100644 index 61a4b17d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoDtoFactory.cs +++ /dev/null @@ -1,190 +0,0 @@ -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text.Json; -using Microsoft.Extensions.Logging; -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public class CiscoDtoFactory -{ - private readonly ICiscoCsafClient _csafClient; - private readonly ILogger _logger; - - public CiscoDtoFactory(ICiscoCsafClient csafClient, ILogger logger) - { - _csafClient = csafClient ?? throw new ArgumentNullException(nameof(csafClient)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task CreateAsync(CiscoRawAdvisory raw, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(raw); - - var advisoryId = raw.AdvisoryId?.Trim(); - if (string.IsNullOrWhiteSpace(advisoryId)) - { - throw new InvalidOperationException("Cisco advisory is missing advisoryId."); - } - - var title = string.IsNullOrWhiteSpace(raw.AdvisoryTitle) ? advisoryId : raw.AdvisoryTitle!.Trim(); - var severity = SeverityNormalization.Normalize(raw.Sir); - var published = ParseDate(raw.FirstPublished); - var updated = ParseDate(raw.LastUpdated); - - CiscoCsafData? csafData = null; - if (!string.IsNullOrWhiteSpace(raw.CsafUrl)) - { - var csafContent = await _csafClient.TryFetchAsync(raw.CsafUrl, cancellationToken).ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(csafContent)) - { - try - { - csafData = CiscoCsafParser.Parse(csafContent!); - } - catch (JsonException ex) - { - _logger.LogWarning(ex, "Cisco CSAF payload parsing failed for {AdvisoryId}", advisoryId); - } - } - } - - var products = BuildProducts(raw, csafData); - var cves = NormalizeList(raw.Cves); - var bugIds = NormalizeList(raw.BugIds); - var cvss = ParseDouble(raw.CvssBaseScore); - - return new CiscoAdvisoryDto( - AdvisoryId: advisoryId, - Title: title, - Summary: string.IsNullOrWhiteSpace(raw.Summary) ? null : raw.Summary!.Trim(), - Severity: severity, - Published: published, - Updated: updated, - PublicationUrl: NormalizeUrl(raw.PublicationUrl), - CsafUrl: NormalizeUrl(raw.CsafUrl), - CvrfUrl: NormalizeUrl(raw.CvrfUrl), - CvssBaseScore: cvss, - Cves: cves, - BugIds: bugIds, - Products: products); - } - - private static IReadOnlyList BuildProducts(CiscoRawAdvisory raw, CiscoCsafData? csafData) - { - var map = new Dictionary(StringComparer.OrdinalIgnoreCase); - - if (csafData is not null) - { - foreach (var entry in csafData.ProductStatuses) - { - var productId = entry.Key; - var name = csafData.Products.TryGetValue(productId, out var product) - ? product.Name - : productId; - - var statuses = NormalizeStatuses(entry.Value); - map[name] = new CiscoAffectedProductDto( - Name: name, - ProductId: productId, - Version: raw.Version?.Trim(), - Statuses: statuses); - } - } - - var rawProducts = NormalizeList(raw.ProductNames); - foreach (var productName in rawProducts) - { - if (map.ContainsKey(productName)) - { - continue; - } - - map[productName] = new CiscoAffectedProductDto( - Name: productName, - ProductId: null, - Version: raw.Version?.Trim(), - Statuses: new[] { AffectedPackageStatusCatalog.KnownAffected }); - } - - return map.Count == 0 - ? Array.Empty() - : map.Values - .OrderBy(static p => p.Name, StringComparer.OrdinalIgnoreCase) - .ThenBy(static p => p.ProductId, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyCollection NormalizeStatuses(IEnumerable statuses) - { - var set = new SortedSet(StringComparer.OrdinalIgnoreCase); - foreach (var status in statuses) - { - if (AffectedPackageStatusCatalog.TryNormalize(status, out var normalized)) - { - set.Add(normalized); - } - else if (!string.IsNullOrWhiteSpace(status)) - { - set.Add(status.Trim().ToLowerInvariant()); - } - } - - if (set.Count == 0) - { - set.Add(AffectedPackageStatusCatalog.KnownAffected); - } - - return set; - } - - private static IReadOnlyList NormalizeList(IEnumerable? items) - { - if (items is null) - { - return Array.Empty(); - } - - var set = new SortedSet(StringComparer.OrdinalIgnoreCase); - foreach (var item in items) - { - if (!string.IsNullOrWhiteSpace(item)) - { - set.Add(item.Trim()); - } - } - - return set.Count == 0 ? Array.Empty() : set.ToArray(); - } - - private static double? ParseDouble(string? value) - => double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) - ? parsed - : null; - - private static DateTimeOffset? ParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) - { - return parsed.ToUniversalTime(); - } - - return null; - } - - private static string? NormalizeUrl(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - return Uri.TryCreate(value.Trim(), UriKind.Absolute, out var uri) ? uri.ToString() : null; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoMapper.cs deleted file mode 100644 index 204b849f..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoMapper.cs +++ /dev/null @@ -1,263 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Common.Packages; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public static class CiscoMapper -{ - public static Advisory Map(CiscoAdvisoryDto dto, DocumentRecord document, DtoRecord dtoRecord) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - ArgumentNullException.ThrowIfNull(dtoRecord); - - var recordedAt = dtoRecord.ValidatedAt.ToUniversalTime(); - var fetchProvenance = new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - "document", - document.Uri, - document.FetchedAt.ToUniversalTime()); - - var mapProvenance = new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - "map", - dto.AdvisoryId, - recordedAt); - - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var affected = BuildAffectedPackages(dto, recordedAt); - - return new Advisory( - advisoryKey: dto.AdvisoryId, - title: dto.Title, - summary: dto.Summary, - language: "en", - published: dto.Published, - modified: dto.Updated, - severity: dto.Severity, - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affected, - cvssMetrics: Array.Empty(), - provenance: new[] { fetchProvenance, mapProvenance }); - } - - private static IReadOnlyList BuildAliases(CiscoAdvisoryDto dto) - { - var set = new HashSet(StringComparer.OrdinalIgnoreCase) - { - dto.AdvisoryId, - }; - - foreach (var cve in dto.Cves) - { - if (!string.IsNullOrWhiteSpace(cve)) - { - set.Add(cve.Trim()); - } - } - - foreach (var bugId in dto.BugIds) - { - if (!string.IsNullOrWhiteSpace(bugId)) - { - set.Add(bugId.Trim()); - } - } - - if (dto.PublicationUrl is not null) - { - set.Add(dto.PublicationUrl); - } - - return set.Count == 0 - ? Array.Empty() - : set.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase).ToArray(); - } - - private static IReadOnlyList BuildReferences(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) - { - var list = new List(3); - AddReference(list, dto.PublicationUrl, "publication", recordedAt); - AddReference(list, dto.CvrfUrl, "cvrf", recordedAt); - AddReference(list, dto.CsafUrl, "csaf", recordedAt); - - return list.Count == 0 - ? Array.Empty() - : list.OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase).ToArray(); - } - - private static void AddReference(ICollection references, string? url, string kind, DateTimeOffset recordedAt) - { - if (string.IsNullOrWhiteSpace(url)) - { - return; - } - - if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) - { - return; - } - - var provenance = new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - $"reference:{kind}", - uri.ToString(), - recordedAt); - - try - { - references.Add(new AdvisoryReference( - url: uri.ToString(), - kind: kind, - sourceTag: null, - summary: null, - provenance: provenance)); - } - catch (ArgumentException) - { - // ignore invalid URLs - } - } - - private static IReadOnlyList BuildAffectedPackages(CiscoAdvisoryDto dto, DateTimeOffset recordedAt) - { - if (dto.Products.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Products.Count); - foreach (var product in dto.Products) - { - if (string.IsNullOrWhiteSpace(product.Name)) - { - continue; - } - - var range = BuildVersionRange(product, recordedAt); - var statuses = BuildStatuses(product, recordedAt); - var provenance = new[] - { - new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - "affected", - product.ProductId ?? product.Name, - recordedAt), - }; - - packages.Add(new AffectedPackage( - type: AffectedPackageTypes.Vendor, - identifier: product.Name, - platform: null, - versionRanges: range is null ? Array.Empty() : new[] { range }, - statuses: statuses, - provenance: provenance, - normalizedVersions: Array.Empty())); - } - - return packages.Count == 0 - ? Array.Empty() - : packages.OrderBy(static p => p.Identifier, StringComparer.OrdinalIgnoreCase).ToArray(); - } - - private static AffectedVersionRange? BuildVersionRange(CiscoAffectedProductDto product, DateTimeOffset recordedAt) - { - if (string.IsNullOrWhiteSpace(product.Version)) - { - return null; - } - - var version = product.Version.Trim(); - RangePrimitives? primitives = null; - string rangeKind = "vendor"; - string? rangeExpression = version; - - if (PackageCoordinateHelper.TryParseSemVer(version, out _, out var normalized)) - { - var semver = new SemVerPrimitive( - Introduced: null, - IntroducedInclusive: true, - Fixed: null, - FixedInclusive: false, - LastAffected: null, - LastAffectedInclusive: true, - ConstraintExpression: null, - ExactValue: normalized); - - primitives = new RangePrimitives(semver, null, null, BuildVendorExtensions(product)); - rangeKind = "semver"; - rangeExpression = normalized; - } - else - { - primitives = new RangePrimitives(null, null, null, BuildVendorExtensions(product, includeVersion: true)); - } - - var provenance = new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - "range", - product.ProductId ?? product.Name, - recordedAt); - - return new AffectedVersionRange( - rangeKind: rangeKind, - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: rangeExpression, - provenance: provenance, - primitives: primitives); - } - - private static IReadOnlyDictionary? BuildVendorExtensions(CiscoAffectedProductDto product, bool includeVersion = false) - { - var dictionary = new Dictionary(StringComparer.Ordinal); - if (!string.IsNullOrWhiteSpace(product.ProductId)) - { - dictionary["cisco.productId"] = product.ProductId!; - } - - if (includeVersion && !string.IsNullOrWhiteSpace(product.Version)) - { - dictionary["cisco.version.raw"] = product.Version!; - } - - return dictionary.Count == 0 ? null : dictionary; - } - - private static IReadOnlyList BuildStatuses(CiscoAffectedProductDto product, DateTimeOffset recordedAt) - { - if (product.Statuses is null || product.Statuses.Count == 0) - { - return Array.Empty(); - } - - var list = new List(product.Statuses.Count); - foreach (var status in product.Statuses) - { - if (!AffectedPackageStatusCatalog.TryNormalize(status, out var normalized) - || string.IsNullOrWhiteSpace(normalized)) - { - continue; - } - - var provenance = new AdvisoryProvenance( - VndrCiscoConnectorPlugin.SourceName, - "status", - product.ProductId ?? product.Name, - recordedAt); - - list.Add(new AffectedPackageStatus(normalized, provenance)); - } - - return list.Count == 0 ? Array.Empty() : list; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs deleted file mode 100644 index 1d166753..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOAuthMessageHandler.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.IO; -using System.Net; -using System.Net.Http.Headers; -using Microsoft.Extensions.Logging; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -internal sealed class CiscoOAuthMessageHandler : DelegatingHandler -{ - private readonly CiscoAccessTokenProvider _tokenProvider; - private readonly ILogger _logger; - - public CiscoOAuthMessageHandler( - CiscoAccessTokenProvider tokenProvider, - ILogger logger) - { - _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(request); - - HttpRequestMessage? retryTemplate = null; - try - { - retryTemplate = await CloneRequestAsync(request, cancellationToken).ConfigureAwait(false); - } - catch (IOException) - { - // Unable to buffer content; retry will fail if needed. - retryTemplate = null; - } - - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); - var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); - if (response.StatusCode != HttpStatusCode.Unauthorized) - { - return response; - } - - response.Dispose(); - _logger.LogWarning("Cisco openVuln request returned 401 Unauthorized; refreshing access token."); - await _tokenProvider.RefreshAsync(cancellationToken).ConfigureAwait(false); - - if (retryTemplate is null) - { - _tokenProvider.Invalidate(); - throw new HttpRequestException("Cisco openVuln request returned 401 Unauthorized and could not be retried."); - } - - retryTemplate.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await _tokenProvider.GetTokenAsync(cancellationToken).ConfigureAwait(false)); - - try - { - var retryResponse = await base.SendAsync(retryTemplate, cancellationToken).ConfigureAwait(false); - if (retryResponse.StatusCode == HttpStatusCode.Unauthorized) - { - _tokenProvider.Invalidate(); - } - - return retryResponse; - } - finally - { - retryTemplate.Dispose(); - } - } - - private static async Task CloneRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - var clone = new HttpRequestMessage(request.Method, request.RequestUri) - { - Version = request.Version, - VersionPolicy = request.VersionPolicy, - }; - - foreach (var header in request.Headers) - { - clone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - if (request.Content is not null) - { - using var memory = new MemoryStream(); - await request.Content.CopyToAsync(memory, cancellationToken).ConfigureAwait(false); - memory.Position = 0; - var buffer = memory.ToArray(); - var contentClone = new ByteArrayContent(buffer); - foreach (var header in request.Content.Headers) - { - contentClone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - clone.Content = contentClone; - } - - return clone; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs deleted file mode 100644 index 8331bcf0..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoOpenVulnClient.cs +++ /dev/null @@ -1,196 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Source.Vndr.Cisco.Configuration; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public sealed class CiscoOpenVulnClient -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - NumberHandling = JsonNumberHandling.AllowReadingFromString, - }; - - private readonly SourceFetchService _fetchService; - private readonly IOptionsMonitor _options; - private readonly ILogger _logger; - private readonly string _sourceName; - - public CiscoOpenVulnClient( - SourceFetchService fetchService, - IOptionsMonitor options, - ILogger logger, - string sourceName) - { - _fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService)); - _options = options ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - _sourceName = sourceName ?? throw new ArgumentNullException(nameof(sourceName)); - } - - internal async Task FetchAsync(DateOnly date, int pageIndex, CancellationToken cancellationToken) - { - var options = _options.CurrentValue; - var requestUri = options.BuildLastModifiedUri(date, pageIndex, options.PageSize); - var request = new SourceFetchRequest(CiscoOptions.HttpClientName, _sourceName, requestUri) - { - AcceptHeaders = new[] { "application/json" }, - TimeoutOverride = options.RequestTimeout, - }; - - var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false); - if (!result.IsSuccess || result.Content is null) - { - _logger.LogDebug("Cisco openVuln request returned empty payload for {Uri} (status {Status})", requestUri, result.StatusCode); - return null; - } - - return CiscoAdvisoryPage.Parse(result.Content); - } -} - -internal sealed record CiscoAdvisoryPage( - IReadOnlyList Advisories, - CiscoPagination Pagination) -{ - public bool HasMore => Pagination.PageIndex < Pagination.TotalPages; - - public static CiscoAdvisoryPage Parse(byte[] content) - { - using var document = JsonDocument.Parse(content); - var root = document.RootElement; - var advisories = new List(); - - if (root.TryGetProperty("advisories", out var advisoriesElement) && advisoriesElement.ValueKind == JsonValueKind.Array) - { - foreach (var advisory in advisoriesElement.EnumerateArray()) - { - if (!TryCreateItem(advisory, out var item)) - { - continue; - } - - advisories.Add(item); - } - } - - var pagination = CiscoPagination.FromJson(root.TryGetProperty("pagination", out var paginationElement) ? paginationElement : default); - return new CiscoAdvisoryPage(advisories, pagination); - } - - private static bool TryCreateItem(JsonElement advisory, [NotNullWhen(true)] out CiscoAdvisoryItem? item) - { - var rawJson = advisory.GetRawText(); - var advisoryId = GetString(advisory, "advisoryId"); - if (string.IsNullOrWhiteSpace(advisoryId)) - { - item = null; - return false; - } - - var lastUpdated = ParseDate(GetString(advisory, "lastUpdated")); - var firstPublished = ParseDate(GetString(advisory, "firstPublished")); - var severity = GetString(advisory, "sir"); - var publicationUrl = GetString(advisory, "publicationUrl"); - var csafUrl = GetString(advisory, "csafUrl"); - var cvrfUrl = GetString(advisory, "cvrfUrl"); - var cvss = GetString(advisory, "cvssBaseScore"); - - var cves = ReadStringArray(advisory, "cves"); - var bugIds = ReadStringArray(advisory, "bugIDs"); - var productNames = ReadStringArray(advisory, "productNames"); - - item = new CiscoAdvisoryItem( - advisoryId, - lastUpdated, - firstPublished, - severity, - publicationUrl, - csafUrl, - cvrfUrl, - cvss, - cves, - bugIds, - productNames, - rawJson); - return true; - } - - private static string? GetString(JsonElement element, string propertyName) - => element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String - ? value.GetString() - : null; - - private static DateTimeOffset? ParseDate(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return null; - } - - if (DateTimeOffset.TryParse(value, out var parsed)) - { - return parsed.ToUniversalTime(); - } - - return null; - } - - private static IReadOnlyList ReadStringArray(JsonElement element, string property) - { - if (!element.TryGetProperty(property, out var value) || value.ValueKind != JsonValueKind.Array) - { - return Array.Empty(); - } - - var results = new List(); - foreach (var child in value.EnumerateArray()) - { - if (child.ValueKind == JsonValueKind.String) - { - var text = child.GetString(); - if (!string.IsNullOrWhiteSpace(text)) - { - results.Add(text.Trim()); - } - } - } - - return results; - } -} - -internal sealed record CiscoAdvisoryItem( - string AdvisoryId, - DateTimeOffset? LastUpdated, - DateTimeOffset? FirstPublished, - string? Severity, - string? PublicationUrl, - string? CsafUrl, - string? CvrfUrl, - string? CvssBaseScore, - IReadOnlyList Cves, - IReadOnlyList BugIds, - IReadOnlyList ProductNames, - string RawJson) -{ - public byte[] GetRawBytes() => Encoding.UTF8.GetBytes(RawJson); -} - -internal sealed record CiscoPagination(int PageIndex, int PageSize, int TotalPages, int TotalRecords) -{ - public static CiscoPagination FromJson(JsonElement element) - { - var pageIndex = element.TryGetProperty("pageIndex", out var index) && index.TryGetInt32(out var parsedIndex) ? parsedIndex : 1; - var pageSize = element.TryGetProperty("pageSize", out var size) && size.TryGetInt32(out var parsedSize) ? parsedSize : 0; - var totalPages = element.TryGetProperty("totalPages", out var pages) && pages.TryGetInt32(out var parsedPages) ? parsedPages : pageIndex; - var totalRecords = element.TryGetProperty("totalRecords", out var records) && records.TryGetInt32(out var parsedRecords) ? parsedRecords : 0; - return new CiscoPagination(pageIndex, pageSize, totalPages, totalRecords); - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoRawAdvisory.cs b/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoRawAdvisory.cs deleted file mode 100644 index 37a745d6..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/Internal/CiscoRawAdvisory.cs +++ /dev/null @@ -1,64 +0,0 @@ -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Vndr.Cisco.Internal; - -public class CiscoRawAdvisory -{ - [JsonPropertyName("advisoryId")] - public string? AdvisoryId { get; set; } - - [JsonPropertyName("advisoryTitle")] - public string? AdvisoryTitle { get; set; } - - [JsonPropertyName("publicationUrl")] - public string? PublicationUrl { get; set; } - - [JsonPropertyName("cvrfUrl")] - public string? CvrfUrl { get; set; } - - [JsonPropertyName("csafUrl")] - public string? CsafUrl { get; set; } - - [JsonPropertyName("summary")] - public string? Summary { get; set; } - - [JsonPropertyName("sir")] - public string? Sir { get; set; } - - [JsonPropertyName("firstPublished")] - public string? FirstPublished { get; set; } - - [JsonPropertyName("lastUpdated")] - public string? LastUpdated { get; set; } - - [JsonPropertyName("productNames")] - public List? ProductNames { get; set; } - - [JsonPropertyName("version")] - public string? Version { get; set; } - - [JsonPropertyName("iosRelease")] - public string? IosRelease { get; set; } - - [JsonPropertyName("cves")] - public List? Cves { get; set; } - - [JsonPropertyName("bugIDs")] - public List? BugIds { get; set; } - - [JsonPropertyName("cvssBaseScore")] - public string? CvssBaseScore { get; set; } - - [JsonPropertyName("cvssTemporalScore")] - public string? CvssTemporalScore { get; set; } - - [JsonPropertyName("cvssEnvironmentalScore")] - public string? CvssEnvironmentalScore { get; set; } - - [JsonPropertyName("cvssBaseScoreVersion2")] - public string? CvssBaseScoreV2 { get; set; } - - [JsonPropertyName("status")] - public string? Status { get; set; } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj b/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj deleted file mode 100644 index effa7961..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Cisco/StellaOps.Feedser.Source.Vndr.Cisco.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/StellaOps.Feedser.Source.Vndr.Msrc.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/StellaOps.Feedser.Source.Vndr.Msrc.Tests.csproj deleted file mode 100644 index f3e0a677..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc.Tests/StellaOps.Feedser.Source.Vndr.Msrc.Tests.csproj +++ /dev/null @@ -1,24 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - - - - - PreserveNewest - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Configuration/MsrcOptions.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Configuration/MsrcOptions.cs deleted file mode 100644 index 3e23fdf6..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Configuration/MsrcOptions.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System.Globalization; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Configuration; - -public sealed class MsrcOptions -{ - public const string HttpClientName = "feedser.source.vndr.msrc"; - public const string TokenClientName = "feedser.source.vndr.msrc.token"; - - public Uri BaseUri { get; set; } = new("https://api.msrc.microsoft.com/sug/v2.0/", UriKind.Absolute); - - public string Locale { get; set; } = "en-US"; - - public string ApiVersion { get; set; } = "2024-08-01"; - - /// - /// Azure AD tenant identifier used for client credential flow. - /// - public string TenantId { get; set; } = string.Empty; - - /// - /// Azure AD application (client) identifier. - /// - public string ClientId { get; set; } = string.Empty; - - /// - /// Azure AD client secret used for token acquisition. - /// - public string ClientSecret { get; set; } = string.Empty; - - /// - /// Scope requested during client-credential token acquisition. - /// - public string Scope { get; set; } = "api://api.msrc.microsoft.com/.default"; - - /// - /// Maximum advisories to fetch per cycle. - /// - public int MaxAdvisoriesPerFetch { get; set; } = 200; - - /// - /// Page size used when iterating the MSRC API. - /// - public int PageSize { get; set; } = 100; - - /// - /// Overlap window added when resuming from the last modified cursor. - /// - public TimeSpan CursorOverlap { get; set; } = TimeSpan.FromMinutes(10); - - /// - /// When enabled the connector downloads the CVRF artefact referenced by each advisory. - /// - public bool DownloadCvrf { get; set; } = false; - - public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250); - - public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5); - - /// - /// Optional lower bound for the initial sync if the cursor is empty. - /// - public DateTimeOffset? InitialLastModified { get; set; } = DateTimeOffset.UtcNow.AddDays(-30); - - public void Validate() - { - if (BaseUri is null || !BaseUri.IsAbsoluteUri) - { - throw new InvalidOperationException("MSRC base URI must be absolute."); - } - - if (string.IsNullOrWhiteSpace(Locale)) - { - throw new InvalidOperationException("Locale must be provided."); - } - - if (!string.IsNullOrWhiteSpace(Locale) && !CultureInfo.GetCultures(CultureTypes.AllCultures).Any(c => string.Equals(c.Name, Locale, StringComparison.OrdinalIgnoreCase))) - { - throw new InvalidOperationException($"Locale '{Locale}' is not recognised."); - } - - if (string.IsNullOrWhiteSpace(ApiVersion)) - { - throw new InvalidOperationException("API version must be provided."); - } - - if (!Guid.TryParse(TenantId, out _)) - { - throw new InvalidOperationException("TenantId must be a valid GUID."); - } - - if (string.IsNullOrWhiteSpace(ClientId)) - { - throw new InvalidOperationException("ClientId must be provided."); - } - - if (string.IsNullOrWhiteSpace(ClientSecret)) - { - throw new InvalidOperationException("ClientSecret must be provided."); - } - - if (string.IsNullOrWhiteSpace(Scope)) - { - throw new InvalidOperationException("Scope must be provided."); - } - - if (MaxAdvisoriesPerFetch <= 0) - { - throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero."); - } - - if (PageSize <= 0 || PageSize > 500) - { - throw new InvalidOperationException($"{nameof(PageSize)} must be between 1 and 500."); - } - - if (CursorOverlap < TimeSpan.Zero || CursorOverlap > TimeSpan.FromHours(6)) - { - throw new InvalidOperationException($"{nameof(CursorOverlap)} must be within 0-6 hours."); - } - - if (RequestDelay < TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative."); - } - - if (FailureBackoff <= TimeSpan.Zero) - { - throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive."); - } - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs deleted file mode 100644 index a5b0b61b..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcAdvisoryDto.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed record MsrcAdvisoryDto -{ - public string AdvisoryId { get; init; } = string.Empty; - - public string Title { get; init; } = string.Empty; - - public string? Description { get; init; } - - public string? Severity { get; init; } - - public DateTimeOffset? ReleaseDate { get; init; } - - public DateTimeOffset? LastModifiedDate { get; init; } - - public IReadOnlyList CveIds { get; init; } = Array.Empty(); - - public IReadOnlyList KbIds { get; init; } = Array.Empty(); - - public IReadOnlyList Threats { get; init; } = Array.Empty(); - - public IReadOnlyList Remediations { get; init; } = Array.Empty(); - - public IReadOnlyList Products { get; init; } = Array.Empty(); - - public double? CvssBaseScore { get; init; } - - public string? CvssVector { get; init; } - - public string? ReleaseNoteUrl { get; init; } - - public string? CvrfUrl { get; init; } -} - -public sealed record MsrcAdvisoryThreat(string Type, string? Description, string? Severity); - -public sealed record MsrcAdvisoryRemediation(string Type, string? Description, string? Url, string? Kb); - -public sealed record MsrcAdvisoryProduct( - string Identifier, - string? ProductName, - string? Platform, - string? Architecture, - string? BuildNumber, - string? Cpe); diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcApiClient.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcApiClient.cs deleted file mode 100644 index 9a4fa7e6..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcApiClient.cs +++ /dev/null @@ -1,138 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Net.Http.Json; -using System.Text; -using System.Text.Json; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed class MsrcApiClient -{ - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) - { - PropertyNameCaseInsensitive = true, - WriteIndented = false, - }; - - private readonly IHttpClientFactory _httpClientFactory; - private readonly IMsrcTokenProvider _tokenProvider; - private readonly MsrcOptions _options; - private readonly ILogger _logger; - - public MsrcApiClient( - IHttpClientFactory httpClientFactory, - IMsrcTokenProvider tokenProvider, - IOptions options, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _tokenProvider = tokenProvider ?? throw new ArgumentNullException(nameof(tokenProvider)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task> FetchSummariesAsync(DateTimeOffset fromInclusive, DateTimeOffset toExclusive, CancellationToken cancellationToken) - { - var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); - - var results = new List(); - var requestUri = BuildSummaryUri(fromInclusive, toExclusive); - - while (requestUri is not null) - { - using var request = new HttpRequestMessage(HttpMethod.Get, requestUri); - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) - { - var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - throw new HttpRequestException($"MSRC summary fetch failed with {(int)response.StatusCode}. Body: {preview}"); - } - - var payload = await response.Content.ReadFromJsonAsync(SerializerOptions, cancellationToken).ConfigureAwait(false) - ?? new MsrcSummaryResponse(); - - results.AddRange(payload.Value); - - if (string.IsNullOrWhiteSpace(payload.NextLink)) - { - break; - } - - requestUri = new Uri(payload.NextLink, UriKind.Absolute); - } - - return results; - } - - public Uri BuildDetailUri(string vulnerabilityId) - { - var uri = CreateDetailUriInternal(vulnerabilityId); - return uri; - } - - public async Task FetchDetailAsync(string vulnerabilityId, CancellationToken cancellationToken) - { - var client = await CreateAuthenticatedClientAsync(cancellationToken).ConfigureAwait(false); - var uri = CreateDetailUriInternal(vulnerabilityId); - - using var request = new HttpRequestMessage(HttpMethod.Get, uri); - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) - { - var preview = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - throw new HttpRequestException($"MSRC detail fetch failed for {vulnerabilityId} with {(int)response.StatusCode}. Body: {preview}"); - } - - return await response.Content.ReadAsByteArrayAsync(cancellationToken).ConfigureAwait(false); - } - - private async Task CreateAuthenticatedClientAsync(CancellationToken cancellationToken) - { - var token = await _tokenProvider.GetAccessTokenAsync(cancellationToken).ConfigureAwait(false); - var client = _httpClientFactory.CreateClient(MsrcOptions.HttpClientName); - client.DefaultRequestHeaders.Remove("Authorization"); - client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}"); - client.DefaultRequestHeaders.Remove("Accept"); - client.DefaultRequestHeaders.Add("Accept", "application/json"); - client.DefaultRequestHeaders.Remove("api-version"); - client.DefaultRequestHeaders.Add("api-version", _options.ApiVersion); - client.DefaultRequestHeaders.Remove("Accept-Language"); - client.DefaultRequestHeaders.Add("Accept-Language", _options.Locale); - return client; - } - - private Uri BuildSummaryUri(DateTimeOffset fromInclusive, DateTimeOffset toExclusive) - { - var builder = new StringBuilder(); - builder.Append(_options.BaseUri.ToString().TrimEnd('/')); - builder.Append("/vulnerabilities?"); - builder.Append("$top=").Append(_options.PageSize); - builder.Append("&lastModifiedStartDateTime=").Append(Uri.EscapeDataString(fromInclusive.ToUniversalTime().ToString("O"))); - builder.Append("&lastModifiedEndDateTime=").Append(Uri.EscapeDataString(toExclusive.ToUniversalTime().ToString("O"))); - builder.Append("&$orderby=lastModifiedDate"); - builder.Append("&locale=").Append(Uri.EscapeDataString(_options.Locale)); - builder.Append("&api-version=").Append(Uri.EscapeDataString(_options.ApiVersion)); - - return new Uri(builder.ToString(), UriKind.Absolute); - } - - private Uri CreateDetailUriInternal(string vulnerabilityId) - { - if (string.IsNullOrWhiteSpace(vulnerabilityId)) - { - throw new ArgumentException("Vulnerability identifier must be provided.", nameof(vulnerabilityId)); - } - - var baseUri = _options.BaseUri.ToString().TrimEnd('/'); - var path = $"{baseUri}/vulnerability/{Uri.EscapeDataString(vulnerabilityId)}?api-version={Uri.EscapeDataString(_options.ApiVersion)}&locale={Uri.EscapeDataString(_options.Locale)}"; - return new Uri(path, UriKind.Absolute); - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcCursor.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcCursor.cs deleted file mode 100644 index cc1e0763..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcCursor.cs +++ /dev/null @@ -1,87 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using MongoDB.Bson; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -internal sealed record MsrcCursor( - IReadOnlyCollection PendingDocuments, - IReadOnlyCollection PendingMappings, - DateTimeOffset? LastModifiedCursor) -{ - private static readonly IReadOnlyCollection EmptyGuidSet = Array.Empty(); - - public static MsrcCursor Empty { get; } = new(EmptyGuidSet, EmptyGuidSet, null); - - public MsrcCursor WithPendingDocuments(IEnumerable documents) - => this with { PendingDocuments = Distinct(documents) }; - - public MsrcCursor WithPendingMappings(IEnumerable mappings) - => this with { PendingMappings = Distinct(mappings) }; - - public MsrcCursor WithLastModifiedCursor(DateTimeOffset? timestamp) - => this with { LastModifiedCursor = timestamp }; - - public BsonDocument ToBsonDocument() - { - var document = new BsonDocument - { - ["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())), - ["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())), - }; - - if (LastModifiedCursor.HasValue) - { - document["lastModifiedCursor"] = LastModifiedCursor.Value.UtcDateTime; - } - - return document; - } - - public static MsrcCursor FromBson(BsonDocument? document) - { - if (document is null || document.ElementCount == 0) - { - return Empty; - } - - var pendingDocuments = ReadGuidArray(document, "pendingDocuments"); - var pendingMappings = ReadGuidArray(document, "pendingMappings"); - var lastModified = document.TryGetValue("lastModifiedCursor", out var value) - ? ParseDate(value) - : null; - - return new MsrcCursor(pendingDocuments, pendingMappings, lastModified); - } - - private static IReadOnlyCollection Distinct(IEnumerable? values) - => values?.Distinct().ToArray() ?? EmptyGuidSet; - - private static IReadOnlyCollection ReadGuidArray(BsonDocument document, string field) - { - if (!document.TryGetValue(field, out var value) || value is not BsonArray array) - { - return EmptyGuidSet; - } - - var items = new List(array.Count); - foreach (var element in array) - { - if (Guid.TryParse(element?.ToString(), out var id)) - { - items.Add(id); - } - } - - return items; - } - - private static DateTimeOffset? ParseDate(BsonValue value) - => value.BsonType switch - { - BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc), - BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(), - _ => null, - }; -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailDto.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailDto.cs deleted file mode 100644 index 7526e232..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailDto.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed record MsrcVulnerabilityDetailDto -{ - [JsonPropertyName("id")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("vulnerabilityId")] - public string VulnerabilityId { get; init; } = string.Empty; - - [JsonPropertyName("cveNumber")] - public string? CveNumber { get; init; } - - [JsonPropertyName("cveNumbers")] - public IReadOnlyList CveNumbers { get; init; } = Array.Empty(); - - [JsonPropertyName("title")] - public string Title { get; init; } = string.Empty; - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("releaseDate")] - public DateTimeOffset? ReleaseDate { get; init; } - - [JsonPropertyName("lastModifiedDate")] - public DateTimeOffset? LastModifiedDate { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("threats")] - public IReadOnlyList Threats { get; init; } = Array.Empty(); - - [JsonPropertyName("remediations")] - public IReadOnlyList Remediations { get; init; } = Array.Empty(); - - [JsonPropertyName("affectedProducts")] - public IReadOnlyList AffectedProducts { get; init; } = Array.Empty(); - - [JsonPropertyName("cvssV3")] - public MsrcCvssDto? Cvss { get; init; } - - [JsonPropertyName("releaseNoteUrl")] - public string? ReleaseNoteUrl { get; init; } - - [JsonPropertyName("cvrfUrl")] - public string? CvrfUrl { get; init; } -} - -public sealed record MsrcThreatDto -{ - [JsonPropertyName("type")] - public string? Type { get; init; } - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } -} - -public sealed record MsrcRemediationDto -{ - [JsonPropertyName("id")] - public string? Id { get; init; } - - [JsonPropertyName("type")] - public string? Type { get; init; } - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("url")] - public string? Url { get; init; } - - [JsonPropertyName("kbNumber")] - public string? KbNumber { get; init; } -} - -public sealed record MsrcAffectedProductDto -{ - [JsonPropertyName("productId")] - public string? ProductId { get; init; } - - [JsonPropertyName("productName")] - public string? ProductName { get; init; } - - [JsonPropertyName("cpe")] - public string? Cpe { get; init; } - - [JsonPropertyName("platform")] - public string? Platform { get; init; } - - [JsonPropertyName("architecture")] - public string? Architecture { get; init; } - - [JsonPropertyName("buildNumber")] - public string? BuildNumber { get; init; } -} - -public sealed record MsrcCvssDto -{ - [JsonPropertyName("baseScore")] - public double? BaseScore { get; init; } - - [JsonPropertyName("vectorString")] - public string? VectorString { get; init; } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailParser.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailParser.cs deleted file mode 100644 index a684599d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDetailParser.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Linq; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed class MsrcDetailParser -{ - public MsrcAdvisoryDto Parse(MsrcVulnerabilityDetailDto detail) - { - ArgumentNullException.ThrowIfNull(detail); - - var advisoryId = string.IsNullOrWhiteSpace(detail.VulnerabilityId) ? detail.Id : detail.VulnerabilityId; - var cveIds = detail.CveNumbers?.Where(static c => !string.IsNullOrWhiteSpace(c)).Select(static c => c.Trim()).ToArray() - ?? (string.IsNullOrWhiteSpace(detail.CveNumber) ? Array.Empty() : new[] { detail.CveNumber! }); - - var kbIds = detail.Remediations? - .Where(static remediation => !string.IsNullOrWhiteSpace(remediation.KbNumber)) - .Select(static remediation => remediation.KbNumber!.Trim()) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToArray() ?? Array.Empty(); - - return new MsrcAdvisoryDto - { - AdvisoryId = advisoryId, - Title = string.IsNullOrWhiteSpace(detail.Title) ? advisoryId : detail.Title.Trim(), - Description = detail.Description, - Severity = detail.Severity, - ReleaseDate = detail.ReleaseDate, - LastModifiedDate = detail.LastModifiedDate, - CveIds = cveIds, - KbIds = kbIds, - Threats = detail.Threats?.Select(static threat => new MsrcAdvisoryThreat( - threat.Type ?? "unspecified", - threat.Description, - threat.Severity)).ToArray() ?? Array.Empty(), - Remediations = detail.Remediations?.Select(static remediation => new MsrcAdvisoryRemediation( - remediation.Type ?? "unspecified", - remediation.Description, - remediation.Url, - remediation.KbNumber)).ToArray() ?? Array.Empty(), - Products = detail.AffectedProducts?.Select(product => - new MsrcAdvisoryProduct( - BuildProductIdentifier(product), - product.ProductName, - product.Platform, - product.Architecture, - product.BuildNumber, - product.Cpe)).ToArray() ?? Array.Empty(), - CvssBaseScore = detail.Cvss?.BaseScore, - CvssVector = detail.Cvss?.VectorString, - ReleaseNoteUrl = detail.ReleaseNoteUrl, - CvrfUrl = detail.CvrfUrl, - }; - } - - private static string BuildProductIdentifier(MsrcAffectedProductDto product) - { - var name = string.IsNullOrWhiteSpace(product.ProductName) ? product.ProductId : product.ProductName; - if (string.IsNullOrWhiteSpace(name)) - { - name = "Unknown Product"; - } - - if (!string.IsNullOrWhiteSpace(product.BuildNumber)) - { - return $"{name} build {product.BuildNumber}"; - } - - return name; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDiagnostics.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDiagnostics.cs deleted file mode 100644 index 00ada34d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDiagnostics.cs +++ /dev/null @@ -1,129 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.Metrics; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed class MsrcDiagnostics : IDisposable -{ - private const string MeterName = "StellaOps.Feedser.Source.Vndr.Msrc"; - private const string MeterVersion = "1.0.0"; - - private readonly Meter _meter; - private readonly Counter _summaryFetchAttempts; - private readonly Counter _summaryFetchSuccess; - private readonly Counter _summaryFetchFailures; - private readonly Histogram _summaryItemCount; - private readonly Histogram _summaryWindowHours; - private readonly Counter _detailFetchAttempts; - private readonly Counter _detailFetchSuccess; - private readonly Counter _detailFetchNotModified; - private readonly Counter _detailFetchFailures; - private readonly Histogram _detailEnqueued; - private readonly Counter _parseSuccess; - private readonly Counter _parseFailures; - private readonly Histogram _parseProductCount; - private readonly Histogram _parseKbCount; - private readonly Counter _mapSuccess; - private readonly Counter _mapFailures; - private readonly Histogram _mapAliasCount; - private readonly Histogram _mapAffectedCount; - - public MsrcDiagnostics() - { - _meter = new Meter(MeterName, MeterVersion); - _summaryFetchAttempts = _meter.CreateCounter("msrc.summary.fetch.attempts", "operations"); - _summaryFetchSuccess = _meter.CreateCounter("msrc.summary.fetch.success", "operations"); - _summaryFetchFailures = _meter.CreateCounter("msrc.summary.fetch.failures", "operations"); - _summaryItemCount = _meter.CreateHistogram("msrc.summary.items.count", "items"); - _summaryWindowHours = _meter.CreateHistogram("msrc.summary.window.hours", "hours"); - _detailFetchAttempts = _meter.CreateCounter("msrc.detail.fetch.attempts", "operations"); - _detailFetchSuccess = _meter.CreateCounter("msrc.detail.fetch.success", "operations"); - _detailFetchNotModified = _meter.CreateCounter("msrc.detail.fetch.not_modified", "operations"); - _detailFetchFailures = _meter.CreateCounter("msrc.detail.fetch.failures", "operations"); - _detailEnqueued = _meter.CreateHistogram("msrc.detail.enqueued.count", "documents"); - _parseSuccess = _meter.CreateCounter("msrc.parse.success", "documents"); - _parseFailures = _meter.CreateCounter("msrc.parse.failures", "documents"); - _parseProductCount = _meter.CreateHistogram("msrc.parse.products.count", "products"); - _parseKbCount = _meter.CreateHistogram("msrc.parse.kb.count", "kb"); - _mapSuccess = _meter.CreateCounter("msrc.map.success", "advisories"); - _mapFailures = _meter.CreateCounter("msrc.map.failures", "advisories"); - _mapAliasCount = _meter.CreateHistogram("msrc.map.aliases.count", "aliases"); - _mapAffectedCount = _meter.CreateHistogram("msrc.map.affected.count", "packages"); - } - - public void SummaryFetchAttempt() => _summaryFetchAttempts.Add(1); - - public void SummaryFetchSuccess(int count, double? windowHours) - { - _summaryFetchSuccess.Add(1); - if (count >= 0) - { - _summaryItemCount.Record(count); - } - - if (windowHours is { } value && value >= 0) - { - _summaryWindowHours.Record(value); - } - } - - public void SummaryFetchFailure(string reason) - => _summaryFetchFailures.Add(1, ReasonTag(reason)); - - public void DetailFetchAttempt() => _detailFetchAttempts.Add(1); - - public void DetailFetchSuccess() => _detailFetchSuccess.Add(1); - - public void DetailFetchNotModified() => _detailFetchNotModified.Add(1); - - public void DetailFetchFailure(string reason) - => _detailFetchFailures.Add(1, ReasonTag(reason)); - - public void DetailEnqueued(int count) - { - if (count >= 0) - { - _detailEnqueued.Record(count); - } - } - - public void ParseSuccess(int productCount, int kbCount) - { - _parseSuccess.Add(1); - if (productCount >= 0) - { - _parseProductCount.Record(productCount); - } - - if (kbCount >= 0) - { - _parseKbCount.Record(kbCount); - } - } - - public void ParseFailure(string reason) - => _parseFailures.Add(1, ReasonTag(reason)); - - public void MapSuccess(int aliasCount, int packageCount) - { - _mapSuccess.Add(1); - if (aliasCount >= 0) - { - _mapAliasCount.Record(aliasCount); - } - - if (packageCount >= 0) - { - _mapAffectedCount.Record(packageCount); - } - } - - public void MapFailure(string reason) - => _mapFailures.Add(1, ReasonTag(reason)); - - private static KeyValuePair ReasonTag(string reason) - => new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant()); - - public void Dispose() => _meter.Dispose(); -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs deleted file mode 100644 index 26d0528d..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcDocumentMetadata.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System; -using System.Collections.Generic; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -internal static class MsrcDocumentMetadata -{ - public static Dictionary CreateMetadata(MsrcVulnerabilitySummary summary) - { - var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - ["msrc.vulnerabilityId"] = summary.VulnerabilityId ?? summary.Id, - ["msrc.id"] = summary.Id, - }; - - if (summary.LastModifiedDate.HasValue) - { - metadata["msrc.lastModified"] = summary.LastModifiedDate.Value.ToString("O"); - } - - if (summary.ReleaseDate.HasValue) - { - metadata["msrc.releaseDate"] = summary.ReleaseDate.Value.ToString("O"); - } - - if (!string.IsNullOrWhiteSpace(summary.CvrfUrl)) - { - metadata["msrc.cvrfUrl"] = summary.CvrfUrl!; - } - - if (summary.CveNumbers.Count > 0) - { - metadata["msrc.cves"] = string.Join(",", summary.CveNumbers); - } - - return metadata; - } - - public static Dictionary CreateCvrfMetadata(MsrcVulnerabilitySummary summary) - { - var metadata = CreateMetadata(summary); - metadata["msrc.cvrf"] = "true"; - return metadata; - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcMapper.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcMapper.cs deleted file mode 100644 index ff507360..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcMapper.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Storage.Mongo.Documents; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -internal static class MsrcMapper -{ - public static Advisory Map(MsrcAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt) - { - ArgumentNullException.ThrowIfNull(dto); - ArgumentNullException.ThrowIfNull(document); - - var advisoryKey = dto.AdvisoryId; - var aliases = BuildAliases(dto); - var references = BuildReferences(dto, recordedAt); - var affectedPackages = BuildPackages(dto, recordedAt); - var cvssMetrics = BuildCvss(dto, recordedAt); - - var provenance = new AdvisoryProvenance( - source: MsrcConnectorPlugin.SourceName, - kind: "advisory", - value: advisoryKey, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - advisoryKey: advisoryKey, - title: dto.Title, - summary: dto.Description, - language: "en", - published: dto.ReleaseDate, - modified: dto.LastModifiedDate, - severity: NormalizeSeverity(dto.Severity), - exploitKnown: false, - aliases: aliases, - references: references, - affectedPackages: affectedPackages, - cvssMetrics: cvssMetrics, - provenance: new[] { provenance }); - } - - private static IReadOnlyList BuildAliases(MsrcAdvisoryDto dto) - { - var aliases = new List { dto.AdvisoryId }; - foreach (var cve in dto.CveIds) - { - if (!string.IsNullOrWhiteSpace(cve)) - { - aliases.Add(cve); - } - } - - foreach (var kb in dto.KbIds) - { - if (!string.IsNullOrWhiteSpace(kb)) - { - aliases.Add(kb.StartsWith("KB", StringComparison.OrdinalIgnoreCase) ? kb : $"KB{kb}"); - } - } - - return aliases - .Where(static alias => !string.IsNullOrWhiteSpace(alias)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildReferences(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) - { - var references = new List(); - - if (!string.IsNullOrWhiteSpace(dto.ReleaseNoteUrl)) - { - references.Add(CreateReference(dto.ReleaseNoteUrl!, "details", recordedAt)); - } - - if (!string.IsNullOrWhiteSpace(dto.CvrfUrl)) - { - references.Add(CreateReference(dto.CvrfUrl!, "cvrf", recordedAt)); - } - - foreach (var remediation in dto.Remediations) - { - if (!string.IsNullOrWhiteSpace(remediation.Url)) - { - references.Add(CreateReference( - remediation.Url!, - string.Equals(remediation.Type, "security update", StringComparison.OrdinalIgnoreCase) ? "remediation" : remediation.Type ?? "reference", - recordedAt, - remediation.Description)); - } - } - - return references - .DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .OrderBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static AdvisoryReference CreateReference(string url, string kind, DateTimeOffset recordedAt, string? summary = null) - => new( - url, - kind: kind.ToLowerInvariant(), - sourceTag: "msrc", - summary: summary, - provenance: new AdvisoryProvenance( - MsrcConnectorPlugin.SourceName, - "reference", - url, - recordedAt, - new[] { ProvenanceFieldMasks.References })); - - private static IReadOnlyList BuildPackages(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) - { - if (dto.Products.Count == 0) - { - return Array.Empty(); - } - - var packages = new List(dto.Products.Count); - foreach (var product in dto.Products) - { - var identifier = string.IsNullOrWhiteSpace(product.Identifier) ? "Unknown Product" : product.Identifier; - var provenance = new AdvisoryProvenance( - MsrcConnectorPlugin.SourceName, - "package", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.AffectedPackages }); - - var notes = new List(); - if (!string.IsNullOrWhiteSpace(product.Platform)) - { - notes.Add($"platform:{product.Platform}"); - } - - if (!string.IsNullOrWhiteSpace(product.Architecture)) - { - notes.Add($"arch:{product.Architecture}"); - } - - if (!string.IsNullOrWhiteSpace(product.Cpe)) - { - notes.Add($"cpe:{product.Cpe}"); - } - - var range = !string.IsNullOrWhiteSpace(product.BuildNumber) - ? new[] - { - new AffectedVersionRange( - rangeKind: "custom", - introducedVersion: null, - fixedVersion: null, - lastAffectedVersion: null, - rangeExpression: $"build:{product.BuildNumber}", - provenance: new AdvisoryProvenance( - MsrcConnectorPlugin.SourceName, - "package-range", - identifier, - recordedAt, - new[] { ProvenanceFieldMasks.VersionRanges })), - } - : Array.Empty(); - - var normalizedRules = !string.IsNullOrWhiteSpace(product.BuildNumber) - ? new[] - { - new NormalizedVersionRule( - scheme: "msrc.build", - type: NormalizedVersionRuleTypes.Exact, - value: product.BuildNumber, - notes: string.Join(";", notes.Where(static n => !string.IsNullOrWhiteSpace(n)))) - } - : Array.Empty(); - - packages.Add(new AffectedPackage( - type: AffectedPackageTypes.Vendor, - identifier: identifier, - platform: product.Platform, - versionRanges: range, - statuses: Array.Empty(), - provenance: new[] { provenance }, - normalizedVersions: normalizedRules)); - } - - return packages - .DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase) - .ToArray(); - } - - private static IReadOnlyList BuildCvss(MsrcAdvisoryDto dto, DateTimeOffset recordedAt) - { - if (dto.CvssBaseScore is null || string.IsNullOrWhiteSpace(dto.CvssVector)) - { - return Array.Empty(); - } - - var severity = CvssSeverityFromScore(dto.CvssBaseScore.Value); - - return new[] - { - new CvssMetric( - version: "3.1", - vector: dto.CvssVector!, - baseScore: dto.CvssBaseScore.Value, - baseSeverity: severity, - provenance: new AdvisoryProvenance( - MsrcConnectorPlugin.SourceName, - "cvss", - dto.AdvisoryId, - recordedAt, - new[] { ProvenanceFieldMasks.CvssMetrics })), - }; - } - - private static string CvssSeverityFromScore(double score) - => score switch - { - < 0 => "none", - < 4 => "low", - < 7 => "medium", - < 9 => "high", - _ => "critical", - }; - - private static string? NormalizeSeverity(string? severity) - { - if (string.IsNullOrWhiteSpace(severity)) - { - return null; - } - - return severity.Trim().ToLowerInvariant(); - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcSummaryResponse.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcSummaryResponse.cs deleted file mode 100644 index a8fadee1..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcSummaryResponse.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public sealed record MsrcSummaryResponse -{ - [JsonPropertyName("value")] - public List Value { get; init; } = new(); - - [JsonPropertyName("@odata.nextLink")] - public string? NextLink { get; init; } -} - -public sealed record MsrcVulnerabilitySummary -{ - [JsonPropertyName("id")] - public string Id { get; init; } = string.Empty; - - [JsonPropertyName("vulnerabilityId")] - public string? VulnerabilityId { get; init; } - - [JsonPropertyName("cveNumber")] - public string? CveNumber { get; init; } - - [JsonPropertyName("cveNumbers")] - public IReadOnlyList CveNumbers { get; init; } = Array.Empty(); - - [JsonPropertyName("title")] - public string? Title { get; init; } - - [JsonPropertyName("description")] - public string? Description { get; init; } - - [JsonPropertyName("releaseDate")] - public DateTimeOffset? ReleaseDate { get; init; } - - [JsonPropertyName("lastModifiedDate")] - public DateTimeOffset? LastModifiedDate { get; init; } - - [JsonPropertyName("severity")] - public string? Severity { get; init; } - - [JsonPropertyName("cvrfUrl")] - public string? CvrfUrl { get; init; } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcTokenProvider.cs b/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcTokenProvider.cs deleted file mode 100644 index 8511a3c8..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Msrc/Internal/MsrcTokenProvider.cs +++ /dev/null @@ -1,106 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; -using System.Net.Http.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Source.Vndr.Msrc.Configuration; - -namespace StellaOps.Feedser.Source.Vndr.Msrc.Internal; - -public interface IMsrcTokenProvider -{ - Task GetAccessTokenAsync(CancellationToken cancellationToken); -} - -public sealed class MsrcTokenProvider : IMsrcTokenProvider, IDisposable -{ - private readonly IHttpClientFactory _httpClientFactory; - private readonly MsrcOptions _options; - private readonly TimeProvider _timeProvider; - private readonly ILogger _logger; - private readonly SemaphoreSlim _refreshLock = new(1, 1); - - private AccessToken? _currentToken; - - public MsrcTokenProvider( - IHttpClientFactory httpClientFactory, - IOptions options, - TimeProvider? timeProvider, - ILogger logger) - { - _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); - _options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options)); - _options.Validate(); - _timeProvider = timeProvider ?? TimeProvider.System; - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public async Task GetAccessTokenAsync(CancellationToken cancellationToken) - { - var token = _currentToken; - if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) - { - return token.Token; - } - - await _refreshLock.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - token = _currentToken; - if (token is not null && !token.IsExpired(_timeProvider.GetUtcNow())) - { - return token.Token; - } - - _logger.LogInformation("Requesting new MSRC access token"); - var client = _httpClientFactory.CreateClient(MsrcOptions.TokenClientName); - var request = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) - { - Content = new FormUrlEncodedContent(new Dictionary - { - ["client_id"] = _options.ClientId, - ["client_secret"] = _options.ClientSecret, - ["grant_type"] = "client_credentials", - ["scope"] = _options.Scope, - }), - }; - - using var response = await client.SendAsync(request, cancellationToken).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - - var payload = await response.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false) - ?? throw new InvalidOperationException("AAD token response was null."); - - var expiresAt = _timeProvider.GetUtcNow().AddSeconds(payload.ExpiresIn - 60); - _currentToken = new AccessToken(payload.AccessToken, expiresAt); - return payload.AccessToken; - } - finally - { - _refreshLock.Release(); - } - } - - private Uri BuildTokenUri() - => new($"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"); - - public void Dispose() => _refreshLock.Dispose(); - - private sealed record AccessToken(string Token, DateTimeOffset ExpiresAt) - { - public bool IsExpired(DateTimeOffset now) => now >= ExpiresAt; - } - - private sealed record TokenResponse - { - [JsonPropertyName("access_token")] - public string AccessToken { get; init; } = string.Empty; - - [JsonPropertyName("expires_in")] - public int ExpiresIn { get; init; } - } -} diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj deleted file mode 100644 index 4316bac2..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle.Tests/StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs deleted file mode 100644 index b773c235..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Oracle.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj b/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj deleted file mode 100644 index 75922ca4..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Oracle/StellaOps.Feedser.Source.Vndr.Oracle.csproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj deleted file mode 100644 index 88018948..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware.Tests/StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - - - PreserveNewest - - - diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs deleted file mode 100644 index 1e127068..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Vndr.Vmware.Tests")] diff --git a/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj b/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj deleted file mode 100644 index 76cc57c2..00000000 --- a/src/StellaOps.Feedser.Source.Vndr.Vmware/StellaOps.Feedser.Source.Vndr.Vmware.csproj +++ /dev/null @@ -1,23 +0,0 @@ - - - - net10.0 - enable - enable - - - - - - - - - - - - - <_Parameter1>StellaOps.Feedser.Tests - - - - diff --git a/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj b/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj deleted file mode 100644 index 910f1f4e..00000000 --- a/src/StellaOps.Feedser.Storage.Mongo.Tests/StellaOps.Feedser.Storage.Mongo.Tests.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - diff --git a/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs deleted file mode 100644 index f7f3209b..00000000 --- a/src/StellaOps.Feedser.Storage.Mongo/Advisories/IAdvisoryStore.cs +++ /dev/null @@ -1,14 +0,0 @@ -using StellaOps.Feedser.Models; - -namespace StellaOps.Feedser.Storage.Mongo.Advisories; - -public interface IAdvisoryStore -{ - Task UpsertAsync(Advisory advisory, CancellationToken cancellationToken); - - Task FindAsync(string advisoryKey, CancellationToken cancellationToken); - - Task> GetRecentAsync(int limit, CancellationToken cancellationToken); - - IAsyncEnumerable StreamAsync(CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs deleted file mode 100644 index 5ce08818..00000000 --- a/src/StellaOps.Feedser.Storage.Mongo/Documents/IDocumentStore.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace StellaOps.Feedser.Storage.Mongo.Documents; - -public interface IDocumentStore -{ - Task UpsertAsync(DocumentRecord record, CancellationToken cancellationToken); - - Task FindBySourceAndUriAsync(string sourceName, string uri, CancellationToken cancellationToken); - - Task FindAsync(Guid id, CancellationToken cancellationToken); - - Task UpdateStatusAsync(Guid id, string status, CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs b/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs deleted file mode 100644 index 07806e94..00000000 --- a/src/StellaOps.Feedser.Storage.Mongo/Dtos/IDtoStore.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace StellaOps.Feedser.Storage.Mongo.Dtos; - -public interface IDtoStore -{ - Task UpsertAsync(DtoRecord record, CancellationToken cancellationToken); - - Task FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken); - - Task> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs deleted file mode 100644 index 6a4ba72a..00000000 --- a/src/StellaOps.Feedser.Storage.Mongo/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Feedser.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj b/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj deleted file mode 100644 index 51c5b3b7..00000000 --- a/src/StellaOps.Feedser.WebService.Tests/StellaOps.Feedser.WebService.Tests.csproj +++ /dev/null @@ -1,13 +0,0 @@ - - - net10.0 - enable - enable - - - - - - - - diff --git a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs b/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs deleted file mode 100644 index 73920247..00000000 --- a/src/StellaOps.Feedser.WebService/Extensions/JobRegistrationExtensions.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Feedser.Core.Jobs; -using StellaOps.Feedser.Merge.Jobs; - -namespace StellaOps.Feedser.WebService.Extensions; - -internal static class JobRegistrationExtensions -{ - private sealed record BuiltInJob( - string Kind, - string JobType, - string AssemblyName, - TimeSpan Timeout, - TimeSpan LeaseDuration, - string? CronExpression = null); - - private static readonly IReadOnlyList BuiltInJobs = new List - { - new("source:redhat:fetch", "StellaOps.Feedser.Source.Distro.RedHat.RedHatFetchJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(12), TimeSpan.FromMinutes(6), "0,15,30,45 * * * *"), - new("source:redhat:parse", "StellaOps.Feedser.Source.Distro.RedHat.RedHatParseJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(6), "5,20,35,50 * * * *"), - new("source:redhat:map", "StellaOps.Feedser.Source.Distro.RedHat.RedHatMapJob", "StellaOps.Feedser.Source.Distro.RedHat", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(6), "10,25,40,55 * * * *"), - - new("source:cert-in:fetch", "StellaOps.Feedser.Source.CertIn.CertInFetchJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:cert-in:parse", "StellaOps.Feedser.Source.CertIn.CertInParseJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:cert-in:map", "StellaOps.Feedser.Source.CertIn.CertInMapJob", "StellaOps.Feedser.Source.CertIn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:cert-fr:fetch", "StellaOps.Feedser.Source.CertFr.CertFrFetchJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:cert-fr:parse", "StellaOps.Feedser.Source.CertFr.CertFrParseJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:cert-fr:map", "StellaOps.Feedser.Source.CertFr.CertFrMapJob", "StellaOps.Feedser.Source.CertFr", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:jvn:fetch", "StellaOps.Feedser.Source.Jvn.JvnFetchJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:jvn:parse", "StellaOps.Feedser.Source.Jvn.JvnParseJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:jvn:map", "StellaOps.Feedser.Source.Jvn.JvnMapJob", "StellaOps.Feedser.Source.Jvn", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:ics-kaspersky:fetch", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyFetchJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:ics-kaspersky:parse", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyParseJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:ics-kaspersky:map", "StellaOps.Feedser.Source.Ics.Kaspersky.KasperskyMapJob", "StellaOps.Feedser.Source.Ics.Kaspersky", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:osv:fetch", "StellaOps.Feedser.Source.Osv.OsvFetchJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:osv:parse", "StellaOps.Feedser.Source.Osv.OsvParseJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:osv:map", "StellaOps.Feedser.Source.Osv.OsvMapJob", "StellaOps.Feedser.Source.Osv", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:vmware:fetch", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareFetchJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:vmware:parse", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareParseJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:vmware:map", "StellaOps.Feedser.Source.Vndr.Vmware.VmwareMapJob", "StellaOps.Feedser.Source.Vndr.Vmware", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("source:vndr-oracle:fetch", "StellaOps.Feedser.Source.Vndr.Oracle.OracleFetchJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:vndr-oracle:parse", "StellaOps.Feedser.Source.Vndr.Oracle.OracleParseJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - new("source:vndr-oracle:map", "StellaOps.Feedser.Source.Vndr.Oracle.OracleMapJob", "StellaOps.Feedser.Source.Vndr.Oracle", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)), - - new("export:json", "StellaOps.Feedser.Exporter.Json.JsonExportJob", "StellaOps.Feedser.Exporter.Json", TimeSpan.FromMinutes(10), TimeSpan.FromMinutes(5)), - new("export:trivy-db", "StellaOps.Feedser.Exporter.TrivyDb.TrivyDbExportJob", "StellaOps.Feedser.Exporter.TrivyDb", TimeSpan.FromMinutes(20), TimeSpan.FromMinutes(10)), - new("merge:reconcile", "StellaOps.Feedser.Merge.Jobs.MergeReconcileJob", "StellaOps.Feedser.Merge", TimeSpan.FromMinutes(15), TimeSpan.FromMinutes(5)) - }; - - public static IServiceCollection AddBuiltInFeedserJobs(this IServiceCollection services) - { - ArgumentNullException.ThrowIfNull(services); - - services.PostConfigure(options => - { - foreach (var registration in BuiltInJobs) - { - if (options.Definitions.ContainsKey(registration.Kind)) - { - continue; - } - - var jobType = Type.GetType( - $"{registration.JobType}, {registration.AssemblyName}", - throwOnError: false, - ignoreCase: false); - - if (jobType is null) - { - continue; - } - - var timeout = registration.Timeout > TimeSpan.Zero ? registration.Timeout : options.DefaultTimeout; - var lease = registration.LeaseDuration > TimeSpan.Zero ? registration.LeaseDuration : options.DefaultLeaseDuration; - - options.Definitions[registration.Kind] = new JobDefinition( - registration.Kind, - jobType, - timeout, - lease, - registration.CronExpression, - Enabled: true); - } - }); - - return services; - } -} diff --git a/src/StellaOps.Feedser.WebService/TASKS.md b/src/StellaOps.Feedser.WebService/TASKS.md deleted file mode 100644 index 8191864c..00000000 --- a/src/StellaOps.Feedser.WebService/TASKS.md +++ /dev/null @@ -1,24 +0,0 @@ -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|Bind & validate FeedserOptions|BE-Base|WebService|DONE – options bound/validated with failure logging.| -|Mongo service wiring|BE-Base|Storage.Mongo|DONE – wiring delegated to `AddMongoStorage`.| -|Bootstrapper execution on start|BE-Base|Storage.Mongo|DONE – startup calls `MongoBootstrapper.InitializeAsync`.| -|Plugin host options finalization|BE-Base|Plugins|DONE – default plugin directories/search patterns configured.| -|Jobs API contract tests|QA|Core|DONE – WebServiceEndpointsTests now cover success payloads, filtering, and trigger outcome mapping.| -|Health/Ready probes|DevOps|Ops|DONE – `/health` and `/ready` endpoints implemented.| -|Serilog + OTEL integration hooks|BE-Base|Observability|DONE – `TelemetryExtensions` wires Serilog + OTEL with configurable exporters.| -|Register built-in jobs (sources/exporters)|BE-Base|Core|DONE – AddBuiltInFeedserJobs adds fallback scheduler definitions for core connectors and exporters via reflection.| -|HTTP problem details consistency|BE-Base|WebService|DONE – API errors now emit RFC7807 responses with trace identifiers and typed problem categories.| -|Request logging and metrics|BE-Base|Observability|DONE – Serilog request logging enabled with enriched context and web.jobs counters published via OpenTelemetry.| -|Endpoint smoke tests (health/ready/jobs error paths)|QA|WebService|DONE – WebServiceEndpointsTests assert success and problem responses for health, ready, and job trigger error paths.| -|Batch job definition last-run lookup|BE-Base|Core|DONE – definitions endpoint now precomputes kinds array and reuses batched last-run dictionary; manual smoke verified via local GET `/jobs/definitions`.| -|Add no-cache headers to health/readiness/jobs APIs|BE-Base|WebService|DONE – helper applies Cache-Control/Pragma/Expires on all health/ready/jobs endpoints; awaiting automated probe tests once connector fixtures stabilize.| -|Authority configuration parity (FSR1)|DevEx/Feedser|Authority options schema|**DONE (2025-10-10)** – Options post-config loads clientSecretFile fallback, validators normalize scopes/audiences, and sample config documents issuer/credential/bypass settings.| -|Document authority toggle & scope requirements|Docs/Feedser|Authority integration|**DOING (2025-10-10)** – Quickstart updated with staging flag, client credentials, env overrides; operator guide refresh pending Docs guild review.| -|Plumb Authority client resilience options|BE-Base|Auth libraries LIB5|**DONE (2025-10-12)** – `Program.cs` wires `authority.resilience.*` + client scopes into `AddStellaOpsAuthClient`; new integration test asserts binding and retries.| -|Author ops guidance for resilience tuning|Docs/Feedser|Plumb Authority client resilience options|**DONE (2025-10-12)** – `docs/21_INSTALL_GUIDE.md` + `docs/ops/feedser-authority-audit-runbook.md` document resilience profiles for connected vs air-gapped installs and reference monitoring cues.| -|Document authority bypass logging patterns|Docs/Feedser|FSR3 logging|**DONE (2025-10-12)** – Updated operator guides clarify `Feedser.Authorization.Audit` fields (route/status/subject/clientId/scopes/bypass/remote) and SIEM triggers.| -|Update Feedser operator guide for enforcement cutoff|Docs/Feedser|FSR1 rollout|**DONE (2025-10-12)** – Installation guide emphasises disabling `allowAnonymousFallback` before 2025-12-31 UTC and connects audit signals to the rollout checklist.| -|Rename plugin drop directory to namespaced path|BE-Base|Plugins|**TODO** – Point Feedser source/exporter build outputs to `StellaOps.Feedser.PluginBinaries`, update PluginHost defaults/search patterns to match, ensure Offline Kit packaging/tests expect the new folder, and document migration guidance for operators.| -|Authority resilience adoption|Feedser WebService, Docs|Plumb Authority client resilience options|**BLOCKED (2025-10-10)** – Roll out retry/offline knobs to deployment docs and confirm CLI parity once LIB5 lands; unblock after resilience options wired and tested.| diff --git a/src/StellaOps.Notify.Connectors.Email/AGENTS.md b/src/StellaOps.Notify.Connectors.Email/AGENTS.md new file mode 100644 index 00000000..c85e6afe --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Connectors.Email — Agent Charter + +## Mission +Implement SMTP connector plug-in per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs new file mode 100644 index 00000000..96ae3e6b --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/EmailChannelTestProvider.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Email; + +[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] +public sealed class EmailChannelTestProvider : INotifyChannelTestProvider +{ + public NotifyChannelType ChannelType => NotifyChannelType.Email; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var subject = context.Request.Title ?? "Stella Ops Notify Preview"; + var summary = context.Request.Summary ?? $"Preview generated at {context.Timestamp:O}."; + var htmlBody = !string.IsNullOrWhiteSpace(context.Request.Body) + ? context.Request.Body! + : $"

          {summary}

          Trace: {context.TraceId}

          "; + var textBody = context.Request.TextBody ?? $"{summary}{Environment.NewLine}Trace: {context.TraceId}"; + + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Email, + NotifyDeliveryFormat.Email, + context.Target, + subject, + htmlBody, + summary, + textBody, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(htmlBody), + context.Request.Attachments); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["email.to"] = context.Target + }; + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj b/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj new file mode 100644 index 00000000..54540bcb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/StellaOps.Notify.Connectors.Email.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Email/TASKS.md b/src/StellaOps.Notify.Connectors.Email/TASKS.md new file mode 100644 index 00000000..9fb0f18b --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Email/TASKS.md @@ -0,0 +1,7 @@ +# Notify Email Connector Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-CONN-EMAIL-15-701 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement SMTP connector with STARTTLS/implicit TLS support, HTML+text rendering, attachment policy enforcement. | Integration tests with SMTP stub pass; TLS enforced; attachments blocked per policy. | +| NOTIFY-CONN-EMAIL-15-702 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-701 | Add DKIM signing optional support and health/test-send flows. | DKIM optional config verified; test-send passes; secrets handled securely. | +| NOTIFY-CONN-EMAIL-15-703 | TODO | Notify Connectors Guild | NOTIFY-CONN-EMAIL-15-702 | Package Email connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/email/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Slack/AGENTS.md b/src/StellaOps.Notify.Connectors.Slack/AGENTS.md new file mode 100644 index 00000000..1829cf9a --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Connectors.Slack — Agent Charter + +## Mission +Deliver Slack connector plug-in per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs new file mode 100644 index 00000000..6c0daa7e --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/SlackChannelTestProvider.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Slack; + +[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] +public sealed class SlackChannelTestProvider : INotifyChannelTestProvider +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public NotifyChannelType ChannelType => NotifyChannelType.Slack; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var title = context.Request.Title ?? $"Stella Ops Notify Preview"; + var summary = context.Request.Summary ?? $"Preview generated for Slack destination at {context.Timestamp:O}."; + var bodyText = context.Request.Body ?? summary; + + var payload = new + { + text = $"{title}\n{bodyText}", + blocks = new object[] + { + new + { + type = "section", + text = new { type = "mrkdwn", text = $"*{title}*\n{bodyText}" } + }, + new + { + type = "context", + elements = new object[] + { + new { type = "mrkdwn", text = $"Preview generated {context.Timestamp:O} · Trace `{context.TraceId}`" } + } + } + } + }; + + var body = JsonSerializer.Serialize(payload, JsonOptions); + + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Slack, + NotifyDeliveryFormat.Slack, + context.Target, + title, + body, + summary, + context.Request.TextBody ?? bodyText, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["slack.channel"] = context.Target + }; + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj b/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj new file mode 100644 index 00000000..54540bcb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/StellaOps.Notify.Connectors.Slack.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Slack/TASKS.md b/src/StellaOps.Notify.Connectors.Slack/TASKS.md new file mode 100644 index 00000000..45cb428a --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Slack/TASKS.md @@ -0,0 +1,7 @@ +# Notify Slack Connector Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-CONN-SLACK-15-501 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement Slack connector with bot token auth, message rendering (blocks), rate limit handling, retries/backoff. | Integration tests stub Slack API; retries/jitter validated; 429 handling documented. | +| NOTIFY-CONN-SLACK-15-502 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-501 | Health check & test-send support with minimal scopes and redacted tokens. | `/channels/{id}/test` hitting Slack stub passes; secrets never logged; health endpoint returns diagnostics. | +| NOTIFY-CONN-SLACK-15-503 | TODO | Notify Connectors Guild | NOTIFY-CONN-SLACK-15-502 | Package Slack connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/slack/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Teams/AGENTS.md b/src/StellaOps.Notify.Connectors.Teams/AGENTS.md new file mode 100644 index 00000000..ad1102e6 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Connectors.Teams — Agent Charter + +## Mission +Implement Microsoft Teams connector plug-in per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj b/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj new file mode 100644 index 00000000..54540bcb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/StellaOps.Notify.Connectors.Teams.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Teams/TASKS.md b/src/StellaOps.Notify.Connectors.Teams/TASKS.md new file mode 100644 index 00000000..4b49b6dc --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/TASKS.md @@ -0,0 +1,7 @@ +# Notify Teams Connector Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-CONN-TEAMS-15-601 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement Teams connector using Adaptive Cards 1.5, handle webhook auth, size limits, retries. | Adaptive card payloads validated; 413/429 handling implemented; integration tests cover success/fail. | +| NOTIFY-CONN-TEAMS-15-602 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-601 | Provide health/test-send support with fallback text for legacy clients. | Test-send returns card preview; fallback text logged; docs updated. | +| NOTIFY-CONN-TEAMS-15-603 | TODO | Notify Connectors Guild | NOTIFY-CONN-TEAMS-15-602 | Package Teams connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/teams/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs new file mode 100644 index 00000000..2e2120e0 --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Teams/TeamsChannelTestProvider.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Teams; + +[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] +public sealed class TeamsChannelTestProvider : INotifyChannelTestProvider +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public NotifyChannelType ChannelType => NotifyChannelType.Teams; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var title = context.Request.Title ?? "Stella Ops Notify Preview"; + var summary = context.Request.Summary ?? $"Preview generated at {context.Timestamp:O}."; + var bodyContent = context.Request.Body ?? summary; + + var card = new + { + type = "AdaptiveCard", + version = "1.5", + body = new object[] + { + new { type = "TextBlock", weight = "Bolder", text = title, wrap = true }, + new { type = "TextBlock", text = bodyContent, wrap = true }, + new { type = "TextBlock", spacing = "None", isSubtle = true, text = $"Trace: {context.TraceId}", wrap = true } + } + }; + + var payload = new + { + type = "message", + attachments = new object[] + { + new + { + contentType = "application/vnd.microsoft.card.adaptive", + content = card + } + } + }; + + var body = JsonSerializer.Serialize(payload, JsonOptions); + + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Teams, + NotifyDeliveryFormat.Teams, + context.Target, + title, + body, + summary, + context.Request.TextBody ?? bodyContent, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["teams.webhook"] = context.Target + }; + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Connectors.Webhook/AGENTS.md b/src/StellaOps.Notify.Connectors.Webhook/AGENTS.md new file mode 100644 index 00000000..04a6541a --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Connectors.Webhook — Agent Charter + +## Mission +Implement generic webhook connector plug-in per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj b/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj new file mode 100644 index 00000000..54540bcb --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/StellaOps.Notify.Connectors.Webhook.csproj @@ -0,0 +1,13 @@ + + + net10.0 + enable + enable + + + + + + + + diff --git a/src/StellaOps.Notify.Connectors.Webhook/TASKS.md b/src/StellaOps.Notify.Connectors.Webhook/TASKS.md new file mode 100644 index 00000000..51a8d5db --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/TASKS.md @@ -0,0 +1,7 @@ +# Notify Webhook Connector Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-CONN-WEBHOOK-15-801 | TODO | Notify Connectors Guild | NOTIFY-ENGINE-15-303 | Implement webhook connector: JSON payload, signature (HMAC/Ed25519), retries/backoff, status code handling. | Integration tests with webhook stub validate signatures, retries, error handling; payload schema documented. | +| NOTIFY-CONN-WEBHOOK-15-802 | DOING (2025-10-19) | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-801 | Health/test-send support with signature validation hints and secret management. | Test-send returns success with sample payload; docs include verification guide; secrets never logged. | +| NOTIFY-CONN-WEBHOOK-15-803 | TODO | Notify Connectors Guild | NOTIFY-CONN-WEBHOOK-15-802 | Package Webhook connector as restart-time plug-in (manifest + host registration). | Plugin manifest added; host loads connector from `plugins/notify/webhook/`; restart validation passes. | diff --git a/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs b/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs new file mode 100644 index 00000000..17cfaeba --- /dev/null +++ b/src/StellaOps.Notify.Connectors.Webhook/WebhookChannelTestProvider.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Connectors.Webhook; + +[ServiceBinding(typeof(INotifyChannelTestProvider), ServiceLifetime.Singleton)] +public sealed class WebhookChannelTestProvider : INotifyChannelTestProvider +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public NotifyChannelType ChannelType => NotifyChannelType.Webhook; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var title = context.Request.Title ?? "Stella Ops Notify Preview"; + var summary = context.Request.Summary ?? $"Preview generated at {context.Timestamp:O}."; + + var payload = new + { + title, + summary, + traceId = context.TraceId, + timestamp = context.Timestamp, + body = context.Request.Body, + metadata = context.Request.Metadata + }; + + var body = JsonSerializer.Serialize(payload, JsonOptions); + + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Webhook, + NotifyDeliveryFormat.Webhook, + context.Target, + title, + body, + summary, + context.Request.TextBody ?? summary, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["webhook.endpoint"] = context.Target + }; + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } +} diff --git a/src/StellaOps.Notify.Engine/AGENTS.md b/src/StellaOps.Notify.Engine/AGENTS.md new file mode 100644 index 00000000..b9598a95 --- /dev/null +++ b/src/StellaOps.Notify.Engine/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Engine — Agent Charter + +## Mission +Deliver rule evaluation, digest, and rendering logic per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Engine/ChannelTestPreviewContracts.cs b/src/StellaOps.Notify.Engine/ChannelTestPreviewContracts.cs new file mode 100644 index 00000000..1851b6c1 --- /dev/null +++ b/src/StellaOps.Notify.Engine/ChannelTestPreviewContracts.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Engine; + +/// +/// Contract implemented by Notify channel plug-ins to generate channel-specific test preview payloads. +/// +public interface INotifyChannelTestProvider +{ + /// + /// Channel type supported by the provider. + /// + NotifyChannelType ChannelType { get; } + + /// + /// Builds a channel-specific preview for a test-send request. + /// + Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken); +} + +/// +/// Sanitised request payload passed to channel plug-ins when building a preview. +/// +public sealed record ChannelTestPreviewRequest( + string? TargetOverride, + string? TemplateId, + string? Title, + string? Summary, + string? Body, + string? TextBody, + string? Locale, + IReadOnlyDictionary Metadata, + IReadOnlyList Attachments); + +/// +/// Immutable context describing the channel and request for a test preview. +/// +public sealed record ChannelTestPreviewContext( + string TenantId, + NotifyChannel Channel, + string Target, + ChannelTestPreviewRequest Request, + DateTimeOffset Timestamp, + string TraceId); + +/// +/// Result returned by channel plug-ins for test preview generation. +/// +public sealed record ChannelTestPreviewResult( + NotifyDeliveryRendered Preview, + IReadOnlyDictionary? Metadata); + +/// +/// Exception thrown by plug-ins when preview input is invalid. +/// +public sealed class ChannelTestPreviewException : Exception +{ + public ChannelTestPreviewException(string message) + : base(message) + { + } +} + +/// +/// Shared helpers for channel preview generation. +/// +public static class ChannelTestPreviewUtilities +{ + /// + /// Computes a lowercase hex SHA-256 body hash for preview payloads. + /// + public static string ComputeBodyHash(string body) + { + var bytes = Encoding.UTF8.GetBytes(body); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Notify.Engine/StellaOps.Notify.Engine.csproj b/src/StellaOps.Notify.Engine/StellaOps.Notify.Engine.csproj new file mode 100644 index 00000000..8cbfad7e --- /dev/null +++ b/src/StellaOps.Notify.Engine/StellaOps.Notify.Engine.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Notify.Engine/TASKS.md b/src/StellaOps.Notify.Engine/TASKS.md new file mode 100644 index 00000000..605b3ba4 --- /dev/null +++ b/src/StellaOps.Notify.Engine/TASKS.md @@ -0,0 +1,8 @@ +# Notify Engine Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-ENGINE-15-301 | TODO | Notify Engine Guild | NOTIFY-MODELS-15-101 | Rules evaluation core: tenant/kind filters, severity/delta gates, VEX gating, throttling, idempotency key generation. | Unit tests cover rule permutations; idempotency keys deterministic; documentation updated. | +| NOTIFY-ENGINE-15-302 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-301 | Action planner + digest coalescer with window management and dedupe per architecture §4. | Digest windows tested; throttles and digests recorded; metrics counters exposed. | +| NOTIFY-ENGINE-15-303 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-302 | Template rendering engine (Slack, Teams, Email, Webhook) with helpers and i18n support. | Rendering fixtures validated; helpers documented; deterministic output proven via golden tests. | +| NOTIFY-ENGINE-15-304 | TODO | Notify Engine Guild | NOTIFY-ENGINE-15-303 | Test-send sandbox + preview utilities for WebService. | Preview/test functions validated; sample outputs returned; no state persisted. | diff --git a/src/StellaOps.Notify.Models.Tests/DocSampleTests.cs b/src/StellaOps.Notify.Models.Tests/DocSampleTests.cs new file mode 100644 index 00000000..4d09c66d --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/DocSampleTests.cs @@ -0,0 +1,47 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Xunit.Sdk; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class DocSampleTests +{ + [Theory] + [InlineData("notify-rule@1.sample.json")] + [InlineData("notify-channel@1.sample.json")] + [InlineData("notify-template@1.sample.json")] + [InlineData("notify-event@1.sample.json")] + public void CanonicalSamplesStayInSync(string fileName) + { + var json = LoadSample(fileName); + var node = JsonNode.Parse(json) ?? throw new InvalidOperationException("Sample JSON null."); + + string canonical = fileName switch + { + "notify-rule@1.sample.json" => NotifyCanonicalJsonSerializer.Serialize(NotifySchemaMigration.UpgradeRule(node)), + "notify-channel@1.sample.json" => NotifyCanonicalJsonSerializer.Serialize(NotifySchemaMigration.UpgradeChannel(node)), + "notify-template@1.sample.json" => NotifyCanonicalJsonSerializer.Serialize(NotifySchemaMigration.UpgradeTemplate(node)), + "notify-event@1.sample.json" => NotifyCanonicalJsonSerializer.Serialize(NotifyCanonicalJsonSerializer.Deserialize(json)), + _ => throw new ArgumentOutOfRangeException(nameof(fileName), fileName, "Unsupported sample.") + }; + + var canonicalNode = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical JSON null."); + if (!JsonNode.DeepEquals(node, canonicalNode)) + { + var expected = canonicalNode.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + var actual = node.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + throw new XunitException($"Sample '{fileName}' must remain canonical.\nExpected:\n{expected}\nActual:\n{actual}"); + } + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Notify.Models.Tests/NotifyCanonicalJsonSerializerTests.cs b/src/StellaOps.Notify.Models.Tests/NotifyCanonicalJsonSerializerTests.cs new file mode 100644 index 00000000..aad1b8b6 --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/NotifyCanonicalJsonSerializerTests.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Nodes; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class NotifyCanonicalJsonSerializerTests +{ + [Fact] + public void SerializeRuleIsDeterministic() + { + var ruleA = NotifyRule.Create( + ruleId: "rule-1", + tenantId: "tenant-a", + name: "critical", + match: NotifyRuleMatch.Create(eventKinds: new[] { NotifyEventKinds.ScannerReportReady }), + actions: new[] + { + NotifyRuleAction.Create(actionId: "b", channel: "slack:sec"), + NotifyRuleAction.Create(actionId: "a", channel: "email:soc") + }, + metadata: new Dictionary + { + ["beta"] = "2", + ["alpha"] = "1" + }, + createdAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z"), + updatedAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z")); + + var ruleB = NotifyRule.Create( + ruleId: "rule-1", + tenantId: "tenant-a", + name: "critical", + match: NotifyRuleMatch.Create(eventKinds: new[] { NotifyEventKinds.ScannerReportReady }), + actions: new[] + { + NotifyRuleAction.Create(actionId: "a", channel: "email:soc"), + NotifyRuleAction.Create(actionId: "b", channel: "slack:sec") + }, + metadata: new Dictionary + { + ["alpha"] = "1", + ["beta"] = "2" + }, + createdAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z"), + updatedAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z")); + + var jsonA = NotifyCanonicalJsonSerializer.Serialize(ruleA); + var jsonB = NotifyCanonicalJsonSerializer.Serialize(ruleB); + + Assert.Equal(jsonA, jsonB); + Assert.Contains("\"schemaVersion\":\"notify.rule@1\"", jsonA, StringComparison.Ordinal); + } + + [Fact] + public void SerializeEventOrdersPayloadKeys() + { + var payload = JsonNode.Parse("{\"b\":2,\"a\":1}"); + var @event = NotifyEvent.Create( + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + tenant: "tenant-a", + ts: DateTimeOffset.Parse("2025-10-18T05:41:22Z"), + payload: payload, + scope: NotifyEventScope.Create(repo: "ghcr.io/acme/api", digest: "sha256:123")); + + var json = NotifyCanonicalJsonSerializer.Serialize(@event); + + var payloadIndex = json.IndexOf("\"payload\":{", StringComparison.Ordinal); + Assert.NotEqual(-1, payloadIndex); + + var aIndex = json.IndexOf("\"a\":1", payloadIndex, StringComparison.Ordinal); + var bIndex = json.IndexOf("\"b\":2", payloadIndex, StringComparison.Ordinal); + + Assert.True(aIndex is >= 0 && bIndex is >= 0 && aIndex < bIndex, "Payload keys should be ordered alphabetically."); + } +} diff --git a/src/StellaOps.Notify.Models.Tests/NotifyDeliveryTests.cs b/src/StellaOps.Notify.Models.Tests/NotifyDeliveryTests.cs new file mode 100644 index 00000000..36d96fd4 --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/NotifyDeliveryTests.cs @@ -0,0 +1,46 @@ +using System; +using System.Linq; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class NotifyDeliveryTests +{ + [Fact] + public void AttemptsAreSortedChronologically() + { + var attempts = new[] + { + new NotifyDeliveryAttempt(DateTimeOffset.Parse("2025-10-19T12:25:00Z"), NotifyDeliveryAttemptStatus.Succeeded), + new NotifyDeliveryAttempt(DateTimeOffset.Parse("2025-10-19T12:15:00Z"), NotifyDeliveryAttemptStatus.Sending), + }; + + var delivery = NotifyDelivery.Create( + deliveryId: "delivery-1", + tenantId: "tenant-a", + ruleId: "rule-1", + actionId: "action-1", + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + status: NotifyDeliveryStatus.Sent, + attempts: attempts); + + Assert.Collection( + delivery.Attempts, + attempt => Assert.Equal(NotifyDeliveryAttemptStatus.Sending, attempt.Status), + attempt => Assert.Equal(NotifyDeliveryAttemptStatus.Succeeded, attempt.Status)); + } + + [Fact] + public void RenderedNormalizesAttachments() + { + var rendered = NotifyDeliveryRendered.Create( + channelType: NotifyChannelType.Slack, + format: NotifyDeliveryFormat.Slack, + target: "#sec", + title: "Alert", + body: "Body", + attachments: new[] { "B", "a", "a" }); + + Assert.Equal(new[] { "B", "a" }.OrderBy(x => x, StringComparer.Ordinal), rendered.Attachments); + } +} diff --git a/src/StellaOps.Notify.Models.Tests/NotifyRuleTests.cs b/src/StellaOps.Notify.Models.Tests/NotifyRuleTests.cs new file mode 100644 index 00000000..99d60140 --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/NotifyRuleTests.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class NotifyRuleTests +{ + [Fact] + public void ConstructorThrowsWhenActionsMissing() + { + var match = NotifyRuleMatch.Create(eventKinds: new[] { NotifyEventKinds.ScannerReportReady }); + + var exception = Assert.Throws(() => + NotifyRule.Create( + ruleId: "rule-1", + tenantId: "tenant-a", + name: "critical", + match: match, + actions: Array.Empty())); + + Assert.Contains("At least one action is required", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void ConstructorNormalizesCollections() + { + var rule = NotifyRule.Create( + ruleId: "rule-1", + tenantId: "tenant-a", + name: "critical", + match: NotifyRuleMatch.Create( + eventKinds: new[] { "Zastava.Admission", NotifyEventKinds.ScannerReportReady }), + actions: new[] + { + NotifyRuleAction.Create(actionId: "b", channel: "slack:sec-alerts", throttle: TimeSpan.FromMinutes(5)), + NotifyRuleAction.Create(actionId: "a", channel: "email:soc", metadata: new Dictionary + { + [" locale "] = " EN-us " + }) + }, + labels: new Dictionary + { + [" team "] = " SecOps " + }, + metadata: new Dictionary + { + ["source"] = "tests" + }); + + Assert.Equal(NotifySchemaVersions.Rule, rule.SchemaVersion); + Assert.Equal(new[] { "scanner.report.ready", "zastava.admission" }, rule.Match.EventKinds); + Assert.Equal(new[] { "a", "b" }, rule.Actions.Select(action => action.ActionId)); + Assert.Equal(TimeSpan.FromMinutes(5), rule.Actions.Last().Throttle); + Assert.Equal("secops", rule.Labels.Single().Value.ToLowerInvariant()); + Assert.Equal("en-us", rule.Actions.First().Metadata["locale"].ToLowerInvariant()); + + var json = NotifyCanonicalJsonSerializer.Serialize(rule); + Assert.Contains("\"schemaVersion\":\"notify.rule@1\"", json, StringComparison.Ordinal); + Assert.Contains("\"actions\":[{\"actionId\":\"a\"", json, StringComparison.Ordinal); + Assert.Contains("\"throttle\":\"PT5M\"", json, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Notify.Models.Tests/NotifySchemaMigrationTests.cs b/src/StellaOps.Notify.Models.Tests/NotifySchemaMigrationTests.cs new file mode 100644 index 00000000..7fa4881b --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/NotifySchemaMigrationTests.cs @@ -0,0 +1,101 @@ +using System; +using System.Text.Json.Nodes; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class NotifySchemaMigrationTests +{ + [Fact] + public void UpgradeRuleAddsSchemaVersionWhenMissing() + { + var json = JsonNode.Parse( + """ + { + "ruleId": "rule-legacy", + "tenantId": "tenant-1", + "name": "legacy", + "enabled": true, + "match": { "eventKinds": ["scanner.report.ready"] }, + "actions": [ { "actionId": "send", "channel": "email:legacy", "enabled": true } ], + "createdAt": "2025-10-18T00:00:00Z", + "updatedAt": "2025-10-18T00:00:00Z" + } + """)!; + + var rule = NotifySchemaMigration.UpgradeRule(json); + + Assert.Equal(NotifySchemaVersions.Rule, rule.SchemaVersion); + Assert.Equal("rule-legacy", rule.RuleId); + } + + [Fact] + public void UpgradeRuleThrowsOnUnknownSchema() + { + var json = JsonNode.Parse( + """ + { + "schemaVersion": "notify.rule@2", + "ruleId": "rule-future", + "tenantId": "tenant-1", + "name": "future", + "enabled": true, + "match": { "eventKinds": ["scanner.report.ready"] }, + "actions": [ { "actionId": "send", "channel": "email:soc", "enabled": true } ], + "createdAt": "2025-10-18T00:00:00Z", + "updatedAt": "2025-10-18T00:00:00Z" + } + """)!; + + var exception = Assert.Throws(() => NotifySchemaMigration.UpgradeRule(json)); + Assert.Contains("notify rule schema version", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void UpgradeChannelDefaultsMissingVersion() + { + var json = JsonNode.Parse( + """ + { + "channelId": "channel-email", + "tenantId": "tenant-1", + "name": "email:soc", + "type": "email", + "config": { "secretRef": "ref://notify/channels/email/soc" }, + "enabled": true, + "createdAt": "2025-10-18T00:00:00Z", + "updatedAt": "2025-10-18T00:00:00Z" + } + """)!; + + var channel = NotifySchemaMigration.UpgradeChannel(json); + + Assert.Equal(NotifySchemaVersions.Channel, channel.SchemaVersion); + Assert.Equal("channel-email", channel.ChannelId); + } + + [Fact] + public void UpgradeTemplateDefaultsMissingVersion() + { + var json = JsonNode.Parse( + """ + { + "templateId": "tmpl-slack-concise", + "tenantId": "tenant-1", + "channelType": "slack", + "key": "concise", + "locale": "en-us", + "body": "{{summary}}", + "renderMode": "markdown", + "format": "slack", + "createdAt": "2025-10-18T00:00:00Z", + "updatedAt": "2025-10-18T00:00:00Z" + } + """)!; + + var template = NotifySchemaMigration.UpgradeTemplate(json); + + Assert.Equal(NotifySchemaVersions.Template, template.SchemaVersion); + Assert.Equal("tmpl-slack-concise", template.TemplateId); + } + +} diff --git a/src/StellaOps.Notify.Models.Tests/PlatformEventSamplesTests.cs b/src/StellaOps.Notify.Models.Tests/PlatformEventSamplesTests.cs new file mode 100644 index 00000000..92df96ef --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/PlatformEventSamplesTests.cs @@ -0,0 +1,52 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; +using StellaOps.Notify.Models; +using Xunit.Sdk; + +namespace StellaOps.Notify.Models.Tests; + +public sealed class PlatformEventSamplesTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + [Theory] + [InlineData("scanner.report.ready@1.sample.json", NotifyEventKinds.ScannerReportReady)] + [InlineData("scanner.scan.completed@1.sample.json", NotifyEventKinds.ScannerScanCompleted)] + [InlineData("scheduler.rescan.delta@1.sample.json", NotifyEventKinds.SchedulerRescanDelta)] + [InlineData("attestor.logged@1.sample.json", NotifyEventKinds.AttestorLogged)] + public void PlatformEventSamplesRoundtripThroughNotifySerializer(string fileName, string expectedKind) + { + var json = LoadSample(fileName); + var notifyEvent = JsonSerializer.Deserialize(json, SerializerOptions); + + Assert.NotNull(notifyEvent); + Assert.Equal(expectedKind, notifyEvent!.Kind); + Assert.NotEqual(Guid.Empty, notifyEvent.EventId); + Assert.False(string.IsNullOrWhiteSpace(notifyEvent.Tenant)); + Assert.Equal(TimeSpan.Zero, notifyEvent.Ts.Offset); + + var canonicalJson = NotifyCanonicalJsonSerializer.Serialize(notifyEvent); + var canonicalNode = JsonNode.Parse(canonicalJson) ?? throw new InvalidOperationException("Canonical JSON null."); + var sampleNode = JsonNode.Parse(json) ?? throw new InvalidOperationException("Sample JSON null."); + + if (!JsonNode.DeepEquals(sampleNode, canonicalNode)) + { + var expected = canonicalNode.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + var actual = sampleNode.ToJsonString(new JsonSerializerOptions { WriteIndented = true }); + throw new Xunit.Sdk.XunitException($"Sample '{fileName}' must remain canonical.\nExpected:\n{expected}\nActual:\n{actual}"); + } + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to locate sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Notify.Models.Tests/StellaOps.Notify.Models.Tests.csproj b/src/StellaOps.Notify.Models.Tests/StellaOps.Notify.Models.Tests.csproj new file mode 100644 index 00000000..5db9f630 --- /dev/null +++ b/src/StellaOps.Notify.Models.Tests/StellaOps.Notify.Models.Tests.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + + + + + + + + + Always + + + Always + + + diff --git a/src/StellaOps.Notify.Models/AGENTS.md b/src/StellaOps.Notify.Models/AGENTS.md new file mode 100644 index 00000000..a7935047 --- /dev/null +++ b/src/StellaOps.Notify.Models/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Models — Agent Charter + +## Mission +Define Notify DTOs and contracts per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Models/Iso8601DurationConverter.cs b/src/StellaOps.Notify.Models/Iso8601DurationConverter.cs new file mode 100644 index 00000000..d8817ed8 --- /dev/null +++ b/src/StellaOps.Notify.Models/Iso8601DurationConverter.cs @@ -0,0 +1,28 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Xml; + +namespace StellaOps.Notify.Models; + +internal sealed class Iso8601DurationConverter : JsonConverter +{ + public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType is JsonTokenType.String) + { + var value = reader.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + return XmlConvert.ToTimeSpan(value); + } + } + + throw new JsonException("Expected ISO 8601 duration string."); + } + + public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) + { + var normalized = XmlConvert.ToString(value); + writer.WriteStringValue(normalized); + } +} diff --git a/src/StellaOps.Notify.Models/NotifyCanonicalJsonSerializer.cs b/src/StellaOps.Notify.Models/NotifyCanonicalJsonSerializer.cs new file mode 100644 index 00000000..8ce91579 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyCanonicalJsonSerializer.cs @@ -0,0 +1,637 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Notify.Models; + +/// +/// Deterministic JSON serializer tuned for Notify canonical documents. +/// +public static class NotifyCanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false, useDeterministicResolver: true); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true, useDeterministicResolver: true); + private static readonly JsonSerializerOptions ReadOptions = CreateOptions(writeIndented: false, useDeterministicResolver: false); + + private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary + { + { + typeof(NotifyRule), + new[] + { + "schemaVersion", + "ruleId", + "tenantId", + "name", + "description", + "enabled", + "match", + "actions", + "labels", + "metadata", + "createdBy", + "createdAt", + "updatedBy", + "updatedAt", + } + }, + { + typeof(NotifyRuleMatch), + new[] + { + "eventKinds", + "namespaces", + "repositories", + "digests", + "labels", + "componentPurls", + "minSeverity", + "verdicts", + "kevOnly", + "vex", + } + }, + { + typeof(NotifyRuleAction), + new[] + { + "actionId", + "channel", + "template", + "locale", + "digest", + "throttle", + "metadata", + "enabled", + } + }, + { + typeof(NotifyChannel), + new[] + { + "schemaVersion", + "channelId", + "tenantId", + "name", + "type", + "displayName", + "description", + "config", + "enabled", + "labels", + "metadata", + "createdBy", + "createdAt", + "updatedBy", + "updatedAt", + } + }, + { + typeof(NotifyChannelConfig), + new[] + { + "secretRef", + "target", + "endpoint", + "properties", + "limits", + } + }, + { + typeof(NotifyTemplate), + new[] + { + "schemaVersion", + "templateId", + "tenantId", + "channelType", + "key", + "locale", + "description", + "renderMode", + "body", + "format", + "metadata", + "createdBy", + "createdAt", + "updatedBy", + "updatedAt", + } + }, + { + typeof(NotifyEvent), + new[] + { + "eventId", + "kind", + "version", + "tenant", + "ts", + "actor", + "scope", + "payload", + "attributes", + } + }, + { + typeof(NotifyEventScope), + new[] + { + "namespace", + "repo", + "digest", + "component", + "image", + "labels", + "attributes", + } + }, + { + typeof(NotifyDelivery), + new[] + { + "deliveryId", + "tenantId", + "ruleId", + "actionId", + "eventId", + "kind", + "status", + "statusReason", + "createdAt", + "sentAt", + "completedAt", + "rendered", + "attempts", + "metadata", + } + }, + { + typeof(NotifyDeliveryAttempt), + new[] + { + "timestamp", + "status", + "statusCode", + "reason", + } + }, + { + typeof(NotifyDeliveryRendered), + new[] + { + "title", + "summary", + "target", + "locale", + "channelType", + "format", + "body", + "textBody", + "bodyHash", + "attachments", + } + }, + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static T Deserialize(string json) + { + if (typeof(T) == typeof(NotifyRule)) + { + var dto = JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException("Unable to deserialize NotifyRule payload."); + return (T)(object)dto.ToModel(); + } + + if (typeof(T) == typeof(NotifyChannel)) + { + var dto = JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException("Unable to deserialize NotifyChannel payload."); + return (T)(object)dto.ToModel(); + } + + if (typeof(T) == typeof(NotifyTemplate)) + { + var dto = JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException("Unable to deserialize NotifyTemplate payload."); + return (T)(object)dto.ToModel(); + } + + if (typeof(T) == typeof(NotifyEvent)) + { + var dto = JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException("Unable to deserialize NotifyEvent payload."); + return (T)(object)dto.ToModel(); + } + + if (typeof(T) == typeof(NotifyDelivery)) + { + var dto = JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException("Unable to deserialize NotifyDelivery payload."); + return (T)(object)dto.ToModel(); + } + + return JsonSerializer.Deserialize(json, ReadOptions) + ?? throw new InvalidOperationException($"Unable to deserialize type {typeof(T).Name}."); + } + + private static JsonSerializerOptions CreateOptions(bool writeIndented, bool useDeterministicResolver) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + if (useDeterministicResolver) + { + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + } + + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); + options.Converters.Add(new Iso8601DurationConverter()); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options) + ?? throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetPropertyOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetPropertyOrder(Type type, string propertyName) + { + if (PropertyOrderOverrides.TryGetValue(type, out var order) && Array.IndexOf(order, propertyName) is { } index and >= 0) + { + return index; + } + + return int.MaxValue; + } + } +} + +internal sealed class NotifyRuleDto +{ + public string? SchemaVersion { get; set; } + public string? RuleId { get; set; } + public string? TenantId { get; set; } + public string? Name { get; set; } + public string? Description { get; set; } + public bool? Enabled { get; set; } + public NotifyRuleMatchDto? Match { get; set; } + public List? Actions { get; set; } + public Dictionary? Labels { get; set; } + public Dictionary? Metadata { get; set; } + public string? CreatedBy { get; set; } + public DateTimeOffset? CreatedAt { get; set; } + public string? UpdatedBy { get; set; } + public DateTimeOffset? UpdatedAt { get; set; } + + public NotifyRule ToModel() + => NotifyRule.Create( + RuleId ?? throw new InvalidOperationException("ruleId missing"), + TenantId ?? throw new InvalidOperationException("tenantId missing"), + Name ?? throw new InvalidOperationException("name missing"), + (Match ?? new NotifyRuleMatchDto()).ToModel(), + Actions?.Select(action => action.ToModel()) ?? Array.Empty(), + Enabled.GetValueOrDefault(true), + Description, + Labels, + Metadata, + CreatedBy, + CreatedAt, + UpdatedBy, + UpdatedAt, + SchemaVersion); +} + +internal sealed class NotifyRuleMatchDto +{ + public List? EventKinds { get; set; } + public List? Namespaces { get; set; } + public List? Repositories { get; set; } + public List? Digests { get; set; } + public List? Labels { get; set; } + public List? ComponentPurls { get; set; } + public string? MinSeverity { get; set; } + public List? Verdicts { get; set; } + public bool? KevOnly { get; set; } + public NotifyRuleMatchVexDto? Vex { get; set; } + + public NotifyRuleMatch ToModel() + => NotifyRuleMatch.Create( + EventKinds, + Namespaces, + Repositories, + Digests, + Labels, + ComponentPurls, + MinSeverity, + Verdicts, + KevOnly, + Vex?.ToModel()); +} + +internal sealed class NotifyRuleMatchVexDto +{ + public bool IncludeAcceptedJustifications { get; set; } = true; + public bool IncludeRejectedJustifications { get; set; } + public bool IncludeUnknownJustifications { get; set; } + public List? JustificationKinds { get; set; } + + public NotifyRuleMatchVex ToModel() + => NotifyRuleMatchVex.Create( + IncludeAcceptedJustifications, + IncludeRejectedJustifications, + IncludeUnknownJustifications, + JustificationKinds); +} + +internal sealed class NotifyRuleActionDto +{ + public string? ActionId { get; set; } + public string? Channel { get; set; } + public string? Template { get; set; } + public string? Digest { get; set; } + public TimeSpan? Throttle { get; set; } + public string? Locale { get; set; } + public bool? Enabled { get; set; } + public Dictionary? Metadata { get; set; } + + public NotifyRuleAction ToModel() + => NotifyRuleAction.Create( + ActionId ?? throw new InvalidOperationException("actionId missing"), + Channel ?? throw new InvalidOperationException("channel missing"), + Template, + Digest, + Throttle, + Locale, + Enabled.GetValueOrDefault(true), + Metadata); +} + +internal sealed class NotifyChannelDto +{ + public string? SchemaVersion { get; set; } + public string? ChannelId { get; set; } + public string? TenantId { get; set; } + public string? Name { get; set; } + public NotifyChannelType Type { get; set; } + public NotifyChannelConfigDto? Config { get; set; } + public string? DisplayName { get; set; } + public string? Description { get; set; } + public bool? Enabled { get; set; } + public Dictionary? Labels { get; set; } + public Dictionary? Metadata { get; set; } + public string? CreatedBy { get; set; } + public DateTimeOffset? CreatedAt { get; set; } + public string? UpdatedBy { get; set; } + public DateTimeOffset? UpdatedAt { get; set; } + + public NotifyChannel ToModel() + => NotifyChannel.Create( + ChannelId ?? throw new InvalidOperationException("channelId missing"), + TenantId ?? throw new InvalidOperationException("tenantId missing"), + Name ?? throw new InvalidOperationException("name missing"), + Type, + (Config ?? new NotifyChannelConfigDto()).ToModel(), + DisplayName, + Description, + Enabled.GetValueOrDefault(true), + Labels, + Metadata, + CreatedBy, + CreatedAt, + UpdatedBy, + UpdatedAt, + SchemaVersion); +} + +internal sealed class NotifyChannelConfigDto +{ + public string? SecretRef { get; set; } + public string? Target { get; set; } + public string? Endpoint { get; set; } + public Dictionary? Properties { get; set; } + public NotifyChannelLimitsDto? Limits { get; set; } + + public NotifyChannelConfig ToModel() + => NotifyChannelConfig.Create( + SecretRef ?? throw new InvalidOperationException("secretRef missing"), + Target, + Endpoint, + Properties, + Limits?.ToModel()); +} + +internal sealed class NotifyChannelLimitsDto +{ + public int? Concurrency { get; set; } + public int? RequestsPerMinute { get; set; } + public TimeSpan? Timeout { get; set; } + public int? MaxBatchSize { get; set; } + + public NotifyChannelLimits ToModel() + => new( + Concurrency, + RequestsPerMinute, + Timeout, + MaxBatchSize); +} + +internal sealed class NotifyTemplateDto +{ + public string? SchemaVersion { get; set; } + public string? TemplateId { get; set; } + public string? TenantId { get; set; } + public NotifyChannelType ChannelType { get; set; } + public string? Key { get; set; } + public string? Locale { get; set; } + public string? Body { get; set; } + public NotifyTemplateRenderMode RenderMode { get; set; } = NotifyTemplateRenderMode.Markdown; + public NotifyDeliveryFormat Format { get; set; } = NotifyDeliveryFormat.Json; + public string? Description { get; set; } + public Dictionary? Metadata { get; set; } + public string? CreatedBy { get; set; } + public DateTimeOffset? CreatedAt { get; set; } + public string? UpdatedBy { get; set; } + public DateTimeOffset? UpdatedAt { get; set; } + + public NotifyTemplate ToModel() + => NotifyTemplate.Create( + TemplateId ?? throw new InvalidOperationException("templateId missing"), + TenantId ?? throw new InvalidOperationException("tenantId missing"), + ChannelType, + Key ?? throw new InvalidOperationException("key missing"), + Locale ?? throw new InvalidOperationException("locale missing"), + Body ?? throw new InvalidOperationException("body missing"), + RenderMode, + Format, + Description, + Metadata, + CreatedBy, + CreatedAt, + UpdatedBy, + UpdatedAt, + SchemaVersion); +} + +internal sealed class NotifyEventDto +{ + public Guid EventId { get; set; } + public string? Kind { get; set; } + public string? Tenant { get; set; } + public DateTimeOffset Ts { get; set; } + public JsonNode? Payload { get; set; } + public NotifyEventScopeDto? Scope { get; set; } + public string? Version { get; set; } + public string? Actor { get; set; } + public Dictionary? Attributes { get; set; } + + public NotifyEvent ToModel() + => NotifyEvent.Create( + EventId, + Kind ?? throw new InvalidOperationException("kind missing"), + Tenant ?? throw new InvalidOperationException("tenant missing"), + Ts, + Payload, + Scope?.ToModel(), + Version, + Actor, + Attributes); +} + +internal sealed class NotifyEventScopeDto +{ + public string? Namespace { get; set; } + public string? Repo { get; set; } + public string? Digest { get; set; } + public string? Component { get; set; } + public string? Image { get; set; } + public Dictionary? Labels { get; set; } + public Dictionary? Attributes { get; set; } + + public NotifyEventScope ToModel() + => NotifyEventScope.Create( + Namespace, + Repo, + Digest, + Component, + Image, + Labels, + Attributes); +} + +internal sealed class NotifyDeliveryDto +{ + public string? DeliveryId { get; set; } + public string? TenantId { get; set; } + public string? RuleId { get; set; } + public string? ActionId { get; set; } + public Guid EventId { get; set; } + public string? Kind { get; set; } + public NotifyDeliveryStatus Status { get; set; } + public string? StatusReason { get; set; } + public NotifyDeliveryRenderedDto? Rendered { get; set; } + public List? Attempts { get; set; } + public Dictionary? Metadata { get; set; } + public DateTimeOffset? CreatedAt { get; set; } + public DateTimeOffset? SentAt { get; set; } + public DateTimeOffset? CompletedAt { get; set; } + + public NotifyDelivery ToModel() + => NotifyDelivery.Create( + DeliveryId ?? throw new InvalidOperationException("deliveryId missing"), + TenantId ?? throw new InvalidOperationException("tenantId missing"), + RuleId ?? throw new InvalidOperationException("ruleId missing"), + ActionId ?? throw new InvalidOperationException("actionId missing"), + EventId, + Kind ?? throw new InvalidOperationException("kind missing"), + Status, + StatusReason, + Rendered?.ToModel(), + Attempts?.Select(attempt => attempt.ToModel()), + Metadata, + CreatedAt, + SentAt, + CompletedAt); +} + +internal sealed class NotifyDeliveryAttemptDto +{ + public DateTimeOffset Timestamp { get; set; } + public NotifyDeliveryAttemptStatus Status { get; set; } + public int? StatusCode { get; set; } + public string? Reason { get; set; } + + public NotifyDeliveryAttempt ToModel() + => new(Timestamp, Status, StatusCode, Reason); +} + +internal sealed class NotifyDeliveryRenderedDto +{ + public NotifyChannelType ChannelType { get; set; } + public NotifyDeliveryFormat Format { get; set; } + public string? Target { get; set; } + public string? Title { get; set; } + public string? Body { get; set; } + public string? Summary { get; set; } + public string? TextBody { get; set; } + public string? Locale { get; set; } + public string? BodyHash { get; set; } + public List? Attachments { get; set; } + + public NotifyDeliveryRendered ToModel() + => NotifyDeliveryRendered.Create( + ChannelType, + Format, + Target ?? throw new InvalidOperationException("target missing"), + Title ?? throw new InvalidOperationException("title missing"), + Body ?? throw new InvalidOperationException("body missing"), + Summary, + TextBody, + Locale, + BodyHash, + Attachments); +} diff --git a/src/StellaOps.Notify.Models/NotifyChannel.cs b/src/StellaOps.Notify.Models/NotifyChannel.cs new file mode 100644 index 00000000..b4c438d0 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyChannel.cs @@ -0,0 +1,235 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Configured delivery channel (Slack workspace, Teams webhook, SMTP profile, etc.). +/// +public sealed record NotifyChannel +{ + [JsonConstructor] + public NotifyChannel( + string channelId, + string tenantId, + string name, + NotifyChannelType type, + NotifyChannelConfig config, + string? displayName = null, + string? description = null, + bool enabled = true, + ImmutableDictionary? labels = null, + ImmutableDictionary? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + SchemaVersion = NotifySchemaVersions.EnsureChannel(schemaVersion); + ChannelId = NotifyValidation.EnsureNotNullOrWhiteSpace(channelId, nameof(channelId)); + TenantId = NotifyValidation.EnsureNotNullOrWhiteSpace(tenantId, nameof(tenantId)); + Name = NotifyValidation.EnsureNotNullOrWhiteSpace(name, nameof(name)); + Type = type; + Config = config ?? throw new ArgumentNullException(nameof(config)); + DisplayName = NotifyValidation.TrimToNull(displayName); + Description = NotifyValidation.TrimToNull(description); + Enabled = enabled; + + Labels = NotifyValidation.NormalizeStringDictionary(labels); + Metadata = NotifyValidation.NormalizeStringDictionary(metadata); + + CreatedBy = NotifyValidation.TrimToNull(createdBy); + CreatedAt = NotifyValidation.EnsureUtc(createdAt ?? DateTimeOffset.UtcNow); + UpdatedBy = NotifyValidation.TrimToNull(updatedBy); + UpdatedAt = NotifyValidation.EnsureUtc(updatedAt ?? CreatedAt); + } + + public static NotifyChannel Create( + string channelId, + string tenantId, + string name, + NotifyChannelType type, + NotifyChannelConfig config, + string? displayName = null, + string? description = null, + bool enabled = true, + IEnumerable>? labels = null, + IEnumerable>? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + return new NotifyChannel( + channelId, + tenantId, + name, + type, + config, + displayName, + description, + enabled, + ToImmutableDictionary(labels), + ToImmutableDictionary(metadata), + createdBy, + createdAt, + updatedBy, + updatedAt, + schemaVersion); + } + + public string SchemaVersion { get; } + + public string ChannelId { get; } + + public string TenantId { get; } + + public string Name { get; } + + public NotifyChannelType Type { get; } + + public NotifyChannelConfig Config { get; } + + public string? DisplayName { get; } + + public string? Description { get; } + + public bool Enabled { get; } + + public ImmutableDictionary Labels { get; } + + public ImmutableDictionary Metadata { get; } + + public string? CreatedBy { get; } + + public DateTimeOffset CreatedAt { get; } + + public string? UpdatedBy { get; } + + public DateTimeOffset UpdatedAt { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} + +/// +/// Channel configuration payload (secret reference, destination coordinates, connector-specific metadata). +/// +public sealed record NotifyChannelConfig +{ + [JsonConstructor] + public NotifyChannelConfig( + string secretRef, + string? target = null, + string? endpoint = null, + ImmutableDictionary? properties = null, + NotifyChannelLimits? limits = null) + { + SecretRef = NotifyValidation.EnsureNotNullOrWhiteSpace(secretRef, nameof(secretRef)); + Target = NotifyValidation.TrimToNull(target); + Endpoint = NotifyValidation.TrimToNull(endpoint); + Properties = NotifyValidation.NormalizeStringDictionary(properties); + Limits = limits; + } + + public static NotifyChannelConfig Create( + string secretRef, + string? target = null, + string? endpoint = null, + IEnumerable>? properties = null, + NotifyChannelLimits? limits = null) + { + return new NotifyChannelConfig( + secretRef, + target, + endpoint, + ToImmutableDictionary(properties), + limits); + } + + public string SecretRef { get; } + + public string? Target { get; } + + public string? Endpoint { get; } + + public ImmutableDictionary Properties { get; } + + public NotifyChannelLimits? Limits { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} + +/// +/// Optional per-channel limits that influence worker behaviour. +/// +public sealed record NotifyChannelLimits +{ + [JsonConstructor] + public NotifyChannelLimits( + int? concurrency = null, + int? requestsPerMinute = null, + TimeSpan? timeout = null, + int? maxBatchSize = null) + { + if (concurrency is < 1) + { + throw new ArgumentOutOfRangeException(nameof(concurrency), "Concurrency must be positive when specified."); + } + + if (requestsPerMinute is < 1) + { + throw new ArgumentOutOfRangeException(nameof(requestsPerMinute), "Requests per minute must be positive when specified."); + } + + if (maxBatchSize is < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxBatchSize), "Max batch size must be positive when specified."); + } + + Concurrency = concurrency; + RequestsPerMinute = requestsPerMinute; + Timeout = timeout is { Ticks: > 0 } ? timeout : null; + MaxBatchSize = maxBatchSize; + } + + public int? Concurrency { get; } + + public int? RequestsPerMinute { get; } + + public TimeSpan? Timeout { get; } + + public int? MaxBatchSize { get; } +} diff --git a/src/StellaOps.Notify.Models/NotifyDelivery.cs b/src/StellaOps.Notify.Models/NotifyDelivery.cs new file mode 100644 index 00000000..8978cf29 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyDelivery.cs @@ -0,0 +1,252 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Delivery ledger entry capturing render output, attempts, and status transitions. +/// +public sealed record NotifyDelivery +{ + [JsonConstructor] + public NotifyDelivery( + string deliveryId, + string tenantId, + string ruleId, + string actionId, + Guid eventId, + string kind, + NotifyDeliveryStatus status, + string? statusReason = null, + NotifyDeliveryRendered? rendered = null, + ImmutableArray attempts = default, + ImmutableDictionary? metadata = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? sentAt = null, + DateTimeOffset? completedAt = null) + { + DeliveryId = NotifyValidation.EnsureNotNullOrWhiteSpace(deliveryId, nameof(deliveryId)); + TenantId = NotifyValidation.EnsureNotNullOrWhiteSpace(tenantId, nameof(tenantId)); + RuleId = NotifyValidation.EnsureNotNullOrWhiteSpace(ruleId, nameof(ruleId)); + ActionId = NotifyValidation.EnsureNotNullOrWhiteSpace(actionId, nameof(actionId)); + EventId = eventId; + Kind = NotifyValidation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)).ToLowerInvariant(); + Status = status; + StatusReason = NotifyValidation.TrimToNull(statusReason); + Rendered = rendered; + + Attempts = NormalizeAttempts(attempts); + Metadata = NotifyValidation.NormalizeStringDictionary(metadata); + + CreatedAt = NotifyValidation.EnsureUtc(createdAt ?? DateTimeOffset.UtcNow); + SentAt = NotifyValidation.EnsureUtc(sentAt); + CompletedAt = NotifyValidation.EnsureUtc(completedAt); + } + + public static NotifyDelivery Create( + string deliveryId, + string tenantId, + string ruleId, + string actionId, + Guid eventId, + string kind, + NotifyDeliveryStatus status, + string? statusReason = null, + NotifyDeliveryRendered? rendered = null, + IEnumerable? attempts = null, + IEnumerable>? metadata = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? sentAt = null, + DateTimeOffset? completedAt = null) + { + return new NotifyDelivery( + deliveryId, + tenantId, + ruleId, + actionId, + eventId, + kind, + status, + statusReason, + rendered, + ToImmutableArray(attempts), + ToImmutableDictionary(metadata), + createdAt, + sentAt, + completedAt); + } + + public string DeliveryId { get; } + + public string TenantId { get; } + + public string RuleId { get; } + + public string ActionId { get; } + + public Guid EventId { get; } + + public string Kind { get; } + + public NotifyDeliveryStatus Status { get; } + + public string? StatusReason { get; } + + public NotifyDeliveryRendered? Rendered { get; } + + public ImmutableArray Attempts { get; } + + public ImmutableDictionary Metadata { get; } + + public DateTimeOffset CreatedAt { get; } + + public DateTimeOffset? SentAt { get; } + + public DateTimeOffset? CompletedAt { get; } + + private static ImmutableArray NormalizeAttempts(ImmutableArray attempts) + { + var source = attempts.IsDefault ? Array.Empty() : attempts.AsEnumerable(); + return source + .Where(static attempt => attempt is not null) + .OrderBy(static attempt => attempt.Timestamp) + .ToImmutableArray(); + } + + private static ImmutableArray ToImmutableArray(IEnumerable? attempts) + { + if (attempts is null) + { + return ImmutableArray.Empty; + } + + return attempts.ToImmutableArray(); + } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} + +/// +/// Individual delivery attempt outcome. +/// +public sealed record NotifyDeliveryAttempt +{ + [JsonConstructor] + public NotifyDeliveryAttempt( + DateTimeOffset timestamp, + NotifyDeliveryAttemptStatus status, + int? statusCode = null, + string? reason = null) + { + Timestamp = NotifyValidation.EnsureUtc(timestamp); + Status = status; + if (statusCode is < 0) + { + throw new ArgumentOutOfRangeException(nameof(statusCode), "Status code must be positive when specified."); + } + + StatusCode = statusCode; + Reason = NotifyValidation.TrimToNull(reason); + } + + public DateTimeOffset Timestamp { get; } + + public NotifyDeliveryAttemptStatus Status { get; } + + public int? StatusCode { get; } + + public string? Reason { get; } +} + +/// +/// Rendered payload snapshot for audit purposes (redacted as needed). +/// +public sealed record NotifyDeliveryRendered +{ + [JsonConstructor] + public NotifyDeliveryRendered( + NotifyChannelType channelType, + NotifyDeliveryFormat format, + string target, + string title, + string body, + string? summary = null, + string? textBody = null, + string? locale = null, + string? bodyHash = null, + ImmutableArray attachments = default) + { + ChannelType = channelType; + Format = format; + Target = NotifyValidation.EnsureNotNullOrWhiteSpace(target, nameof(target)); + Title = NotifyValidation.EnsureNotNullOrWhiteSpace(title, nameof(title)); + Body = NotifyValidation.EnsureNotNullOrWhiteSpace(body, nameof(body)); + Summary = NotifyValidation.TrimToNull(summary); + TextBody = NotifyValidation.TrimToNull(textBody); + Locale = NotifyValidation.TrimToNull(locale)?.ToLowerInvariant(); + BodyHash = NotifyValidation.TrimToNull(bodyHash); + Attachments = NotifyValidation.NormalizeStringSet(attachments.IsDefault ? Array.Empty() : attachments.AsEnumerable()); + } + + public static NotifyDeliveryRendered Create( + NotifyChannelType channelType, + NotifyDeliveryFormat format, + string target, + string title, + string body, + string? summary = null, + string? textBody = null, + string? locale = null, + string? bodyHash = null, + IEnumerable? attachments = null) + { + return new NotifyDeliveryRendered( + channelType, + format, + target, + title, + body, + summary, + textBody, + locale, + bodyHash, + attachments is null ? ImmutableArray.Empty : attachments.ToImmutableArray()); + } + + public NotifyChannelType ChannelType { get; } + + public NotifyDeliveryFormat Format { get; } + + public string Target { get; } + + public string Title { get; } + + public string Body { get; } + + public string? Summary { get; } + + public string? TextBody { get; } + + public string? Locale { get; } + + public string? BodyHash { get; } + + public ImmutableArray Attachments { get; } +} diff --git a/src/StellaOps.Notify.Models/NotifyEnums.cs b/src/StellaOps.Notify.Models/NotifyEnums.cs new file mode 100644 index 00000000..4d341b82 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyEnums.cs @@ -0,0 +1,70 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Supported Notify channel types. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum NotifyChannelType +{ + Slack, + Teams, + Email, + Webhook, + Custom, +} + +/// +/// Delivery lifecycle states tracked for audit and retries. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum NotifyDeliveryStatus +{ + Pending, + Sent, + Failed, + Throttled, + Digested, + Dropped, +} + +/// +/// Individual attempt status recorded during delivery retries. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum NotifyDeliveryAttemptStatus +{ + Enqueued, + Sending, + Succeeded, + Failed, + Throttled, + Skipped, +} + +/// +/// Rendering modes for templates to help connectors decide format handling. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum NotifyTemplateRenderMode +{ + Markdown, + Html, + AdaptiveCard, + PlainText, + Json, +} + +/// +/// Structured representation of rendered payload format. +/// +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum NotifyDeliveryFormat +{ + Slack, + Teams, + Email, + Webhook, + Json, +} diff --git a/src/StellaOps.Notify.Models/NotifyEvent.cs b/src/StellaOps.Notify.Models/NotifyEvent.cs new file mode 100644 index 00000000..900ea640 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyEvent.cs @@ -0,0 +1,168 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Canonical platform event envelope consumed by Notify. +/// +public sealed record NotifyEvent +{ + [JsonConstructor] + public NotifyEvent( + Guid eventId, + string kind, + string tenant, + DateTimeOffset ts, + JsonNode? payload, + NotifyEventScope? scope = null, + string? version = null, + string? actor = null, + ImmutableDictionary? attributes = null) + { + EventId = eventId; + Kind = NotifyValidation.EnsureNotNullOrWhiteSpace(kind, nameof(kind)).ToLowerInvariant(); + Tenant = NotifyValidation.EnsureNotNullOrWhiteSpace(tenant, nameof(tenant)); + Ts = NotifyValidation.EnsureUtc(ts); + Payload = NotifyValidation.NormalizeJsonNode(payload); + Scope = scope; + Version = NotifyValidation.TrimToNull(version); + Actor = NotifyValidation.TrimToNull(actor); + Attributes = NotifyValidation.NormalizeStringDictionary(attributes); + } + + public static NotifyEvent Create( + Guid eventId, + string kind, + string tenant, + DateTimeOffset ts, + JsonNode? payload, + NotifyEventScope? scope = null, + string? version = null, + string? actor = null, + IEnumerable>? attributes = null) + { + return new NotifyEvent( + eventId, + kind, + tenant, + ts, + payload, + scope, + version, + actor, + ToImmutableDictionary(attributes)); + } + + public Guid EventId { get; } + + public string Kind { get; } + + public string Tenant { get; } + + public DateTimeOffset Ts { get; } + + public JsonNode? Payload { get; } + + public NotifyEventScope? Scope { get; } + + public string? Version { get; } + + public string? Actor { get; } + + public ImmutableDictionary Attributes { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} + +/// +/// Optional scope block describing where the event originated (namespace/repo/digest/etc.). +/// +public sealed record NotifyEventScope +{ + [JsonConstructor] + public NotifyEventScope( + string? @namespace = null, + string? repo = null, + string? digest = null, + string? component = null, + string? image = null, + ImmutableDictionary? labels = null, + ImmutableDictionary? attributes = null) + { + Namespace = NotifyValidation.TrimToNull(@namespace); + Repo = NotifyValidation.TrimToNull(repo); + Digest = NotifyValidation.TrimToNull(digest); + Component = NotifyValidation.TrimToNull(component); + Image = NotifyValidation.TrimToNull(image); + Labels = NotifyValidation.NormalizeStringDictionary(labels); + Attributes = NotifyValidation.NormalizeStringDictionary(attributes); + } + + public static NotifyEventScope Create( + string? @namespace = null, + string? repo = null, + string? digest = null, + string? component = null, + string? image = null, + IEnumerable>? labels = null, + IEnumerable>? attributes = null) + { + return new NotifyEventScope( + @namespace, + repo, + digest, + component, + image, + ToImmutableDictionary(labels), + ToImmutableDictionary(attributes)); + } + + public string? Namespace { get; } + + public string? Repo { get; } + + public string? Digest { get; } + + public string? Component { get; } + + public string? Image { get; } + + public ImmutableDictionary Labels { get; } + + public ImmutableDictionary Attributes { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Notify.Models/NotifyEventKinds.cs b/src/StellaOps.Notify.Models/NotifyEventKinds.cs new file mode 100644 index 00000000..4e98d100 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyEventKinds.cs @@ -0,0 +1,15 @@ +namespace StellaOps.Notify.Models; + +/// +/// Known platform event kind identifiers consumed by Notify. +/// +public static class NotifyEventKinds +{ + public const string ScannerReportReady = "scanner.report.ready"; + public const string ScannerScanCompleted = "scanner.scan.completed"; + public const string SchedulerRescanDelta = "scheduler.rescan.delta"; + public const string AttestorLogged = "attestor.logged"; + public const string ZastavaAdmission = "zastava.admission"; + public const string FeedserExportCompleted = "feedser.export.completed"; + public const string VexerExportCompleted = "vexer.export.completed"; +} diff --git a/src/StellaOps.Notify.Models/NotifyRule.cs b/src/StellaOps.Notify.Models/NotifyRule.cs new file mode 100644 index 00000000..8b185092 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyRule.cs @@ -0,0 +1,388 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Rule definition describing how platform events are matched and routed to delivery actions. +/// +public sealed record NotifyRule +{ + [JsonConstructor] + public NotifyRule( + string ruleId, + string tenantId, + string name, + NotifyRuleMatch match, + ImmutableArray actions, + bool enabled = true, + string? description = null, + ImmutableDictionary? labels = null, + ImmutableDictionary? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + SchemaVersion = NotifySchemaVersions.EnsureRule(schemaVersion); + RuleId = NotifyValidation.EnsureNotNullOrWhiteSpace(ruleId, nameof(ruleId)); + TenantId = NotifyValidation.EnsureNotNullOrWhiteSpace(tenantId, nameof(tenantId)); + Name = NotifyValidation.EnsureNotNullOrWhiteSpace(name, nameof(name)); + Description = NotifyValidation.TrimToNull(description); + Match = match ?? throw new ArgumentNullException(nameof(match)); + Enabled = enabled; + + Actions = NormalizeActions(actions); + if (Actions.IsDefaultOrEmpty) + { + throw new ArgumentException("At least one action is required.", nameof(actions)); + } + + Labels = NotifyValidation.NormalizeStringDictionary(labels); + Metadata = NotifyValidation.NormalizeStringDictionary(metadata); + + CreatedBy = NotifyValidation.TrimToNull(createdBy); + CreatedAt = NotifyValidation.EnsureUtc(createdAt ?? DateTimeOffset.UtcNow); + UpdatedBy = NotifyValidation.TrimToNull(updatedBy); + UpdatedAt = NotifyValidation.EnsureUtc(updatedAt ?? CreatedAt); + } + + public static NotifyRule Create( + string ruleId, + string tenantId, + string name, + NotifyRuleMatch match, + IEnumerable? actions, + bool enabled = true, + string? description = null, + IEnumerable>? labels = null, + IEnumerable>? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + return new NotifyRule( + ruleId, + tenantId, + name, + match, + ToImmutableArray(actions), + enabled, + description, + ToImmutableDictionary(labels), + ToImmutableDictionary(metadata), + createdBy, + createdAt, + updatedBy, + updatedAt, + schemaVersion); + } + + public string SchemaVersion { get; } + + public string RuleId { get; } + + public string TenantId { get; } + + public string Name { get; } + + public string? Description { get; } + + public bool Enabled { get; } + + public NotifyRuleMatch Match { get; } + + public ImmutableArray Actions { get; } + + public ImmutableDictionary Labels { get; } + + public ImmutableDictionary Metadata { get; } + + public string? CreatedBy { get; } + + public DateTimeOffset CreatedAt { get; } + + public string? UpdatedBy { get; } + + public DateTimeOffset UpdatedAt { get; } + + private static ImmutableArray NormalizeActions(ImmutableArray actions) + { + var source = actions.IsDefault ? Array.Empty() : actions.AsEnumerable(); + return source + .Where(static action => action is not null) + .Distinct() + .OrderBy(static action => action.ActionId, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray ToImmutableArray(IEnumerable? actions) + { + if (actions is null) + { + return ImmutableArray.Empty; + } + + return actions.ToImmutableArray(); + } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} + +/// +/// Matching criteria used to evaluate whether an event should trigger the rule. +/// +public sealed record NotifyRuleMatch +{ + [JsonConstructor] + public NotifyRuleMatch( + ImmutableArray eventKinds, + ImmutableArray namespaces, + ImmutableArray repositories, + ImmutableArray digests, + ImmutableArray labels, + ImmutableArray componentPurls, + string? minSeverity, + ImmutableArray verdicts, + bool? kevOnly, + NotifyRuleMatchVex? vex) + { + EventKinds = NormalizeStringSet(eventKinds, lowerCase: true); + Namespaces = NormalizeStringSet(namespaces); + Repositories = NormalizeStringSet(repositories); + Digests = NormalizeStringSet(digests, lowerCase: true); + Labels = NormalizeStringSet(labels); + ComponentPurls = NormalizeStringSet(componentPurls); + Verdicts = NormalizeStringSet(verdicts, lowerCase: true); + MinSeverity = NotifyValidation.TrimToNull(minSeverity)?.ToLowerInvariant(); + KevOnly = kevOnly; + Vex = vex; + } + + public static NotifyRuleMatch Create( + IEnumerable? eventKinds = null, + IEnumerable? namespaces = null, + IEnumerable? repositories = null, + IEnumerable? digests = null, + IEnumerable? labels = null, + IEnumerable? componentPurls = null, + string? minSeverity = null, + IEnumerable? verdicts = null, + bool? kevOnly = null, + NotifyRuleMatchVex? vex = null) + { + return new NotifyRuleMatch( + ToImmutableArray(eventKinds), + ToImmutableArray(namespaces), + ToImmutableArray(repositories), + ToImmutableArray(digests), + ToImmutableArray(labels), + ToImmutableArray(componentPurls), + minSeverity, + ToImmutableArray(verdicts), + kevOnly, + vex); + } + + public ImmutableArray EventKinds { get; } + + public ImmutableArray Namespaces { get; } + + public ImmutableArray Repositories { get; } + + public ImmutableArray Digests { get; } + + public ImmutableArray Labels { get; } + + public ImmutableArray ComponentPurls { get; } + + public string? MinSeverity { get; } + + public ImmutableArray Verdicts { get; } + + public bool? KevOnly { get; } + + public NotifyRuleMatchVex? Vex { get; } + + private static ImmutableArray NormalizeStringSet(ImmutableArray values, bool lowerCase = false) + { + var enumerable = values.IsDefault ? Array.Empty() : values.AsEnumerable(); + var normalized = NotifyValidation.NormalizeStringSet(enumerable); + + if (!lowerCase) + { + return normalized; + } + + return normalized + .Select(static value => value.ToLowerInvariant()) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + } + + private static ImmutableArray ToImmutableArray(IEnumerable? values) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + return values.ToImmutableArray(); + } +} + +/// +/// Additional VEX (Vulnerability Exploitability eXchange) gating options. +/// +public sealed record NotifyRuleMatchVex +{ + [JsonConstructor] + public NotifyRuleMatchVex( + bool includeAcceptedJustifications = true, + bool includeRejectedJustifications = false, + bool includeUnknownJustifications = false, + ImmutableArray justificationKinds = default) + { + IncludeAcceptedJustifications = includeAcceptedJustifications; + IncludeRejectedJustifications = includeRejectedJustifications; + IncludeUnknownJustifications = includeUnknownJustifications; + JustificationKinds = NormalizeStringSet(justificationKinds); + } + + public static NotifyRuleMatchVex Create( + bool includeAcceptedJustifications = true, + bool includeRejectedJustifications = false, + bool includeUnknownJustifications = false, + IEnumerable? justificationKinds = null) + { + return new NotifyRuleMatchVex( + includeAcceptedJustifications, + includeRejectedJustifications, + includeUnknownJustifications, + ToImmutableArray(justificationKinds)); + } + + public bool IncludeAcceptedJustifications { get; } + + public bool IncludeRejectedJustifications { get; } + + public bool IncludeUnknownJustifications { get; } + + public ImmutableArray JustificationKinds { get; } + + private static ImmutableArray NormalizeStringSet(ImmutableArray values) + { + var enumerable = values.IsDefault ? Array.Empty() : values.AsEnumerable(); + return NotifyValidation.NormalizeStringSet(enumerable); + } + + private static ImmutableArray ToImmutableArray(IEnumerable? values) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + return values.ToImmutableArray(); + } +} + +/// +/// Action executed when a rule matches an event. +/// +public sealed record NotifyRuleAction +{ + [JsonConstructor] + public NotifyRuleAction( + string actionId, + string channel, + string? template = null, + string? digest = null, + TimeSpan? throttle = null, + string? locale = null, + bool enabled = true, + ImmutableDictionary? metadata = null) + { + ActionId = NotifyValidation.EnsureNotNullOrWhiteSpace(actionId, nameof(actionId)); + Channel = NotifyValidation.EnsureNotNullOrWhiteSpace(channel, nameof(channel)); + Template = NotifyValidation.TrimToNull(template); + Digest = NotifyValidation.TrimToNull(digest); + Locale = NotifyValidation.TrimToNull(locale)?.ToLowerInvariant(); + Enabled = enabled; + Throttle = throttle is { Ticks: > 0 } ? throttle : null; + Metadata = NotifyValidation.NormalizeStringDictionary(metadata); + } + + public static NotifyRuleAction Create( + string actionId, + string channel, + string? template = null, + string? digest = null, + TimeSpan? throttle = null, + string? locale = null, + bool enabled = true, + IEnumerable>? metadata = null) + { + return new NotifyRuleAction( + actionId, + channel, + template, + digest, + throttle, + locale, + enabled, + ToImmutableDictionary(metadata)); + } + + public string ActionId { get; } + + public string Channel { get; } + + public string? Template { get; } + + public string? Digest { get; } + + public TimeSpan? Throttle { get; } + + public string? Locale { get; } + + public bool Enabled { get; } + + public ImmutableDictionary Metadata { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Notify.Models/NotifySchemaMigration.cs b/src/StellaOps.Notify.Models/NotifySchemaMigration.cs new file mode 100644 index 00000000..5097eff7 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifySchemaMigration.cs @@ -0,0 +1,74 @@ +using System.Text.Json.Nodes; + +namespace StellaOps.Notify.Models; + +/// +/// Upgrades Notify documents emitted by older schema revisions to the current DTOs. +/// +public static class NotifySchemaMigration +{ + public static NotifyRule UpgradeRule(JsonNode document) + { + ArgumentNullException.ThrowIfNull(document); + var (clone, schemaVersion) = Normalize(document, NotifySchemaVersions.Rule); + + return schemaVersion switch + { + NotifySchemaVersions.Rule => Deserialize(clone), + _ => throw new NotSupportedException($"Unsupported notify rule schema version '{schemaVersion}'.") + }; + } + + public static NotifyChannel UpgradeChannel(JsonNode document) + { + ArgumentNullException.ThrowIfNull(document); + var (clone, schemaVersion) = Normalize(document, NotifySchemaVersions.Channel); + + return schemaVersion switch + { + NotifySchemaVersions.Channel => Deserialize(clone), + _ => throw new NotSupportedException($"Unsupported notify channel schema version '{schemaVersion}'.") + }; + } + + public static NotifyTemplate UpgradeTemplate(JsonNode document) + { + ArgumentNullException.ThrowIfNull(document); + var (clone, schemaVersion) = Normalize(document, NotifySchemaVersions.Template); + + return schemaVersion switch + { + NotifySchemaVersions.Template => Deserialize(clone), + _ => throw new NotSupportedException($"Unsupported notify template schema version '{schemaVersion}'.") + }; + } + + private static (JsonObject Clone, string SchemaVersion) Normalize(JsonNode node, string fallback) + { + if (node is not JsonObject obj) + { + throw new ArgumentException("Document must be a JSON object.", nameof(node)); + } + + if (obj.DeepClone() is not JsonObject clone) + { + throw new InvalidOperationException("Unable to clone document as JsonObject."); + } + + string schemaVersion; + if (clone.TryGetPropertyValue("schemaVersion", out var value) && value is JsonValue jsonValue && jsonValue.TryGetValue(out string? version) && !string.IsNullOrWhiteSpace(version)) + { + schemaVersion = version.Trim(); + } + else + { + schemaVersion = fallback; + clone["schemaVersion"] = schemaVersion; + } + + return (clone, schemaVersion); + } + + private static T Deserialize(JsonObject json) + => NotifyCanonicalJsonSerializer.Deserialize(json.ToJsonString()); +} diff --git a/src/StellaOps.Notify.Models/NotifySchemaVersions.cs b/src/StellaOps.Notify.Models/NotifySchemaVersions.cs new file mode 100644 index 00000000..94926cb0 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifySchemaVersions.cs @@ -0,0 +1,23 @@ +namespace StellaOps.Notify.Models; + +/// +/// Canonical schema version identifiers for Notify documents. +/// +public static class NotifySchemaVersions +{ + public const string Rule = "notify.rule@1"; + public const string Channel = "notify.channel@1"; + public const string Template = "notify.template@1"; + + public static string EnsureRule(string? value) + => Normalize(value, Rule); + + public static string EnsureChannel(string? value) + => Normalize(value, Channel); + + public static string EnsureTemplate(string? value) + => Normalize(value, Template); + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); +} diff --git a/src/StellaOps.Notify.Models/NotifyTemplate.cs b/src/StellaOps.Notify.Models/NotifyTemplate.cs new file mode 100644 index 00000000..4c68e569 --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyTemplate.cs @@ -0,0 +1,130 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.Models; + +/// +/// Stored template metadata and content for channel-specific rendering. +/// +public sealed record NotifyTemplate +{ + [JsonConstructor] + public NotifyTemplate( + string templateId, + string tenantId, + NotifyChannelType channelType, + string key, + string locale, + string body, + NotifyTemplateRenderMode renderMode = NotifyTemplateRenderMode.Markdown, + NotifyDeliveryFormat format = NotifyDeliveryFormat.Json, + string? description = null, + ImmutableDictionary? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + SchemaVersion = NotifySchemaVersions.EnsureTemplate(schemaVersion); + TemplateId = NotifyValidation.EnsureNotNullOrWhiteSpace(templateId, nameof(templateId)); + TenantId = NotifyValidation.EnsureNotNullOrWhiteSpace(tenantId, nameof(tenantId)); + ChannelType = channelType; + Key = NotifyValidation.EnsureNotNullOrWhiteSpace(key, nameof(key)); + Locale = NotifyValidation.EnsureNotNullOrWhiteSpace(locale, nameof(locale)).ToLowerInvariant(); + Body = NotifyValidation.EnsureNotNullOrWhiteSpace(body, nameof(body)); + Description = NotifyValidation.TrimToNull(description); + RenderMode = renderMode; + Format = format; + Metadata = NotifyValidation.NormalizeStringDictionary(metadata); + + CreatedBy = NotifyValidation.TrimToNull(createdBy); + CreatedAt = NotifyValidation.EnsureUtc(createdAt ?? DateTimeOffset.UtcNow); + UpdatedBy = NotifyValidation.TrimToNull(updatedBy); + UpdatedAt = NotifyValidation.EnsureUtc(updatedAt ?? CreatedAt); + } + + public static NotifyTemplate Create( + string templateId, + string tenantId, + NotifyChannelType channelType, + string key, + string locale, + string body, + NotifyTemplateRenderMode renderMode = NotifyTemplateRenderMode.Markdown, + NotifyDeliveryFormat format = NotifyDeliveryFormat.Json, + string? description = null, + IEnumerable>? metadata = null, + string? createdBy = null, + DateTimeOffset? createdAt = null, + string? updatedBy = null, + DateTimeOffset? updatedAt = null, + string? schemaVersion = null) + { + return new NotifyTemplate( + templateId, + tenantId, + channelType, + key, + locale, + body, + renderMode, + format, + description, + ToImmutableDictionary(metadata), + createdBy, + createdAt, + updatedBy, + updatedAt, + schemaVersion); + } + + public string SchemaVersion { get; } + + public string TemplateId { get; } + + public string TenantId { get; } + + public NotifyChannelType ChannelType { get; } + + public string Key { get; } + + public string Locale { get; } + + public string Body { get; } + + public string? Description { get; } + + public NotifyTemplateRenderMode RenderMode { get; } + + public NotifyDeliveryFormat Format { get; } + + public ImmutableDictionary Metadata { get; } + + public string? CreatedBy { get; } + + public DateTimeOffset CreatedAt { get; } + + public string? UpdatedBy { get; } + + public DateTimeOffset UpdatedAt { get; } + + private static ImmutableDictionary? ToImmutableDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return null; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + builder[key] = value; + } + + return builder.ToImmutable(); + } +} diff --git a/src/StellaOps.Notify.Models/NotifyValidation.cs b/src/StellaOps.Notify.Models/NotifyValidation.cs new file mode 100644 index 00000000..cfcb7c3b --- /dev/null +++ b/src/StellaOps.Notify.Models/NotifyValidation.cs @@ -0,0 +1,98 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Nodes; + +namespace StellaOps.Notify.Models; + +/// +/// Lightweight validation helpers shared across Notify model constructors. +/// +public static class NotifyValidation +{ + public static string EnsureNotNullOrWhiteSpace(string value, string paramName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Value cannot be null or whitespace.", paramName); + } + + return value.Trim(); + } + + public static string? TrimToNull(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + public static ImmutableArray NormalizeStringSet(IEnumerable? values) + => (values ?? Array.Empty()) + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + + public static ImmutableDictionary NormalizeStringDictionary(IEnumerable>? pairs) + { + if (pairs is null) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var (key, value) in pairs) + { + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + var normalizedKey = key.Trim(); + var normalizedValue = value?.Trim() ?? string.Empty; + builder[normalizedKey] = normalizedValue; + } + + return ImmutableDictionary.CreateRange(StringComparer.Ordinal, builder); + } + + public static DateTimeOffset EnsureUtc(DateTimeOffset value) + => value.ToUniversalTime(); + + public static DateTimeOffset? EnsureUtc(DateTimeOffset? value) + => value?.ToUniversalTime(); + + public static JsonNode? NormalizeJsonNode(JsonNode? node) + { + if (node is null) + { + return null; + } + + switch (node) + { + case JsonObject jsonObject: + { + var normalized = new JsonObject(); + foreach (var property in jsonObject + .Where(static pair => pair.Key is not null) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + normalized[property.Key!] = NormalizeJsonNode(property.Value?.DeepClone()); + } + + return normalized; + } + case JsonArray jsonArray: + { + var normalized = new JsonArray(); + foreach (var element in jsonArray) + { + normalized.Add(NormalizeJsonNode(element?.DeepClone())); + } + + return normalized; + } + default: + return node.DeepClone(); + } + } +} diff --git a/src/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj b/src/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj new file mode 100644 index 00000000..6d665dea --- /dev/null +++ b/src/StellaOps.Notify.Models/StellaOps.Notify.Models.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/src/StellaOps.Notify.Models/TASKS.md b/src/StellaOps.Notify.Models/TASKS.md new file mode 100644 index 00000000..6738ad87 --- /dev/null +++ b/src/StellaOps.Notify.Models/TASKS.md @@ -0,0 +1,7 @@ +# Notify Models Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-MODELS-15-101 | DONE (2025-10-19) | Notify Models Guild | — | Define core DTOs (Rule, Channel, Template, Event envelope, Delivery) with validation helpers and canonical JSON serialization. | DTOs merged with tests; documented; serialization deterministic. | +| NOTIFY-MODELS-15-102 | DONE (2025-10-19) | Notify Models Guild | NOTIFY-MODELS-15-101 | Publish schema docs + sample payloads for channels, rules, events (used by UI + connectors). | Markdown/JSON schema generated; linked in docs; integration tests reference samples. | +| NOTIFY-MODELS-15-103 | DONE (2025-10-19) | Notify Models Guild | NOTIFY-MODELS-15-101 | Provide versioning and migration helpers (e.g., rule evolution, template revisions). | Migration helpers implemented; tests cover upgrade/downgrade; guidance captured in docs. | diff --git a/src/StellaOps.Notify.Queue/AGENTS.md b/src/StellaOps.Notify.Queue/AGENTS.md new file mode 100644 index 00000000..ba755be3 --- /dev/null +++ b/src/StellaOps.Notify.Queue/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Queue — Agent Charter + +## Mission +Provide event & delivery queues for Notify per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj b/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj new file mode 100644 index 00000000..6d665dea --- /dev/null +++ b/src/StellaOps.Notify.Queue/StellaOps.Notify.Queue.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/src/StellaOps.Notify.Queue/TASKS.md b/src/StellaOps.Notify.Queue/TASKS.md new file mode 100644 index 00000000..dae08651 --- /dev/null +++ b/src/StellaOps.Notify.Queue/TASKS.md @@ -0,0 +1,7 @@ +# Notify Queue Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-QUEUE-15-401 | TODO | Notify Queue Guild | NOTIFY-MODELS-15-101 | Build queue abstraction + Redis Streams adapter with ack/claim APIs, idempotency tokens, serialization contracts. | Adapter integration tests cover enqueue/dequeue/ack; ordering preserved; idempotency tokens supported. | +| NOTIFY-QUEUE-15-402 | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; integration tests exercise both adapters. | +| NOTIFY-QUEUE-15-403 | TODO | Notify Queue Guild | NOTIFY-QUEUE-15-401 | Delivery queue for channel actions with retry schedules, poison queues, and metrics instrumentation. | Delivery queue integration tests cover retries/dead-letter; metrics/logging emitted per spec. | diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/AssemblyInfo.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/AssemblyInfo.cs new file mode 100644 index 00000000..e43661c3 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/GlobalUsings.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/GlobalUsings.cs new file mode 100644 index 00000000..e1065597 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Internal/NotifyMongoMigrationTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Internal/NotifyMongoMigrationTests.cs new file mode 100644 index 00000000..a6359d80 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Internal/NotifyMongoMigrationTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Internal; + +public sealed class NotifyMongoMigrationTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + + public NotifyMongoMigrationTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-migration-tests", + DeliveryHistoryRetention = TimeSpan.FromDays(45), + MigrationsCollection = "notify_migrations_tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task EnsureIndexesCreatesExpectedDefinitions() + { + // run twice to ensure idempotency + await _initializer.EnsureIndexesAsync(); + + var deliveriesIndexes = await GetIndexesAsync(_context.Options.DeliveriesCollection); + Assert.Contains("tenant_sortKey", deliveriesIndexes.Select(doc => doc["name"].AsString)); + Assert.Contains("tenant_status", deliveriesIndexes.Select(doc => doc["name"].AsString)); + var ttlIndex = deliveriesIndexes.Single(doc => doc["name"].AsString == "completedAt_ttl"); + Assert.Equal(_context.Options.DeliveryHistoryRetention.TotalSeconds, ttlIndex["expireAfterSeconds"].ToDouble()); + + var locksIndexes = await GetIndexesAsync(_context.Options.LocksCollection); + Assert.Contains("tenant_resource", locksIndexes.Select(doc => doc["name"].AsString)); + Assert.True(locksIndexes.Single(doc => doc["name"].AsString == "tenant_resource")["unique"].ToBoolean()); + Assert.Contains("expiresAt_ttl", locksIndexes.Select(doc => doc["name"].AsString)); + + var digestsIndexes = await GetIndexesAsync(_context.Options.DigestsCollection); + Assert.Contains("tenant_actionKey", digestsIndexes.Select(doc => doc["name"].AsString)); + + var rulesIndexes = await GetIndexesAsync(_context.Options.RulesCollection); + Assert.Contains("tenant_enabled", rulesIndexes.Select(doc => doc["name"].AsString)); + + var migrationsIndexes = await GetIndexesAsync(_context.Options.MigrationsCollection); + Assert.Contains("migrationId_unique", migrationsIndexes.Select(doc => doc["name"].AsString)); + } + + private async Task> GetIndexesAsync(string collectionName) + { + var collection = _context.Database.GetCollection(collectionName); + var cursor = await collection.Indexes.ListAsync().ConfigureAwait(false); + return await cursor.ToListAsync().ConfigureAwait(false); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyAuditRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyAuditRepositoryTests.cs new file mode 100644 index 00000000..3ab0ae4a --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyAuditRepositoryTests.cs @@ -0,0 +1,75 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using MongoDB.Bson; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyAuditRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyAuditRepository _repository; + + public NotifyAuditRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-audit-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyAuditRepository(_context); + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task AppendAndQuery() + { + var entry = new NotifyAuditEntryDocument + { + TenantId = "tenant-a", + Actor = "user@example.com", + Action = "create-rule", + EntityId = "rule-1", + EntityType = "rule", + Timestamp = DateTimeOffset.UtcNow, + Payload = new BsonDocument("ruleId", "rule-1") + }; + + await _repository.AppendAsync(entry); + var list = await _repository.QueryAsync("tenant-a", DateTimeOffset.UtcNow.AddMinutes(-5), 10); + Assert.Single(list); + Assert.Equal("create-rule", list[0].Action); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyChannelRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyChannelRepositoryTests.cs new file mode 100644 index 00000000..4a3e294b --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyChannelRepositoryTests.cs @@ -0,0 +1,77 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyChannelRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyChannelRepository _repository; + + public NotifyChannelRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-channel-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyChannelRepository(_context); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + [Fact] + public async Task UpsertChannelPersistsData() + { + var channel = NotifyChannel.Create( + channelId: "channel-1", + tenantId: "tenant-a", + name: "slack:sec", + type: NotifyChannelType.Slack, + config: NotifyChannelConfig.Create(secretRef: "ref://secret")); + + await _repository.UpsertAsync(channel); + + var fetched = await _repository.GetAsync("tenant-a", "channel-1"); + Assert.NotNull(fetched); + Assert.Equal(channel.ChannelId, fetched!.ChannelId); + + var listed = await _repository.ListAsync("tenant-a"); + Assert.Single(listed); + + await _repository.DeleteAsync("tenant-a", "channel-1"); + Assert.Null(await _repository.GetAsync("tenant-a", "channel-1")); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDeliveryRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDeliveryRepositoryTests.cs new file mode 100644 index 00000000..bd814665 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDeliveryRepositoryTests.cs @@ -0,0 +1,119 @@ +using System; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyDeliveryRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyDeliveryRepository _repository; + + public NotifyDeliveryRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-delivery-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyDeliveryRepository(_context); + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task AppendAndQueryWithPaging() + { + var now = DateTimeOffset.UtcNow; + var deliveries = new[] + { + NotifyDelivery.Create( + deliveryId: "delivery-1", + tenantId: "tenant-a", + ruleId: "rule-1", + actionId: "action-1", + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + status: NotifyDeliveryStatus.Sent, + createdAt: now.AddMinutes(-2), + sentAt: now.AddMinutes(-2)), + NotifyDelivery.Create( + deliveryId: "delivery-2", + tenantId: "tenant-a", + ruleId: "rule-2", + actionId: "action-2", + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + status: NotifyDeliveryStatus.Failed, + createdAt: now.AddMinutes(-1), + completedAt: now.AddMinutes(-1)), + NotifyDelivery.Create( + deliveryId: "delivery-3", + tenantId: "tenant-a", + ruleId: "rule-3", + actionId: "action-3", + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + status: NotifyDeliveryStatus.Sent, + createdAt: now, + sentAt: now) + }; + + foreach (var delivery in deliveries) + { + await _repository.AppendAsync(delivery); + } + + var fetched = await _repository.GetAsync("tenant-a", "delivery-3"); + Assert.NotNull(fetched); + Assert.Equal("delivery-3", fetched!.DeliveryId); + + var page1 = await _repository.QueryAsync("tenant-a", now.AddHours(-1), "sent", 1); + Assert.Single(page1.Items); + Assert.Equal("delivery-3", page1.Items[0].DeliveryId); + Assert.False(string.IsNullOrWhiteSpace(page1.ContinuationToken)); + + var page2 = await _repository.QueryAsync("tenant-a", now.AddHours(-1), "sent", 1, page1.ContinuationToken); + Assert.Single(page2.Items); + Assert.Equal("delivery-1", page2.Items[0].DeliveryId); + Assert.Null(page2.ContinuationToken); + } + + [Fact] + public async Task QueryAsyncWithInvalidContinuationThrows() + { + await Assert.ThrowsAsync(() => _repository.QueryAsync("tenant-a", null, null, 10, "not-a-token")); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDigestRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDigestRepositoryTests.cs new file mode 100644 index 00000000..fa8a8882 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyDigestRepositoryTests.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyDigestRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyDigestRepository _repository; + + public NotifyDigestRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-digest-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyDigestRepository(_context); + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task UpsertAndRemove() + { + var digest = new NotifyDigestDocument + { + TenantId = "tenant-a", + ActionKey = "action-1", + Window = "hourly", + OpenedAt = DateTimeOffset.UtcNow, + Status = "open", + Items = new List + { + new() { EventId = Guid.NewGuid().ToString() } + } + }; + + await _repository.UpsertAsync(digest); + var fetched = await _repository.GetAsync("tenant-a", "action-1"); + Assert.NotNull(fetched); + Assert.Equal("action-1", fetched!.ActionKey); + + await _repository.RemoveAsync("tenant-a", "action-1"); + Assert.Null(await _repository.GetAsync("tenant-a", "action-1")); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyLockRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyLockRepositoryTests.cs new file mode 100644 index 00000000..6d534319 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyLockRepositoryTests.cs @@ -0,0 +1,67 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyLockRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyLockRepository _repository; + + public NotifyLockRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-lock-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyLockRepository(_context); + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + [Fact] + public async Task AcquireAndRelease() + { + var acquired = await _repository.TryAcquireAsync("tenant-a", "resource-1", "owner-1", TimeSpan.FromMinutes(1)); + Assert.True(acquired); + + var second = await _repository.TryAcquireAsync("tenant-a", "resource-1", "owner-2", TimeSpan.FromMinutes(1)); + Assert.False(second); + + await _repository.ReleaseAsync("tenant-a", "resource-1", "owner-1"); + var third = await _repository.TryAcquireAsync("tenant-a", "resource-1", "owner-2", TimeSpan.FromMinutes(1)); + Assert.True(third); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyRuleRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyRuleRepositoryTests.cs new file mode 100644 index 00000000..20a30b71 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyRuleRepositoryTests.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyRuleRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyRuleRepository _repository; + + public NotifyRuleRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-rule-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyRuleRepository(_context); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + [Fact] + public async Task UpsertRoundtripsData() + { + var rule = NotifyRule.Create( + ruleId: "rule-1", + tenantId: "tenant-a", + name: "Critical Alerts", + match: NotifyRuleMatch.Create(eventKinds: new[] { NotifyEventKinds.ScannerReportReady }), + actions: new[] { new NotifyRuleAction("action-1", "slack:sec") }); + + await _repository.UpsertAsync(rule); + + var fetched = await _repository.GetAsync("tenant-a", "rule-1"); + Assert.NotNull(fetched); + Assert.Equal(rule.RuleId, fetched!.RuleId); + Assert.Equal(rule.SchemaVersion, fetched.SchemaVersion); + + var listed = await _repository.ListAsync("tenant-a"); + Assert.Single(listed); + + await _repository.DeleteAsync("tenant-a", "rule-1"); + var deleted = await _repository.GetAsync("tenant-a", "rule-1"); + Assert.Null(deleted); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyTemplateRepositoryTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyTemplateRepositoryTests.cs new file mode 100644 index 00000000..9f105754 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Repositories/NotifyTemplateRepositoryTests.cs @@ -0,0 +1,80 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Mongo2Go; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Repositories; + +public sealed class NotifyTemplateRepositoryTests : IAsyncLifetime +{ + private readonly MongoDbRunner _runner = MongoDbRunner.Start(singleNodeReplSet: true); + private readonly NotifyMongoContext _context; + private readonly NotifyMongoInitializer _initializer; + private readonly NotifyTemplateRepository _repository; + + public NotifyTemplateRepositoryTests() + { + var options = Microsoft.Extensions.Options.Options.Create(new NotifyMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = "notify-template-tests" + }); + + _context = new NotifyMongoContext(options, NullLogger.Instance); + _initializer = CreateInitializer(_context); + _repository = new NotifyTemplateRepository(_context); + } + + public Task DisposeAsync() + { + _runner.Dispose(); + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + await _initializer.EnsureIndexesAsync(); + } + + [Fact] + public async Task UpsertTemplatePersistsData() + { + var template = NotifyTemplate.Create( + templateId: "template-1", + tenantId: "tenant-a", + channelType: NotifyChannelType.Slack, + key: "concise", + locale: "en-us", + body: "{{summary}}", + renderMode: NotifyTemplateRenderMode.Markdown, + format: NotifyDeliveryFormat.Slack); + + await _repository.UpsertAsync(template); + + var fetched = await _repository.GetAsync("tenant-a", "template-1"); + Assert.NotNull(fetched); + Assert.Equal(template.TemplateId, fetched!.TemplateId); + + var listed = await _repository.ListAsync("tenant-a"); + Assert.Single(listed); + + await _repository.DeleteAsync("tenant-a", "template-1"); + Assert.Null(await _repository.GetAsync("tenant-a", "template-1")); + } + + private static NotifyMongoInitializer CreateInitializer(NotifyMongoContext context) + { + var migrations = new INotifyMongoMigration[] + { + new EnsureNotifyCollectionsMigration(NullLogger.Instance), + new EnsureNotifyIndexesMigration() + }; + + var runner = new NotifyMongoMigrationRunner(context, migrations, NullLogger.Instance); + return new NotifyMongoInitializer(context, runner, NullLogger.Instance); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyChannelDocumentMapperTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyChannelDocumentMapperTests.cs new file mode 100644 index 00000000..f3a12957 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyChannelDocumentMapperTests.cs @@ -0,0 +1,35 @@ +using System.Text.Json.Nodes; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Serialization; + +public sealed class NotifyChannelDocumentMapperTests +{ + [Fact] + public void RoundTripSampleChannelMaintainsCanonicalShape() + { + var sample = LoadSample("notify-channel@1.sample.json"); + var node = JsonNode.Parse(sample) ?? throw new InvalidOperationException("Sample JSON null."); + + var channel = NotifySchemaMigration.UpgradeChannel(node); + var bson = NotifyChannelDocumentMapper.ToBsonDocument(channel); + var restored = NotifyChannelDocumentMapper.FromBsonDocument(bson); + + var canonical = NotifyCanonicalJsonSerializer.Serialize(restored); + var canonicalNode = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical JSON null."); + + Assert.True(JsonNode.DeepEquals(node, canonicalNode), "Canonical JSON should match sample document."); + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyRuleDocumentMapperTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyRuleDocumentMapperTests.cs new file mode 100644 index 00000000..4d1c4974 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyRuleDocumentMapperTests.cs @@ -0,0 +1,36 @@ +using System.Text.Json.Nodes; +using MongoDB.Bson; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Serialization; + +public sealed class NotifyRuleDocumentMapperTests +{ + [Fact] + public void RoundTripSampleRuleMaintainsCanonicalShape() + { + var sample = LoadSample("notify-rule@1.sample.json"); + var node = JsonNode.Parse(sample) ?? throw new InvalidOperationException("Sample JSON null."); + + var rule = NotifySchemaMigration.UpgradeRule(node); + var bson = NotifyRuleDocumentMapper.ToBsonDocument(rule); + var restored = NotifyRuleDocumentMapper.FromBsonDocument(bson); + + var canonical = NotifyCanonicalJsonSerializer.Serialize(restored); + var canonicalNode = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical JSON null."); + + Assert.True(JsonNode.DeepEquals(node, canonicalNode), "Canonical JSON should match sample document."); + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyTemplateDocumentMapperTests.cs b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyTemplateDocumentMapperTests.cs new file mode 100644 index 00000000..b8126b29 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/Serialization/NotifyTemplateDocumentMapperTests.cs @@ -0,0 +1,35 @@ +using System.Text.Json.Nodes; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Tests.Serialization; + +public sealed class NotifyTemplateDocumentMapperTests +{ + [Fact] + public void RoundTripSampleTemplateMaintainsCanonicalShape() + { + var sample = LoadSample("notify-template@1.sample.json"); + var node = JsonNode.Parse(sample) ?? throw new InvalidOperationException("Sample JSON null."); + + var template = NotifySchemaMigration.UpgradeTemplate(node); + var bson = NotifyTemplateDocumentMapper.ToBsonDocument(template); + var restored = NotifyTemplateDocumentMapper.FromBsonDocument(bson); + + var canonical = NotifyCanonicalJsonSerializer.Serialize(restored); + var canonicalNode = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical JSON null."); + + Assert.True(JsonNode.DeepEquals(node, canonicalNode), "Canonical JSON should match sample document."); + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj b/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..ccf0cbe4 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo.Tests/StellaOps.Notify.Storage.Mongo.Tests.csproj @@ -0,0 +1,28 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + Always + + + diff --git a/src/StellaOps.Notify.Storage.Mongo/AGENTS.md b/src/StellaOps.Notify.Storage.Mongo/AGENTS.md new file mode 100644 index 00000000..22cb2910 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Storage.Mongo — Agent Charter + +## Mission +Implement Mongo persistence (rules, channels, deliveries, digests, locks, audit) per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyAuditEntryDocument.cs b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyAuditEntryDocument.cs new file mode 100644 index 00000000..7a376dba --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyAuditEntryDocument.cs @@ -0,0 +1,31 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Notify.Storage.Mongo.Documents; + +public sealed class NotifyAuditEntryDocument +{ + [BsonId] + public ObjectId Id { get; init; } + + [BsonElement("tenantId")] + public required string TenantId { get; init; } + + [BsonElement("actor")] + public required string Actor { get; init; } + + [BsonElement("action")] + public required string Action { get; init; } + + [BsonElement("entityId")] + public string EntityId { get; init; } = string.Empty; + + [BsonElement("entityType")] + public string EntityType { get; init; } = string.Empty; + + [BsonElement("timestamp")] + public required DateTimeOffset Timestamp { get; init; } + + [BsonElement("payload")] + public BsonDocument Payload { get; init; } = new(); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyDigestDocument.cs b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyDigestDocument.cs new file mode 100644 index 00000000..3febd3a7 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyDigestDocument.cs @@ -0,0 +1,39 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Notify.Storage.Mongo.Documents; + +public sealed class NotifyDigestDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("tenantId")] + public required string TenantId { get; init; } + + [BsonElement("actionKey")] + public required string ActionKey { get; init; } + + [BsonElement("window")] + public required string Window { get; init; } + + [BsonElement("openedAt")] + public required DateTimeOffset OpenedAt { get; init; } + + [BsonElement("status")] + public required string Status { get; init; } + + [BsonElement("items")] + public List Items { get; init; } = new(); +} + +public sealed class NotifyDigestItemDocument +{ + [BsonElement("eventId")] + public string EventId { get; init; } = string.Empty; + + [BsonElement("scope")] + public Dictionary Scope { get; init; } = new(); + + [BsonElement("delta")] + public Dictionary Delta { get; init; } = new(); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyLockDocument.cs b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyLockDocument.cs new file mode 100644 index 00000000..ac21273f --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Documents/NotifyLockDocument.cs @@ -0,0 +1,24 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Notify.Storage.Mongo.Documents; + +public sealed class NotifyLockDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("tenantId")] + public required string TenantId { get; init; } + + [BsonElement("resource")] + public required string Resource { get; init; } + + [BsonElement("acquiredAt")] + public required DateTimeOffset AcquiredAt { get; init; } + + [BsonElement("expiresAt")] + public required DateTimeOffset ExpiresAt { get; init; } + + [BsonElement("owner")] + public string Owner { get; init; } = string.Empty; +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoContext.cs b/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoContext.cs new file mode 100644 index 00000000..aa21764c --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoContext.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Options; + +namespace StellaOps.Notify.Storage.Mongo.Internal; + +internal sealed class NotifyMongoContext +{ + public NotifyMongoContext(IOptions options, ILogger logger) + { + ArgumentNullException.ThrowIfNull(logger); + var value = options?.Value ?? throw new ArgumentNullException(nameof(options)); + + if (string.IsNullOrWhiteSpace(value.ConnectionString)) + { + throw new InvalidOperationException("Notify Mongo connection string is not configured."); + } + + if (string.IsNullOrWhiteSpace(value.Database)) + { + throw new InvalidOperationException("Notify Mongo database name is not configured."); + } + + Client = new MongoClient(value.ConnectionString); + var settings = new MongoDatabaseSettings(); + if (value.UseMajorityReadConcern) + { + settings.ReadConcern = ReadConcern.Majority; + } + if (value.UseMajorityWriteConcern) + { + settings.WriteConcern = WriteConcern.WMajority; + } + + Database = Client.GetDatabase(value.Database, settings); + Options = value; + } + + public MongoClient Client { get; } + + public IMongoDatabase Database { get; } + + public NotifyMongoOptions Options { get; } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoInitializer.cs b/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoInitializer.cs new file mode 100644 index 00000000..dc0ecc1b --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Internal/NotifyMongoInitializer.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Logging; +using StellaOps.Notify.Storage.Mongo.Migrations; + +namespace StellaOps.Notify.Storage.Mongo.Internal; + +internal interface INotifyMongoInitializer +{ + Task EnsureIndexesAsync(CancellationToken cancellationToken = default); +} + +internal sealed class NotifyMongoInitializer : INotifyMongoInitializer +{ + private readonly NotifyMongoContext _context; + private readonly NotifyMongoMigrationRunner _migrationRunner; + private readonly ILogger _logger; + + public NotifyMongoInitializer( + NotifyMongoContext context, + NotifyMongoMigrationRunner migrationRunner, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + _migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task EnsureIndexesAsync(CancellationToken cancellationToken = default) + { + _logger.LogInformation("Ensuring Notify Mongo migrations are applied for database {Database}.", _context.Options.Database); + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyCollectionsMigration.cs b/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyCollectionsMigration.cs new file mode 100644 index 00000000..7cd4b891 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyCollectionsMigration.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Migrations; + +internal sealed class EnsureNotifyCollectionsMigration : INotifyMongoMigration +{ + private readonly ILogger _logger; + + public EnsureNotifyCollectionsMigration(ILogger logger) + => _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + public string Id => "20251019_notify_collections_v1"; + + public async ValueTask ExecuteAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var requiredCollections = new[] + { + context.Options.RulesCollection, + context.Options.ChannelsCollection, + context.Options.TemplatesCollection, + context.Options.DeliveriesCollection, + context.Options.DigestsCollection, + context.Options.LocksCollection, + context.Options.AuditCollection, + context.Options.MigrationsCollection + }; + + var cursor = await context.Database + .ListCollectionNamesAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var existingNames = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + foreach (var collection in requiredCollections) + { + if (existingNames.Contains(collection, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Creating Notify Mongo collection '{CollectionName}'.", collection); + await context.Database.CreateCollectionAsync(collection, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyIndexesMigration.cs b/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyIndexesMigration.cs new file mode 100644 index 00000000..5b206e8a --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Migrations/EnsureNotifyIndexesMigration.cs @@ -0,0 +1,165 @@ +using System; +using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Migrations; + +internal sealed class EnsureNotifyIndexesMigration : INotifyMongoMigration +{ + public string Id => "20251019_notify_indexes_v1"; + + public async ValueTask ExecuteAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + await EnsureRulesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureChannelsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureTemplatesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureDeliveriesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureDigestsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureLocksIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureAuditIndexesAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureRulesIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.RulesCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Ascending("enabled"); + + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_enabled" + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureChannelsIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.ChannelsCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Ascending("type") + .Ascending("enabled"); + + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_type_enabled" + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureTemplatesIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.TemplatesCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Ascending("channelType") + .Ascending("key") + .Ascending("locale"); + + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_channel_key_locale", + Unique = true + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureDeliveriesIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.DeliveriesCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Descending("sortKey"); + + var sortModel = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_sortKey" + }); + + await collection.Indexes.CreateOneAsync(sortModel, cancellationToken: cancellationToken).ConfigureAwait(false); + + var statusModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("tenantId").Ascending("status"), + new CreateIndexOptions + { + Name = "tenant_status" + }); + + await collection.Indexes.CreateOneAsync(statusModel, cancellationToken: cancellationToken).ConfigureAwait(false); + + if (context.Options.DeliveryHistoryRetention > TimeSpan.Zero) + { + var ttlModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("completedAt"), + new CreateIndexOptions + { + Name = "completedAt_ttl", + ExpireAfter = context.Options.DeliveryHistoryRetention + }); + + await collection.Indexes.CreateOneAsync(ttlModel, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + private static async Task EnsureDigestsIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.DigestsCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Ascending("actionKey"); + + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_actionKey" + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureLocksIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.LocksCollection); + var uniqueModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("tenantId").Ascending("resource"), + new CreateIndexOptions + { + Name = "tenant_resource", + Unique = true + }); + + await collection.Indexes.CreateOneAsync(uniqueModel, cancellationToken: cancellationToken).ConfigureAwait(false); + + var ttlModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("expiresAt"), + new CreateIndexOptions + { + Name = "expiresAt_ttl", + ExpireAfter = TimeSpan.Zero + }); + + await collection.Indexes.CreateOneAsync(ttlModel, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureAuditIndexesAsync(NotifyMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.AuditCollection); + var keys = Builders.IndexKeys + .Ascending("tenantId") + .Descending("timestamp"); + + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "tenant_timestamp" + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Migrations/INotifyMongoMigration.cs b/src/StellaOps.Notify.Storage.Mongo/Migrations/INotifyMongoMigration.cs new file mode 100644 index 00000000..d1721f0b --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Migrations/INotifyMongoMigration.cs @@ -0,0 +1,12 @@ +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Migrations; + +internal interface INotifyMongoMigration +{ + string Id { get; } + + ValueTask ExecuteAsync(NotifyMongoContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRecord.cs b/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRecord.cs new file mode 100644 index 00000000..af255061 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRecord.cs @@ -0,0 +1,16 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Notify.Storage.Mongo.Migrations; + +internal sealed class NotifyMongoMigrationRecord +{ + [BsonId] + public ObjectId Id { get; init; } + + [BsonElement("migrationId")] + public required string MigrationId { get; init; } + + [BsonElement("appliedAt")] + public required DateTimeOffset AppliedAt { get; init; } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRunner.cs b/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRunner.cs new file mode 100644 index 00000000..5c9c3672 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Migrations/NotifyMongoMigrationRunner.cs @@ -0,0 +1,78 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Migrations; + +internal sealed class NotifyMongoMigrationRunner +{ + private readonly NotifyMongoContext _context; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + + public NotifyMongoMigrationRunner( + NotifyMongoContext context, + IEnumerable migrations, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + ArgumentNullException.ThrowIfNull(migrations); + _migrations = migrations.OrderBy(migration => migration.Id, StringComparer.Ordinal).ToArray(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + var collection = _context.Database.GetCollection(_context.Options.MigrationsCollection); + await EnsureMigrationIndexAsync(collection, cancellationToken).ConfigureAwait(false); + + var applied = await collection + .Find(FilterDefinition.Empty) + .Project(record => record.MigrationId) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var appliedSet = applied.ToHashSet(StringComparer.Ordinal); + + foreach (var migration in _migrations) + { + if (appliedSet.Contains(migration.Id)) + { + continue; + } + + _logger.LogInformation("Applying Notify Mongo migration {MigrationId}.", migration.Id); + await migration.ExecuteAsync(_context, cancellationToken).ConfigureAwait(false); + + var record = new NotifyMongoMigrationRecord + { + Id = ObjectId.GenerateNewId(), + MigrationId = migration.Id, + AppliedAt = DateTimeOffset.UtcNow + }; + + await collection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed Notify Mongo migration {MigrationId}.", migration.Id); + } + } + + private static async Task EnsureMigrationIndexAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + var keys = Builders.IndexKeys.Ascending(record => record.MigrationId); + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "migrationId_unique", + Unique = true + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Options/NotifyMongoOptions.cs b/src/StellaOps.Notify.Storage.Mongo/Options/NotifyMongoOptions.cs new file mode 100644 index 00000000..c4217d0e --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Options/NotifyMongoOptions.cs @@ -0,0 +1,32 @@ +using System; + +namespace StellaOps.Notify.Storage.Mongo.Options; + +public sealed class NotifyMongoOptions +{ + public string ConnectionString { get; set; } = "mongodb://localhost:27017"; + + public string Database { get; set; } = "stellaops_notify"; + + public string RulesCollection { get; set; } = "rules"; + + public string ChannelsCollection { get; set; } = "channels"; + + public string TemplatesCollection { get; set; } = "templates"; + + public string DeliveriesCollection { get; set; } = "deliveries"; + + public string DigestsCollection { get; set; } = "digests"; + + public string LocksCollection { get; set; } = "locks"; + + public string AuditCollection { get; set; } = "audit"; + + public string MigrationsCollection { get; set; } = "_notify_migrations"; + + public TimeSpan DeliveryHistoryRetention { get; set; } = TimeSpan.FromDays(90); + + public bool UseMajorityReadConcern { get; set; } = true; + + public bool UseMajorityWriteConcern { get; set; } = true; +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Notify.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..8b53fafe --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Notify.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyAuditRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyAuditRepository.cs new file mode 100644 index 00000000..ae3a0901 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyAuditRepository.cs @@ -0,0 +1,10 @@ +using StellaOps.Notify.Storage.Mongo.Documents; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyAuditRepository +{ + Task AppendAsync(NotifyAuditEntryDocument entry, CancellationToken cancellationToken = default); + + Task> QueryAsync(string tenantId, DateTimeOffset? since, int? limit, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyChannelRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyChannelRepository.cs new file mode 100644 index 00000000..96a01781 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyChannelRepository.cs @@ -0,0 +1,14 @@ +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyChannelRepository +{ + Task UpsertAsync(NotifyChannel channel, CancellationToken cancellationToken = default); + + Task GetAsync(string tenantId, string channelId, CancellationToken cancellationToken = default); + + Task> ListAsync(string tenantId, CancellationToken cancellationToken = default); + + Task DeleteAsync(string tenantId, string channelId, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDeliveryRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDeliveryRepository.cs new file mode 100644 index 00000000..47233da9 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDeliveryRepository.cs @@ -0,0 +1,20 @@ +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyDeliveryRepository +{ + Task AppendAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default); + + Task UpdateAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default); + + Task GetAsync(string tenantId, string deliveryId, CancellationToken cancellationToken = default); + + Task QueryAsync( + string tenantId, + DateTimeOffset? since, + string? status, + int? limit, + string? continuationToken = null, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDigestRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDigestRepository.cs new file mode 100644 index 00000000..bbaf4a60 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyDigestRepository.cs @@ -0,0 +1,12 @@ +using StellaOps.Notify.Storage.Mongo.Documents; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyDigestRepository +{ + Task GetAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default); + + Task UpsertAsync(NotifyDigestDocument document, CancellationToken cancellationToken = default); + + Task RemoveAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyLockRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyLockRepository.cs new file mode 100644 index 00000000..db6eca31 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyLockRepository.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyLockRepository +{ + Task TryAcquireAsync(string tenantId, string resource, string owner, TimeSpan ttl, CancellationToken cancellationToken = default); + + Task ReleaseAsync(string tenantId, string resource, string owner, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyRuleRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyRuleRepository.cs new file mode 100644 index 00000000..d01394da --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyRuleRepository.cs @@ -0,0 +1,14 @@ +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyRuleRepository +{ + Task UpsertAsync(NotifyRule rule, CancellationToken cancellationToken = default); + + Task GetAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default); + + Task> ListAsync(string tenantId, CancellationToken cancellationToken = default); + + Task DeleteAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyTemplateRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyTemplateRepository.cs new file mode 100644 index 00000000..150416e6 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/INotifyTemplateRepository.cs @@ -0,0 +1,14 @@ +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public interface INotifyTemplateRepository +{ + Task UpsertAsync(NotifyTemplate template, CancellationToken cancellationToken = default); + + Task GetAsync(string tenantId, string templateId, CancellationToken cancellationToken = default); + + Task> ListAsync(string tenantId, CancellationToken cancellationToken = default); + + Task DeleteAsync(string tenantId, string templateId, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyAuditRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyAuditRepository.cs new file mode 100644 index 00000000..a4a5388d --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyAuditRepository.cs @@ -0,0 +1,40 @@ +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyAuditRepository : INotifyAuditRepository +{ + private readonly IMongoCollection _collection; + + public NotifyAuditRepository(NotifyMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.AuditCollection); + } + + public async Task AppendAsync(NotifyAuditEntryDocument entry, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(entry); + await _collection.InsertOneAsync(entry, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task> QueryAsync(string tenantId, DateTimeOffset? since, int? limit, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.TenantId, tenantId); + if (since is not null) + { + filter &= Builders.Filter.Gte(x => x.Timestamp, since.Value); + } + + var ordered = _collection.Find(filter).SortByDescending(x => x.Timestamp); + IFindFluent query = ordered; + if (limit is > 0) + { + query = query.Limit(limit); + } + + return await query.ToListAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyChannelRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyChannelRepository.cs new file mode 100644 index 00000000..52f9f835 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyChannelRepository.cs @@ -0,0 +1,70 @@ +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyChannelRepository : INotifyChannelRepository +{ + private readonly IMongoCollection _collection; + + public NotifyChannelRepository(NotifyMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.ChannelsCollection); + } + + public async Task UpsertAsync(NotifyChannel channel, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(channel); + var document = NotifyChannelDocumentMapper.ToBsonDocument(channel); + var filter = Builders.Filter.Eq("_id", CreateDocumentId(channel.TenantId, channel.ChannelId)); + + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string tenantId, string channelId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, channelId)) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : NotifyChannelDocumentMapper.FromBsonDocument(document); + } + + public async Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("tenantId", tenantId) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + var cursor = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return cursor.Select(NotifyChannelDocumentMapper.FromBsonDocument).ToArray(); + } + + public async Task DeleteAsync(string tenantId, string channelId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, channelId)); + await _collection.UpdateOneAsync(filter, + Builders.Update.Set("deletedAt", DateTime.UtcNow).Set("enabled", false), + new UpdateOptions { IsUpsert = false }, + cancellationToken).ConfigureAwait(false); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryQueryResult.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryQueryResult.cs new file mode 100644 index 00000000..48e9c50e --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryQueryResult.cs @@ -0,0 +1,6 @@ +using System.Collections.Generic; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +public sealed record NotifyDeliveryQueryResult(IReadOnlyList Items, string? ContinuationToken); diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryRepository.cs new file mode 100644 index 00000000..af7e5f35 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDeliveryRepository.cs @@ -0,0 +1,179 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyDeliveryRepository : INotifyDeliveryRepository +{ + private readonly IMongoCollection _collection; + + public NotifyDeliveryRepository(NotifyMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.DeliveriesCollection); + } + + public Task AppendAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default) + => UpdateAsync(delivery, cancellationToken); + + public async Task UpdateAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(delivery); + var document = NotifyDeliveryDocumentMapper.ToBsonDocument(delivery); + var filter = Builders.Filter.Eq("_id", CreateDocumentId(delivery.TenantId, delivery.DeliveryId)); + + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string tenantId, string deliveryId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, deliveryId)); + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : NotifyDeliveryDocumentMapper.FromBsonDocument(document); + } + + public async Task QueryAsync( + string tenantId, + DateTimeOffset? since, + string? status, + int? limit, + string? continuationToken = null, + CancellationToken cancellationToken = default) + { + var builder = Builders.Filter; + var filter = builder.Eq("tenantId", tenantId); + if (since is not null) + { + filter &= builder.Gte("sortKey", since.Value.UtcDateTime); + } + + if (!string.IsNullOrWhiteSpace(status)) + { + var statuses = status + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(static value => value.ToLowerInvariant()) + .ToArray(); + + if (statuses.Length == 1) + { + filter &= builder.Eq("status", statuses[0]); + } + else if (statuses.Length > 1) + { + filter &= builder.In("status", statuses); + } + } + + if (!string.IsNullOrWhiteSpace(continuationToken)) + { + if (!TryParseContinuationToken(continuationToken, out var continuationSortKey, out var continuationId)) + { + throw new ArgumentException("The continuation token is invalid.", nameof(continuationToken)); + } + + var lessThanSort = builder.Lt("sortKey", continuationSortKey); + var equalSortLowerId = builder.And(builder.Eq("sortKey", continuationSortKey), builder.Lte("_id", continuationId)); + filter &= builder.Or(lessThanSort, equalSortLowerId); + } + + var find = _collection.Find(filter) + .Sort(Builders.Sort.Descending("sortKey").Descending("_id")); + + List documents; + if (limit is > 0) + { + documents = await find.Limit(limit.Value + 1).ToListAsync(cancellationToken).ConfigureAwait(false); + } + else + { + documents = await find.ToListAsync(cancellationToken).ConfigureAwait(false); + } + + string? nextToken = null; + if (limit is > 0 && documents.Count > limit.Value) + { + var overflow = documents[^1]; + documents.RemoveAt(documents.Count - 1); + nextToken = BuildContinuationToken(overflow); + } + + var deliveries = documents.Select(NotifyDeliveryDocumentMapper.FromBsonDocument).ToArray(); + return new NotifyDeliveryQueryResult(deliveries, nextToken); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); + + private static string BuildContinuationToken(BsonDocument document) + { + var sortKey = ResolveSortKey(document); + if (!document.TryGetValue("_id", out var idValue) || !idValue.IsString) + { + throw new InvalidOperationException("Delivery document missing string _id required for continuation token."); + } + + return BuildContinuationToken(sortKey, idValue.AsString); + } + + private static DateTime ResolveSortKey(BsonDocument document) + { + if (document.TryGetValue("sortKey", out var sortValue) && sortValue.IsValidDateTime) + { + return sortValue.ToUniversalTime(); + } + + if (document.TryGetValue("completedAt", out var completed) && completed.IsValidDateTime) + { + return completed.ToUniversalTime(); + } + + if (document.TryGetValue("sentAt", out var sent) && sent.IsValidDateTime) + { + return sent.ToUniversalTime(); + } + + var created = document["createdAt"]; + return created.ToUniversalTime(); + } + + private static string BuildContinuationToken(DateTime sortKey, string id) + => FormattableString.Invariant($"{sortKey:O}|{id}"); + + private static bool TryParseContinuationToken(string token, out DateTime sortKey, out string id) + { + sortKey = default; + id = string.Empty; + + var parts = token.Split('|', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return false; + } + + if (!DateTime.TryParseExact(parts[0], "O", CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var parsedSort)) + { + return false; + } + + if (string.IsNullOrWhiteSpace(parts[1])) + { + return false; + } + + sortKey = parsedSort.ToUniversalTime(); + id = parts[1]; + return true; + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDigestRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDigestRepository.cs new file mode 100644 index 00000000..88709f85 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyDigestRepository.cs @@ -0,0 +1,44 @@ +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyDigestRepository : INotifyDigestRepository +{ + private readonly IMongoCollection _collection; + + public NotifyDigestRepository(NotifyMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.DigestsCollection); + } + + public async Task GetAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.Id, CreateDocumentId(tenantId, actionKey)); + return await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + } + + public async Task UpsertAsync(NotifyDigestDocument document, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(document); + document.Id = CreateDocumentId(document.TenantId, document.ActionKey); + var filter = Builders.Filter.Eq(x => x.Id, document.Id); + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + public async Task RemoveAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.Id, CreateDocumentId(tenantId, actionKey)); + await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); + } + + private static string CreateDocumentId(string tenantId, string actionKey) + => string.Create(tenantId.Length + actionKey.Length + 1, (tenantId, actionKey), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.actionKey.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyLockRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyLockRepository.cs new file mode 100644 index 00000000..1b479653 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyLockRepository.cs @@ -0,0 +1,71 @@ +using MongoDB.Driver; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Internal; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyLockRepository : INotifyLockRepository +{ + private readonly IMongoCollection _collection; + + public NotifyLockRepository(NotifyMongoContext context) + { + ArgumentNullException.ThrowIfNull(context); + _collection = context.Database.GetCollection(context.Options.LocksCollection); + } + + public async Task TryAcquireAsync(string tenantId, string resource, string owner, TimeSpan ttl, CancellationToken cancellationToken = default) + { + var now = DateTimeOffset.UtcNow; + var document = new NotifyLockDocument + { + Id = CreateDocumentId(tenantId, resource), + TenantId = tenantId, + Resource = resource, + Owner = owner, + AcquiredAt = now, + ExpiresAt = now.Add(ttl) + }; + + var candidateFilter = Builders.Filter.Eq(x => x.Id, document.Id); + var takeoverFilter = candidateFilter & Builders.Filter.Lt(x => x.ExpiresAt, now.UtcDateTime); + var sameOwnerFilter = candidateFilter & Builders.Filter.Eq(x => x.Owner, owner); + + var update = Builders.Update + .Set(x => x.TenantId, document.TenantId) + .Set(x => x.Resource, document.Resource) + .Set(x => x.Owner, document.Owner) + .Set(x => x.AcquiredAt, document.AcquiredAt) + .Set(x => x.ExpiresAt, document.ExpiresAt); + + try + { + var result = await _collection.UpdateOneAsync( + takeoverFilter | sameOwnerFilter, + update.SetOnInsert(x => x.Id, document.Id), + new UpdateOptions { IsUpsert = true }, + cancellationToken).ConfigureAwait(false); + + return result.MatchedCount > 0 || result.UpsertedId != null; + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + return false; + } + } + + public async Task ReleaseAsync(string tenantId, string resource, string owner, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq(x => x.Id, CreateDocumentId(tenantId, resource)) + & Builders.Filter.Eq(x => x.Owner, owner); + await _collection.DeleteOneAsync(filter, cancellationToken).ConfigureAwait(false); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyRuleRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyRuleRepository.cs new file mode 100644 index 00000000..9ffae84e --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyRuleRepository.cs @@ -0,0 +1,73 @@ +using System; +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyRuleRepository : INotifyRuleRepository +{ + private readonly IMongoCollection _collection; + + public NotifyRuleRepository(NotifyMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.RulesCollection); + } + + public async Task UpsertAsync(NotifyRule rule, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(rule); + var document = NotifyRuleDocumentMapper.ToBsonDocument(rule); + var filter = Builders.Filter.Eq("_id", CreateDocumentId(rule.TenantId, rule.RuleId)); + + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, ruleId)) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : NotifyRuleDocumentMapper.FromBsonDocument(document); + } + + public async Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("tenantId", tenantId) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + var cursor = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return cursor.Select(NotifyRuleDocumentMapper.FromBsonDocument).ToArray(); + } + + public async Task DeleteAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, ruleId)); + await _collection.UpdateOneAsync(filter, + Builders.Update + .Set("deletedAt", DateTime.UtcNow) + .Set("enabled", false), + new UpdateOptions { IsUpsert = false }, + cancellationToken).ConfigureAwait(false); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyTemplateRepository.cs b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyTemplateRepository.cs new file mode 100644 index 00000000..97af9611 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Repositories/NotifyTemplateRepository.cs @@ -0,0 +1,70 @@ +using System.Linq; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Serialization; + +namespace StellaOps.Notify.Storage.Mongo.Repositories; + +internal sealed class NotifyTemplateRepository : INotifyTemplateRepository +{ + private readonly IMongoCollection _collection; + + public NotifyTemplateRepository(NotifyMongoContext context) + { + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + _collection = context.Database.GetCollection(context.Options.TemplatesCollection); + } + + public async Task UpsertAsync(NotifyTemplate template, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(template); + var document = NotifyTemplateDocumentMapper.ToBsonDocument(template); + var filter = Builders.Filter.Eq("_id", CreateDocumentId(template.TenantId, template.TemplateId)); + + await _collection.ReplaceOneAsync(filter, document, new ReplaceOptions { IsUpsert = true }, cancellationToken).ConfigureAwait(false); + } + + public async Task GetAsync(string tenantId, string templateId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, templateId)) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + + var document = await _collection.Find(filter).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); + return document is null ? null : NotifyTemplateDocumentMapper.FromBsonDocument(document); + } + + public async Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("tenantId", tenantId) + & Builders.Filter.Or( + Builders.Filter.Exists("deletedAt", false), + Builders.Filter.Eq("deletedAt", BsonNull.Value)); + var cursor = await _collection.Find(filter).ToListAsync(cancellationToken).ConfigureAwait(false); + return cursor.Select(NotifyTemplateDocumentMapper.FromBsonDocument).ToArray(); + } + + public async Task DeleteAsync(string tenantId, string templateId, CancellationToken cancellationToken = default) + { + var filter = Builders.Filter.Eq("_id", CreateDocumentId(tenantId, templateId)); + await _collection.UpdateOneAsync(filter, + Builders.Update.Set("deletedAt", DateTime.UtcNow), + new UpdateOptions { IsUpsert = false }, + cancellationToken).ConfigureAwait(false); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs b/src/StellaOps.Notify.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs new file mode 100644 index 00000000..406bbcd6 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Serialization/BsonDocumentJsonExtensions.cs @@ -0,0 +1,129 @@ +using System.Globalization; +using System.Text.Json.Nodes; +using MongoDB.Bson; +using MongoDB.Bson.IO; + +namespace StellaOps.Notify.Storage.Mongo.Serialization; + +internal static class BsonDocumentJsonExtensions +{ + public static JsonNode ToCanonicalJsonNode(this BsonDocument document, params string[] fieldsToRemove) + { + ArgumentNullException.ThrowIfNull(document); + + var clone = document.DeepClone().AsBsonDocument; + clone.Remove("_id"); + if (fieldsToRemove is { Length: > 0 }) + { + foreach (var field in fieldsToRemove) + { + clone.Remove(field); + } + } + + var json = clone.ToJson(new JsonWriterSettings + { + OutputMode = JsonOutputMode.RelaxedExtendedJson, + Indent = false + }); + + var node = JsonNode.Parse(json) ?? throw new InvalidOperationException("Unable to parse BsonDocument JSON."); + return NormalizeExtendedJson(node); + } + + private static JsonNode NormalizeExtendedJson(JsonNode node) + { + if (node is JsonObject obj) + { + if (TryConvertExtendedDate(obj, out var replacement)) + { + return replacement; + } + + foreach (var property in obj.ToList()) + { + if (property.Value is null) + { + continue; + } + + var normalized = NormalizeExtendedJson(property.Value); + if (!ReferenceEquals(normalized, property.Value)) + { + obj[property.Key] = normalized; + } + } + + return obj; + } + + if (node is JsonArray array) + { + for (var i = 0; i < array.Count; i++) + { + if (array[i] is null) + { + continue; + } + + var normalized = NormalizeExtendedJson(array[i]!); + if (!ReferenceEquals(normalized, array[i])) + { + array[i] = normalized; + } + } + + return array; + } + + return node; + } + + private static bool TryConvertExtendedDate(JsonObject obj, out JsonNode replacement) + { + replacement = obj; + if (obj.Count != 1 || !obj.TryGetPropertyValue("$date", out var value) || value is null) + { + return false; + } + + if (value is JsonValue directValue) + { + if (directValue.TryGetValue(out string? dateString) && TryParseIso(dateString, out var iso)) + { + replacement = JsonValue.Create(iso); + return true; + } + + if (directValue.TryGetValue(out long epochMilliseconds)) + { + replacement = JsonValue.Create(DateTimeOffset.FromUnixTimeMilliseconds(epochMilliseconds).ToString("O")); + return true; + } + } + else if (value is JsonObject nested && nested.TryGetPropertyValue("$numberLong", out var numberNode) && numberNode is JsonValue numberValue && numberValue.TryGetValue(out string? numberString) && long.TryParse(numberString, NumberStyles.Integer, CultureInfo.InvariantCulture, out var ms)) + { + replacement = JsonValue.Create(DateTimeOffset.FromUnixTimeMilliseconds(ms).ToString("O")); + return true; + } + + return false; + } + + private static bool TryParseIso(string? value, out string iso) + { + iso = string.Empty; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var parsed)) + { + iso = parsed.ToUniversalTime().ToString("O"); + return true; + } + + return false; + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyChannelDocumentMapper.cs b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyChannelDocumentMapper.cs new file mode 100644 index 00000000..152e61ce --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyChannelDocumentMapper.cs @@ -0,0 +1,33 @@ +using System.Text.Json.Nodes; +using MongoDB.Bson; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Serialization; + +internal static class NotifyChannelDocumentMapper +{ + public static BsonDocument ToBsonDocument(NotifyChannel channel) + { + ArgumentNullException.ThrowIfNull(channel); + var json = NotifyCanonicalJsonSerializer.Serialize(channel); + var document = BsonDocument.Parse(json); + document["_id"] = BsonValue.Create(CreateDocumentId(channel.TenantId, channel.ChannelId)); + return document; + } + + public static NotifyChannel FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var node = document.ToCanonicalJsonNode(); + return NotifySchemaMigration.UpgradeChannel(node); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyDeliveryDocumentMapper.cs b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyDeliveryDocumentMapper.cs new file mode 100644 index 00000000..f649b958 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyDeliveryDocumentMapper.cs @@ -0,0 +1,46 @@ +using System.Text.Json.Nodes; +using MongoDB.Bson; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Serialization; + +internal static class NotifyDeliveryDocumentMapper +{ + public static BsonDocument ToBsonDocument(NotifyDelivery delivery) + { + ArgumentNullException.ThrowIfNull(delivery); + var json = NotifyCanonicalJsonSerializer.Serialize(delivery); + var document = BsonDocument.Parse(json); + document["_id"] = BsonValue.Create(CreateDocumentId(delivery.TenantId, delivery.DeliveryId)); + document["tenantId"] = delivery.TenantId; + document["createdAt"] = delivery.CreatedAt.UtcDateTime; + if (delivery.SentAt is not null) + { + document["sentAt"] = delivery.SentAt.Value.UtcDateTime; + } + + if (delivery.CompletedAt is not null) + { + document["completedAt"] = delivery.CompletedAt.Value.UtcDateTime; + } + + var sortTimestamp = delivery.CompletedAt ?? delivery.SentAt ?? delivery.CreatedAt; + document["sortKey"] = sortTimestamp.UtcDateTime; + return document; + } + + public static NotifyDelivery FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + var node = document.ToCanonicalJsonNode("sortKey"); + return NotifyCanonicalJsonSerializer.Deserialize(node.ToJsonString()); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyRuleDocumentMapper.cs b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyRuleDocumentMapper.cs new file mode 100644 index 00000000..d66a6e25 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyRuleDocumentMapper.cs @@ -0,0 +1,33 @@ +using System.Text.Json.Nodes; +using MongoDB.Bson; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Serialization; + +internal static class NotifyRuleDocumentMapper +{ + public static BsonDocument ToBsonDocument(NotifyRule rule) + { + ArgumentNullException.ThrowIfNull(rule); + var json = NotifyCanonicalJsonSerializer.Serialize(rule); + var document = BsonDocument.Parse(json); + document["_id"] = BsonValue.Create(CreateDocumentId(rule.TenantId, rule.RuleId)); + return document; + } + + public static NotifyRule FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var node = document.ToCanonicalJsonNode(); + return NotifySchemaMigration.UpgradeRule(node); + } + + private static string CreateDocumentId(string tenantId, string ruleId) + => string.Create(tenantId.Length + ruleId.Length + 1, (tenantId, ruleId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.ruleId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyTemplateDocumentMapper.cs b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyTemplateDocumentMapper.cs new file mode 100644 index 00000000..36c5f834 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/Serialization/NotifyTemplateDocumentMapper.cs @@ -0,0 +1,33 @@ +using System.Text.Json.Nodes; +using MongoDB.Bson; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.Storage.Mongo.Serialization; + +internal static class NotifyTemplateDocumentMapper +{ + public static BsonDocument ToBsonDocument(NotifyTemplate template) + { + ArgumentNullException.ThrowIfNull(template); + var json = NotifyCanonicalJsonSerializer.Serialize(template); + var document = BsonDocument.Parse(json); + document["_id"] = BsonValue.Create(CreateDocumentId(template.TenantId, template.TemplateId)); + return document; + } + + public static NotifyTemplate FromBsonDocument(BsonDocument document) + { + ArgumentNullException.ThrowIfNull(document); + + var node = document.ToCanonicalJsonNode(); + return NotifySchemaMigration.UpgradeTemplate(node); + } + + private static string CreateDocumentId(string tenantId, string resourceId) + => string.Create(tenantId.Length + resourceId.Length + 1, (tenantId, resourceId), static (span, value) => + { + value.tenantId.AsSpan().CopyTo(span); + span[value.tenantId.Length] = ':'; + value.resourceId.AsSpan().CopyTo(span[(value.tenantId.Length + 1)..]); + }); +} diff --git a/src/StellaOps.Notify.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Notify.Storage.Mongo/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..11079598 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/ServiceCollectionExtensions.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Notify.Storage.Mongo.Internal; +using StellaOps.Notify.Storage.Mongo.Migrations; +using StellaOps.Notify.Storage.Mongo.Options; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.Storage.Mongo; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddNotifyMongoStorage(this IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.Configure(configuration); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj b/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj new file mode 100644 index 00000000..c6563738 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/StellaOps.Notify.Storage.Mongo.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.Storage.Mongo/TASKS.md b/src/StellaOps.Notify.Storage.Mongo/TASKS.md new file mode 100644 index 00000000..12090060 --- /dev/null +++ b/src/StellaOps.Notify.Storage.Mongo/TASKS.md @@ -0,0 +1,7 @@ +# Notify Storage Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-STORAGE-15-201 | DONE (2025-10-19) | Notify Storage Guild | NOTIFY-MODELS-15-101 | Create Mongo schemas/collections (rules, channels, deliveries, digests, locks, audit) with indexes per architecture §7. | Migration scripts authored; indexes tested; integration tests cover CRUD/read paths. | +| NOTIFY-STORAGE-15-202 | DONE (2025-10-19) | Notify Storage Guild | NOTIFY-STORAGE-15-201 | Implement repositories/services with tenant scoping, soft deletes, TTL, causal consistency (majority) options. | Repositories unit-tested; soft delete + TTL validated; majority read/write configuration documented. | +| NOTIFY-STORAGE-15-203 | DONE (2025-10-19) | Notify Storage Guild | NOTIFY-STORAGE-15-201 | Delivery history retention + query APIs (paging, filters). | History queries return expected data; paging verified; docs updated. | diff --git a/src/StellaOps.Notify.WebService.Tests/CrudEndpointsTests.cs b/src/StellaOps.Notify.WebService.Tests/CrudEndpointsTests.cs new file mode 100644 index 00000000..84b47072 --- /dev/null +++ b/src/StellaOps.Notify.WebService.Tests/CrudEndpointsTests.cs @@ -0,0 +1,417 @@ +using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json.Nodes; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.IdentityModel.Tokens; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.WebService.Tests; + +public sealed class CrudEndpointsTests : IClassFixture>, IAsyncLifetime +{ + private const string SigningKey = "super-secret-test-key-1234567890"; + private const string Issuer = "test-issuer"; + private const string Audience = "notify"; + + private readonly WebApplicationFactory _factory; + private readonly string _adminToken; + private readonly string _readToken; + + public CrudEndpointsTests(WebApplicationFactory factory) + { + _factory = factory.WithWebHostBuilder(builder => + { + builder.UseSetting("notify:storage:driver", "memory"); + builder.UseSetting("notify:authority:enabled", "false"); + builder.UseSetting("notify:authority:developmentSigningKey", SigningKey); + builder.UseSetting("notify:authority:issuer", Issuer); + builder.UseSetting("notify:authority:audiences:0", Audience); + builder.UseSetting("notify:authority:adminScope", "notify.admin"); + builder.UseSetting("notify:authority:readScope", "notify.read"); + builder.UseSetting("notify:telemetry:enableRequestLogging", "false"); + builder.UseSetting("notify:api:rateLimits:testSend:tokenLimit", "10"); + builder.UseSetting("notify:api:rateLimits:testSend:tokensPerPeriod", "10"); + builder.UseSetting("notify:api:rateLimits:testSend:queueLimit", "5"); + builder.UseSetting("notify:api:rateLimits:deliveryHistory:tokenLimit", "30"); + builder.UseSetting("notify:api:rateLimits:deliveryHistory:tokensPerPeriod", "30"); + builder.UseSetting("notify:api:rateLimits:deliveryHistory:queueLimit", "10"); + }); + + _adminToken = CreateToken("notify.admin"); + _readToken = CreateToken("notify.read"); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task RuleCrudLifecycle() + { + var client = _factory.CreateClient(); + var payload = LoadSample("notify-rule@1.sample.json"); + payload["ruleId"] = "rule-web"; + payload["tenantId"] = "tenant-web"; + payload["actions"]!.AsArray()[0]! ["actionId"] = "action-web"; + + await PostAsync(client, "/api/v1/notify/rules", payload); + + var list = await GetJsonArrayAsync(client, "/api/v1/notify/rules", useAdminToken: false); + Assert.Equal("rule-web", list?[0]? ["ruleId"]?.GetValue()); + + var single = await GetJsonObjectAsync(client, "/api/v1/notify/rules/rule-web", useAdminToken: false); + Assert.Equal("tenant-web", single? ["tenantId"]?.GetValue()); + + await DeleteAsync(client, "/api/v1/notify/rules/rule-web"); + var afterDelete = await SendAsync(client, HttpMethod.Get, "/api/v1/notify/rules/rule-web", useAdminToken: false); + Assert.Equal(HttpStatusCode.NotFound, afterDelete.StatusCode); + } + + [Fact] + public async Task ChannelTemplateDeliveryAndAuditFlows() + { + var client = _factory.CreateClient(); + + var channelPayload = LoadSample("notify-channel@1.sample.json"); + channelPayload["channelId"] = "channel-web"; + channelPayload["tenantId"] = "tenant-web"; + await PostAsync(client, "/api/v1/notify/channels", channelPayload); + + var templatePayload = LoadSample("notify-template@1.sample.json"); + templatePayload["templateId"] = "template-web"; + templatePayload["tenantId"] = "tenant-web"; + await PostAsync(client, "/api/v1/notify/templates", templatePayload); + + var delivery = NotifyDelivery.Create( + deliveryId: "delivery-web", + tenantId: "tenant-web", + ruleId: "rule-web", + actionId: "channel-web", + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + status: NotifyDeliveryStatus.Sent, + createdAt: DateTimeOffset.UtcNow, + sentAt: DateTimeOffset.UtcNow); + + var deliveryNode = JsonNode.Parse(NotifyCanonicalJsonSerializer.Serialize(delivery))!; + await PostAsync(client, "/api/v1/notify/deliveries", deliveryNode); + + var deliveriesEnvelope = await GetJsonObjectAsync(client, "/api/v1/notify/deliveries?limit=10", useAdminToken: false); + Assert.NotNull(deliveriesEnvelope); + Assert.Equal(1, deliveriesEnvelope? ["count"]?.GetValue()); + Assert.Null(deliveriesEnvelope? ["continuationToken"]?.GetValue()); + var deliveries = deliveriesEnvelope? ["items"] as JsonArray; + Assert.NotNull(deliveries); + Assert.NotEmpty(deliveries!.OfType()); + + var digestNode = new JsonObject + { + ["tenantId"] = "tenant-web", + ["actionKey"] = "channel-web", + ["window"] = "hourly", + ["openedAt"] = DateTimeOffset.UtcNow.ToString("O"), + ["status"] = "open", + ["items"] = new JsonArray() + }; + await PostAsync(client, "/api/v1/notify/digests", digestNode); + + var digest = await GetJsonObjectAsync(client, "/api/v1/notify/digests/channel-web", useAdminToken: false); + Assert.Equal("channel-web", digest? ["actionKey"]?.GetValue()); + + var auditPayload = JsonNode.Parse(""" + { + "action": "create-rule", + "entityType": "rule", + "entityId": "rule-web", + "payload": {"ruleId": "rule-web"} + } + """)!; + await PostAsync(client, "/api/v1/notify/audit", auditPayload); + + var audits = await GetJsonArrayAsync(client, "/api/v1/notify/audit", useAdminToken: false); + Assert.NotNull(audits); + Assert.Contains(audits!.OfType(), entry => entry?["action"]?.GetValue() == "create-rule"); + + await DeleteAsync(client, "/api/v1/notify/digests/channel-web"); + var digestAfterDelete = await SendAsync(client, HttpMethod.Get, "/api/v1/notify/digests/channel-web", useAdminToken: false); + Assert.Equal(HttpStatusCode.NotFound, digestAfterDelete.StatusCode); + } + + [Fact] + public async Task LockEndpointsAllowAcquireAndRelease() + { + var client = _factory.CreateClient(); + var acquirePayload = JsonNode.Parse(""" + { + "resource": "workers", + "owner": "worker-1", + "ttlSeconds": 30 + } + """)!; + + var acquireResponse = await PostAsync(client, "/api/v1/notify/locks/acquire", acquirePayload); + var acquireContent = JsonNode.Parse(await acquireResponse.Content.ReadAsStringAsync()); + Assert.True(acquireContent? ["acquired"]?.GetValue()); + + await PostAsync(client, "/api/v1/notify/locks/release", JsonNode.Parse(""" + { + "resource": "workers", + "owner": "worker-1" + } + """)!); + + var secondAcquire = await PostAsync(client, "/api/v1/notify/locks/acquire", acquirePayload); + var secondContent = JsonNode.Parse(await secondAcquire.Content.ReadAsStringAsync()); + Assert.True(secondContent? ["acquired"]?.GetValue()); + } + + [Fact] + public async Task ChannelTestSendReturnsPreview() + { + var client = _factory.CreateClient(); + + var channelPayload = LoadSample("notify-channel@1.sample.json"); + channelPayload["channelId"] = "channel-test"; + channelPayload["tenantId"] = "tenant-web"; + channelPayload["config"]! ["target"] = "#ops-alerts"; + await PostAsync(client, "/api/v1/notify/channels", channelPayload); + + var payload = JsonNode.Parse(""" + { + "target": "#ops-alerts", + "title": "Smoke test", + "body": "Sample body" + } + """)!; + + var response = await PostAsync(client, "/api/v1/notify/channels/channel-test/test", payload); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + var json = JsonNode.Parse(await response.Content.ReadAsStringAsync())!.AsObject(); + Assert.Equal("tenant-web", json["tenantId"]?.GetValue()); + Assert.Equal("channel-test", json["channelId"]?.GetValue()); + Assert.NotNull(json["queuedAt"]); + Assert.NotNull(json["traceId"]); + + var preview = json["preview"]?.AsObject(); + Assert.NotNull(preview); + Assert.Equal("#ops-alerts", preview? ["target"]?.GetValue()); + Assert.Equal("Smoke test", preview? ["title"]?.GetValue()); + Assert.Equal("Sample body", preview? ["body"]?.GetValue()); + + var expectedHash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes("Sample body"))).ToLowerInvariant(); + Assert.Equal(expectedHash, preview? ["bodyHash"]?.GetValue()); + + var metadata = json["metadata"] as JsonObject; + Assert.NotNull(metadata); + Assert.Equal("#ops-alerts", metadata?["target"]?.GetValue()); + Assert.Equal("slack", metadata?["channelType"]?.GetValue()); + Assert.Equal("fallback", metadata?["previewProvider"]?.GetValue()); + Assert.Equal(json["traceId"]?.GetValue(), metadata?["traceId"]?.GetValue()); + } + + [Fact] + public async Task ChannelTestSendHonoursRateLimit() + { + using var limitedFactory = _factory.WithWebHostBuilder(builder => + { + builder.UseSetting("notify:api:rateLimits:testSend:tokenLimit", "1"); + builder.UseSetting("notify:api:rateLimits:testSend:tokensPerPeriod", "1"); + builder.UseSetting("notify:api:rateLimits:testSend:queueLimit", "0"); + }); + + var client = limitedFactory.CreateClient(); + + var channelPayload = LoadSample("notify-channel@1.sample.json"); + channelPayload["channelId"] = "channel-rate-limit"; + channelPayload["tenantId"] = "tenant-web"; + channelPayload["config"]! ["target"] = "#ops-alerts"; + await PostAsync(client, "/api/v1/notify/channels", channelPayload); + + var payload = JsonNode.Parse(""" + { + "body": "First" + } + """)!; + + var first = await PostAsync(client, "/api/v1/notify/channels/channel-rate-limit/test", payload); + Assert.Equal(HttpStatusCode.Accepted, first.StatusCode); + + var secondRequest = new HttpRequestMessage(HttpMethod.Post, "/api/v1/notify/channels/channel-rate-limit/test") + { + Content = new StringContent(payload.ToJsonString(), Encoding.UTF8, "application/json") + }; + + var second = await SendAsync(client, secondRequest); + Assert.Equal(HttpStatusCode.TooManyRequests, second.StatusCode); + Assert.NotNull(second.Headers.RetryAfter); + } + + [Fact] + public async Task ChannelTestSendUsesRegisteredProvider() + { + var providerName = typeof(FakeSlackTestProvider).FullName!; + + using var providerFactory = _factory.WithWebHostBuilder(builder => + { + builder.ConfigureServices(services => + { + services.AddSingleton(); + }); + }); + + var client = providerFactory.CreateClient(); + + var channelPayload = LoadSample("notify-channel@1.sample.json"); + channelPayload["channelId"] = "channel-provider"; + channelPayload["tenantId"] = "tenant-web"; + channelPayload["config"]! ["target"] = "#ops-alerts"; + await PostAsync(client, "/api/v1/notify/channels", channelPayload); + + var payload = JsonNode.Parse(""" + { + "target": "#ops-alerts", + "title": "Provider Title", + "summary": "Provider Summary" + } + """)!; + + var response = await PostAsync(client, "/api/v1/notify/channels/channel-provider/test", payload); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + var json = JsonNode.Parse(await response.Content.ReadAsStringAsync())!.AsObject(); + var preview = json["preview"]?.AsObject(); + Assert.NotNull(preview); + Assert.Equal("#ops-alerts", preview?["target"]?.GetValue()); + Assert.Equal("Provider Title", preview?["title"]?.GetValue()); + Assert.Equal("{\"provider\":\"fake\"}", preview?["body"]?.GetValue()); + + var metadata = json["metadata"]?.AsObject(); + Assert.NotNull(metadata); + Assert.Equal(providerName, metadata?["previewProvider"]?.GetValue()); + Assert.Equal("fake-provider", metadata?["provider.name"]?.GetValue()); + } + + private sealed class FakeSlackTestProvider : INotifyChannelTestProvider + { + public NotifyChannelType ChannelType => NotifyChannelType.Slack; + + public Task BuildPreviewAsync(ChannelTestPreviewContext context, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var body = "{\"provider\":\"fake\"}"; + var preview = NotifyDeliveryRendered.Create( + NotifyChannelType.Slack, + NotifyDeliveryFormat.Slack, + context.Target, + context.Request.Title ?? "Provider Title", + body, + context.Request.Summary ?? "Provider Summary", + context.Request.TextBody, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["provider.name"] = "fake-provider" + }; + + return Task.FromResult(new ChannelTestPreviewResult(preview, metadata)); + } + } + + private static JsonNode LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return JsonNode.Parse(File.ReadAllText(path)) ?? throw new InvalidOperationException("Sample JSON null."); + } + + private async Task GetJsonArrayAsync(HttpClient client, string path, bool useAdminToken) + { + var response = await SendAsync(client, HttpMethod.Get, path, useAdminToken); + response.EnsureSuccessStatusCode(); + var content = await response.Content.ReadAsStringAsync(); + return JsonNode.Parse(content) as JsonArray; + } + + private async Task GetJsonObjectAsync(HttpClient client, string path, bool useAdminToken) + { + var response = await SendAsync(client, HttpMethod.Get, path, useAdminToken); + response.EnsureSuccessStatusCode(); + var content = await response.Content.ReadAsStringAsync(); + return JsonNode.Parse(content) as JsonObject; + } + + private async Task PostAsync(HttpClient client, string path, JsonNode payload, bool useAdminToken = true) + { + var request = new HttpRequestMessage(HttpMethod.Post, path) + { + Content = new StringContent(payload.ToJsonString(), Encoding.UTF8, "application/json") + }; + + var response = await SendAsync(client, request, useAdminToken); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(); + throw new InvalidOperationException($"Request to {path} failed with {(int)response.StatusCode} {response.StatusCode}: {body}"); + } + + return response; + } + + private Task PostAsync(HttpClient client, string path, JsonNode payload) + => PostAsync(client, path, payload, useAdminToken: true); + + private async Task DeleteAsync(HttpClient client, string path) + { + var response = await SendAsync(client, HttpMethod.Delete, path); + response.EnsureSuccessStatusCode(); + } + + private Task SendAsync(HttpClient client, HttpMethod method, string path, bool useAdminToken = true) + => SendAsync(client, new HttpRequestMessage(method, path), useAdminToken); + + private Task SendAsync(HttpClient client, HttpRequestMessage request, bool useAdminToken = true) + { + request.Headers.Add("X-StellaOps-Tenant", "tenant-web"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", useAdminToken ? _adminToken : _readToken); + return client.SendAsync(request); + } + + private static string CreateToken(string scope) + { + var handler = new JwtSecurityTokenHandler(); + var key = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(SigningKey)); + var descriptor = new SecurityTokenDescriptor + { + Issuer = Issuer, + Audience = Audience, + Expires = DateTime.UtcNow.AddMinutes(10), + SigningCredentials = new SigningCredentials(key, SecurityAlgorithms.HmacSha256), + Subject = new System.Security.Claims.ClaimsIdentity(new[] + { + new System.Security.Claims.Claim("scope", scope), + new System.Security.Claims.Claim(System.Security.Claims.ClaimTypes.Name, "integration-test") + }) + }; + + var token = handler.CreateToken(descriptor); + return handler.WriteToken(token); + } +} diff --git a/src/StellaOps.Notify.WebService.Tests/NormalizeEndpointsTests.cs b/src/StellaOps.Notify.WebService.Tests/NormalizeEndpointsTests.cs new file mode 100644 index 00000000..ce48466b --- /dev/null +++ b/src/StellaOps.Notify.WebService.Tests/NormalizeEndpointsTests.cs @@ -0,0 +1,86 @@ +using System.Net.Http.Json; +using System.Text.Json.Nodes; +using Microsoft.AspNetCore.Mvc.Testing; + +namespace StellaOps.Notify.WebService.Tests; + +public sealed class NormalizeEndpointsTests : IClassFixture>, IAsyncLifetime +{ + private readonly WebApplicationFactory _factory; + + public NormalizeEndpointsTests(WebApplicationFactory factory) + { + _factory = factory.WithWebHostBuilder(builder => + { + builder.UseSetting("notify:storage:driver", "memory"); + builder.UseSetting("notify:authority:enabled", "false"); + builder.UseSetting("notify:authority:developmentSigningKey", "normalize-tests-signing-key-1234567890"); + builder.UseSetting("notify:authority:issuer", "test-issuer"); + builder.UseSetting("notify:authority:audiences:0", "notify"); + builder.UseSetting("notify:telemetry:enableRequestLogging", "false"); + }); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() => Task.CompletedTask; + + [Fact] + public async Task RuleNormalizeAddsSchemaVersion() + { + var client = _factory.CreateClient(); + var payload = LoadSampleNode("notify-rule@1.sample.json"); + payload!.AsObject().Remove("schemaVersion"); + + var response = await client.PostAsJsonAsync("/internal/notify/rules/normalize", payload); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(); + var normalized = JsonNode.Parse(content); + + Assert.Equal("notify.rule@1", normalized?["schemaVersion"]?.GetValue()); + } + + [Fact] + public async Task ChannelNormalizeAddsSchemaVersion() + { + var client = _factory.CreateClient(); + var payload = LoadSampleNode("notify-channel@1.sample.json"); + payload!.AsObject().Remove("schemaVersion"); + + var response = await client.PostAsJsonAsync("/internal/notify/channels/normalize", payload); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(); + var normalized = JsonNode.Parse(content); + + Assert.Equal("notify.channel@1", normalized?["schemaVersion"]?.GetValue()); + } + + [Fact] + public async Task TemplateNormalizeAddsSchemaVersion() + { + var client = _factory.CreateClient(); + var payload = LoadSampleNode("notify-template@1.sample.json"); + payload!.AsObject().Remove("schemaVersion"); + + var response = await client.PostAsJsonAsync("/internal/notify/templates/normalize", payload); + response.EnsureSuccessStatusCode(); + + var content = await response.Content.ReadAsStringAsync(); + var normalized = JsonNode.Parse(content); + + Assert.Equal("notify.template@1", normalized?["schemaVersion"]?.GetValue()); + } + + private static JsonNode? LoadSampleNode(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to load sample '{fileName}'.", path); + } + + return JsonNode.Parse(File.ReadAllText(path)); + } +} diff --git a/src/StellaOps.Notify.WebService.Tests/StellaOps.Notify.WebService.Tests.csproj b/src/StellaOps.Notify.WebService.Tests/StellaOps.Notify.WebService.Tests.csproj new file mode 100644 index 00000000..0ee02931 --- /dev/null +++ b/src/StellaOps.Notify.WebService.Tests/StellaOps.Notify.WebService.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + + + + + + + + + + Always + + + diff --git a/src/StellaOps.Notify.WebService/AGENTS.md b/src/StellaOps.Notify.WebService/AGENTS.md new file mode 100644 index 00000000..7ff7827b --- /dev/null +++ b/src/StellaOps.Notify.WebService/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.WebService — Agent Charter + +## Mission +Implement Notify control plane per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendRequest.cs b/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendRequest.cs new file mode 100644 index 00000000..6db5467d --- /dev/null +++ b/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendRequest.cs @@ -0,0 +1,56 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Notify.WebService.Contracts; + +/// +/// Payload for Notify channel test send requests. +/// +public sealed record ChannelTestSendRequest +{ + /// + /// Optional override for the default channel destination (email address, webhook URL, Slack channel, etc.). + /// + public string? Target { get; init; } + + /// + /// Optional template identifier to drive future rendering hooks. + /// + public string? TemplateId { get; init; } + + /// + /// Preview title (fallback supplied when omitted). + /// + public string? Title { get; init; } + + /// + /// Optional short summary to show in UI cards. + /// + public string? Summary { get; init; } + + /// + /// Primary body payload rendered for the connector. + /// + public string? Body { get; init; } + + /// + /// Optional text-only representation (used by email/plaintext destinations). + /// + public string? TextBody { get; init; } + + /// + /// Optional locale hint (RFC 5646). + /// + public string? Locale { get; init; } + + /// + /// Optional metadata for future expansion (headers, context labels, etc.). + /// + public IDictionary? Metadata { get; init; } + + /// + /// Optional attachment references emitted with the preview. + /// + [JsonPropertyName("attachments")] + public IList? AttachmentRefs { get; init; } +} diff --git a/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendResponse.cs b/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendResponse.cs new file mode 100644 index 00000000..6c94e12c --- /dev/null +++ b/src/StellaOps.Notify.WebService/Contracts/ChannelTestSendResponse.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.WebService.Contracts; + +/// +/// Response payload summarising a Notify channel test send preview. +/// +public sealed record ChannelTestSendResponse( + string TenantId, + string ChannelId, + NotifyDeliveryRendered Preview, + DateTimeOffset QueuedAt, + string TraceId, + IReadOnlyDictionary Metadata); diff --git a/src/StellaOps.Notify.WebService/Contracts/LockRequests.cs b/src/StellaOps.Notify.WebService/Contracts/LockRequests.cs new file mode 100644 index 00000000..1b6e4de1 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Contracts/LockRequests.cs @@ -0,0 +1,5 @@ +namespace StellaOps.Notify.WebService.Contracts; + +internal sealed record AcquireLockRequest(string Resource, string Owner, int TtlSeconds); + +internal sealed record ReleaseLockRequest(string Resource, string Owner); diff --git a/src/StellaOps.Notify.WebService/Diagnostics/ServiceStatus.cs b/src/StellaOps.Notify.WebService/Diagnostics/ServiceStatus.cs new file mode 100644 index 00000000..29b3ae29 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Diagnostics/ServiceStatus.cs @@ -0,0 +1,47 @@ +using System; + +namespace StellaOps.Notify.WebService.Diagnostics; + +/// +/// Tracks Notify WebService readiness information for `/readyz`. +/// +internal sealed class ServiceStatus +{ + private readonly TimeProvider _timeProvider; + private readonly DateTimeOffset _startedAt; + private ReadySnapshot _readySnapshot; + + public ServiceStatus(TimeProvider timeProvider) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _startedAt = _timeProvider.GetUtcNow(); + _readySnapshot = ReadySnapshot.CreateInitial(_startedAt); + } + + public ServiceSnapshot CreateSnapshot() + { + var now = _timeProvider.GetUtcNow(); + return new ServiceSnapshot(_startedAt, now, _readySnapshot); + } + + public void RecordReadyCheck(bool success, TimeSpan latency, string? errorMessage = null) + { + var timestamp = _timeProvider.GetUtcNow(); + _readySnapshot = new ReadySnapshot(timestamp, latency, success, success ? null : errorMessage); + } + + public readonly record struct ServiceSnapshot( + DateTimeOffset StartedAt, + DateTimeOffset CapturedAt, + ReadySnapshot Ready); + + public readonly record struct ReadySnapshot( + DateTimeOffset CheckedAt, + TimeSpan? Latency, + bool IsReady, + string? Error) + { + public static ReadySnapshot CreateInitial(DateTimeOffset timestamp) + => new(timestamp, null, false, "initialising"); + } +} diff --git a/src/StellaOps.Notify.WebService/Extensions/ConfigurationExtensions.cs b/src/StellaOps.Notify.WebService/Extensions/ConfigurationExtensions.cs new file mode 100644 index 00000000..dbf22e41 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Extensions/ConfigurationExtensions.cs @@ -0,0 +1,37 @@ +using System.IO; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Notify.WebService.Extensions; + +internal static class ConfigurationExtensions +{ + public static IConfigurationBuilder AddNotifyYaml(this IConfigurationBuilder builder, string path) + { + ArgumentNullException.ThrowIfNull(builder); + + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return builder; + } + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + using var reader = File.OpenText(path); + var yamlObject = deserializer.Deserialize(reader); + + if (yamlObject is null) + { + return builder; + } + + var payload = JsonSerializer.Serialize(yamlObject); + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(payload)); + return builder.AddJsonStream(stream); + } +} diff --git a/src/StellaOps.Notify.WebService/Hosting/NotifyPluginHostFactory.cs b/src/StellaOps.Notify.WebService/Hosting/NotifyPluginHostFactory.cs new file mode 100644 index 00000000..ab298935 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Hosting/NotifyPluginHostFactory.cs @@ -0,0 +1,44 @@ +using System; +using System.IO; +using StellaOps.Notify.WebService.Options; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Notify.WebService.Hosting; + +internal static class NotifyPluginHostFactory +{ + public static PluginHostOptions Build(NotifyWebServiceOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(contentRootPath); + + var hostOptions = new PluginHostOptions + { + BaseDirectory = options.Plugins.BaseDirectory ?? Path.Combine(contentRootPath, ".."), + PluginsDirectory = options.Plugins.Directory ?? Path.Combine("plugins", "notify"), + PrimaryPrefix = "StellaOps.Notify" + }; + + if (!Path.IsPathRooted(hostOptions.BaseDirectory)) + { + hostOptions.BaseDirectory = Path.GetFullPath(Path.Combine(contentRootPath, hostOptions.BaseDirectory)); + } + + if (!Path.IsPathRooted(hostOptions.PluginsDirectory)) + { + hostOptions.PluginsDirectory = Path.Combine(hostOptions.BaseDirectory, hostOptions.PluginsDirectory); + } + + foreach (var pattern in options.Plugins.SearchPatterns) + { + hostOptions.SearchPatterns.Add(pattern); + } + + foreach (var prefix in options.Plugins.OrderedPlugins) + { + hostOptions.PluginOrder.Add(prefix); + } + + return hostOptions; + } +} diff --git a/src/StellaOps.Notify.WebService/Internal/JsonHttpResult.cs b/src/StellaOps.Notify.WebService/Internal/JsonHttpResult.cs new file mode 100644 index 00000000..83fb69ff --- /dev/null +++ b/src/StellaOps.Notify.WebService/Internal/JsonHttpResult.cs @@ -0,0 +1,29 @@ +using Microsoft.AspNetCore.Http; + +namespace StellaOps.Notify.WebService.Internal; + +internal sealed class JsonHttpResult : IResult +{ + private readonly string _payload; + private readonly int _statusCode; + private readonly string? _location; + + public JsonHttpResult(string payload, int statusCode, string? location) + { + _payload = payload; + _statusCode = statusCode; + _location = location; + } + + public async Task ExecuteAsync(HttpContext httpContext) + { + httpContext.Response.StatusCode = _statusCode; + httpContext.Response.ContentType = "application/json"; + if (!string.IsNullOrWhiteSpace(_location)) + { + httpContext.Response.Headers.Location = _location; + } + + await httpContext.Response.WriteAsync(_payload); + } +} diff --git a/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptions.cs b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptions.cs new file mode 100644 index 00000000..b6abc08c --- /dev/null +++ b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptions.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; + +namespace StellaOps.Notify.WebService.Options; + +/// +/// Strongly typed configuration for the Notify WebService host. +/// +public sealed class NotifyWebServiceOptions +{ + public const string SectionName = "notify"; + + /// + /// Schema version that downstream consumers can use to detect breaking changes. + /// + public int SchemaVersion { get; set; } = 1; + + /// + /// Authority / authentication configuration. + /// + public AuthorityOptions Authority { get; set; } = new(); + + /// + /// Mongo storage configuration for configuration state and audit logs. + /// + public StorageOptions Storage { get; set; } = new(); + + /// + /// Plug-in loader configuration. + /// + public PluginOptions Plugins { get; set; } = new(); + + /// + /// HTTP API behaviour. + /// + public ApiOptions Api { get; set; } = new(); + + /// + /// Telemetry configuration toggles. + /// + public TelemetryOptions Telemetry { get; set; } = new(); + + public sealed class AuthorityOptions + { + public bool Enabled { get; set; } = true; + + public bool AllowAnonymousFallback { get; set; } + + public string Issuer { get; set; } = "https://authority.local"; + + public string? MetadataAddress { get; set; } + + public bool RequireHttpsMetadata { get; set; } = true; + + public int BackchannelTimeoutSeconds { get; set; } = 30; + + public int TokenClockSkewSeconds { get; set; } = 60; + + public IList Audiences { get; set; } = new List { "notify" }; + + public string ReadScope { get; set; } = "notify.read"; + + public string AdminScope { get; set; } = "notify.admin"; + + /// + /// Optional development signing key for symmetric JWT validation when Authority is disabled. + /// + public string? DevelopmentSigningKey { get; set; } + } + + public sealed class StorageOptions + { + public string Driver { get; set; } = "mongo"; + + public string ConnectionString { get; set; } = string.Empty; + + public string Database { get; set; } = "notify"; + + public int CommandTimeoutSeconds { get; set; } = 30; + } + + public sealed class PluginOptions + { + public string? BaseDirectory { get; set; } + + public string? Directory { get; set; } + + public IList SearchPatterns { get; set; } = new List(); + + public IList OrderedPlugins { get; set; } = new List(); + } + + public sealed class ApiOptions + { + public string BasePath { get; set; } = "/api/v1/notify"; + + public string InternalBasePath { get; set; } = "/internal/notify"; + + public string TenantHeader { get; set; } = "X-StellaOps-Tenant"; + + public RateLimitOptions RateLimits { get; set; } = new(); + } + + public sealed class RateLimitOptions + { + public RateLimitPolicyOptions DeliveryHistory { get; set; } = RateLimitPolicyOptions.CreateDefault( + tokenLimit: 60, + tokensPerPeriod: 30, + replenishmentPeriodSeconds: 60, + queueLimit: 20); + + public RateLimitPolicyOptions TestSend { get; set; } = RateLimitPolicyOptions.CreateDefault( + tokenLimit: 5, + tokensPerPeriod: 5, + replenishmentPeriodSeconds: 60, + queueLimit: 2); + } + + public sealed class RateLimitPolicyOptions + { + public bool Enabled { get; set; } = true; + + public int TokenLimit { get; set; } = 10; + + public int TokensPerPeriod { get; set; } = 10; + + public int ReplenishmentPeriodSeconds { get; set; } = 60; + + public int QueueLimit { get; set; } = 0; + + public static RateLimitPolicyOptions CreateDefault(int tokenLimit, int tokensPerPeriod, int replenishmentPeriodSeconds, int queueLimit) + { + return new RateLimitPolicyOptions + { + TokenLimit = tokenLimit, + TokensPerPeriod = tokensPerPeriod, + ReplenishmentPeriodSeconds = replenishmentPeriodSeconds, + QueueLimit = queueLimit + }; + } + } + + public sealed class TelemetryOptions + { + public bool EnableRequestLogging { get; set; } = true; + + public string MinimumLogLevel { get; set; } = "Information"; + } +} diff --git a/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsPostConfigure.cs b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsPostConfigure.cs new file mode 100644 index 00000000..751da5b8 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsPostConfigure.cs @@ -0,0 +1,47 @@ +using System; +using System.IO; + +namespace StellaOps.Notify.WebService.Options; + +internal static class NotifyWebServiceOptionsPostConfigure +{ + public static void Apply(NotifyWebServiceOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(contentRootPath); + + NormalizePluginOptions(options.Plugins, contentRootPath); + } + + private static void NormalizePluginOptions(NotifyWebServiceOptions.PluginOptions plugins, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(plugins); + + var baseDirectory = plugins.BaseDirectory; + if (string.IsNullOrWhiteSpace(baseDirectory)) + { + baseDirectory = Path.Combine(contentRootPath, ".."); + } + else if (!Path.IsPathRooted(baseDirectory)) + { + baseDirectory = Path.GetFullPath(Path.Combine(contentRootPath, baseDirectory)); + } + + plugins.BaseDirectory = baseDirectory; + + if (string.IsNullOrWhiteSpace(plugins.Directory)) + { + plugins.Directory = Path.Combine("plugins", "notify"); + } + + if (!Path.IsPathRooted(plugins.Directory)) + { + plugins.Directory = Path.Combine(baseDirectory, plugins.Directory); + } + + if (plugins.SearchPatterns.Count == 0) + { + plugins.SearchPatterns.Add("StellaOps.Notify.Connectors.*.dll"); + } + } +} diff --git a/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsValidator.cs b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsValidator.cs new file mode 100644 index 00000000..ec87c3df --- /dev/null +++ b/src/StellaOps.Notify.WebService/Options/NotifyWebServiceOptionsValidator.cs @@ -0,0 +1,136 @@ +using System; +using System.Linq; + +namespace StellaOps.Notify.WebService.Options; + +internal static class NotifyWebServiceOptionsValidator +{ + public static void Validate(NotifyWebServiceOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + ValidateStorage(options.Storage); + ValidateAuthority(options.Authority); + ValidateApi(options.Api); + } + + private static void ValidateStorage(NotifyWebServiceOptions.StorageOptions storage) + { + ArgumentNullException.ThrowIfNull(storage); + + var driver = storage.Driver ?? string.Empty; + if (!string.Equals(driver, "mongo", StringComparison.OrdinalIgnoreCase) && + !string.Equals(driver, "memory", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"Unsupported storage driver '{storage.Driver}'."); + } + + if (string.Equals(driver, "mongo", StringComparison.OrdinalIgnoreCase)) + { + if (string.IsNullOrWhiteSpace(storage.ConnectionString)) + { + throw new InvalidOperationException("notify:storage:connectionString must be provided."); + } + + if (string.IsNullOrWhiteSpace(storage.Database)) + { + throw new InvalidOperationException("notify:storage:database must be provided."); + } + + if (storage.CommandTimeoutSeconds <= 0) + { + throw new InvalidOperationException("notify:storage:commandTimeoutSeconds must be positive."); + } + } + } + + private static void ValidateAuthority(NotifyWebServiceOptions.AuthorityOptions authority) + { + ArgumentNullException.ThrowIfNull(authority); + + if (authority.Enabled) + { + if (string.IsNullOrWhiteSpace(authority.Issuer)) + { + throw new InvalidOperationException("notify:authority:issuer must be provided when authority is enabled."); + } + + if (authority.Audiences is null || authority.Audiences.Count == 0) + { + throw new InvalidOperationException("notify:authority:audiences must include at least one value."); + } + + if (string.IsNullOrWhiteSpace(authority.AdminScope) || string.IsNullOrWhiteSpace(authority.ReadScope)) + { + throw new InvalidOperationException("notify:authority admin and read scopes must be configured."); + } + } + else + { + if (string.IsNullOrWhiteSpace(authority.DevelopmentSigningKey) || authority.DevelopmentSigningKey.Length < 32) + { + throw new InvalidOperationException("notify:authority:developmentSigningKey must be at least 32 characters when authority is disabled."); + } + } + } + + private static void ValidateApi(NotifyWebServiceOptions.ApiOptions api) + { + ArgumentNullException.ThrowIfNull(api); + + if (!api.BasePath.StartsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("notify:api:basePath must start with '/'."); + } + + if (!api.InternalBasePath.StartsWith("/", StringComparison.Ordinal)) + { + throw new InvalidOperationException("notify:api:internalBasePath must start with '/'."); + } + + if (string.IsNullOrWhiteSpace(api.TenantHeader)) + { + throw new InvalidOperationException("notify:api:tenantHeader must be provided."); + } + + ValidateRateLimits(api.RateLimits); + } + + private static void ValidateRateLimits(NotifyWebServiceOptions.RateLimitOptions rateLimits) + { + ArgumentNullException.ThrowIfNull(rateLimits); + + ValidatePolicy(rateLimits.DeliveryHistory, "notify:api:rateLimits:deliveryHistory"); + ValidatePolicy(rateLimits.TestSend, "notify:api:rateLimits:testSend"); + + static void ValidatePolicy(NotifyWebServiceOptions.RateLimitPolicyOptions options, string prefix) + { + ArgumentNullException.ThrowIfNull(options); + + if (!options.Enabled) + { + return; + } + + if (options.TokenLimit <= 0) + { + throw new InvalidOperationException($"{prefix}:tokenLimit must be positive when enabled."); + } + + if (options.TokensPerPeriod <= 0) + { + throw new InvalidOperationException($"{prefix}:tokensPerPeriod must be positive when enabled."); + } + + if (options.ReplenishmentPeriodSeconds <= 0) + { + throw new InvalidOperationException($"{prefix}:replenishmentPeriodSeconds must be positive when enabled."); + } + + if (options.QueueLimit < 0) + { + throw new InvalidOperationException($"{prefix}:queueLimit cannot be negative."); + } + } + } +} diff --git a/src/StellaOps.Notify.WebService/Plugins/NotifyPluginRegistry.cs b/src/StellaOps.Notify.WebService/Plugins/NotifyPluginRegistry.cs new file mode 100644 index 00000000..0f3f443b --- /dev/null +++ b/src/StellaOps.Notify.WebService/Plugins/NotifyPluginRegistry.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin.Hosting; + +namespace StellaOps.Notify.WebService.Plugins; + +internal interface INotifyPluginRegistry +{ + Task WarmupAsync(CancellationToken cancellationToken = default); +} + +internal sealed class NotifyPluginRegistry : INotifyPluginRegistry +{ + private readonly PluginHostOptions _hostOptions; + private readonly ILogger _logger; + + public NotifyPluginRegistry( + PluginHostOptions hostOptions, + ILogger logger) + { + _hostOptions = hostOptions ?? throw new ArgumentNullException(nameof(hostOptions)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task WarmupAsync(CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var result = PluginHost.LoadPlugins(_hostOptions, _logger); + + if (result.Plugins.Count == 0) + { + _logger.LogWarning( + "No Notify plug-ins discovered under '{PluginDirectory}'.", + result.PluginDirectory); + } + else + { + _logger.LogInformation( + "Loaded {PluginCount} Notify plug-in(s) from '{PluginDirectory}'.", + result.Plugins.Count, + result.PluginDirectory); + } + + if (result.MissingOrderedPlugins.Count > 0) + { + _logger.LogWarning( + "Configured plug-ins missing from disk: {Missing}.", + string.Join(", ", result.MissingOrderedPlugins)); + } + + return Task.FromResult(result.Plugins.Count); + } +} diff --git a/src/StellaOps.Notify.WebService/Program.Partial.cs b/src/StellaOps.Notify.WebService/Program.Partial.cs new file mode 100644 index 00000000..c308f9cb --- /dev/null +++ b/src/StellaOps.Notify.WebService/Program.Partial.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Notify.WebService; + +public partial class Program; diff --git a/src/StellaOps.Notify.WebService/Program.cs b/src/StellaOps.Notify.WebService/Program.cs new file mode 100644 index 00000000..64d1dee5 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Program.cs @@ -0,0 +1,831 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Claims; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.RateLimiting; +using Microsoft.AspNetCore.Authentication.JwtBearer; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.RateLimiting; +using Microsoft.Extensions.Options; +using Microsoft.IdentityModel.Tokens; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using Serilog; +using Serilog.Events; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Configuration; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Repositories; +using StellaOps.Notify.WebService.Diagnostics; +using StellaOps.Notify.WebService.Extensions; +using StellaOps.Notify.WebService.Hosting; +using StellaOps.Notify.WebService.Options; +using StellaOps.Notify.WebService.Plugins; +using StellaOps.Notify.WebService.Security; +using StellaOps.Notify.WebService.Services; +using StellaOps.Notify.WebService.Internal; +using StellaOps.Notify.WebService.Storage.InMemory; +using StellaOps.Plugin.DependencyInjection; +using MongoDB.Bson; +using StellaOps.Notify.WebService.Contracts; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "NOTIFY_"; + options.ConfigureBuilder = configurationBuilder => + { + configurationBuilder.AddNotifyYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/notify.yaml")); + }; +}); + +var contentRootPath = builder.Environment.ContentRootPath; + +var bootstrapOptions = builder.Configuration.BindOptions( + NotifyWebServiceOptions.SectionName, + (opts, _) => + { + NotifyWebServiceOptionsPostConfigure.Apply(opts, contentRootPath); + NotifyWebServiceOptionsValidator.Validate(opts); + }); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(NotifyWebServiceOptions.SectionName)) + .PostConfigure(options => + { + NotifyWebServiceOptionsPostConfigure.Apply(options, contentRootPath); + NotifyWebServiceOptionsValidator.Validate(options); + }) + .ValidateOnStart(); + +builder.Host.UseSerilog((context, services, loggerConfiguration) => +{ + var minimumLevel = MapLogLevel(bootstrapOptions.Telemetry.MinimumLogLevel); + + loggerConfiguration + .MinimumLevel.Is(minimumLevel) + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) + .Enrich.FromLogContext() + .WriteTo.Console(); +}); + +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +if (string.Equals(bootstrapOptions.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase)) +{ + builder.Services.AddNotifyMongoStorage(builder.Configuration.GetSection("notify:storage")); +} +else +{ + builder.Services.AddInMemoryNotifyStorage(); +} + +var pluginHostOptions = NotifyPluginHostFactory.Build(bootstrapOptions, contentRootPath); +builder.Services.AddSingleton(pluginHostOptions); +builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +ConfigureAuthentication(builder, bootstrapOptions); +ConfigureRateLimiting(builder, bootstrapOptions); + +builder.Services.AddEndpointsApiExplorer(); + +var app = builder.Build(); + +var readyStatus = app.Services.GetRequiredService(); + +var resolvedOptions = app.Services.GetRequiredService>().Value; +await InitialiseAsync(app.Services, readyStatus, app.Logger, resolvedOptions); + +ConfigureRequestPipeline(app, bootstrapOptions); +ConfigureEndpoints(app); + +await app.RunAsync(); + +static void ConfigureAuthentication(WebApplicationBuilder builder, NotifyWebServiceOptions options) +{ + if (options.Authority.Enabled) + { + builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = options.Authority.Issuer; + resourceOptions.RequireHttpsMetadata = options.Authority.RequireHttpsMetadata; + resourceOptions.MetadataAddress = options.Authority.MetadataAddress; + resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(options.Authority.BackchannelTimeoutSeconds); + resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(options.Authority.TokenClockSkewSeconds); + + resourceOptions.Audiences.Clear(); + foreach (var audience in options.Authority.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + }); + + builder.Services.AddAuthorization(auth => + { + auth.AddStellaOpsScopePolicy(NotifyPolicies.Read, options.Authority.ReadScope); + auth.AddStellaOpsScopePolicy(NotifyPolicies.Admin, options.Authority.AdminScope); + }); + } + else + { + builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme) + .AddJwtBearer(jwt => + { + jwt.RequireHttpsMetadata = false; + jwt.TokenValidationParameters = new TokenValidationParameters + { + ValidateIssuer = true, + ValidIssuer = options.Authority.Issuer, + ValidateAudience = options.Authority.Audiences.Count > 0, + ValidAudiences = options.Authority.Audiences, + ValidateIssuerSigningKey = true, + IssuerSigningKey = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(options.Authority.DevelopmentSigningKey!)), + ValidateLifetime = true, + ClockSkew = TimeSpan.FromSeconds(options.Authority.TokenClockSkewSeconds), + NameClaimType = ClaimTypes.Name + }; + }); + + builder.Services.AddAuthorization(auth => + { + auth.AddPolicy( + NotifyPolicies.Read, + policy => policy + .RequireAuthenticatedUser() + .RequireAssertion(ctx => + HasScope(ctx.User, options.Authority.ReadScope) || + HasScope(ctx.User, options.Authority.AdminScope))); + + auth.AddPolicy( + NotifyPolicies.Admin, + policy => policy + .RequireAuthenticatedUser() + .RequireAssertion(ctx => HasScope(ctx.User, options.Authority.AdminScope))); + }); + } +} + +static void ConfigureRateLimiting(WebApplicationBuilder builder, NotifyWebServiceOptions options) +{ + ArgumentNullException.ThrowIfNull(options); + var tenantHeader = options.Api.TenantHeader; + var limits = options.Api.RateLimits; + + builder.Services.AddRateLimiter(rateLimiterOptions => + { + rateLimiterOptions.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + rateLimiterOptions.OnRejected = static (context, _) => + { + context.HttpContext.Response.Headers.TryAdd("Retry-After", "1"); + return ValueTask.CompletedTask; + }; + + ConfigurePolicy(rateLimiterOptions, NotifyRateLimitPolicies.DeliveryHistory, limits.DeliveryHistory, tenantHeader, "deliveries"); + ConfigurePolicy(rateLimiterOptions, NotifyRateLimitPolicies.TestSend, limits.TestSend, tenantHeader, "channel-test"); + }); + + static void ConfigurePolicy( + RateLimiterOptions rateLimiterOptions, + string policyName, + NotifyWebServiceOptions.RateLimitPolicyOptions policy, + string tenantHeader, + string prefix) + { + rateLimiterOptions.AddPolicy(policyName, httpContext => + { + if (policy is null || !policy.Enabled) + { + return RateLimitPartition.GetNoLimiter("notify-disabled"); + } + + var identity = ResolveIdentity(httpContext, tenantHeader, prefix); + + return RateLimitPartition.GetTokenBucketLimiter(identity, _ => new TokenBucketRateLimiterOptions + { + TokenLimit = policy.TokenLimit, + TokensPerPeriod = policy.TokensPerPeriod, + ReplenishmentPeriod = TimeSpan.FromSeconds(policy.ReplenishmentPeriodSeconds), + QueueLimit = policy.QueueLimit, + QueueProcessingOrder = QueueProcessingOrder.OldestFirst, + AutoReplenishment = true + }); + }); + } + + static string ResolveIdentity(HttpContext httpContext, string tenantHeader, string prefix) + { + var tenant = httpContext.Request.Headers.TryGetValue(tenantHeader, out var header) && !StringValues.IsNullOrEmpty(header) + ? header.ToString().Trim() + : "anonymous"; + + var subject = httpContext.User.FindFirst("sub")?.Value + ?? httpContext.User.Identity?.Name + ?? httpContext.Connection.RemoteIpAddress?.ToString() + ?? "anonymous"; + + return string.Concat(prefix, ':', tenant, ':', subject); + } +} + +static async Task InitialiseAsync(IServiceProvider services, ServiceStatus status, Microsoft.Extensions.Logging.ILogger logger, NotifyWebServiceOptions options) +{ + var stopwatch = Stopwatch.StartNew(); + + try + { + await using var scope = services.CreateAsyncScope(); + if (string.Equals(options.Storage.Driver, "mongo", StringComparison.OrdinalIgnoreCase)) + { + await RunMongoMigrationsAsync(scope.ServiceProvider); + } + + var registry = scope.ServiceProvider.GetRequiredService(); + var count = await registry.WarmupAsync(); + + stopwatch.Stop(); + status.RecordReadyCheck(success: true, stopwatch.Elapsed); + logger.LogInformation("Notify WebService initialised in {ElapsedMs} ms; loaded {PluginCount} plug-in(s).", stopwatch.Elapsed.TotalMilliseconds, count); + } + catch (Exception ex) + { + stopwatch.Stop(); + status.RecordReadyCheck(success: false, stopwatch.Elapsed, ex.Message); + logger.LogError(ex, "Failed to initialise Notify WebService."); + throw; + } +} + +static async Task RunMongoMigrationsAsync(IServiceProvider services) +{ + var initializerType = Type.GetType("StellaOps.Notify.Storage.Mongo.Internal.NotifyMongoInitializer, StellaOps.Notify.Storage.Mongo"); + if (initializerType is null) + { + return; + } + + var initializer = services.GetService(initializerType); + if (initializer is null) + { + return; + } + + var method = initializerType.GetMethod("EnsureIndexesAsync", new[] { typeof(CancellationToken) }); + if (method is null) + { + return; + } + + if (method.Invoke(initializer, new object[] { CancellationToken.None }) is Task task) + { + await task.ConfigureAwait(false); + } +} + +static void ConfigureRequestPipeline(WebApplication app, NotifyWebServiceOptions options) +{ + if (options.Telemetry.EnableRequestLogging) + { + app.UseSerilogRequestLogging(c => + { + c.IncludeQueryInRequestPath = true; + c.GetLevel = (_, _, exception) => exception is null ? LogEventLevel.Information : LogEventLevel.Error; + }); + } + + app.UseAuthentication(); + app.UseRateLimiter(); + app.UseAuthorization(); +} + +static void ConfigureEndpoints(WebApplication app) +{ + app.MapGet("/healthz", () => Results.Ok(new { status = "ok" })); + + app.MapGet("/readyz", (ServiceStatus status) => + { + var snapshot = status.CreateSnapshot(); + if (snapshot.Ready.IsReady) + { + return Results.Ok(new + { + status = "ready", + checkedAt = snapshot.Ready.CheckedAt, + latencyMs = snapshot.Ready.Latency?.TotalMilliseconds, + snapshot.StartedAt + }); + } + + return JsonResponse( + new + { + status = "unready", + snapshot.Ready.Error, + checkedAt = snapshot.Ready.CheckedAt, + latencyMs = snapshot.Ready.Latency?.TotalMilliseconds + }, + StatusCodes.Status503ServiceUnavailable); + }); + + var options = app.Services.GetRequiredService>().Value; + var tenantHeader = options.Api.TenantHeader; + var apiBasePath = options.Api.BasePath.TrimEnd('/'); + var apiGroup = app.MapGroup(options.Api.BasePath); + var internalGroup = app.MapGroup(options.Api.InternalBasePath); + + internalGroup.MapPost("/rules/normalize", (JsonNode? body, NotifySchemaMigrationService service) => Normalize(body, service.UpgradeRule)) + .WithName("notify.rules.normalize") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest); + + internalGroup.MapPost("/channels/normalize", (JsonNode? body, NotifySchemaMigrationService service) => Normalize(body, service.UpgradeChannel)) + .WithName("notify.channels.normalize"); + + internalGroup.MapPost("/templates/normalize", (JsonNode? body, NotifySchemaMigrationService service) => Normalize(body, service.UpgradeTemplate)) + .WithName("notify.templates.normalize"); + + apiGroup.MapGet("/rules", async ([FromServices] INotifyRuleRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var rules = await repository.ListAsync(tenant, cancellationToken); + return JsonResponse(rules); + }) + .RequireAuthorization(NotifyPolicies.Read); + + apiGroup.MapGet("/rules/{ruleId}", async (string ruleId, [FromServices] INotifyRuleRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var rule = await repository.GetAsync(tenant, ruleId, cancellationToken); + return rule is null ? Results.NotFound() : JsonResponse(rule); + }) + .RequireAuthorization(NotifyPolicies.Read); + + apiGroup.MapPost("/rules", async (JsonNode? body, NotifySchemaMigrationService service, INotifyRuleRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var rule = service.UpgradeRule(body); + if (!string.Equals(rule.TenantId, tenant, StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Tenant mismatch between header and payload." }); + } + + await repository.UpsertAsync(rule, cancellationToken); + + return CreatedJson(BuildResourceLocation(apiBasePath, "rules", rule.RuleId), rule); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapDelete("/rules/{ruleId}", async (string ruleId, INotifyRuleRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + await repository.DeleteAsync(tenant, ruleId, cancellationToken); + return Results.NoContent(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapGet("/channels", async (INotifyChannelRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var channels = await repository.ListAsync(tenant, cancellationToken); + return JsonResponse(channels); + }) + .RequireAuthorization(NotifyPolicies.Read); + + apiGroup.MapPost("/channels", async (JsonNode? body, NotifySchemaMigrationService service, INotifyChannelRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var channel = service.UpgradeChannel(body); + if (!string.Equals(channel.TenantId, tenant, StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Tenant mismatch between header and payload." }); + } + + await repository.UpsertAsync(channel, cancellationToken); + return CreatedJson(BuildResourceLocation(apiBasePath, "channels", channel.ChannelId), channel); +}) +.RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapPost("/channels/{channelId}/test", async (string channelId, [FromBody] ChannelTestSendRequest? request, INotifyChannelRepository repository, INotifyChannelTestService testService, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (request is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var channel = await repository.GetAsync(tenant, channelId, cancellationToken); + if (channel is null) + { + return Results.NotFound(); + } + + try + { + var response = await testService.SendAsync(tenant, channel, request, context.TraceIdentifier, cancellationToken).ConfigureAwait(false); + return JsonResponse(response, StatusCodes.Status202Accepted); + } + catch (ChannelTestSendValidationException ex) + { + return Results.BadRequest(new { error = ex.Message }); + } + }) + .RequireAuthorization(NotifyPolicies.Admin) + .RequireRateLimiting(NotifyRateLimitPolicies.TestSend); + + apiGroup.MapDelete("/channels/{channelId}", async (string channelId, INotifyChannelRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + await repository.DeleteAsync(tenant, channelId, cancellationToken); + return Results.NoContent(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapGet("/templates", async (INotifyTemplateRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var templates = await repository.ListAsync(tenant, cancellationToken); + return JsonResponse(templates); + }) + .RequireAuthorization(NotifyPolicies.Read); + + apiGroup.MapPost("/templates", async (JsonNode? body, NotifySchemaMigrationService service, INotifyTemplateRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var template = service.UpgradeTemplate(body); + if (!string.Equals(template.TenantId, tenant, StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Tenant mismatch between header and payload." }); + } + + await repository.UpsertAsync(template, cancellationToken); + return CreatedJson(BuildResourceLocation(apiBasePath, "templates", template.TemplateId), template); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapDelete("/templates/{templateId}", async (string templateId, INotifyTemplateRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + await repository.DeleteAsync(tenant, templateId, cancellationToken); + return Results.NoContent(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapPost("/deliveries", async (JsonNode? body, INotifyDeliveryRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var delivery = NotifyCanonicalJsonSerializer.Deserialize(body.ToJsonString()); + if (!string.Equals(delivery.TenantId, tenant, StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Tenant mismatch between header and payload." }); + } + + await repository.UpdateAsync(delivery, cancellationToken); + return CreatedJson(BuildResourceLocation(apiBasePath, "deliveries", delivery.DeliveryId), delivery); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapGet("/deliveries", async ([FromServices] INotifyDeliveryRepository repository, HttpContext context, [FromQuery] DateTimeOffset? since, [FromQuery] string? status, [FromQuery] int? limit, [FromQuery] string? continuationToken, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var effectiveLimit = NormalizeLimit(limit); + var result = await repository.QueryAsync(tenant, since, status, effectiveLimit, continuationToken, cancellationToken).ConfigureAwait(false); + var payload = new + { + items = result.Items, + continuationToken = result.ContinuationToken, + count = result.Items.Count + }; + + return JsonResponse(payload); + }) + .RequireAuthorization(NotifyPolicies.Read) + .RequireRateLimiting(NotifyRateLimitPolicies.DeliveryHistory); + + apiGroup.MapGet("/deliveries/{deliveryId}", async (string deliveryId, INotifyDeliveryRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var delivery = await repository.GetAsync(tenant, deliveryId, cancellationToken); + return delivery is null ? Results.NotFound() : JsonResponse(delivery); + }) + .RequireAuthorization(NotifyPolicies.Read) + .RequireRateLimiting(NotifyRateLimitPolicies.DeliveryHistory); + + apiGroup.MapPost("/digests", async ([FromBody] NotifyDigestDocument payload, INotifyDigestRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (!string.Equals(payload.TenantId, tenant, StringComparison.Ordinal)) + { + return Results.BadRequest(new { error = "Tenant mismatch between header and payload." }); + } + + await repository.UpsertAsync(payload, cancellationToken); + return Results.Ok(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapGet("/digests/{actionKey}", async (string actionKey, INotifyDigestRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var digest = await repository.GetAsync(tenant, actionKey, cancellationToken); + return digest is null ? Results.NotFound() : JsonResponse(digest); + }) + .RequireAuthorization(NotifyPolicies.Read); + + apiGroup.MapDelete("/digests/{actionKey}", async (string actionKey, INotifyDigestRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + await repository.RemoveAsync(tenant, actionKey, cancellationToken); + return Results.NoContent(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapPost("/locks/acquire", async ([FromBody] AcquireLockRequest request, INotifyLockRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var acquired = await repository.TryAcquireAsync(tenant, request.Resource, request.Owner, TimeSpan.FromSeconds(request.TtlSeconds), cancellationToken); + return JsonResponse(new { acquired }); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapPost("/locks/release", async ([FromBody] ReleaseLockRequest request, INotifyLockRepository repository, HttpContext context, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + await repository.ReleaseAsync(tenant, request.Resource, request.Owner, cancellationToken); + return Results.NoContent(); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapPost("/audit", async ([FromBody] JsonNode? body, INotifyAuditRepository repository, HttpContext context, ClaimsPrincipal user, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + var action = body["action"]?.GetValue(); + if (string.IsNullOrWhiteSpace(action)) + { + return Results.BadRequest(new { error = "Action is required." }); + } + + var entry = new NotifyAuditEntryDocument + { + Id = ObjectId.GenerateNewId(), + TenantId = tenant, + Action = action, + Actor = user.Identity?.Name ?? "unknown", + EntityId = body["entityId"]?.GetValue() ?? string.Empty, + EntityType = body["entityType"]?.GetValue() ?? string.Empty, + Timestamp = DateTimeOffset.UtcNow, + Payload = body["payload"] is JsonObject payloadObj + ? BsonDocument.Parse(payloadObj.ToJsonString()) + : new BsonDocument() + }; + + await repository.AppendAsync(entry, cancellationToken); + return CreatedJson(BuildResourceLocation(apiBasePath, "audit", entry.Id.ToString()), new { entry.Id }); + }) + .RequireAuthorization(NotifyPolicies.Admin); + + apiGroup.MapGet("/audit", async (INotifyAuditRepository repository, HttpContext context, [FromQuery] DateTimeOffset? since, [FromQuery] int? limit, CancellationToken cancellationToken) => + { + if (!TryResolveTenant(context, tenantHeader, out var tenant, out var error)) + { + return error!; + } + + var entries = await repository.QueryAsync(tenant, since, limit, cancellationToken); + var response = entries.Select(e => new + { + e.Id, + e.TenantId, + e.Actor, + e.Action, + e.EntityId, + e.EntityType, + e.Timestamp, + Payload = JsonNode.Parse(e.Payload.ToJson()) + }); + + return JsonResponse(response); + }) + .RequireAuthorization(NotifyPolicies.Read); +} + +static int NormalizeLimit(int? value) +{ + if (value is null || value <= 0) + { + return 50; + } + + return Math.Min(value.Value, 200); +} + +static bool TryResolveTenant(HttpContext context, string tenantHeader, out string tenant, out IResult? error) +{ + if (!context.Request.Headers.TryGetValue(tenantHeader, out var header) || string.IsNullOrWhiteSpace(header)) + { + tenant = string.Empty; + error = Results.BadRequest(new { error = $"{tenantHeader} header is required." }); + return false; + } + + tenant = header.ToString().Trim(); + error = null; + return true; +} + +static string BuildResourceLocation(string basePath, params string[] segments) +{ + if (segments.Length == 0) + { + return basePath; + } + + var builder = new StringBuilder(basePath); + foreach (var segment in segments) + { + builder.Append('/'); + builder.Append(Uri.EscapeDataString(segment)); + } + + return builder.ToString(); +} + +static IResult JsonResponse(T value, int statusCode = StatusCodes.Status200OK, string? location = null) +{ + var payload = JsonSerializer.Serialize(value, new JsonSerializerOptions(JsonSerializerDefaults.Web)); + return new JsonHttpResult(payload, statusCode, location); +} + +static IResult CreatedJson(string location, T value) + => JsonResponse(value, StatusCodes.Status201Created, location); + +static IResult Normalize(JsonNode? body, Func upgrade) +{ + if (body is null) + { + return Results.BadRequest(new { error = "Request body is required." }); + } + + try + { + var model = upgrade(body); + var json = NotifyCanonicalJsonSerializer.Serialize(model); + return Results.Content(json, "application/json"); + } + catch (Exception ex) + { + return Results.BadRequest(new { error = ex.Message }); + } +} + +static bool HasScope(ClaimsPrincipal principal, string scope) +{ + if (principal is null || string.IsNullOrWhiteSpace(scope)) + { + return false; + } + + foreach (var claim in principal.FindAll("scope")) + { + if (string.Equals(claim.Value, scope, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; +} + +static LogEventLevel MapLogLevel(string configuredLevel) +{ + return configuredLevel?.ToLowerInvariant() switch + { + "verbose" => LogEventLevel.Verbose, + "debug" => LogEventLevel.Debug, + "warning" => LogEventLevel.Warning, + "error" => LogEventLevel.Error, + "fatal" => LogEventLevel.Fatal, + _ => LogEventLevel.Information + }; +} diff --git a/src/StellaOps.Notify.WebService/Security/NotifyPolicies.cs b/src/StellaOps.Notify.WebService/Security/NotifyPolicies.cs new file mode 100644 index 00000000..b6803793 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Security/NotifyPolicies.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Notify.WebService.Security; + +internal static class NotifyPolicies +{ + public const string Read = "notify.read"; + public const string Admin = "notify.admin"; +} diff --git a/src/StellaOps.Notify.WebService/Security/NotifyRateLimitPolicies.cs b/src/StellaOps.Notify.WebService/Security/NotifyRateLimitPolicies.cs new file mode 100644 index 00000000..de5b57e6 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Security/NotifyRateLimitPolicies.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Notify.WebService.Security; + +internal static class NotifyRateLimitPolicies +{ + public const string DeliveryHistory = "notify-deliveries"; + + public const string TestSend = "notify-test-send"; +} diff --git a/src/StellaOps.Notify.WebService/Services/NotifyChannelTestService.cs b/src/StellaOps.Notify.WebService/Services/NotifyChannelTestService.cs new file mode 100644 index 00000000..9c0180bb --- /dev/null +++ b/src/StellaOps.Notify.WebService/Services/NotifyChannelTestService.cs @@ -0,0 +1,313 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Extensions.Logging; +using StellaOps.Notify.Engine; +using StellaOps.Notify.Models; +using StellaOps.Notify.WebService.Contracts; + +namespace StellaOps.Notify.WebService.Services; + +internal interface INotifyChannelTestService +{ + Task SendAsync( + string tenantId, + NotifyChannel channel, + ChannelTestSendRequest request, + string traceId, + CancellationToken cancellationToken); +} + +internal sealed class NotifyChannelTestService : INotifyChannelTestService +{ + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly IReadOnlyDictionary _providers; + + public NotifyChannelTestService( + TimeProvider timeProvider, + ILogger logger, + IEnumerable providers) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _providers = BuildProviderMap(providers ?? Array.Empty(), _logger); + } + + public async Task SendAsync( + string tenantId, + NotifyChannel channel, + ChannelTestSendRequest request, + string traceId, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(channel); + ArgumentNullException.ThrowIfNull(request); + + cancellationToken.ThrowIfCancellationRequested(); + + if (!channel.Enabled) + { + throw new ChannelTestSendValidationException("Channel is disabled. Enable it before issuing test sends."); + } + + var target = ResolveTarget(channel, request.Target); + var timestamp = _timeProvider.GetUtcNow(); + var previewRequest = BuildPreviewRequest(request); + var context = new ChannelTestPreviewContext( + tenantId, + channel, + target, + previewRequest, + timestamp, + traceId); + + ChannelTestPreviewResult? providerResult = null; + var providerName = "fallback"; + + if (_providers.TryGetValue(channel.Type, out var provider)) + { + try + { + providerResult = await provider.BuildPreviewAsync(context, cancellationToken).ConfigureAwait(false); + providerName = provider.GetType().FullName ?? provider.GetType().Name; + } + catch (ChannelTestPreviewException ex) + { + throw new ChannelTestSendValidationException(ex.Message); + } + } + + var rendered = providerResult is not null + ? EnsureBodyHash(providerResult.Preview) + : CreateFallbackPreview(context); + + var metadata = MergeMetadata( + context, + providerName, + providerResult?.Metadata); + + var response = new ChannelTestSendResponse( + tenantId, + channel.ChannelId, + rendered, + timestamp, + traceId, + metadata); + + _logger.LogInformation( + "Notify test send preview generated for tenant {TenantId}, channel {ChannelId} ({ChannelType}) using provider {Provider}.", + tenantId, + channel.ChannelId, + channel.Type, + providerName); + + return response; + } + + private static IReadOnlyDictionary BuildProviderMap( + IEnumerable providers, + ILogger logger) + { + var map = new Dictionary(); + foreach (var provider in providers) + { + if (provider is null) + { + continue; + } + + if (map.TryGetValue(provider.ChannelType, out var existing)) + { + logger?.LogWarning( + "Multiple Notify channel test providers registered for {ChannelType}. Keeping {ExistingProvider} and ignoring {NewProvider}.", + provider.ChannelType, + existing.GetType().FullName, + provider.GetType().FullName); + continue; + } + + map[provider.ChannelType] = provider; + } + + return map; + } + + private static ChannelTestPreviewRequest BuildPreviewRequest(ChannelTestSendRequest request) + { + return new ChannelTestPreviewRequest( + TrimToNull(request.Target), + TrimToNull(request.TemplateId), + TrimToNull(request.Title), + TrimToNull(request.Summary), + request.Body, + TrimToNull(request.TextBody), + TrimToLowerInvariant(request.Locale), + NormalizeInputMetadata(request.Metadata), + NormalizeAttachments(request.AttachmentRefs)); + } + + private static string ResolveTarget(NotifyChannel channel, string? overrideTarget) + { + var target = string.IsNullOrWhiteSpace(overrideTarget) + ? channel.Config.Target ?? channel.Config.Endpoint + : overrideTarget.Trim(); + + if (string.IsNullOrWhiteSpace(target)) + { + throw new ChannelTestSendValidationException("Channel target is required. Provide 'target' or configure channel.config.target/endpoint."); + } + + return target; + } + + private static NotifyDeliveryRendered CreateFallbackPreview(ChannelTestPreviewContext context) + { + var format = MapFormat(context.Channel.Type); + var title = context.Request.Title ?? $"Stella Ops Notify Test ({context.Channel.Name})"; + var body = context.Request.Body ?? BuildDefaultBody(context.Channel, context.Timestamp); + var summary = context.Request.Summary ?? $"Preview generated for {context.Channel.Type} destination."; + + return NotifyDeliveryRendered.Create( + context.Channel.Type, + format, + context.Target, + title, + body, + summary, + context.Request.TextBody, + context.Request.Locale, + ChannelTestPreviewUtilities.ComputeBodyHash(body), + context.Request.Attachments); + } + + private static NotifyDeliveryRendered EnsureBodyHash(NotifyDeliveryRendered preview) + { + if (!string.IsNullOrWhiteSpace(preview.BodyHash)) + { + return preview; + } + + var hash = ChannelTestPreviewUtilities.ComputeBodyHash(preview.Body); + return NotifyDeliveryRendered.Create( + preview.ChannelType, + preview.Format, + preview.Target, + preview.Title, + preview.Body, + preview.Summary, + preview.TextBody, + preview.Locale, + hash, + preview.Attachments); + } + + private static IReadOnlyDictionary MergeMetadata( + ChannelTestPreviewContext context, + string providerName, + IReadOnlyDictionary? providerMetadata) + { + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["channelType"] = context.Channel.Type.ToString().ToLowerInvariant(), + ["target"] = context.Target, + ["previewProvider"] = providerName, + ["traceId"] = context.TraceId + }; + + foreach (var pair in context.Request.Metadata) + { + metadata[pair.Key] = pair.Value; + } + + if (providerMetadata is not null) + { + foreach (var pair in providerMetadata) + { + if (string.IsNullOrWhiteSpace(pair.Key) || pair.Value is null) + { + continue; + } + + metadata[pair.Key.Trim()] = pair.Value; + } + } + + return metadata; + } + + private static NotifyDeliveryFormat MapFormat(NotifyChannelType type) + => type switch + { + NotifyChannelType.Slack => NotifyDeliveryFormat.Slack, + NotifyChannelType.Teams => NotifyDeliveryFormat.Teams, + NotifyChannelType.Email => NotifyDeliveryFormat.Email, + NotifyChannelType.Webhook => NotifyDeliveryFormat.Webhook, + _ => NotifyDeliveryFormat.Json + }; + + private static string BuildDefaultBody(NotifyChannel channel, DateTimeOffset timestamp) + { + return $"This is a Stella Ops Notify test message for channel '{channel.Name}' " + + $"({channel.ChannelId}, type {channel.Type}). Generated at {timestamp:O}."; + } + + private static IReadOnlyDictionary NormalizeInputMetadata(IDictionary? source) + { + if (source is null || source.Count == 0) + { + return new Dictionary(StringComparer.Ordinal); + } + + var result = new Dictionary(source.Count, StringComparer.Ordinal); + foreach (var pair in source) + { + if (string.IsNullOrWhiteSpace(pair.Key) || string.IsNullOrWhiteSpace(pair.Value)) + { + continue; + } + + result[pair.Key.Trim()] = pair.Value.Trim(); + } + + return result; + } + + private static IReadOnlyList NormalizeAttachments(IList? attachments) + { + if (attachments is null || attachments.Count == 0) + { + return Array.Empty(); + } + + var list = new List(attachments.Count); + foreach (var attachment in attachments) + { + if (string.IsNullOrWhiteSpace(attachment)) + { + continue; + } + + list.Add(attachment.Trim()); + } + + return list.Count == 0 ? Array.Empty() : list; + } + + private static string? TrimToNull(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string? TrimToLowerInvariant(string? value) + { + var trimmed = TrimToNull(value); + return trimmed?.ToLowerInvariant(); + } +} + +internal sealed class ChannelTestSendValidationException : Exception +{ + public ChannelTestSendValidationException(string message) + : base(message) + { + } +} diff --git a/src/StellaOps.Notify.WebService/Services/NotifySchemaMigrationService.cs b/src/StellaOps.Notify.WebService/Services/NotifySchemaMigrationService.cs new file mode 100644 index 00000000..267e0b46 --- /dev/null +++ b/src/StellaOps.Notify.WebService/Services/NotifySchemaMigrationService.cs @@ -0,0 +1,17 @@ +using System; +using System.Text.Json.Nodes; +using StellaOps.Notify.Models; + +namespace StellaOps.Notify.WebService.Services; + +internal sealed class NotifySchemaMigrationService +{ + public NotifyRule UpgradeRule(JsonNode json) + => NotifySchemaMigration.UpgradeRule(json ?? throw new ArgumentNullException(nameof(json))); + + public NotifyChannel UpgradeChannel(JsonNode json) + => NotifySchemaMigration.UpgradeChannel(json ?? throw new ArgumentNullException(nameof(json))); + + public NotifyTemplate UpgradeTemplate(JsonNode json) + => NotifySchemaMigration.UpgradeTemplate(json ?? throw new ArgumentNullException(nameof(json))); +} diff --git a/src/StellaOps.Notify.WebService/StellaOps.Notify.WebService.csproj b/src/StellaOps.Notify.WebService/StellaOps.Notify.WebService.csproj new file mode 100644 index 00000000..0a52aeae --- /dev/null +++ b/src/StellaOps.Notify.WebService/StellaOps.Notify.WebService.csproj @@ -0,0 +1,27 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Notify.WebService/Storage/InMemory/InMemoryStorageModule.cs b/src/StellaOps.Notify.WebService/Storage/InMemory/InMemoryStorageModule.cs new file mode 100644 index 00000000..861a244b --- /dev/null +++ b/src/StellaOps.Notify.WebService/Storage/InMemory/InMemoryStorageModule.cs @@ -0,0 +1,360 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Notify.Models; +using StellaOps.Notify.Storage.Mongo.Documents; +using StellaOps.Notify.Storage.Mongo.Repositories; + +namespace StellaOps.Notify.WebService.Storage.InMemory; + +internal static class InMemoryStorageModule +{ + public static IServiceCollection AddInMemoryNotifyStorage(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + return services; + } + + private sealed class InMemoryStore + { + public ConcurrentDictionary> Rules { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> Channels { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> Templates { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> Deliveries { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> Digests { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> Locks { get; } = new(StringComparer.Ordinal); + + public ConcurrentDictionary> AuditEntries { get; } = new(StringComparer.Ordinal); + } + + private sealed class InMemoryRuleRepository : INotifyRuleRepository + { + private readonly InMemoryStore _store; + + public InMemoryRuleRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task UpsertAsync(NotifyRule rule, CancellationToken cancellationToken = default) + { + var map = _store.Rules.GetOrAdd(rule.TenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + map[rule.RuleId] = rule; + return Task.CompletedTask; + } + + public Task GetAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default) + { + if (_store.Rules.TryGetValue(tenantId, out var map) && map.TryGetValue(ruleId, out var rule)) + { + return Task.FromResult(rule); + } + + return Task.FromResult(null); + } + + public Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + if (_store.Rules.TryGetValue(tenantId, out var map)) + { + return Task.FromResult>(map.Values.OrderBy(static r => r.RuleId, StringComparer.Ordinal).ToList()); + } + + return Task.FromResult>(Array.Empty()); + } + + public Task DeleteAsync(string tenantId, string ruleId, CancellationToken cancellationToken = default) + { + if (_store.Rules.TryGetValue(tenantId, out var map)) + { + map.TryRemove(ruleId, out _); + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryChannelRepository : INotifyChannelRepository + { + private readonly InMemoryStore _store; + + public InMemoryChannelRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task UpsertAsync(NotifyChannel channel, CancellationToken cancellationToken = default) + { + var map = _store.Channels.GetOrAdd(channel.TenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + map[channel.ChannelId] = channel; + return Task.CompletedTask; + } + + public Task GetAsync(string tenantId, string channelId, CancellationToken cancellationToken = default) + { + if (_store.Channels.TryGetValue(tenantId, out var map) && map.TryGetValue(channelId, out var channel)) + { + return Task.FromResult(channel); + } + + return Task.FromResult(null); + } + + public Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + if (_store.Channels.TryGetValue(tenantId, out var map)) + { + return Task.FromResult>(map.Values.OrderBy(static c => c.ChannelId, StringComparer.Ordinal).ToList()); + } + + return Task.FromResult>(Array.Empty()); + } + + public Task DeleteAsync(string tenantId, string channelId, CancellationToken cancellationToken = default) + { + if (_store.Channels.TryGetValue(tenantId, out var map)) + { + map.TryRemove(channelId, out _); + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryTemplateRepository : INotifyTemplateRepository + { + private readonly InMemoryStore _store; + + public InMemoryTemplateRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task UpsertAsync(NotifyTemplate template, CancellationToken cancellationToken = default) + { + var map = _store.Templates.GetOrAdd(template.TenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + map[template.TemplateId] = template; + return Task.CompletedTask; + } + + public Task GetAsync(string tenantId, string templateId, CancellationToken cancellationToken = default) + { + if (_store.Templates.TryGetValue(tenantId, out var map) && map.TryGetValue(templateId, out var template)) + { + return Task.FromResult(template); + } + + return Task.FromResult(null); + } + + public Task> ListAsync(string tenantId, CancellationToken cancellationToken = default) + { + if (_store.Templates.TryGetValue(tenantId, out var map)) + { + return Task.FromResult>(map.Values.OrderBy(static t => t.TemplateId, StringComparer.Ordinal).ToList()); + } + + return Task.FromResult>(Array.Empty()); + } + + public Task DeleteAsync(string tenantId, string templateId, CancellationToken cancellationToken = default) + { + if (_store.Templates.TryGetValue(tenantId, out var map)) + { + map.TryRemove(templateId, out _); + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryDeliveryRepository : INotifyDeliveryRepository + { + private readonly InMemoryStore _store; + + public InMemoryDeliveryRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task AppendAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default) + => UpdateAsync(delivery, cancellationToken); + + public Task UpdateAsync(NotifyDelivery delivery, CancellationToken cancellationToken = default) + { + var map = _store.Deliveries.GetOrAdd(delivery.TenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + map[delivery.DeliveryId] = delivery; + return Task.CompletedTask; + } + + public Task GetAsync(string tenantId, string deliveryId, CancellationToken cancellationToken = default) + { + if (_store.Deliveries.TryGetValue(tenantId, out var map) && map.TryGetValue(deliveryId, out var delivery)) + { + return Task.FromResult(delivery); + } + + return Task.FromResult(null); + } + + public Task QueryAsync(string tenantId, DateTimeOffset? since, string? status, int? limit, string? continuationToken = null, CancellationToken cancellationToken = default) + { + if (!_store.Deliveries.TryGetValue(tenantId, out var map)) + { + return Task.FromResult(new NotifyDeliveryQueryResult(Array.Empty(), null)); + } + + var query = map.Values.AsEnumerable(); + if (since.HasValue) + { + query = query.Where(d => d.CreatedAt >= since.Value); + } + + if (!string.IsNullOrWhiteSpace(status) && Enum.TryParse(status, true, out var parsed)) + { + query = query.Where(d => d.Status == parsed); + } + + query = query.OrderByDescending(d => d.CreatedAt).ThenBy(d => d.DeliveryId, StringComparer.Ordinal); + + if (limit.HasValue && limit.Value > 0) + { + query = query.Take(limit.Value); + } + + var items = query.ToList(); + return Task.FromResult(new NotifyDeliveryQueryResult(items, ContinuationToken: null)); + } + } + + private sealed class InMemoryDigestRepository : INotifyDigestRepository + { + private readonly InMemoryStore _store; + + public InMemoryDigestRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task UpsertAsync(NotifyDigestDocument document, CancellationToken cancellationToken = default) + { + var map = _store.Digests.GetOrAdd(document.TenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + map[document.ActionKey] = document; + return Task.CompletedTask; + } + + public Task GetAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default) + { + if (_store.Digests.TryGetValue(tenantId, out var map) && map.TryGetValue(actionKey, out var document)) + { + return Task.FromResult(document); + } + + return Task.FromResult(null); + } + + public Task RemoveAsync(string tenantId, string actionKey, CancellationToken cancellationToken = default) + { + if (_store.Digests.TryGetValue(tenantId, out var map)) + { + map.TryRemove(actionKey, out _); + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryLockRepository : INotifyLockRepository + { + private readonly InMemoryStore _store; + + public InMemoryLockRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task TryAcquireAsync(string tenantId, string resource, string owner, TimeSpan ttl, CancellationToken cancellationToken = default) + { + var map = _store.Locks.GetOrAdd(tenantId, _ => new ConcurrentDictionary(StringComparer.Ordinal)); + + var now = DateTimeOffset.UtcNow; + var entry = map.GetOrAdd(resource, _ => new LockEntry(owner, now, now.Add(ttl))); + + lock (entry) + { + if (entry.Owner == owner || entry.ExpiresAt <= now) + { + entry.Owner = owner; + entry.AcquiredAt = now; + entry.ExpiresAt = now.Add(ttl); + return Task.FromResult(true); + } + + return Task.FromResult(false); + } + } + + public Task ReleaseAsync(string tenantId, string resource, string owner, CancellationToken cancellationToken = default) + { + if (_store.Locks.TryGetValue(tenantId, out var map) && map.TryGetValue(resource, out var entry)) + { + lock (entry) + { + if (entry.Owner == owner) + { + map.TryRemove(resource, out _); + } + } + } + + return Task.CompletedTask; + } + } + + private sealed class InMemoryAuditRepository : INotifyAuditRepository + { + private readonly InMemoryStore _store; + + public InMemoryAuditRepository(InMemoryStore store) => _store = store ?? throw new ArgumentNullException(nameof(store)); + + public Task AppendAsync(NotifyAuditEntryDocument entry, CancellationToken cancellationToken = default) + { + var queue = _store.AuditEntries.GetOrAdd(entry.TenantId, _ => new ConcurrentQueue()); + queue.Enqueue(entry); + return Task.CompletedTask; + } + + public Task> QueryAsync(string tenantId, DateTimeOffset? since, int? limit, CancellationToken cancellationToken = default) + { + if (!_store.AuditEntries.TryGetValue(tenantId, out var queue)) + { + return Task.FromResult>(Array.Empty()); + } + + var items = queue + .Where(entry => !since.HasValue || entry.Timestamp >= since.Value) + .OrderByDescending(entry => entry.Timestamp) + .ThenBy(entry => entry.Id.ToString(), StringComparer.Ordinal) + .ToList(); + + if (limit.HasValue && limit.Value > 0 && items.Count > limit.Value) + { + items = items.Take(limit.Value).ToList(); + } + + return Task.FromResult>(items); + } + } + + private sealed class LockEntry + { + public LockEntry(string owner, DateTimeOffset acquiredAt, DateTimeOffset expiresAt) + { + Owner = owner; + AcquiredAt = acquiredAt; + ExpiresAt = expiresAt; + } + + public string Owner { get; set; } + + public DateTimeOffset AcquiredAt { get; set; } + + public DateTimeOffset ExpiresAt { get; set; } + } +} diff --git a/src/StellaOps.Notify.WebService/TASKS.md b/src/StellaOps.Notify.WebService/TASKS.md new file mode 100644 index 00000000..9cd8614d --- /dev/null +++ b/src/StellaOps.Notify.WebService/TASKS.md @@ -0,0 +1,8 @@ +# Notify WebService Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-WEB-15-101 | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-MODELS-15-101 | Bootstrap minimal API host with Authority auth, health endpoints, and plug-in discovery per architecture. | Service starts with config validation, `/healthz`/`/readyz` pass, plug-ins loaded at restart. | +| NOTIFY-WEB-15-102 | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-101 | Rules/channel/template CRUD endpoints with tenant scoping, validation, audit logging. | CRUD endpoints tested; invalid inputs rejected; audit entries persisted. | +| NOTIFY-WEB-15-103 | DONE (2025-10-19) | Notify WebService Guild | NOTIFY-WEB-15-102 | Delivery history + test-send endpoints with rate limits. | `/deliveries` and `/channels/{id}/test` tested; rate limits enforced. | +| NOTIFY-WEB-15-104 | TODO | Notify WebService Guild | NOTIFY-STORAGE-15-201, NOTIFY-QUEUE-15-401 | Configuration binding for Mongo/queue/secrets; startup diagnostics. | Misconfiguration fails fast; diagnostics logged; integration tests cover env overrides. | diff --git a/src/StellaOps.Notify.Worker/AGENTS.md b/src/StellaOps.Notify.Worker/AGENTS.md new file mode 100644 index 00000000..6ce16fe6 --- /dev/null +++ b/src/StellaOps.Notify.Worker/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Notify.Worker — Agent Charter + +## Mission +Consume events, evaluate rules, and dispatch deliveries per `docs/ARCHITECTURE_NOTIFY.md`. diff --git a/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj b/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj new file mode 100644 index 00000000..c444aa16 --- /dev/null +++ b/src/StellaOps.Notify.Worker/StellaOps.Notify.Worker.csproj @@ -0,0 +1,8 @@ + + + net10.0 + enable + enable + Exe + + diff --git a/src/StellaOps.Notify.Worker/TASKS.md b/src/StellaOps.Notify.Worker/TASKS.md new file mode 100644 index 00000000..b35be7f6 --- /dev/null +++ b/src/StellaOps.Notify.Worker/TASKS.md @@ -0,0 +1,8 @@ +# Notify Worker Task Board (Sprint 15) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| NOTIFY-WORKER-15-201 | TODO | Notify Worker Guild | NOTIFY-QUEUE-15-401 | Implement bus subscription + leasing loop with correlation IDs, backoff, dead-letter handling (§1–§5). | Worker consumes events from queue, ack/retry behaviour proven in integration tests; logs include correlation IDs. | +| NOTIFY-WORKER-15-202 | TODO | Notify Worker Guild | NOTIFY-ENGINE-15-301 | Wire rules evaluation pipeline (tenant scoping, filters, throttles, digests, idempotency) with deterministic decisions. | Evaluation unit tests cover rule combinations; throttles/digests produce expected suppression; idempotency keys validated. | +| NOTIFY-WORKER-15-203 | TODO | Notify Worker Guild | NOTIFY-ENGINE-15-302 | Channel dispatch orchestration: invoke connectors, manage retries/jitter, record delivery outcomes. | Connector mocks show retries/backoff; delivery results stored; metrics incremented per outcome. | +| NOTIFY-WORKER-15-204 | TODO | Notify Worker Guild | NOTIFY-WORKER-15-203 | Metrics/telemetry: `notify.sent_total`, `notify.dropped_total`, latency histograms, tracing integration. | Metrics emitted per spec; OTLP spans annotated; dashboards documented. | diff --git a/src/StellaOps.Plugin.Tests/DependencyInjection/PluginServiceRegistrationTests.cs b/src/StellaOps.Plugin.Tests/DependencyInjection/PluginServiceRegistrationTests.cs new file mode 100644 index 00000000..9ec4166a --- /dev/null +++ b/src/StellaOps.Plugin.Tests/DependencyInjection/PluginServiceRegistrationTests.cs @@ -0,0 +1,112 @@ +using System.Linq; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.DependencyInjection; +using Xunit; + +namespace StellaOps.Plugin.Tests.DependencyInjection; + +public sealed class PluginServiceRegistrationTests +{ + [Fact] + public void RegisterAssemblyMetadata_RegistersScopedDescriptor() + { + var services = new ServiceCollection(); + + PluginServiceRegistration.RegisterAssemblyMetadata( + services, + typeof(ScopedTestService).Assembly, + NullLogger.Instance); + + var descriptor = Assert.Single(services, static d => d.ServiceType == typeof(IScopedService)); + Assert.Equal(ServiceLifetime.Scoped, descriptor.Lifetime); + Assert.Equal(typeof(ScopedTestService), descriptor.ImplementationType); + } + + [Fact] + public void RegisterAssemblyMetadata_HonoursRegisterAsSelf() + { + var services = new ServiceCollection(); + + PluginServiceRegistration.RegisterAssemblyMetadata( + services, + typeof(SelfRegisteringService).Assembly, + NullLogger.Instance); + + Assert.Contains(services, static d => + d.ServiceType == typeof(SelfRegisteringService) && + d.ImplementationType == typeof(SelfRegisteringService)); + } + + [Fact] + public void RegisterAssemblyMetadata_ReplacesExistingDescriptorsWhenRequested() + { + var services = new ServiceCollection(); + services.AddSingleton(); + + PluginServiceRegistration.RegisterAssemblyMetadata( + services, + typeof(ReplacementService).Assembly, + NullLogger.Instance); + + var descriptor = Assert.Single( + services, + static d => d.ServiceType == typeof(IReplacementService) && + d.ImplementationType == typeof(ReplacementService)); + Assert.Equal(ServiceLifetime.Transient, descriptor.Lifetime); + } + + [Fact] + public void RegisterAssemblyMetadata_SkipsInvalidAssignments() + { + var services = new ServiceCollection(); + + PluginServiceRegistration.RegisterAssemblyMetadata( + services, + typeof(InvalidServiceBinding).Assembly, + NullLogger.Instance); + + Assert.DoesNotContain(services, static d => d.ServiceType == typeof(IAnotherService)); + } + + private interface IScopedService + { + } + + private interface ISelfContract + { + } + + private interface IReplacementService + { + } + + private interface IAnotherService + { + } + + private sealed class ExistingReplacementService : IReplacementService + { + } + + [ServiceBinding(typeof(IScopedService), ServiceLifetime.Scoped)] + private sealed class ScopedTestService : IScopedService + { + } + + [ServiceBinding(typeof(ISelfContract), ServiceLifetime.Singleton, RegisterAsSelf = true)] + private sealed class SelfRegisteringService : ISelfContract + { + } + + [ServiceBinding(typeof(IReplacementService), ServiceLifetime.Transient, ReplaceExisting = true)] + private sealed class ReplacementService : IReplacementService + { + } + + [ServiceBinding(typeof(IAnotherService), ServiceLifetime.Singleton)] + private sealed class InvalidServiceBinding + { + } +} diff --git a/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj b/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj new file mode 100644 index 00000000..04c131b3 --- /dev/null +++ b/src/StellaOps.Plugin.Tests/StellaOps.Plugin.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + diff --git a/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs index 40d72992..24de33ec 100644 --- a/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs +++ b/src/StellaOps.Plugin/DependencyInjection/PluginDependencyInjectionExtensions.cs @@ -33,15 +33,17 @@ public static class PluginDependencyInjectionExtensions throw new ArgumentNullException(nameof(options)); } - var loadResult = PluginHost.LoadPlugins(options, logger); - - foreach (var plugin in loadResult.Plugins) - { - foreach (var routine in CreateRoutines(plugin.Assembly)) - { - logger?.LogDebug( - "Registering DI routine '{RoutineType}' from plugin '{PluginAssembly}'.", - routine.GetType().FullName, + var loadResult = PluginHost.LoadPlugins(options, logger); + + foreach (var plugin in loadResult.Plugins) + { + PluginServiceRegistration.RegisterAssemblyMetadata(services, plugin.Assembly, logger); + + foreach (var routine in CreateRoutines(plugin.Assembly)) + { + logger?.LogDebug( + "Registering DI routine '{RoutineType}' from plugin '{PluginAssembly}'.", + routine.GetType().FullName, plugin.Assembly.FullName); routine.Register(services, configuration); @@ -88,4 +90,4 @@ public static class PluginDependencyInjectionExtensions } } } -} \ No newline at end of file +} diff --git a/src/StellaOps.Plugin/DependencyInjection/PluginServiceRegistration.cs b/src/StellaOps.Plugin/DependencyInjection/PluginServiceRegistration.cs new file mode 100644 index 00000000..eb08dae7 --- /dev/null +++ b/src/StellaOps.Plugin/DependencyInjection/PluginServiceRegistration.cs @@ -0,0 +1,169 @@ +using System; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.DependencyInjection; +using StellaOps.Plugin.Internal; + +namespace StellaOps.Plugin.DependencyInjection; + +public static class PluginServiceRegistration +{ + public static void RegisterAssemblyMetadata(IServiceCollection services, Assembly assembly, ILogger? logger) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(assembly); + + foreach (var implementationType in assembly.GetLoadableTypes()) + { + if (implementationType is null || !implementationType.IsClass || implementationType.IsAbstract) + { + continue; + } + + var attributes = implementationType.GetCustomAttributes(inherit: false); + if (!attributes.Any()) + { + continue; + } + + foreach (var attribute in attributes) + { + try + { + ApplyBinding(services, implementationType, attribute, logger); + } + catch (Exception ex) + { + logger?.LogWarning( + ex, + "Failed to register service binding for implementation '{Implementation}' declared in assembly '{Assembly}'.", + implementationType.FullName ?? implementationType.Name, + assembly.FullName ?? assembly.GetName().Name); + } + } + } + } + + private static void ApplyBinding( + IServiceCollection services, + Type implementationType, + ServiceBindingAttribute attribute, + ILogger? logger) + { + var serviceType = attribute.ServiceType ?? implementationType; + + if (!IsValidBinding(serviceType, implementationType)) + { + logger?.LogWarning( + "Service binding metadata ignored: implementation '{Implementation}' is not assignable to service '{Service}'.", + implementationType.FullName ?? implementationType.Name, + serviceType.FullName ?? serviceType.Name); + return; + } + + if (attribute.ReplaceExisting) + { + RemoveExistingDescriptors(services, serviceType); + } + + AddDescriptorIfMissing(services, serviceType, implementationType, attribute.Lifetime, logger); + + if (attribute.RegisterAsSelf && serviceType != implementationType) + { + AddDescriptorIfMissing(services, implementationType, implementationType, attribute.Lifetime, logger); + } + } + + private static bool IsValidBinding(Type serviceType, Type implementationType) + { + if (serviceType.IsGenericTypeDefinition) + { + return implementationType.IsGenericTypeDefinition + && implementationType.IsClass + && implementationType.IsAssignableToGenericTypeDefinition(serviceType); + } + + return serviceType.IsAssignableFrom(implementationType); + } + + private static void AddDescriptorIfMissing( + IServiceCollection services, + Type serviceType, + Type implementationType, + ServiceLifetime lifetime, + ILogger? logger) + { + if (services.Any(descriptor => + descriptor.ServiceType == serviceType && + descriptor.ImplementationType == implementationType)) + { + logger?.LogDebug( + "Skipping duplicate service binding for {ServiceType} -> {ImplementationType}.", + serviceType.FullName ?? serviceType.Name, + implementationType.FullName ?? implementationType.Name); + return; + } + + ServiceDescriptor descriptor; + if (serviceType.IsGenericTypeDefinition || implementationType.IsGenericTypeDefinition) + { + descriptor = ServiceDescriptor.Describe(serviceType, implementationType, lifetime); + } + else + { + descriptor = new ServiceDescriptor(serviceType, implementationType, lifetime); + } + + services.Add(descriptor); + logger?.LogDebug( + "Registered service binding {ServiceType} -> {ImplementationType} with {Lifetime} lifetime.", + serviceType.FullName ?? serviceType.Name, + implementationType.FullName ?? implementationType.Name, + lifetime); + } + + private static void RemoveExistingDescriptors(IServiceCollection services, Type serviceType) + { + for (var i = services.Count - 1; i >= 0; i--) + { + if (services[i].ServiceType == serviceType) + { + services.RemoveAt(i); + } + } + } + + private static bool IsAssignableToGenericTypeDefinition(this Type implementationType, Type serviceTypeDefinition) + { + if (!serviceTypeDefinition.IsGenericTypeDefinition) + { + return false; + } + + if (implementationType == serviceTypeDefinition) + { + return true; + } + + if (implementationType.IsGenericType && implementationType.GetGenericTypeDefinition() == serviceTypeDefinition) + { + return true; + } + + var interfaces = implementationType.GetInterfaces(); + foreach (var iface in interfaces) + { + if (iface.IsGenericType && iface.GetGenericTypeDefinition() == serviceTypeDefinition) + { + return true; + } + } + + var baseType = implementationType.BaseType; + return baseType is not null && baseType.IsGenericTypeDefinition + ? baseType.GetGenericTypeDefinition() == serviceTypeDefinition + : baseType is not null && baseType.IsAssignableToGenericTypeDefinition(serviceTypeDefinition); + } +} diff --git a/src/StellaOps.Plugin/Hosting/PluginHost.cs b/src/StellaOps.Plugin/Hosting/PluginHost.cs index b37a1c03..3b1b843b 100644 --- a/src/StellaOps.Plugin/Hosting/PluginHost.cs +++ b/src/StellaOps.Plugin/Hosting/PluginHost.cs @@ -71,10 +71,13 @@ public static class PluginHost private static string ResolvePluginDirectory(PluginHostOptions options, string baseDirectory) { - if (string.IsNullOrWhiteSpace(options.PluginsDirectory)) - { - return Path.Combine(baseDirectory, "PluginBinaries"); - } + if (string.IsNullOrWhiteSpace(options.PluginsDirectory)) + { + var defaultDirectory = !string.IsNullOrWhiteSpace(options.PrimaryPrefix) + ? $"{options.PrimaryPrefix}.PluginBinaries" + : "PluginBinaries"; + return Path.Combine(baseDirectory, defaultDirectory); + } if (Path.IsPathRooted(options.PluginsDirectory)) { @@ -213,4 +216,4 @@ public static class PluginHost return false; } -} \ No newline at end of file +} diff --git a/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs index 634a2f2e..71146697 100644 --- a/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs +++ b/src/StellaOps.Plugin/Hosting/PluginHostOptions.cs @@ -16,8 +16,8 @@ public sealed class PluginHostOptions public string? BaseDirectory { get; set; } /// - /// Directory that contains plugin assemblies. Relative values are resolved against . - /// Defaults to PluginBinaries under the base directory. + /// Directory that contains plugin assemblies. Relative values are resolved against . + /// Defaults to {PrimaryPrefix}.PluginBinaries when a primary prefix is provided, otherwise PluginBinaries. /// public string? PluginsDirectory { get; set; } @@ -56,4 +56,4 @@ public sealed class PluginHostOptions => string.IsNullOrWhiteSpace(BaseDirectory) ? AppContext.BaseDirectory : Path.GetFullPath(BaseDirectory); -} \ No newline at end of file +} diff --git a/src/StellaOps.Plugin/PluginContracts.cs b/src/StellaOps.Plugin/PluginContracts.cs index 924ca656..d50c26a9 100644 --- a/src/StellaOps.Plugin/PluginContracts.cs +++ b/src/StellaOps.Plugin/PluginContracts.cs @@ -60,7 +60,7 @@ public sealed class PluginCatalog return this; } - public PluginCatalog AddFromDirectory(string directory, string searchPattern = "StellaOps.Feedser.*.dll") + public PluginCatalog AddFromDirectory(string directory, string searchPattern = "StellaOps.Concelier.*.dll") { if (string.IsNullOrWhiteSpace(directory)) throw new ArgumentException("Directory is required", nameof(directory)); diff --git a/src/StellaOps.Plugin/Properties/AssemblyInfo.cs b/src/StellaOps.Plugin/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..c187681c --- /dev/null +++ b/src/StellaOps.Plugin/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Plugin.Tests")] diff --git a/src/StellaOps.Plugin/TASKS.md b/src/StellaOps.Plugin/TASKS.md new file mode 100644 index 00000000..d7b8ccdd --- /dev/null +++ b/src/StellaOps.Plugin/TASKS.md @@ -0,0 +1,6 @@ +# TASKS +| Task | Owner(s) | Depends on | Notes | +|---|---|---|---| +|PLUGIN-DI-08-001 Scoped service support in plugin bootstrap|Plugin Platform Guild (DONE 2025-10-19)|StellaOps.DependencyInjection|Introduced `ServiceBindingAttribute` metadata for scoped DI, taught plugin/job loaders to consume it with duplicate-safe registration, added coverage, and refreshed the plug-in SDK guide.| +|PLUGIN-DI-08-002.COORD Authority scoped-service handshake|Plugin Platform Guild, Authority Core (DOING 2025-10-19)|PLUGIN-DI-08-001|Workshop scheduled for 2025-10-20 15:00–16:00 UTC; agenda + attendee list tracked in `docs/dev/authority-plugin-di-coordination.md`; pending pre-read contributions prior to session.| +|PLUGIN-DI-08-002 Authority plugin integration updates|Plugin Platform Guild, Authority Core|PLUGIN-DI-08-001, PLUGIN-DI-08-002.COORD|Update Authority identity-provider plugin registrar/registry to resolve scoped services correctly; adjust bootstrap flows and background services to create scopes when needed; add regression tests.| diff --git a/src/StellaOps.Policy.Tests/PolicyBinderTests.cs b/src/StellaOps.Policy.Tests/PolicyBinderTests.cs new file mode 100644 index 00000000..f6e1d557 --- /dev/null +++ b/src/StellaOps.Policy.Tests/PolicyBinderTests.cs @@ -0,0 +1,86 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace StellaOps.Policy.Tests; + +public sealed class PolicyBinderTests +{ + [Fact] + public void Bind_ValidYaml_ReturnsSuccess() + { + const string yaml = """ + version: "1.0" + rules: + - name: Block Critical + severity: [Critical] + sources: [NVD] + action: block + """; + + var result = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml); + + Assert.True(result.Success); + Assert.Equal("1.0", result.Document.Version); + Assert.Single(result.Document.Rules); + Assert.Empty(result.Issues); + } + + [Fact] + public void Bind_InvalidSeverity_ReturnsError() + { + const string yaml = """ + version: "1.0" + rules: + - name: Invalid Severity + severity: [Nope] + action: block + """; + + var result = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml); + + Assert.False(result.Success); + Assert.Contains(result.Issues, issue => issue.Code == "policy.severity.invalid"); + } + + [Fact] + public async Task Cli_StrictMode_FailsOnWarnings() + { + const string yaml = """ + version: "1.0" + rules: + - name: Quiet Warning + sources: ["", "NVD"] + action: ignore + """; + + var path = Path.Combine(Path.GetTempPath(), $"policy-{Guid.NewGuid():N}.yaml"); + await File.WriteAllTextAsync(path, yaml); + + try + { + using var output = new StringWriter(); + using var error = new StringWriter(); + var cli = new PolicyValidationCli(output, error); + var options = new PolicyValidationCliOptions + { + Inputs = new[] { path }, + Strict = true, + }; + + var exitCode = await cli.RunAsync(options, CancellationToken.None); + + Assert.Equal(2, exitCode); + Assert.Contains("WARNING", output.ToString()); + } + finally + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + } +} diff --git a/src/StellaOps.Policy.Tests/PolicyEvaluationTests.cs b/src/StellaOps.Policy.Tests/PolicyEvaluationTests.cs new file mode 100644 index 00000000..77d16a75 --- /dev/null +++ b/src/StellaOps.Policy.Tests/PolicyEvaluationTests.cs @@ -0,0 +1,133 @@ +using System.Collections.Immutable; +using Xunit; + +namespace StellaOps.Policy.Tests; + +public sealed class PolicyEvaluationTests +{ + [Fact] + public void EvaluateFinding_AppliesTrustAndReachabilityWeights() + { + var action = new PolicyAction(PolicyActionType.Block, null, null, null, false); + var rule = PolicyRule.Create( + "BlockMedium", + action, + ImmutableArray.Create(PolicySeverity.Medium), + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + PolicyRuleMatchCriteria.Empty, + expires: null, + justification: null); + var document = new PolicyDocument( + PolicySchema.CurrentVersion, + ImmutableArray.Create(rule), + ImmutableDictionary.Empty); + + var config = PolicyScoringConfig.Default; + var finding = PolicyFinding.Create( + "finding-medium", + PolicySeverity.Medium, + source: "community", + tags: ImmutableArray.Create("reachability:indirect")); + + var verdict = PolicyEvaluation.EvaluateFinding(document, config, finding); + + Assert.Equal(PolicyVerdictStatus.Blocked, verdict.Status); + Assert.Equal(19.5, verdict.Score, 3); + + var inputs = verdict.GetInputs(); + Assert.Equal(50, inputs["severityWeight"]); + Assert.Equal(0.65, inputs["trustWeight"], 3); + Assert.Equal(0.6, inputs["reachabilityWeight"], 3); + Assert.Equal(19.5, inputs["baseScore"], 3); + } + + [Fact] + public void EvaluateFinding_QuietWithRequireVexAppliesQuietPenalty() + { + var ignoreOptions = new PolicyIgnoreOptions(null, null); + var requireVexOptions = new PolicyRequireVexOptions( + ImmutableArray.Empty, + ImmutableArray.Empty); + var action = new PolicyAction(PolicyActionType.Ignore, ignoreOptions, null, requireVexOptions, true); + var rule = PolicyRule.Create( + "QuietIgnore", + action, + ImmutableArray.Create(PolicySeverity.Critical), + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + PolicyRuleMatchCriteria.Empty, + expires: null, + justification: null); + + var document = new PolicyDocument( + PolicySchema.CurrentVersion, + ImmutableArray.Create(rule), + ImmutableDictionary.Empty); + + var config = PolicyScoringConfig.Default; + var finding = PolicyFinding.Create( + "finding-critical", + PolicySeverity.Critical, + tags: ImmutableArray.Create("reachability:entrypoint")); + + var verdict = PolicyEvaluation.EvaluateFinding(document, config, finding); + + Assert.Equal(PolicyVerdictStatus.Ignored, verdict.Status); + Assert.True(verdict.Quiet); + Assert.Equal("QuietIgnore", verdict.QuietedBy); + Assert.Equal(10, verdict.Score, 3); + + var inputs = verdict.GetInputs(); + Assert.Equal(90, inputs["baseScore"], 3); + Assert.Equal(config.IgnorePenalty, inputs["ignorePenalty"]); + Assert.Equal(config.QuietPenalty, inputs["quietPenalty"]); + } + + [Fact] + public void EvaluateFinding_UnknownSeverityComputesConfidence() + { + var action = new PolicyAction(PolicyActionType.Block, null, null, null, false); + var rule = PolicyRule.Create( + "BlockUnknown", + action, + ImmutableArray.Create(PolicySeverity.Unknown), + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + PolicyRuleMatchCriteria.Empty, + expires: null, + justification: null); + + var document = new PolicyDocument( + PolicySchema.CurrentVersion, + ImmutableArray.Create(rule), + ImmutableDictionary.Empty); + + var config = PolicyScoringConfig.Default; + var finding = PolicyFinding.Create( + "finding-unknown", + PolicySeverity.Unknown, + tags: ImmutableArray.Create("reachability:unknown", "unknown-age-days:5")); + + var verdict = PolicyEvaluation.EvaluateFinding(document, config, finding); + + Assert.Equal(PolicyVerdictStatus.Blocked, verdict.Status); + Assert.Equal(30, verdict.Score, 3); // 60 * 1 * 0.5 + Assert.Equal(0.55, verdict.UnknownConfidence ?? 0, 3); + Assert.Equal("medium", verdict.ConfidenceBand); + Assert.Equal(5, verdict.UnknownAgeDays ?? 0, 3); + + var inputs = verdict.GetInputs(); + Assert.Equal(0.55, inputs["unknownConfidence"], 3); + Assert.Equal(5, inputs["unknownAgeDays"], 3); + } +} diff --git a/src/StellaOps.Policy.Tests/PolicyPreviewServiceTests.cs b/src/StellaOps.Policy.Tests/PolicyPreviewServiceTests.cs new file mode 100644 index 00000000..d8ca0065 --- /dev/null +++ b/src/StellaOps.Policy.Tests/PolicyPreviewServiceTests.cs @@ -0,0 +1,185 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Xunit; +using Xunit.Abstractions; + +namespace StellaOps.Policy.Tests; + +public sealed class PolicyPreviewServiceTests +{ + private readonly ITestOutputHelper _output; + + public PolicyPreviewServiceTests(ITestOutputHelper output) + { + _output = output ?? throw new ArgumentNullException(nameof(output)); + } + + [Fact] + public async Task PreviewAsync_ComputesDiffs_ForBlockingRule() + { + const string yaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var timeProvider = new FakeTimeProvider(); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger.Instance); + + await store.SaveAsync(new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, null), CancellationToken.None); + + var service = new PolicyPreviewService(store, NullLogger.Instance); + + var findings = ImmutableArray.Create( + PolicyFinding.Create("finding-1", PolicySeverity.Critical, environment: "prod", source: "NVD"), + PolicyFinding.Create("finding-2", PolicySeverity.Low)); + + var baseline = ImmutableArray.Create( + new PolicyVerdict("finding-1", PolicyVerdictStatus.Pass), + new PolicyVerdict("finding-2", PolicyVerdictStatus.Pass)); + + var response = await service.PreviewAsync(new PolicyPreviewRequest( + "sha256:abc", + findings, + baseline), + CancellationToken.None); + + Assert.True(response.Success); + Assert.Equal(1, response.ChangedCount); + var diff1 = Assert.Single(response.Diffs.Where(diff => diff.Projected.FindingId == "finding-1")); + Assert.Equal(PolicyVerdictStatus.Pass, diff1.Baseline.Status); + Assert.Equal(PolicyVerdictStatus.Blocked, diff1.Projected.Status); + Assert.Equal("Block Critical", diff1.Projected.RuleName); + Assert.True(diff1.Projected.Score > 0); + Assert.Equal(PolicyScoringConfig.Default.Version, diff1.Projected.ConfigVersion); + Assert.Equal(PolicyVerdictStatus.Pass, response.Diffs.First(diff => diff.Projected.FindingId == "finding-2").Projected.Status); + } + + [Fact] + public async Task PreviewAsync_UsesProposedPolicy_WhenProvided() + { + const string yaml = """ +version: "1.0" +rules: + - name: Ignore Dev + environments: [dev] + action: + type: ignore + justification: dev waiver +"""; + + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger.Instance); + var service = new PolicyPreviewService(store, NullLogger.Instance); + + var findings = ImmutableArray.Create( + PolicyFinding.Create("finding-1", PolicySeverity.Medium, environment: "dev")); + + var baseline = ImmutableArray.Create(new PolicyVerdict("finding-1", PolicyVerdictStatus.Blocked)); + + var response = await service.PreviewAsync(new PolicyPreviewRequest( + "sha256:def", + findings, + baseline, + SnapshotOverride: null, + ProposedPolicy: new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, "dev override")), + CancellationToken.None); + + Assert.True(response.Success); + var diff = Assert.Single(response.Diffs); + Assert.Equal(PolicyVerdictStatus.Blocked, diff.Baseline.Status); + Assert.Equal(PolicyVerdictStatus.Ignored, diff.Projected.Status); + Assert.Equal("Ignore Dev", diff.Projected.RuleName); + Assert.True(diff.Projected.Score >= 0); + Assert.Equal(1, response.ChangedCount); + } + + [Fact] + public async Task PreviewAsync_ReturnsIssues_WhenPolicyInvalid() + { + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger.Instance); + var service = new PolicyPreviewService(store, NullLogger.Instance); + + const string invalid = "version: 1.0"; + var request = new PolicyPreviewRequest( + "sha256:ghi", + ImmutableArray.Empty, + ImmutableArray.Empty, + SnapshotOverride: null, + ProposedPolicy: new PolicySnapshotContent(invalid, PolicyDocumentFormat.Yaml, null, null, null)); + + var response = await service.PreviewAsync(request, CancellationToken.None); + + Assert.False(response.Success); + Assert.NotEmpty(response.Issues); + } + + [Fact] + public async Task PreviewAsync_QuietWithoutVexDowngradesToWarn() + { + const string yaml = """ +version: "1.0" +rules: + - name: Quiet Without VEX + severity: [Low] + quiet: true + action: + type: ignore +"""; + + var binding = PolicyBinder.Bind(yaml, PolicyDocumentFormat.Yaml); + if (!binding.Success) + { + foreach (var issue in binding.Issues) + { + _output.WriteLine($"{issue.Severity} {issue.Code} {issue.Path} :: {issue.Message}"); + } + + var parseMethod = typeof(PolicyBinder).GetMethod("ParseToNode", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + var node = (System.Text.Json.Nodes.JsonNode?)parseMethod?.Invoke(null, new object[] { yaml, PolicyDocumentFormat.Yaml }); + _output.WriteLine(node?.ToJsonString() ?? ""); + } + Assert.True(binding.Success); + Assert.Empty(binding.Issues); + Assert.False(binding.Document.Rules[0].Metadata.ContainsKey("quiet")); + Assert.True(binding.Document.Rules[0].Action.Quiet); + + var store = new PolicySnapshotStore(new InMemoryPolicySnapshotRepository(), new InMemoryPolicyAuditRepository(), TimeProvider.System, NullLogger.Instance); + await store.SaveAsync(new PolicySnapshotContent(yaml, PolicyDocumentFormat.Yaml, "tester", null, "quiet test"), CancellationToken.None); + var snapshot = await store.GetLatestAsync(); + Assert.NotNull(snapshot); + Assert.True(snapshot!.Document.Rules[0].Action.Quiet); + Assert.Null(snapshot.Document.Rules[0].Action.RequireVex); + Assert.Equal(PolicyActionType.Ignore, snapshot.Document.Rules[0].Action.Type); + var manualVerdict = PolicyEvaluation.EvaluateFinding(snapshot.Document, snapshot.ScoringConfig, PolicyFinding.Create("finding-quiet", PolicySeverity.Low)); + Assert.Equal(PolicyVerdictStatus.Warned, manualVerdict.Status); + + var service = new PolicyPreviewService(store, NullLogger.Instance); + + var findings = ImmutableArray.Create(PolicyFinding.Create("finding-quiet", PolicySeverity.Low)); + var baseline = ImmutableArray.Empty; + + var response = await service.PreviewAsync(new PolicyPreviewRequest( + "sha256:quiet", + findings, + baseline), + CancellationToken.None); + + Assert.True(response.Success); + var verdict = Assert.Single(response.Diffs).Projected; + Assert.Equal(PolicyVerdictStatus.Warned, verdict.Status); + Assert.Contains("requireVex", verdict.Notes, System.StringComparison.OrdinalIgnoreCase); + Assert.True(verdict.Score >= 0); + } +} diff --git a/src/StellaOps.Policy.Tests/PolicyScoringConfigTests.cs b/src/StellaOps.Policy.Tests/PolicyScoringConfigTests.cs new file mode 100644 index 00000000..ea019b56 --- /dev/null +++ b/src/StellaOps.Policy.Tests/PolicyScoringConfigTests.cs @@ -0,0 +1,66 @@ +using System; +using System.IO; +using Xunit; + +namespace StellaOps.Policy.Tests; + +public sealed class PolicyScoringConfigTests +{ + [Fact] + public void LoadDefaultReturnsConfig() + { + var config = PolicyScoringConfigBinder.LoadDefault(); + Assert.NotNull(config); + Assert.Equal("1.0", config.Version); + Assert.NotEmpty(config.SeverityWeights); + Assert.True(config.SeverityWeights.ContainsKey(PolicySeverity.Critical)); + Assert.True(config.QuietPenalty > 0); + Assert.NotEmpty(config.ReachabilityBuckets); + Assert.Contains("entrypoint", config.ReachabilityBuckets.Keys); + Assert.False(config.UnknownConfidence.Bands.IsDefaultOrEmpty); + Assert.Equal("high", config.UnknownConfidence.Bands[0].Name); + } + + [Fact] + public void BindRejectsEmptyContent() + { + var result = PolicyScoringConfigBinder.Bind(string.Empty, PolicyDocumentFormat.Json); + Assert.False(result.Success); + Assert.NotEmpty(result.Issues); + } + + [Fact] + public void BindRejectsInvalidSchema() + { + const string json = """ +{ + "version": "1.0", + "severityWeights": { + "Critical": 90.0 + } +} +"""; + + var result = PolicyScoringConfigBinder.Bind(json, PolicyDocumentFormat.Json); + Assert.False(result.Success); + Assert.Contains(result.Issues, issue => issue.Code.StartsWith("scoring.schema", StringComparison.OrdinalIgnoreCase)); + Assert.Null(result.Config); + } + + [Fact] + public void DefaultResourceDigestMatchesGolden() + { + var assembly = typeof(PolicyScoringConfig).Assembly; + using var stream = assembly.GetManifestResourceStream("StellaOps.Policy.Schemas.policy-scoring-default.json") + ?? throw new InvalidOperationException("Unable to locate embedded scoring default resource."); + using var reader = new StreamReader(stream); + var json = reader.ReadToEnd(); + + var binding = PolicyScoringConfigBinder.Bind(json, PolicyDocumentFormat.Json); + Assert.True(binding.Success); + Assert.NotNull(binding.Config); + + var digest = PolicyScoringConfigDigest.Compute(binding.Config!); + Assert.Equal("5ef2e43a112cb00753beb7811dd2e1720f2385e2289d0fb6abcf7bbbb8cda2d2", digest); + } +} diff --git a/src/StellaOps.Policy.Tests/PolicySnapshotStoreTests.cs b/src/StellaOps.Policy.Tests/PolicySnapshotStoreTests.cs new file mode 100644 index 00000000..3e596d24 --- /dev/null +++ b/src/StellaOps.Policy.Tests/PolicySnapshotStoreTests.cs @@ -0,0 +1,94 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Xunit; + +namespace StellaOps.Policy.Tests; + +public sealed class PolicySnapshotStoreTests +{ + private const string BasePolicyYaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + + [Fact] + public async Task SaveAsync_CreatesNewSnapshotAndAuditEntry() + { + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 10, 0, 0, TimeSpan.Zero)); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger.Instance); + + var content = new PolicySnapshotContent(BasePolicyYaml, PolicyDocumentFormat.Yaml, "cli", "test", null); + + var result = await store.SaveAsync(content, CancellationToken.None); + + Assert.True(result.Success); + Assert.True(result.Created); + Assert.NotNull(result.Snapshot); + Assert.Equal("rev-1", result.Snapshot!.RevisionId); + Assert.Equal(result.Digest, result.Snapshot.Digest); + Assert.Equal(timeProvider.GetUtcNow(), result.Snapshot.CreatedAt); + Assert.Equal(PolicyScoringConfig.Default.Version, result.Snapshot.ScoringConfig.Version); + + var latest = await store.GetLatestAsync(); + Assert.Equal(result.Snapshot, latest); + + var audits = await auditRepo.ListAsync(10); + Assert.Single(audits); + Assert.Equal(result.Digest, audits[0].Digest); + Assert.Equal("snapshot.created", audits[0].Action); + Assert.Equal("rev-1", audits[0].RevisionId); + } + + [Fact] + public async Task SaveAsync_DoesNotCreateNewRevisionWhenDigestUnchanged() + { + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 10, 0, 0, TimeSpan.Zero)); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, timeProvider, NullLogger.Instance); + + var content = new PolicySnapshotContent(BasePolicyYaml, PolicyDocumentFormat.Yaml, "cli", "test", null); + var first = await store.SaveAsync(content, CancellationToken.None); + Assert.True(first.Created); + + timeProvider.Advance(TimeSpan.FromHours(1)); + var second = await store.SaveAsync(content, CancellationToken.None); + + Assert.True(second.Success); + Assert.False(second.Created); + Assert.Equal(first.Digest, second.Digest); + Assert.Equal("rev-1", second.Snapshot!.RevisionId); + Assert.Equal(PolicyScoringConfig.Default.Version, second.Snapshot.ScoringConfig.Version); + + var audits = await auditRepo.ListAsync(10); + Assert.Single(audits); + } + + [Fact] + public async Task SaveAsync_ReturnsFailureWhenValidationFails() + { + var snapshotRepo = new InMemoryPolicySnapshotRepository(); + var auditRepo = new InMemoryPolicyAuditRepository(); + var store = new PolicySnapshotStore(snapshotRepo, auditRepo, TimeProvider.System, NullLogger.Instance); + + const string invalidYaml = "version: '1.0'\nrules: []"; + var content = new PolicySnapshotContent(invalidYaml, PolicyDocumentFormat.Yaml, null, null, null); + + var result = await store.SaveAsync(content, CancellationToken.None); + + Assert.False(result.Success); + Assert.False(result.Created); + Assert.Null(result.Snapshot); + + var audits = await auditRepo.ListAsync(5); + Assert.Empty(audits); + } +} diff --git a/src/StellaOps.Vexer.Policy.Tests/StellaOps.Vexer.Policy.Tests.csproj b/src/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj similarity index 77% rename from src/StellaOps.Vexer.Policy.Tests/StellaOps.Vexer.Policy.Tests.csproj rename to src/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj index a4bbbfe9..77bb0e5e 100644 --- a/src/StellaOps.Vexer.Policy.Tests/StellaOps.Vexer.Policy.Tests.csproj +++ b/src/StellaOps.Policy.Tests/StellaOps.Policy.Tests.csproj @@ -1,12 +1,13 @@ - - - net10.0 - preview - enable - enable - true - - - - - + + + net10.0 + preview + enable + enable + true + + + + + + diff --git a/src/StellaOps.Policy/AGENTS.md b/src/StellaOps.Policy/AGENTS.md new file mode 100644 index 00000000..14a6c360 --- /dev/null +++ b/src/StellaOps.Policy/AGENTS.md @@ -0,0 +1,12 @@ +# StellaOps.Policy — Agent Charter + +## Mission +Deliver the policy engine outlined in `docs/ARCHITECTURE_SCANNER.md` and related prose: +- Define YAML schema (ignore rules, VEX inclusion/exclusion, vendor precedence, license gates). +- Provide policy snapshot storage with revision digests and diagnostics. +- Offer preview APIs to compare policy impacts on existing reports. + +## Expectations +- Coordinate with Scanner.WebService, Feedser, Vexer, UI, Notify. +- Maintain deterministic serialization and unit tests for precedence rules. +- Update `TASKS.md` and broadcast contract changes. diff --git a/src/StellaOps.Policy/Audit/IPolicyAuditRepository.cs b/src/StellaOps.Policy/Audit/IPolicyAuditRepository.cs new file mode 100644 index 00000000..1c382f4a --- /dev/null +++ b/src/StellaOps.Policy/Audit/IPolicyAuditRepository.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Policy; + +public interface IPolicyAuditRepository +{ + Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default); + + Task> ListAsync(int limit, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Policy/Audit/InMemoryPolicyAuditRepository.cs b/src/StellaOps.Policy/Audit/InMemoryPolicyAuditRepository.cs new file mode 100644 index 00000000..d61e2d7c --- /dev/null +++ b/src/StellaOps.Policy/Audit/InMemoryPolicyAuditRepository.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Policy; + +public sealed class InMemoryPolicyAuditRepository : IPolicyAuditRepository +{ + private readonly List _entries = new(); + private readonly SemaphoreSlim _mutex = new(1, 1); + + public async Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default) + { + if (entry is null) + { + throw new ArgumentNullException(nameof(entry)); + } + + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _entries.Add(entry); + _entries.Sort(static (left, right) => left.CreatedAt.CompareTo(right.CreatedAt)); + } + finally + { + _mutex.Release(); + } + } + + public async Task> ListAsync(int limit, CancellationToken cancellationToken = default) + { + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + IEnumerable query = _entries; + if (limit > 0) + { + query = query.TakeLast(limit); + } + + return query.ToImmutableArray(); + } + finally + { + _mutex.Release(); + } + } +} diff --git a/src/StellaOps.Policy/PolicyAuditEntry.cs b/src/StellaOps.Policy/PolicyAuditEntry.cs new file mode 100644 index 00000000..2b608ddc --- /dev/null +++ b/src/StellaOps.Policy/PolicyAuditEntry.cs @@ -0,0 +1,12 @@ +using System; + +namespace StellaOps.Policy; + +public sealed record PolicyAuditEntry( + Guid Id, + DateTimeOffset CreatedAt, + string Action, + string RevisionId, + string Digest, + string? Actor, + string Message); diff --git a/src/StellaOps.Policy/PolicyBinder.cs b/src/StellaOps.Policy/PolicyBinder.cs new file mode 100644 index 00000000..4cc9f7d0 --- /dev/null +++ b/src/StellaOps.Policy/PolicyBinder.cs @@ -0,0 +1,915 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Policy; + +public enum PolicyDocumentFormat +{ + Json, + Yaml, +} + +public sealed record PolicyBindingResult( + bool Success, + PolicyDocument Document, + ImmutableArray Issues, + PolicyDocumentFormat Format); + +public static class PolicyBinder +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true, + NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString, + Converters = + { + new JsonStringEnumConverter() + }, + }; + + private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + public static PolicyBindingResult Bind(string content, PolicyDocumentFormat format) + { + if (string.IsNullOrWhiteSpace(content)) + { + var issues = ImmutableArray.Create( + PolicyIssue.Error("policy.empty", "Policy document is empty.", "$")); + return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format); + } + + try + { + var node = ParseToNode(content, format); + if (node is not JsonObject obj) + { + var issues = ImmutableArray.Create( + PolicyIssue.Error("policy.document.invalid", "Policy document must be an object.", "$")); + return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format); + } + + var model = obj.Deserialize(SerializerOptions) ?? new PolicyDocumentModel(); + var normalization = PolicyNormalizer.Normalize(model); + var success = normalization.Issues.All(static issue => issue.Severity != PolicyIssueSeverity.Error); + return new PolicyBindingResult(success, normalization.Document, normalization.Issues, format); + } + catch (JsonException ex) + { + var issues = ImmutableArray.Create( + PolicyIssue.Error("policy.parse.json", $"Failed to parse policy JSON: {ex.Message}", "$")); + return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format); + } + catch (YamlDotNet.Core.YamlException ex) + { + var issues = ImmutableArray.Create( + PolicyIssue.Error("policy.parse.yaml", $"Failed to parse policy YAML: {ex.Message}", "$")); + return new PolicyBindingResult(false, PolicyDocument.Empty, issues, format); + } + } + + public static PolicyBindingResult Bind(Stream stream, PolicyDocumentFormat format, Encoding? encoding = null) + { + if (stream is null) + { + throw new ArgumentNullException(nameof(stream)); + } + + encoding ??= Encoding.UTF8; + using var reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var content = reader.ReadToEnd(); + return Bind(content, format); + } + + private static JsonNode? ParseToNode(string content, PolicyDocumentFormat format) + { + return format switch + { + PolicyDocumentFormat.Json => JsonNode.Parse(content, documentOptions: new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + }), + PolicyDocumentFormat.Yaml => ConvertYamlToJsonNode(content), + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported policy document format."), + }; + } + + private static JsonNode? ConvertYamlToJsonNode(string content) + { + var yamlObject = YamlDeserializer.Deserialize(content); + return ConvertYamlObject(yamlObject); + } + + private static JsonNode? ConvertYamlObject(object? value) + { + switch (value) + { + case null: + return null; + case string s when bool.TryParse(s, out var boolValue): + return JsonValue.Create(boolValue); + case string s: + return JsonValue.Create(s); + case bool b: + return JsonValue.Create(b); + case sbyte or byte or short or ushort or int or uint or long or ulong or float or double or decimal: + return JsonValue.Create(Convert.ToDecimal(value, CultureInfo.InvariantCulture)); + case DateTime dt: + return JsonValue.Create(dt.ToString("O", CultureInfo.InvariantCulture)); + case DateTimeOffset dto: + return JsonValue.Create(dto.ToString("O", CultureInfo.InvariantCulture)); + case Enum e: + return JsonValue.Create(e.ToString()); + case IDictionary dictionary: + { + var obj = new JsonObject(); + foreach (DictionaryEntry entry in dictionary) + { + if (entry.Key is null) + { + continue; + } + + var key = Convert.ToString(entry.Key, CultureInfo.InvariantCulture); + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + obj[key!] = ConvertYamlObject(entry.Value); + } + + return obj; + } + case IEnumerable enumerable: + { + var array = new JsonArray(); + foreach (var item in enumerable) + { + array.Add(ConvertYamlObject(item)); + } + + return array; + } + default: + return JsonValue.Create(value.ToString()); + } + } + + private sealed record PolicyDocumentModel + { + [JsonPropertyName("version")] + public JsonNode? Version { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("metadata")] + public Dictionary? Metadata { get; init; } + + [JsonPropertyName("rules")] + public List? Rules { get; init; } + + [JsonExtensionData] + public Dictionary? Extensions { get; init; } + } + + private sealed record PolicyRuleModel + { + [JsonPropertyName("id")] + public string? Identifier { get; init; } + + [JsonPropertyName("name")] + public string? Name { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } + + [JsonPropertyName("severity")] + public List? Severity { get; init; } + + [JsonPropertyName("sources")] + public List? Sources { get; init; } + + [JsonPropertyName("vendors")] + public List? Vendors { get; init; } + + [JsonPropertyName("licenses")] + public List? Licenses { get; init; } + + [JsonPropertyName("tags")] + public List? Tags { get; init; } + + [JsonPropertyName("environments")] + public List? Environments { get; init; } + + [JsonPropertyName("images")] + public List? Images { get; init; } + + [JsonPropertyName("repositories")] + public List? Repositories { get; init; } + + [JsonPropertyName("packages")] + public List? Packages { get; init; } + + [JsonPropertyName("purls")] + public List? Purls { get; init; } + + [JsonPropertyName("cves")] + public List? Cves { get; init; } + + [JsonPropertyName("paths")] + public List? Paths { get; init; } + + [JsonPropertyName("layerDigests")] + public List? LayerDigests { get; init; } + + [JsonPropertyName("usedByEntrypoint")] + public List? UsedByEntrypoint { get; init; } + + [JsonPropertyName("action")] + public JsonNode? Action { get; init; } + + [JsonPropertyName("expires")] + public JsonNode? Expires { get; init; } + + [JsonPropertyName("until")] + public JsonNode? Until { get; init; } + + [JsonPropertyName("justification")] + public string? Justification { get; init; } + + [JsonPropertyName("quiet")] + public bool? Quiet { get; init; } + + [JsonPropertyName("metadata")] + public Dictionary? Metadata { get; init; } + + [JsonExtensionData] + public Dictionary? Extensions { get; init; } + } + + private sealed class PolicyNormalizer + { + private static readonly ImmutableDictionary SeverityMap = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["critical"] = PolicySeverity.Critical, + ["high"] = PolicySeverity.High, + ["medium"] = PolicySeverity.Medium, + ["moderate"] = PolicySeverity.Medium, + ["low"] = PolicySeverity.Low, + ["informational"] = PolicySeverity.Informational, + ["info"] = PolicySeverity.Informational, + ["none"] = PolicySeverity.None, + ["unknown"] = PolicySeverity.Unknown, + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + public static (PolicyDocument Document, ImmutableArray Issues) Normalize(PolicyDocumentModel model) + { + var issues = ImmutableArray.CreateBuilder(); + + var version = NormalizeVersion(model.Version, issues); + var metadata = NormalizeMetadata(model.Metadata, "$.metadata", issues); + var rules = NormalizeRules(model.Rules, issues); + + if (model.Extensions is { Count: > 0 }) + { + foreach (var pair in model.Extensions) + { + issues.Add(PolicyIssue.Warning( + "policy.document.extension", + $"Unrecognized document property '{pair.Key}' has been ignored.", + $"$.{pair.Key}")); + } + } + + var document = new PolicyDocument( + version ?? PolicySchema.CurrentVersion, + rules, + metadata); + + var orderedIssues = SortIssues(issues); + return (document, orderedIssues); + } + + private static string? NormalizeVersion(JsonNode? versionNode, ImmutableArray.Builder issues) + { + if (versionNode is null) + { + issues.Add(PolicyIssue.Warning("policy.version.missing", "Policy version not specified; defaulting to 1.0.", "$.version")); + return PolicySchema.CurrentVersion; + } + + if (versionNode is JsonValue value) + { + if (value.TryGetValue(out string? versionText)) + { + versionText = versionText?.Trim(); + if (string.IsNullOrEmpty(versionText)) + { + issues.Add(PolicyIssue.Error("policy.version.empty", "Policy version is empty.", "$.version")); + return null; + } + + if (IsSupportedVersion(versionText)) + { + return CanonicalizeVersion(versionText); + } + + issues.Add(PolicyIssue.Error("policy.version.unsupported", $"Unsupported policy version '{versionText}'. Expected '{PolicySchema.CurrentVersion}'.", "$.version")); + return null; + } + + if (value.TryGetValue(out double numericVersion)) + { + var numericText = numericVersion.ToString("0.0###", CultureInfo.InvariantCulture); + if (IsSupportedVersion(numericText)) + { + return CanonicalizeVersion(numericText); + } + + issues.Add(PolicyIssue.Error("policy.version.unsupported", $"Unsupported policy version '{numericText}'.", "$.version")); + return null; + } + } + + var raw = versionNode.ToJsonString(); + issues.Add(PolicyIssue.Error("policy.version.invalid", $"Policy version must be a string. Received: {raw}", "$.version")); + return null; + } + + private static bool IsSupportedVersion(string versionText) + => string.Equals(versionText, "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(versionText, "1.0", StringComparison.OrdinalIgnoreCase) + || string.Equals(versionText, PolicySchema.CurrentVersion, StringComparison.OrdinalIgnoreCase); + + private static string CanonicalizeVersion(string versionText) + => string.Equals(versionText, "1", StringComparison.OrdinalIgnoreCase) + ? "1.0" + : versionText; + + private static ImmutableDictionary NormalizeMetadata( + Dictionary? metadata, + string path, + ImmutableArray.Builder issues) + { + if (metadata is null || metadata.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in metadata) + { + var key = pair.Key?.Trim(); + if (string.IsNullOrEmpty(key)) + { + issues.Add(PolicyIssue.Warning("policy.metadata.key.empty", "Metadata keys must be non-empty strings.", path)); + continue; + } + + var value = ConvertNodeToString(pair.Value); + builder[key] = value; + } + + return builder.ToImmutable(); + } + + private static ImmutableArray NormalizeRules( + List? rules, + ImmutableArray.Builder issues) + { + if (rules is null || rules.Count == 0) + { + issues.Add(PolicyIssue.Error("policy.rules.empty", "At least one rule must be defined.", "$.rules")); + return ImmutableArray.Empty; + } + + var normalized = new List<(PolicyRule Rule, int Index)>(rules.Count); + var seenNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + for (var index = 0; index < rules.Count; index++) + { + var model = rules[index]; + var normalizedRule = NormalizeRule(model, index, issues); + if (normalizedRule is null) + { + continue; + } + + if (!seenNames.Add(normalizedRule.Name)) + { + issues.Add(PolicyIssue.Warning( + "policy.rules.duplicateName", + $"Duplicate rule name '{normalizedRule.Name}' detected; evaluation order may be ambiguous.", + $"$.rules[{index}].name")); + } + + normalized.Add((normalizedRule, index)); + } + + return normalized + .OrderBy(static tuple => tuple.Rule.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(static tuple => tuple.Rule.Identifier ?? string.Empty, StringComparer.OrdinalIgnoreCase) + .ThenBy(static tuple => tuple.Index) + .Select(static tuple => tuple.Rule) + .ToImmutableArray(); + } + + private static PolicyRule? NormalizeRule( + PolicyRuleModel model, + int index, + ImmutableArray.Builder issues) + { + var basePath = $"$.rules[{index}]"; + + var name = NormalizeRequiredString(model.Name, $"{basePath}.name", "Rule name", issues); + if (name is null) + { + return null; + } + + var identifier = NormalizeOptionalString(model.Identifier); + var description = NormalizeOptionalString(model.Description); + var metadata = NormalizeMetadata(model.Metadata, $"{basePath}.metadata", issues); + + var severities = NormalizeSeverityList(model.Severity, $"{basePath}.severity", issues); + var environments = NormalizeStringList(model.Environments, $"{basePath}.environments", issues); + var sources = NormalizeStringList(model.Sources, $"{basePath}.sources", issues); + var vendors = NormalizeStringList(model.Vendors, $"{basePath}.vendors", issues); + var licenses = NormalizeStringList(model.Licenses, $"{basePath}.licenses", issues); + var tags = NormalizeStringList(model.Tags, $"{basePath}.tags", issues); + + var match = new PolicyRuleMatchCriteria( + NormalizeStringList(model.Images, $"{basePath}.images", issues), + NormalizeStringList(model.Repositories, $"{basePath}.repositories", issues), + NormalizeStringList(model.Packages, $"{basePath}.packages", issues), + NormalizeStringList(model.Purls, $"{basePath}.purls", issues), + NormalizeStringList(model.Cves, $"{basePath}.cves", issues), + NormalizeStringList(model.Paths, $"{basePath}.paths", issues), + NormalizeStringList(model.LayerDigests, $"{basePath}.layerDigests", issues), + NormalizeStringList(model.UsedByEntrypoint, $"{basePath}.usedByEntrypoint", issues)); + + var action = NormalizeAction(model, basePath, issues); + var justification = NormalizeOptionalString(model.Justification); + var expires = NormalizeTemporal(model.Expires ?? model.Until, $"{basePath}.expires", issues); + + if (model.Extensions is { Count: > 0 }) + { + foreach (var pair in model.Extensions) + { + issues.Add(PolicyIssue.Warning( + "policy.rule.extension", + $"Unrecognized rule property '{pair.Key}' has been ignored.", + $"{basePath}.{pair.Key}")); + } + } + + return PolicyRule.Create( + name, + action, + severities, + environments, + sources, + vendors, + licenses, + tags, + match, + expires, + justification, + identifier, + description, + metadata); + } + + private static PolicyAction NormalizeAction( + PolicyRuleModel model, + string basePath, + ImmutableArray.Builder issues) + { + var actionNode = model.Action; + var quiet = model.Quiet ?? false; + if (!quiet && model.Extensions is not null && model.Extensions.TryGetValue("quiet", out var quietExtension) && quietExtension.ValueKind == JsonValueKind.True) + { + quiet = true; + } + string? justification = NormalizeOptionalString(model.Justification); + DateTimeOffset? until = NormalizeTemporal(model.Until, $"{basePath}.until", issues); + DateTimeOffset? expires = NormalizeTemporal(model.Expires, $"{basePath}.expires", issues); + + var effectiveUntil = until ?? expires; + + if (actionNode is null) + { + issues.Add(PolicyIssue.Error("policy.action.missing", "Rule action is required.", $"{basePath}.action")); + return new PolicyAction(PolicyActionType.Block, null, null, null, Quiet: false); + } + + string? actionType = null; + JsonObject? actionObject = null; + + switch (actionNode) + { + case JsonValue value when value.TryGetValue(out string? text): + actionType = text; + break; + case JsonValue value when value.TryGetValue(out bool booleanValue): + actionType = booleanValue ? "block" : "ignore"; + break; + case JsonObject obj: + actionObject = obj; + if (obj.TryGetPropertyValue("type", out var typeNode) && typeNode is JsonValue typeValue && typeValue.TryGetValue(out string? typeText)) + { + actionType = typeText; + } + else + { + issues.Add(PolicyIssue.Error("policy.action.type", "Action object must contain a 'type' property.", $"{basePath}.action.type")); + } + + if (obj.TryGetPropertyValue("quiet", out var quietNode) && quietNode is JsonValue quietValue && quietValue.TryGetValue(out bool quietFlag)) + { + quiet = quietFlag; + } + + if (obj.TryGetPropertyValue("until", out var untilNode)) + { + effectiveUntil ??= NormalizeTemporal(untilNode, $"{basePath}.action.until", issues); + } + + if (obj.TryGetPropertyValue("justification", out var justificationNode) && justificationNode is JsonValue justificationValue && justificationValue.TryGetValue(out string? justificationText)) + { + justification = NormalizeOptionalString(justificationText); + } + + break; + default: + actionType = actionNode.ToString(); + break; + } + + if (string.IsNullOrWhiteSpace(actionType)) + { + issues.Add(PolicyIssue.Error("policy.action.type", "Action type is required.", $"{basePath}.action")); + return new PolicyAction(PolicyActionType.Block, null, null, null, Quiet: quiet); + } + + actionType = actionType.Trim(); + var (type, typeIssues) = MapActionType(actionType, $"{basePath}.action"); + foreach (var issue in typeIssues) + { + issues.Add(issue); + } + + PolicyIgnoreOptions? ignoreOptions = null; + PolicyEscalateOptions? escalateOptions = null; + PolicyRequireVexOptions? requireVexOptions = null; + + if (type == PolicyActionType.Ignore) + { + ignoreOptions = new PolicyIgnoreOptions(effectiveUntil, justification); + } + else if (type == PolicyActionType.Escalate) + { + escalateOptions = NormalizeEscalateOptions(actionObject, $"{basePath}.action", issues); + } + else if (type == PolicyActionType.RequireVex) + { + requireVexOptions = NormalizeRequireVexOptions(actionObject, $"{basePath}.action", issues); + } + + return new PolicyAction(type, ignoreOptions, escalateOptions, requireVexOptions, quiet); + } + + private static (PolicyActionType Type, ImmutableArray Issues) MapActionType(string value, string path) + { + var issues = ImmutableArray.Empty; + var lower = value.ToLowerInvariant(); + return lower switch + { + "block" or "fail" or "deny" => (PolicyActionType.Block, issues), + "ignore" or "mute" => (PolicyActionType.Ignore, issues), + "warn" or "warning" => (PolicyActionType.Warn, issues), + "defer" => (PolicyActionType.Defer, issues), + "escalate" => (PolicyActionType.Escalate, issues), + "requirevex" or "require_vex" or "require-vex" => (PolicyActionType.RequireVex, issues), + _ => (PolicyActionType.Block, ImmutableArray.Create(PolicyIssue.Warning( + "policy.action.unknown", + $"Unknown action '{value}' encountered. Defaulting to 'block'.", + path))), + }; + } + + private static PolicyEscalateOptions? NormalizeEscalateOptions( + JsonObject? actionObject, + string path, + ImmutableArray.Builder issues) + { + if (actionObject is null) + { + return null; + } + + PolicySeverity? minSeverity = null; + bool requireKev = false; + double? minEpss = null; + + if (actionObject.TryGetPropertyValue("severity", out var severityNode) && severityNode is JsonValue severityValue && severityValue.TryGetValue(out string? severityText)) + { + if (SeverityMap.TryGetValue(severityText ?? string.Empty, out var mapped)) + { + minSeverity = mapped; + } + else + { + issues.Add(PolicyIssue.Warning("policy.action.escalate.severity", $"Unknown escalate severity '{severityText}'.", $"{path}.severity")); + } + } + + if (actionObject.TryGetPropertyValue("kev", out var kevNode) && kevNode is JsonValue kevValue && kevValue.TryGetValue(out bool kevFlag)) + { + requireKev = kevFlag; + } + + if (actionObject.TryGetPropertyValue("epss", out var epssNode)) + { + var parsed = ParseDouble(epssNode, $"{path}.epss", issues); + if (parsed is { } epssValue) + { + if (epssValue < 0 || epssValue > 1) + { + issues.Add(PolicyIssue.Warning("policy.action.escalate.epssRange", "EPS score must be between 0 and 1.", $"{path}.epss")); + } + else + { + minEpss = epssValue; + } + } + } + + return new PolicyEscalateOptions(minSeverity, requireKev, minEpss); + } + + private static PolicyRequireVexOptions? NormalizeRequireVexOptions( + JsonObject? actionObject, + string path, + ImmutableArray.Builder issues) + { + if (actionObject is null) + { + return null; + } + + var vendors = ImmutableArray.Empty; + var justifications = ImmutableArray.Empty; + + if (actionObject.TryGetPropertyValue("vendors", out var vendorsNode)) + { + vendors = NormalizeJsonStringArray(vendorsNode, $"{path}.vendors", issues); + } + + if (actionObject.TryGetPropertyValue("justifications", out var justificationsNode)) + { + justifications = NormalizeJsonStringArray(justificationsNode, $"{path}.justifications", issues); + } + + return new PolicyRequireVexOptions(vendors, justifications); + } + + private static ImmutableArray NormalizeStringList( + List? values, + string path, + ImmutableArray.Builder issues) + { + if (values is null || values.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableHashSet.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var value in values) + { + var normalized = NormalizeOptionalString(value); + if (string.IsNullOrEmpty(normalized)) + { + issues.Add(PolicyIssue.Warning("policy.list.blank", $"Blank entry detected; ignoring value at {path}.", path)); + continue; + } + + builder.Add(normalized); + } + + return builder.ToImmutable() + .OrderBy(static item => item, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + private static ImmutableArray NormalizeSeverityList( + List? values, + string path, + ImmutableArray.Builder issues) + { + if (values is null || values.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + foreach (var value in values) + { + var normalized = NormalizeOptionalString(value); + if (string.IsNullOrEmpty(normalized)) + { + issues.Add(PolicyIssue.Warning("policy.severity.blank", "Blank severity was ignored.", path)); + continue; + } + + if (SeverityMap.TryGetValue(normalized, out var severity)) + { + builder.Add(severity); + } + else + { + issues.Add(PolicyIssue.Error("policy.severity.invalid", $"Unknown severity '{value}'.", path)); + } + } + + return builder.Distinct().OrderBy(static sev => sev).ToImmutableArray(); + } + + private static ImmutableArray NormalizeJsonStringArray( + JsonNode? node, + string path, + ImmutableArray.Builder issues) + { + if (node is null) + { + return ImmutableArray.Empty; + } + + if (node is JsonArray array) + { + var values = new List(array.Count); + foreach (var element in array) + { + var text = ConvertNodeToString(element); + if (string.IsNullOrWhiteSpace(text)) + { + issues.Add(PolicyIssue.Warning("policy.list.blank", $"Blank entry detected; ignoring value at {path}.", path)); + } + else + { + values.Add(text); + } + } + + return values + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static entry => entry, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + } + + var single = ConvertNodeToString(node); + return ImmutableArray.Create(single); + } + + private static double? ParseDouble(JsonNode? node, string path, ImmutableArray.Builder issues) + { + if (node is null) + { + return null; + } + + if (node is JsonValue value) + { + if (value.TryGetValue(out double numeric)) + { + return numeric; + } + + if (value.TryGetValue(out string? text) && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out numeric)) + { + return numeric; + } + } + + issues.Add(PolicyIssue.Warning("policy.number.invalid", $"Value '{node.ToJsonString()}' is not a valid number.", path)); + return null; + } + + private static DateTimeOffset? NormalizeTemporal(JsonNode? node, string path, ImmutableArray.Builder issues) + { + if (node is null) + { + return null; + } + + if (node is JsonValue value) + { + if (value.TryGetValue(out DateTimeOffset dto)) + { + return dto; + } + + if (value.TryGetValue(out DateTime dt)) + { + return new DateTimeOffset(DateTime.SpecifyKind(dt, DateTimeKind.Utc)); + } + + if (value.TryGetValue(out string? text)) + { + if (DateTimeOffset.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)) + { + return parsed; + } + + if (DateTime.TryParse(text, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsedDate)) + { + return new DateTimeOffset(parsedDate); + } + } + } + + issues.Add(PolicyIssue.Warning("policy.date.invalid", $"Value '{node.ToJsonString()}' is not a valid ISO-8601 timestamp.", path)); + return null; + } + + private static string? NormalizeRequiredString( + string? value, + string path, + string fieldDescription, + ImmutableArray.Builder issues) + { + var normalized = NormalizeOptionalString(value); + if (!string.IsNullOrEmpty(normalized)) + { + return normalized; + } + + issues.Add(PolicyIssue.Error( + "policy.required", + $"{fieldDescription} is required.", + path)); + return null; + } + + private static string? NormalizeOptionalString(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim(); + } + + private static string ConvertNodeToString(JsonNode? node) + { + if (node is null) + { + return string.Empty; + } + + return node switch + { + JsonValue value when value.TryGetValue(out string? text) => text ?? string.Empty, + JsonValue value when value.TryGetValue(out bool boolean) => boolean ? "true" : "false", + JsonValue value when value.TryGetValue(out double numeric) => numeric.ToString(CultureInfo.InvariantCulture), + JsonObject obj => obj.ToJsonString(), + JsonArray array => array.ToJsonString(), + _ => node.ToJsonString(), + }; + } + + private static ImmutableArray SortIssues(ImmutableArray.Builder issues) + { + return issues.ToImmutable() + .OrderBy(static issue => issue.Severity switch + { + PolicyIssueSeverity.Error => 0, + PolicyIssueSeverity.Warning => 1, + _ => 2, + }) + .ThenBy(static issue => issue.Path, StringComparer.Ordinal) + .ThenBy(static issue => issue.Code, StringComparer.Ordinal) + .ToImmutableArray(); + } + } +} diff --git a/src/StellaOps.Policy/PolicyDiagnostics.cs b/src/StellaOps.Policy/PolicyDiagnostics.cs new file mode 100644 index 00000000..a6783c8b --- /dev/null +++ b/src/StellaOps.Policy/PolicyDiagnostics.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Policy; + +public sealed record PolicyDiagnosticsReport( + string Version, + int RuleCount, + int ErrorCount, + int WarningCount, + DateTimeOffset GeneratedAt, + ImmutableArray Issues, + ImmutableArray Recommendations); + +public static class PolicyDiagnostics +{ + public static PolicyDiagnosticsReport Create(PolicyBindingResult bindingResult, TimeProvider? timeProvider = null) + { + if (bindingResult is null) + { + throw new ArgumentNullException(nameof(bindingResult)); + } + + var time = (timeProvider ?? TimeProvider.System).GetUtcNow(); + var errorCount = bindingResult.Issues.Count(static issue => issue.Severity == PolicyIssueSeverity.Error); + var warningCount = bindingResult.Issues.Count(static issue => issue.Severity == PolicyIssueSeverity.Warning); + + var recommendations = BuildRecommendations(bindingResult.Document, errorCount, warningCount); + + return new PolicyDiagnosticsReport( + bindingResult.Document.Version, + bindingResult.Document.Rules.Length, + errorCount, + warningCount, + time, + bindingResult.Issues, + recommendations); + } + + private static ImmutableArray BuildRecommendations(PolicyDocument document, int errorCount, int warningCount) + { + var messages = ImmutableArray.CreateBuilder(); + + if (errorCount > 0) + { + messages.Add("Resolve policy errors before promoting the revision; fallback rules may be applied while errors remain."); + } + + if (warningCount > 0) + { + messages.Add("Review policy warnings and ensure intentional overrides are documented."); + } + + if (document.Rules.Length == 0) + { + messages.Add("Add at least one policy rule to enforce gating logic."); + } + + var quietRules = document.Rules + .Where(static rule => rule.Action.Quiet) + .Select(static rule => rule.Name) + .ToArray(); + + if (quietRules.Length > 0) + { + messages.Add($"Quiet rules detected ({string.Join(", ", quietRules)}); verify scoring behaviour aligns with expectations."); + } + + if (messages.Count == 0) + { + messages.Add("Policy validated successfully; no additional action required."); + } + + return messages.ToImmutable(); + } +} diff --git a/src/StellaOps.Policy/PolicyDigest.cs b/src/StellaOps.Policy/PolicyDigest.cs new file mode 100644 index 00000000..e22cb2d1 --- /dev/null +++ b/src/StellaOps.Policy/PolicyDigest.cs @@ -0,0 +1,211 @@ +using System; +using System.Buffers; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; + +namespace StellaOps.Policy; + +public static class PolicyDigest +{ + public static string Compute(PolicyDocument document) + { + if (document is null) + { + throw new ArgumentNullException(nameof(document)); + } + + var buffer = new ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions + { + SkipValidation = true, + })) + { + WriteDocument(writer, document); + } + + var hash = SHA256.HashData(buffer.WrittenSpan); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static void WriteDocument(Utf8JsonWriter writer, PolicyDocument document) + { + writer.WriteStartObject(); + writer.WriteString("version", document.Version); + + if (!document.Metadata.IsEmpty) + { + writer.WritePropertyName("metadata"); + writer.WriteStartObject(); + foreach (var pair in document.Metadata.OrderBy(static kvp => kvp.Key, StringComparer.Ordinal)) + { + writer.WriteString(pair.Key, pair.Value); + } + writer.WriteEndObject(); + } + + writer.WritePropertyName("rules"); + writer.WriteStartArray(); + foreach (var rule in document.Rules) + { + WriteRule(writer, rule); + } + writer.WriteEndArray(); + + writer.WriteEndObject(); + writer.Flush(); + } + + private static void WriteRule(Utf8JsonWriter writer, PolicyRule rule) + { + writer.WriteStartObject(); + writer.WriteString("name", rule.Name); + + if (!string.IsNullOrWhiteSpace(rule.Identifier)) + { + writer.WriteString("id", rule.Identifier); + } + + if (!string.IsNullOrWhiteSpace(rule.Description)) + { + writer.WriteString("description", rule.Description); + } + + WriteMetadata(writer, rule.Metadata); + WriteSeverities(writer, rule.Severities); + WriteStringArray(writer, "environments", rule.Environments); + WriteStringArray(writer, "sources", rule.Sources); + WriteStringArray(writer, "vendors", rule.Vendors); + WriteStringArray(writer, "licenses", rule.Licenses); + WriteStringArray(writer, "tags", rule.Tags); + + if (!rule.Match.IsEmpty) + { + writer.WritePropertyName("match"); + writer.WriteStartObject(); + WriteStringArray(writer, "images", rule.Match.Images); + WriteStringArray(writer, "repositories", rule.Match.Repositories); + WriteStringArray(writer, "packages", rule.Match.Packages); + WriteStringArray(writer, "purls", rule.Match.Purls); + WriteStringArray(writer, "cves", rule.Match.Cves); + WriteStringArray(writer, "paths", rule.Match.Paths); + WriteStringArray(writer, "layerDigests", rule.Match.LayerDigests); + WriteStringArray(writer, "usedByEntrypoint", rule.Match.UsedByEntrypoint); + writer.WriteEndObject(); + } + + WriteAction(writer, rule.Action); + + if (rule.Expires is DateTimeOffset expires) + { + writer.WriteString("expires", expires.ToUniversalTime().ToString("O")); + } + + if (!string.IsNullOrWhiteSpace(rule.Justification)) + { + writer.WriteString("justification", rule.Justification); + } + + writer.WriteEndObject(); + } + + private static void WriteAction(Utf8JsonWriter writer, PolicyAction action) + { + writer.WritePropertyName("action"); + writer.WriteStartObject(); + writer.WriteString("type", action.Type.ToString().ToLowerInvariant()); + + if (action.Quiet) + { + writer.WriteBoolean("quiet", true); + } + + if (action.Ignore is { } ignore) + { + if (ignore.Until is DateTimeOffset until) + { + writer.WriteString("until", until.ToUniversalTime().ToString("O")); + } + + if (!string.IsNullOrWhiteSpace(ignore.Justification)) + { + writer.WriteString("justification", ignore.Justification); + } + } + + if (action.Escalate is { } escalate) + { + if (escalate.MinimumSeverity is { } severity) + { + writer.WriteString("severity", severity.ToString()); + } + + if (escalate.RequireKev) + { + writer.WriteBoolean("kev", true); + } + + if (escalate.MinimumEpss is double epss) + { + writer.WriteNumber("epss", epss); + } + } + + if (action.RequireVex is { } requireVex) + { + WriteStringArray(writer, "vendors", requireVex.Vendors); + WriteStringArray(writer, "justifications", requireVex.Justifications); + } + + writer.WriteEndObject(); + } + + private static void WriteMetadata(Utf8JsonWriter writer, ImmutableDictionary metadata) + { + if (metadata.IsEmpty) + { + return; + } + + writer.WritePropertyName("metadata"); + writer.WriteStartObject(); + foreach (var pair in metadata.OrderBy(static kvp => kvp.Key, StringComparer.Ordinal)) + { + writer.WriteString(pair.Key, pair.Value); + } + writer.WriteEndObject(); + } + + private static void WriteSeverities(Utf8JsonWriter writer, ImmutableArray severities) + { + if (severities.IsDefaultOrEmpty) + { + return; + } + + writer.WritePropertyName("severity"); + writer.WriteStartArray(); + foreach (var severity in severities) + { + writer.WriteStringValue(severity.ToString()); + } + writer.WriteEndArray(); + } + + private static void WriteStringArray(Utf8JsonWriter writer, string propertyName, ImmutableArray values) + { + if (values.IsDefaultOrEmpty) + { + return; + } + + writer.WritePropertyName(propertyName); + writer.WriteStartArray(); + foreach (var value in values) + { + writer.WriteStringValue(value); + } + writer.WriteEndArray(); + } +} diff --git a/src/StellaOps.Policy/PolicyDocument.cs b/src/StellaOps.Policy/PolicyDocument.cs new file mode 100644 index 00000000..a086109d --- /dev/null +++ b/src/StellaOps.Policy/PolicyDocument.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +/// +/// Canonical representation of a StellaOps policy document. +/// +public sealed record PolicyDocument( + string Version, + ImmutableArray Rules, + ImmutableDictionary Metadata) +{ + public static PolicyDocument Empty { get; } = new( + PolicySchema.CurrentVersion, + ImmutableArray.Empty, + ImmutableDictionary.Empty); +} + +public static class PolicySchema +{ + public const string SchemaId = "https://schemas.stella-ops.org/policy/policy-schema@1.json"; + public const string CurrentVersion = "1.0"; + + public static PolicyDocumentFormat DetectFormat(string fileName) + { + if (fileName is null) + { + throw new ArgumentNullException(nameof(fileName)); + } + + var lower = fileName.Trim().ToLowerInvariant(); + if (lower.EndsWith(".yaml", StringComparison.Ordinal) || lower.EndsWith(".yml", StringComparison.Ordinal)) + { + return PolicyDocumentFormat.Yaml; + } + + return PolicyDocumentFormat.Json; + } +} + +public sealed record PolicyRule( + string Name, + string? Identifier, + string? Description, + PolicyAction Action, + ImmutableArray Severities, + ImmutableArray Environments, + ImmutableArray Sources, + ImmutableArray Vendors, + ImmutableArray Licenses, + ImmutableArray Tags, + PolicyRuleMatchCriteria Match, + DateTimeOffset? Expires, + string? Justification, + ImmutableDictionary Metadata) +{ + public static PolicyRuleMatchCriteria EmptyMatch { get; } = new( + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty); + + public static PolicyRule Create( + string name, + PolicyAction action, + ImmutableArray severities, + ImmutableArray environments, + ImmutableArray sources, + ImmutableArray vendors, + ImmutableArray licenses, + ImmutableArray tags, + PolicyRuleMatchCriteria match, + DateTimeOffset? expires, + string? justification, + string? identifier = null, + string? description = null, + ImmutableDictionary? metadata = null) + { + metadata ??= ImmutableDictionary.Empty; + return new PolicyRule( + name, + identifier, + description, + action, + severities, + environments, + sources, + vendors, + licenses, + tags, + match, + expires, + justification, + metadata); + } + + public bool MatchesAnyEnvironment => Environments.IsDefaultOrEmpty; +} + +public sealed record PolicyRuleMatchCriteria( + ImmutableArray Images, + ImmutableArray Repositories, + ImmutableArray Packages, + ImmutableArray Purls, + ImmutableArray Cves, + ImmutableArray Paths, + ImmutableArray LayerDigests, + ImmutableArray UsedByEntrypoint) +{ + public static PolicyRuleMatchCriteria Create( + ImmutableArray images, + ImmutableArray repositories, + ImmutableArray packages, + ImmutableArray purls, + ImmutableArray cves, + ImmutableArray paths, + ImmutableArray layerDigests, + ImmutableArray usedByEntrypoint) + => new( + images, + repositories, + packages, + purls, + cves, + paths, + layerDigests, + usedByEntrypoint); + + public static PolicyRuleMatchCriteria Empty { get; } = new( + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty, + ImmutableArray.Empty); + + public bool IsEmpty => + Images.IsDefaultOrEmpty && + Repositories.IsDefaultOrEmpty && + Packages.IsDefaultOrEmpty && + Purls.IsDefaultOrEmpty && + Cves.IsDefaultOrEmpty && + Paths.IsDefaultOrEmpty && + LayerDigests.IsDefaultOrEmpty && + UsedByEntrypoint.IsDefaultOrEmpty; +} + +public sealed record PolicyAction( + PolicyActionType Type, + PolicyIgnoreOptions? Ignore, + PolicyEscalateOptions? Escalate, + PolicyRequireVexOptions? RequireVex, + bool Quiet); + +public enum PolicyActionType +{ + Block, + Ignore, + Warn, + Defer, + Escalate, + RequireVex, +} + +public sealed record PolicyIgnoreOptions(DateTimeOffset? Until, string? Justification); + +public sealed record PolicyEscalateOptions( + PolicySeverity? MinimumSeverity, + bool RequireKev, + double? MinimumEpss); + +public sealed record PolicyRequireVexOptions( + ImmutableArray Vendors, + ImmutableArray Justifications); + +public enum PolicySeverity +{ + Critical, + High, + Medium, + Low, + Informational, + None, + Unknown, +} diff --git a/src/StellaOps.Policy/PolicyEvaluation.cs b/src/StellaOps.Policy/PolicyEvaluation.cs new file mode 100644 index 00000000..6829471a --- /dev/null +++ b/src/StellaOps.Policy/PolicyEvaluation.cs @@ -0,0 +1,552 @@ +using System; +using System.Collections.Immutable; +using System.Globalization; + +namespace StellaOps.Policy; + +public static class PolicyEvaluation +{ + public static PolicyVerdict EvaluateFinding(PolicyDocument document, PolicyScoringConfig scoringConfig, PolicyFinding finding) + { + if (document is null) + { + throw new ArgumentNullException(nameof(document)); + } + + if (scoringConfig is null) + { + throw new ArgumentNullException(nameof(scoringConfig)); + } + + if (finding is null) + { + throw new ArgumentNullException(nameof(finding)); + } + + var severityWeight = scoringConfig.SeverityWeights.TryGetValue(finding.Severity, out var weight) + ? weight + : scoringConfig.SeverityWeights.GetValueOrDefault(PolicySeverity.Unknown, 0); + var trustKey = ResolveTrustKey(finding); + var trustWeight = ResolveTrustWeight(scoringConfig, trustKey); + var reachabilityKey = ResolveReachabilityKey(finding); + var reachabilityWeight = ResolveReachabilityWeight(scoringConfig, reachabilityKey, out var resolvedReachabilityKey); + var baseScore = severityWeight * trustWeight * reachabilityWeight; + var components = new ScoringComponents( + severityWeight, + trustWeight, + reachabilityWeight, + baseScore, + trustKey, + resolvedReachabilityKey); + var unknownConfidence = ComputeUnknownConfidence(scoringConfig.UnknownConfidence, finding); + + foreach (var rule in document.Rules) + { + if (!RuleMatches(rule, finding)) + { + continue; + } + + return BuildVerdict(rule, finding, scoringConfig, components, unknownConfidence); + } + + var baseline = PolicyVerdict.CreateBaseline(finding.FindingId, scoringConfig); + return ApplyUnknownConfidence(baseline, unknownConfidence); + } + + private static PolicyVerdict BuildVerdict( + PolicyRule rule, + PolicyFinding finding, + PolicyScoringConfig config, + ScoringComponents components, + UnknownConfidenceResult? unknownConfidence) + { + var action = rule.Action; + var status = MapAction(action); + var notes = BuildNotes(action); + var inputs = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + inputs["severityWeight"] = components.SeverityWeight; + inputs["trustWeight"] = components.TrustWeight; + inputs["reachabilityWeight"] = components.ReachabilityWeight; + inputs["baseScore"] = components.BaseScore; + if (!string.IsNullOrWhiteSpace(components.TrustKey)) + { + inputs[$"trustWeight.{components.TrustKey}"] = components.TrustWeight; + } + if (!string.IsNullOrWhiteSpace(components.ReachabilityKey)) + { + inputs[$"reachability.{components.ReachabilityKey}"] = components.ReachabilityWeight; + } + if (unknownConfidence is { Band.Description: { Length: > 0 } description }) + { + notes = AppendNote(notes, description); + } + if (unknownConfidence is { } unknownDetails) + { + inputs["unknownConfidence"] = unknownDetails.Confidence; + inputs["unknownAgeDays"] = unknownDetails.AgeDays; + } + + double score = components.BaseScore; + string? quietedBy = null; + var quiet = false; + + var quietRequested = action.Quiet; + var quietAllowed = quietRequested && (action.RequireVex is not null || action.Type == PolicyActionType.RequireVex); + + if (quietRequested && !quietAllowed) + { + var warnInputs = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var pair in inputs) + { + warnInputs[pair.Key] = pair.Value; + } + if (unknownConfidence is { } unknownInfo) + { + warnInputs["unknownConfidence"] = unknownInfo.Confidence; + warnInputs["unknownAgeDays"] = unknownInfo.AgeDays; + } + + var warnPenalty = config.WarnPenalty; + warnInputs["warnPenalty"] = warnPenalty; + var warnScore = Math.Max(0, components.BaseScore - warnPenalty); + var warnNotes = AppendNote(notes, "Quiet flag ignored: rule must specify requireVex justifications."); + + return new PolicyVerdict( + finding.FindingId, + PolicyVerdictStatus.Warned, + rule.Name, + action.Type.ToString(), + warnNotes, + warnScore, + config.Version, + warnInputs.ToImmutable(), + QuietedBy: null, + Quiet: false, + UnknownConfidence: unknownConfidence?.Confidence, + ConfidenceBand: unknownConfidence?.Band.Name, + UnknownAgeDays: unknownConfidence?.AgeDays, + SourceTrust: components.TrustKey, + Reachability: components.ReachabilityKey); + } + + switch (status) + { + case PolicyVerdictStatus.Ignored: + score = ApplyPenalty(score, config.IgnorePenalty, inputs, "ignorePenalty"); + break; + case PolicyVerdictStatus.Warned: + score = ApplyPenalty(score, config.WarnPenalty, inputs, "warnPenalty"); + break; + case PolicyVerdictStatus.Deferred: + var deferPenalty = config.WarnPenalty / 2; + score = ApplyPenalty(score, deferPenalty, inputs, "deferPenalty"); + break; + } + + if (quietAllowed) + { + score = ApplyPenalty(score, config.QuietPenalty, inputs, "quietPenalty"); + quietedBy = rule.Name; + quiet = true; + } + + return new PolicyVerdict( + finding.FindingId, + status, + rule.Name, + action.Type.ToString(), + notes, + score, + config.Version, + inputs.ToImmutable(), + quietedBy, + quiet, + unknownConfidence?.Confidence, + unknownConfidence?.Band.Name, + unknownConfidence?.AgeDays, + components.TrustKey, + components.ReachabilityKey); + } + + private static double ApplyPenalty(double score, double penalty, ImmutableDictionary.Builder inputs, string key) + { + if (penalty <= 0) + { + return score; + } + + inputs[key] = penalty; + return Math.Max(0, score - penalty); + } + + private static PolicyVerdict ApplyUnknownConfidence(PolicyVerdict verdict, UnknownConfidenceResult? unknownConfidence) + { + if (unknownConfidence is null) + { + return verdict; + } + + var inputsBuilder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var pair in verdict.GetInputs()) + { + inputsBuilder[pair.Key] = pair.Value; + } + + inputsBuilder["unknownConfidence"] = unknownConfidence.Value.Confidence; + inputsBuilder["unknownAgeDays"] = unknownConfidence.Value.AgeDays; + + return verdict with + { + Inputs = inputsBuilder.ToImmutable(), + UnknownConfidence = unknownConfidence.Value.Confidence, + ConfidenceBand = unknownConfidence.Value.Band.Name, + UnknownAgeDays = unknownConfidence.Value.AgeDays, + }; + } + + private static UnknownConfidenceResult? ComputeUnknownConfidence(PolicyUnknownConfidenceConfig config, PolicyFinding finding) + { + if (!IsUnknownFinding(finding)) + { + return null; + } + + var ageDays = ResolveUnknownAgeDays(finding); + var rawConfidence = config.Initial - (ageDays * config.DecayPerDay); + var confidence = config.Clamp(rawConfidence); + var band = config.ResolveBand(confidence); + return new UnknownConfidenceResult(ageDays, confidence, band); + } + + private static bool IsUnknownFinding(PolicyFinding finding) + { + if (finding.Severity == PolicySeverity.Unknown) + { + return true; + } + + if (!finding.Tags.IsDefaultOrEmpty) + { + foreach (var tag in finding.Tags) + { + if (string.Equals(tag, "state:unknown", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + } + + return false; + } + + private static double ResolveUnknownAgeDays(PolicyFinding finding) + { + var ageTag = TryGetTagValue(finding.Tags, "unknown-age-days:"); + if (!string.IsNullOrWhiteSpace(ageTag) && + double.TryParse(ageTag, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsedAge) && + parsedAge >= 0) + { + return parsedAge; + } + + var sinceTag = TryGetTagValue(finding.Tags, "unknown-since:"); + if (string.IsNullOrWhiteSpace(sinceTag)) + { + return 0; + } + + if (!DateTimeOffset.TryParse(sinceTag, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var since)) + { + return 0; + } + + var observedTag = TryGetTagValue(finding.Tags, "observed-at:"); + if (!string.IsNullOrWhiteSpace(observedTag) && + DateTimeOffset.TryParse(observedTag, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var observed) && + observed > since) + { + return Math.Max(0, (observed - since).TotalDays); + } + + return 0; + } + + private static string? ResolveTrustKey(PolicyFinding finding) + { + if (!finding.Tags.IsDefaultOrEmpty) + { + var tagged = TryGetTagValue(finding.Tags, "trust:"); + if (!string.IsNullOrWhiteSpace(tagged)) + { + return tagged; + } + } + + if (!string.IsNullOrWhiteSpace(finding.Source)) + { + return finding.Source; + } + + if (!string.IsNullOrWhiteSpace(finding.Vendor)) + { + return finding.Vendor; + } + + return null; + } + + private static double ResolveTrustWeight(PolicyScoringConfig config, string? key) + { + if (string.IsNullOrWhiteSpace(key) || config.TrustOverrides.IsEmpty) + { + return 1.0; + } + + return config.TrustOverrides.TryGetValue(key, out var weight) ? weight : 1.0; + } + + private static string? ResolveReachabilityKey(PolicyFinding finding) + { + if (finding.Tags.IsDefaultOrEmpty) + { + return null; + } + + var reachability = TryGetTagValue(finding.Tags, "reachability:"); + if (!string.IsNullOrWhiteSpace(reachability)) + { + return reachability; + } + + var usage = TryGetTagValue(finding.Tags, "usage:"); + if (!string.IsNullOrWhiteSpace(usage)) + { + return usage; + } + + return null; + } + + private static double ResolveReachabilityWeight(PolicyScoringConfig config, string? key, out string? resolvedKey) + { + if (!string.IsNullOrWhiteSpace(key) && config.ReachabilityBuckets.TryGetValue(key, out var weight)) + { + resolvedKey = key; + return weight; + } + + if (config.ReachabilityBuckets.TryGetValue("unknown", out var unknownWeight)) + { + resolvedKey = "unknown"; + return unknownWeight; + } + + resolvedKey = key; + return 1.0; + } + + private static string? TryGetTagValue(ImmutableArray tags, string prefix) + { + if (tags.IsDefaultOrEmpty) + { + return null; + } + + foreach (var tag in tags) + { + if (string.IsNullOrWhiteSpace(tag)) + { + continue; + } + + if (tag.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + var value = tag[prefix.Length..].Trim(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } + } + } + + return null; + } + + private readonly record struct ScoringComponents( + double SeverityWeight, + double TrustWeight, + double ReachabilityWeight, + double BaseScore, + string? TrustKey, + string? ReachabilityKey); + + private readonly struct UnknownConfidenceResult + { + public UnknownConfidenceResult(double ageDays, double confidence, PolicyUnknownConfidenceBand band) + { + AgeDays = ageDays; + Confidence = confidence; + Band = band; + } + + public double AgeDays { get; } + + public double Confidence { get; } + + public PolicyUnknownConfidenceBand Band { get; } + } + + private static bool RuleMatches(PolicyRule rule, PolicyFinding finding) + { + if (!rule.Severities.IsDefaultOrEmpty && !rule.Severities.Contains(finding.Severity)) + { + return false; + } + + if (!Matches(rule.Environments, finding.Environment)) + { + return false; + } + + if (!Matches(rule.Sources, finding.Source)) + { + return false; + } + + if (!Matches(rule.Vendors, finding.Vendor)) + { + return false; + } + + if (!Matches(rule.Licenses, finding.License)) + { + return false; + } + + if (!RuleMatchCriteria(rule.Match, finding)) + { + return false; + } + + return true; + } + + private static bool Matches(ImmutableArray ruleValues, string? candidate) + { + if (ruleValues.IsDefaultOrEmpty) + { + return true; + } + + if (string.IsNullOrWhiteSpace(candidate)) + { + return false; + } + + return ruleValues.Contains(candidate, StringComparer.OrdinalIgnoreCase); + } + + private static bool RuleMatchCriteria(PolicyRuleMatchCriteria criteria, PolicyFinding finding) + { + if (!criteria.Images.IsDefaultOrEmpty && !ContainsValue(criteria.Images, finding.Image, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.Repositories.IsDefaultOrEmpty && !ContainsValue(criteria.Repositories, finding.Repository, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.Packages.IsDefaultOrEmpty && !ContainsValue(criteria.Packages, finding.Package, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.Purls.IsDefaultOrEmpty && !ContainsValue(criteria.Purls, finding.Purl, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.Cves.IsDefaultOrEmpty && !ContainsValue(criteria.Cves, finding.Cve, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.Paths.IsDefaultOrEmpty && !ContainsValue(criteria.Paths, finding.Path, StringComparer.Ordinal)) + { + return false; + } + + if (!criteria.LayerDigests.IsDefaultOrEmpty && !ContainsValue(criteria.LayerDigests, finding.LayerDigest, StringComparer.OrdinalIgnoreCase)) + { + return false; + } + + if (!criteria.UsedByEntrypoint.IsDefaultOrEmpty) + { + var match = false; + foreach (var tag in criteria.UsedByEntrypoint) + { + if (finding.Tags.Contains(tag, StringComparer.OrdinalIgnoreCase)) + { + match = true; + break; + } + } + + if (!match) + { + return false; + } + } + + return true; + } + + private static bool ContainsValue(ImmutableArray values, string? candidate, StringComparer comparer) + { + if (values.IsDefaultOrEmpty) + { + return true; + } + + if (string.IsNullOrWhiteSpace(candidate)) + { + return false; + } + + return values.Contains(candidate, comparer); + } + + private static PolicyVerdictStatus MapAction(PolicyAction action) + => action.Type switch + { + PolicyActionType.Block => PolicyVerdictStatus.Blocked, + PolicyActionType.Ignore => PolicyVerdictStatus.Ignored, + PolicyActionType.Warn => PolicyVerdictStatus.Warned, + PolicyActionType.Defer => PolicyVerdictStatus.Deferred, + PolicyActionType.Escalate => PolicyVerdictStatus.Escalated, + PolicyActionType.RequireVex => PolicyVerdictStatus.RequiresVex, + _ => PolicyVerdictStatus.Pass, + }; + + private static string? BuildNotes(PolicyAction action) + { + if (action.Ignore is { } ignore && !string.IsNullOrWhiteSpace(ignore.Justification)) + { + return ignore.Justification; + } + + if (action.Escalate is { } escalate && escalate.MinimumSeverity is { } severity) + { + return $"Escalate >= {severity}"; + } + + return null; + } + + private static string? AppendNote(string? existing, string addition) + => string.IsNullOrWhiteSpace(existing) ? addition : string.Concat(existing, " | ", addition); +} diff --git a/src/StellaOps.Policy/PolicyFinding.cs b/src/StellaOps.Policy/PolicyFinding.cs new file mode 100644 index 00000000..4d51966c --- /dev/null +++ b/src/StellaOps.Policy/PolicyFinding.cs @@ -0,0 +1,51 @@ +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public sealed record PolicyFinding( + string FindingId, + PolicySeverity Severity, + string? Environment, + string? Source, + string? Vendor, + string? License, + string? Image, + string? Repository, + string? Package, + string? Purl, + string? Cve, + string? Path, + string? LayerDigest, + ImmutableArray Tags) +{ + public static PolicyFinding Create( + string findingId, + PolicySeverity severity, + string? environment = null, + string? source = null, + string? vendor = null, + string? license = null, + string? image = null, + string? repository = null, + string? package = null, + string? purl = null, + string? cve = null, + string? path = null, + string? layerDigest = null, + ImmutableArray? tags = null) + => new( + findingId, + severity, + environment, + source, + vendor, + license, + image, + repository, + package, + purl, + cve, + path, + layerDigest, + tags ?? ImmutableArray.Empty); +} diff --git a/src/StellaOps.Policy/PolicyIssue.cs b/src/StellaOps.Policy/PolicyIssue.cs new file mode 100644 index 00000000..0d889576 --- /dev/null +++ b/src/StellaOps.Policy/PolicyIssue.cs @@ -0,0 +1,28 @@ +using System; + +namespace StellaOps.Policy; + +/// +/// Represents a validation or normalization issue discovered while processing a policy document. +/// +public sealed record PolicyIssue(string Code, string Message, PolicyIssueSeverity Severity, string Path) +{ + public static PolicyIssue Error(string code, string message, string path) + => new(code, message, PolicyIssueSeverity.Error, path); + + public static PolicyIssue Warning(string code, string message, string path) + => new(code, message, PolicyIssueSeverity.Warning, path); + + public static PolicyIssue Info(string code, string message, string path) + => new(code, message, PolicyIssueSeverity.Info, path); + + public PolicyIssue EnsurePath(string fallbackPath) + => string.IsNullOrWhiteSpace(Path) ? this with { Path = fallbackPath } : this; +} + +public enum PolicyIssueSeverity +{ + Error, + Warning, + Info, +} diff --git a/src/StellaOps.Policy/PolicyPreviewModels.cs b/src/StellaOps.Policy/PolicyPreviewModels.cs new file mode 100644 index 00000000..3f817330 --- /dev/null +++ b/src/StellaOps.Policy/PolicyPreviewModels.cs @@ -0,0 +1,18 @@ +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public sealed record PolicyPreviewRequest( + string ImageDigest, + ImmutableArray Findings, + ImmutableArray BaselineVerdicts, + PolicySnapshot? SnapshotOverride = null, + PolicySnapshotContent? ProposedPolicy = null); + +public sealed record PolicyPreviewResponse( + bool Success, + string PolicyDigest, + string? RevisionId, + ImmutableArray Issues, + ImmutableArray Diffs, + int ChangedCount); diff --git a/src/StellaOps.Policy/PolicyPreviewService.cs b/src/StellaOps.Policy/PolicyPreviewService.cs new file mode 100644 index 00000000..3b81270d --- /dev/null +++ b/src/StellaOps.Policy/PolicyPreviewService.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Policy; + +public sealed class PolicyPreviewService +{ + private readonly PolicySnapshotStore _snapshotStore; + private readonly ILogger _logger; + + public PolicyPreviewService(PolicySnapshotStore snapshotStore, ILogger logger) + { + _snapshotStore = snapshotStore ?? throw new ArgumentNullException(nameof(snapshotStore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PreviewAsync(PolicyPreviewRequest request, CancellationToken cancellationToken = default) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + var (snapshot, bindingIssues) = await ResolveSnapshotAsync(request, cancellationToken).ConfigureAwait(false); + if (snapshot is null) + { + _logger.LogWarning("Policy preview failed: snapshot unavailable or validation errors. Issues={Count}", bindingIssues.Length); + return new PolicyPreviewResponse(false, string.Empty, null, bindingIssues, ImmutableArray.Empty, 0); + } + + var projected = Evaluate(snapshot.Document, snapshot.ScoringConfig, request.Findings); + var baseline = BuildBaseline(request.BaselineVerdicts, projected, snapshot.ScoringConfig); + var diffs = BuildDiffs(baseline, projected); + var changed = diffs.Count(static diff => diff.Changed); + + _logger.LogDebug("Policy preview computed for {ImageDigest}. Changed={Changed}", request.ImageDigest, changed); + + return new PolicyPreviewResponse(true, snapshot.Digest, snapshot.RevisionId, bindingIssues, diffs, changed); + } + + private async Task<(PolicySnapshot? Snapshot, ImmutableArray Issues)> ResolveSnapshotAsync(PolicyPreviewRequest request, CancellationToken cancellationToken) + { + if (request.ProposedPolicy is not null) + { + var binding = PolicyBinder.Bind(request.ProposedPolicy.Content, request.ProposedPolicy.Format); + if (!binding.Success) + { + return (null, binding.Issues); + } + + var digest = PolicyDigest.Compute(binding.Document); + var snapshot = new PolicySnapshot( + request.SnapshotOverride?.RevisionNumber + 1 ?? 0, + request.SnapshotOverride?.RevisionId ?? "preview", + digest, + DateTimeOffset.UtcNow, + request.ProposedPolicy.Actor, + request.ProposedPolicy.Format, + binding.Document, + binding.Issues, + PolicyScoringConfig.Default); + + return (snapshot, binding.Issues); + } + + if (request.SnapshotOverride is not null) + { + return (request.SnapshotOverride, ImmutableArray.Empty); + } + + var latest = await _snapshotStore.GetLatestAsync(cancellationToken).ConfigureAwait(false); + if (latest is not null) + { + return (latest, ImmutableArray.Empty); + } + + return (null, ImmutableArray.Create(PolicyIssue.Error("policy.preview.snapshot_missing", "No policy snapshot is available for preview.", "$"))); + } + + private static ImmutableArray Evaluate(PolicyDocument document, PolicyScoringConfig scoringConfig, ImmutableArray findings) + { + if (findings.IsDefaultOrEmpty) + { + return ImmutableArray.Empty; + } + + var results = ImmutableArray.CreateBuilder(findings.Length); + foreach (var finding in findings) + { + var verdict = PolicyEvaluation.EvaluateFinding(document, scoringConfig, finding); + results.Add(verdict); + } + + return results.ToImmutable(); + } + + private static ImmutableDictionary BuildBaseline(ImmutableArray baseline, ImmutableArray projected, PolicyScoringConfig scoringConfig) + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + if (!baseline.IsDefaultOrEmpty) + { + foreach (var verdict in baseline) + { + if (!string.IsNullOrEmpty(verdict.FindingId) && !builder.ContainsKey(verdict.FindingId)) + { + builder.Add(verdict.FindingId, verdict); + } + } + } + + foreach (var verdict in projected) + { + if (!builder.ContainsKey(verdict.FindingId)) + { + builder.Add(verdict.FindingId, PolicyVerdict.CreateBaseline(verdict.FindingId, scoringConfig)); + } + } + + return builder.ToImmutable(); + } + + private static ImmutableArray BuildDiffs(ImmutableDictionary baseline, ImmutableArray projected) + { + var diffs = ImmutableArray.CreateBuilder(projected.Length); + foreach (var verdict in projected.OrderBy(static v => v.FindingId, StringComparer.Ordinal)) + { + var baseVerdict = baseline.TryGetValue(verdict.FindingId, out var existing) + ? existing + : new PolicyVerdict(verdict.FindingId, PolicyVerdictStatus.Pass); + + diffs.Add(new PolicyVerdictDiff(baseVerdict, verdict)); + } + + return diffs.ToImmutable(); + } +} diff --git a/src/StellaOps.Policy/PolicySchemaResource.cs b/src/StellaOps.Policy/PolicySchemaResource.cs new file mode 100644 index 00000000..bbea1749 --- /dev/null +++ b/src/StellaOps.Policy/PolicySchemaResource.cs @@ -0,0 +1,30 @@ +using System; +using System.IO; +using System.Reflection; +using System.Text; + +namespace StellaOps.Policy; + +public static class PolicySchemaResource +{ + private const string SchemaResourceName = "StellaOps.Policy.Schemas.policy-schema@1.json"; + + public static Stream OpenSchemaStream() + { + var assembly = Assembly.GetExecutingAssembly(); + var stream = assembly.GetManifestResourceStream(SchemaResourceName); + if (stream is null) + { + throw new InvalidOperationException($"Unable to locate embedded schema resource '{SchemaResourceName}'."); + } + + return stream; + } + + public static string ReadSchemaJson() + { + using var stream = OpenSchemaStream(); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + return reader.ReadToEnd(); + } +} diff --git a/src/StellaOps.Policy/PolicyScoringConfig.cs b/src/StellaOps.Policy/PolicyScoringConfig.cs new file mode 100644 index 00000000..de3a1873 --- /dev/null +++ b/src/StellaOps.Policy/PolicyScoringConfig.cs @@ -0,0 +1,18 @@ +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public sealed record PolicyScoringConfig( + string Version, + ImmutableDictionary SeverityWeights, + double QuietPenalty, + double WarnPenalty, + double IgnorePenalty, + ImmutableDictionary TrustOverrides, + ImmutableDictionary ReachabilityBuckets, + PolicyUnknownConfidenceConfig UnknownConfidence) +{ + public static string BaselineVersion => "1.0"; + + public static PolicyScoringConfig Default { get; } = PolicyScoringConfigBinder.LoadDefault(); +} diff --git a/src/StellaOps.Policy/PolicyScoringConfigBinder.cs b/src/StellaOps.Policy/PolicyScoringConfigBinder.cs new file mode 100644 index 00000000..26ff0919 --- /dev/null +++ b/src/StellaOps.Policy/PolicyScoringConfigBinder.cs @@ -0,0 +1,603 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using Json.Schema; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Policy; + +public sealed record PolicyScoringBindingResult( + bool Success, + PolicyScoringConfig? Config, + ImmutableArray Issues); + +public static class PolicyScoringConfigBinder +{ + private const string DefaultResourceName = "StellaOps.Policy.Schemas.policy-scoring-default.json"; + + private static readonly JsonSchema ScoringSchema = PolicyScoringSchema.Schema; + + private static readonly ImmutableDictionary DefaultReachabilityBuckets = CreateDefaultReachabilityBuckets(); + + private static readonly PolicyUnknownConfidenceConfig DefaultUnknownConfidence = CreateDefaultUnknownConfidence(); + + private static readonly IDeserializer YamlDeserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + public static PolicyScoringConfig LoadDefault() + { + var assembly = Assembly.GetExecutingAssembly(); + using var stream = assembly.GetManifestResourceStream(DefaultResourceName) + ?? throw new InvalidOperationException($"Embedded resource '{DefaultResourceName}' not found."); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + var json = reader.ReadToEnd(); + var binding = Bind(json, PolicyDocumentFormat.Json); + if (!binding.Success || binding.Config is null) + { + throw new InvalidOperationException("Failed to load default policy scoring configuration."); + } + + return binding.Config; + } + + public static PolicyScoringBindingResult Bind(string content, PolicyDocumentFormat format) + { + if (string.IsNullOrWhiteSpace(content)) + { + var issue = PolicyIssue.Error("scoring.empty", "Scoring configuration content is empty.", "$"); + return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue)); + } + + try + { + var root = Parse(content, format); + if (root is not JsonObject obj) + { + var issue = PolicyIssue.Error("scoring.invalid", "Scoring configuration must be a JSON object.", "$"); + return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue)); + } + + var issues = ImmutableArray.CreateBuilder(); + var schemaIssues = ValidateAgainstSchema(root); + issues.AddRange(schemaIssues); + if (schemaIssues.Any(static issue => issue.Severity == PolicyIssueSeverity.Error)) + { + return new PolicyScoringBindingResult(false, null, issues.ToImmutable()); + } + + var config = BuildConfig(obj, issues); + var hasErrors = issues.Any(issue => issue.Severity == PolicyIssueSeverity.Error); + return new PolicyScoringBindingResult(!hasErrors, config, issues.ToImmutable()); + } + catch (JsonException ex) + { + var issue = PolicyIssue.Error("scoring.parse.json", $"Failed to parse scoring JSON: {ex.Message}", "$"); + return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue)); + } + catch (YamlDotNet.Core.YamlException ex) + { + var issue = PolicyIssue.Error("scoring.parse.yaml", $"Failed to parse scoring YAML: {ex.Message}", "$"); + return new PolicyScoringBindingResult(false, null, ImmutableArray.Create(issue)); + } + } + + private static JsonNode? Parse(string content, PolicyDocumentFormat format) + { + return format switch + { + PolicyDocumentFormat.Json => JsonNode.Parse(content, new JsonNodeOptions { PropertyNameCaseInsensitive = true }), + PolicyDocumentFormat.Yaml => ConvertYamlToJsonNode(content), + _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Unsupported scoring configuration format."), + }; + } + + private static JsonNode? ConvertYamlToJsonNode(string content) + { + var yamlObject = YamlDeserializer.Deserialize(content); + return PolicyBinderUtilities.ConvertYamlObject(yamlObject); + } + + private static ImmutableArray ValidateAgainstSchema(JsonNode root) + { + try + { + using var document = JsonDocument.Parse(root.ToJsonString(new JsonSerializerOptions + { + WriteIndented = false, + })); + + var result = ScoringSchema.Evaluate(document.RootElement, new EvaluationOptions + { + OutputFormat = OutputFormat.List, + RequireFormatValidation = true, + }); + + if (result.IsValid) + { + return ImmutableArray.Empty; + } + + var issues = ImmutableArray.CreateBuilder(); + var seen = new HashSet(StringComparer.Ordinal); + CollectSchemaIssues(result, issues, seen); + return issues.ToImmutable(); + } + catch (JsonException ex) + { + return ImmutableArray.Create(PolicyIssue.Error("scoring.schema.normalize", $"Failed to normalize scoring configuration for schema validation: {ex.Message}", "$")); + } + } + + private static void CollectSchemaIssues(EvaluationResults result, ImmutableArray.Builder issues, HashSet seen) + { + if (result.Errors is { Count: > 0 }) + { + foreach (var pair in result.Errors) + { + var keyword = SanitizeKeyword(pair.Key); + var path = ConvertPointerToPath(result.InstanceLocation?.ToString() ?? "#"); + var message = pair.Value ?? "Schema violation."; + var key = $"{path}|{keyword}|{message}"; + if (seen.Add(key)) + { + issues.Add(PolicyIssue.Error($"scoring.schema.{keyword}", message, path)); + } + } + } + + if (result.Details is null) + { + return; + } + + foreach (var detail in result.Details) + { + CollectSchemaIssues(detail, issues, seen); + } + } + + private static string ConvertPointerToPath(string pointer) + { + if (string.IsNullOrEmpty(pointer) || pointer == "#") + { + return "$"; + } + + if (pointer[0] == '#') + { + pointer = pointer.Length > 1 ? pointer[1..] : string.Empty; + } + + if (string.IsNullOrEmpty(pointer)) + { + return "$"; + } + + var segments = pointer.Split('/', StringSplitOptions.RemoveEmptyEntries); + var builder = new StringBuilder("$"); + foreach (var segment in segments) + { + var unescaped = segment.Replace("~1", "/").Replace("~0", "~"); + if (int.TryParse(unescaped, out var index)) + { + builder.Append('[').Append(index).Append(']'); + } + else + { + builder.Append('.').Append(unescaped); + } + } + + return builder.ToString(); + } + + private static string SanitizeKeyword(string keyword) + { + if (string.IsNullOrWhiteSpace(keyword)) + { + return "unknown"; + } + + var builder = new StringBuilder(keyword.Length); + foreach (var ch in keyword) + { + if (char.IsLetterOrDigit(ch)) + { + builder.Append(char.ToLowerInvariant(ch)); + } + else if (ch is '.' or '_' or '-') + { + builder.Append(ch); + } + else + { + builder.Append('_'); + } + } + + return builder.Length == 0 ? "unknown" : builder.ToString(); + } + + private static PolicyScoringConfig BuildConfig(JsonObject obj, ImmutableArray.Builder issues) + { + var version = ReadString(obj, "version", issues, required: true) ?? PolicyScoringConfig.BaselineVersion; + + var severityWeights = ReadSeverityWeights(obj, issues); + var quietPenalty = ReadDouble(obj, "quietPenalty", issues, defaultValue: 45); + var warnPenalty = ReadDouble(obj, "warnPenalty", issues, defaultValue: 15); + var ignorePenalty = ReadDouble(obj, "ignorePenalty", issues, defaultValue: 35); + var trustOverrides = ReadTrustOverrides(obj, issues); + var reachabilityBuckets = ReadReachabilityBuckets(obj, issues); + var unknownConfidence = ReadUnknownConfidence(obj, issues); + + return new PolicyScoringConfig( + version, + severityWeights, + quietPenalty, + warnPenalty, + ignorePenalty, + trustOverrides, + reachabilityBuckets, + unknownConfidence); + } + + private static ImmutableDictionary CreateDefaultReachabilityBuckets() + { + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + builder["entrypoint"] = 1.0; + builder["direct"] = 0.85; + builder["indirect"] = 0.6; + builder["runtime"] = 0.45; + builder["unreachable"] = 0.25; + builder["unknown"] = 0.5; + return builder.ToImmutable(); + } + + private static PolicyUnknownConfidenceConfig CreateDefaultUnknownConfidence() + { + var bands = ImmutableArray.Create( + new PolicyUnknownConfidenceBand("high", 0.65, "Fresh unknowns with recent telemetry."), + new PolicyUnknownConfidenceBand("medium", 0.35, "Unknowns aging toward action required."), + new PolicyUnknownConfidenceBand("low", 0.0, "Stale unknowns that must be triaged.")); + return new PolicyUnknownConfidenceConfig(0.8, 0.05, 0.2, bands); + } + + private static ImmutableDictionary ReadReachabilityBuckets(JsonObject obj, ImmutableArray.Builder issues) + { + if (!obj.TryGetPropertyValue("reachabilityBuckets", out var node)) + { + issues.Add(PolicyIssue.Warning("scoring.reachability.default", "reachabilityBuckets not specified; defaulting to baseline weights.", "$.reachabilityBuckets")); + return DefaultReachabilityBuckets; + } + + if (node is not JsonObject bucketsObj) + { + issues.Add(PolicyIssue.Error("scoring.reachability.type", "reachabilityBuckets must be an object.", "$.reachabilityBuckets")); + return DefaultReachabilityBuckets; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var pair in bucketsObj) + { + if (pair.Value is null) + { + issues.Add(PolicyIssue.Warning("scoring.reachability.null", $"Bucket '{pair.Key}' is null; defaulting to 0.", $"$.reachabilityBuckets.{pair.Key}")); + builder[pair.Key] = 0; + continue; + } + + var value = ExtractDouble(pair.Value, issues, $"$.reachabilityBuckets.{pair.Key}"); + builder[pair.Key] = value; + } + + if (builder.Count == 0) + { + issues.Add(PolicyIssue.Warning("scoring.reachability.empty", "No reachability buckets defined; using defaults.", "$.reachabilityBuckets")); + return DefaultReachabilityBuckets; + } + + return builder.ToImmutable(); + } + + private static PolicyUnknownConfidenceConfig ReadUnknownConfidence(JsonObject obj, ImmutableArray.Builder issues) + { + if (!obj.TryGetPropertyValue("unknownConfidence", out var node)) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.default", "unknownConfidence not specified; defaulting to baseline decay settings.", "$.unknownConfidence")); + return DefaultUnknownConfidence; + } + + if (node is not JsonObject configObj) + { + issues.Add(PolicyIssue.Error("scoring.unknown.type", "unknownConfidence must be an object.", "$.unknownConfidence")); + return DefaultUnknownConfidence; + } + + var initial = DefaultUnknownConfidence.Initial; + if (configObj.TryGetPropertyValue("initial", out var initialNode)) + { + initial = ExtractDouble(initialNode, issues, "$.unknownConfidence.initial"); + } + else + { + issues.Add(PolicyIssue.Warning("scoring.unknown.initial.default", "initial not specified; using baseline value.", "$.unknownConfidence.initial")); + } + + var decay = DefaultUnknownConfidence.DecayPerDay; + if (configObj.TryGetPropertyValue("decayPerDay", out var decayNode)) + { + decay = ExtractDouble(decayNode, issues, "$.unknownConfidence.decayPerDay"); + } + else + { + issues.Add(PolicyIssue.Warning("scoring.unknown.decay.default", "decayPerDay not specified; using baseline value.", "$.unknownConfidence.decayPerDay")); + } + + var floor = DefaultUnknownConfidence.Floor; + if (configObj.TryGetPropertyValue("floor", out var floorNode)) + { + floor = ExtractDouble(floorNode, issues, "$.unknownConfidence.floor"); + } + else + { + issues.Add(PolicyIssue.Warning("scoring.unknown.floor.default", "floor not specified; using baseline value.", "$.unknownConfidence.floor")); + } + + var bands = ReadConfidenceBands(configObj, issues); + if (bands.IsDefaultOrEmpty) + { + bands = DefaultUnknownConfidence.Bands; + } + + if (initial < 0 || initial > 1) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.initial.range", "initial confidence should be between 0 and 1. Clamping to valid range.", "$.unknownConfidence.initial")); + initial = Math.Clamp(initial, 0, 1); + } + + if (decay < 0 || decay > 1) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.decay.range", "decayPerDay should be between 0 and 1. Clamping to valid range.", "$.unknownConfidence.decayPerDay")); + decay = Math.Clamp(decay, 0, 1); + } + + if (floor < 0 || floor > 1) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.floor.range", "floor should be between 0 and 1. Clamping to valid range.", "$.unknownConfidence.floor")); + floor = Math.Clamp(floor, 0, 1); + } + + return new PolicyUnknownConfidenceConfig(initial, decay, floor, bands); + } + + private static ImmutableArray ReadConfidenceBands(JsonObject configObj, ImmutableArray.Builder issues) + { + if (!configObj.TryGetPropertyValue("bands", out var node)) + { + return ImmutableArray.Empty; + } + + if (node is not JsonArray array) + { + issues.Add(PolicyIssue.Error("scoring.unknown.bands.type", "unknownConfidence.bands must be an array.", "$.unknownConfidence.bands")); + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(); + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var index = 0; index < array.Count; index++) + { + var element = array[index]; + if (element is not JsonObject bandObj) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.band.type", "Band entry must be an object.", $"$.unknownConfidence.bands[{index}]")); + continue; + } + + string? name = null; + if (bandObj.TryGetPropertyValue("name", out var nameNode) && nameNode is JsonValue nameValue && nameValue.TryGetValue(out string? text)) + { + name = text?.Trim(); + } + + if (string.IsNullOrWhiteSpace(name)) + { + issues.Add(PolicyIssue.Error("scoring.unknown.band.name", "Band entry requires a non-empty 'name'.", $"$.unknownConfidence.bands[{index}].name")); + continue; + } + + if (!seen.Add(name)) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.band.duplicate", $"Duplicate band '{name}' encountered.", $"$.unknownConfidence.bands[{index}].name")); + continue; + } + + if (!bandObj.TryGetPropertyValue("min", out var minNode)) + { + issues.Add(PolicyIssue.Error("scoring.unknown.band.min", $"Band '{name}' is missing 'min'.", $"$.unknownConfidence.bands[{index}].min")); + continue; + } + + var min = ExtractDouble(minNode, issues, $"$.unknownConfidence.bands[{index}].min"); + if (min < 0 || min > 1) + { + issues.Add(PolicyIssue.Warning("scoring.unknown.band.range", $"Band '{name}' min should be between 0 and 1. Clamping to valid range.", $"$.unknownConfidence.bands[{index}].min")); + min = Math.Clamp(min, 0, 1); + } + + string? description = null; + if (bandObj.TryGetPropertyValue("description", out var descriptionNode) && descriptionNode is JsonValue descriptionValue && descriptionValue.TryGetValue(out string? descriptionText)) + { + description = descriptionText?.Trim(); + } + + builder.Add(new PolicyUnknownConfidenceBand(name, min, description)); + } + + if (builder.Count == 0) + { + return ImmutableArray.Empty; + } + + return builder.ToImmutable() + .OrderByDescending(static band => band.Min) + .ToImmutableArray(); + } + + private static ImmutableDictionary ReadSeverityWeights(JsonObject obj, ImmutableArray.Builder issues) + { + if (!obj.TryGetPropertyValue("severityWeights", out var node) || node is not JsonObject severityObj) + { + issues.Add(PolicyIssue.Error("scoring.severityWeights.missing", "severityWeights section is required.", "$.severityWeights")); + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var severity in Enum.GetValues()) + { + var key = severity.ToString(); + if (!severityObj.TryGetPropertyValue(key, out var valueNode)) + { + issues.Add(PolicyIssue.Warning("scoring.severityWeights.default", $"Severity '{key}' not specified; defaulting to 0.", $"$.severityWeights.{key}")); + builder[severity] = 0; + continue; + } + + var value = ExtractDouble(valueNode, issues, $"$.severityWeights.{key}"); + builder[severity] = value; + } + + return builder.ToImmutable(); + } + + private static double ReadDouble(JsonObject obj, string property, ImmutableArray.Builder issues, double defaultValue) + { + if (!obj.TryGetPropertyValue(property, out var node)) + { + issues.Add(PolicyIssue.Warning("scoring.numeric.default", $"{property} not specified; defaulting to {defaultValue:0.##}.", $"$.{property}")); + return defaultValue; + } + + return ExtractDouble(node, issues, $"$.{property}"); + } + + private static double ExtractDouble(JsonNode? node, ImmutableArray.Builder issues, string path) + { + if (node is null) + { + issues.Add(PolicyIssue.Warning("scoring.numeric.null", $"Value at {path} missing; defaulting to 0.", path)); + return 0; + } + + if (node is JsonValue value) + { + if (value.TryGetValue(out double number)) + { + return number; + } + + if (value.TryGetValue(out string? text) && double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out number)) + { + return number; + } + } + + issues.Add(PolicyIssue.Error("scoring.numeric.invalid", $"Value at {path} is not numeric.", path)); + return 0; + } + + private static ImmutableDictionary ReadTrustOverrides(JsonObject obj, ImmutableArray.Builder issues) + { + if (!obj.TryGetPropertyValue("trustOverrides", out var node) || node is not JsonObject trustObj) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.OrdinalIgnoreCase); + foreach (var pair in trustObj) + { + var value = ExtractDouble(pair.Value, issues, $"$.trustOverrides.{pair.Key}"); + builder[pair.Key] = value; + } + + return builder.ToImmutable(); + } + + private static string? ReadString(JsonObject obj, string property, ImmutableArray.Builder issues, bool required) + { + if (!obj.TryGetPropertyValue(property, out var node) || node is null) + { + if (required) + { + issues.Add(PolicyIssue.Error("scoring.string.missing", $"{property} is required.", $"$.{property}")); + } + return null; + } + + if (node is JsonValue value && value.TryGetValue(out string? text)) + { + return text?.Trim(); + } + + issues.Add(PolicyIssue.Error("scoring.string.invalid", $"{property} must be a string.", $"$.{property}")); + return null; + } +} + +internal static class PolicyBinderUtilities +{ + public static JsonNode? ConvertYamlObject(object? value) + { + switch (value) + { + case null: + return null; + case string s when bool.TryParse(s, out var boolValue): + return JsonValue.Create(boolValue); + case string s: + return JsonValue.Create(s); + case bool b: + return JsonValue.Create(b); + case sbyte or byte or short or ushort or int or uint or long or ulong or float or double or decimal: + return JsonValue.Create(Convert.ToDouble(value, CultureInfo.InvariantCulture)); + case IDictionary dictionary: + { + var obj = new JsonObject(); + foreach (DictionaryEntry entry in dictionary) + { + if (entry.Key is null) + { + continue; + } + + obj[entry.Key.ToString()!] = ConvertYamlObject(entry.Value); + } + + return obj; + } + case IEnumerable enumerable: + { + var array = new JsonArray(); + foreach (var item in enumerable) + { + array.Add(ConvertYamlObject(item)); + } + + return array; + } + default: + return JsonValue.Create(value.ToString()); + } + } +} diff --git a/src/StellaOps.Policy/PolicyScoringConfigDigest.cs b/src/StellaOps.Policy/PolicyScoringConfigDigest.cs new file mode 100644 index 00000000..83bcd2da --- /dev/null +++ b/src/StellaOps.Policy/PolicyScoringConfigDigest.cs @@ -0,0 +1,100 @@ +using System; +using System.Buffers; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text.Json; + +namespace StellaOps.Policy; + +public static class PolicyScoringConfigDigest +{ + public static string Compute(PolicyScoringConfig config) + { + ArgumentNullException.ThrowIfNull(config); + + var buffer = new ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions + { + SkipValidation = true, + })) + { + WriteConfig(writer, config); + } + + var hash = SHA256.HashData(buffer.WrittenSpan); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static void WriteConfig(Utf8JsonWriter writer, PolicyScoringConfig config) + { + writer.WriteStartObject(); + writer.WriteString("version", config.Version); + + writer.WritePropertyName("severityWeights"); + writer.WriteStartObject(); + foreach (var severity in Enum.GetValues()) + { + var key = severity.ToString(); + var value = config.SeverityWeights.TryGetValue(severity, out var weight) ? weight : 0; + writer.WriteNumber(key, value); + } + writer.WriteEndObject(); + + writer.WriteNumber("quietPenalty", config.QuietPenalty); + writer.WriteNumber("warnPenalty", config.WarnPenalty); + writer.WriteNumber("ignorePenalty", config.IgnorePenalty); + + if (!config.TrustOverrides.IsEmpty) + { + writer.WritePropertyName("trustOverrides"); + writer.WriteStartObject(); + foreach (var pair in config.TrustOverrides.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)) + { + writer.WriteNumber(pair.Key, pair.Value); + } + writer.WriteEndObject(); + } + + if (!config.ReachabilityBuckets.IsEmpty) + { + writer.WritePropertyName("reachabilityBuckets"); + writer.WriteStartObject(); + foreach (var pair in config.ReachabilityBuckets.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)) + { + writer.WriteNumber(pair.Key, pair.Value); + } + writer.WriteEndObject(); + } + + writer.WritePropertyName("unknownConfidence"); + writer.WriteStartObject(); + writer.WriteNumber("initial", config.UnknownConfidence.Initial); + writer.WriteNumber("decayPerDay", config.UnknownConfidence.DecayPerDay); + writer.WriteNumber("floor", config.UnknownConfidence.Floor); + + if (!config.UnknownConfidence.Bands.IsDefaultOrEmpty) + { + writer.WritePropertyName("bands"); + writer.WriteStartArray(); + foreach (var band in config.UnknownConfidence.Bands + .OrderByDescending(static b => b.Min) + .ThenBy(static b => b.Name, StringComparer.OrdinalIgnoreCase)) + { + writer.WriteStartObject(); + writer.WriteString("name", band.Name); + writer.WriteNumber("min", band.Min); + if (!string.IsNullOrWhiteSpace(band.Description)) + { + writer.WriteString("description", band.Description); + } + writer.WriteEndObject(); + } + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + writer.WriteEndObject(); + writer.Flush(); + } +} diff --git a/src/StellaOps.Policy/PolicyScoringSchema.cs b/src/StellaOps.Policy/PolicyScoringSchema.cs new file mode 100644 index 00000000..531fcf44 --- /dev/null +++ b/src/StellaOps.Policy/PolicyScoringSchema.cs @@ -0,0 +1,27 @@ +using System; +using System.IO; +using System.Reflection; +using System.Text; +using System.Threading; +using Json.Schema; + +namespace StellaOps.Policy; + +public static class PolicyScoringSchema +{ + private const string SchemaResourceName = "StellaOps.Policy.Schemas.policy-scoring-schema@1.json"; + + private static readonly Lazy CachedSchema = new(LoadSchema, LazyThreadSafetyMode.ExecutionAndPublication); + + public static JsonSchema Schema => CachedSchema.Value; + + private static JsonSchema LoadSchema() + { + var assembly = Assembly.GetExecutingAssembly(); + using var stream = assembly.GetManifestResourceStream(SchemaResourceName) + ?? throw new InvalidOperationException($"Embedded resource '{SchemaResourceName}' was not found."); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + var schemaJson = reader.ReadToEnd(); + return JsonSchema.FromText(schemaJson); + } +} diff --git a/src/StellaOps.Policy/PolicySnapshot.cs b/src/StellaOps.Policy/PolicySnapshot.cs new file mode 100644 index 00000000..ed6e5ffa --- /dev/null +++ b/src/StellaOps.Policy/PolicySnapshot.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public sealed record PolicySnapshot( + long RevisionNumber, + string RevisionId, + string Digest, + DateTimeOffset CreatedAt, + string? CreatedBy, + PolicyDocumentFormat Format, + PolicyDocument Document, + ImmutableArray Issues, + PolicyScoringConfig ScoringConfig); + +public sealed record PolicySnapshotContent( + string Content, + PolicyDocumentFormat Format, + string? Actor, + string? Source, + string? Description); + +public sealed record PolicySnapshotSaveResult( + bool Success, + bool Created, + string Digest, + PolicySnapshot? Snapshot, + PolicyBindingResult BindingResult); diff --git a/src/StellaOps.Policy/PolicySnapshotStore.cs b/src/StellaOps.Policy/PolicySnapshotStore.cs new file mode 100644 index 00000000..d101955a --- /dev/null +++ b/src/StellaOps.Policy/PolicySnapshotStore.cs @@ -0,0 +1,101 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Policy; + +public sealed class PolicySnapshotStore +{ + private readonly IPolicySnapshotRepository _snapshotRepository; + private readonly IPolicyAuditRepository _auditRepository; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + private readonly SemaphoreSlim _mutex = new(1, 1); + + public PolicySnapshotStore( + IPolicySnapshotRepository snapshotRepository, + IPolicyAuditRepository auditRepository, + TimeProvider? timeProvider, + ILogger logger) + { + _snapshotRepository = snapshotRepository ?? throw new ArgumentNullException(nameof(snapshotRepository)); + _auditRepository = auditRepository ?? throw new ArgumentNullException(nameof(auditRepository)); + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task SaveAsync(PolicySnapshotContent content, CancellationToken cancellationToken = default) + { + if (content is null) + { + throw new ArgumentNullException(nameof(content)); + } + + var bindingResult = PolicyBinder.Bind(content.Content, content.Format); + if (!bindingResult.Success) + { + _logger.LogWarning("Policy snapshot rejected due to validation errors (Format: {Format})", content.Format); + return new PolicySnapshotSaveResult(false, false, string.Empty, null, bindingResult); + } + + var digest = PolicyDigest.Compute(bindingResult.Document); + + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var latest = await _snapshotRepository.GetLatestAsync(cancellationToken).ConfigureAwait(false); + if (latest is not null && string.Equals(latest.Digest, digest, StringComparison.Ordinal)) + { + _logger.LogInformation("Policy snapshot unchanged; digest {Digest} matches revision {RevisionId}", digest, latest.RevisionId); + return new PolicySnapshotSaveResult(true, false, digest, latest, bindingResult); + } + + var revisionNumber = (latest?.RevisionNumber ?? 0) + 1; + var revisionId = $"rev-{revisionNumber}"; + var createdAt = _timeProvider.GetUtcNow(); + + var scoringConfig = PolicyScoringConfig.Default; + + var snapshot = new PolicySnapshot( + revisionNumber, + revisionId, + digest, + createdAt, + content.Actor, + content.Format, + bindingResult.Document, + bindingResult.Issues, + scoringConfig); + + await _snapshotRepository.AddAsync(snapshot, cancellationToken).ConfigureAwait(false); + + var auditMessage = content.Description ?? "Policy snapshot created"; + var auditEntry = new PolicyAuditEntry( + Guid.NewGuid(), + createdAt, + "snapshot.created", + revisionId, + digest, + content.Actor, + auditMessage); + + await _auditRepository.AddAsync(auditEntry, cancellationToken).ConfigureAwait(false); + + _logger.LogInformation( + "Policy snapshot saved. Revision {RevisionId}, digest {Digest}, issues {IssueCount}", + revisionId, + digest, + bindingResult.Issues.Length); + + return new PolicySnapshotSaveResult(true, true, digest, snapshot, bindingResult); + } + finally + { + _mutex.Release(); + } + } + + public Task GetLatestAsync(CancellationToken cancellationToken = default) + => _snapshotRepository.GetLatestAsync(cancellationToken); +} diff --git a/src/StellaOps.Policy/PolicyUnknownConfidenceConfig.cs b/src/StellaOps.Policy/PolicyUnknownConfidenceConfig.cs new file mode 100644 index 00000000..9c599c58 --- /dev/null +++ b/src/StellaOps.Policy/PolicyUnknownConfidenceConfig.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public sealed record PolicyUnknownConfidenceConfig( + double Initial, + double DecayPerDay, + double Floor, + ImmutableArray Bands) +{ + public double Clamp(double value) + => Math.Clamp(value, Floor, 1.0); + + public PolicyUnknownConfidenceBand ResolveBand(double value) + { + if (Bands.IsDefaultOrEmpty) + { + return PolicyUnknownConfidenceBand.Default; + } + + foreach (var band in Bands) + { + if (value >= band.Min) + { + return band; + } + } + + return Bands[Bands.Length - 1]; + } +} + +public sealed record PolicyUnknownConfidenceBand(string Name, double Min, string? Description = null) +{ + public static PolicyUnknownConfidenceBand Default { get; } = new("unspecified", 0, null); +} diff --git a/src/StellaOps.Policy/PolicyValidationCli.cs b/src/StellaOps.Policy/PolicyValidationCli.cs new file mode 100644 index 00000000..98884988 --- /dev/null +++ b/src/StellaOps.Policy/PolicyValidationCli.cs @@ -0,0 +1,241 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Policy; + +public sealed record PolicyValidationCliOptions +{ + public IReadOnlyList Inputs { get; init; } = Array.Empty(); + + /// + /// Writes machine-readable JSON instead of human-formatted text. + /// + public bool OutputJson { get; init; } + + /// + /// When enabled, warnings cause a non-zero exit code. + /// + public bool Strict { get; init; } +} + +public sealed record PolicyValidationFileResult( + string Path, + PolicyBindingResult BindingResult, + PolicyDiagnosticsReport Diagnostics); + +public sealed class PolicyValidationCli +{ + private readonly TextWriter _output; + private readonly TextWriter _error; + + public PolicyValidationCli(TextWriter? output = null, TextWriter? error = null) + { + _output = output ?? Console.Out; + _error = error ?? Console.Error; + } + + public async Task RunAsync(PolicyValidationCliOptions options, CancellationToken cancellationToken = default) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (options.Inputs.Count == 0) + { + await _error.WriteLineAsync("No input files provided. Supply one or more policy file paths."); + return 64; // EX_USAGE + } + + var results = new List(); + foreach (var input in options.Inputs) + { + cancellationToken.ThrowIfCancellationRequested(); + + var resolvedPaths = ResolveInput(input); + if (resolvedPaths.Count == 0) + { + await _error.WriteLineAsync($"No files matched '{input}'."); + continue; + } + + foreach (var path in resolvedPaths) + { + cancellationToken.ThrowIfCancellationRequested(); + + var format = PolicySchema.DetectFormat(path); + var content = await File.ReadAllTextAsync(path, cancellationToken); + var bindingResult = PolicyBinder.Bind(content, format); + var diagnostics = PolicyDiagnostics.Create(bindingResult); + + results.Add(new PolicyValidationFileResult(path, bindingResult, diagnostics)); + } + } + + if (results.Count == 0) + { + await _error.WriteLineAsync("No files were processed."); + return 65; // EX_DATAERR + } + + if (options.OutputJson) + { + WriteJson(results); + } + else + { + await WriteTextAsync(results, cancellationToken); + } + + var hasErrors = results.Any(static result => !result.BindingResult.Success); + var hasWarnings = results.Any(static result => result.BindingResult.Issues.Any(static issue => issue.Severity == PolicyIssueSeverity.Warning)); + + if (hasErrors) + { + return 1; + } + + if (options.Strict && hasWarnings) + { + return 2; + } + + return 0; + } + + private async Task WriteTextAsync(IReadOnlyList results, CancellationToken cancellationToken) + { + foreach (var result in results) + { + cancellationToken.ThrowIfCancellationRequested(); + + var relativePath = MakeRelative(result.Path); + await _output.WriteLineAsync($"{relativePath} [{result.BindingResult.Format}]"); + + if (result.BindingResult.Issues.Length == 0) + { + await _output.WriteLineAsync(" OK"); + continue; + } + + foreach (var issue in result.BindingResult.Issues) + { + var severity = issue.Severity.ToString().ToUpperInvariant().PadRight(7); + await _output.WriteLineAsync($" {severity} {issue.Path} :: {issue.Message} ({issue.Code})"); + } + } + } + + private void WriteJson(IReadOnlyList results) + { + var payload = results.Select(static result => new + { + path = result.Path, + format = result.BindingResult.Format.ToString().ToLowerInvariant(), + success = result.BindingResult.Success, + issues = result.BindingResult.Issues.Select(static issue => new + { + code = issue.Code, + message = issue.Message, + severity = issue.Severity.ToString().ToLowerInvariant(), + path = issue.Path, + }), + diagnostics = new + { + version = result.Diagnostics.Version, + ruleCount = result.Diagnostics.RuleCount, + errorCount = result.Diagnostics.ErrorCount, + warningCount = result.Diagnostics.WarningCount, + generatedAt = result.Diagnostics.GeneratedAt, + recommendations = result.Diagnostics.Recommendations, + }, + }) + .ToArray(); + + var json = JsonSerializer.Serialize(payload, new JsonSerializerOptions + { + WriteIndented = true, + }); + _output.WriteLine(json); + } + + private static IReadOnlyList ResolveInput(string input) + { + if (string.IsNullOrWhiteSpace(input)) + { + return Array.Empty(); + } + + var expanded = Environment.ExpandEnvironmentVariables(input.Trim()); + if (File.Exists(expanded)) + { + return new[] { Path.GetFullPath(expanded) }; + } + + if (Directory.Exists(expanded)) + { + return Directory.EnumerateFiles(expanded, "*.*", SearchOption.TopDirectoryOnly) + .Where(static path => MatchesPolicyExtension(path)) + .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) + .Select(Path.GetFullPath) + .ToArray(); + } + + var directory = Path.GetDirectoryName(expanded); + var searchPattern = Path.GetFileName(expanded); + + if (string.IsNullOrEmpty(searchPattern)) + { + return Array.Empty(); + } + + if (string.IsNullOrEmpty(directory)) + { + directory = "."; + } + + if (!Directory.Exists(directory)) + { + return Array.Empty(); + } + + return Directory.EnumerateFiles(directory, searchPattern, SearchOption.TopDirectoryOnly) + .Where(static path => MatchesPolicyExtension(path)) + .OrderBy(static path => path, StringComparer.OrdinalIgnoreCase) + .Select(Path.GetFullPath) + .ToArray(); + } + + private static bool MatchesPolicyExtension(string path) + { + var extension = Path.GetExtension(path); + return extension.Equals(".yaml", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".yml", StringComparison.OrdinalIgnoreCase) + || extension.Equals(".json", StringComparison.OrdinalIgnoreCase); + } + + private static string MakeRelative(string path) + { + try + { + var fullPath = Path.GetFullPath(path); + var current = Directory.GetCurrentDirectory(); + if (fullPath.StartsWith(current, StringComparison.OrdinalIgnoreCase)) + { + return fullPath[current.Length..].TrimStart(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + } + + return fullPath; + } + catch + { + return path; + } + } +} diff --git a/src/StellaOps.Policy/PolicyVerdict.cs b/src/StellaOps.Policy/PolicyVerdict.cs new file mode 100644 index 00000000..8136dde7 --- /dev/null +++ b/src/StellaOps.Policy/PolicyVerdict.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Immutable; + +namespace StellaOps.Policy; + +public enum PolicyVerdictStatus +{ + Pass, + Blocked, + Ignored, + Warned, + Deferred, + Escalated, + RequiresVex, +} + +public sealed record PolicyVerdict( + string FindingId, + PolicyVerdictStatus Status, + string? RuleName = null, + string? RuleAction = null, + string? Notes = null, + double Score = 0, + string ConfigVersion = "1.0", + ImmutableDictionary? Inputs = null, + string? QuietedBy = null, + bool Quiet = false, + double? UnknownConfidence = null, + string? ConfidenceBand = null, + double? UnknownAgeDays = null, + string? SourceTrust = null, + string? Reachability = null) +{ + public static PolicyVerdict CreateBaseline(string findingId, PolicyScoringConfig scoringConfig) + { + var inputs = ImmutableDictionary.Empty; + return new PolicyVerdict( + findingId, + PolicyVerdictStatus.Pass, + RuleName: null, + RuleAction: null, + Notes: null, + Score: 0, + ConfigVersion: scoringConfig.Version, + Inputs: inputs, + QuietedBy: null, + Quiet: false, + UnknownConfidence: null, + ConfidenceBand: null, + UnknownAgeDays: null, + SourceTrust: null, + Reachability: null); + } + + public ImmutableDictionary GetInputs() + => Inputs ?? ImmutableDictionary.Empty; +} + +public sealed record PolicyVerdictDiff( + PolicyVerdict Baseline, + PolicyVerdict Projected) +{ + public bool Changed + { + get + { + if (Baseline.Status != Projected.Status) + { + return true; + } + + if (!string.Equals(Baseline.RuleName, Projected.RuleName, StringComparison.Ordinal)) + { + return true; + } + + if (Math.Abs(Baseline.Score - Projected.Score) > 0.0001) + { + return true; + } + + if (!string.Equals(Baseline.QuietedBy, Projected.QuietedBy, StringComparison.Ordinal)) + { + return true; + } + + var baselineConfidence = Baseline.UnknownConfidence ?? 0; + var projectedConfidence = Projected.UnknownConfidence ?? 0; + if (Math.Abs(baselineConfidence - projectedConfidence) > 0.0001) + { + return true; + } + + if (!string.Equals(Baseline.ConfidenceBand, Projected.ConfidenceBand, StringComparison.Ordinal)) + { + return true; + } + + if (!string.Equals(Baseline.SourceTrust, Projected.SourceTrust, StringComparison.Ordinal)) + { + return true; + } + + if (!string.Equals(Baseline.Reachability, Projected.Reachability, StringComparison.Ordinal)) + { + return true; + } + + return false; + } + } +} diff --git a/src/StellaOps.Policy/Schemas/policy-schema@1.json b/src/StellaOps.Policy/Schemas/policy-schema@1.json new file mode 100644 index 00000000..48049c87 --- /dev/null +++ b/src/StellaOps.Policy/Schemas/policy-schema@1.json @@ -0,0 +1,176 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.stella-ops.org/policy/policy-schema@1.json", + "title": "StellaOps Policy Schema v1", + "type": "object", + "required": ["version", "rules"], + "properties": { + "version": { + "type": ["string", "number"], + "enum": ["1", "1.0", 1, 1.0] + }, + "description": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": ["string", "number", "boolean"] + } + }, + "rules": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/rule" + } + } + }, + "additionalProperties": true, + "$defs": { + "identifier": { + "type": "string", + "minLength": 1 + }, + "severity": { + "type": "string", + "enum": ["Critical", "High", "Medium", "Low", "Informational", "None", "Unknown"] + }, + "stringArray": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "uniqueItems": true + }, + "rule": { + "type": "object", + "required": ["name", "action"], + "properties": { + "id": { + "$ref": "#/$defs/identifier" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "description": { + "type": "string" + }, + "severity": { + "type": "array", + "items": { + "$ref": "#/$defs/severity" + }, + "uniqueItems": true + }, + "sources": { + "$ref": "#/$defs/stringArray" + }, + "vendors": { + "$ref": "#/$defs/stringArray" + }, + "licenses": { + "$ref": "#/$defs/stringArray" + }, + "tags": { + "$ref": "#/$defs/stringArray" + }, + "environments": { + "$ref": "#/$defs/stringArray" + }, + "images": { + "$ref": "#/$defs/stringArray" + }, + "repositories": { + "$ref": "#/$defs/stringArray" + }, + "packages": { + "$ref": "#/$defs/stringArray" + }, + "purls": { + "$ref": "#/$defs/stringArray" + }, + "cves": { + "$ref": "#/$defs/stringArray" + }, + "paths": { + "$ref": "#/$defs/stringArray" + }, + "layerDigests": { + "$ref": "#/$defs/stringArray" + }, + "usedByEntrypoint": { + "$ref": "#/$defs/stringArray" + }, + "justification": { + "type": "string" + }, + "quiet": { + "type": "boolean" + }, + "action": { + "oneOf": [ + { + "type": "string", + "enum": ["block", "fail", "deny", "ignore", "warn", "defer", "escalate", "requireVex"] + }, + { + "type": "object", + "required": ["type"], + "properties": { + "type": { + "type": "string" + }, + "quiet": { + "type": "boolean" + }, + "until": { + "type": "string", + "format": "date-time" + }, + "justification": { + "type": "string" + }, + "severity": { + "$ref": "#/$defs/severity" + }, + "vendors": { + "$ref": "#/$defs/stringArray" + }, + "justifications": { + "$ref": "#/$defs/stringArray" + }, + "epss": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "kev": { + "type": "boolean" + } + }, + "additionalProperties": true + } + ] + }, + "expires": { + "type": "string", + "format": "date-time" + }, + "until": { + "type": "string", + "format": "date-time" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": ["string", "number", "boolean"] + } + } + }, + "additionalProperties": true + } + } +} diff --git a/src/StellaOps.Policy/Schemas/policy-scoring-default.json b/src/StellaOps.Policy/Schemas/policy-scoring-default.json new file mode 100644 index 00000000..d589a5b9 --- /dev/null +++ b/src/StellaOps.Policy/Schemas/policy-scoring-default.json @@ -0,0 +1,51 @@ +{ + "version": "1.0", + "severityWeights": { + "Critical": 90.0, + "High": 75.0, + "Medium": 50.0, + "Low": 25.0, + "Informational": 10.0, + "None": 0.0, + "Unknown": 60.0 + }, + "quietPenalty": 45.0, + "warnPenalty": 15.0, + "ignorePenalty": 35.0, + "trustOverrides": { + "vendor": 1.0, + "distro": 0.85, + "platform": 0.75, + "community": 0.65 + }, + "reachabilityBuckets": { + "entrypoint": 1.0, + "direct": 0.85, + "indirect": 0.6, + "runtime": 0.45, + "unreachable": 0.25, + "unknown": 0.5 + }, + "unknownConfidence": { + "initial": 0.8, + "decayPerDay": 0.05, + "floor": 0.2, + "bands": [ + { + "name": "high", + "min": 0.65, + "description": "Fresh unknowns with recent telemetry." + }, + { + "name": "medium", + "min": 0.35, + "description": "Unknowns aging toward action required." + }, + { + "name": "low", + "min": 0.0, + "description": "Stale unknowns that must be triaged." + } + ] + } +} diff --git a/src/StellaOps.Policy/Schemas/policy-scoring-schema@1.json b/src/StellaOps.Policy/Schemas/policy-scoring-schema@1.json new file mode 100644 index 00000000..a01e8b2e --- /dev/null +++ b/src/StellaOps.Policy/Schemas/policy-scoring-schema@1.json @@ -0,0 +1,156 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.stella-ops.org/policy/policy-scoring-schema@1.json", + "title": "StellaOps Policy Scoring Configuration v1", + "type": "object", + "additionalProperties": false, + "required": [ + "version", + "severityWeights" + ], + "properties": { + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+$" + }, + "severityWeights": { + "type": "object", + "additionalProperties": false, + "required": [ + "Critical", + "High", + "Medium", + "Low", + "Informational", + "None", + "Unknown" + ], + "properties": { + "Critical": { + "$ref": "#/$defs/weight" + }, + "High": { + "$ref": "#/$defs/weight" + }, + "Medium": { + "$ref": "#/$defs/weight" + }, + "Low": { + "$ref": "#/$defs/weight" + }, + "Informational": { + "$ref": "#/$defs/weight" + }, + "None": { + "$ref": "#/$defs/weight" + }, + "Unknown": { + "$ref": "#/$defs/weight" + } + } + }, + "quietPenalty": { + "$ref": "#/$defs/penalty" + }, + "warnPenalty": { + "$ref": "#/$defs/penalty" + }, + "ignorePenalty": { + "$ref": "#/$defs/penalty" + }, + "trustOverrides": { + "type": "object", + "propertyNames": { + "pattern": "^[a-z][a-z0-9_.-]*$" + }, + "additionalProperties": { + "$ref": "#/$defs/trustWeight" + } + }, + "reachabilityBuckets": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[a-z][a-z0-9_.-]*$" + }, + "additionalProperties": { + "$ref": "#/$defs/reachabilityWeight" + } + }, + "unknownConfidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "initial", + "decayPerDay", + "floor", + "bands" + ], + "properties": { + "initial": { + "$ref": "#/$defs/confidence" + }, + "decayPerDay": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "floor": { + "$ref": "#/$defs/confidence" + }, + "bands": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "min" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[a-z][a-z0-9_.-]*$" + }, + "min": { + "$ref": "#/$defs/confidence" + }, + "description": { + "type": "string", + "maxLength": 256 + } + } + } + } + } + } + }, + "$defs": { + "weight": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "penalty": { + "type": "number", + "minimum": 0, + "maximum": 200 + }, + "trustWeight": { + "type": "number", + "minimum": 0, + "maximum": 5 + }, + "reachabilityWeight": { + "type": "number", + "minimum": 0, + "maximum": 1.5 + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + } +} diff --git a/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj b/src/StellaOps.Policy/StellaOps.Policy.csproj similarity index 50% rename from src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj rename to src/StellaOps.Policy/StellaOps.Policy.csproj index e39dc64a..4b97757c 100644 --- a/src/StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj +++ b/src/StellaOps.Policy/StellaOps.Policy.csproj @@ -1,19 +1,22 @@ net10.0 - preview - enable enable + enable + preview true + - - - + + + + - - + + + diff --git a/src/StellaOps.Policy/Storage/IPolicySnapshotRepository.cs b/src/StellaOps.Policy/Storage/IPolicySnapshotRepository.cs new file mode 100644 index 00000000..b432a058 --- /dev/null +++ b/src/StellaOps.Policy/Storage/IPolicySnapshotRepository.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Policy; + +public interface IPolicySnapshotRepository +{ + Task GetLatestAsync(CancellationToken cancellationToken = default); + + Task> ListAsync(int limit, CancellationToken cancellationToken = default); + + Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Policy/Storage/InMemoryPolicySnapshotRepository.cs b/src/StellaOps.Policy/Storage/InMemoryPolicySnapshotRepository.cs new file mode 100644 index 00000000..69a033c8 --- /dev/null +++ b/src/StellaOps.Policy/Storage/InMemoryPolicySnapshotRepository.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Policy; + +public sealed class InMemoryPolicySnapshotRepository : IPolicySnapshotRepository +{ + private readonly List _snapshots = new(); + private readonly SemaphoreSlim _mutex = new(1, 1); + + public async Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default) + { + if (snapshot is null) + { + throw new ArgumentNullException(nameof(snapshot)); + } + + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _snapshots.Add(snapshot); + _snapshots.Sort(static (left, right) => left.RevisionNumber.CompareTo(right.RevisionNumber)); + } + finally + { + _mutex.Release(); + } + } + + public async Task GetLatestAsync(CancellationToken cancellationToken = default) + { + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return _snapshots.Count == 0 ? null : _snapshots[^1]; + } + finally + { + _mutex.Release(); + } + } + + public async Task> ListAsync(int limit, CancellationToken cancellationToken = default) + { + await _mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + IEnumerable query = _snapshots; + if (limit > 0) + { + query = query.TakeLast(limit); + } + + return query.ToImmutableArray(); + } + finally + { + _mutex.Release(); + } + } +} diff --git a/src/StellaOps.Policy/TASKS.md b/src/StellaOps.Policy/TASKS.md new file mode 100644 index 00000000..5e3bf339 --- /dev/null +++ b/src/StellaOps.Policy/TASKS.md @@ -0,0 +1,19 @@ +# Policy Engine Task Board (Sprint 9) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| POLICY-CORE-09-001 | DONE | Policy Guild | SCANNER-WEB-09-101 | Define YAML schema/binder, diagnostics, CLI validation for policy files. | Schema doc published; binder loads sample policy; validation errors actionable. | +| POLICY-CORE-09-002 | DONE | Policy Guild | POLICY-CORE-09-001 | Implement policy snapshot store + revision digests + audit logging. | Snapshots persisted with digest; tests compare revisions; audit entries created. | +| POLICY-CORE-09-003 | DONE | Policy Guild | POLICY-CORE-09-002 | `/policy/preview` API (image digest → projected verdict delta). | Preview returns diff JSON; integration tests with mocked report; docs updated. | +| POLICY-CORE-09-004 | DOING (2025-10-19) | Policy Guild | — | Versioned scoring config with schema validation, trust table, and golden fixtures. | Scoring config documented; fixtures stored; validation CLI passes. | +| POLICY-CORE-09-005 | DOING (2025-10-19) | Policy Guild | — | Scoring/quiet engine – compute score, enforce VEX-only quiet rules, emit inputs and provenance. | Engine unit tests cover severity weighting; outputs include provenance data. | +| POLICY-CORE-09-006 | DOING (2025-10-19) | Policy Guild | — | Unknown state & confidence decay – deterministic bands surfaced in policy outputs. | Confidence decay tests pass; docs updated; preview endpoint displays banding. | +| POLICY-CORE-09-004 | DONE | Policy Guild | POLICY-CORE-09-001 | Versioned scoring config (weights, trust table, reachability buckets) with schema validation, binder, and golden fixtures. | Config serialized with semantic version, binder loads defaults, fixtures assert deterministic hash. | +| POLICY-CORE-09-005 | DONE | Policy Guild | POLICY-CORE-09-004, POLICY-CORE-09-002 | Implement scoring/quiet engine: compute score from config, enforce VEX-only quiet rules, emit inputs + `quietedBy` metadata in policy verdicts. | `/reports` policy result includes score, inputs, configVersion, quiet provenance; unit/integration tests prove reproducibility. | +| POLICY-CORE-09-006 | DONE | Policy Guild | POLICY-CORE-09-005, FEEDCORE-ENGINE-07-003 | Track unknown states with deterministic confidence bands that decay over time; expose state in policy outputs and docs. | Unknown flags + confidence band persisted, decay job deterministic, preview/report APIs show state with tests covering decay math. | +| POLICY-RUNTIME-17-201 | TODO | Policy Guild, Scanner WebService Guild | ZASTAVA-OBS-17-005 | Define runtime reachability feed contract and alignment plan for `SCANNER-RUNTIME-17-401` once Zastava endpoints land; document policy expectations for reachability tags. | Contract note published, sample payload agreed with Scanner team, dependencies captured in scanner/runtime task boards. | + +## Notes +- 2025-10-18: POLICY-CORE-09-001 completed. Binder + diagnostics + CLI scaffolding landed with tests; schema embedded at `src/StellaOps.Policy/Schemas/policy-schema@1.json` and referenced by docs/11_DATA_SCHEMAS.md. +- 2025-10-18: POLICY-CORE-09-002 completed. Snapshot store + audit trail implemented with deterministic digest hashing and tests covering revision increments and dedupe. +- 2025-10-18: POLICY-CORE-09-003 delivered. Preview service evaluates policy projections vs. baseline, returns verdict diffs, and ships with unit coverage. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.DotNet/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/AGENTS.md new file mode 100644 index 00000000..5731b819 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/AGENTS.md @@ -0,0 +1,29 @@ +# StellaOps.Scanner.Analyzers.Lang.DotNet — Agent Charter + +## Role +Create the .NET analyzer plug-in that inspects `*.deps.json`, `runtimeconfig.json`, assemblies, and RID-specific assets to deliver accurate NuGet components with signing metadata. + +## Scope +- Parse dependency graphs from `*.deps.json` and merge with `runtimeconfig.json` and bundle manifests. +- Capture assembly metadata (strong name, file version, Authenticode) and correlate with packages. +- Handle RID-specific asset selection, self-contained apps, and crossgen/native dependency hints. +- Package plug-in manifest, determinism fixtures, benchmarks, and Offline Kit documentation. + +## Out of Scope +- Policy evaluation or Signer integration (handled elsewhere). +- Native dependency resolution outside RID mapping. +- Windows-specific MSI/SxS analyzers (covered by native analyzer roadmap). + +## Expectations +- Performance target: multi-target app fixture <1.2 s, memory <250 MB. +- Deterministic RID collapsing to reduce component duplication by ≥40 % vs naive approach. +- Offline-first; support air-gapped strong-name/Authenticode validation using cached root store. +- Rich telemetry (components per RID, strong-name validations) conforming to Scanner metrics. + +## Dependencies +- Shared language analyzer infrastructure; Worker dispatcher; optional security key store for signature verification. + +## Testing & Artifacts +- Fixtures for framework-dependent and self-contained apps (linux-musl, win-x64). +- Golden outputs capturing signature metadata and RID grouping. +- Benchmark comparing analyzer fidelity vs market competitors. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.DotNet/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/GlobalUsings.cs new file mode 100644 index 00000000..69c494f5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/GlobalUsings.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.DotNet/Placeholder.cs b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/Placeholder.cs new file mode 100644 index 00000000..cb504b1e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/Placeholder.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Analyzers.Lang.DotNet; + +internal static class Placeholder +{ + // Analyzer implementation will be added during Sprint LA4. +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.DotNet/StellaOps.Scanner.Analyzers.Lang.DotNet.csproj b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/StellaOps.Scanner.Analyzers.Lang.DotNet.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/StellaOps.Scanner.Analyzers.Lang.DotNet.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md new file mode 100644 index 00000000..da8c8ebe --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.DotNet/TASKS.md @@ -0,0 +1,10 @@ +# .NET Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-305A | TODO | SCANNER-ANALYZERS-LANG-10-307 | Parse `*.deps.json` + `runtimeconfig.json`, build RID graph, and normalize to `pkg:nuget` components. | RID graph deterministic; fixtures confirm consistent component ordering; fallback to `bin:{sha256}` documented. | +| 2 | SCANNER-ANALYZERS-LANG-10-305B | TODO | SCANNER-ANALYZERS-LANG-10-305A | Extract assembly metadata (strong name, file/product info) and optional Authenticode details when offline cert bundle provided. | Signing metadata captured for signed assemblies; offline trust store documented; hash validations deterministic. | +| 3 | SCANNER-ANALYZERS-LANG-10-305C | TODO | SCANNER-ANALYZERS-LANG-10-305B | Handle self-contained apps and native assets; merge with EntryTrace usage hints. | Self-contained fixtures map to components with RID flags; usage hints propagate; tests cover linux/win variants. | +| 4 | SCANNER-ANALYZERS-LANG-10-307D | TODO | SCANNER-ANALYZERS-LANG-10-305C | Integrate shared helpers (license mapping, quiet provenance) and concurrency-safe caches. | Shared helpers reused; concurrency tests for parallel layer scans pass; no redundant allocations. | +| 5 | SCANNER-ANALYZERS-LANG-10-308D | TODO | SCANNER-ANALYZERS-LANG-10-307D | Determinism fixtures + benchmark harness; compare to competitor scanners for accuracy/perf. | Fixtures in `Fixtures/lang/dotnet/`; determinism CI guard; benchmark demonstrates lower duplication + faster runtime. | +| 6 | SCANNER-ANALYZERS-LANG-10-309D | TODO | SCANNER-ANALYZERS-LANG-10-308D | Package plug-in (manifest, DI registration) and update Offline Kit instructions. | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer; Offline Kit doc updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Go/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang.Go/AGENTS.md new file mode 100644 index 00000000..8abf0f95 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Go/AGENTS.md @@ -0,0 +1,29 @@ +# StellaOps.Scanner.Analyzers.Lang.Go — Agent Charter + +## Role +Build the Go analyzer plug-in that reads Go build info, module metadata, and DWARF notes to attribute binaries with rich provenance inside Scanner. + +## Scope +- Inspect binaries for build info (`.note.go.buildid`, Go build info blob) and extract module, version, VCS metadata. +- Parse DWARF-lite sections for commit hash / dirty flag and map to components. +- Manage shared hash cache to dedupe identical binaries across layers. +- Provide benchmarks and determinism fixtures; package plug-in manifest. + +## Out of Scope +- Native library link analysis (belongs to native analyzer). +- VCS remote fetching or symbol download. +- Policy decisions or vulnerability joins. + +## Expectations +- Latency targets: ≤400 µs (hot) / ≤2 ms (cold) per binary; minimal allocations via buffer pooling. +- Deterministic fallback to `bin:{sha256}` when metadata absent; heuristics clearly identified. +- Offline-first: rely solely on embedded metadata. +- Telemetry for binaries processed, metadata coverage, heuristics usage. + +## Dependencies +- Shared language analyzer core; Worker dispatcher; caching infrastructure (layer cache + file CAS). + +## Testing & Artifacts +- Golden fixtures for modules with/without VCS info, stripped binaries, cross-compiled variants. +- Benchmark comparison with competitor scanners to demonstrate speed/fidelity advantages. +- ADR documenting heuristics and risk mitigation. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Go/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.Go/GlobalUsings.cs new file mode 100644 index 00000000..69c494f5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Go/GlobalUsings.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Go/Placeholder.cs b/src/StellaOps.Scanner.Analyzers.Lang.Go/Placeholder.cs new file mode 100644 index 00000000..533a970d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Go/Placeholder.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Go; + +internal static class Placeholder +{ + // Analyzer implementation will be added during Sprint LA3. +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Go/StellaOps.Scanner.Analyzers.Lang.Go.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Go/StellaOps.Scanner.Analyzers.Lang.Go.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Go/StellaOps.Scanner.Analyzers.Lang.Go.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md new file mode 100644 index 00000000..3a5a533b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Go/TASKS.md @@ -0,0 +1,10 @@ +# Go Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-304A | TODO | SCANNER-ANALYZERS-LANG-10-307 | Parse Go build info blob (`runtime/debug` format) and `.note.go.buildid`; map to module/version and evidence. | Build info extracted across Go 1.18–1.23 fixtures; evidence includes VCS, module path, and build settings. | +| 2 | SCANNER-ANALYZERS-LANG-10-304B | TODO | SCANNER-ANALYZERS-LANG-10-304A | Implement DWARF-lite reader for VCS metadata + dirty flag; add cache to avoid re-reading identical binaries. | DWARF reader supplies commit hash for ≥95 % fixtures; cache reduces duplicated IO by ≥70 %. | +| 3 | SCANNER-ANALYZERS-LANG-10-304C | TODO | SCANNER-ANALYZERS-LANG-10-304B | Fallback heuristics for stripped binaries with deterministic `bin:{sha256}` labeling and quiet provenance. | Heuristic labels clearly separated; tests ensure no false “observed” provenance; documentation updated. | +| 4 | SCANNER-ANALYZERS-LANG-10-307G | TODO | SCANNER-ANALYZERS-LANG-10-304C | Wire shared helpers (license mapping, usage flags) and ensure concurrency-safe buffer reuse. | Analyzer reuses shared infrastructure; concurrency tests with parallel scans pass; no data races. | +| 5 | SCANNER-ANALYZERS-LANG-10-308G | TODO | SCANNER-ANALYZERS-LANG-10-307G | Determinism fixtures + benchmark harness (Vs competitor). | Fixtures under `Fixtures/lang/go/`; CI determinism check; benchmark runs showing ≥20 % speed advantage. | +| 6 | SCANNER-ANALYZERS-LANG-10-309G | TODO | SCANNER-ANALYZERS-LANG-10-308G | Package plug-in manifest + Offline Kit notes; ensure Worker DI registration. | Manifest copied; Worker loads analyzer; Offline Kit docs updated with Go analyzer presence. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Java/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.Java/GlobalUsings.cs new file mode 100644 index 00000000..a4e0b12e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Java/GlobalUsings.cs @@ -0,0 +1,10 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.IO.Compression; +global using System.Security.Cryptography; +global using System.Text; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs new file mode 100644 index 00000000..89c4e1ae --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Java/JavaLanguageAnalyzer.cs @@ -0,0 +1,360 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Java; + +public sealed class JavaLanguageAnalyzer : ILanguageAnalyzer +{ + private static readonly HashSet SupportedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".jar", + ".war", + ".ear", + ".jmod", + }; + + private static readonly EnumerationOptions EnumerationOptions = new() + { + RecurseSubdirectories = true, + IgnoreInaccessible = true, + AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint, + }; + + public string Id => "java"; + + public string DisplayName => "Java/Maven Analyzer"; + + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(writer); + + foreach (var jarPath in EnumerateCandidateArchives(context.RootPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + await ProcessArchiveAsync(jarPath, Id, context, writer, cancellationToken).ConfigureAwait(false); + } + catch (IOException) + { + // Ignore corrupt archives to keep scans resilient. + } + catch (InvalidDataException) + { + // Non-zip payloads should not break the scan. + } + } + } + + private static IEnumerable EnumerateCandidateArchives(string root) + { + foreach (var file in Directory.EnumerateFiles(root, "*", EnumerationOptions)) + { + if (!SupportedExtensions.Contains(Path.GetExtension(file))) + { + continue; + } + + yield return file; + } + } + + private static async ValueTask ProcessArchiveAsync(string archivePath, string analyzerId, LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) + { + using var fileStream = new FileStream(archivePath, FileMode.Open, FileAccess.Read, FileShare.Read); + using var archive = new ZipArchive(fileStream, ZipArchiveMode.Read, leaveOpen: false); + + ManifestMetadata? manifestMetadata = null; + var manifestEntry = archive.GetEntry("META-INF/MANIFEST.MF"); + if (manifestEntry is not null) + { + manifestMetadata = await ParseManifestAsync(manifestEntry, cancellationToken).ConfigureAwait(false); + } + + foreach (var entry in archive.Entries) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (IsManifestEntry(entry.FullName)) + { + continue; + } + + if (!IsPomPropertiesEntry(entry.FullName)) + { + continue; + } + + var artifact = await ParsePomPropertiesAsync(entry, cancellationToken).ConfigureAwait(false); + if (artifact is null) + { + continue; + } + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["groupId"] = artifact.GroupId, + ["artifactId"] = artifact.ArtifactId, + ["jarPath"] = context.GetRelativePath(archivePath), + }; + + if (!string.IsNullOrEmpty(artifact.Packaging)) + { + metadata["packaging"] = artifact.Packaging; + } + + if (!string.IsNullOrEmpty(artifact.Name)) + { + metadata["displayName"] = artifact.Name; + } + + if (manifestMetadata is not null) + { + manifestMetadata.ApplyMetadata(metadata); + } + + var evidence = new List + { + new(LanguageEvidenceKind.File, "pom.properties", BuildLocator(context, archivePath, entry.FullName), null, artifact.PomSha256), + }; + + if (manifestMetadata is not null) + { + evidence.Add(manifestMetadata.CreateEvidence(context, archivePath)); + } + + var usedByEntrypoint = context.UsageHints.IsPathUsed(archivePath); + + writer.AddFromPurl( + analyzerId: analyzerId, + purl: artifact.Purl, + name: artifact.ArtifactId, + version: artifact.Version, + type: "maven", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: usedByEntrypoint); + } + } + + private static string BuildLocator(LanguageAnalyzerContext context, string archivePath, string entryPath) + { + var relativeArchive = context.GetRelativePath(archivePath); + if (string.IsNullOrEmpty(relativeArchive) || relativeArchive == ".") + { + return NormalizeEntry(entryPath); + } + + return string.Concat(relativeArchive, "!", NormalizeEntry(entryPath)); + } + + private static string NormalizeEntry(string entryPath) + => entryPath.Replace('\\', '/'); + + private static bool IsPomPropertiesEntry(string entryName) + => entryName.StartsWith("META-INF/maven/", StringComparison.OrdinalIgnoreCase) + && entryName.EndsWith("/pom.properties", StringComparison.OrdinalIgnoreCase); + + private static bool IsManifestEntry(string entryName) + => string.Equals(entryName, "META-INF/MANIFEST.MF", StringComparison.OrdinalIgnoreCase); + + private static async ValueTask ParsePomPropertiesAsync(ZipArchiveEntry entry, CancellationToken cancellationToken) + { + await using var entryStream = entry.Open(); + using var buffer = new MemoryStream(); + await entryStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + + using var reader = new StreamReader(buffer, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: true); + var properties = new Dictionary(StringComparer.OrdinalIgnoreCase); + + while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + line = line.Trim(); + if (line.Length == 0 || line.StartsWith('#')) + { + continue; + } + + var separatorIndex = line.IndexOf('='); + if (separatorIndex <= 0) + { + continue; + } + + var key = line[..separatorIndex].Trim(); + var value = line[(separatorIndex + 1)..].Trim(); + if (key.Length == 0) + { + continue; + } + + properties[key] = value; + } + + if (!properties.TryGetValue("groupId", out var groupId) || string.IsNullOrWhiteSpace(groupId)) + { + return null; + } + + if (!properties.TryGetValue("artifactId", out var artifactId) || string.IsNullOrWhiteSpace(artifactId)) + { + return null; + } + + if (!properties.TryGetValue("version", out var version) || string.IsNullOrWhiteSpace(version)) + { + return null; + } + + var packaging = properties.TryGetValue("packaging", out var packagingValue) ? packagingValue : "jar"; + var name = properties.TryGetValue("name", out var nameValue) ? nameValue : null; + + var purl = BuildPurl(groupId, artifactId, version, packaging); + buffer.Position = 0; + var pomSha = Convert.ToHexString(SHA256.HashData(buffer)).ToLowerInvariant(); + + return new MavenArtifact( + GroupId: groupId.Trim(), + ArtifactId: artifactId.Trim(), + Version: version.Trim(), + Packaging: packaging?.Trim(), + Name: name?.Trim(), + Purl: purl, + PomSha256: pomSha); + } + + private static async ValueTask ParseManifestAsync(ZipArchiveEntry entry, CancellationToken cancellationToken) + { + await using var entryStream = entry.Open(); + using var reader = new StreamReader(entryStream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, leaveOpen: false); + + string? title = null; + string? version = null; + string? vendor = null; + + while (await reader.ReadLineAsync().ConfigureAwait(false) is { } line) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var separatorIndex = line.IndexOf(':'); + if (separatorIndex <= 0) + { + continue; + } + + var key = line[..separatorIndex].Trim(); + var value = line[(separatorIndex + 1)..].Trim(); + + if (key.Equals("Implementation-Title", StringComparison.OrdinalIgnoreCase)) + { + title ??= value; + } + else if (key.Equals("Implementation-Version", StringComparison.OrdinalIgnoreCase)) + { + version ??= value; + } + else if (key.Equals("Implementation-Vendor", StringComparison.OrdinalIgnoreCase)) + { + vendor ??= value; + } + } + + if (title is null && version is null && vendor is null) + { + return null; + } + + return new ManifestMetadata(title, version, vendor); + } + + private static string BuildPurl(string groupId, string artifactId, string version, string? packaging) + { + var normalizedGroup = groupId.Replace('.', '/'); + var builder = new StringBuilder(); + builder.Append("pkg:maven/"); + builder.Append(normalizedGroup); + builder.Append('/'); + builder.Append(artifactId); + builder.Append('@'); + builder.Append(version); + + if (!string.IsNullOrWhiteSpace(packaging) && !packaging.Equals("jar", StringComparison.OrdinalIgnoreCase)) + { + builder.Append("?type="); + builder.Append(packaging); + } + + return builder.ToString(); + } + + private sealed record MavenArtifact( + string GroupId, + string ArtifactId, + string Version, + string? Packaging, + string? Name, + string Purl, + string PomSha256); + + private sealed record ManifestMetadata(string? ImplementationTitle, string? ImplementationVersion, string? ImplementationVendor) + { + public void ApplyMetadata(IDictionary target) + { + if (!string.IsNullOrWhiteSpace(ImplementationTitle)) + { + target["manifestTitle"] = ImplementationTitle; + } + + if (!string.IsNullOrWhiteSpace(ImplementationVersion)) + { + target["manifestVersion"] = ImplementationVersion; + } + + if (!string.IsNullOrWhiteSpace(ImplementationVendor)) + { + target["manifestVendor"] = ImplementationVendor; + } + } + + public LanguageComponentEvidence CreateEvidence(LanguageAnalyzerContext context, string archivePath) + { + var locator = BuildLocator(context, archivePath, "META-INF/MANIFEST.MF"); + var valueBuilder = new StringBuilder(); + + if (!string.IsNullOrWhiteSpace(ImplementationTitle)) + { + valueBuilder.Append("title=").Append(ImplementationTitle); + } + + if (!string.IsNullOrWhiteSpace(ImplementationVersion)) + { + if (valueBuilder.Length > 0) + { + valueBuilder.Append(';'); + } + + valueBuilder.Append("version=").Append(ImplementationVersion); + } + + if (!string.IsNullOrWhiteSpace(ImplementationVendor)) + { + if (valueBuilder.Length > 0) + { + valueBuilder.Append(';'); + } + + valueBuilder.Append("vendor=").Append(ImplementationVendor); + } + + var value = valueBuilder.Length > 0 ? valueBuilder.ToString() : null; + return new LanguageComponentEvidence(LanguageEvidenceKind.File, "MANIFEST.MF", locator, value, null); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Java/StellaOps.Scanner.Analyzers.Lang.Java.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Java/StellaOps.Scanner.Analyzers.Lang.Java.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Java/StellaOps.Scanner.Analyzers.Lang.Java.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json b/src/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json new file mode 100644 index 00000000..096ef4d6 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Java/manifest.json @@ -0,0 +1,22 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzer.lang.java", + "displayName": "StellaOps Java / Maven Analyzer", + "version": "0.1.0", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.Lang.Java.dll", + "typeName": "StellaOps.Scanner.Analyzers.Lang.Java.JavaLanguageAnalyzer" + }, + "capabilities": [ + "language-analyzer", + "java", + "maven" + ], + "metadata": { + "org.stellaops.analyzer.language": "java", + "org.stellaops.analyzer.kind": "language", + "org.stellaops.restart.required": "true" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang.Node/AGENTS.md new file mode 100644 index 00000000..16f84ea4 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/AGENTS.md @@ -0,0 +1,39 @@ +# StellaOps.Scanner.Analyzers.Lang.Node — Agent Charter + +## Role +Deliver the Node.js / npm / Yarn / PNPM analyzer plug-in that resolves workspace graphs, symlinks, and script metadata for Scanner Workers. + +## Scope +- Deterministic filesystem walker for `node_modules`, PNPM store, Yarn Plug'n'Play, and workspace roots. +- Component identity normalization to `pkg:npm` with provenance evidence (manifest path, integrity hashes, lockfile references). +- Workspace + symlink attribution, script metadata (postinstall, lifecycle), and policy hints for risky scripts. +- Plug-in manifest authoring, DI bootstrap, and benchmark harness integration. + +## Out of Scope +- OS package detection, native library linkage, or vulnerability joins. +- Language analyzers for other ecosystems (Python, Go, .NET, Rust). +- CLI/UI surfacing of analyzer diagnostics (handed to UI guild post-gate). + +## Expectations +- Deterministic output across Yarn/NPM/PNPM variations; normalized casing and path separators. +- Performance targets: 10 k-module fixture <1.8 s, <220 MB RSS on 4 vCPU runner. +- Offline-first; no network dependency to resolve registries. +- Emit structured metrics + logs (`analyzer=node`) compatible with Scanner telemetry model. +- Update `TASKS.md`, `SPRINTS_LANG_IMPLEMENTATION_PLAN.md`, and corresponding fixtures as progress occurs. + +## Dependencies +- Shared language analyzer core (`StellaOps.Scanner.Analyzers.Lang`). +- Worker dispatcher for plug-in discovery. +- EntryTrace usage hints (for script usage classification). + +## Testing & Artifacts +- Determinism golden fixtures under `Fixtures/lang/node/`. +- Benchmark CSV + flamegraph stored in `bench/Scanner.Analyzers/`. +- Plug-in manifest + cosign workflow added to Offline Kit instructions once analyzer is production-ready. + +## Telemetry & Policy Hints +- Metrics: `scanner_analyzer_node_scripts_total{script}` increments for each install lifecycle script discovered. +- Metadata keys: + - `policyHint.installLifecycle` lists lifecycle scripts (`preinstall;install;postinstall`) observed for a package. + - `script.` stores the canonical command string for each lifecycle script. +- Evidence: lifecycle script entries emit `LanguageEvidenceKind.Metadata` pointing to `package.json#scripts.` with SHA-256 hashes for determinism. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/GlobalUsings.cs new file mode 100644 index 00000000..0c3ca905 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/GlobalUsings.cs @@ -0,0 +1,9 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Linq; +global using System.Text.Json; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeAnalyzerMetrics.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeAnalyzerMetrics.cs new file mode 100644 index 00000000..a1eaf2c9 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeAnalyzerMetrics.cs @@ -0,0 +1,31 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal static class NodeAnalyzerMetrics +{ + private static readonly Meter Meter = new("StellaOps.Scanner.Analyzers.Lang.Node", "1.0.0"); + private static readonly Counter LifecycleScriptsCounter = Meter.CreateCounter( + "scanner_analyzer_node_scripts_total", + unit: "scripts", + description: "Counts Node.js install lifecycle scripts discovered by the language analyzer."); + + public static void RecordLifecycleScript(string scriptName) + { + var normalized = Normalize(scriptName); + LifecycleScriptsCounter.Add( + 1, + new KeyValuePair("script", normalized)); + } + + private static string Normalize(string? scriptName) + { + if (string.IsNullOrWhiteSpace(scriptName)) + { + return "unknown"; + } + + return scriptName.Trim().ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLifecycleScript.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLifecycleScript.cs new file mode 100644 index 00000000..815c1ee0 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLifecycleScript.cs @@ -0,0 +1,37 @@ +using System.Diagnostics.CodeAnalysis; +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal sealed record NodeLifecycleScript +{ + public NodeLifecycleScript(string name, string command) + { + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentException.ThrowIfNullOrWhiteSpace(command); + + Name = name.Trim(); + Command = command.Trim(); + Sha256 = ComputeSha256(Command); + } + + public string Name { get; } + + public string Command { get; } + + public string Sha256 { get; } + + [SuppressMessage("Security", "CA5350:Do Not Use Weak Cryptographic Algorithms", Justification = "SHA256 is required for deterministic evidence hashing.")] + private static string ComputeSha256(string value) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + var bytes = Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs new file mode 100644 index 00000000..33f3fcb9 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockData.cs @@ -0,0 +1,446 @@ +using System.Text.Json; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal sealed class NodeLockData +{ + private static readonly NodeLockData Empty = new(new Dictionary(StringComparer.Ordinal), new Dictionary(StringComparer.OrdinalIgnoreCase)); + + private readonly Dictionary _byPath; + private readonly Dictionary _byName; + + private NodeLockData(Dictionary byPath, Dictionary byName) + { + _byPath = byPath; + _byName = byName; + } + + public static ValueTask LoadAsync(string rootPath, CancellationToken cancellationToken) + { + var byPath = new Dictionary(StringComparer.Ordinal); + var byName = new Dictionary(StringComparer.OrdinalIgnoreCase); + + LoadPackageLockJson(rootPath, byPath, byName, cancellationToken); + LoadYarnLock(rootPath, byName); + LoadPnpmLock(rootPath, byName); + + if (byPath.Count == 0 && byName.Count == 0) + { + return ValueTask.FromResult(Empty); + } + + return ValueTask.FromResult(new NodeLockData(byPath, byName)); + } + + public bool TryGet(string relativePath, string packageName, out NodeLockEntry? entry) + { + var normalizedPath = NormalizeLockPath(relativePath); + if (_byPath.TryGetValue(normalizedPath, out var byPathEntry)) + { + entry = byPathEntry; + return true; + } + + if (!string.IsNullOrEmpty(packageName)) + { + var normalizedName = packageName.StartsWith('@') ? packageName : packageName; + if (_byName.TryGetValue(normalizedName, out var byNameEntry)) + { + entry = byNameEntry; + return true; + } + } + + entry = null; + return false; + } + + private static NodeLockEntry? CreateEntry(JsonElement element) + { + string? version = null; + string? resolved = null; + string? integrity = null; + + if (element.TryGetProperty("version", out var versionElement) && versionElement.ValueKind == JsonValueKind.String) + { + version = versionElement.GetString(); + } + + if (element.TryGetProperty("resolved", out var resolvedElement) && resolvedElement.ValueKind == JsonValueKind.String) + { + resolved = resolvedElement.GetString(); + } + + if (element.TryGetProperty("integrity", out var integrityElement) && integrityElement.ValueKind == JsonValueKind.String) + { + integrity = integrityElement.GetString(); + } + + if (version is null && resolved is null && integrity is null) + { + return null; + } + + return new NodeLockEntry(version, resolved, integrity); + } + + private static void TraverseLegacyDependencies( + string currentPath, + JsonElement dependenciesElement, + IDictionary byPath, + IDictionary byName) + { + foreach (var dependency in dependenciesElement.EnumerateObject()) + { + var depValue = dependency.Value; + var path = $"{currentPath}/{dependency.Name}"; + var entry = CreateEntry(depValue); + if (entry is not null) + { + var normalizedPath = NormalizeLockPath(path); + byPath[normalizedPath] = entry; + byName[dependency.Name] = entry; + } + + if (depValue.TryGetProperty("dependencies", out var childDependencies) && childDependencies.ValueKind == JsonValueKind.Object) + { + TraverseLegacyDependencies(path + "/node_modules", childDependencies, byPath, byName); + } + } + } + + private static void LoadPackageLockJson(string rootPath, IDictionary byPath, IDictionary byName, CancellationToken cancellationToken) + { + var packageLockPath = Path.Combine(rootPath, "package-lock.json"); + if (!File.Exists(packageLockPath)) + { + return; + } + + try + { + using var stream = File.OpenRead(packageLockPath); + using var document = JsonDocument.Parse(stream); + cancellationToken.ThrowIfCancellationRequested(); + + var root = document.RootElement; + + if (root.TryGetProperty("packages", out var packagesElement) && packagesElement.ValueKind == JsonValueKind.Object) + { + foreach (var packageProperty in packagesElement.EnumerateObject()) + { + var entry = CreateEntry(packageProperty.Value); + if (entry is null) + { + continue; + } + + var key = NormalizeLockPath(packageProperty.Name); + byPath[key] = entry; + + var name = ExtractNameFromPath(key); + if (!string.IsNullOrEmpty(name)) + { + byName[name] = entry; + } + + if (packageProperty.Value.TryGetProperty("name", out var explicitNameElement) && explicitNameElement.ValueKind == JsonValueKind.String) + { + var explicitName = explicitNameElement.GetString(); + if (!string.IsNullOrWhiteSpace(explicitName)) + { + byName[explicitName] = entry; + } + } + } + } + else if (root.TryGetProperty("dependencies", out var dependenciesElement) && dependenciesElement.ValueKind == JsonValueKind.Object) + { + TraverseLegacyDependencies("node_modules", dependenciesElement, byPath, byName); + } + } + catch (IOException) + { + // Ignore unreadable package-lock. + } + catch (JsonException) + { + // Ignore malformed package-lock. + } + } + + private static void LoadYarnLock(string rootPath, IDictionary byName) + { + var yarnLockPath = Path.Combine(rootPath, "yarn.lock"); + if (!File.Exists(yarnLockPath)) + { + return; + } + + try + { + var lines = File.ReadAllLines(yarnLockPath); + string? currentName = null; + string? version = null; + string? resolved = null; + string? integrity = null; + + void Flush() + { + if (string.IsNullOrWhiteSpace(currentName)) + { + version = null; + resolved = null; + integrity = null; + return; + } + + var simpleName = ExtractPackageNameFromYarnKey(currentName!); + if (string.IsNullOrEmpty(simpleName)) + { + version = null; + resolved = null; + integrity = null; + return; + } + + var entry = new NodeLockEntry(version, resolved, integrity); + byName[simpleName] = entry; + version = null; + resolved = null; + integrity = null; + } + + foreach (var line in lines) + { + var trimmed = line.Trim(); + if (string.IsNullOrEmpty(trimmed)) + { + Flush(); + currentName = null; + continue; + } + + if (!char.IsWhiteSpace(line, 0) && trimmed.EndsWith(':')) + { + Flush(); + currentName = trimmed.TrimEnd(':').Trim('"'); + continue; + } + + if (trimmed.StartsWith("version", StringComparison.OrdinalIgnoreCase)) + { + version = ExtractQuotedValue(trimmed); + } + else if (trimmed.StartsWith("resolved", StringComparison.OrdinalIgnoreCase)) + { + resolved = ExtractQuotedValue(trimmed); + } + else if (trimmed.StartsWith("integrity", StringComparison.OrdinalIgnoreCase)) + { + integrity = ExtractQuotedValue(trimmed); + } + } + + Flush(); + } + catch (IOException) + { + // Ignore unreadable yarn.lock + } + } + + private static void LoadPnpmLock(string rootPath, IDictionary byName) + { + var pnpmLockPath = Path.Combine(rootPath, "pnpm-lock.yaml"); + if (!File.Exists(pnpmLockPath)) + { + return; + } + + try + { + using var reader = new StreamReader(pnpmLockPath); + string? currentPackage = null; + string? version = null; + string? resolved = null; + string? integrity = null; + var inPackages = false; + + while (reader.ReadLine() is { } line) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + if (!inPackages) + { + if (line.StartsWith("packages:", StringComparison.Ordinal)) + { + inPackages = true; + } + continue; + } + + if (line.StartsWith(" /", StringComparison.Ordinal)) + { + if (!string.IsNullOrEmpty(currentPackage) && !string.IsNullOrEmpty(integrity)) + { + var name = ExtractNameFromPnpmKey(currentPackage); + if (!string.IsNullOrEmpty(name)) + { + byName[name] = new NodeLockEntry(version, resolved, integrity); + } + } + + currentPackage = line.Trim().TrimEnd(':').TrimStart('/'); + version = null; + resolved = null; + integrity = null; + continue; + } + + if (string.IsNullOrEmpty(currentPackage)) + { + continue; + } + + var trimmed = line.Trim(); + if (trimmed.StartsWith("resolution:", StringComparison.Ordinal)) + { + var integrityIndex = trimmed.IndexOf("integrity", StringComparison.OrdinalIgnoreCase); + if (integrityIndex >= 0) + { + var integrityValue = trimmed[(integrityIndex + 9)..].Trim(' ', ':', '{', '}', '"'); + integrity = integrityValue; + } + + var tarballIndex = trimmed.IndexOf("tarball", StringComparison.OrdinalIgnoreCase); + if (tarballIndex >= 0) + { + var tarballValue = trimmed[(tarballIndex + 7)..].Trim(' ', ':', '{', '}', '"'); + resolved = tarballValue; + } + } + else if (trimmed.StartsWith("integrity:", StringComparison.Ordinal)) + { + integrity = trimmed[("integrity:".Length)..].Trim(); + } + else if (trimmed.StartsWith("tarball:", StringComparison.Ordinal)) + { + resolved = trimmed[("tarball:".Length)..].Trim(); + } + else if (trimmed.StartsWith("version:", StringComparison.Ordinal)) + { + version = trimmed[("version:".Length)..].Trim(); + } + } + + if (!string.IsNullOrEmpty(currentPackage) && !string.IsNullOrEmpty(integrity)) + { + var name = ExtractNameFromPnpmKey(currentPackage); + if (!string.IsNullOrEmpty(name)) + { + byName[name] = new NodeLockEntry(version, resolved, integrity); + } + } + } + catch (IOException) + { + // Ignore unreadable pnpm lock file. + } + } + + private static string? ExtractQuotedValue(string line) + { + var quoteStart = line.IndexOf('"'); + if (quoteStart < 0) + { + return null; + } + + var quoteEnd = line.LastIndexOf('"'); + if (quoteEnd <= quoteStart) + { + return null; + } + + return line.Substring(quoteStart + 1, quoteEnd - quoteStart - 1); + } + + private static string ExtractPackageNameFromYarnKey(string key) + { + var commaIndex = key.IndexOf(','); + var trimmed = commaIndex > 0 ? key[..commaIndex] : key; + trimmed = trimmed.Trim('"'); + + var atIndex = trimmed.IndexOf('@', 1); + if (atIndex > 0) + { + return trimmed[..atIndex]; + } + + return trimmed; + } + + private static string ExtractNameFromPnpmKey(string key) + { + var parts = key.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length == 0) + { + return string.Empty; + } + + if (parts[0].StartsWith('@')) + { + return parts.Length >= 2 ? $"{parts[0]}/{parts[1]}" : parts[0]; + } + + return parts[0]; + } + + private static string NormalizeLockPath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return string.Empty; + } + + var normalized = path.Replace('\\', '/'); + normalized = normalized.TrimStart('.', '/'); + return normalized; + } + + private static string ExtractNameFromPath(string normalizedPath) + { + if (string.IsNullOrEmpty(normalizedPath)) + { + return string.Empty; + } + + var segments = normalizedPath.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length == 0) + { + return string.Empty; + } + + if (segments[0] == "node_modules") + { + if (segments.Length >= 3 && segments[1].StartsWith('@')) + { + return $"{segments[1]}/{segments[2]}"; + } + + return segments.Length >= 2 ? segments[1] : string.Empty; + } + + var last = segments[^1]; + if (last.StartsWith('@') && segments.Length >= 2) + { + return $"{segments[^2]}/{last}"; + } + + return last; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs new file mode 100644 index 00000000..7ac54352 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeLockEntry.cs @@ -0,0 +1,3 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal sealed record NodeLockEntry(string? Version, string? Resolved, string? Integrity); diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs new file mode 100644 index 00000000..c2b23175 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackage.cs @@ -0,0 +1,179 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal sealed class NodePackage +{ + public NodePackage( + string name, + string version, + string relativePath, + string packageJsonLocator, + bool? isPrivate, + NodeLockEntry? lockEntry, + bool isWorkspaceMember, + string? workspaceRoot, + IReadOnlyList workspaceTargets, + string? workspaceLink, + IReadOnlyList lifecycleScripts, + bool usedByEntrypoint) + { + Name = name; + Version = version; + RelativePath = relativePath; + PackageJsonLocator = packageJsonLocator; + IsPrivate = isPrivate; + LockEntry = lockEntry; + IsWorkspaceMember = isWorkspaceMember; + WorkspaceRoot = workspaceRoot; + WorkspaceTargets = workspaceTargets; + WorkspaceLink = workspaceLink; + LifecycleScripts = lifecycleScripts ?? Array.Empty(); + IsUsedByEntrypoint = usedByEntrypoint; + } + + public string Name { get; } + + public string Version { get; } + + public string RelativePath { get; } + + public string PackageJsonLocator { get; } + + public bool? IsPrivate { get; } + + public NodeLockEntry? LockEntry { get; } + + public bool IsWorkspaceMember { get; } + + public string? WorkspaceRoot { get; } + + public IReadOnlyList WorkspaceTargets { get; } + + public string? WorkspaceLink { get; } + + public IReadOnlyList LifecycleScripts { get; } + + public bool HasInstallScripts => LifecycleScripts.Count > 0; + + public bool IsUsedByEntrypoint { get; } + + public string RelativePathNormalized => string.IsNullOrEmpty(RelativePath) ? string.Empty : RelativePath.Replace(Path.DirectorySeparatorChar, '/'); + + public string ComponentKey => $"purl::{Purl}"; + + public string Purl => BuildPurl(Name, Version); + + public IReadOnlyCollection CreateEvidence() + { + var evidence = new List + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "package.json", PackageJsonLocator, Value: null, Sha256: null) + }; + + foreach (var script in LifecycleScripts) + { + var locator = string.IsNullOrEmpty(PackageJsonLocator) + ? $"package.json#scripts.{script.Name}" + : $"{PackageJsonLocator}#scripts.{script.Name}"; + + evidence.Add(new LanguageComponentEvidence( + LanguageEvidenceKind.Metadata, + "package.json:scripts", + locator, + script.Command, + script.Sha256)); + } + + return evidence; + } + + public IReadOnlyCollection> CreateMetadata() + { + var entries = new List>(8) + { + new("path", string.IsNullOrEmpty(RelativePathNormalized) ? "." : RelativePathNormalized) + }; + + if (IsPrivate is bool isPrivate) + { + entries.Add(new KeyValuePair("private", isPrivate ? "true" : "false")); + } + + if (LockEntry is not null) + { + if (!string.IsNullOrWhiteSpace(LockEntry.Resolved)) + { + entries.Add(new KeyValuePair("resolved", LockEntry.Resolved)); + } + + if (!string.IsNullOrWhiteSpace(LockEntry.Integrity)) + { + entries.Add(new KeyValuePair("integrity", LockEntry.Integrity)); + } + } + + if (IsWorkspaceMember) + { + entries.Add(new KeyValuePair("workspaceMember", "true")); + if (!string.IsNullOrWhiteSpace(WorkspaceRoot)) + { + entries.Add(new KeyValuePair("workspaceRoot", WorkspaceRoot)); + } + } + + if (!string.IsNullOrWhiteSpace(WorkspaceLink)) + { + entries.Add(new KeyValuePair("workspaceLink", WorkspaceLink)); + } + + if (WorkspaceTargets.Count > 0) + { + entries.Add(new KeyValuePair("workspaceTargets", string.Join(';', WorkspaceTargets))); + } + + if (HasInstallScripts) + { + entries.Add(new KeyValuePair("installScripts", "true")); + var lifecycleNames = LifecycleScripts + .Select(static script => script.Name) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static name => name, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (lifecycleNames.Length > 0) + { + entries.Add(new KeyValuePair("policyHint.installLifecycle", string.Join(';', lifecycleNames))); + } + + foreach (var script in LifecycleScripts.OrderBy(static script => script.Name, StringComparer.OrdinalIgnoreCase)) + { + entries.Add(new KeyValuePair($"script.{script.Name}", script.Command)); + } + } + + return entries + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .ToArray(); + } + + private static string BuildPurl(string name, string version) + { + var normalizedName = NormalizeName(name); + return $"pkg:npm/{normalizedName}@{version}"; + } + + private static string NormalizeName(string name) + { + if (string.IsNullOrWhiteSpace(name)) + { + return name; + } + + if (name[0] == '@') + { + var scopeAndName = name[1..]; + return $"%40{scopeAndName}"; + } + + return name; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs new file mode 100644 index 00000000..e8fc738a --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodePackageCollector.cs @@ -0,0 +1,378 @@ +using System.Text.Json; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal static class NodePackageCollector +{ + private static readonly string[] IgnoredDirectories = + { + ".bin", + ".cache", + ".store", + "__pycache__" + }; + + public static IReadOnlyList CollectPackages(LanguageAnalyzerContext context, NodeLockData lockData, CancellationToken cancellationToken) + { + var packages = new List(); + var visited = new HashSet(StringComparer.OrdinalIgnoreCase); + var pendingNodeModuleRoots = new List(); + + var rootPackageJson = Path.Combine(context.RootPath, "package.json"); + var workspaceIndex = NodeWorkspaceIndex.Create(context.RootPath); + + if (File.Exists(rootPackageJson)) + { + var rootPackage = TryCreatePackage(context, rootPackageJson, string.Empty, lockData, workspaceIndex, cancellationToken); + if (rootPackage is not null) + { + packages.Add(rootPackage); + visited.Add(rootPackage.RelativePathNormalized); + } + } + + foreach (var workspaceRelative in workspaceIndex.GetMembers()) + { + var workspaceAbsolute = Path.Combine(context.RootPath, workspaceRelative.Replace('/', Path.DirectorySeparatorChar)); + if (!Directory.Exists(workspaceAbsolute)) + { + continue; + } + + ProcessPackageDirectory(context, workspaceAbsolute, lockData, workspaceIndex, includeNestedNodeModules: false, packages, visited, cancellationToken); + + var workspaceNodeModules = Path.Combine(workspaceAbsolute, "node_modules"); + if (Directory.Exists(workspaceNodeModules)) + { + pendingNodeModuleRoots.Add(workspaceNodeModules); + } + } + + var nodeModules = Path.Combine(context.RootPath, "node_modules"); + TraverseDirectory(context, nodeModules, lockData, workspaceIndex, packages, visited, cancellationToken); + + foreach (var pendingRoot in pendingNodeModuleRoots.OrderBy(static path => path, StringComparer.Ordinal)) + { + TraverseDirectory(context, pendingRoot, lockData, workspaceIndex, packages, visited, cancellationToken); + } + + return packages; + } + + private static void TraverseDirectory( + LanguageAnalyzerContext context, + string directory, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + List packages, + HashSet visited, + CancellationToken cancellationToken) + { + if (!Directory.Exists(directory)) + { + return; + } + + foreach (var child in Directory.EnumerateDirectories(directory)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var name = Path.GetFileName(child); + if (string.IsNullOrEmpty(name)) + { + continue; + } + + if (ShouldSkipDirectory(name)) + { + continue; + } + + if (string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase)) + { + TraversePnpmStore(context, child, lockData, workspaceIndex, packages, visited, cancellationToken); + continue; + } + + if (name.StartsWith('@')) + { + foreach (var scoped in Directory.EnumerateDirectories(child)) + { + ProcessPackageDirectory(context, scoped, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, cancellationToken); + } + continue; + } + + ProcessPackageDirectory(context, child, lockData, workspaceIndex, includeNestedNodeModules: true, packages, visited, cancellationToken); + } + } + + private static void TraversePnpmStore( + LanguageAnalyzerContext context, + string pnpmDirectory, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + List packages, + HashSet visited, + CancellationToken cancellationToken) + { + foreach (var storeEntry in Directory.EnumerateDirectories(pnpmDirectory)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var nestedNodeModules = Path.Combine(storeEntry, "node_modules"); + if (Directory.Exists(nestedNodeModules)) + { + TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, cancellationToken); + } + } + } + + private static void ProcessPackageDirectory( + LanguageAnalyzerContext context, + string directory, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + bool includeNestedNodeModules, + List packages, + HashSet visited, + CancellationToken cancellationToken) + { + var packageJsonPath = Path.Combine(directory, "package.json"); + var relativeDirectory = NormalizeRelativeDirectory(context, directory); + + if (!visited.Add(relativeDirectory)) + { + // Already processed this path. + if (includeNestedNodeModules) + { + TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, cancellationToken); + } + return; + } + + if (File.Exists(packageJsonPath)) + { + var package = TryCreatePackage(context, packageJsonPath, relativeDirectory, lockData, workspaceIndex, cancellationToken); + if (package is not null) + { + packages.Add(package); + } + } + + if (includeNestedNodeModules) + { + TraverseNestedNodeModules(context, directory, lockData, workspaceIndex, packages, visited, cancellationToken); + } + } + + private static void TraverseNestedNodeModules( + LanguageAnalyzerContext context, + string directory, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + List packages, + HashSet visited, + CancellationToken cancellationToken) + { + var nestedNodeModules = Path.Combine(directory, "node_modules"); + TraverseDirectory(context, nestedNodeModules, lockData, workspaceIndex, packages, visited, cancellationToken); + } + + private static NodePackage? TryCreatePackage( + LanguageAnalyzerContext context, + string packageJsonPath, + string relativeDirectory, + NodeLockData lockData, + NodeWorkspaceIndex workspaceIndex, + CancellationToken cancellationToken) + { + try + { + using var stream = File.OpenRead(packageJsonPath); + using var document = JsonDocument.Parse(stream); + + var root = document.RootElement; + if (!root.TryGetProperty("name", out var nameElement)) + { + return null; + } + + var name = nameElement.GetString(); + if (string.IsNullOrWhiteSpace(name)) + { + return null; + } + + if (!root.TryGetProperty("version", out var versionElement)) + { + return null; + } + + var version = versionElement.GetString(); + if (string.IsNullOrWhiteSpace(version)) + { + return null; + } + + bool? isPrivate = null; + if (root.TryGetProperty("private", out var privateElement) && privateElement.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + isPrivate = privateElement.GetBoolean(); + } + + var lockEntry = lockData.TryGet(relativeDirectory, name, out var entry) ? entry : null; + var locator = BuildLocator(relativeDirectory); + var usedByEntrypoint = context.UsageHints.IsPathUsed(packageJsonPath); + + var isWorkspaceMember = workspaceIndex.TryGetMember(relativeDirectory, out var workspaceRoot); + var workspaceTargets = ExtractWorkspaceTargets(relativeDirectory, root, workspaceIndex); + var workspaceLink = !isWorkspaceMember && workspaceIndex.TryGetWorkspacePathByName(name, out var workspacePathByName) + ? NormalizeRelativeDirectory(context, Path.Combine(context.RootPath, relativeDirectory)) + : null; + var lifecycleScripts = ExtractLifecycleScripts(root); + + return new NodePackage( + name: name.Trim(), + version: version.Trim(), + relativePath: relativeDirectory, + packageJsonLocator: locator, + isPrivate: isPrivate, + lockEntry: lockEntry, + isWorkspaceMember: isWorkspaceMember, + workspaceRoot: workspaceRoot, + workspaceTargets: workspaceTargets, + workspaceLink: workspaceLink, + lifecycleScripts: lifecycleScripts, + usedByEntrypoint: usedByEntrypoint); + } + catch (IOException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + private static string NormalizeRelativeDirectory(LanguageAnalyzerContext context, string directory) + { + var relative = context.GetRelativePath(directory); + if (string.IsNullOrEmpty(relative) || relative == ".") + { + return string.Empty; + } + + return relative.Replace(Path.DirectorySeparatorChar, '/'); + } + + private static string BuildLocator(string relativeDirectory) + { + if (string.IsNullOrEmpty(relativeDirectory)) + { + return "package.json"; + } + + return relativeDirectory + "/package.json"; + } + + private static bool ShouldSkipDirectory(string name) + { + if (name.Length == 0) + { + return true; + } + + if (name[0] == '.') + { + return !string.Equals(name, ".pnpm", StringComparison.OrdinalIgnoreCase); + } + + return IgnoredDirectories.Any(ignored => string.Equals(name, ignored, StringComparison.OrdinalIgnoreCase)); + } + + private static IReadOnlyList ExtractWorkspaceTargets(string relativeDirectory, JsonElement root, NodeWorkspaceIndex workspaceIndex) + { + var dependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "dependencies")); + var devDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "devDependencies")); + var peerDependencies = workspaceIndex.ResolveWorkspaceTargets(relativeDirectory, TryGetProperty(root, "peerDependencies")); + + if (dependencies.Count == 0 && devDependencies.Count == 0 && peerDependencies.Count == 0) + { + return Array.Empty(); + } + + var combined = new HashSet(StringComparer.Ordinal); + foreach (var item in dependencies) + { + combined.Add(item); + } + foreach (var item in devDependencies) + { + combined.Add(item); + } + foreach (var item in peerDependencies) + { + combined.Add(item); + } + + return combined.OrderBy(static x => x, StringComparer.Ordinal).ToArray(); + } + + private static JsonElement? TryGetProperty(JsonElement element, string propertyName) + => element.TryGetProperty(propertyName, out var property) ? property : null; + + private static IReadOnlyList ExtractLifecycleScripts(JsonElement root) + { + if (!root.TryGetProperty("scripts", out var scriptsElement) || scriptsElement.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var lifecycleScripts = new Dictionary(StringComparer.OrdinalIgnoreCase); + + foreach (var script in scriptsElement.EnumerateObject()) + { + if (!IsLifecycleScriptName(script.Name)) + { + continue; + } + + if (script.Value.ValueKind != JsonValueKind.String) + { + continue; + } + + var command = script.Value.GetString(); + if (string.IsNullOrWhiteSpace(command)) + { + continue; + } + + var canonicalName = script.Name.Trim().ToLowerInvariant(); + var lifecycleScript = new NodeLifecycleScript(canonicalName, command); + + if (!lifecycleScripts.ContainsKey(canonicalName)) + { + NodeAnalyzerMetrics.RecordLifecycleScript(canonicalName); + } + + lifecycleScripts[canonicalName] = lifecycleScript; + } + + if (lifecycleScripts.Count == 0) + { + return Array.Empty(); + } + + return lifecycleScripts.Values + .OrderBy(static script => script.Name, StringComparer.Ordinal) + .ToArray(); + } + + private static bool IsLifecycleScriptName(string name) + => name.Equals("preinstall", StringComparison.OrdinalIgnoreCase) + || name.Equals("install", StringComparison.OrdinalIgnoreCase) + || name.Equals("postinstall", StringComparison.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs new file mode 100644 index 00000000..89af3039 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Internal/NodeWorkspaceIndex.cs @@ -0,0 +1,278 @@ +using System.Text.Json; + +namespace StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +internal sealed class NodeWorkspaceIndex +{ + private readonly string _rootPath; + private readonly HashSet _workspacePaths; + private readonly Dictionary _workspaceByName; + + private NodeWorkspaceIndex(string rootPath, HashSet workspacePaths, Dictionary workspaceByName) + { + _rootPath = rootPath; + _workspacePaths = workspacePaths; + _workspaceByName = workspaceByName; + } + + public static NodeWorkspaceIndex Create(string rootPath) + { + var normalizedRoot = Path.GetFullPath(rootPath); + var workspacePaths = new HashSet(StringComparer.Ordinal); + var workspaceByName = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var packageJsonPath = Path.Combine(normalizedRoot, "package.json"); + if (!File.Exists(packageJsonPath)) + { + return new NodeWorkspaceIndex(normalizedRoot, workspacePaths, workspaceByName); + } + + try + { + using var stream = File.OpenRead(packageJsonPath); + using var document = JsonDocument.Parse(stream); + var root = document.RootElement; + if (!root.TryGetProperty("workspaces", out var workspacesElement)) + { + return new NodeWorkspaceIndex(normalizedRoot, workspacePaths, workspaceByName); + } + + var patterns = ExtractPatterns(workspacesElement); + foreach (var pattern in patterns) + { + foreach (var workspacePath in ExpandPattern(normalizedRoot, pattern)) + { + if (string.IsNullOrWhiteSpace(workspacePath)) + { + continue; + } + + workspacePaths.Add(workspacePath); + var packagePath = Path.Combine(normalizedRoot, workspacePath.Replace('/', Path.DirectorySeparatorChar), "package.json"); + if (!File.Exists(packagePath)) + { + continue; + } + + try + { + using var workspaceStream = File.OpenRead(packagePath); + using var workspaceDoc = JsonDocument.Parse(workspaceStream); + if (workspaceDoc.RootElement.TryGetProperty("name", out var nameElement)) + { + var name = nameElement.GetString(); + if (!string.IsNullOrWhiteSpace(name)) + { + workspaceByName[name] = workspacePath!; + } + } + } + catch (IOException) + { + // Ignore unreadable workspace package definitions. + } + catch (JsonException) + { + // Ignore malformed workspace package definitions. + } + } + } + } + catch (IOException) + { + // If the root package.json is unreadable we treat as no workspaces. + } + catch (JsonException) + { + // Malformed root package.json: treat as no workspaces. + } + + return new NodeWorkspaceIndex(normalizedRoot, workspacePaths, workspaceByName); + } + + public IEnumerable GetMembers() + => _workspacePaths.OrderBy(static path => path, StringComparer.Ordinal); + + public bool TryGetMember(string relativePath, out string normalizedPath) + { + if (string.IsNullOrEmpty(relativePath)) + { + normalizedPath = string.Empty; + return false; + } + + var normalized = NormalizeRelative(relativePath); + if (_workspacePaths.Contains(normalized)) + { + normalizedPath = normalized; + return true; + } + + normalizedPath = string.Empty; + return false; + } + + public bool TryGetWorkspacePathByName(string packageName, out string? relativePath) + => _workspaceByName.TryGetValue(packageName, out relativePath); + + public IReadOnlyList ResolveWorkspaceTargets(string relativeDirectory, JsonElement? dependencies) + { + if (dependencies is null || dependencies.Value.ValueKind != JsonValueKind.Object) + { + return Array.Empty(); + } + + var result = new HashSet(StringComparer.Ordinal); + foreach (var property in dependencies.Value.EnumerateObject()) + { + var value = property.Value; + if (value.ValueKind != JsonValueKind.String) + { + continue; + } + + var targetSpec = value.GetString(); + if (string.IsNullOrWhiteSpace(targetSpec)) + { + continue; + } + + const string workspacePrefix = "workspace:"; + if (!targetSpec.StartsWith(workspacePrefix, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var descriptor = targetSpec[workspacePrefix.Length..].Trim(); + if (string.IsNullOrEmpty(descriptor) || descriptor is "*" or "^") + { + if (_workspaceByName.TryGetValue(property.Name, out var workspaceByName)) + { + result.Add(workspaceByName); + } + + continue; + } + + if (TryResolveWorkspaceTarget(relativeDirectory, descriptor, out var resolved)) + { + result.Add(resolved); + } + } + + if (result.Count == 0) + { + return Array.Empty(); + } + + return result.OrderBy(static x => x, StringComparer.Ordinal).ToArray(); + } + + public bool TryResolveWorkspaceTarget(string relativeDirectory, string descriptor, out string normalized) + { + normalized = string.Empty; + var baseDirectory = string.IsNullOrEmpty(relativeDirectory) ? string.Empty : relativeDirectory; + var baseAbsolute = Path.GetFullPath(Path.Combine(_rootPath, baseDirectory)); + var candidate = Path.GetFullPath(Path.Combine(baseAbsolute, descriptor.Replace('/', Path.DirectorySeparatorChar))); + if (!IsUnderRoot(_rootPath, candidate)) + { + return false; + } + + var relative = NormalizeRelative(Path.GetRelativePath(_rootPath, candidate)); + if (_workspacePaths.Contains(relative)) + { + normalized = relative; + return true; + } + + return false; + } + + private static IEnumerable ExtractPatterns(JsonElement workspacesElement) + { + if (workspacesElement.ValueKind == JsonValueKind.Array) + { + foreach (var item in workspacesElement.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.String) + { + var value = item.GetString(); + if (!string.IsNullOrWhiteSpace(value)) + { + yield return value.Trim(); + } + } + } + } + else if (workspacesElement.ValueKind == JsonValueKind.Object) + { + if (workspacesElement.TryGetProperty("packages", out var packagesElement) && packagesElement.ValueKind == JsonValueKind.Array) + { + foreach (var pattern in ExtractPatterns(packagesElement)) + { + yield return pattern; + } + } + } + } + + private static IEnumerable ExpandPattern(string rootPath, string pattern) + { + var cleanedPattern = pattern.Replace('\\', '/').Trim(); + if (cleanedPattern.EndsWith("/*", StringComparison.Ordinal)) + { + var baseSegment = cleanedPattern[..^2]; + var baseAbsolute = CombineAndNormalize(rootPath, baseSegment); + if (baseAbsolute is null || !Directory.Exists(baseAbsolute)) + { + yield break; + } + + foreach (var directory in Directory.EnumerateDirectories(baseAbsolute)) + { + var normalized = NormalizeRelative(Path.GetRelativePath(rootPath, directory)); + yield return normalized; + } + } + else + { + var absolute = CombineAndNormalize(rootPath, cleanedPattern); + if (absolute is null || !Directory.Exists(absolute)) + { + yield break; + } + + var normalized = NormalizeRelative(Path.GetRelativePath(rootPath, absolute)); + yield return normalized; + } + } + + private static string? CombineAndNormalize(string rootPath, string relative) + { + var candidate = Path.GetFullPath(Path.Combine(rootPath, relative.Replace('/', Path.DirectorySeparatorChar))); + return IsUnderRoot(rootPath, candidate) ? candidate : null; + } + + private static string NormalizeRelative(string relativePath) + { + if (string.IsNullOrEmpty(relativePath) || relativePath == ".") + { + return string.Empty; + } + + var normalized = relativePath.Replace('\\', '/'); + normalized = normalized.TrimStart('.', '/'); + return normalized; + } + + private static bool IsUnderRoot(string rootPath, string absolutePath) + { + if (OperatingSystem.IsWindows()) + { + return absolutePath.StartsWith(rootPath, StringComparison.OrdinalIgnoreCase); + } + + return absolutePath.StartsWith(rootPath, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs new file mode 100644 index 00000000..76bea37f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/NodeLanguageAnalyzer.cs @@ -0,0 +1,37 @@ +using StellaOps.Scanner.Analyzers.Lang.Node.Internal; + +namespace StellaOps.Scanner.Analyzers.Lang.Node; + +public sealed class NodeLanguageAnalyzer : ILanguageAnalyzer +{ + public string Id => "node"; + + public string DisplayName => "Node.js Analyzer"; + + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentNullException.ThrowIfNull(writer); + + var lockData = await NodeLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false); + var packages = NodePackageCollector.CollectPackages(context, lockData, cancellationToken); + + foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var metadata = package.CreateMetadata(); + var evidence = package.CreateEvidence(); + + writer.AddFromPurl( + analyzerId: Id, + purl: package.Purl, + name: package.Name, + version: package.Version, + type: "npm", + metadata: metadata, + evidence: evidence, + usedByEntrypoint: package.IsUsedByEntrypoint); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/Placeholder.cs b/src/StellaOps.Scanner.Analyzers.Lang.Node/Placeholder.cs new file mode 100644 index 00000000..51045302 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/Placeholder.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Node; + +internal static class Placeholder +{ + // Analyzer implementation will be added during Sprint LA1. +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md new file mode 100644 index 00000000..ae5e95c2 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Node/TASKS.md @@ -0,0 +1,10 @@ +# Node Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-302A | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-307 | Build deterministic module graph walker covering npm, Yarn, and PNPM; capture package.json provenance and integrity metadata. | Walker indexes >100 k modules in <1.5 s (hot cache); golden fixtures verify deterministic ordering and path normalization. | +| 2 | SCANNER-ANALYZERS-LANG-10-302B | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302A | Resolve workspaces/symlinks and attribute components to originating package with usage hints; guard against directory traversal. | Workspace attribution accurate on multi-workspace fixture; symlink resolver proves canonical path; security tests ensure no traversal. | +| 3 | SCANNER-ANALYZERS-LANG-10-302C | DONE (2025-10-19) | SCANNER-ANALYZERS-LANG-10-302B | Surface script metadata (postinstall/preinstall) and policy hints; emit telemetry counters and evidence records. | Analyzer output includes script metadata + evidence; metrics `scanner_analyzer_node_scripts_total` recorded; policy hints documented. | +| 4 | SCANNER-ANALYZERS-LANG-10-307N | TODO | SCANNER-ANALYZERS-LANG-10-302C | Integrate shared helpers for license/licence evidence, canonical JSON serialization, and usage flag propagation. | Reuse shared helpers without duplication; unit tests confirm stable metadata merge; no analyzer-specific serializer drift. | +| 5 | SCANNER-ANALYZERS-LANG-10-308N | TODO | SCANNER-ANALYZERS-LANG-10-307N | Author determinism harness + fixtures for Node analyzer; add benchmark suite. | Fixtures committed under `Fixtures/lang/node/`; determinism CI job compares JSON snapshots; benchmark CSV published. | +| 6 | SCANNER-ANALYZERS-LANG-10-309N | TODO | SCANNER-ANALYZERS-LANG-10-308N | Package Node analyzer as restart-time plug-in (manifest, DI registration, Offline Kit notes). | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer after restart; Offline Kit docs updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang.Python/AGENTS.md new file mode 100644 index 00000000..977b828a --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/AGENTS.md @@ -0,0 +1,32 @@ +# StellaOps.Scanner.Analyzers.Lang.Python — Agent Charter + +## Role +Implement the Python analyzer plug-in that inspects installed distributions, RECORD hashes, entry points, and editable installs to feed Scanner SBOM views. + +## Scope +- Parse `*.dist-info` and `*.data` directories, validating `METADATA`, `RECORD`, and `entry_points.txt`. +- Detect editable installs and pip caches, reconciling metadata with actual files. +- Integrate EntryTrace usage hints for runtime entry points and flag missing RECORD hashes. +- Package plug-in manifest and ensure deterministic fixtures + benchmarks. + +## Out of Scope +- Language analyzers for other ecosystems. +- Policy evaluation, vulnerability correlation, or packaging into UI flows. +- Building Python interpreters or executing scripts (analysis is static only). + +## Expectations +- Deterministic RECORD hashing with streaming IO; fallback heuristics clearly flagged. +- Performance target: ≥75 MB/s RECORD verification, end-to-end fixture <2.0 s. +- Offline-first: no PyPI calls; relies on local metadata only. +- Rich telemetry (components counted, hash mismatches) following Scanner metrics schema. +- Keep `TASKS.md` and `SPRINTS_LANG_IMPLEMENTATION_PLAN.md` in sync. + +## Dependencies +- Shared language analyzer infrastructure. +- EntryTrace usage hints (for script activation). +- Worker dispatcher for plug-in loading. + +## Testing & Artifacts +- Golden fixtures for venv, virtualenv, pipx, and editable installs. +- Benchmark results comparing hash-check throughput against competitor tools. +- Offline Kit guidance for bundling standard library metadata if required. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.Python/GlobalUsings.cs new file mode 100644 index 00000000..69c494f5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/GlobalUsings.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/Placeholder.cs b/src/StellaOps.Scanner.Analyzers.Lang.Python/Placeholder.cs new file mode 100644 index 00000000..8379b716 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/Placeholder.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Python; + +internal static class Placeholder +{ + // Analyzer implementation will be added during Sprint LA2. +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md new file mode 100644 index 00000000..864d7a69 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md @@ -0,0 +1,10 @@ +# Python Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-303A | TODO | SCANNER-ANALYZERS-LANG-10-307 | STREAM-based parser for `*.dist-info` (`METADATA`, `WHEEL`, `entry_points.txt`) with normalization + evidence capture. | Parser handles CPython 3.8–3.12 metadata variations; fixtures confirm canonical ordering and UTF-8 handling. | +| 2 | SCANNER-ANALYZERS-LANG-10-303B | TODO | SCANNER-ANALYZERS-LANG-10-303A | RECORD hash verifier with chunked hashing, Zip64 support, and mismatch diagnostics. | Verifier processes 5 GB RECORD fixture without allocations >2 MB; mismatches produce deterministic evidence records. | +| 3 | SCANNER-ANALYZERS-LANG-10-303C | TODO | SCANNER-ANALYZERS-LANG-10-303B | Editable install + pip cache detection; integrate EntryTrace hints for runtime usage flags. | Editable installs resolved to source path; usage flags propagated; regression tests cover mixed editable + wheel installs. | +| 4 | SCANNER-ANALYZERS-LANG-10-307P | TODO | SCANNER-ANALYZERS-LANG-10-303C | Shared helper integration (license metadata, quiet provenance, component merging). | Shared helpers reused; analyzer-specific metadata minimal; deterministic merge tests pass. | +| 5 | SCANNER-ANALYZERS-LANG-10-308P | TODO | SCANNER-ANALYZERS-LANG-10-307P | Golden fixtures + determinism harness for Python analyzer; add benchmark and hash throughput reporting. | Fixtures under `Fixtures/lang/python/`; determinism CI guard; benchmark CSV added with threshold alerts. | +| 6 | SCANNER-ANALYZERS-LANG-10-309P | TODO | SCANNER-ANALYZERS-LANG-10-308P | Package plug-in (manifest, DI registration) and document Offline Kit bundling of Python stdlib metadata if needed. | Manifest copied to `plugins/scanner/analyzers/lang/`; Worker loads analyzer; Offline Kit doc updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang.Rust/AGENTS.md new file mode 100644 index 00000000..c41c745c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/AGENTS.md @@ -0,0 +1,29 @@ +# StellaOps.Scanner.Analyzers.Lang.Rust — Agent Charter + +## Role +Develop the Rust analyzer plug-in that resolves crates from metadata (`.fingerprint`, Cargo.lock, embedded markers) and provides deterministic fallbacks for stripped binaries. + +## Scope +- Locate Cargo metadata in container layers (registry cache, target fingerprints, embedded Git info). +- Parse symbol tables / section data to heuristically identify crates when metadata missing, tagging provenance appropriately. +- Integrate binary hash fallback with quiet provenance classification. +- Package plug-in manifest, determinism fixtures, and performance/coverage benchmarks. + +## Out of Scope +- Native linker analysis beyond crate attribution. +- Fetching Cargo registry metadata from the network. +- Policy decisions or UI surfacing. + +## Expectations +- Accurate crate attribution (≥85 % on curated fixtures) with explicit heuristic labeling. +- Analyzer runtime <1 s over 500 binary corpus; minimal allocations through pooling. +- Offline-first; rely on local Cargo data. +- Telemetry capturing heuristic vs verified evidence ratios. + +## Dependencies +- Shared language analyzer infrastructure; Worker dispatcher; optionally EntryTrace hints for runtime coverage. + +## Testing & Artifacts +- Fixtures for cargo workspaces, release builds, stripped binaries, vendor caches. +- Determinism + benchmark artifacts comparing to competitor scanners. +- ADR documenting heuristic boundaries + risk mitigations. diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang.Rust/GlobalUsings.cs new file mode 100644 index 00000000..69c494f5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/GlobalUsings.cs @@ -0,0 +1,7 @@ +global using System; +global using System.Collections.Generic; +global using System.IO; +global using System.Threading; +global using System.Threading.Tasks; + +global using StellaOps.Scanner.Analyzers.Lang; diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/Placeholder.cs b/src/StellaOps.Scanner.Analyzers.Lang.Rust/Placeholder.cs new file mode 100644 index 00000000..ce64b9ad --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/Placeholder.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Rust; + +internal static class Placeholder +{ + // Analyzer implementation will be added during Sprint LA5. +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/StellaOps.Scanner.Analyzers.Lang.Rust.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Rust/StellaOps.Scanner.Analyzers.Lang.Rust.csproj new file mode 100644 index 00000000..16dcc2e5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/StellaOps.Scanner.Analyzers.Lang.Rust.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md new file mode 100644 index 00000000..a3ea6341 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Rust/TASKS.md @@ -0,0 +1,10 @@ +# Rust Analyzer Task Flow + +| Seq | ID | Status | Depends on | Description | Exit Criteria | +|-----|----|--------|------------|-------------|---------------| +| 1 | SCANNER-ANALYZERS-LANG-10-306A | TODO | SCANNER-ANALYZERS-LANG-10-307 | Parse Cargo metadata (`Cargo.lock`, `.fingerprint`, `.metadata`) and map crates to components with evidence. | Fixtures confirm crate attribution ≥85 % coverage; metadata normalized; evidence includes path + hash. | +| 2 | SCANNER-ANALYZERS-LANG-10-306B | TODO | SCANNER-ANALYZERS-LANG-10-306A | Implement heuristic classifier using ELF section names, symbol mangling, and `.comment` data for stripped binaries. | Heuristic output flagged as `heuristic`; regression tests ensure no false “observed” classifications. | +| 3 | SCANNER-ANALYZERS-LANG-10-306C | TODO | SCANNER-ANALYZERS-LANG-10-306B | Integrate binary hash fallback (`bin:{sha256}`) and tie into shared quiet provenance helpers. | Fallback path deterministic; shared helpers reused; tests verify consistent hashing. | +| 4 | SCANNER-ANALYZERS-LANG-10-307R | TODO | SCANNER-ANALYZERS-LANG-10-306C | Finalize shared helper usage (license, usage flags) and concurrency-safe caches. | Analyzer uses shared utilities; concurrency tests pass; no race conditions. | +| 5 | SCANNER-ANALYZERS-LANG-10-308R | TODO | SCANNER-ANALYZERS-LANG-10-307R | Determinism fixtures + performance benchmarks; compare against competitor heuristic coverage. | Fixtures `Fixtures/lang/rust/` committed; determinism guard; benchmark shows ≥15 % better coverage vs competitor. | +| 6 | SCANNER-ANALYZERS-LANG-10-309R | TODO | SCANNER-ANALYZERS-LANG-10-308R | Package plug-in manifest + Offline Kit documentation; ensure Worker integration. | Manifest copied; Worker loads analyzer; Offline Kit doc updated. | diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageAnalyzerResultTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageAnalyzerResultTests.cs new file mode 100644 index 00000000..e4020d20 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageAnalyzerResultTests.cs @@ -0,0 +1,88 @@ +using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Core; + +public sealed class LanguageAnalyzerResultTests +{ + [Fact] + public async Task MergesDuplicateComponentsDeterministicallyAsync() + { + var analyzer = new DuplicateComponentAnalyzer(); + var engine = new LanguageAnalyzerEngine(new[] { analyzer }); + var root = TestPaths.CreateTemporaryDirectory(); + try + { + var context = new LanguageAnalyzerContext(root, TimeProvider.System); + var result = await engine.AnalyzeAsync(context, CancellationToken.None); + + var component = Assert.Single(result.Components); + Assert.Equal("purl::pkg:example/acme@2.0.0", component.ComponentKey); + Assert.Equal("pkg:example/acme@2.0.0", component.Purl); + Assert.True(component.UsedByEntrypoint); + Assert.Equal(2, component.Evidence.Count); + Assert.Equal(3, component.Metadata.Count); + + // Metadata retains stable ordering (sorted by key) + var keys = component.Metadata.Keys.ToArray(); + Assert.Equal(new[] { "artifactId", "groupId", "path" }, keys); + + // Evidence de-duplicates via comparison key + Assert.Equal(2, component.Evidence.Count); + } + finally + { + TestPaths.SafeDelete(root); + } + } + + private sealed class DuplicateComponentAnalyzer : ILanguageAnalyzer + { + public string Id => "duplicate"; + + public string DisplayName => "Duplicate Analyzer"; + + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) + { + await Task.Yield(); + + var metadataA = new[] + { + new KeyValuePair("groupId", "example"), + new KeyValuePair("artifactId", "acme") + }; + + var metadataB = new[] + { + new KeyValuePair("artifactId", "acme"), + new KeyValuePair("path", ".") + }; + + var evidence = new[] + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "manifest", "META-INF/MANIFEST.MF", null, null), + new LanguageComponentEvidence(LanguageEvidenceKind.Metadata, "pom", "pom.xml", "groupId=example", null) + }; + + writer.AddFromPurl( + analyzerId: Id, + purl: "pkg:example/acme@2.0.0", + name: "acme", + version: "2.0.0", + type: "example", + metadata: metadataA, + evidence: evidence, + usedByEntrypoint: true); + + // duplicate insert with different metadata ordering + writer.AddFromPurl( + analyzerId: Id, + purl: "pkg:example/acme@2.0.0", + name: "acme", + version: "2.0.0", + type: "example", + metadata: metadataB, + evidence: evidence, + usedByEntrypoint: false); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageComponentMapperTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageComponentMapperTests.cs new file mode 100644 index 00000000..a9c1db97 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Core/LanguageComponentMapperTests.cs @@ -0,0 +1,70 @@ +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Core; + +public sealed class LanguageComponentMapperTests +{ + [Fact] + public void ToComponentRecordsProjectsDeterministicComponents() + { + // Arrange + var analyzerId = "node"; + var records = new[] + { + LanguageComponentRecord.FromPurl( + analyzerId: analyzerId, + purl: "pkg:npm/example@1.0.0", + name: "example", + version: "1.0.0", + type: "npm", + metadata: new Dictionary() + { + ["path"] = "packages/app", + ["license"] = "MIT" + }, + evidence: new[] + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "package.json", "packages/app/package.json", null, "abc123") + }, + usedByEntrypoint: true), + LanguageComponentRecord.FromExplicitKey( + analyzerId: analyzerId, + componentKey: "bin::sha256:deadbeef", + purl: null, + name: "app-binary", + version: null, + type: "binary", + metadata: new Dictionary() + { + ["description"] = "Utility binary" + }, + evidence: new[] + { + new LanguageComponentEvidence(LanguageEvidenceKind.Derived, "entrypoint", "/usr/local/bin/app", "ENTRYPOINT", null) + }) + }; + + // Act + var layerDigest = LanguageComponentMapper.ComputeLayerDigest(analyzerId); + var results = LanguageComponentMapper.ToComponentRecords(analyzerId, records, layerDigest); + + // Assert + Assert.Equal(2, results.Length); + Assert.All(results, component => Assert.Equal(layerDigest, component.LayerDigest)); + + var first = results[0]; + Assert.Equal("bin::sha256:deadbeef", first.Identity.Key); + Assert.Equal("Utility binary", first.Metadata!.Properties!["stellaops.lang.meta.description"]); + Assert.Equal("derived", first.Evidence.Single().Kind); + + var second = results[1]; + Assert.Equal("pkg:npm/example@1.0.0", second.Identity.Key); // prefix removed + Assert.True(second.Usage.UsedByEntrypoint); + Assert.Contains("MIT", second.Metadata!.Licenses!); + Assert.Equal("packages/app", second.Metadata.Properties!["stellaops.lang.meta.path"]); + Assert.Equal("abc123", second.Metadata.Properties!["stellaops.lang.evidence.0.sha256"]); + Assert.Equal("file", second.Evidence.Single().Kind); + Assert.Equal("packages/app/package.json", second.Evidence.Single().Value); + Assert.Equal("package.json", second.Evidence.Single().Source); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Determinism/LanguageAnalyzerHarnessTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Determinism/LanguageAnalyzerHarnessTests.cs new file mode 100644 index 00000000..05e42803 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Determinism/LanguageAnalyzerHarnessTests.cs @@ -0,0 +1,102 @@ +using StellaOps.Scanner.Analyzers.Lang; +using StellaOps.Scanner.Analyzers.Lang.Tests.Harness; +using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Determinism; + +public sealed class LanguageAnalyzerHarnessTests +{ + [Fact] + public async Task HarnessProducesDeterministicOutputAsync() + { + var fixturePath = TestPaths.ResolveFixture("determinism", "basic", "input"); + var goldenPath = TestPaths.ResolveFixture("determinism", "basic", "expected.json"); + var cancellationToken = TestContext.Current.CancellationToken; + + var analyzers = new ILanguageAnalyzer[] + { + new FakeLanguageAnalyzer( + "fake-java", + LanguageComponentRecord.FromPurl( + analyzerId: "fake-java", + purl: "pkg:maven/org.example/example-lib@1.2.3", + name: "example-lib", + version: "1.2.3", + type: "maven", + metadata: new Dictionary + { + ["groupId"] = "org.example", + ["artifactId"] = "example-lib", + }, + evidence: new [] + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "pom.properties", "META-INF/maven/org.example/example-lib/pom.properties", null, "abc123"), + }), + LanguageComponentRecord.FromExplicitKey( + analyzerId: "fake-java", + componentKey: "bin::sha256:deadbeef", + purl: null, + name: "example-cli", + version: null, + type: "bin", + metadata: new Dictionary + { + ["sha256"] = "deadbeef", + }, + evidence: new [] + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "binary", "usr/local/bin/example", null, "deadbeef"), + })), + new FakeLanguageAnalyzer( + "fake-node", + LanguageComponentRecord.FromPurl( + analyzerId: "fake-node", + purl: "pkg:npm/example-package@4.5.6", + name: "example-package", + version: "4.5.6", + type: "npm", + metadata: new Dictionary + { + ["workspace"] = "packages/example", + }, + evidence: new [] + { + new LanguageComponentEvidence(LanguageEvidenceKind.File, "package.json", "packages/example/package.json", null, null), + }, + usedByEntrypoint: true)), + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync(fixturePath, goldenPath, analyzers, cancellationToken); + + var first = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixturePath, analyzers, cancellationToken); + var second = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixturePath, analyzers, cancellationToken); + Assert.Equal(first, second); + } + + private sealed class FakeLanguageAnalyzer : ILanguageAnalyzer + { + private readonly IReadOnlyList _components; + + public FakeLanguageAnalyzer(string id, params LanguageComponentRecord[] components) + { + Id = id; + DisplayName = id; + _components = components ?? Array.Empty(); + } + + public string Id { get; } + + public string DisplayName { get; } + + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken) + { + await Task.Delay(5, cancellationToken).ConfigureAwait(false); // ensure asynchrony is handled + + // Intentionally add in reverse order to prove determinism. + foreach (var component in _components.Reverse()) + { + writer.Add(component); + } + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/expected.json new file mode 100644 index 00000000..53370f06 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/expected.json @@ -0,0 +1,60 @@ +[ + { + "analyzerId": "fake-java", + "componentKey": "bin::sha256:deadbeef", + "name": "example-cli", + "type": "bin", + "usedByEntrypoint": false, + "metadata": { + "sha256": "deadbeef" + }, + "evidence": [ + { + "kind": "file", + "source": "binary", + "locator": "usr/local/bin/example", + "sha256": "deadbeef" + } + ] + }, + { + "analyzerId": "fake-java", + "componentKey": "purl::pkg:maven/org.example/example-lib@1.2.3", + "purl": "pkg:maven/org.example/example-lib@1.2.3", + "name": "example-lib", + "version": "1.2.3", + "type": "maven", + "usedByEntrypoint": false, + "metadata": { + "artifactId": "example-lib", + "groupId": "org.example" + }, + "evidence": [ + { + "kind": "file", + "source": "pom.properties", + "locator": "META-INF/maven/org.example/example-lib/pom.properties", + "sha256": "abc123" + } + ] + }, + { + "analyzerId": "fake-node", + "componentKey": "purl::pkg:npm/example-package@4.5.6", + "purl": "pkg:npm/example-package@4.5.6", + "name": "example-package", + "version": "4.5.6", + "type": "npm", + "usedByEntrypoint": true, + "metadata": { + "workspace": "packages/example" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/example/package.json" + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/input/placeholder.txt b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/input/placeholder.txt new file mode 100644 index 00000000..63a8db7b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/determinism/basic/input/placeholder.txt @@ -0,0 +1 @@ +sample diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/java/basic/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/java/basic/expected.json new file mode 100644 index 00000000..4f78fe2e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/java/basic/expected.json @@ -0,0 +1,35 @@ +[ + { + "analyzerId": "java", + "componentKey": "purl::pkg:maven/com/example/demo@1.0.0", + "purl": "pkg:maven/com/example/demo@1.0.0", + "name": "demo", + "version": "1.0.0", + "type": "maven", + "usedByEntrypoint": true, + "metadata": { + "artifactId": "demo", + "displayName": "Demo Library", + "groupId": "com.example", + "jarPath": "libs/demo.jar", + "manifestTitle": "Demo", + "manifestVendor": "Example Corp", + "manifestVersion": "1.0.0", + "packaging": "jar" + }, + "evidence": [ + { + "kind": "file", + "source": "MANIFEST.MF", + "locator": "libs/demo.jar!META-INF/MANIFEST.MF", + "value": "title=Demo;version=1.0.0;vendor=Example Corp" + }, + { + "kind": "file", + "source": "pom.properties", + "locator": "libs/demo.jar!META-INF/maven/com.example/demo/pom.properties", + "sha256": "c20f36aa1b9d89d28cf9ed131519ffd6287a4dac0c7cb926130496f3f8157bf1" + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/expected.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/expected.json new file mode 100644 index 00000000..e7335182 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/expected.json @@ -0,0 +1,134 @@ +[ + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/left-pad@1.3.0", + "purl": "pkg:npm/left-pad@1.3.0", + "name": "left-pad", + "version": "1.3.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "integrity": "sha512-LEFTPAD", + "path": "packages/app/node_modules/left-pad", + "resolved": "https://registry.example/left-pad-1.3.0.tgz" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/app/node_modules/left-pad/package.json" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/lib@2.0.1", + "purl": "pkg:npm/lib@2.0.1", + "name": "lib", + "version": "2.0.1", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "integrity": "sha512-LIB", + "path": "packages/lib", + "resolved": "https://registry.example/lib-2.0.1.tgz", + "workspaceLink": "packages/app/node_modules/lib", + "workspaceMember": "true", + "workspaceRoot": "packages/lib" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/app/node_modules/lib/package.json" + }, + { + "kind": "file", + "source": "package.json", + "locator": "packages/lib/package.json" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/root-workspace@1.0.0", + "purl": "pkg:npm/root-workspace@1.0.0", + "name": "root-workspace", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "path": ".", + "private": "true" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "package.json" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/shared@3.1.4", + "purl": "pkg:npm/shared@3.1.4", + "name": "shared", + "version": "3.1.4", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "integrity": "sha512-SHARED", + "path": "packages/shared", + "resolved": "https://registry.example/shared-3.1.4.tgz", + "workspaceLink": "packages/app/node_modules/shared", + "workspaceMember": "true", + "workspaceRoot": "packages/shared", + "workspaceTargets": "packages/lib" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/app/node_modules/shared/package.json" + }, + { + "kind": "file", + "source": "package.json", + "locator": "packages/shared/package.json" + } + ] + }, + { + "analyzerId": "node", + "componentKey": "purl::pkg:npm/workspace-app@1.0.0", + "purl": "pkg:npm/workspace-app@1.0.0", + "name": "workspace-app", + "version": "1.0.0", + "type": "npm", + "usedByEntrypoint": false, + "metadata": { + "installScripts": "true", + "path": "packages/app", + "policyHint.installLifecycle": "postinstall", + "script.postinstall": "node scripts/setup.js", + "workspaceMember": "true", + "workspaceRoot": "packages/app", + "workspaceTargets": "packages/lib;packages/shared" + }, + "evidence": [ + { + "kind": "file", + "source": "package.json", + "locator": "packages/app/package.json" + }, + { + "kind": "metadata", + "source": "package.json:scripts", + "locator": "packages/app/package.json#scripts.postinstall", + "value": "node scripts/setup.js", + "sha256": "f9ae4e4c9313857d1acc31947cee9984232cbefe93c8a56c718804744992728a" + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package-lock.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package-lock.json new file mode 100644 index 00000000..84abb8da --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package-lock.json @@ -0,0 +1,49 @@ +{ + "name": "root-workspace", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": { + "name": "root-workspace", + "version": "1.0.0", + "private": true, + "workspaces": [ + "packages/*" + ] + }, + "packages/app": { + "name": "workspace-app", + "version": "1.0.0" + }, + "packages/lib": { + "name": "lib", + "version": "2.0.1", + "resolved": "https://registry.example/lib-2.0.1.tgz", + "integrity": "sha512-LIB" + }, + "packages/shared": { + "name": "shared", + "version": "3.1.4", + "resolved": "https://registry.example/shared-3.1.4.tgz", + "integrity": "sha512-SHARED" + }, + "packages/app/node_modules/lib": { + "name": "lib", + "version": "2.0.1", + "resolved": "https://registry.example/lib-2.0.1.tgz", + "integrity": "sha512-LIB" + }, + "packages/app/node_modules/shared": { + "name": "shared", + "version": "3.1.4", + "resolved": "https://registry.example/shared-3.1.4.tgz", + "integrity": "sha512-SHARED" + }, + "packages/app/node_modules/left-pad": { + "name": "left-pad", + "version": "1.3.0", + "resolved": "https://registry.example/left-pad-1.3.0.tgz", + "integrity": "sha512-LEFTPAD" + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package.json new file mode 100644 index 00000000..03fded30 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/package.json @@ -0,0 +1,10 @@ +{ + "name": "root-workspace", + "version": "1.0.0", + "private": true, + "workspaces": [ + "packages/app", + "packages/lib", + "packages/shared" + ] +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/left-pad/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/left-pad/package.json new file mode 100644 index 00000000..0a32ad2c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/left-pad/package.json @@ -0,0 +1,5 @@ +{ + "name": "left-pad", + "version": "1.3.0", + "main": "index.js" +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/lib/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/lib/package.json new file mode 100644 index 00000000..4a4395f9 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/lib/package.json @@ -0,0 +1,5 @@ +{ + "name": "lib", + "version": "2.0.1", + "main": "index.js" +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/shared/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/shared/package.json new file mode 100644 index 00000000..cea61bc5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/node_modules/shared/package.json @@ -0,0 +1,5 @@ +{ + "name": "shared", + "version": "3.1.4", + "main": "index.js" +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/package.json new file mode 100644 index 00000000..191c13ae --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/package.json @@ -0,0 +1,11 @@ +{ + "name": "workspace-app", + "version": "1.0.0", + "dependencies": { + "lib": "workspace:../lib", + "shared": "workspace:../shared" + }, + "scripts": { + "postinstall": "node scripts/setup.js" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/scripts/setup.js b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/scripts/setup.js new file mode 100644 index 00000000..13f1a33c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/app/scripts/setup.js @@ -0,0 +1 @@ +console.log('setup'); diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/lib/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/lib/package.json new file mode 100644 index 00000000..9fba5e27 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/lib/package.json @@ -0,0 +1,7 @@ +{ + "name": "lib", + "version": "2.0.1", + "dependencies": { + "left-pad": "1.3.0" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/shared/package.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/shared/package.json new file mode 100644 index 00000000..0141611d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/node/workspaces/packages/shared/package.json @@ -0,0 +1,7 @@ +{ + "name": "shared", + "version": "3.1.4", + "dependencies": { + "lib": "workspace:../lib" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Harness/LanguageAnalyzerTestHarness.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Harness/LanguageAnalyzerTestHarness.cs new file mode 100644 index 00000000..f62d5c68 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Harness/LanguageAnalyzerTestHarness.cs @@ -0,0 +1,46 @@ +using StellaOps.Scanner.Analyzers.Lang; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Harness; + +public static class LanguageAnalyzerTestHarness +{ + public static async Task RunToJsonAsync(string fixturePath, IEnumerable analyzers, CancellationToken cancellationToken = default, LanguageUsageHints? usageHints = null) + { + if (string.IsNullOrWhiteSpace(fixturePath)) + { + throw new ArgumentException("Fixture path is required", nameof(fixturePath)); + } + + var engine = new LanguageAnalyzerEngine(analyzers ?? Array.Empty()); + var context = new LanguageAnalyzerContext(fixturePath, TimeProvider.System, usageHints); + var result = await engine.AnalyzeAsync(context, cancellationToken).ConfigureAwait(false); + return result.ToJson(indent: true); + } + + public static async Task AssertDeterministicAsync(string fixturePath, string goldenPath, IEnumerable analyzers, CancellationToken cancellationToken = default, LanguageUsageHints? usageHints = null) + { + var actual = await RunToJsonAsync(fixturePath, analyzers, cancellationToken, usageHints).ConfigureAwait(false); + var expected = await File.ReadAllTextAsync(goldenPath, cancellationToken).ConfigureAwait(false); + + // Normalize newlines for portability. + actual = NormalizeLineEndings(actual).TrimEnd(); + expected = NormalizeLineEndings(expected).TrimEnd(); + + if (!string.Equals(expected, actual, StringComparison.Ordinal)) + { + var actualPath = goldenPath + ".actual"; + var directory = Path.GetDirectoryName(actualPath); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllTextAsync(actualPath, actual, cancellationToken).ConfigureAwait(false); + } + + Assert.Equal(expected, actual); + } + + private static string NormalizeLineEndings(string value) + => value.Replace("\r\n", "\n", StringComparison.Ordinal); +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Java/JavaLanguageAnalyzerTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Java/JavaLanguageAnalyzerTests.cs new file mode 100644 index 00000000..975b5557 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Java/JavaLanguageAnalyzerTests.cs @@ -0,0 +1,33 @@ +using StellaOps.Scanner.Analyzers.Lang.Java; +using StellaOps.Scanner.Analyzers.Lang.Tests.Harness; +using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Java; + +public sealed class JavaLanguageAnalyzerTests +{ + [Fact] + public async Task ExtractsMavenArtifactFromJarAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var root = TestPaths.CreateTemporaryDirectory(); + try + { + var jarPath = JavaFixtureBuilder.CreateSampleJar(root); + var usageHints = new LanguageUsageHints(new[] { jarPath }); + var analyzers = new ILanguageAnalyzer[] { new JavaLanguageAnalyzer() }; + var goldenPath = TestPaths.ResolveFixture("java", "basic", "expected.json"); + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath: root, + goldenPath: goldenPath, + analyzers: analyzers, + cancellationToken: cancellationToken, + usageHints: usageHints); + } + finally + { + TestPaths.SafeDelete(root); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/Node/NodeLanguageAnalyzerTests.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Node/NodeLanguageAnalyzerTests.cs new file mode 100644 index 00000000..9dba7b62 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/Node/NodeLanguageAnalyzerTests.cs @@ -0,0 +1,27 @@ +using StellaOps.Scanner.Analyzers.Lang.Node; +using StellaOps.Scanner.Analyzers.Lang.Tests.Harness; +using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.Node; + +public sealed class NodeLanguageAnalyzerTests +{ + [Fact] + public async Task WorkspaceFixtureProducesDeterministicOutputAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var fixturePath = TestPaths.ResolveFixture("lang", "node", "workspaces"); + var goldenPath = Path.Combine(fixturePath, "expected.json"); + + var analyzers = new ILanguageAnalyzer[] + { + new NodeLanguageAnalyzer() + }; + + await LanguageAnalyzerTestHarness.AssertDeterministicAsync( + fixturePath, + goldenPath, + analyzers, + cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj b/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj new file mode 100644 index 00000000..5b19ea6e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/StellaOps.Scanner.Analyzers.Lang.Tests.csproj @@ -0,0 +1,46 @@ + + + net10.0 + preview + enable + enable + true + false + Exe + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs new file mode 100644 index 00000000..b14f0bed --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/JavaFixtureBuilder.cs @@ -0,0 +1,46 @@ +using System.IO.Compression; +using System.Text; + +namespace StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +public static class JavaFixtureBuilder +{ + public static string CreateSampleJar(string rootDirectory, string relativePath = "libs/demo.jar") + { + ArgumentNullException.ThrowIfNull(rootDirectory); + ArgumentException.ThrowIfNullOrEmpty(relativePath); + + var jarPath = Path.Combine(rootDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(jarPath)!); + + using var fileStream = new FileStream(jarPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: false); + + var timestamp = new DateTimeOffset(2024, 01, 01, 0, 0, 0, TimeSpan.Zero); + + var pomEntry = archive.CreateEntry("META-INF/maven/com.example/demo/pom.properties", CompressionLevel.NoCompression); + pomEntry.LastWriteTime = timestamp; + using (var writer = new StreamWriter(pomEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("# Test pom.properties"); + writer.WriteLine("groupId=com.example"); + writer.WriteLine("artifactId=demo"); + writer.WriteLine("version=1.0.0"); + writer.WriteLine("name=Demo Library"); + writer.WriteLine("packaging=jar"); + } + + var manifestEntry = archive.CreateEntry("META-INF/MANIFEST.MF", CompressionLevel.NoCompression); + manifestEntry.LastWriteTime = timestamp; + using (var writer = new StreamWriter(manifestEntry.Open(), Encoding.UTF8, leaveOpen: false)) + { + writer.WriteLine("Manifest-Version: 1.0"); + writer.WriteLine("Implementation-Title: Demo"); + writer.WriteLine("Implementation-Version: 1.0.0"); + writer.WriteLine("Implementation-Vendor: Example Corp"); + writer.WriteLine(); + } + + return jarPath; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/TestPaths.cs b/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/TestPaths.cs new file mode 100644 index 00000000..b0adf80b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/TestUtilities/TestPaths.cs @@ -0,0 +1,53 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; + +public static class TestPaths +{ + public static string ResolveFixture(params string[] segments) + { + var baseDirectory = AppContext.BaseDirectory; + var parts = new List { baseDirectory }; + parts.AddRange(new[] { "Fixtures" }); + parts.AddRange(segments); + return Path.GetFullPath(Path.Combine(parts.ToArray())); + } + + public static string CreateTemporaryDirectory() + { + var root = Path.Combine(AppContext.BaseDirectory, "tmp", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + return root; + } + + public static void SafeDelete(string directory) + { + if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory)) + { + return; + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch + { + // Swallow cleanup exceptions to avoid masking test failures. + } + } + + public static string ResolveProjectRoot() + { + var directory = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(directory)) + { + if (File.Exists(Path.Combine(directory, "StellaOps.Scanner.Analyzers.Lang.Tests.csproj"))) + { + return directory; + } + + directory = Path.GetDirectoryName(directory) ?? string.Empty; + } + + throw new InvalidOperationException("Unable to locate project root."); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang.Tests/xunit.runner.json b/src/StellaOps.Scanner.Analyzers.Lang.Tests/xunit.runner.json new file mode 100644 index 00000000..249d815c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang.Tests/xunit.runner.json @@ -0,0 +1,3 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json" +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/AGENTS.md b/src/StellaOps.Scanner.Analyzers.Lang/AGENTS.md new file mode 100644 index 00000000..c8b87654 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/AGENTS.md @@ -0,0 +1,33 @@ +# StellaOps.Scanner.Analyzers.Lang — Agent Charter + +## Role +Deliver deterministic language ecosystem analyzers that run inside Scanner Workers, emit component evidence for SBOM assembly, and package as restart-time plug-ins. + +## Scope +- Shared analyzer abstractions for installed application ecosystems (Java, Node.js, Python, Go, .NET, Rust). +- Evidence helpers that map on-disk artefacts to canonical component identities (purl/bin sha) with provenance and usage flags. +- File-system traversal, metadata parsing, and normalization for language-specific package formats. +- Plug-in bootstrap, manifest authoring, and DI registration so Workers load analyzers at start-up. + +## Out of Scope +- OS package analyzers, native link graph, or EntryTrace plug-ins (handled by other guilds). +- SBOM composition, diffing, or signing (owned by Emit/Diff/Signer groups). +- Policy adjudication or vulnerability joins. + +## Expectations +- Deterministic output: identical inputs → identical component ordering and hashes. +- Memory discipline: streaming walkers, avoid loading entire trees; reuse buffers. +- Cancellation-aware and timeboxed per layer. +- Enrich telemetry (counters + timings) via Scanner.Core primitives. +- Update `TASKS.md` as work progresses (TODO → DOING → DONE/BLOCKED). + +## Dependencies +- Scanner.Core contracts + observability helpers. +- Scanner.Worker analyzer dispatcher. +- Upcoming Scanner.Emit models for SBOM assembly. +- Plugin host infrastructure under `StellaOps.Plugin`. + +## Testing & Artifacts +- Determinism harness with golden fixtures under `Fixtures/`. +- Microbench benchmarks recorded per language where feasible. +- Plugin manifests stored under `plugins/scanner/analyzers/lang/` with cosign workflow documented. diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/ILanguageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/ILanguageAnalyzer.cs new file mode 100644 index 00000000..617fddad --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/ILanguageAnalyzer.cs @@ -0,0 +1,24 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +/// +/// Contract implemented by language ecosystem analyzers. Analyzers must be deterministic, +/// cancellation-aware, and refrain from mutating shared state. +/// +public interface ILanguageAnalyzer +{ + /// + /// Stable identifier (e.g., java, node). + /// + string Id { get; } + + /// + /// Human-readable display name for diagnostics. + /// + string DisplayName { get; } + + /// + /// Executes the analyzer against the resolved filesystem. + /// + ValueTask AnalyzeAsync(LanguageAnalyzerContext context, LanguageComponentWriter writer, CancellationToken cancellationToken); +} + diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/Internal/LanguageAnalyzerJson.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/Internal/LanguageAnalyzerJson.cs new file mode 100644 index 00000000..a0160e6d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/Internal/LanguageAnalyzerJson.cs @@ -0,0 +1,16 @@ +namespace StellaOps.Scanner.Analyzers.Lang.Internal; + +internal static class LanguageAnalyzerJson +{ + public static JsonSerializerOptions CreateDefault(bool indent = false) + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = indent, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + return options; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerContext.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerContext.cs new file mode 100644 index 00000000..7fe029ef --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerContext.cs @@ -0,0 +1,67 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +public sealed class LanguageAnalyzerContext +{ + public LanguageAnalyzerContext(string rootPath, TimeProvider timeProvider, LanguageUsageHints? usageHints = null, IServiceProvider? services = null) + { + if (string.IsNullOrWhiteSpace(rootPath)) + { + throw new ArgumentException("Root path is required", nameof(rootPath)); + } + + RootPath = Path.GetFullPath(rootPath); + if (!Directory.Exists(RootPath)) + { + throw new DirectoryNotFoundException($"Root path '{RootPath}' does not exist."); + } + + TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + UsageHints = usageHints ?? LanguageUsageHints.Empty; + Services = services; + } + + public string RootPath { get; } + + public TimeProvider TimeProvider { get; } + + public LanguageUsageHints UsageHints { get; } + + public IServiceProvider? Services { get; } + + public bool TryGetService([NotNullWhen(true)] out T? service) where T : class + { + if (Services is null) + { + service = null; + return false; + } + + service = Services.GetService(typeof(T)) as T; + return service is not null; + } + + public string ResolvePath(ReadOnlySpan relative) + { + if (relative.IsEmpty) + { + return RootPath; + } + + var relativeString = new string(relative); + var combined = Path.Combine(RootPath, relativeString); + return Path.GetFullPath(combined); + } + + public string GetRelativePath(string absolutePath) + { + if (string.IsNullOrWhiteSpace(absolutePath)) + { + return string.Empty; + } + + var relative = Path.GetRelativePath(RootPath, absolutePath); + return OperatingSystem.IsWindows() + ? relative.Replace('\\', '/') + : relative; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerEngine.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerEngine.cs new file mode 100644 index 00000000..53af91c0 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerEngine.cs @@ -0,0 +1,59 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +public sealed class LanguageAnalyzerEngine +{ + private readonly IReadOnlyList _analyzers; + + public LanguageAnalyzerEngine(IEnumerable analyzers) + { + if (analyzers is null) + { + throw new ArgumentNullException(nameof(analyzers)); + } + + _analyzers = analyzers + .Where(static analyzer => analyzer is not null) + .Distinct(new AnalyzerIdComparer()) + .OrderBy(static analyzer => analyzer.Id, StringComparer.Ordinal) + .ToArray(); + } + + public IReadOnlyList Analyzers => _analyzers; + + public async ValueTask AnalyzeAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(context); + + var builder = new LanguageAnalyzerResultBuilder(); + var writer = new LanguageComponentWriter(builder); + + foreach (var analyzer in _analyzers) + { + cancellationToken.ThrowIfCancellationRequested(); + await analyzer.AnalyzeAsync(context, writer, cancellationToken).ConfigureAwait(false); + } + + return builder.Build(); + } + + private sealed class AnalyzerIdComparer : IEqualityComparer + { + public bool Equals(ILanguageAnalyzer? x, ILanguageAnalyzer? y) + { + if (ReferenceEquals(x, y)) + { + return true; + } + + if (x is null || y is null) + { + return false; + } + + return string.Equals(x.Id, y.Id, StringComparison.Ordinal); + } + + public int GetHashCode(ILanguageAnalyzer obj) + => obj?.Id is null ? 0 : StringComparer.Ordinal.GetHashCode(obj.Id); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerResult.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerResult.cs new file mode 100644 index 00000000..2c789fe5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageAnalyzerResult.cs @@ -0,0 +1,111 @@ +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Analyzers.Lang; + +public sealed class LanguageAnalyzerResult +{ + private readonly ImmutableArray _components; + + internal LanguageAnalyzerResult(IEnumerable components) + { + _components = components + .OrderBy(static record => record.ComponentKey, StringComparer.Ordinal) + .ThenBy(static record => record.AnalyzerId, StringComparer.Ordinal) + .ToImmutableArray(); + } + + public IReadOnlyList Components => _components; + + public ImmutableArray ToComponentRecords(string analyzerId, string? layerDigest = null) + => LanguageComponentMapper.ToComponentRecords(analyzerId, _components, layerDigest); + + public LayerComponentFragment ToLayerFragment(string analyzerId, string? layerDigest = null) + => LanguageComponentMapper.ToLayerFragment(analyzerId, _components, layerDigest); + + public IReadOnlyList ToSnapshots() + => _components.Select(static component => component.ToSnapshot()).ToImmutableArray(); + + public string ToJson(bool indent = true) + { + var snapshots = ToSnapshots(); + var options = Internal.LanguageAnalyzerJson.CreateDefault(indent); + return JsonSerializer.Serialize(snapshots, options); + } +} + +internal sealed class LanguageAnalyzerResultBuilder +{ + private readonly Dictionary _records = new(StringComparer.Ordinal); + private readonly object _sync = new(); + + public void Add(LanguageComponentRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + lock (_sync) + { + if (_records.TryGetValue(record.ComponentKey, out var existing)) + { + existing.Merge(record); + return; + } + + _records[record.ComponentKey] = record; + } + } + + public void AddRange(IEnumerable records) + { + foreach (var record in records ?? Array.Empty()) + { + Add(record); + } + } + + public LanguageAnalyzerResult Build() + { + lock (_sync) + { + return new LanguageAnalyzerResult(_records.Values.ToArray()); + } + } +} + +public sealed class LanguageComponentWriter +{ + private readonly LanguageAnalyzerResultBuilder _builder; + + internal LanguageComponentWriter(LanguageAnalyzerResultBuilder builder) + { + _builder = builder ?? throw new ArgumentNullException(nameof(builder)); + } + + public void Add(LanguageComponentRecord record) + => _builder.Add(record); + + public void AddRange(IEnumerable records) + => _builder.AddRange(records); + + public void AddFromPurl( + string analyzerId, + string purl, + string name, + string? version, + string type, + IEnumerable>? metadata = null, + IEnumerable? evidence = null, + bool usedByEntrypoint = false) + => Add(LanguageComponentRecord.FromPurl(analyzerId, purl, name, version, type, metadata, evidence, usedByEntrypoint)); + + public void AddFromExplicitKey( + string analyzerId, + string componentKey, + string? purl, + string name, + string? version, + string type, + IEnumerable>? metadata = null, + IEnumerable? evidence = null, + bool usedByEntrypoint = false) + => Add(LanguageComponentRecord.FromExplicitKey(analyzerId, componentKey, purl, name, version, type, metadata, evidence, usedByEntrypoint)); +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentEvidence.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentEvidence.cs new file mode 100644 index 00000000..4e18b836 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentEvidence.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +public enum LanguageEvidenceKind +{ + File, + Metadata, + Derived, +} + +public sealed record LanguageComponentEvidence( + LanguageEvidenceKind Kind, + string Source, + string Locator, + string? Value, + string? Sha256) +{ + public string ComparisonKey => string.Join('|', Kind, Source, Locator, Value, Sha256); +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentMapper.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentMapper.cs new file mode 100644 index 00000000..91215e82 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentMapper.cs @@ -0,0 +1,223 @@ +using System.Collections.Immutable; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Analyzers.Lang; + +/// +/// Helpers converting language analyzer component records into canonical scanner component models. +/// +public static class LanguageComponentMapper +{ + private const string LayerHashPrefix = "stellaops:lang:"; + private const string MetadataPrefix = "stellaops.lang"; + + /// + /// Computes a deterministic synthetic layer digest for the supplied analyzer identifier. + /// + public static string ComputeLayerDigest(string analyzerId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + + var payload = $"{LayerHashPrefix}{analyzerId.Trim().ToLowerInvariant()}"; + var bytes = Encoding.UTF8.GetBytes(payload); + var hash = SHA256.HashData(bytes); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + /// + /// Projects language component records into a deterministic set of component records. + /// + public static ImmutableArray ToComponentRecords( + string analyzerId, + IEnumerable components, + string? layerDigest = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + ArgumentNullException.ThrowIfNull(components); + + var effectiveLayer = string.IsNullOrWhiteSpace(layerDigest) + ? ComputeLayerDigest(analyzerId) + : layerDigest!; + + var builder = ImmutableArray.CreateBuilder(); + foreach (var record in components.OrderBy(static component => component.ComponentKey, StringComparer.Ordinal)) + { + builder.Add(CreateComponentRecord(analyzerId, effectiveLayer, record)); + } + + return builder.ToImmutable(); + } + + /// + /// Creates a layer component fragment using the supplied component records. + /// + public static LayerComponentFragment ToLayerFragment( + string analyzerId, + IEnumerable components, + string? layerDigest = null) + { + var componentRecords = ToComponentRecords(analyzerId, components, layerDigest); + if (componentRecords.IsEmpty) + { + return LayerComponentFragment.Create(ComputeLayerDigest(analyzerId), componentRecords); + } + + return LayerComponentFragment.Create(componentRecords[0].LayerDigest, componentRecords); + } + + private static ComponentRecord CreateComponentRecord( + string analyzerId, + string layerDigest, + LanguageComponentRecord record) + { + ArgumentNullException.ThrowIfNull(record); + + var identity = ComponentIdentity.Create( + key: ResolveIdentityKey(record), + name: record.Name, + version: record.Version, + purl: record.Purl, + componentType: record.Type); + + var evidence = MapEvidence(record); + var metadata = BuildMetadata(analyzerId, record); + var usage = record.UsedByEntrypoint + ? ComponentUsage.Create(usedByEntrypoint: true) + : ComponentUsage.Unused; + + return new ComponentRecord + { + Identity = identity, + LayerDigest = layerDigest, + Evidence = evidence, + Dependencies = ImmutableArray.Empty, + Metadata = metadata, + Usage = usage, + }; + } + + private static ImmutableArray MapEvidence(LanguageComponentRecord record) + { + var builder = ImmutableArray.CreateBuilder(); + foreach (var item in record.Evidence) + { + if (item is null) + { + continue; + } + + var kind = item.Kind switch + { + LanguageEvidenceKind.File => "file", + LanguageEvidenceKind.Metadata => "metadata", + LanguageEvidenceKind.Derived => "derived", + _ => "unknown", + }; + + var value = string.IsNullOrWhiteSpace(item.Locator) ? item.Source : item.Locator; + if (string.IsNullOrWhiteSpace(value)) + { + value = kind; + } + + builder.Add(new ComponentEvidence + { + Kind = kind, + Value = value, + Source = string.IsNullOrWhiteSpace(item.Source) ? null : item.Source, + }); + } + + return builder.Count == 0 + ? ImmutableArray.Empty + : builder.ToImmutable(); + } + + private static ComponentMetadata? BuildMetadata(string analyzerId, LanguageComponentRecord record) + { + var properties = new SortedDictionary(StringComparer.Ordinal) + { + [$"{MetadataPrefix}.analyzerId"] = analyzerId + }; + + var licenseList = new List(); + + foreach (var pair in record.Metadata) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(pair.Value)) + { + var value = pair.Value.Trim(); + properties[$"{MetadataPrefix}.meta.{pair.Key}"] = value; + + if (IsLicenseKey(pair.Key) && value.Length > 0) + { + foreach (var candidate in value.Split(new[] { ',', ';' }, StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) + { + if (candidate.Length > 0) + { + licenseList.Add(candidate); + } + } + } + } + } + + var evidenceIndex = 0; + foreach (var evidence in record.Evidence) + { + if (evidence is null) + { + continue; + } + + var prefix = $"{MetadataPrefix}.evidence.{evidenceIndex}"; + if (!string.IsNullOrWhiteSpace(evidence.Value)) + { + properties[$"{prefix}.value"] = evidence.Value.Trim(); + } + + if (!string.IsNullOrWhiteSpace(evidence.Sha256)) + { + properties[$"{prefix}.sha256"] = evidence.Sha256.Trim(); + } + + evidenceIndex++; + } + + IReadOnlyList? licenses = null; + if (licenseList.Count > 0) + { + licenses = licenseList + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static license => license, StringComparer.Ordinal) + .ToArray(); + } + + return new ComponentMetadata + { + Licenses = licenses, + Properties = properties.Count == 0 ? null : properties, + }; + } + + private static string ResolveIdentityKey(LanguageComponentRecord record) + { + var key = record.ComponentKey; + if (key.StartsWith("purl::", StringComparison.Ordinal)) + { + return key[6..]; + } + + return key; + } + + private static bool IsLicenseKey(string key) + => key.Contains("license", StringComparison.OrdinalIgnoreCase); +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentRecord.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentRecord.cs new file mode 100644 index 00000000..b023017e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageComponentRecord.cs @@ -0,0 +1,219 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +public sealed class LanguageComponentRecord +{ + private readonly SortedDictionary _metadata; + private readonly SortedDictionary _evidence; + + private LanguageComponentRecord( + string analyzerId, + string componentKey, + string? purl, + string name, + string? version, + string type, + IEnumerable> metadata, + IEnumerable evidence, + bool usedByEntrypoint) + { + AnalyzerId = analyzerId ?? throw new ArgumentNullException(nameof(analyzerId)); + ComponentKey = componentKey ?? throw new ArgumentNullException(nameof(componentKey)); + Purl = string.IsNullOrWhiteSpace(purl) ? null : purl.Trim(); + Name = name ?? throw new ArgumentNullException(nameof(name)); + Version = string.IsNullOrWhiteSpace(version) ? null : version.Trim(); + Type = string.IsNullOrWhiteSpace(type) ? throw new ArgumentException("Type is required", nameof(type)) : type.Trim(); + UsedByEntrypoint = usedByEntrypoint; + + _metadata = new SortedDictionary(StringComparer.Ordinal); + foreach (var entry in metadata ?? Array.Empty>()) + { + if (string.IsNullOrWhiteSpace(entry.Key)) + { + continue; + } + + _metadata[entry.Key.Trim()] = entry.Value; + } + + _evidence = new SortedDictionary(StringComparer.Ordinal); + foreach (var evidenceItem in evidence ?? Array.Empty()) + { + if (evidenceItem is null) + { + continue; + } + + _evidence[evidenceItem.ComparisonKey] = evidenceItem; + } + } + + public string AnalyzerId { get; } + + public string ComponentKey { get; } + + public string? Purl { get; } + + public string Name { get; } + + public string? Version { get; } + + public string Type { get; } + + public bool UsedByEntrypoint { get; private set; } + + public IReadOnlyDictionary Metadata => _metadata; + + public IReadOnlyCollection Evidence => _evidence.Values; + + public static LanguageComponentRecord FromPurl( + string analyzerId, + string purl, + string name, + string? version, + string type, + IEnumerable>? metadata = null, + IEnumerable? evidence = null, + bool usedByEntrypoint = false) + { + if (string.IsNullOrWhiteSpace(purl)) + { + throw new ArgumentException("purl is required", nameof(purl)); + } + + var key = $"purl::{purl.Trim()}"; + return new LanguageComponentRecord( + analyzerId, + key, + purl, + name, + version, + type, + metadata ?? Array.Empty>(), + evidence ?? Array.Empty(), + usedByEntrypoint); + } + + public static LanguageComponentRecord FromExplicitKey( + string analyzerId, + string componentKey, + string? purl, + string name, + string? version, + string type, + IEnumerable>? metadata = null, + IEnumerable? evidence = null, + bool usedByEntrypoint = false) + { + if (string.IsNullOrWhiteSpace(componentKey)) + { + throw new ArgumentException("Component key is required", nameof(componentKey)); + } + + return new LanguageComponentRecord( + analyzerId, + componentKey.Trim(), + purl, + name, + version, + type, + metadata ?? Array.Empty>(), + evidence ?? Array.Empty(), + usedByEntrypoint); + } + + internal void Merge(LanguageComponentRecord other) + { + ArgumentNullException.ThrowIfNull(other); + + if (!ComponentKey.Equals(other.ComponentKey, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Cannot merge component '{ComponentKey}' with '{other.ComponentKey}'."); + } + + UsedByEntrypoint |= other.UsedByEntrypoint; + + foreach (var entry in other._metadata) + { + if (!_metadata.TryGetValue(entry.Key, out var existing) || string.IsNullOrEmpty(existing)) + { + _metadata[entry.Key] = entry.Value; + } + } + + foreach (var evidenceItem in other._evidence) + { + _evidence[evidenceItem.Key] = evidenceItem.Value; + } + } + + public LanguageComponentSnapshot ToSnapshot() + { + return new LanguageComponentSnapshot + { + AnalyzerId = AnalyzerId, + ComponentKey = ComponentKey, + Purl = Purl, + Name = Name, + Version = Version, + Type = Type, + UsedByEntrypoint = UsedByEntrypoint, + Metadata = _metadata.ToDictionary(static pair => pair.Key, static pair => pair.Value, StringComparer.Ordinal), + Evidence = _evidence.Values.Select(static item => new LanguageComponentEvidenceSnapshot + { + Kind = item.Kind, + Source = item.Source, + Locator = item.Locator, + Value = item.Value, + Sha256 = item.Sha256, + }).ToArray(), + }; + } +} + +public sealed class LanguageComponentSnapshot +{ + [JsonPropertyName("analyzerId")] + public string AnalyzerId { get; set; } = string.Empty; + + [JsonPropertyName("componentKey")] + public string ComponentKey { get; set; } = string.Empty; + + [JsonPropertyName("purl")] + public string? Purl { get; set; } + + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + + [JsonPropertyName("version")] + public string? Version { get; set; } + + [JsonPropertyName("type")] + public string Type { get; set; } = string.Empty; + + [JsonPropertyName("usedByEntrypoint")] + public bool UsedByEntrypoint { get; set; } + + [JsonPropertyName("metadata")] + public IDictionary Metadata { get; set; } = new Dictionary(StringComparer.Ordinal); + + [JsonPropertyName("evidence")] + public IReadOnlyList Evidence { get; set; } = Array.Empty(); +} + +public sealed class LanguageComponentEvidenceSnapshot +{ + [JsonPropertyName("kind")] + public LanguageEvidenceKind Kind { get; set; } + + [JsonPropertyName("source")] + public string Source { get; set; } = string.Empty; + + [JsonPropertyName("locator")] + public string Locator { get; set; } = string.Empty; + + [JsonPropertyName("value")] + public string? Value { get; set; } + + [JsonPropertyName("sha256")] + public string? Sha256 { get; set; } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageUsageHints.cs b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageUsageHints.cs new file mode 100644 index 00000000..0c9952e2 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/Core/LanguageUsageHints.cs @@ -0,0 +1,49 @@ +namespace StellaOps.Scanner.Analyzers.Lang; + +public sealed class LanguageUsageHints +{ + private static readonly StringComparer Comparer = OperatingSystem.IsWindows() + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; + + private readonly ImmutableHashSet _usedPaths; + + public static LanguageUsageHints Empty { get; } = new(Array.Empty()); + + public LanguageUsageHints(IEnumerable usedPaths) + { + if (usedPaths is null) + { + throw new ArgumentNullException(nameof(usedPaths)); + } + + _usedPaths = usedPaths + .Select(Normalize) + .Where(static path => path.Length > 0) + .ToImmutableHashSet(Comparer); + } + + public bool IsPathUsed(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return false; + } + + var normalized = Normalize(path); + return _usedPaths.Contains(normalized); + } + + private static string Normalize(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return string.Empty; + } + + var full = Path.GetFullPath(path); + return OperatingSystem.IsWindows() + ? full.Replace('\\', '/').TrimEnd('/') + : full; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.Lang/GlobalUsings.cs b/src/StellaOps.Scanner.Analyzers.Lang/GlobalUsings.cs new file mode 100644 index 00000000..6b54c31d --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/GlobalUsings.cs @@ -0,0 +1,11 @@ +global using System; +global using System.Collections.Concurrent; +global using System.Collections.Generic; +global using System.Collections.Immutable; +global using System.Diagnostics.CodeAnalysis; +global using System.IO; +global using System.Linq; +global using System.Text.Json; +global using System.Text.Json.Serialization; +global using System.Threading; +global using System.Threading.Tasks; diff --git a/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md b/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..173fb9dc --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/SPRINTS_LANG_IMPLEMENTATION_PLAN.md @@ -0,0 +1,114 @@ +# StellaOps Scanner — Language Analyzer Implementation Plan (2025Q4) + +> **Goal.** Deliver best-in-class language analyzers that outperform competitors on fidelity, determinism, and offline readiness while integrating tightly with Scanner Worker orchestration and SBOM composition. + +All sprints below assume prerequisites from SP10-G2 (core scaffolding + Java analyzer) are complete. Each sprint is sized for a focused guild (≈1–1.5 weeks) and produces definitive gates for downstream teams (Emit, Policy, Scheduler). + +--- + +## Sprint LA1 — Node Analyzer & Workspace Intelligence (Tasks 10-302, 10-307, 10-308, 10-309 subset) *(DOING — 2025-10-19)* +- **Scope:** Resolve hoisted `node_modules`, PNPM structures, Yarn Berry Plug'n'Play, symlinked workspaces, and detect security-sensitive scripts. +- **Deliverables:** + - `StellaOps.Scanner.Analyzers.Lang.Node` plug-in with manifest + DI registration. + - Deterministic walker supporting >100 k modules with streaming JSON parsing. + - Workspace graph persisted as analyzer metadata (`package.json` provenance + symlink target proofs). +- **Acceptance Metrics:** + - 10 k module fixture scans <1.8 s on 4 vCPU (p95). + - Memory ceiling <220 MB (tracked via deterministic benchmark harness). + - All symlink targets canonicalized; path traversal guarded. +- **Gate Artifacts:** + - `Fixtures/lang/node/**` golden outputs. + - Analyzer benchmark CSV + flamegraph (commit under `bench/Scanner.Analyzers`). + - Worker integration sample enabling Node analyzer via manifest. +- **Progress (2025-10-19):** Module walker with package-lock/yarn/pnpm resolution, workspace attribution, integrity metadata, and deterministic fixture harness committed; Node tasks 10-302A/B marked DONE. Shared component mapper + canonical result harness landed, closing tasks 10-307/308. Script metadata & telemetry (10-302C) emit policy hints, hashed evidence, and feed `scanner_analyzer_node_scripts_total` into Worker OpenTelemetry pipeline. + +## Sprint LA2 — Python Analyzer & Entry Point Attribution (Tasks 10-303, 10-307, 10-308, 10-309 subset) +- **Scope:** Parse `*.dist-info`, `RECORD` hashes, entry points, and pip-installed editable packages; integrate usage hints from EntryTrace. +- **Deliverables:** + - `StellaOps.Scanner.Analyzers.Lang.Python` plug-in. + - RECORD hash validation with optional Zip64 support for `.whl` caches. + - Entry-point mapping into `UsageFlags` for Emit stage. +- **Acceptance Metrics:** + - Hash verification throughput ≥75 MB/s sustained with streaming reader. + - False-positive rate for editable installs <1 % on curated fixtures. + - Determinism check across CPython 3.8–3.12 generated metadata. +- **Gate Artifacts:** + - Golden fixtures for `site-packages`, virtualenv, and layered pip caches. + - Usage hint propagation tests (EntryTrace → analyzer → SBOM). + - Metrics counters (`scanner_analyzer_python_components_total`) documented. + +## Sprint LA3 — Go Analyzer & Build Info Synthesis (Tasks 10-304, 10-307, 10-308, 10-309 subset) +- **Scope:** Extract Go build metadata from `.note.go.buildid`, embedded module info, and fallback to `bin:{sha256}`; surface VCS provenance. +- **Deliverables:** + - `StellaOps.Scanner.Analyzers.Lang.Go` plug-in. + - DWARF-lite parser to enrich component origin (commit hash + dirty flag) when available. + - Shared hash cache to dedupe repeated binaries across layers. +- **Acceptance Metrics:** + - Analyzer latency ≤400 µs per binary (hot cache) / ≤2 ms (cold). + - Provenance coverage ≥95 % on representative Go fixture suite. + - Zero allocations in happy path beyond pooled buffers (validated via BenchmarkDotNet). +- **Gate Artifacts:** + - Benchmarks vs competitor open-source tool (Trivy or Syft) demonstrating faster metadata extraction. + - Documentation snippet explaining VCS metadata fields for Policy team. + +## Sprint LA4 — .NET Analyzer & RID Variants (Tasks 10-305, 10-307, 10-308, 10-309 subset) +- **Scope:** Parse `*.deps.json`, `runtimeconfig.json`, assembly metadata, and RID-specific assets; correlate with native dependencies. +- **Deliverables:** + - `StellaOps.Scanner.Analyzers.Lang.DotNet` plug-in. + - Strong-name + Authenticode optional verification when offline cert bundle provided. + - RID-aware component grouping with fallback to `bin:{sha256}` for self-contained apps. +- **Acceptance Metrics:** + - Multi-target app fixture processed <1.2 s; memory <250 MB. + - RID variant collapse reduces component explosion by ≥40 % vs naive listing. + - All security metadata (signing Publisher, timestamp) surfaced deterministically. +- **Gate Artifacts:** + - Signed .NET sample apps (framework-dependent & self-contained) under `samples/scanner/lang/dotnet/`. + - Tests verifying dual runtimeconfig merge logic. + - Guidance for Policy on license propagation from NuGet metadata. + +## Sprint LA5 — Rust Analyzer & Binary Fingerprinting (Tasks 10-306, 10-307, 10-308, 10-309 subset) +- **Scope:** Detect crates via metadata in `.fingerprint`, Cargo.lock fragments, or embedded `rustc` markers; robust fallback to binary hash classification. +- **Deliverables:** + - `StellaOps.Scanner.Analyzers.Lang.Rust` plug-in. + - Symbol table heuristics capable of attributing stripped binaries by leveraging `.comment` and section names without violating determinism. + - Quiet-provenance flags to differentiate heuristics from hard evidence. +- **Acceptance Metrics:** + - Accurate crate attribution ≥85 % on curated Cargo workspace fixtures. + - Heuristic fallback clearly labeled; no false “certain” claims. + - Analyzer completes <1 s on 500 binary corpus. +- **Gate Artifacts:** + - Fixtures covering cargo workspaces, binaries with embedded metadata stripped. + - ADR documenting heuristic boundaries + risk mitigations. + +## Sprint LA6 — Shared Evidence Enhancements & Worker Integration (Tasks 10-307, 10-308, 10-309 finalization) +- **Scope:** Finalize shared helpers, deterministic harness expansion, Worker/Emit wiring, and macro benchmarks. +- **Deliverables:** + - Consolidated `LanguageComponentWriter` extensions for license, vulnerability hints, and usage propagation. + - Worker dispatcher loading plug-ins via manifest registry + health checks. + - Combined analyzer benchmark suite executed in CI with regression thresholds. +- **Acceptance Metrics:** + - Worker executes mixed analyzer suite (Java+Node+Python+Go+.NET+Rust) within SLA: warm scan <6 s, cold <25 s. + - CI determinism guard catches output drift (>0 diff tolerance) across all fixtures. + - Telemetry coverage: each analyzer emits timing + component counters. +- **Gate Artifacts:** + - `SPRINTS_LANG_IMPLEMENTATION_PLAN.md` progress log updated (this file). + - `bench/Scanner.Analyzers/lang-matrix.csv` recorded + referenced in docs. + - Ops notes for packaging plug-ins into Offline Kit. + +--- + +## Cross-Sprint Considerations +- **Security:** All analyzers must enforce path canonicalization, guard against zip-slip, and expose provenance classifications (`observed`, `heuristic`, `attested`). +- **Offline-first:** No network calls; rely on cached metadata and optional offline bundles (license texts, signature roots). +- **Determinism:** Normalise timestamps to `0001-01-01T00:00:00Z` when persisting synthetic data; sort collections by stable keys. +- **Benchmarking:** Extend `bench/Scanner.Analyzers` to compare against open-source scanners (Syft/Trivy) and document performance wins. +- **Hand-offs:** Emit guild requires consistent component schemas; Policy needs license + provenance metadata; Scheduler depends on usage flags for ImpactIndex. + +## Tracking & Reporting +- Update `TASKS.md` per sprint (TODO → DOING → DONE) with date stamps. +- Log sprint summaries in `docs/updates/` once each sprint lands. +- Use module-specific CI pipeline to run analyzer suites nightly (determinism + perf). + +--- + +**Next Action:** Start Sprint LA1 (Node Analyzer) — move tasks 10-302, 10-307, 10-308, 10-309 → DOING and spin up fixtures + benchmarks. diff --git a/src/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj b/src/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj new file mode 100644 index 00000000..1ad5f85c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj @@ -0,0 +1,21 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.Lang/TASKS.md b/src/StellaOps.Scanner.Analyzers.Lang/TASKS.md new file mode 100644 index 00000000..aabce91e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.Lang/TASKS.md @@ -0,0 +1,13 @@ +# Language Analyzer Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-LANG-10-301 | DONE (2025-10-19) | Language Analyzer Guild | SCANNER-CORE-09-501, SCANNER-WORKER-09-203 | Java analyzer emitting deterministic `pkg:maven` components using pom.properties / MANIFEST evidence. | Java analyzer extracts coordinates+version+licenses with provenance; golden fixtures deterministic; microbenchmark meets target. | +| SCANNER-ANALYZERS-LANG-10-302 | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Node analyzer resolving workspaces/symlinks into `pkg:npm` identities. | Node analyzer handles symlinks/workspaces; outputs sorted components; determinism harness covers hoisted deps. | +| SCANNER-ANALYZERS-LANG-10-303 | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Python analyzer consuming `*.dist-info` metadata and RECORD hashes. | Analyzer binds METADATA + RECORD evidence, includes entry points, determinism fixtures stable. | +| SCANNER-ANALYZERS-LANG-10-304 | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Go analyzer leveraging buildinfo for `pkg:golang` components. | Buildinfo parser emits module path/version + vcs metadata; binaries without buildinfo downgraded gracefully. | +| SCANNER-ANALYZERS-LANG-10-305 | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | .NET analyzer parsing `*.deps.json`, assembly metadata, and RID variants. | Analyzer merges deps.json + assembly info; dedupes per RID; determinism verified. | +| SCANNER-ANALYZERS-LANG-10-306 | TODO | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Rust analyzer detecting crate provenance or falling back to `bin:{sha256}`. | Analyzer emits `pkg:cargo` when metadata present; falls back to binary hash; fixtures cover both paths. | +| SCANNER-ANALYZERS-LANG-10-307 | DONE (2025-10-19) | Language Analyzer Guild | SCANNER-CORE-09-501 | Shared language evidence helpers + usage flag propagation. | Shared abstractions implemented; analyzers reuse helpers; evidence includes usage hints; unit tests cover canonical ordering. | +| SCANNER-ANALYZERS-LANG-10-308 | DONE (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-307 | Determinism + fixture harness for language analyzers. | Harness executes analyzers against fixtures; golden JSON stored; CI helper ensures stable hashes. | +| SCANNER-ANALYZERS-LANG-10-309 | DOING (2025-10-19) | Language Analyzer Guild | SCANNER-ANALYZERS-LANG-10-301..308 | Package language analyzers as restart-time plug-ins (manifest + host registration). | Plugin manifests authored under `plugins/scanner/analyzers/lang`; Worker loads via DI; restart required flag enforced; tests confirm manifest integrity. | diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkAnalyzerPlugin.cs b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkAnalyzerPlugin.cs new file mode 100644 index 00000000..d2a059c6 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkAnalyzerPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Plugin; + +namespace StellaOps.Scanner.Analyzers.OS.Apk; + +public sealed class ApkAnalyzerPlugin : IOSAnalyzerPlugin +{ + public string Name => "StellaOps.Scanner.Analyzers.OS.Apk"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + var loggerFactory = services.GetRequiredService(); + return new ApkPackageAnalyzer(loggerFactory.CreateLogger()); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkDatabaseParser.cs b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkDatabaseParser.cs new file mode 100644 index 00000000..bdd12152 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkDatabaseParser.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace StellaOps.Scanner.Analyzers.OS.Apk; + +internal sealed class ApkDatabaseParser +{ + public IReadOnlyList Parse(Stream stream, CancellationToken cancellationToken) + { + var packages = new List(); + var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: true); + + var current = new ApkPackageEntry(); + string? currentDirectory = "/"; + string? pendingDigest = null; + bool pendingConfig = false; + + string? line; + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(line)) + { + CommitCurrent(); + current = new ApkPackageEntry(); + currentDirectory = "/"; + pendingDigest = null; + pendingConfig = false; + continue; + } + + if (line.Length < 2) + { + continue; + } + + var key = line[0]; + var value = line.Length > 2 ? line[2..] : string.Empty; + + switch (key) + { + case 'C': + current.Channel = value; + break; + case 'P': + current.Name = value; + break; + case 'V': + current.Version = value; + break; + case 'A': + current.Architecture = value; + break; + case 'S': + current.InstalledSize = value; + break; + case 'I': + current.PackageSize = value; + break; + case 'T': + current.Description = value; + break; + case 'U': + current.Url = value; + break; + case 'L': + current.License = value; + break; + case 'o': + current.Origin = value; + break; + case 'm': + current.Maintainer = value; + break; + case 't': + current.BuildTime = value; + break; + case 'c': + current.Checksum = value; + break; + case 'D': + current.Depends.AddRange(SplitList(value)); + break; + case 'p': + current.Provides.AddRange(SplitList(value)); + break; + case 'F': + currentDirectory = NormalizeDirectory(value); + current.Files.Add(new ApkFileEntry(currentDirectory, true, false, null)); + break; + case 'R': + if (currentDirectory is null) + { + currentDirectory = "/"; + } + + var fullPath = CombinePath(currentDirectory, value); + current.Files.Add(new ApkFileEntry(fullPath, false, pendingConfig, pendingDigest)); + pendingDigest = null; + pendingConfig = false; + break; + case 'Z': + pendingDigest = string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + break; + case 'a': + pendingConfig = value.Contains("cfg", StringComparison.OrdinalIgnoreCase); + break; + default: + current.Metadata[key.ToString()] = value; + break; + } + } + + CommitCurrent(); + return packages; + + void CommitCurrent() + { + if (!string.IsNullOrWhiteSpace(current.Name) && + !string.IsNullOrWhiteSpace(current.Version) && + !string.IsNullOrWhiteSpace(current.Architecture)) + { + packages.Add(current); + } + } + } + + private static IEnumerable SplitList(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + yield break; + } + + foreach (var token in value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + yield return token; + } + } + + private static string NormalizeDirectory(string value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return "/"; + } + + var path = value.Trim(); + if (!path.StartsWith('/')) + { + path = "/" + path; + } + + if (!path.EndsWith('/')) + { + path += "/"; + } + + return path.Replace("//", "/"); + } + + private static string CombinePath(string directory, string relative) + { + if (string.IsNullOrWhiteSpace(relative)) + { + return directory.TrimEnd('/'); + } + + if (!directory.EndsWith('/')) + { + directory += "/"; + } + + return (directory + relative.TrimStart('/')).Replace("//", "/"); + } +} + +internal sealed class ApkPackageEntry +{ + public string? Channel { get; set; } + public string? Name { get; set; } + public string? Version { get; set; } + public string? Architecture { get; set; } + public string? InstalledSize { get; set; } + public string? PackageSize { get; set; } + public string? Description { get; set; } + public string? Url { get; set; } + public string? License { get; set; } + public string? Origin { get; set; } + public string? Maintainer { get; set; } + public string? BuildTime { get; set; } + public string? Checksum { get; set; } + public List Depends { get; } = new(); + public List Provides { get; } = new(); + public List Files { get; } = new(); + public Dictionary Metadata { get; } = new(StringComparer.Ordinal); +} + +internal sealed record ApkFileEntry(string Path, bool IsDirectory, bool IsConfig, string? Digest); diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkPackageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkPackageAnalyzer.cs new file mode 100644 index 00000000..44383918 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/ApkPackageAnalyzer.cs @@ -0,0 +1,106 @@ +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Analyzers; +using StellaOps.Scanner.Analyzers.OS.Helpers; + +namespace StellaOps.Scanner.Analyzers.OS.Apk; + +internal sealed class ApkPackageAnalyzer : OsPackageAnalyzerBase +{ + private static readonly IReadOnlyList EmptyPackages = + new ReadOnlyCollection(System.Array.Empty()); + + private readonly ApkDatabaseParser _parser = new(); + + public ApkPackageAnalyzer(ILogger logger) + : base(logger) + { + } + + public override string AnalyzerId => "apk"; + + protected override ValueTask> ExecuteCoreAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken) + { + var installedPath = Path.Combine(context.RootPath, "lib", "apk", "db", "installed"); + if (!File.Exists(installedPath)) + { + Logger.LogInformation("Apk installed database not found at {Path}; skipping analyzer.", installedPath); + return ValueTask.FromResult>(EmptyPackages); + } + + using var stream = File.OpenRead(installedPath); + var entries = _parser.Parse(stream, cancellationToken); + + var records = new List(entries.Count); + foreach (var entry in entries) + { + if (string.IsNullOrWhiteSpace(entry.Name) || + string.IsNullOrWhiteSpace(entry.Version) || + string.IsNullOrWhiteSpace(entry.Architecture)) + { + continue; + } + + var versionParts = PackageVersionParser.ParseApkVersion(entry.Version); + var purl = PackageUrlBuilder.BuildAlpine(entry.Name, entry.Version, entry.Architecture); + + var vendorMetadata = new Dictionary(StringComparer.Ordinal) + { + ["origin"] = entry.Origin, + ["description"] = entry.Description, + ["homepage"] = entry.Url, + ["maintainer"] = entry.Maintainer, + ["checksum"] = entry.Checksum, + ["buildTime"] = entry.BuildTime, + }; + + foreach (var pair in entry.Metadata) + { + vendorMetadata[$"apk:{pair.Key}"] = pair.Value; + } + + var files = new List(entry.Files.Count); + foreach (var file in entry.Files) + { + files.Add(new OSPackageFileEvidence( + file.Path, + layerDigest: null, + sha256: file.Digest, + sizeBytes: null, + isConfigFile: file.IsConfig)); + } + + var cveHints = CveHintExtractor.Extract( + string.Join(' ', entry.Depends), + string.Join(' ', entry.Provides)); + + var record = new OSPackageRecord( + AnalyzerId, + purl, + entry.Name, + versionParts.BaseVersion, + entry.Architecture, + PackageEvidenceSource.ApkDatabase, + epoch: null, + release: versionParts.Release, + sourcePackage: entry.Origin, + license: entry.License, + cveHints: cveHints, + provides: entry.Provides, + depends: entry.Depends, + files: files, + vendorMetadata: vendorMetadata); + + records.Add(record); + } + + records.Sort(); + return ValueTask.FromResult>(records); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/Properties/AssemblyInfo.cs b/src/StellaOps.Scanner.Analyzers.OS.Apk/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..308f76f1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Tests")] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj b/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj new file mode 100644 index 00000000..f8879698 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/StellaOps.Scanner.Analyzers.OS.Apk.csproj @@ -0,0 +1,15 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json b/src/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json new file mode 100644 index 00000000..0aaa333c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Apk/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.apk", + "displayName": "StellaOps Alpine APK Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Apk.dll" + }, + "capabilities": [ + "os-analyzer", + "apk" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "apk" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgAnalyzerPlugin.cs b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgAnalyzerPlugin.cs new file mode 100644 index 00000000..ff3f7263 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgAnalyzerPlugin.cs @@ -0,0 +1,21 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Plugin; + +namespace StellaOps.Scanner.Analyzers.OS.Dpkg; + +public sealed class DpkgAnalyzerPlugin : IOSAnalyzerPlugin +{ + public string Name => "StellaOps.Scanner.Analyzers.OS.Dpkg"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + var loggerFactory = services.GetRequiredService(); + return new DpkgPackageAnalyzer(loggerFactory.CreateLogger()); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgPackageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgPackageAnalyzer.cs new file mode 100644 index 00000000..8d05e588 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgPackageAnalyzer.cs @@ -0,0 +1,267 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Analyzers; +using StellaOps.Scanner.Analyzers.OS.Helpers; + +namespace StellaOps.Scanner.Analyzers.OS.Dpkg; + +internal sealed class DpkgPackageAnalyzer : OsPackageAnalyzerBase +{ + private static readonly IReadOnlyList EmptyPackages = + new ReadOnlyCollection(System.Array.Empty()); + + private readonly DpkgStatusParser _parser = new(); + + public DpkgPackageAnalyzer(ILogger logger) + : base(logger) + { + } + + public override string AnalyzerId => "dpkg"; + + protected override ValueTask> ExecuteCoreAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken) + { + var statusPath = Path.Combine(context.RootPath, "var", "lib", "dpkg", "status"); + if (!File.Exists(statusPath)) + { + Logger.LogInformation("dpkg status file not found at {Path}; skipping analyzer.", statusPath); + return ValueTask.FromResult>(EmptyPackages); + } + + using var stream = File.OpenRead(statusPath); + var entries = _parser.Parse(stream, cancellationToken); + + var infoDirectory = Path.Combine(context.RootPath, "var", "lib", "dpkg", "info"); + var records = new List(); + + foreach (var entry in entries) + { + if (!IsInstalled(entry.Status)) + { + continue; + } + + if (string.IsNullOrWhiteSpace(entry.Name) || string.IsNullOrWhiteSpace(entry.Version) || string.IsNullOrWhiteSpace(entry.Architecture)) + { + continue; + } + + var versionParts = PackageVersionParser.ParseDebianVersion(entry.Version); + var sourceName = ParseSource(entry.Source) ?? entry.Name; + var distribution = entry.Origin; + if (distribution is null && entry.Metadata.TryGetValue("origin", out var originValue)) + { + distribution = originValue; + } + distribution ??= "debian"; + + var purl = PackageUrlBuilder.BuildDebian(distribution!, entry.Name, entry.Version, entry.Architecture); + + var vendorMetadata = new Dictionary(StringComparer.Ordinal) + { + ["source"] = entry.Source, + ["homepage"] = entry.Homepage, + ["maintainer"] = entry.Maintainer, + ["origin"] = entry.Origin, + ["priority"] = entry.Priority, + ["section"] = entry.Section, + }; + + foreach (var kvp in entry.Metadata) + { + vendorMetadata[$"dpkg:{kvp.Key}"] = kvp.Value; + } + + var dependencies = entry.Depends.Concat(entry.PreDepends).ToArray(); + var provides = entry.Provides.ToArray(); + + var fileEvidence = BuildFileEvidence(infoDirectory, entry, cancellationToken); + + var cveHints = CveHintExtractor.Extract(entry.Description, string.Join(' ', dependencies), string.Join(' ', provides)); + + var record = new OSPackageRecord( + AnalyzerId, + purl, + entry.Name, + versionParts.UpstreamVersion, + entry.Architecture, + PackageEvidenceSource.DpkgStatus, + epoch: versionParts.Epoch, + release: versionParts.Revision, + sourcePackage: sourceName, + license: entry.License, + cveHints: cveHints, + provides: provides, + depends: dependencies, + files: fileEvidence, + vendorMetadata: vendorMetadata); + + records.Add(record); + } + + records.Sort(); + return ValueTask.FromResult>(records); + } + + private static bool IsInstalled(string? status) + => status?.Contains("install ok installed", System.StringComparison.OrdinalIgnoreCase) == true; + + private static string? ParseSource(string? sourceField) + { + if (string.IsNullOrWhiteSpace(sourceField)) + { + return null; + } + + var parts = sourceField.Split(' ', 2, System.StringSplitOptions.TrimEntries | System.StringSplitOptions.RemoveEmptyEntries); + return parts.Length == 0 ? null : parts[0]; + } + + private static IReadOnlyList BuildFileEvidence(string infoDirectory, DpkgPackageEntry entry, CancellationToken cancellationToken) + { + if (!Directory.Exists(infoDirectory)) + { + return Array.Empty(); + } + + var files = new Dictionary(StringComparer.Ordinal); + void EnsureFile(string path) + { + if (!files.TryGetValue(path, out _)) + { + files[path] = new FileEvidenceBuilder(path); + } + } + + foreach (var conffile in entry.Conffiles) + { + var normalized = conffile.Path.Trim(); + if (string.IsNullOrWhiteSpace(normalized)) + { + continue; + } + + EnsureFile(normalized); + files[normalized].IsConfig = true; + if (!string.IsNullOrWhiteSpace(conffile.Checksum)) + { + files[normalized].Digests["md5"] = conffile.Checksum.Trim(); + } + } + + foreach (var candidate in GetInfoFileCandidates(entry.Name!, entry.Architecture!)) + { + var listPath = Path.Combine(infoDirectory, candidate + ".list"); + if (File.Exists(listPath)) + { + foreach (var line in File.ReadLines(listPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + var trimmed = line.Trim(); + if (string.IsNullOrWhiteSpace(trimmed)) + { + continue; + } + + EnsureFile(trimmed); + } + } + + var confFilePath = Path.Combine(infoDirectory, candidate + ".conffiles"); + if (File.Exists(confFilePath)) + { + foreach (var line in File.ReadLines(confFilePath)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(' ', System.StringSplitOptions.RemoveEmptyEntries | System.StringSplitOptions.TrimEntries); + if (parts.Length == 0) + { + continue; + } + + var path = parts[0]; + EnsureFile(path); + files[path].IsConfig = true; + if (parts.Length >= 2) + { + files[path].Digests["md5"] = parts[1]; + } + } + } + + var md5sumsPath = Path.Combine(infoDirectory, candidate + ".md5sums"); + if (File.Exists(md5sumsPath)) + { + foreach (var line in File.ReadLines(md5sumsPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var parts = line.Split(' ', 2, System.StringSplitOptions.RemoveEmptyEntries | System.StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + continue; + } + + var hash = parts[0]; + var path = parts[1]; + EnsureFile(path); + files[path].Digests["md5"] = hash; + } + } + } + + if (files.Count == 0) + { + return Array.Empty(); + } + + var evidence = files.Values + .Select(builder => builder.ToEvidence()) + .OrderBy(e => e) + .ToArray(); + + return new ReadOnlyCollection(evidence); + } + + private static IEnumerable GetInfoFileCandidates(string packageName, string architecture) + { + yield return packageName + ":" + architecture; + yield return packageName; + } + + private sealed class FileEvidenceBuilder + { + public FileEvidenceBuilder(string path) + { + Path = path; + } + + public string Path { get; } + + public bool IsConfig { get; set; } + + public Dictionary Digests { get; } = new(StringComparer.OrdinalIgnoreCase); + + public OSPackageFileEvidence ToEvidence() + { + return new OSPackageFileEvidence(Path, isConfigFile: IsConfig, digests: Digests); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgStatusParser.cs b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgStatusParser.cs new file mode 100644 index 00000000..272a6bf8 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/DpkgStatusParser.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; + +namespace StellaOps.Scanner.Analyzers.OS.Dpkg; + +internal sealed class DpkgStatusParser +{ + public IReadOnlyList Parse(Stream stream, CancellationToken cancellationToken) + { + var packages = new List(); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true, bufferSize: 4096, leaveOpen: true); + + var current = new DpkgPackageEntry(); + string? currentField = null; + + string? line; + while ((line = reader.ReadLine()) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (string.IsNullOrWhiteSpace(line)) + { + CommitField(); + CommitPackage(); + current = new DpkgPackageEntry(); + currentField = null; + continue; + } + + if (char.IsWhiteSpace(line, 0)) + { + var continuation = line.Trim(); + if (currentField is not null) + { + current.AppendContinuation(currentField, continuation); + } + continue; + } + + var separator = line.IndexOf(':'); + if (separator <= 0) + { + continue; + } + + CommitField(); + + var fieldName = line[..separator]; + var value = line[(separator + 1)..].TrimStart(); + + currentField = fieldName; + current.SetField(fieldName, value); + } + + CommitField(); + CommitPackage(); + + return packages; + + void CommitField() + { + if (currentField is not null) + { + current.FieldCompleted(currentField); + } + } + + void CommitPackage() + { + if (current.IsValid) + { + packages.Add(current); + } + } + } +} + +internal sealed class DpkgPackageEntry +{ + private readonly StringBuilder _descriptionBuilder = new(); + private readonly Dictionary _metadata = new(StringComparer.OrdinalIgnoreCase); + private string? _currentMultilineField; + + public string? Name { get; private set; } + public string? Version { get; private set; } + public string? Architecture { get; private set; } + public string? Status { get; private set; } + public string? Source { get; private set; } + public string? Description { get; private set; } + public string? Homepage { get; private set; } + public string? Maintainer { get; private set; } + public string? Origin { get; private set; } + public string? Priority { get; private set; } + public string? Section { get; private set; } + public string? License { get; private set; } + public List Depends { get; } = new(); + public List PreDepends { get; } = new(); + public List Provides { get; } = new(); + public List Recommends { get; } = new(); + public List Suggests { get; } = new(); + public List Replaces { get; } = new(); + public List Conffiles { get; } = new(); + + public IReadOnlyDictionary Metadata => _metadata; + + public bool IsValid => !string.IsNullOrWhiteSpace(Name) + && !string.IsNullOrWhiteSpace(Version) + && !string.IsNullOrWhiteSpace(Architecture) + && !string.IsNullOrWhiteSpace(Status); + + public void SetField(string fieldName, string value) + { + switch (fieldName) + { + case "Package": + Name = value; + break; + case "Version": + Version = value; + break; + case "Architecture": + Architecture = value; + break; + case "Status": + Status = value; + break; + case "Source": + Source = value; + break; + case "Description": + _descriptionBuilder.Clear(); + _descriptionBuilder.Append(value); + Description = _descriptionBuilder.ToString(); + _currentMultilineField = fieldName; + break; + case "Homepage": + Homepage = value; + break; + case "Maintainer": + Maintainer = value; + break; + case "Origin": + Origin = value; + break; + case "Priority": + Priority = value; + break; + case "Section": + Section = value; + break; + case "License": + License = value; + break; + case "Depends": + Depends.AddRange(ParseRelations(value)); + break; + case "Pre-Depends": + PreDepends.AddRange(ParseRelations(value)); + break; + case "Provides": + Provides.AddRange(ParseRelations(value)); + break; + case "Recommends": + Recommends.AddRange(ParseRelations(value)); + break; + case "Suggests": + Suggests.AddRange(ParseRelations(value)); + break; + case "Replaces": + Replaces.AddRange(ParseRelations(value)); + break; + case "Conffiles": + _currentMultilineField = fieldName; + if (!string.IsNullOrWhiteSpace(value)) + { + AddConffile(value); + } + break; + default: + _metadata[fieldName] = value; + break; + } + } + + public void AppendContinuation(string fieldName, string continuation) + { + if (string.Equals(fieldName, "Description", StringComparison.OrdinalIgnoreCase)) + { + if (_descriptionBuilder.Length > 0) + { + _descriptionBuilder.AppendLine(); + } + + _descriptionBuilder.Append(continuation); + Description = _descriptionBuilder.ToString(); + _currentMultilineField = fieldName; + return; + } + + if (string.Equals(fieldName, "Conffiles", StringComparison.OrdinalIgnoreCase)) + { + AddConffile(continuation); + _currentMultilineField = fieldName; + return; + } + + if (_metadata.TryGetValue(fieldName, out var existing) && existing is not null) + { + _metadata[fieldName] = $"{existing}{Environment.NewLine}{continuation}"; + } + else + { + _metadata[fieldName] = continuation; + } + } + + public void FieldCompleted(string fieldName) + { + if (string.Equals(fieldName, _currentMultilineField, StringComparison.OrdinalIgnoreCase)) + { + _currentMultilineField = null; + } + } + + private void AddConffile(string value) + { + var tokens = value.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (tokens.Length >= 1) + { + var path = tokens[0]; + var checksum = tokens.Length >= 2 ? tokens[1] : null; + Conffiles.Add(new DpkgConffileEntry(path, checksum)); + } + } + + private static IEnumerable ParseRelations(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + yield break; + } + + foreach (var segment in value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + yield return segment; + } + } +} + +internal sealed record DpkgConffileEntry(string Path, string? Checksum); diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/Properties/AssemblyInfo.cs b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..308f76f1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Tests")] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj new file mode 100644 index 00000000..f8879698 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/StellaOps.Scanner.Analyzers.OS.Dpkg.csproj @@ -0,0 +1,15 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json new file mode 100644 index 00000000..8ff2ab98 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Dpkg/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.dpkg", + "displayName": "StellaOps Debian dpkg Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Dpkg.dll" + }, + "capabilities": [ + "os-analyzer", + "dpkg" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "dpkg" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/IRpmDatabaseReader.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/IRpmDatabaseReader.cs new file mode 100644 index 00000000..a4e6e743 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/IRpmDatabaseReader.cs @@ -0,0 +1,10 @@ +using System.Collections.Generic; +using System.Threading; +using StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm; + +internal interface IRpmDatabaseReader +{ + IReadOnlyList ReadHeaders(string rootPath, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeader.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeader.cs new file mode 100644 index 00000000..60a8bfdd --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeader.cs @@ -0,0 +1,86 @@ +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +internal sealed class RpmHeader +{ + public RpmHeader( + string name, + string version, + string architecture, + string? release, + string? epoch, + string? summary, + string? description, + string? license, + string? sourceRpm, + string? url, + string? vendor, + long? buildTime, + long? installTime, + IReadOnlyList provides, + IReadOnlyList provideVersions, + IReadOnlyList requires, + IReadOnlyList requireVersions, + IReadOnlyList files, + IReadOnlyList changeLogs, + IReadOnlyDictionary metadata) + { + Name = name; + Version = version; + Architecture = architecture; + Release = release; + Epoch = epoch; + Summary = summary; + Description = description; + License = license; + SourceRpm = sourceRpm; + Url = url; + Vendor = vendor; + BuildTime = buildTime; + InstallTime = installTime; + Provides = provides; + ProvideVersions = provideVersions; + Requires = requires; + RequireVersions = requireVersions; + Files = files; + ChangeLogs = changeLogs; + Metadata = metadata; + } + + public string Name { get; } + public string Version { get; } + public string Architecture { get; } + public string? Release { get; } + public string? Epoch { get; } + public string? Summary { get; } + public string? Description { get; } + public string? License { get; } + public string? SourceRpm { get; } + public string? Url { get; } + public string? Vendor { get; } + public long? BuildTime { get; } + public long? InstallTime { get; } + public IReadOnlyList Provides { get; } + public IReadOnlyList ProvideVersions { get; } + public IReadOnlyList Requires { get; } + public IReadOnlyList RequireVersions { get; } + public IReadOnlyList Files { get; } + public IReadOnlyList ChangeLogs { get; } + public IReadOnlyDictionary Metadata { get; } +} + +internal sealed class RpmFileEntry +{ + public RpmFileEntry(string path, bool isConfig, IReadOnlyDictionary digests) + { + Path = path; + IsConfig = isConfig; + Digests = digests; + } + + public string Path { get; } + public bool IsConfig { get; } + public IReadOnlyDictionary Digests { get; } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeaderParser.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeaderParser.cs new file mode 100644 index 00000000..ee55dbf5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmHeaderParser.cs @@ -0,0 +1,479 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +internal sealed class RpmHeaderParser +{ + private const uint HeaderMagic = 0x8eade8ab; + private const int RpmFileConfigFlag = 1; + + public RpmHeader Parse(ReadOnlySpan buffer) + { + if (buffer.Length < 16) + { + throw new InvalidOperationException("RPM header buffer too small."); + } + + var reader = new HeaderReader(buffer); + var magic = reader.ReadUInt32(); + if (magic != HeaderMagic) + { + throw new InvalidOperationException("Invalid RPM header magic."); + } + + reader.ReadByte(); // version + reader.ReadByte(); // reserved + reader.ReadUInt16(); // reserved + + var indexCount = reader.ReadInt32(); + var storeSize = reader.ReadInt32(); + + if (indexCount < 0 || storeSize < 0) + { + throw new InvalidOperationException("Corrupt RPM header lengths."); + } + + var entries = new IndexEntry[indexCount]; + for (var i = 0; i < indexCount; i++) + { + var tag = reader.ReadInt32(); + var type = (RpmDataType)reader.ReadInt32(); + var offset = reader.ReadInt32(); + var count = reader.ReadInt32(); + entries[i] = new IndexEntry(tag, type, offset, count); + } + + var store = reader.ReadBytes(storeSize); + + for (var i = 0; i < entries.Length; i++) + { + var current = entries[i]; + var nextOffset = i + 1 < entries.Length ? entries[i + 1].Offset : storeSize; + var length = Math.Max(0, nextOffset - current.Offset); + current.SetLength(length); + entries[i] = current; + } + + var values = new Dictionary(entries.Length); + foreach (var entry in entries) + { + if (entry.Offset < 0 || entry.Offset + entry.Length > store.Length) + { + continue; + } + + var slice = store.Slice(entry.Offset, entry.Length); + values[entry.Tag] = entry.Type switch + { + RpmDataType.Null => null, + RpmDataType.Char => slice.ToArray(), + RpmDataType.Int8 => ReadSByteArray(slice, entry.Count), + RpmDataType.Int16 => ReadInt16Array(slice, entry.Count), + RpmDataType.Int32 => ReadInt32Array(slice, entry.Count), + RpmDataType.Int64 => ReadInt64Array(slice, entry.Count), + RpmDataType.String => ReadString(slice), + RpmDataType.Bin => slice.ToArray(), + RpmDataType.StringArray => ReadStringArray(slice, entry.Count), + RpmDataType.I18NString => ReadStringArray(slice, entry.Count), + _ => null, + }; + } + + var name = RequireString(values, RpmTags.Name); + var version = RequireString(values, RpmTags.Version); + var arch = GetString(values, RpmTags.Arch) ?? "noarch"; + var release = GetString(values, RpmTags.Release); + var epoch = GetEpoch(values); + var summary = GetString(values, RpmTags.Summary); + var description = GetString(values, RpmTags.Description); + var license = GetString(values, RpmTags.License); + var sourceRpm = GetString(values, RpmTags.SourceRpm); + var url = GetString(values, RpmTags.Url); + var vendor = GetString(values, RpmTags.Vendor); + var buildTime = GetFirstInt64(values, RpmTags.BuildTime); + var installTime = GetFirstInt64(values, RpmTags.InstallTime); + var provides = GetStringArray(values, RpmTags.ProvideName); + var provideVersions = GetStringArray(values, RpmTags.ProvideVersion); + var requires = GetStringArray(values, RpmTags.RequireName); + var requireVersions = GetStringArray(values, RpmTags.RequireVersion); + var changeLogs = GetStringArray(values, RpmTags.ChangeLogText); + + var fileEntries = BuildFiles(values); + + var metadata = new SortedDictionary(StringComparer.Ordinal) + { + ["summary"] = summary, + ["description"] = description, + ["vendor"] = vendor, + ["url"] = url, + ["packager"] = GetString(values, RpmTags.Packager), + ["group"] = GetString(values, RpmTags.Group), + ["buildHost"] = GetString(values, RpmTags.BuildHost), + ["size"] = GetFirstInt64(values, RpmTags.Size)?.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["buildTime"] = buildTime?.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["installTime"] = installTime?.ToString(System.Globalization.CultureInfo.InvariantCulture), + ["os"] = GetString(values, RpmTags.Os), + }; + + return new RpmHeader( + name, + version, + arch, + release, + epoch, + summary, + description, + license, + sourceRpm, + url, + vendor, + buildTime, + installTime, + provides, + provideVersions, + requires, + requireVersions, + fileEntries, + changeLogs, + new ReadOnlyDictionary(metadata)); + } + + private static IReadOnlyList BuildFiles(Dictionary values) + { + var directories = GetStringArray(values, RpmTags.DirNames); + var basenames = GetStringArray(values, RpmTags.BaseNames); + var dirIndexes = GetInt32Array(values, RpmTags.DirIndexes); + var fileFlags = GetInt32Array(values, RpmTags.FileFlags); + var fileMd5 = GetStringArray(values, RpmTags.FileMd5); + var fileDigests = GetStringArray(values, RpmTags.FileDigests); + var digestAlgorithm = GetFirstInt32(values, RpmTags.FileDigestAlgorithm) ?? 1; + + if (basenames.Count == 0) + { + return Array.Empty(); + } + + var result = new List(basenames.Count); + for (var i = 0; i < basenames.Count; i++) + { + var dirIndex = dirIndexes.Count > i ? dirIndexes[i] : 0; + var directory = directories.Count > dirIndex ? directories[dirIndex] : "/"; + if (!directory.EndsWith('/')) + { + directory += "/"; + } + + var fullPath = (directory + basenames[i]).Replace("//", "/"); + var isConfig = fileFlags.Count > i && (fileFlags[i] & RpmFileConfigFlag) == RpmFileConfigFlag; + + var digests = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (fileDigests.Count > i && !string.IsNullOrWhiteSpace(fileDigests[i])) + { + digests[ResolveDigestName(digestAlgorithm)] = fileDigests[i]; + } + else if (fileMd5.Count > i && !string.IsNullOrWhiteSpace(fileMd5[i])) + { + digests["md5"] = fileMd5[i]; + } + + result.Add(new RpmFileEntry(fullPath, isConfig, new ReadOnlyDictionary(digests))); + } + + return new ReadOnlyCollection(result); + } + + private static string ResolveDigestName(int algorithm) + => algorithm switch + { + 1 => "md5", + 2 => "sha1", + 8 => "sha256", + 9 => "sha384", + 10 => "sha512", + _ => "md5", + }; + + private static string RequireString(Dictionary values, int tag) + { + var value = GetString(values, tag); + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"Required RPM tag {tag} missing."); + } + + return value; + } + + private static string? GetString(Dictionary values, int tag) + { + if (!values.TryGetValue(tag, out var value) || value is null) + { + return null; + } + + return value switch + { + string s => s, + string[] array when array.Length > 0 => array[0], + byte[] bytes => Encoding.UTF8.GetString(bytes).TrimEnd('\0'), + _ => value.ToString(), + }; + } + + private static IReadOnlyList GetStringArray(Dictionary values, int tag) + { + if (!values.TryGetValue(tag, out var value) || value is null) + { + return Array.Empty(); + } + + return value switch + { + string[] array => array, + string s => new[] { s }, + _ => Array.Empty(), + }; + } + + private static IReadOnlyList GetInt32Array(Dictionary values, int tag) + { + if (!values.TryGetValue(tag, out var value) || value is null) + { + return Array.Empty(); + } + + return value switch + { + int[] array => array, + int i => new[] { i }, + _ => Array.Empty(), + }; + } + + private static long? GetFirstInt64(Dictionary values, int tag) + { + if (!values.TryGetValue(tag, out var value) || value is null) + { + return null; + } + + return value switch + { + long[] array when array.Length > 0 => array[0], + long l => l, + int[] ints when ints.Length > 0 => ints[0], + int i => i, + _ => null, + }; + } + + private static int? GetFirstInt32(Dictionary values, int tag) + { + if (!values.TryGetValue(tag, out var value) || value is null) + { + return null; + } + + return value switch + { + int[] array when array.Length > 0 => array[0], + int i => i, + _ => null, + }; + } + + private static string? GetEpoch(Dictionary values) + { + if (!values.TryGetValue(RpmTags.Epoch, out var value) || value is null) + { + return null; + } + + return value switch + { + int i when i > 0 => i.ToString(System.Globalization.CultureInfo.InvariantCulture), + int[] array when array.Length > 0 => array[0].ToString(System.Globalization.CultureInfo.InvariantCulture), + string s => s, + _ => null, + }; + } + + private static sbyte[] ReadSByteArray(ReadOnlySpan slice, int count) + { + if (count <= 0) + { + return Array.Empty(); + } + + var result = new sbyte[count]; + for (var i = 0; i < count && i < slice.Length; i++) + { + result[i] = unchecked((sbyte)slice[i]); + } + + return result; + } + + private static short[] ReadInt16Array(ReadOnlySpan slice, int count) + { + if (count <= 0) + { + return Array.Empty(); + } + + var result = new short[count]; + for (var i = 0; i < count && (i * 2 + 2) <= slice.Length; i++) + { + result[i] = unchecked((short)BinaryPrimitives.ReadInt16BigEndian(slice[(i * 2)..])); + } + + return result; + } + + private static int[] ReadInt32Array(ReadOnlySpan slice, int count) + { + if (count <= 0) + { + return Array.Empty(); + } + + var result = new int[count]; + for (var i = 0; i < count && (i * 4 + 4) <= slice.Length; i++) + { + result[i] = BinaryPrimitives.ReadInt32BigEndian(slice[(i * 4)..]); + } + + return result; + } + + private static long[] ReadInt64Array(ReadOnlySpan slice, int count) + { + if (count <= 0) + { + return Array.Empty(); + } + + var result = new long[count]; + for (var i = 0; i < count && (i * 8 + 8) <= slice.Length; i++) + { + result[i] = BinaryPrimitives.ReadInt64BigEndian(slice[(i * 8)..]); + } + + return result; + } + + private static string ReadString(ReadOnlySpan slice) + { + var zero = slice.IndexOf((byte)0); + if (zero >= 0) + { + slice = slice[..zero]; + } + + return Encoding.UTF8.GetString(slice); + } + + private static string[] ReadStringArray(ReadOnlySpan slice, int count) + { + if (count <= 0) + { + return Array.Empty(); + } + + var list = new List(count); + var span = slice; + for (var i = 0; i < count && span.Length > 0; i++) + { + var zero = span.IndexOf((byte)0); + if (zero < 0) + { + list.Add(Encoding.UTF8.GetString(span).TrimEnd('\0')); + break; + } + + var value = Encoding.UTF8.GetString(span[..zero]); + list.Add(value); + span = span[(zero + 1)..]; + } + + return list.ToArray(); + } + + private struct IndexEntry + { + public IndexEntry(int tag, RpmDataType type, int offset, int count) + { + Tag = tag; + Type = type; + Offset = offset; + Count = count; + Length = 0; + } + + public int Tag { get; } + public RpmDataType Type { get; } + public int Offset { get; } + public int Count { get; } + public int Length { readonly get; private set; } + public void SetLength(int length) => Length = length; + } + + private enum RpmDataType + { + Null = 0, + Char = 1, + Int8 = 2, + Int16 = 3, + Int32 = 4, + Int64 = 5, + String = 6, + Bin = 7, + StringArray = 8, + I18NString = 9, + } + + private ref struct HeaderReader + { + private readonly ReadOnlySpan _buffer; + private int _offset; + + public HeaderReader(ReadOnlySpan buffer) + { + _buffer = buffer; + _offset = 0; + } + + public uint ReadUInt32() + { + var value = BinaryPrimitives.ReadUInt32BigEndian(_buffer[_offset..]); + _offset += 4; + return value; + } + + public int ReadInt32() => (int)ReadUInt32(); + + public ushort ReadUInt16() + { + var value = BinaryPrimitives.ReadUInt16BigEndian(_buffer[_offset..]); + _offset += 2; + return value; + } + + public byte ReadByte() + { + return _buffer[_offset++]; + } + + public ReadOnlySpan ReadBytes(int length) + { + var slice = _buffer.Slice(_offset, length); + _offset += length; + return slice; + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmTags.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmTags.cs new file mode 100644 index 00000000..4ad3db19 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Internal/RpmTags.cs @@ -0,0 +1,36 @@ +namespace StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +internal static class RpmTags +{ + public const int Name = 1000; + public const int Version = 1001; + public const int Release = 1002; + public const int Epoch = 1003; + public const int Summary = 1004; + public const int Description = 1005; + public const int BuildTime = 1006; + public const int InstallTime = 1008; + public const int Size = 1009; + public const int Vendor = 1011; + public const int License = 1014; + public const int Packager = 1015; + public const int BuildHost = 1007; + public const int Group = 1016; + public const int Url = 1020; + public const int Os = 1021; + public const int Arch = 1022; + public const int SourceRpm = 1044; + public const int ProvideName = 1047; + public const int ProvideVersion = 1048; + public const int RequireName = 1049; + public const int RequireVersion = 1050; + public const int DirNames = 1098; + public const int ChangeLogText = 1082; + public const int DirIndexes = 1116; + public const int BaseNames = 1117; + public const int FileFlags = 1037; + public const int FileSizes = 1028; + public const int FileMd5 = 1035; + public const int FileDigests = 1146; + public const int FileDigestAlgorithm = 5011; +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/Properties/AssemblyInfo.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..308f76f1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Tests")] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmAnalyzerPlugin.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmAnalyzerPlugin.cs new file mode 100644 index 00000000..86187663 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmAnalyzerPlugin.cs @@ -0,0 +1,23 @@ +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Plugin; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm; + +public sealed class RpmAnalyzerPlugin : IOSAnalyzerPlugin +{ + public string Name => "StellaOps.Scanner.Analyzers.OS.Rpm"; + + public bool IsAvailable(IServiceProvider services) => services is not null; + + public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + var loggerFactory = services.GetRequiredService(); + return new RpmPackageAnalyzer( + loggerFactory.CreateLogger(), + new RpmDatabaseReader(loggerFactory.CreateLogger())); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmDatabaseReader.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmDatabaseReader.cs new file mode 100644 index 00000000..7028053a --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmDatabaseReader.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm; + +internal sealed class RpmDatabaseReader : IRpmDatabaseReader +{ + private readonly ILogger _logger; + private readonly RpmHeaderParser _parser = new(); + + public RpmDatabaseReader(ILogger logger) + { + _logger = logger; + } + + public IReadOnlyList ReadHeaders(string rootPath, CancellationToken cancellationToken) + { + var sqlitePath = ResolveSqlitePath(rootPath); + if (sqlitePath is null) + { + _logger.LogWarning("rpmdb.sqlite not found under root {RootPath}; rpm analyzer will skip.", rootPath); + return Array.Empty(); + } + + var headers = new List(); + try + { + var connectionString = new SqliteConnectionStringBuilder + { + DataSource = sqlitePath, + Mode = SqliteOpenMode.ReadOnly, + }.ToString(); + + using var connection = new SqliteConnection(connectionString); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = "SELECT * FROM Packages"; + + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + cancellationToken.ThrowIfCancellationRequested(); + var blob = ExtractHeaderBlob(reader); + if (blob is null) + { + continue; + } + + try + { + headers.Add(_parser.Parse(blob)); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse RPM header record (pkgKey={PkgKey}).", TryGetPkgKey(reader)); + } + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Unable to read rpmdb.sqlite at {Path}.", sqlitePath); + return Array.Empty(); + } + + return headers; + } + + private static string? ResolveSqlitePath(string rootPath) + { + var candidates = new[] + { + Path.Combine(rootPath, "var", "lib", "rpm", "rpmdb.sqlite"), + Path.Combine(rootPath, "usr", "lib", "sysimage", "rpm", "rpmdb.sqlite"), + }; + + foreach (var candidate in candidates) + { + if (File.Exists(candidate)) + { + return candidate; + } + } + + return null; + } + + private static byte[]? ExtractHeaderBlob(SqliteDataReader reader) + { + for (var i = 0; i < reader.FieldCount; i++) + { + if (reader.GetFieldType(i) == typeof(byte[])) + { + return reader.GetFieldValue(i); + } + } + + return null; + } + + private static object? TryGetPkgKey(SqliteDataReader reader) + { + try + { + var ordinal = reader.GetOrdinal("pkgKey"); + if (ordinal >= 0) + { + return reader.GetValue(ordinal); + } + } + catch + { + } + + return null; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmPackageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmPackageAnalyzer.cs new file mode 100644 index 00000000..1b71a138 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/RpmPackageAnalyzer.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Analyzers; +using StellaOps.Scanner.Analyzers.OS.Helpers; +using StellaOps.Scanner.Analyzers.OS.Rpm.Internal; + +namespace StellaOps.Scanner.Analyzers.OS.Rpm; + +internal sealed class RpmPackageAnalyzer : OsPackageAnalyzerBase +{ + private static readonly IReadOnlyList EmptyPackages = + new ReadOnlyCollection(Array.Empty()); + + private readonly IRpmDatabaseReader _reader; + + public RpmPackageAnalyzer(ILogger logger) + : this(logger, null) + { + } + + internal RpmPackageAnalyzer(ILogger logger, IRpmDatabaseReader? reader) + : base(logger) + { + _reader = reader ?? new RpmDatabaseReader(logger); + } + + public override string AnalyzerId => "rpm"; + + protected override ValueTask> ExecuteCoreAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken) + { + var headers = _reader.ReadHeaders(context.RootPath, cancellationToken); + if (headers.Count == 0) + { + return ValueTask.FromResult>(EmptyPackages); + } + + var records = new List(headers.Count); + foreach (var header in headers) + { + try + { + var purl = PackageUrlBuilder.BuildRpm(header.Name, header.Epoch, header.Version, header.Release, header.Architecture); + + var vendorMetadata = new Dictionary(StringComparer.Ordinal) + { + ["summary"] = header.Summary, + ["description"] = header.Description, + ["vendor"] = header.Vendor, + ["url"] = header.Url, + ["sourceRpm"] = header.SourceRpm, + ["buildTime"] = header.BuildTime?.ToString(CultureInfo.InvariantCulture), + ["installTime"] = header.InstallTime?.ToString(CultureInfo.InvariantCulture), + }; + + foreach (var kvp in header.Metadata) + { + vendorMetadata[$"rpm:{kvp.Key}"] = kvp.Value; + } + + var provides = ComposeRelations(header.Provides, header.ProvideVersions); + var requires = ComposeRelations(header.Requires, header.RequireVersions); + + var files = new List(header.Files.Count); + foreach (var file in header.Files) + { + IDictionary? digests = null; + if (file.Digests.Count > 0) + { + digests = new Dictionary(file.Digests, StringComparer.OrdinalIgnoreCase); + } + + files.Add(new OSPackageFileEvidence(file.Path, isConfigFile: file.IsConfig, digests: digests)); + } + + var cveHints = CveHintExtractor.Extract( + header.Description, + string.Join('\n', header.ChangeLogs)); + + var record = new OSPackageRecord( + AnalyzerId, + purl, + header.Name, + header.Version, + header.Architecture, + PackageEvidenceSource.RpmDatabase, + epoch: header.Epoch, + release: header.Release, + sourcePackage: header.SourceRpm, + license: header.License, + cveHints: cveHints, + provides: provides, + depends: requires, + files: files, + vendorMetadata: vendorMetadata); + + records.Add(record); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to convert RPM header for package {Name}.", header.Name); + } + } + + records.Sort(); + return ValueTask.FromResult>(records); + } + + private static IReadOnlyList ComposeRelations(IReadOnlyList names, IReadOnlyList versions) + { + if (names.Count == 0) + { + return Array.Empty(); + } + + var result = new string[names.Count]; + for (var i = 0; i < names.Count; i++) + { + var version = versions.Count > i ? versions[i] : null; + result[i] = string.IsNullOrWhiteSpace(version) + ? names[i] + : $"{names[i]} = {version}"; + } + + return result; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj b/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj new file mode 100644 index 00000000..e773101b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/StellaOps.Scanner.Analyzers.OS.Rpm.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json b/src/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json new file mode 100644 index 00000000..ed842a2f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Rpm/manifest.json @@ -0,0 +1,19 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.analyzers.os.rpm", + "displayName": "StellaOps RPM Analyzer", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "assembly": "StellaOps.Scanner.Analyzers.OS.Rpm.dll" + }, + "capabilities": [ + "os-analyzer", + "rpm" + ], + "metadata": { + "org.stellaops.analyzer.kind": "os", + "org.stellaops.analyzer.id": "rpm" + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/apk/lib/apk/db/installed b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/apk/lib/apk/db/installed new file mode 100644 index 00000000..a293e330 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/apk/lib/apk/db/installed @@ -0,0 +1,23 @@ +C:Q1 +P:busybox +V:1.37.0-r0 +A:x86_64 +S:4096 +I:8192 +T:BusyBox utility set +U:https://busybox.net/ +L:GPL-2.0-only +o:busybox +m:Stella Ops +t:1729286400 +c:deadbeef12345678 +D:musl>=1.2.5-r0 ssl_client +p:/bin/sh +F:bin +R:busybox +Z:0f1e2d3c4b5a6978ffeeddccbbaa9988 +F:etc +R:profile +Z:11223344556677889900aabbccddeeff +a:0:0:0644 + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.conffiles b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.conffiles new file mode 100644 index 00000000..198895f7 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.conffiles @@ -0,0 +1 @@ +/etc/bash.bashrc abcdef1234567890 diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.list b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.list new file mode 100644 index 00000000..3fb23f06 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.list @@ -0,0 +1,3 @@ +/bin/bash +/etc/bash.bashrc +/usr/share/doc/bash/changelog.Debian.gz diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.md5sums b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.md5sums new file mode 100644 index 00000000..2c154b7b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/info/bash.md5sums @@ -0,0 +1,2 @@ +0123456789abcdef0123456789abcdef /bin/bash +abcdef1234567890abcdef1234567890 /etc/bash.bashrc diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/status b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/status new file mode 100644 index 00000000..cbac17be --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/dpkg/var/lib/dpkg/status @@ -0,0 +1,15 @@ +Package: bash +Status: install ok installed +Priority: important +Section: shells +Installed-Size: 1024 +Maintainer: Debian Developers +Architecture: amd64 +Version: 5.2.21-2 +Source: bash (5.2.21-2) +Homepage: https://www.gnu.org/software/bash/ +Description: GNU Bourne Again Shell + This is the GNU Bourne Again Shell. +Conffiles: + /etc/bash.bashrc abcdef1234567890 + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/apk.json b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/apk.json new file mode 100644 index 00000000..6526f68f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/apk.json @@ -0,0 +1,74 @@ +[ + { + "analyzerId": "apk", + "durationMilliseconds": 0, + "packageCount": 1, + "fileEvidenceCount": 4, + "warnings": [], + "packages": [ + { + "packageUrl": "pkg:alpine/busybox@1.37.0-r0?arch=x86_64", + "name": "busybox", + "version": "1.37.0", + "architecture": "x86_64", + "epoch": null, + "release": "r0", + "sourcePackage": "busybox", + "license": "GPL-2.0-only", + "evidenceSource": "ApkDatabase", + "cveHints": [], + "provides": [ + "/bin/sh" + ], + "depends": [ + "musl\u003E=1.2.5-r0", + "ssl_client" + ], + "files": [ + { + "path": "/bin/", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": false, + "digests": {} + }, + { + "path": "/bin/busybox", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": false, + "digests": {} + }, + { + "path": "/etc/", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": false, + "digests": {} + }, + { + "path": "/etc/profile", + "layerDigest": null, + "sha256": "0f1e2d3c4b5a6978ffeeddccbbaa9988", + "sizeBytes": null, + "isConfigFile": false, + "digests": { + "sha256": "0f1e2d3c4b5a6978ffeeddccbbaa9988" + } + } + ], + "vendorMetadata": { + "buildTime": "1729286400", + "checksum": "deadbeef12345678", + "description": "BusyBox utility set", + "homepage": "https://busybox.net/", + "maintainer": "Stella Ops \u003Cops@stella-ops.org\u003E", + "origin": "busybox" + } + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/dpkg.json b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/dpkg.json new file mode 100644 index 00000000..566833c5 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/dpkg.json @@ -0,0 +1,64 @@ +[ + { + "analyzerId": "dpkg", + "durationMilliseconds": 0, + "packageCount": 1, + "fileEvidenceCount": 3, + "warnings": [], + "packages": [ + { + "packageUrl": "pkg:deb/debian/bash@5.2.21-2?arch=amd64", + "name": "bash", + "version": "5.2.21", + "architecture": "amd64", + "epoch": null, + "release": "2", + "sourcePackage": "bash", + "license": null, + "evidenceSource": "DpkgStatus", + "cveHints": [], + "provides": [], + "depends": [], + "files": [ + { + "path": "/bin/bash", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": false, + "digests": { + "md5": "0123456789abcdef0123456789abcdef" + } + }, + { + "path": "/etc/bash.bashrc", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": true, + "digests": { + "md5": "abcdef1234567890abcdef1234567890" + } + }, + { + "path": "/usr/share/doc/bash/changelog.Debian.gz", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": false, + "digests": {} + } + ], + "vendorMetadata": { + "dpkg:Installed-Size": "1024", + "homepage": "https://www.gnu.org/software/bash/", + "maintainer": "Debian Developers \u003Cdebian-devel@lists.debian.org\u003E", + "origin": null, + "priority": "important", + "section": "shells", + "source": "bash (5.2.21-2)" + } + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/rpm.json b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/rpm.json new file mode 100644 index 00000000..48e11d9b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Fixtures/goldens/rpm.json @@ -0,0 +1,64 @@ +[ + { + "analyzerId": "rpm", + "durationMilliseconds": 0, + "packageCount": 1, + "fileEvidenceCount": 2, + "warnings": [], + "packages": [ + { + "packageUrl": "pkg:rpm/openssl-libs@1:3.2.1-8.el9?arch=x86_64", + "name": "openssl-libs", + "version": "3.2.1", + "architecture": "x86_64", + "epoch": "1", + "release": "8.el9", + "sourcePackage": "openssl-3.2.1-8.el9.src.rpm", + "license": "OpenSSL", + "evidenceSource": "RpmDatabase", + "cveHints": [ + "CVE-2025-1234" + ], + "provides": [ + "libcrypto.so.3()(64bit)", + "openssl-libs" + ], + "depends": [ + "glibc(x86-64) \u003E= 2.34" + ], + "files": [ + { + "path": "/etc/pki/tls/openssl.cnf", + "layerDigest": null, + "sha256": null, + "sizeBytes": null, + "isConfigFile": true, + "digests": { + "md5": "c0ffee" + } + }, + { + "path": "/usr/lib64/libcrypto.so.3", + "layerDigest": null, + "sha256": "abc123", + "sizeBytes": null, + "isConfigFile": false, + "digests": { + "sha256": "abc123" + } + } + ], + "vendorMetadata": { + "buildTime": null, + "description": null, + "installTime": null, + "rpm:summary": "TLS toolkit", + "sourceRpm": "openssl-3.2.1-8.el9.src.rpm", + "summary": "TLS toolkit", + "url": null, + "vendor": null + } + } + ] + } +] diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/Mapping/OsComponentMapperTests.cs b/src/StellaOps.Scanner.Analyzers.OS.Tests/Mapping/OsComponentMapperTests.cs new file mode 100644 index 00000000..39315a16 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/Mapping/OsComponentMapperTests.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using StellaOps.Scanner.Analyzers.OS.Mapping; +using StellaOps.Scanner.Core.Contracts; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.OS.Tests.Mapping; + +public class OsComponentMapperTests +{ + [Fact] + public void ToLayerFragments_ProducesDeterministicComponents() + { + var package = new OSPackageRecord( + analyzerId: "apk", + packageUrl: "pkg:alpine/busybox@1.37.0-r0?arch=x86_64", + name: "busybox", + version: "1.37.0", + architecture: "x86_64", + evidenceSource: PackageEvidenceSource.ApkDatabase, + release: "r0", + sourcePackage: "busybox", + license: "GPL-2.0-only", + depends: new[] { "musl>=1.2.5-r0", "ssl_client" }, + files: new[] + { + new OSPackageFileEvidence("/bin/busybox", sha256: "abc123", isConfigFile: false), + new OSPackageFileEvidence("/etc/profile", isConfigFile: true, digests: new Dictionary { ["md5"] = "deadbeef" }), + }, + vendorMetadata: new Dictionary + { + ["homepage"] = "https://busybox.net/", + }); + + var result = new OSPackageAnalyzerResult( + analyzerId: "apk", + packages: ImmutableArray.Create(package), + telemetry: new OSAnalyzerTelemetry(System.TimeSpan.Zero, 1, 2)); + + var fragments = OsComponentMapper.ToLayerFragments(new[] { result }); + + Assert.Single(fragments); + var fragment = fragments[0]; + Assert.StartsWith("sha256:", fragment.LayerDigest); + Assert.Single(fragment.Components); + + var component = fragment.Components[0]; + Assert.Equal(fragment.LayerDigest, component.LayerDigest); + Assert.Equal("pkg:alpine/busybox@1.37.0-r0?arch=x86_64", component.Identity.Key); + Assert.Equal("busybox", component.Identity.Name); + Assert.Equal("1.37.0", component.Identity.Version); + Assert.Equal("pkg:alpine/busybox@1.37.0-r0?arch=x86_64", component.Identity.Purl); + Assert.Equal("os-package", component.Identity.ComponentType); + Assert.Equal("busybox", component.Identity.Group); + Assert.Collection(component.Evidence, + evidence => + { + Assert.Equal("file", evidence.Kind); + Assert.Equal("/bin/busybox", evidence.Value); + Assert.Equal("abc123", evidence.Source); + }, + evidence => + { + Assert.Equal("config-file", evidence.Kind); + Assert.Equal("/etc/profile", evidence.Value); + Assert.Null(evidence.Source); + }); + Assert.Equal(new[] { "musl>=1.2.5-r0", "ssl_client" }, component.Dependencies); + Assert.False(component.Usage.UsedByEntrypoint); + Assert.NotNull(component.Metadata); + Assert.Equal(new[] { "GPL-2.0-only" }, component.Metadata!.Licenses); + Assert.Contains("stellaops.os.analyzer", component.Metadata.Properties!.Keys); + Assert.Equal("apk", component.Metadata.Properties!["stellaops.os.analyzer"]); + Assert.Equal("https://busybox.net/", component.Metadata.Properties!["vendor.homepage"]); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/OsAnalyzerDeterminismTests.cs b/src/StellaOps.Scanner.Analyzers.OS.Tests/OsAnalyzerDeterminismTests.cs new file mode 100644 index 00000000..4f1275b1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/OsAnalyzerDeterminismTests.cs @@ -0,0 +1,137 @@ +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Scanner.Analyzers.OS; +using StellaOps.Scanner.Analyzers.OS.Apk; +using StellaOps.Scanner.Analyzers.OS.Dpkg; +using StellaOps.Scanner.Analyzers.OS.Rpm; +using StellaOps.Scanner.Analyzers.OS.Rpm.Internal; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Analyzers.OS.Tests.TestUtilities; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.OS.Tests; + +public sealed class OsAnalyzerDeterminismTests +{ + [Fact] + public async Task ApkAnalyzerMatchesGolden() + { + using var fixture = FixtureManager.UseFixture("apk", out var rootPath); + var analyzer = new ApkPackageAnalyzer(NullLogger.Instance); + var context = CreateContext(rootPath); + + var result = await analyzer.AnalyzeAsync(context, CancellationToken.None); + var snapshot = SnapshotSerializer.Serialize(new[] { result }); + GoldenAssert.MatchSnapshot(snapshot, FixtureManager.GetGoldenPath("apk.json")); + } + + [Fact] + public async Task DpkgAnalyzerMatchesGolden() + { + using var fixture = FixtureManager.UseFixture("dpkg", out var rootPath); + var analyzer = new DpkgPackageAnalyzer(NullLogger.Instance); + var context = CreateContext(rootPath); + + var result = await analyzer.AnalyzeAsync(context, CancellationToken.None); + var snapshot = SnapshotSerializer.Serialize(new[] { result }); + GoldenAssert.MatchSnapshot(snapshot, FixtureManager.GetGoldenPath("dpkg.json")); + } + + [Fact] + public async Task RpmAnalyzerMatchesGolden() + { + var headers = new[] + { + CreateRpmHeader( + name: "openssl-libs", + version: "3.2.1", + architecture: "x86_64", + release: "8.el9", + epoch: "1", + license: "OpenSSL", + sourceRpm: "openssl-3.2.1-8.el9.src.rpm", + provides: new[] { "libcrypto.so.3()(64bit)", "openssl-libs" }, + requires: new[] { "glibc(x86-64) >= 2.34" }, + files: new[] + { + new RpmFileEntry("/usr/lib64/libcrypto.so.3", false, new Dictionary { ["sha256"] = "abc123" }), + new RpmFileEntry("/etc/pki/tls/openssl.cnf", true, new Dictionary { ["md5"] = "c0ffee" }) + }, + changeLogs: new[] { "Resolves: CVE-2025-1234" }, + metadata: new Dictionary { ["summary"] = "TLS toolkit" }) + }; + + var reader = new StubRpmDatabaseReader(headers); + var analyzer = new RpmPackageAnalyzer( + NullLogger.Instance, + reader); + + var context = CreateContext("/tmp/nonexistent"); + var result = await analyzer.AnalyzeAsync(context, CancellationToken.None); + var snapshot = SnapshotSerializer.Serialize(new[] { result }); + GoldenAssert.MatchSnapshot(snapshot, FixtureManager.GetGoldenPath("rpm.json")); + } + + private static OSPackageAnalyzerContext CreateContext(string rootPath) + { + var metadata = new Dictionary + { + [ScanMetadataKeys.RootFilesystemPath] = rootPath + }; + + return new OSPackageAnalyzerContext(rootPath, workspacePath: null, TimeProvider.System, NullLoggerFactory.Instance.CreateLogger("os-analyzer-tests"), metadata); + } + + private static RpmHeader CreateRpmHeader( + string name, + string version, + string architecture, + string? release, + string? epoch, + string? license, + string? sourceRpm, + IReadOnlyList provides, + IReadOnlyList requires, + IReadOnlyList files, + IReadOnlyList changeLogs, + IReadOnlyDictionary metadata) + { + return new RpmHeader( + name, + version, + architecture, + release, + epoch, + metadata.TryGetValue("summary", out var summary) ? summary : null, + metadata.TryGetValue("description", out var description) ? description : null, + license, + sourceRpm, + metadata.TryGetValue("url", out var url) ? url : null, + metadata.TryGetValue("vendor", out var vendor) ? vendor : null, + buildTime: null, + installTime: null, + provides, + provideVersions: provides.Select(_ => string.Empty).ToArray(), + requires, + requireVersions: requires.Select(_ => string.Empty).ToArray(), + files, + changeLogs, + metadata); + } + + private sealed class StubRpmDatabaseReader : IRpmDatabaseReader + { + private readonly IReadOnlyList _headers; + + public StubRpmDatabaseReader(IReadOnlyList headers) + { + _headers = headers; + } + + public IReadOnlyList ReadHeaders(string rootPath, CancellationToken cancellationToken) + => _headers; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj b/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj new file mode 100644 index 00000000..3a6cd8ce --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/StellaOps.Scanner.Analyzers.OS.Tests.csproj @@ -0,0 +1,28 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/FixtureManager.cs b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/FixtureManager.cs new file mode 100644 index 00000000..a629f013 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/FixtureManager.cs @@ -0,0 +1,75 @@ +using System; +using System.IO; + +namespace StellaOps.Scanner.Analyzers.OS.Tests.TestUtilities; + +internal static class FixtureManager +{ + public static IDisposable UseFixture(string name, out string rootPath) + { + var basePath = Path.Combine(AppContext.BaseDirectory, "Fixtures", name); + if (!Directory.Exists(basePath)) + { + throw new DirectoryNotFoundException($"Fixture '{name}' was not found at '{basePath}'."); + } + + var tempRoot = Path.Combine(Path.GetTempPath(), "stellaops-os-fixture", name, Guid.NewGuid().ToString("n")); + CopyDirectory(basePath, tempRoot); + rootPath = tempRoot; + return new Disposable(() => DeleteDirectory(tempRoot)); + } + + public static string GetGoldenPath(string name) + => Path.Combine(AppContext.BaseDirectory, "Fixtures", "goldens", name); + + private static void CopyDirectory(string source, string destination) + { + Directory.CreateDirectory(destination); + foreach (var file in Directory.GetFiles(source, "*", SearchOption.AllDirectories)) + { + var relative = Path.GetRelativePath(source, file); + var target = Path.Combine(destination, relative); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target); + } + } + + private static void DeleteDirectory(string path) + { + if (!Directory.Exists(path)) + { + return; + } + + try + { + Directory.Delete(path, recursive: true); + } + catch + { + // best-effort cleanup + } + } + + private sealed class Disposable : IDisposable + { + private readonly Action _dispose; + private bool _disposed; + + public Disposable(Action dispose) + { + _dispose = dispose; + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _disposed = true; + _dispose(); + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/GoldenAssert.cs b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/GoldenAssert.cs new file mode 100644 index 00000000..4eec282b --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/GoldenAssert.cs @@ -0,0 +1,41 @@ +using System; +using System.IO; +using Xunit; + +namespace StellaOps.Scanner.Analyzers.OS.Tests.TestUtilities; + +internal static class GoldenAssert +{ + private const string UpdateEnvironmentVariable = "UPDATE_OS_ANALYZER_FIXTURES"; + + public static void MatchSnapshot(string snapshot, string goldenPath) + { + var directory = Path.GetDirectoryName(goldenPath); + if (!string.IsNullOrWhiteSpace(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + snapshot = Normalize(snapshot); + + if (!File.Exists(goldenPath)) + { + File.WriteAllText(goldenPath, snapshot); + return; + } + + if (ShouldUpdate()) + { + File.WriteAllText(goldenPath, snapshot); + } + + var expected = Normalize(File.ReadAllText(goldenPath)); + Assert.Equal(expected.TrimEnd(), snapshot.TrimEnd()); + } + + private static bool ShouldUpdate() + => string.Equals(Environment.GetEnvironmentVariable(UpdateEnvironmentVariable), "1", StringComparison.OrdinalIgnoreCase); + + private static string Normalize(string value) + => value.Replace("\r\n", "\n"); +} diff --git a/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/SnapshotSerializer.cs b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/SnapshotSerializer.cs new file mode 100644 index 00000000..ee1a2f80 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS.Tests/TestUtilities/SnapshotSerializer.cs @@ -0,0 +1,106 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Analyzers.OS; + +namespace StellaOps.Scanner.Analyzers.OS.Tests.TestUtilities; + +internal static class SnapshotSerializer +{ + private static readonly JsonSerializerOptions Options = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = true, + Converters = + { + new JsonStringEnumConverter(JsonNamingPolicy.CamelCase) + } + }; + + public static string Serialize(IEnumerable results) + { + var ordered = results + .OrderBy(r => r.AnalyzerId, StringComparer.OrdinalIgnoreCase) + .Select(result => new AnalyzerSnapshot + { + AnalyzerId = result.AnalyzerId, + PackageCount = result.Telemetry.PackageCount, + FileEvidenceCount = result.Telemetry.FileEvidenceCount, + DurationMilliseconds = 0, + Warnings = result.Warnings.Select(w => new WarningSnapshot(w.Code, w.Message)).ToArray(), + Packages = result.Packages + .OrderBy(p => p, Comparer.Default) + .Select(p => new PackageSnapshot + { + PackageUrl = p.PackageUrl, + Name = p.Name, + Version = p.Version, + Architecture = p.Architecture, + Epoch = p.Epoch, + Release = p.Release, + SourcePackage = p.SourcePackage, + License = p.License, + EvidenceSource = p.EvidenceSource.ToString(), + CveHints = p.CveHints, + Provides = p.Provides, + Depends = p.Depends, + Files = p.Files.Select(f => new FileSnapshot + { + Path = f.Path, + LayerDigest = f.LayerDigest, + Sha256 = f.Sha256, + SizeBytes = f.SizeBytes, + IsConfigFile = f.IsConfigFile, + Digests = f.Digests.OrderBy(kv => kv.Key, StringComparer.OrdinalIgnoreCase).ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.OrdinalIgnoreCase) + }).ToArray(), + VendorMetadata = p.VendorMetadata.OrderBy(kv => kv.Key, StringComparer.Ordinal).ToDictionary(kv => kv.Key, kv => kv.Value, StringComparer.Ordinal) + }).ToArray() + }) + .ToArray(); + + return JsonSerializer.Serialize(ordered, Options); + } + + private sealed record AnalyzerSnapshot + { + public string AnalyzerId { get; init; } = string.Empty; + public double DurationMilliseconds { get; init; } + public int PackageCount { get; init; } + public int FileEvidenceCount { get; init; } + public IReadOnlyList Warnings { get; init; } = Array.Empty(); + public IReadOnlyList Packages { get; init; } = Array.Empty(); + } + + private sealed record WarningSnapshot(string Code, string Message); + + private sealed record PackageSnapshot + { + public string PackageUrl { get; init; } = string.Empty; + public string Name { get; init; } = string.Empty; + public string Version { get; init; } = string.Empty; + public string Architecture { get; init; } = string.Empty; + public string? Epoch { get; init; } + public string? Release { get; init; } + public string? SourcePackage { get; init; } + public string? License { get; init; } + public string EvidenceSource { get; init; } = string.Empty; + public IReadOnlyList CveHints { get; init; } = Array.Empty(); + public IReadOnlyList Provides { get; init; } = Array.Empty(); + public IReadOnlyList Depends { get; init; } = Array.Empty(); + public IReadOnlyList Files { get; init; } = Array.Empty(); + public IReadOnlyDictionary VendorMetadata { get; init; } = new Dictionary(); + } + + private sealed record FileSnapshot + { + public string Path { get; init; } = string.Empty; + public string? LayerDigest { get; init; } + public string? Sha256 { get; init; } + public long? SizeBytes { get; init; } + public bool? IsConfigFile { get; init; } + public IReadOnlyDictionary Digests { get; init; } = new Dictionary(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/AGENTS.md b/src/StellaOps.Scanner.Analyzers.OS/AGENTS.md new file mode 100644 index 00000000..91f164f3 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/AGENTS.md @@ -0,0 +1,40 @@ +# AGENTS +## Role +Design and ship deterministic Linux operating-system analyzers that transform container root filesystems into canonical package evidence for SBOM emission. + +## Scope +- Provide shared helpers for reading apk, dpkg, and rpm metadata and emitting normalized package identities with provenance. +- Implement analyzer plug-ins for Alpine (apk), Debian (dpkg), and RPM-based distributions that operate on extracted rootfs snapshots. +- Enrich package records with vendor-origin metadata (source packages, declared licenses, CVE hints) and evidence linking files to packages. +- Expose restart-time plug-in manifests so the Scanner.Worker can load analyzers in offline or air-gapped environments. +- Supply deterministic fixtures and a regression harness that verifies analyzer outputs remain stable across runs. + +## Participants +- `StellaOps.Scanner.Core` for shared contracts, observability, and plug-in catalog guardrails. +- `StellaOps.Scanner.Worker` which executes analyzers inside the scan pipeline. +- `StellaOps.Scanner.Cache` (future) for layer cache integration; analyzers must be cache-aware via deterministic inputs/outputs. +- `StellaOps.Scanner.Emit` and `StellaOps.Scanner.Diff` rely on analyzer outputs to build SBOMs and change reports. + +## Interfaces & Contracts +- Analyzers implement `IOSPackageAnalyzer` (defined in this module) and register via plug-in manifests; they must be restart-time only. +- Input rootfs paths are read-only; analyzers must never mutate files and must tolerate missing metadata gracefully. +- Package records emit canonical purls (`pkg:alpine`, `pkg:deb`, `pkg:rpm`) plus NEVRA/EVR details, source package identifiers, declared licenses, and evidence (file lists with layer attribution placeholders). +- Outputs must be deterministic: ordering is lexicographic, timestamps removed or normalized, hashes (SHA256) calculated when required. + +## In/Out of Scope +In scope: +- Linux apk/dpkg/rpm analyzers, shared helpers, plug-in manifests, deterministic regression harness. + +Out of scope: +- Windows MSI/SxS analyzers, native (ELF) analyzers, language analyzers, EntryTrace pipeline, or SBOM assembly logic (handled by other guilds). + +## Observability & Security Expectations +- Emit structured logs with correlation/job identifiers provided by `StellaOps.Scanner.Core`. +- Surface metrics for package counts, elapsed time, and cache hits (metrics hooks stubbed until Cache module lands). +- Do not perform outbound network calls; operate entirely on provided filesystem snapshot. +- Validate plug-in manifests via `IPluginCatalogGuard` to enforce restart-only loading. + +## Tests +- `StellaOps.Scanner.Analyzers.OS.Tests` hosts regression tests with canned rootfs fixtures to verify determinism. +- Fixtures store expected analyzer outputs under `Fixtures/` with golden JSON (normalized, sorted). +- Tests cover apk/dpkg/rpm analyzers, shared helper edge cases, and plug-in catalog enforcement. diff --git a/src/StellaOps.Scanner.Analyzers.OS/Abstractions/IOSPackageAnalyzer.cs b/src/StellaOps.Scanner.Analyzers.OS/Abstractions/IOSPackageAnalyzer.cs new file mode 100644 index 00000000..57c34a94 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Abstractions/IOSPackageAnalyzer.cs @@ -0,0 +1,24 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Analyzers.OS.Abstractions; + +/// +/// Represents a deterministic analyzer capable of extracting operating-system package +/// evidence from a container root filesystem snapshot. +/// +public interface IOSPackageAnalyzer +{ + /// + /// Gets the identifier used for logging and manifest composition (e.g. apk, dpkg). + /// + string AnalyzerId { get; } + + /// + /// Executes the analyzer against the provided context, producing a deterministic set of packages. + /// + /// Analysis context surfaced by the worker. + /// Cancellation token propagated from the orchestration pipeline. + /// A result describing discovered packages, metadata, and telemetry. + ValueTask AnalyzeAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Analyzers/OsPackageAnalyzerBase.cs b/src/StellaOps.Scanner.Analyzers.OS/Analyzers/OsPackageAnalyzerBase.cs new file mode 100644 index 00000000..423fdf4f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Analyzers/OsPackageAnalyzerBase.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Analyzers.OS.Abstractions; + +namespace StellaOps.Scanner.Analyzers.OS.Analyzers; + +public abstract class OsPackageAnalyzerBase : IOSPackageAnalyzer +{ + protected OsPackageAnalyzerBase(ILogger logger) + { + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public abstract string AnalyzerId { get; } + + protected ILogger Logger { get; } + + public async ValueTask AnalyzeAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var stopwatch = Stopwatch.StartNew(); + var packages = await ExecuteCoreAsync(context, cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + + var packageCount = packages.Count; + var fileEvidenceCount = 0; + foreach (var package in packages) + { + fileEvidenceCount += package.Files.Count; + } + + var telemetry = new OSAnalyzerTelemetry(stopwatch.Elapsed, packageCount, fileEvidenceCount); + return new OSPackageAnalyzerResult(AnalyzerId, packages, telemetry); + } + + protected abstract ValueTask> ExecuteCoreAsync(OSPackageAnalyzerContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Helpers/CveHintExtractor.cs b/src/StellaOps.Scanner.Analyzers.OS/Helpers/CveHintExtractor.cs new file mode 100644 index 00000000..5315f557 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Helpers/CveHintExtractor.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Text.RegularExpressions; + +namespace StellaOps.Scanner.Analyzers.OS.Helpers; + +public static class CveHintExtractor +{ + private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,7}", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + public static IReadOnlyList Extract(params string?[] inputs) + { + if (inputs is { Length: > 0 }) + { + var set = new SortedSet(StringComparer.OrdinalIgnoreCase); + foreach (var input in inputs) + { + if (string.IsNullOrWhiteSpace(input)) + { + continue; + } + + foreach (Match match in CveRegex.Matches(input)) + { + set.Add(match.Value.ToUpperInvariant()); + } + } + + if (set.Count > 0) + { + return new ReadOnlyCollection(set.ToArray()); + } + } + + return Array.Empty(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageUrlBuilder.cs b/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageUrlBuilder.cs new file mode 100644 index 00000000..4c4b90a2 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageUrlBuilder.cs @@ -0,0 +1,56 @@ +using System; +using System.Text; + +namespace StellaOps.Scanner.Analyzers.OS.Helpers; + +public static class PackageUrlBuilder +{ + public static string BuildAlpine(string name, string version, string architecture) + => $"pkg:alpine/{Escape(name)}@{Escape(version)}?arch={EscapeQuery(architecture)}"; + + public static string BuildDebian(string distribution, string name, string version, string architecture) + { + var distro = string.IsNullOrWhiteSpace(distribution) ? "debian" : distribution.Trim().ToLowerInvariant(); + return $"pkg:deb/{Escape(distro)}/{Escape(name)}@{Escape(version)}?arch={EscapeQuery(architecture)}"; + } + + public static string BuildRpm(string name, string? epoch, string version, string? release, string architecture) + { + var versionComponent = string.IsNullOrWhiteSpace(epoch) + ? Escape(version) + : $"{Escape(epoch)}:{Escape(version)}"; + + var releaseComponent = string.IsNullOrWhiteSpace(release) + ? string.Empty + : $"-{Escape(release!)}"; + + return $"pkg:rpm/{Escape(name)}@{versionComponent}{releaseComponent}?arch={EscapeQuery(architecture)}"; + } + + private static string Escape(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + return Uri.EscapeDataString(value.Trim()); + } + + private static string EscapeQuery(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + var trimmed = value.Trim(); + var builder = new StringBuilder(trimmed.Length); + foreach (var ch in trimmed) + { + if ((ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '-' || ch == '_' || ch == '.' || ch == '~') + { + builder.Append(ch); + } + else + { + builder.Append('%'); + builder.Append(((int)ch).ToString("X2")); + } + } + + return builder.ToString(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageVersionParser.cs b/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageVersionParser.cs new file mode 100644 index 00000000..a05489a1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Helpers/PackageVersionParser.cs @@ -0,0 +1,57 @@ +using System; +using System.Text.RegularExpressions; + +namespace StellaOps.Scanner.Analyzers.OS.Helpers; + +public static class PackageVersionParser +{ + private static readonly Regex DebianVersionRegex = new(@"^(?\d+):(?.+)$", RegexOptions.Compiled); + private static readonly Regex DebianRevisionRegex = new(@"^(?.+?)(?-[^-]+)?$", RegexOptions.Compiled); + private static readonly Regex ApkVersionRegex = new(@"^(?.+?)(?:-(?r\d+))?$", RegexOptions.Compiled); + + public static DebianVersionParts ParseDebianVersion(string version) + { + ArgumentException.ThrowIfNullOrWhiteSpace(version); + + var trimmed = version.Trim(); + string? epoch = null; + string baseVersion = trimmed; + + var epochMatch = DebianVersionRegex.Match(trimmed); + if (epochMatch.Success) + { + epoch = epochMatch.Groups["epoch"].Value; + baseVersion = epochMatch.Groups["version"].Value; + } + + string? revision = null; + var revisionMatch = DebianRevisionRegex.Match(baseVersion); + if (revisionMatch.Success && revisionMatch.Groups["revision"].Success) + { + revision = revisionMatch.Groups["revision"].Value.TrimStart('-'); + baseVersion = revisionMatch.Groups["base"].Value; + } + + return new DebianVersionParts(epoch, baseVersion, revision, trimmed); + } + + public static ApkVersionParts ParseApkVersion(string version) + { + ArgumentException.ThrowIfNullOrWhiteSpace(version); + var match = ApkVersionRegex.Match(version.Trim()); + if (!match.Success) + { + return new ApkVersionParts(null, version.Trim()); + } + + var release = match.Groups["release"].Success ? match.Groups["release"].Value : null; + return new ApkVersionParts(release, match.Groups["version"].Value); + } +} + +public sealed record DebianVersionParts(string? Epoch, string UpstreamVersion, string? Revision, string Original) +{ + public string ForPackageUrl => Epoch is null ? Original : $"{Epoch}:{UpstreamVersion}{(Revision is null ? string.Empty : "-" + Revision)}"; +} + +public sealed record ApkVersionParts(string? Release, string BaseVersion); diff --git a/src/StellaOps.Scanner.Analyzers.OS/Mapping/OsComponentMapper.cs b/src/StellaOps.Scanner.Analyzers.OS/Mapping/OsComponentMapper.cs new file mode 100644 index 00000000..f3154f5f --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Mapping/OsComponentMapper.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Analyzers.OS.Mapping; + +public static class OsComponentMapper +{ + private const string ComponentType = "os-package"; + + public static ImmutableArray ToLayerFragments(IEnumerable results) + { + ArgumentNullException.ThrowIfNull(results); + + var builder = ImmutableArray.CreateBuilder(); + foreach (var result in results) + { + if (result is null || string.IsNullOrWhiteSpace(result.AnalyzerId)) + { + continue; + } + + var layerDigest = ComputeLayerDigest(result.AnalyzerId); + var components = BuildComponentRecords(result.AnalyzerId, layerDigest, result.Packages); + if (components.IsEmpty) + { + continue; + } + + builder.Add(LayerComponentFragment.Create(layerDigest, components)); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray BuildComponentRecords( + string analyzerId, + string layerDigest, + IEnumerable packages) + { + var records = ImmutableArray.CreateBuilder(); + foreach (var package in packages ?? Enumerable.Empty()) + { + records.Add(ToComponentRecord(analyzerId, layerDigest, package)); + } + + return records.ToImmutable(); + } + + private static ComponentRecord ToComponentRecord(string analyzerId, string layerDigest, OSPackageRecord package) + { + var identity = ComponentIdentity.Create( + key: package.PackageUrl, + name: package.Name, + version: package.Version, + purl: package.PackageUrl, + componentType: ComponentType, + group: package.SourcePackage); + + var evidence = package.Files.Select(file => + new ComponentEvidence + { + Kind = file.IsConfigFile is true ? "config-file" : "file", + Value = file.Path, + Source = ResolvePrimaryDigest(file), + }).ToImmutableArray(); + + var dependencies = package.Depends.Count == 0 + ? ImmutableArray.Empty + : ImmutableArray.CreateRange(package.Depends); + + var metadata = BuildMetadata(analyzerId, package); + + return new ComponentRecord + { + Identity = identity, + LayerDigest = layerDigest, + Evidence = evidence, + Dependencies = dependencies, + Metadata = metadata, + Usage = ComponentUsage.Unused, + }; + } + + private static ComponentMetadata? BuildMetadata(string analyzerId, OSPackageRecord package) + { + var properties = new SortedDictionary(StringComparer.Ordinal) + { + ["stellaops.os.analyzer"] = analyzerId, + ["stellaops.os.architecture"] = package.Architecture, + ["stellaops.os.evidenceSource"] = package.EvidenceSource.ToString(), + }; + + if (!string.IsNullOrWhiteSpace(package.SourcePackage)) + { + properties["stellaops.os.sourcePackage"] = package.SourcePackage!; + } + + if (package.CveHints.Count > 0) + { + properties["stellaops.os.cveHints"] = string.Join(",", package.CveHints); + } + + if (package.Provides.Count > 0) + { + properties["stellaops.os.provides"] = string.Join(",", package.Provides); + } + + foreach (var pair in package.VendorMetadata) + { + if (string.IsNullOrWhiteSpace(pair.Key) || string.IsNullOrWhiteSpace(pair.Value)) + { + continue; + } + + properties[$"vendor.{pair.Key}"] = pair.Value!.Trim(); + } + + foreach (var file in package.Files) + { + foreach (var digest in file.Digests) + { + if (string.IsNullOrWhiteSpace(digest.Value)) + { + continue; + } + + properties[$"digest.{digest.Key}.{NormalizePathKey(file.Path)}"] = digest.Value.Trim(); + } + } + + IReadOnlyList? licenses = null; + if (!string.IsNullOrWhiteSpace(package.License)) + { + licenses = new[] { package.License!.Trim() }; + } + + return new ComponentMetadata + { + Licenses = licenses, + Properties = properties.Count == 0 ? null : properties, + }; + } + + private static string NormalizePathKey(string path) + => path.Replace('/', '_').Replace('\\', '_').Trim('_'); + + private static string? ResolvePrimaryDigest(OSPackageFileEvidence file) + { + if (!string.IsNullOrWhiteSpace(file.Sha256)) + { + return file.Sha256; + } + + if (file.Digests.TryGetValue("sha256", out var sha256) && !string.IsNullOrWhiteSpace(sha256)) + { + return sha256; + } + + return null; + } + + private static string ComputeLayerDigest(string analyzerId) + { + var normalized = $"stellaops:os:{analyzerId.Trim().ToLowerInvariant()}"; + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(normalized)); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/AnalyzerWarning.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/AnalyzerWarning.cs new file mode 100644 index 00000000..c6fd46eb --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/AnalyzerWarning.cs @@ -0,0 +1,13 @@ +using System; + +namespace StellaOps.Scanner.Analyzers.OS; + +public sealed record AnalyzerWarning(string Code, string Message) +{ + public static AnalyzerWarning From(string code, string message) + { + ArgumentException.ThrowIfNullOrWhiteSpace(code); + ArgumentException.ThrowIfNullOrWhiteSpace(message); + return new AnalyzerWarning(code.Trim(), message.Trim()); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/OSAnalyzerTelemetry.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/OSAnalyzerTelemetry.cs new file mode 100644 index 00000000..fd14cf87 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/OSAnalyzerTelemetry.cs @@ -0,0 +1,5 @@ +using System; + +namespace StellaOps.Scanner.Analyzers.OS; + +public sealed record OSAnalyzerTelemetry(TimeSpan Duration, int PackageCount, int FileEvidenceCount); diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerContext.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerContext.cs new file mode 100644 index 00000000..f87e9b71 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerContext.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Analyzers.OS; + +/// +/// Carries the immutable context shared across analyzer executions for a given scan job. +/// +public sealed class OSPackageAnalyzerContext +{ + private static readonly IReadOnlyDictionary EmptyMetadata = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)); + + public OSPackageAnalyzerContext( + string rootPath, + string? workspacePath, + TimeProvider timeProvider, + ILogger logger, + IReadOnlyDictionary? metadata = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(rootPath); + RootPath = Path.GetFullPath(rootPath); + WorkspacePath = string.IsNullOrWhiteSpace(workspacePath) ? null : Path.GetFullPath(workspacePath!); + TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + Logger = logger ?? throw new ArgumentNullException(nameof(logger)); + Metadata = metadata is null or { Count: 0 } + ? EmptyMetadata + : new ReadOnlyDictionary(new Dictionary(metadata, StringComparer.Ordinal)); + } + + /// + /// Gets the absolute path to the reconstructed root filesystem of the scanned image/layer set. + /// + public string RootPath { get; } + + /// + /// Gets the absolute path to a writable workspace root the analyzer may use for transient state (optional). + /// The sandbox guarantees cleanup post-run. + /// + public string? WorkspacePath { get; } + + /// + /// Gets the time provider aligned with the scanner's deterministic clock. + /// + public TimeProvider TimeProvider { get; } + + /// + /// Gets the structured logger scoped to the analyzer execution. + /// + public ILogger Logger { get; } + + /// + /// Gets metadata forwarded by prior pipeline stages (image digest, layer digests, tenant, etc.). + /// + public IReadOnlyDictionary Metadata { get; } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerResult.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerResult.cs new file mode 100644 index 00000000..9ea49845 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageAnalyzerResult.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; + +namespace StellaOps.Scanner.Analyzers.OS; + +public sealed class OSPackageAnalyzerResult +{ + private static readonly IReadOnlyList EmptyPackages = + new ReadOnlyCollection(Array.Empty()); + + private static readonly IReadOnlyList EmptyWarnings = + new ReadOnlyCollection(Array.Empty()); + + public OSPackageAnalyzerResult( + string analyzerId, + IEnumerable? packages, + OSAnalyzerTelemetry telemetry, + IEnumerable? warnings = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + AnalyzerId = analyzerId.Trim(); + Packages = packages is null + ? EmptyPackages + : new ReadOnlyCollection(packages.ToArray()); + Telemetry = telemetry; + Warnings = warnings is null + ? EmptyWarnings + : new ReadOnlyCollection(warnings.ToArray()); + } + + public string AnalyzerId { get; } + + public IReadOnlyList Packages { get; } + + public OSAnalyzerTelemetry Telemetry { get; } + + public IReadOnlyList Warnings { get; } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageFileEvidence.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageFileEvidence.cs new file mode 100644 index 00000000..1821f2f3 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageFileEvidence.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Globalization; + +namespace StellaOps.Scanner.Analyzers.OS; + +public sealed class OSPackageFileEvidence : IComparable +{ + public OSPackageFileEvidence( + string path, + string? layerDigest = null, + string? sha256 = null, + long? sizeBytes = null, + bool? isConfigFile = null, + IDictionary? digests = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + Path = Normalize(path); + LayerDigest = NormalizeDigest(layerDigest); + var digestMap = digests is null + ? new SortedDictionary(StringComparer.OrdinalIgnoreCase) + : new SortedDictionary(digests, StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(sha256)) + { + digestMap["sha256"] = NormalizeHash(sha256)!; + } + + Digests = new ReadOnlyDictionary(digestMap); + Sha256 = Digests.TryGetValue("sha256", out var normalizedSha256) ? normalizedSha256 : null; + SizeBytes = sizeBytes; + IsConfigFile = isConfigFile; + } + + public string Path { get; } + + public string? LayerDigest { get; } + + public string? Sha256 { get; } + + public IReadOnlyDictionary Digests { get; } + + public long? SizeBytes { get; } + + public bool? IsConfigFile { get; } + + public int CompareTo(OSPackageFileEvidence? other) + { + if (other is null) + { + return 1; + } + + return string.CompareOrdinal(Path, other.Path); + } + + public override string ToString() + => $"{Path} ({SizeBytes?.ToString("N0", CultureInfo.InvariantCulture) ?? "?"} bytes)"; + + private static string Normalize(string path) + { + var trimmed = path.Trim(); + if (!trimmed.StartsWith('/')) + { + trimmed = "/" + trimmed; + } + + return trimmed.Replace('\\', '/'); + } + + private static string? NormalizeDigest(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var trimmed = digest.Trim(); + if (!trimmed.Contains(':', StringComparison.Ordinal)) + { + return trimmed.ToLowerInvariant(); + } + + var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return parts.Length == 2 + ? $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}" + : trimmed.ToLowerInvariant(); + } + + private static string? NormalizeHash(string? hash) + { + if (string.IsNullOrWhiteSpace(hash)) + { + return null; + } + + return hash.Trim().ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageRecord.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageRecord.cs new file mode 100644 index 00000000..1a87bd85 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/OSPackageRecord.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; + +namespace StellaOps.Scanner.Analyzers.OS; + +public sealed class OSPackageRecord : IComparable +{ + private static readonly IReadOnlyList EmptyList = + new ReadOnlyCollection(Array.Empty()); + + private static readonly IReadOnlyList EmptyFiles = + new ReadOnlyCollection(Array.Empty()); + + private static readonly IReadOnlyDictionary EmptyMetadata = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)); + + public OSPackageRecord( + string analyzerId, + string packageUrl, + string name, + string version, + string architecture, + PackageEvidenceSource evidenceSource, + string? epoch = null, + string? release = null, + string? sourcePackage = null, + string? license = null, + IEnumerable? cveHints = null, + IEnumerable? provides = null, + IEnumerable? depends = null, + IEnumerable? files = null, + IDictionary? vendorMetadata = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(analyzerId); + ArgumentException.ThrowIfNullOrWhiteSpace(packageUrl); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + ArgumentException.ThrowIfNullOrWhiteSpace(version); + ArgumentException.ThrowIfNullOrWhiteSpace(architecture); + + AnalyzerId = analyzerId.Trim(); + PackageUrl = packageUrl.Trim(); + Name = name.Trim(); + Version = version.Trim(); + Architecture = architecture.Trim(); + EvidenceSource = evidenceSource; + Epoch = string.IsNullOrWhiteSpace(epoch) ? null : epoch.Trim(); + Release = string.IsNullOrWhiteSpace(release) ? null : release.Trim(); + SourcePackage = string.IsNullOrWhiteSpace(sourcePackage) ? null : sourcePackage.Trim(); + License = string.IsNullOrWhiteSpace(license) ? null : license.Trim(); + CveHints = AsReadOnlyList(cveHints); + Provides = AsReadOnlyList(provides); + Depends = AsReadOnlyList(depends); + Files = files is null + ? EmptyFiles + : new ReadOnlyCollection(files.OrderBy(f => f).ToArray()); + VendorMetadata = vendorMetadata is null or { Count: 0 } + ? EmptyMetadata + : new ReadOnlyDictionary( + new SortedDictionary(vendorMetadata, StringComparer.Ordinal)); + } + + public string AnalyzerId { get; } + + public string PackageUrl { get; } + + public string Name { get; } + + public string Version { get; } + + public string Architecture { get; } + + public string? Epoch { get; } + + public string? Release { get; } + + public string? SourcePackage { get; } + + public string? License { get; } + + public IReadOnlyList CveHints { get; } + + public IReadOnlyList Provides { get; } + + public IReadOnlyList Depends { get; } + + public IReadOnlyList Files { get; } + + public IReadOnlyDictionary VendorMetadata { get; } + + public PackageEvidenceSource EvidenceSource { get; } + + public int CompareTo(OSPackageRecord? other) + { + if (other is null) + { + return 1; + } + + var cmp = string.CompareOrdinal(PackageUrl, other.PackageUrl); + if (cmp != 0) + { + return cmp; + } + + cmp = string.CompareOrdinal(Name, other.Name); + if (cmp != 0) + { + return cmp; + } + + cmp = string.CompareOrdinal(Version, other.Version); + if (cmp != 0) + { + return cmp; + } + + return string.CompareOrdinal(Architecture, other.Architecture); + } + + private static IReadOnlyList AsReadOnlyList(IEnumerable? values) + { + if (values is null) + { + return EmptyList; + } + + var buffer = values + .Where(static value => !string.IsNullOrWhiteSpace(value)) + .Select(static value => value.Trim()) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToArray(); + + return buffer.Length == 0 ? EmptyList : new ReadOnlyCollection(buffer); + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Model/PackageEvidenceSource.cs b/src/StellaOps.Scanner.Analyzers.OS/Model/PackageEvidenceSource.cs new file mode 100644 index 00000000..f971699c --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Model/PackageEvidenceSource.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Scanner.Analyzers.OS; + +public enum PackageEvidenceSource +{ + Unknown = 0, + ApkDatabase, + DpkgStatus, + RpmDatabase, +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Plugin/IOSAnalyzerPlugin.cs b/src/StellaOps.Scanner.Analyzers.OS/Plugin/IOSAnalyzerPlugin.cs new file mode 100644 index 00000000..4ce2e5ee --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Plugin/IOSAnalyzerPlugin.cs @@ -0,0 +1,16 @@ +using System; +using StellaOps.Plugin; +using StellaOps.Scanner.Analyzers.OS.Abstractions; + +namespace StellaOps.Scanner.Analyzers.OS.Plugin; + +/// +/// Represents a restart-time plug-in that publishes a single . +/// +public interface IOSAnalyzerPlugin : IAvailabilityPlugin +{ + /// + /// Creates the analyzer instance bound to the host service provider. + /// + IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services); +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Plugin/OsAnalyzerPluginCatalog.cs b/src/StellaOps.Scanner.Analyzers.OS/Plugin/OsAnalyzerPluginCatalog.cs new file mode 100644 index 00000000..3c19419e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Plugin/OsAnalyzerPluginCatalog.cs @@ -0,0 +1,138 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Reflection; +using Microsoft.Extensions.Logging; +using StellaOps.Plugin; +using StellaOps.Plugin.Hosting; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Core.Security; + +namespace StellaOps.Scanner.Analyzers.OS.Plugin; + +public sealed class OsAnalyzerPluginCatalog +{ + private readonly ILogger _logger; + private readonly IPluginCatalogGuard _guard; + private readonly ConcurrentDictionary _assemblies = new(StringComparer.OrdinalIgnoreCase); + private IReadOnlyList _plugins = Array.Empty(); + + public OsAnalyzerPluginCatalog(IPluginCatalogGuard guard, ILogger logger) + { + _guard = guard ?? throw new ArgumentNullException(nameof(guard)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public IReadOnlyCollection Plugins => _plugins; + + public void LoadFromDirectory(string directory, bool seal = true) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + var fullDirectory = Path.GetFullPath(directory); + + var options = new PluginHostOptions + { + PluginsDirectory = fullDirectory, + EnsureDirectoryExists = false, + RecursiveSearch = false, + }; + options.SearchPatterns.Add("StellaOps.Scanner.Analyzers.*.dll"); + + var result = PluginHost.LoadPlugins(options, _logger); + if (result.Plugins.Count == 0) + { + _logger.LogWarning("No OS analyzer plug-ins discovered under '{Directory}'.", fullDirectory); + } + + foreach (var descriptor in result.Plugins) + { + try + { + _guard.EnsureRegistrationAllowed(descriptor.AssemblyPath); + _assemblies[descriptor.AssemblyPath] = descriptor.Assembly; + _logger.LogInformation("Registered OS analyzer plug-in assembly '{Assembly}' from '{Path}'.", + descriptor.Assembly.FullName, + descriptor.AssemblyPath); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to register analyzer plug-in '{Path}'.", descriptor.AssemblyPath); + } + } + + RefreshPluginList(); + + if (seal) + { + _guard.Seal(); + } + } + + public IReadOnlyList CreateAnalyzers(IServiceProvider services) + { + ArgumentNullException.ThrowIfNull(services); + + if (_plugins.Count == 0) + { + _logger.LogWarning("No OS analyzer plug-ins available; scanning will skip OS package extraction."); + return Array.Empty(); + } + + var analyzers = new List(_plugins.Count); + foreach (var plugin in _plugins) + { + if (!IsPluginAvailable(plugin, services)) + { + continue; + } + + try + { + var analyzer = plugin.CreateAnalyzer(services); + if (analyzer is null) + { + continue; + } + + analyzers.Add(analyzer); + } + catch (Exception ex) + { + _logger.LogError(ex, "Analyzer plug-in '{Plugin}' failed to create analyzer instance.", plugin.Name); + } + } + + if (analyzers.Count == 0) + { + _logger.LogWarning("All OS analyzer plug-ins were unavailable."); + return Array.Empty(); + } + + analyzers.Sort(static (a, b) => string.CompareOrdinal(a.AnalyzerId, b.AnalyzerId)); + return new ReadOnlyCollection(analyzers); + } + + private void RefreshPluginList() + { + var assemblies = _assemblies.Values.ToArray(); + var plugins = PluginLoader.LoadPlugins(assemblies); + _plugins = plugins is IReadOnlyList list + ? list + : new ReadOnlyCollection(plugins.ToArray()); + } + + private static bool IsPluginAvailable(IOSAnalyzerPlugin plugin, IServiceProvider services) + { + try + { + return plugin.IsAvailable(services); + } + catch + { + return false; + } + } +} diff --git a/src/StellaOps.Scanner.Analyzers.OS/Properties/AssemblyInfo.cs b/src/StellaOps.Scanner.Analyzers.OS/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..308f76f1 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Tests")] diff --git a/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj b/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj new file mode 100644 index 00000000..8cc43a0e --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + diff --git a/src/StellaOps.Scanner.Analyzers.OS/TASKS.md b/src/StellaOps.Scanner.Analyzers.OS/TASKS.md new file mode 100644 index 00000000..6d5532c7 --- /dev/null +++ b/src/StellaOps.Scanner.Analyzers.OS/TASKS.md @@ -0,0 +1,11 @@ +# OS Analyzer Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ANALYZERS-OS-10-201 | DONE (2025-10-19) | OS Analyzer Guild | Scanner Core contracts | Alpine/apk analyzer emitting deterministic package components with provenance evidence. | Analyzer reads `/lib/apk/db/installed`, emits deterministic `pkg:alpine` components with provenance, license, and file evidence; snapshot tests cover fixture. | +| SCANNER-ANALYZERS-OS-10-202 | DONE (2025-10-19) | OS Analyzer Guild | Shared helpers (204) | Debian/dpkg analyzer mapping packages to canonical `pkg:deb` identities with evidence and normalized metadata. | Analyzer parses `status` + `info/*.list`/`md5sums`, outputs normalized packages with config flags and provenance evidence. | +| SCANNER-ANALYZERS-OS-10-203 | DONE (2025-10-19) | OS Analyzer Guild | Shared helpers (204) | RPM analyzer capturing EVR/NEVRA, declared file lists, provenance metadata. | SQLite rpmdb reader parses headers, reconstructs NEVRA, provides/requires, file evidence, and vendor metadata for fixtures. | +| SCANNER-ANALYZERS-OS-10-204 | DONE (2025-10-19) | OS Analyzer Guild | — | Build shared OS evidence helpers for package identity normalization, file attribution, and metadata enrichment used by analyzers. | Shared helpers deliver analyzer base context, PURL builders, CVE hint extraction, and file evidence model reused across plugins. | +| SCANNER-ANALYZERS-OS-10-205 | DONE (2025-10-19) | OS Analyzer Guild | Shared helpers (204) | Vendor metadata enrichment (source packages, declared licenses, CVE hints). | Apk/dpkg/rpm analyzers populate source, license, maintainer, URLs, and CVE hints; metadata stored deterministically. | +| SCANNER-ANALYZERS-OS-10-206 | DONE (2025-10-19) | QA + OS Analyzer Guild | 201–205 | Determinism harness + fixtures for OS analyzers (warm/cold runs). | xUnit snapshot harness with fixtures + goldens ensures byte-stable JSON; helper normalizes newlines and supports env-based regen. | +| SCANNER-ANALYZERS-OS-10-207 | DONE (2025-10-19) | OS Analyzer Guild + DevOps | 201–206 | Package OS analyzers as restart-time plug-ins (manifest + host registration). | Build targets copy analyzer DLLs/manifests to `plugins/scanner/analyzers/os/`; Worker dispatcher loads via restart-only plugin guard. | diff --git a/src/StellaOps.Scanner.Cache.Tests/LayerCacheRoundTripTests.cs b/src/StellaOps.Scanner.Cache.Tests/LayerCacheRoundTripTests.cs new file mode 100644 index 00000000..6d5b15b8 --- /dev/null +++ b/src/StellaOps.Scanner.Cache.Tests/LayerCacheRoundTripTests.cs @@ -0,0 +1,140 @@ +using System.Text; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Cache; +using StellaOps.Scanner.Cache.Abstractions; +using StellaOps.Scanner.Cache.FileCas; +using StellaOps.Scanner.Cache.LayerCache; +using Xunit; + +namespace StellaOps.Scanner.Cache.Tests; + +public sealed class LayerCacheRoundTripTests : IAsyncLifetime +{ + private readonly string _rootPath; + private readonly FakeTimeProvider _timeProvider; + private readonly IOptions _options; + private readonly LayerCacheStore _layerCache; + private readonly FileContentAddressableStore _fileCas; + + public LayerCacheRoundTripTests() + { + _rootPath = Path.Combine(Path.GetTempPath(), "stellaops-cache-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_rootPath); + + _timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero)); + + var optionsValue = new ScannerCacheOptions + { + RootPath = _rootPath, + LayerTtl = TimeSpan.FromHours(1), + FileTtl = TimeSpan.FromHours(2), + MaxBytes = 512 * 1024, // 512 KiB + WarmBytesThreshold = 256 * 1024, + ColdBytesThreshold = 400 * 1024, + MaintenanceInterval = TimeSpan.FromMinutes(5) + }; + + _options = Options.Create(optionsValue); + _layerCache = new LayerCacheStore(_options, NullLogger.Instance, _timeProvider); + _fileCas = new FileContentAddressableStore(_options, NullLogger.Instance, _timeProvider); + } + + [Fact] + public async Task RoundTrip_Succeeds_And_Respects_Ttl_And_ImportExport() + { + var layerDigest = "sha256:abcd1234"; + var metadata = new Dictionary + { + ["image"] = "ghcr.io/stella/sample:1", + ["schema"] = "1.0" + }; + + using var inventoryStream = CreateStream("inventory" + Environment.NewLine + "component:libfoo" + Environment.NewLine); + using var usageStream = CreateStream("usage" + Environment.NewLine + "component:bin" + Environment.NewLine); + + var request = new LayerCachePutRequest( + layerDigest, + architecture: "linux/amd64", + mediaType: "application/vnd.oci.image.layer.v1.tar", + metadata, + new List + { + new("inventory.cdx.json", inventoryStream, "application/json"), + new("usage.cdx.json", usageStream, "application/json") + }); + + var stored = await _layerCache.PutAsync(request, CancellationToken.None); + stored.LayerDigest.Should().Be(layerDigest); + stored.Artifacts.Should().ContainKey("inventory.cdx.json"); + stored.TotalSizeBytes.Should().BeGreaterThan(0); + + var cached = await _layerCache.TryGetAsync(layerDigest, CancellationToken.None); + cached.Should().NotBeNull(); + cached!.Metadata.Should().ContainKey("image"); + + await using (var artifact = await _layerCache.OpenArtifactAsync(layerDigest, "inventory.cdx.json", CancellationToken.None)) + { + artifact.Should().NotBeNull(); + using var reader = new StreamReader(artifact!, Encoding.UTF8); + var content = await reader.ReadToEndAsync(); + content.Should().Contain("component:libfoo"); + } + + // Store file CAS entry and validate export/import lifecycle. + var casHash = "sha256:" + new string('f', 64); + using var casStream = CreateStream("some-cas-content"); + await _fileCas.PutAsync(new FileCasPutRequest(casHash, casStream), CancellationToken.None); + + var exportPath = Path.Combine(_rootPath, "export"); + var exportCount = await _fileCas.ExportAsync(exportPath, CancellationToken.None); + exportCount.Should().Be(1); + + await _fileCas.RemoveAsync(casHash, CancellationToken.None); + (await _fileCas.TryGetAsync(casHash, CancellationToken.None)).Should().BeNull(); + + var importCount = await _fileCas.ImportAsync(exportPath, CancellationToken.None); + importCount.Should().Be(1); + var imported = await _fileCas.TryGetAsync(casHash, CancellationToken.None); + imported.Should().NotBeNull(); + imported!.RelativePath.Should().EndWith("content.bin"); + + // TTL eviction + _timeProvider.Advance(TimeSpan.FromHours(2)); + await _layerCache.EvictExpiredAsync(CancellationToken.None); + (await _layerCache.TryGetAsync(layerDigest, CancellationToken.None)).Should().BeNull(); + + // Compaction removes CAS entry once over threshold. + // Force compaction by writing a large entry. + using var largeStream = CreateStream(new string('x', 400_000)); + var largeHash = "sha256:" + new string('e', 64); + await _fileCas.PutAsync(new FileCasPutRequest(largeHash, largeStream), CancellationToken.None); + _timeProvider.Advance(TimeSpan.FromMinutes(1)); + await _fileCas.CompactAsync(CancellationToken.None); + (await _fileCas.TryGetAsync(casHash, CancellationToken.None)).Should().BeNull(); + } + + public Task InitializeAsync() => Task.CompletedTask; + + public Task DisposeAsync() + { + try + { + if (Directory.Exists(_rootPath)) + { + Directory.Delete(_rootPath, recursive: true); + } + } + catch + { + // Ignored – best effort cleanup. + } + + return Task.CompletedTask; + } + + private static MemoryStream CreateStream(string content) + => new(Encoding.UTF8.GetBytes(content)); +} diff --git a/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj b/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj new file mode 100644 index 00000000..afe325bb --- /dev/null +++ b/src/StellaOps.Scanner.Cache.Tests/StellaOps.Scanner.Cache.Tests.csproj @@ -0,0 +1,25 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Cache/AGENTS.md b/src/StellaOps.Scanner.Cache/AGENTS.md new file mode 100644 index 00000000..69929606 --- /dev/null +++ b/src/StellaOps.Scanner.Cache/AGENTS.md @@ -0,0 +1,15 @@ +# StellaOps.Scanner.Cache — Agent Charter + +## Mission +Provide deterministic, offline-friendly caching primitives for scanner layers and file content so warm scans complete in <5 s and cache reuse remains reproducible across deployments. + +## Responsibilities +- Implement layer cache keyed by layer digest, retaining analyzer metadata and provenance per architecture §3.3. +- Deliver file content-addressable storage (CAS) with deduplication, TTL enforcement, and offline import/export hooks. +- Expose structured metrics, health probes, and configuration toggles for cache sizing, eviction, and warm/cold thresholds. +- Coordinate invalidation workflows (layer purge, TTL expiry, diff invalidation) while keeping deterministic logs and telemetry. + +## Interfaces & Dependencies +- Relies on `StackExchange.Redis` via `StellaOps.DependencyInjection` bindings for cache state. +- Coordinates with `StellaOps.Scanner.Storage` object store when persisting immutable artifacts. +- Targets `net10.0` preview SDK and follows scanner coding standards from `docs/18_CODING_STANDARDS.md`. diff --git a/src/StellaOps.Scanner.Cache/Abstractions/IFileContentAddressableStore.cs b/src/StellaOps.Scanner.Cache/Abstractions/IFileContentAddressableStore.cs new file mode 100644 index 00000000..fb101edc --- /dev/null +++ b/src/StellaOps.Scanner.Cache/Abstractions/IFileContentAddressableStore.cs @@ -0,0 +1,48 @@ +using System.IO; + +namespace StellaOps.Scanner.Cache.Abstractions; + +public interface IFileContentAddressableStore +{ + ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default); + + Task PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default); + + Task RemoveAsync(string sha256, CancellationToken cancellationToken = default); + + Task EvictExpiredAsync(CancellationToken cancellationToken = default); + + Task ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default); + + Task ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default); + + Task CompactAsync(CancellationToken cancellationToken = default); +} + +public sealed record FileCasEntry( + string Sha256, + long SizeBytes, + DateTimeOffset CreatedAt, + DateTimeOffset LastAccessed, + string RelativePath); + +public sealed class FileCasPutRequest +{ + public string Sha256 { get; } + + public Stream Content { get; } + + public bool LeaveOpen { get; } + + public FileCasPutRequest(string sha256, Stream content, bool leaveOpen = false) + { + if (string.IsNullOrWhiteSpace(sha256)) + { + throw new ArgumentException("SHA-256 identifier must be provided.", nameof(sha256)); + } + + Sha256 = sha256; + Content = content ?? throw new ArgumentNullException(nameof(content)); + LeaveOpen = leaveOpen; + } +} diff --git a/src/StellaOps.Scanner.Cache/Abstractions/ILayerCacheStore.cs b/src/StellaOps.Scanner.Cache/Abstractions/ILayerCacheStore.cs new file mode 100644 index 00000000..07f5165c --- /dev/null +++ b/src/StellaOps.Scanner.Cache/Abstractions/ILayerCacheStore.cs @@ -0,0 +1,18 @@ +using System.IO; + +namespace StellaOps.Scanner.Cache.Abstractions; + +public interface ILayerCacheStore +{ + ValueTask TryGetAsync(string layerDigest, CancellationToken cancellationToken = default); + + Task PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default); + + Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default); + + Task EvictExpiredAsync(CancellationToken cancellationToken = default); + + Task OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default); + + Task CompactAsync(CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.Cache/Abstractions/LayerCacheEntry.cs b/src/StellaOps.Scanner.Cache/Abstractions/LayerCacheEntry.cs new file mode 100644 index 00000000..c21d366f --- /dev/null +++ b/src/StellaOps.Scanner.Cache/Abstractions/LayerCacheEntry.cs @@ -0,0 +1,28 @@ +namespace StellaOps.Scanner.Cache.Abstractions; + +/// +/// Represents cached metadata for a single layer digest. +/// +public sealed record LayerCacheEntry( + string LayerDigest, + string Architecture, + string MediaType, + DateTimeOffset CachedAt, + DateTimeOffset LastAccessed, + long TotalSizeBytes, + IReadOnlyDictionary Artifacts, + IReadOnlyDictionary Metadata) +{ + public bool IsExpired(DateTimeOffset utcNow, TimeSpan ttl) + => utcNow - CachedAt >= ttl; +} + +/// +/// Points to a cached artifact stored on disk. +/// +public sealed record LayerCacheArtifactReference( + string Name, + string RelativePath, + string ContentType, + long SizeBytes, + bool IsImmutable = false); diff --git a/src/StellaOps.Scanner.Cache/Abstractions/LayerCachePutRequest.cs b/src/StellaOps.Scanner.Cache/Abstractions/LayerCachePutRequest.cs new file mode 100644 index 00000000..bb36eb1c --- /dev/null +++ b/src/StellaOps.Scanner.Cache/Abstractions/LayerCachePutRequest.cs @@ -0,0 +1,93 @@ +using System.IO; + +namespace StellaOps.Scanner.Cache.Abstractions; + +/// +/// Describes layer cache content to be stored. +/// +public sealed class LayerCachePutRequest +{ + public string LayerDigest { get; } + + public string Architecture { get; } + + public string MediaType { get; } + + public IReadOnlyDictionary Metadata { get; } + + public IReadOnlyList Artifacts { get; } + + public LayerCachePutRequest( + string layerDigest, + string architecture, + string mediaType, + IReadOnlyDictionary metadata, + IReadOnlyList artifacts) + { + if (string.IsNullOrWhiteSpace(layerDigest)) + { + throw new ArgumentException("Layer digest must be provided.", nameof(layerDigest)); + } + + if (string.IsNullOrWhiteSpace(architecture)) + { + throw new ArgumentException("Architecture must be provided.", nameof(architecture)); + } + + if (string.IsNullOrWhiteSpace(mediaType)) + { + throw new ArgumentException("Media type must be provided.", nameof(mediaType)); + } + + Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + Artifacts = artifacts ?? throw new ArgumentNullException(nameof(artifacts)); + if (artifacts.Count == 0) + { + throw new ArgumentException("At least one artifact must be supplied.", nameof(artifacts)); + } + + LayerDigest = layerDigest; + Architecture = architecture; + MediaType = mediaType; + } +} + +/// +/// Stream payload for a cached artifact. +/// +public sealed class LayerCacheArtifactContent +{ + public string Name { get; } + + public Stream Content { get; } + + public string ContentType { get; } + + public bool Immutable { get; } + + public bool LeaveOpen { get; } + + public LayerCacheArtifactContent( + string name, + Stream content, + string contentType, + bool immutable = false, + bool leaveOpen = false) + { + if (string.IsNullOrWhiteSpace(name)) + { + throw new ArgumentException("Artifact name must be provided.", nameof(name)); + } + + if (string.IsNullOrWhiteSpace(contentType)) + { + throw new ArgumentException("Content type must be provided.", nameof(contentType)); + } + + Name = name; + Content = content ?? throw new ArgumentNullException(nameof(content)); + ContentType = contentType; + Immutable = immutable; + LeaveOpen = leaveOpen; + } +} diff --git a/src/StellaOps.Scanner.Cache/FileCas/FileContentAddressableStore.cs b/src/StellaOps.Scanner.Cache/FileCas/FileContentAddressableStore.cs new file mode 100644 index 00000000..a8b8e013 --- /dev/null +++ b/src/StellaOps.Scanner.Cache/FileCas/FileContentAddressableStore.cs @@ -0,0 +1,481 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Cache.FileCas; + +public sealed class FileContentAddressableStore : IFileContentAddressableStore +{ + private const string MetadataFileName = "meta.json"; + private const string ContentFileName = "content.bin"; + + private readonly ScannerCacheOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly JsonSerializerOptions _jsonOptions; + private readonly SemaphoreSlim _initializationLock = new(1, 1); + private volatile bool _initialised; + + public FileContentAddressableStore( + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + } + + public async ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sha256); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + var entryDirectory = GetEntryDirectory(sha256); + var metadataPath = Path.Combine(entryDirectory, MetadataFileName); + if (!File.Exists(metadataPath)) + { + ScannerCacheMetrics.RecordFileCasMiss(sha256); + return null; + } + + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false); + ScannerCacheMetrics.RecordFileCasMiss(sha256); + return null; + } + + var now = _timeProvider.GetUtcNow(); + if (IsExpired(metadata, now)) + { + ScannerCacheMetrics.RecordFileCasEviction(sha256); + await RemoveDirectoryAsync(entryDirectory).ConfigureAwait(false); + return null; + } + + metadata.LastAccessed = now; + await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordFileCasHit(sha256); + + return new FileCasEntry( + metadata.Sha256, + metadata.SizeBytes, + metadata.CreatedAt, + metadata.LastAccessed, + GetRelativeContentPath(metadata.Sha256)); + } + + public async Task PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + var sha = request.Sha256; + var directory = GetEntryDirectory(sha); + Directory.CreateDirectory(directory); + + var contentPath = Path.Combine(directory, ContentFileName); + await using (var destination = new FileStream(contentPath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous)) + { + await request.Content.CopyToAsync(destination, cancellationToken).ConfigureAwait(false); + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + if (!request.LeaveOpen) + { + request.Content.Dispose(); + } + + var now = _timeProvider.GetUtcNow(); + var sizeBytes = new FileInfo(contentPath).Length; + var metadata = new FileCasMetadata + { + Sha256 = NormalizeHash(sha), + CreatedAt = now, + LastAccessed = now, + SizeBytes = sizeBytes + }; + + var metadataPath = Path.Combine(directory, MetadataFileName); + await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordFileCasBytes(sizeBytes); + + await CompactAsync(cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Stored CAS entry {Sha256} ({SizeBytes} bytes)", sha, sizeBytes); + return new FileCasEntry(metadata.Sha256, metadata.SizeBytes, metadata.CreatedAt, metadata.LastAccessed, GetRelativeContentPath(metadata.Sha256)); + } + + public async Task RemoveAsync(string sha256, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sha256); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + var directory = GetEntryDirectory(sha256); + if (!Directory.Exists(directory)) + { + return false; + } + + await RemoveDirectoryAsync(directory).ConfigureAwait(false); + ScannerCacheMetrics.RecordFileCasEviction(sha256); + return true; + } + + public async Task EvictExpiredAsync(CancellationToken cancellationToken = default) + { + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + if (_options.FileTtl <= TimeSpan.Zero) + { + return 0; + } + + var now = _timeProvider.GetUtcNow(); + var evicted = 0; + + foreach (var metadataPath in EnumerateMetadataFiles()) + { + cancellationToken.ThrowIfCancellationRequested(); + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + continue; + } + + if (IsExpired(metadata, now)) + { + var directory = Path.GetDirectoryName(metadataPath)!; + await RemoveDirectoryAsync(directory).ConfigureAwait(false); + ScannerCacheMetrics.RecordFileCasEviction(metadata.Sha256); + evicted++; + } + } + + if (evicted > 0) + { + _logger.LogInformation("Evicted {Count} CAS entries due to TTL", evicted); + } + + return evicted; + } + + public async Task ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(destinationDirectory); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + Directory.CreateDirectory(destinationDirectory); + var exported = 0; + + foreach (var entryDirectory in EnumerateEntryDirectories()) + { + cancellationToken.ThrowIfCancellationRequested(); + var hash = Path.GetFileName(entryDirectory); + if (hash is null) + { + continue; + } + + var target = Path.Combine(destinationDirectory, hash); + if (Directory.Exists(target)) + { + continue; + } + + CopyDirectory(entryDirectory, target); + exported++; + } + + _logger.LogInformation("Exported {Count} CAS entries to {Destination}", exported, destinationDirectory); + return exported; + } + + public async Task ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sourceDirectory); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + if (!Directory.Exists(sourceDirectory)) + { + return 0; + } + + var imported = 0; + foreach (var directory in Directory.EnumerateDirectories(sourceDirectory)) + { + cancellationToken.ThrowIfCancellationRequested(); + var metadataPath = Path.Combine(directory, MetadataFileName); + if (!File.Exists(metadataPath)) + { + continue; + } + + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + continue; + } + + var destination = GetEntryDirectory(metadata.Sha256); + if (Directory.Exists(destination)) + { + // Only overwrite if the source is newer. + var existingMetadataPath = Path.Combine(destination, MetadataFileName); + var existing = await ReadMetadataAsync(existingMetadataPath, cancellationToken).ConfigureAwait(false); + if (existing is not null && existing.CreatedAt >= metadata.CreatedAt) + { + continue; + } + + await RemoveDirectoryAsync(destination).ConfigureAwait(false); + } + + CopyDirectory(directory, destination); + imported++; + } + + if (imported > 0) + { + _logger.LogInformation("Imported {Count} CAS entries from {Source}", imported, sourceDirectory); + } + + return imported; + } + + public async Task CompactAsync(CancellationToken cancellationToken = default) + { + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + if (_options.MaxBytes <= 0) + { + return 0; + } + + var entries = new List<(FileCasMetadata Metadata, string Directory)>(); + long totalBytes = 0; + + foreach (var metadataPath in EnumerateMetadataFiles()) + { + cancellationToken.ThrowIfCancellationRequested(); + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + continue; + } + + var directory = Path.GetDirectoryName(metadataPath)!; + entries.Add((metadata, directory)); + totalBytes += metadata.SizeBytes; + } + + if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes)) + { + return 0; + } + + entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed)); + var target = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2; + var removed = 0; + + foreach (var entry in entries) + { + if (totalBytes <= target) + { + break; + } + + await RemoveDirectoryAsync(entry.Directory).ConfigureAwait(false); + totalBytes -= entry.Metadata.SizeBytes; + removed++; + ScannerCacheMetrics.RecordFileCasEviction(entry.Metadata.Sha256); + } + + if (removed > 0) + { + _logger.LogInformation("Compacted CAS store, removed {Count} entries", removed); + } + + return removed; + } + + private async Task EnsureInitialisedAsync(CancellationToken cancellationToken) + { + if (_initialised) + { + return; + } + + await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_initialised) + { + return; + } + + Directory.CreateDirectory(_options.FileCasDirectoryPath); + _initialised = true; + } + finally + { + _initializationLock.Release(); + } + } + + private IEnumerable EnumerateMetadataFiles() + { + if (!Directory.Exists(_options.FileCasDirectoryPath)) + { + yield break; + } + + foreach (var file in Directory.EnumerateFiles(_options.FileCasDirectoryPath, MetadataFileName, SearchOption.AllDirectories)) + { + yield return file; + } + } + + private IEnumerable EnumerateEntryDirectories() + { + if (!Directory.Exists(_options.FileCasDirectoryPath)) + { + yield break; + } + + foreach (var directory in Directory.EnumerateDirectories(_options.FileCasDirectoryPath, "*", SearchOption.AllDirectories)) + { + if (File.Exists(Path.Combine(directory, MetadataFileName))) + { + yield return directory; + } + } + } + + private async Task ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken) + { + try + { + await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); + return await JsonSerializer.DeserializeAsync(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or JsonException) + { + _logger.LogWarning(ex, "Failed to read CAS metadata from {Path}", metadataPath); + return null; + } + } + + private async Task WriteMetadataAsync(string metadataPath, FileCasMetadata metadata, CancellationToken cancellationToken) + { + var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp"); + await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous)) + { + await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + File.Move(tempFile, metadataPath, overwrite: true); + } + + private static string GetRelativeContentPath(string sha256) + { + var normalized = NormalizeHash(sha256); + return Path.Combine(NormalizedPrefix(normalized, 0, 2), NormalizedPrefix(normalized, 2, 2), normalized, ContentFileName); + } + + private string GetEntryDirectory(string sha256) + { + var normalized = NormalizeHash(sha256); + return Path.Combine( + _options.FileCasDirectoryPath, + NormalizedPrefix(normalized, 0, 2), + NormalizedPrefix(normalized, 2, 2), + normalized); + } + + private static string NormalizeHash(string sha256) + { + if (string.IsNullOrWhiteSpace(sha256)) + { + return "unknown"; + } + + var hash = sha256.Contains(':', StringComparison.Ordinal) ? sha256[(sha256.IndexOf(':') + 1)..] : sha256; + return hash.ToLowerInvariant(); + } + + private static string NormalizedPrefix(string hash, int offset, int length) + { + if (hash.Length <= offset) + { + return "00"; + } + + if (hash.Length < offset + length) + { + length = hash.Length - offset; + } + + return hash.Substring(offset, length); + } + + private bool IsExpired(FileCasMetadata metadata, DateTimeOffset now) + { + if (_options.FileTtl <= TimeSpan.Zero) + { + return false; + } + + return now - metadata.CreatedAt >= _options.FileTtl; + } + + private static void CopyDirectory(string sourceDir, string destDir) + { + Directory.CreateDirectory(destDir); + foreach (var file in Directory.EnumerateFiles(sourceDir, "*", SearchOption.AllDirectories)) + { + var relative = Path.GetRelativePath(sourceDir, file); + var destination = Path.Combine(destDir, relative); + var parent = Path.GetDirectoryName(destination); + if (!string.IsNullOrEmpty(parent)) + { + Directory.CreateDirectory(parent); + } + File.Copy(file, destination, overwrite: true); + } + } + + private Task RemoveDirectoryAsync(string directory) + { + if (!Directory.Exists(directory)) + { + return Task.CompletedTask; + } + + try + { + Directory.Delete(directory, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to delete CAS directory {Directory}", directory); + } + + return Task.CompletedTask; + } + + private sealed class FileCasMetadata + { + public string Sha256 { get; set; } = string.Empty; + + public DateTimeOffset CreatedAt { get; set; } + + public DateTimeOffset LastAccessed { get; set; } + + public long SizeBytes { get; set; } + } +} diff --git a/src/StellaOps.Scanner.Cache/FileCas/NullFileContentAddressableStore.cs b/src/StellaOps.Scanner.Cache/FileCas/NullFileContentAddressableStore.cs new file mode 100644 index 00000000..53e59840 --- /dev/null +++ b/src/StellaOps.Scanner.Cache/FileCas/NullFileContentAddressableStore.cs @@ -0,0 +1,27 @@ +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Cache.FileCas; + +internal sealed class NullFileContentAddressableStore : IFileContentAddressableStore +{ + public ValueTask TryGetAsync(string sha256, CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + + public Task PutAsync(FileCasPutRequest request, CancellationToken cancellationToken = default) + => Task.FromException(new InvalidOperationException("File CAS is disabled via configuration.")); + + public Task RemoveAsync(string sha256, CancellationToken cancellationToken = default) + => Task.FromResult(false); + + public Task EvictExpiredAsync(CancellationToken cancellationToken = default) + => Task.FromResult(0); + + public Task ExportAsync(string destinationDirectory, CancellationToken cancellationToken = default) + => Task.FromResult(0); + + public Task ImportAsync(string sourceDirectory, CancellationToken cancellationToken = default) + => Task.FromResult(0); + + public Task CompactAsync(CancellationToken cancellationToken = default) + => Task.FromResult(0); +} diff --git a/src/StellaOps.Scanner.Cache/LayerCache/LayerCacheStore.cs b/src/StellaOps.Scanner.Cache/LayerCache/LayerCacheStore.cs new file mode 100644 index 00000000..6d187f0f --- /dev/null +++ b/src/StellaOps.Scanner.Cache/LayerCache/LayerCacheStore.cs @@ -0,0 +1,480 @@ +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Cache.LayerCache; + +public sealed class LayerCacheStore : ILayerCacheStore +{ + private const string MetadataFileName = "meta.json"; + + private readonly ScannerCacheOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly JsonSerializerOptions _jsonOptions; + private readonly SemaphoreSlim _initializationLock = new(1, 1); + private volatile bool _initialised; + + public LayerCacheStore( + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _jsonOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = false + }; + } + + public async ValueTask TryGetAsync(string layerDigest, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + var directory = GetLayerDirectory(layerDigest); + if (!Directory.Exists(directory)) + { + _logger.LogTrace("Layer cache miss: directory {Directory} not found for {LayerDigest}", directory, layerDigest); + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + var metadataPath = Path.Combine(directory, MetadataFileName); + if (!File.Exists(metadataPath)) + { + _logger.LogDebug("Layer cache metadata missing at {Path} for {LayerDigest}; removing directory", metadataPath, layerDigest); + await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + var now = _timeProvider.GetUtcNow(); + if (IsExpired(metadata, now)) + { + _logger.LogDebug("Layer cache entry {LayerDigest} expired at {Expiration}", metadata.LayerDigest, metadata.CachedAt + _options.LayerTtl); + await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerEviction(layerDigest); + return null; + } + + metadata.LastAccessed = now; + await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerHit(layerDigest); + + return Map(metadata); + } + + public async Task PutAsync(LayerCachePutRequest request, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + var digest = request.LayerDigest; + var directory = GetLayerDirectory(digest); + var metadataPath = Path.Combine(directory, MetadataFileName); + + if (Directory.Exists(directory)) + { + _logger.LogDebug("Replacing existing layer cache entry for {LayerDigest}", digest); + await RemoveInternalAsync(directory, digest, cancellationToken).ConfigureAwait(false); + } + + Directory.CreateDirectory(directory); + + var artifactMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase); + long totalSize = 0; + + foreach (var artifact in request.Artifacts) + { + cancellationToken.ThrowIfCancellationRequested(); + var fileName = SanitizeArtifactName(artifact.Name); + var relativePath = Path.Combine("artifacts", fileName); + var artifactDirectory = Path.Combine(directory, "artifacts"); + Directory.CreateDirectory(artifactDirectory); + var filePath = Path.Combine(directory, relativePath); + + await using (var target = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, 81920, FileOptions.Asynchronous)) + { + await artifact.Content.CopyToAsync(target, cancellationToken).ConfigureAwait(false); + await target.FlushAsync(cancellationToken).ConfigureAwait(false); + totalSize += target.Length; + } + + if (!artifact.LeaveOpen) + { + artifact.Content.Dispose(); + } + + var sizeBytes = new FileInfo(filePath).Length; + + artifactMetadata[artifact.Name] = new LayerCacheArtifactMetadata + { + Name = artifact.Name, + ContentType = artifact.ContentType, + RelativePath = relativePath, + SizeBytes = sizeBytes, + Immutable = artifact.Immutable + }; + } + + var now = _timeProvider.GetUtcNow(); + var metadata = new LayerCacheMetadata + { + LayerDigest = digest, + Architecture = request.Architecture, + MediaType = request.MediaType, + CachedAt = now, + LastAccessed = now, + Metadata = new Dictionary(request.Metadata, StringComparer.Ordinal), + Artifacts = artifactMetadata, + SizeBytes = totalSize + }; + + await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerBytes(totalSize); + + await CompactAsync(cancellationToken).ConfigureAwait(false); + + _logger.LogInformation("Cached layer {LayerDigest} with {ArtifactCount} artifacts ({SizeBytes} bytes)", digest, artifactMetadata.Count, totalSize); + return Map(metadata); + } + + public async Task RemoveAsync(string layerDigest, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + var directory = GetLayerDirectory(layerDigest); + await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false); + } + + public async Task EvictExpiredAsync(CancellationToken cancellationToken = default) + { + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + if (_options.LayerTtl <= TimeSpan.Zero) + { + return 0; + } + + var now = _timeProvider.GetUtcNow(); + var evicted = 0; + + foreach (var metadataPath in EnumerateMetadataFiles()) + { + cancellationToken.ThrowIfCancellationRequested(); + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + continue; + } + + if (IsExpired(metadata, now)) + { + var directory = Path.GetDirectoryName(metadataPath)!; + await RemoveInternalAsync(directory, metadata.LayerDigest, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerEviction(metadata.LayerDigest); + evicted++; + } + } + + if (evicted > 0) + { + _logger.LogInformation("Evicted {Count} expired layer cache entries", evicted); + } + + return evicted; + } + + public async Task OpenArtifactAsync(string layerDigest, string artifactName, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest); + ArgumentException.ThrowIfNullOrWhiteSpace(artifactName); + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + + var directory = GetLayerDirectory(layerDigest); + var metadataPath = Path.Combine(directory, MetadataFileName); + if (!File.Exists(metadataPath)) + { + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + if (!metadata.Artifacts.TryGetValue(artifactName, out var artifact)) + { + _logger.LogDebug("Layer cache artifact {Artifact} missing for {LayerDigest}", artifactName, layerDigest); + return null; + } + + var filePath = Path.Combine(directory, artifact.RelativePath); + if (!File.Exists(filePath)) + { + _logger.LogDebug("Layer cache file {FilePath} not found for artifact {Artifact}", filePath, artifactName); + await RemoveInternalAsync(directory, layerDigest, cancellationToken).ConfigureAwait(false); + ScannerCacheMetrics.RecordLayerMiss(layerDigest); + return null; + } + + metadata.LastAccessed = _timeProvider.GetUtcNow(); + await WriteMetadataAsync(metadataPath, metadata, cancellationToken).ConfigureAwait(false); + return new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, 81920, FileOptions.Asynchronous | FileOptions.SequentialScan); + } + + public async Task CompactAsync(CancellationToken cancellationToken = default) + { + await EnsureInitialisedAsync(cancellationToken).ConfigureAwait(false); + if (_options.MaxBytes <= 0) + { + return 0; + } + + var entries = new List<(LayerCacheMetadata Metadata, string Directory)>(); + long totalBytes = 0; + + foreach (var metadataPath in EnumerateMetadataFiles()) + { + cancellationToken.ThrowIfCancellationRequested(); + var metadata = await ReadMetadataAsync(metadataPath, cancellationToken).ConfigureAwait(false); + if (metadata is null) + { + continue; + } + + var directory = Path.GetDirectoryName(metadataPath)!; + entries.Add((metadata, directory)); + totalBytes += metadata.SizeBytes; + } + + if (totalBytes <= Math.Min(_options.ColdBytesThreshold > 0 ? _options.ColdBytesThreshold : long.MaxValue, _options.MaxBytes)) + { + return 0; + } + + entries.Sort((left, right) => DateTimeOffset.Compare(left.Metadata.LastAccessed, right.Metadata.LastAccessed)); + var targetBytes = _options.WarmBytesThreshold > 0 ? _options.WarmBytesThreshold : _options.MaxBytes / 2; + var removed = 0; + + foreach (var entry in entries) + { + if (totalBytes <= targetBytes) + { + break; + } + + await RemoveInternalAsync(entry.Directory, entry.Metadata.LayerDigest, cancellationToken).ConfigureAwait(false); + totalBytes -= entry.Metadata.SizeBytes; + removed++; + ScannerCacheMetrics.RecordLayerEviction(entry.Metadata.LayerDigest); + _logger.LogInformation("Evicted layer {LayerDigest} during compaction (remaining bytes: {Bytes})", entry.Metadata.LayerDigest, totalBytes); + } + + return removed; + } + + private async Task EnsureInitialisedAsync(CancellationToken cancellationToken) + { + if (_initialised) + { + return; + } + + await _initializationLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_initialised) + { + return; + } + + Directory.CreateDirectory(_options.LayersDirectoryPath); + _initialised = true; + } + finally + { + _initializationLock.Release(); + } + } + + private IEnumerable EnumerateMetadataFiles() + { + if (!Directory.Exists(_options.LayersDirectoryPath)) + { + yield break; + } + + foreach (var file in Directory.EnumerateFiles(_options.LayersDirectoryPath, MetadataFileName, SearchOption.AllDirectories)) + { + yield return file; + } + } + + private async Task ReadMetadataAsync(string metadataPath, CancellationToken cancellationToken) + { + try + { + await using var stream = new FileStream(metadataPath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); + return await JsonSerializer.DeserializeAsync(stream, _jsonOptions, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException or JsonException) + { + _logger.LogWarning(ex, "Failed to load layer cache metadata from {Path}", metadataPath); + return null; + } + } + + private async Task WriteMetadataAsync(string metadataPath, LayerCacheMetadata metadata, CancellationToken cancellationToken) + { + var tempFile = Path.Combine(Path.GetDirectoryName(metadataPath)!, $"{Guid.NewGuid():N}.tmp"); + await using (var stream = new FileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous)) + { + await JsonSerializer.SerializeAsync(stream, metadata, _jsonOptions, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + File.Move(tempFile, metadataPath, overwrite: true); + } + + private Task RemoveInternalAsync(string directory, string layerDigest, CancellationToken cancellationToken) + { + if (!Directory.Exists(directory)) + { + return Task.CompletedTask; + } + + try + { + Directory.Delete(directory, recursive: true); + _logger.LogDebug("Removed layer cache entry {LayerDigest}", layerDigest); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + _logger.LogWarning(ex, "Failed to delete layer cache directory {Directory}", directory); + } + + return Task.CompletedTask; + } + + private bool IsExpired(LayerCacheMetadata metadata, DateTimeOffset now) + { + if (_options.LayerTtl <= TimeSpan.Zero) + { + return false; + } + + if (metadata.CachedAt == default) + { + return false; + } + + return now - metadata.CachedAt >= _options.LayerTtl; + } + + private LayerCacheEntry Map(LayerCacheMetadata metadata) + { + var artifacts = metadata.Artifacts?.ToDictionary( + pair => pair.Key, + pair => new LayerCacheArtifactReference( + pair.Value.Name, + pair.Value.RelativePath, + pair.Value.ContentType, + pair.Value.SizeBytes, + pair.Value.Immutable), + StringComparer.OrdinalIgnoreCase) + ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + + return new LayerCacheEntry( + metadata.LayerDigest, + metadata.Architecture, + metadata.MediaType, + metadata.CachedAt, + metadata.LastAccessed, + metadata.SizeBytes, + artifacts, + metadata.Metadata is null + ? new Dictionary(StringComparer.Ordinal) + : new Dictionary(metadata.Metadata, StringComparer.Ordinal)); + } + + private string GetLayerDirectory(string layerDigest) + { + var safeDigest = SanitizeDigest(layerDigest); + return Path.Combine(_options.LayersDirectoryPath, safeDigest); + } + + private static string SanitizeArtifactName(string name) + { + var fileName = Path.GetFileName(name); + return string.IsNullOrWhiteSpace(fileName) ? "artifact" : fileName; + } + + private static string SanitizeDigest(string digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return "unknown"; + } + + var hash = digest.Contains(':', StringComparison.Ordinal) + ? digest[(digest.IndexOf(':') + 1)..] + : digest; + + var buffer = new char[hash.Length]; + var count = 0; + foreach (var ch in hash) + { + buffer[count++] = char.IsLetterOrDigit(ch) ? char.ToLowerInvariant(ch) : '_'; + } + + return new string(buffer, 0, count); + } + + private sealed class LayerCacheMetadata + { + public string LayerDigest { get; set; } = string.Empty; + + public string Architecture { get; set; } = string.Empty; + + public string MediaType { get; set; } = string.Empty; + + public DateTimeOffset CachedAt { get; set; } + + public DateTimeOffset LastAccessed { get; set; } + + public Dictionary? Metadata { get; set; } + + public Dictionary Artifacts { get; set; } = new(StringComparer.OrdinalIgnoreCase); + + public long SizeBytes { get; set; } + + } + + private sealed class LayerCacheArtifactMetadata + { + public string Name { get; set; } = string.Empty; + + public string RelativePath { get; set; } = string.Empty; + + public string ContentType { get; set; } = string.Empty; + + public long SizeBytes { get; set; } + + public bool Immutable { get; set; } + } +} diff --git a/src/StellaOps.Scanner.Cache/Maintenance/ScannerCacheMaintenanceService.cs b/src/StellaOps.Scanner.Cache/Maintenance/ScannerCacheMaintenanceService.cs new file mode 100644 index 00000000..3331772b --- /dev/null +++ b/src/StellaOps.Scanner.Cache/Maintenance/ScannerCacheMaintenanceService.cs @@ -0,0 +1,85 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Cache.Abstractions; + +namespace StellaOps.Scanner.Cache.Maintenance; + +public sealed class ScannerCacheMaintenanceService : BackgroundService +{ + private readonly ILayerCacheStore _layerCache; + private readonly IFileContentAddressableStore _fileCas; + private readonly IOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public ScannerCacheMaintenanceService( + ILayerCacheStore layerCache, + IFileContentAddressableStore fileCas, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _layerCache = layerCache ?? throw new ArgumentNullException(nameof(layerCache)); + _fileCas = fileCas ?? throw new ArgumentNullException(nameof(fileCas)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var settings = _options.Value; + if (!settings.Enabled) + { + _logger.LogInformation("Scanner cache disabled; maintenance loop will not start."); + return; + } + + if (!settings.EnableAutoEviction) + { + _logger.LogInformation("Scanner cache automatic eviction disabled by configuration."); + return; + } + + var interval = settings.MaintenanceInterval > TimeSpan.Zero + ? settings.MaintenanceInterval + : TimeSpan.FromMinutes(15); + + _logger.LogInformation("Scanner cache maintenance loop started with interval {Interval}", interval); + + await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false); + + using var timer = new PeriodicTimer(interval, _timeProvider); + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) + { + await RunMaintenanceAsync(stoppingToken).ConfigureAwait(false); + } + } + + private async Task RunMaintenanceAsync(CancellationToken cancellationToken) + { + try + { + var layerExpired = await _layerCache.EvictExpiredAsync(cancellationToken).ConfigureAwait(false); + var layerCompacted = await _layerCache.CompactAsync(cancellationToken).ConfigureAwait(false); + var casExpired = await _fileCas.EvictExpiredAsync(cancellationToken).ConfigureAwait(false); + var casCompacted = await _fileCas.CompactAsync(cancellationToken).ConfigureAwait(false); + + _logger.LogDebug( + "Scanner cache maintenance tick complete (layers expired={LayersExpired}, layers compacted={LayersCompacted}, cas expired={CasExpired}, cas compacted={CasCompacted})", + layerExpired, + layerCompacted, + casExpired, + casCompacted); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Shutting down; ignore. + } + catch (Exception ex) + { + _logger.LogError(ex, "Scanner cache maintenance tick failed"); + } + } +} diff --git a/src/StellaOps.Scanner.Cache/ScannerCacheMetrics.cs b/src/StellaOps.Scanner.Cache/ScannerCacheMetrics.cs new file mode 100644 index 00000000..0af9c413 --- /dev/null +++ b/src/StellaOps.Scanner.Cache/ScannerCacheMetrics.cs @@ -0,0 +1,43 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Scanner.Cache; + +public static class ScannerCacheMetrics +{ + public const string MeterName = "StellaOps.Scanner.Cache"; + + private static readonly Meter Meter = new(MeterName, "1.0.0"); + + private static readonly Counter LayerHits = Meter.CreateCounter("scanner.layer_cache_hits_total"); + private static readonly Counter LayerMisses = Meter.CreateCounter("scanner.layer_cache_misses_total"); + private static readonly Counter LayerEvictions = Meter.CreateCounter("scanner.layer_cache_evictions_total"); + private static readonly Histogram LayerBytes = Meter.CreateHistogram("scanner.layer_cache_bytes"); + private static readonly Counter FileCasHits = Meter.CreateCounter("scanner.file_cas_hits_total"); + private static readonly Counter FileCasMisses = Meter.CreateCounter("scanner.file_cas_misses_total"); + private static readonly Counter FileCasEvictions = Meter.CreateCounter("scanner.file_cas_evictions_total"); + private static readonly Histogram FileCasBytes = Meter.CreateHistogram("scanner.file_cas_bytes"); + + public static void RecordLayerHit(string layerDigest) + => LayerHits.Add(1, new KeyValuePair("layer", layerDigest)); + + public static void RecordLayerMiss(string layerDigest) + => LayerMisses.Add(1, new KeyValuePair("layer", layerDigest)); + + public static void RecordLayerEviction(string layerDigest) + => LayerEvictions.Add(1, new KeyValuePair("layer", layerDigest)); + + public static void RecordLayerBytes(long bytes) + => LayerBytes.Record(bytes); + + public static void RecordFileCasHit(string sha256) + => FileCasHits.Add(1, new KeyValuePair("sha256", sha256)); + + public static void RecordFileCasMiss(string sha256) + => FileCasMisses.Add(1, new KeyValuePair("sha256", sha256)); + + public static void RecordFileCasEviction(string sha256) + => FileCasEvictions.Add(1, new KeyValuePair("sha256", sha256)); + + public static void RecordFileCasBytes(long bytes) + => FileCasBytes.Record(bytes); +} diff --git a/src/StellaOps.Scanner.Cache/ScannerCacheOptions.cs b/src/StellaOps.Scanner.Cache/ScannerCacheOptions.cs new file mode 100644 index 00000000..9c12a180 --- /dev/null +++ b/src/StellaOps.Scanner.Cache/ScannerCacheOptions.cs @@ -0,0 +1,40 @@ +using System.IO; + +namespace StellaOps.Scanner.Cache; + +public sealed class ScannerCacheOptions +{ + private const long DefaultMaxBytes = 5L * 1024 * 1024 * 1024; // 5 GiB + + public bool Enabled { get; set; } = true; + + public string RootPath { get; set; } = Path.Combine("cache", "scanner"); + + public string LayersDirectoryName { get; set; } = "layers"; + + public string FileCasDirectoryName { get; set; } = "cas"; + + public TimeSpan LayerTtl { get; set; } = TimeSpan.FromDays(45); + + public TimeSpan FileTtl { get; set; } = TimeSpan.FromDays(30); + + public long MaxBytes { get; set; } = DefaultMaxBytes; + + public long WarmBytesThreshold { get; set; } = DefaultMaxBytes / 5; // 20 % + + public long ColdBytesThreshold { get; set; } = (DefaultMaxBytes * 4) / 5; // 80 % + + public bool EnableAutoEviction { get; set; } = true; + + public TimeSpan MaintenanceInterval { get; set; } = TimeSpan.FromMinutes(15); + + public bool EnableFileCas { get; set; } = true; + + public string? ImportDirectory { get; set; } + + public string? ExportDirectory { get; set; } + + public string LayersDirectoryPath => Path.Combine(RootPath, LayersDirectoryName); + + public string FileCasDirectoryPath => Path.Combine(RootPath, FileCasDirectoryName); +} diff --git a/src/StellaOps.Scanner.Cache/ScannerCacheServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Cache/ScannerCacheServiceCollectionExtensions.cs new file mode 100644 index 00000000..69841d5d --- /dev/null +++ b/src/StellaOps.Scanner.Cache/ScannerCacheServiceCollectionExtensions.cs @@ -0,0 +1,51 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Cache.Abstractions; +using StellaOps.Scanner.Cache.FileCas; +using StellaOps.Scanner.Cache.LayerCache; +using StellaOps.Scanner.Cache.Maintenance; + +namespace StellaOps.Scanner.Cache; + +public static class ScannerCacheServiceCollectionExtensions +{ + public static IServiceCollection AddScannerCache( + this IServiceCollection services, + IConfiguration configuration, + string sectionName = "scanner:cache") + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOptions() + .Bind(configuration.GetSection(sectionName)) + .Validate(options => !string.IsNullOrWhiteSpace(options.RootPath), "scanner:cache:rootPath must be configured"); + + services.TryAddSingleton(TimeProvider.System); + + services.TryAddSingleton(); + services.TryAddSingleton(sp => + { + var options = sp.GetRequiredService>(); + var timeProvider = sp.GetService() ?? TimeProvider.System; + var loggerFactory = sp.GetRequiredService(); + + if (!options.Value.EnableFileCas) + { + return new NullFileContentAddressableStore(); + } + + return new FileContentAddressableStore( + options, + loggerFactory.CreateLogger(), + timeProvider); + }); + + services.AddHostedService(); + + return services; + } +} diff --git a/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj b/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj new file mode 100644 index 00000000..46556ddc --- /dev/null +++ b/src/StellaOps.Scanner.Cache/StellaOps.Scanner.Cache.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Cache/TASKS.md b/src/StellaOps.Scanner.Cache/TASKS.md new file mode 100644 index 00000000..7821781b --- /dev/null +++ b/src/StellaOps.Scanner.Cache/TASKS.md @@ -0,0 +1,10 @@ +# Scanner Cache Task Board (Sprint 10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-CACHE-10-101 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-WORKER-09-201 | Implement layer cache store keyed by layer digest with metadata retention aligned with architecture §3.3 object layout. | Layer cache API supports get/put/delete by digest; metadata persisted with deterministic serialization; warm lookup covered by tests. | +| SCANNER-CACHE-10-102 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Build file CAS with dedupe, TTL enforcement, and offline import/export hooks for offline kit workflows. | CAS stores content by SHA-256, enforces TTL policy, import/export commands documented and exercised in tests. | +| SCANNER-CACHE-10-103 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Expose cache metrics/logging and configuration toggles for warm/cold thresholds. | Metrics counters/gauges emitted; options validated; logs include correlation IDs; configuration doc references settings. | +| SCANNER-CACHE-10-104 | DONE (2025-10-19) | Scanner Cache Guild | SCANNER-CACHE-10-101 | Implement cache invalidation workflows (layer delete, TTL expiry, diff invalidation). | Invalidation API implemented with deterministic eviction; tests cover TTL expiry + explicit delete; logs instrumented. | + +> Update statuses to DONE once acceptance criteria and tests/documentation are delivered. diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs new file mode 100644 index 00000000..e7d191bd --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentGraphBuilderTests.cs @@ -0,0 +1,93 @@ +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Core.Tests.Contracts; + +public sealed class ComponentGraphBuilderTests +{ + [Fact] + public void Build_AggregatesComponentsAcrossLayers() + { + var layer1 = LayerComponentFragment.Create("sha256:layer1", new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/a", "a", "1.0.0"), + LayerDigest = "sha256:layer1", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/package.json")), + Dependencies = ImmutableArray.Create("pkg:npm/x"), + Usage = ComponentUsage.Create(false), + Metadata = new ComponentMetadata + { + Scope = "runtime", + }, + } + }); + + var layer2 = LayerComponentFragment.Create("sha256:layer2", new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/a", "a", "1.0.0"), + LayerDigest = "sha256:layer2", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/index.js")), + Dependencies = ImmutableArray.Create("pkg:npm/y"), + Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), + }, + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/b", "b", "2.0.0"), + LayerDigest = "sha256:layer2", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/b/package.json")), + } + }); + + var graph = ComponentGraphBuilder.Build(new[] { layer1, layer2 }); + + Assert.Equal(new[] { "sha256:layer1", "sha256:layer2" }, graph.Layers.Select(layer => layer.LayerDigest)); + Assert.Equal(new[] { "pkg:npm/a", "pkg:npm/b" }, graph.Components.Select(component => component.Identity.Key)); + + var componentA = graph.ComponentMap["pkg:npm/a"]; + Assert.Equal("sha256:layer1", componentA.FirstLayerDigest); + Assert.Equal("sha256:layer2", componentA.LastLayerDigest); + Assert.Equal(new[] { "sha256:layer1", "sha256:layer2" }, componentA.LayerDigests); + Assert.True(componentA.Usage.UsedByEntrypoint); + Assert.Contains("/app/start.sh", componentA.Usage.Entrypoints); + Assert.Equal(new[] { "pkg:npm/x", "pkg:npm/y" }, componentA.Dependencies); + Assert.Equal("runtime", componentA.Metadata?.Scope); + Assert.Equal(2, componentA.Evidence.Length); + + var componentB = graph.ComponentMap["pkg:npm/b"]; + Assert.Equal("sha256:layer2", componentB.FirstLayerDigest); + Assert.Null(componentB.LastLayerDigest); + Assert.Single(componentB.LayerDigests, "sha256:layer2"); + Assert.False(componentB.Usage.UsedByEntrypoint); + } + + [Fact] + public void Build_DeterministicOrdering() + { + var fragments = new[] + { + LayerComponentFragment.Create("sha256:layer1", new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/c", "c"), + LayerDigest = "sha256:layer1", + }, + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/a", "a"), + LayerDigest = "sha256:layer1", + } + }) + }; + + var graph1 = ComponentGraphBuilder.Build(fragments); + var graph2 = ComponentGraphBuilder.Build(fragments); + + Assert.Equal(graph1.Components.Select(c => c.Identity.Key), graph2.Components.Select(c => c.Identity.Key)); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs new file mode 100644 index 00000000..48144fb3 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ComponentModelsTests.cs @@ -0,0 +1,85 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Serialization; + +namespace StellaOps.Scanner.Core.Tests.Contracts; + +public sealed class ComponentModelsTests +{ + [Fact] + public void ComponentIdentity_Create_Trimmed() + { + var identity = ComponentIdentity.Create(" pkg:npm/foo ", " Foo ", " 1.0.0 ", " pkg:npm/foo@1.0.0 ", " library ", " group "); + + Assert.Equal("pkg:npm/foo", identity.Key); + Assert.Equal("Foo", identity.Name); + Assert.Equal("1.0.0", identity.Version); + Assert.Equal("pkg:npm/foo@1.0.0", identity.Purl); + Assert.Equal("library", identity.ComponentType); + Assert.Equal("group", identity.Group); + } + + [Fact] + public void ComponentUsage_Create_SortsEntrypoints() + { + var usage = ComponentUsage.Create(true, new[] { "/app/start.sh", "/app/start.sh", "/bin/init", " ", null! }); + + Assert.True(usage.UsedByEntrypoint); + Assert.Equal(new[] { "/app/start.sh", "/bin/init" }, usage.Entrypoints); + } + + [Fact] + public void LayerComponentFragment_Create_SortsComponents() + { + var compB = new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/b", "b"), + LayerDigest = "sha256:layer2", + }; + + var compA = new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/a", "a"), + LayerDigest = "sha256:layer2", + }; + + var fragment = LayerComponentFragment.Create("sha256:layer2", new[] { compB, compA }); + + Assert.Equal("sha256:layer2", fragment.LayerDigest); + Assert.Equal(new[] { compA.Identity.Key, compB.Identity.Key }, fragment.Components.Select(c => c.Identity.Key)); + } + + [Fact] + public void ComponentRecord_Serializes_WithScannerDefaults() + { + var record = new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/test", "test", "1.0.0"), + LayerDigest = "sha256:layer", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/package.json")), + Dependencies = ImmutableArray.Create("pkg:npm/dep"), + Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), + Metadata = new ComponentMetadata + { + Scope = "runtime", + Licenses = new[] { "MIT" }, + Properties = new Dictionary + { + ["source"] = "package-lock.json", + }, + }, + }; + + var json = JsonSerializer.Serialize(record, ScannerJsonOptions.Default); + var deserialized = JsonSerializer.Deserialize(json, ScannerJsonOptions.Default); + + Assert.NotNull(deserialized); + Assert.Equal(record.Identity.Key, deserialized!.Identity.Key); + Assert.Equal(record.Metadata?.Scope, deserialized.Metadata?.Scope); + Assert.True(deserialized.Usage.UsedByEntrypoint); + Assert.Equal(record.Usage.Entrypoints.AsSpan(), deserialized.Usage.Entrypoints.AsSpan()); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ScanJobTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ScanJobTests.cs new file mode 100644 index 00000000..1d50755b --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ScanJobTests.cs @@ -0,0 +1,81 @@ +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Serialization; +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Contracts; + +public sealed class ScanJobTests +{ + [Fact] + public void SerializeAndDeserialize_RoundTripsDeterministically() + { + var createdAt = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.Zero); + var jobId = ScannerIdentifiers.CreateJobId("registry.example.com/stellaops/scanner:1.2.3", "sha256:ABCDEF", "tenant-a", "request-1"); + var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue"); + var error = new ScannerError( + ScannerErrorCode.AnalyzerFailure, + ScannerErrorSeverity.Error, + "Analyzer crashed for layer sha256:abc", + createdAt, + retryable: false, + details: new Dictionary + { + ["stage"] = "analyze-os", + ["layer"] = "sha256:abc" + }); + + var job = new ScanJob( + jobId, + ScanJobStatus.Running, + "registry.example.com/stellaops/scanner:1.2.3", + "SHA256:ABCDEF", + createdAt, + createdAt, + correlationId, + "tenant-a", + new Dictionary + { + ["requestId"] = "request-1" + }, + error); + + var json = JsonSerializer.Serialize(job, ScannerJsonOptions.CreateDefault()); + var deserialized = JsonSerializer.Deserialize(json, ScannerJsonOptions.CreateDefault()); + + Assert.NotNull(deserialized); + Assert.Equal(job.Id, deserialized!.Id); + Assert.Equal(job.ImageDigest, deserialized.ImageDigest); + Assert.Equal(job.CorrelationId, deserialized.CorrelationId); + Assert.Equal(job.Metadata["requestId"], deserialized.Metadata["requestId"]); + + var secondJson = JsonSerializer.Serialize(deserialized, ScannerJsonOptions.CreateDefault()); + Assert.Equal(json, secondJson); + } + + [Fact] + public void WithStatus_UpdatesTimestampDeterministically() + { + var createdAt = new DateTimeOffset(2025, 10, 18, 14, 30, 15, 123, TimeSpan.Zero); + var jobId = ScannerIdentifiers.CreateJobId("example/scanner:latest", "sha256:def", null, null); + var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue"); + + var job = new ScanJob( + jobId, + ScanJobStatus.Pending, + "example/scanner:latest", + "sha256:def", + createdAt, + null, + correlationId, + null, + null, + null); + + var updated = job.WithStatus(ScanJobStatus.Running, createdAt.AddSeconds(5)); + + Assert.Equal(ScanJobStatus.Running, updated.Status); + Assert.Equal(ScannerTimestamps.Normalize(createdAt.AddSeconds(5)), updated.UpdatedAt); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Contracts/ScannerCoreContractsTests.cs b/src/StellaOps.Scanner.Core.Tests/Contracts/ScannerCoreContractsTests.cs new file mode 100644 index 00000000..dc15acc6 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Contracts/ScannerCoreContractsTests.cs @@ -0,0 +1,130 @@ +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Serialization; +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Contracts; + +public sealed class ScannerCoreContractsTests +{ + private static readonly JsonSerializerOptions Options = ScannerJsonOptions.CreateDefault(); + private static readonly ScanJobId SampleJobId = ScanJobId.From(Guid.Parse("8f4cc9c5-8245-4b9d-9b4f-5ae049631b7d")); + private static readonly DateTimeOffset SampleCreatedAt = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.Zero).AddTicks(1_234_560); + + [Fact] + public void ScanJob_RoundTripMatchesGoldenFixture() + { + var job = CreateSampleJob(); + + var json = JsonSerializer.Serialize(job, Options); + var expected = LoadFixture("scan-job.json"); + Assert.Equal(expected, json); + + var deserialized = JsonSerializer.Deserialize(expected, Options); + Assert.NotNull(deserialized); + Assert.Equal(job.Id, deserialized!.Id); + Assert.Equal(job.ImageDigest, deserialized.ImageDigest); + Assert.Equal(job.CorrelationId, deserialized.CorrelationId); + Assert.Equal(job.Metadata, deserialized.Metadata); + Assert.Equal(job.Failure?.Message, deserialized.Failure?.Message); + Assert.Equal(job.Failure?.Details, deserialized.Failure?.Details); + } + + [Fact] + public void ScanProgressEvent_RoundTripMatchesGoldenFixture() + { + var progress = CreateSampleProgressEvent(); + + var json = JsonSerializer.Serialize(progress, Options); + var expected = LoadFixture("scan-progress-event.json"); + Assert.Equal(expected, json); + + var deserialized = JsonSerializer.Deserialize(expected, Options); + Assert.NotNull(deserialized); + Assert.Equal(progress.JobId, deserialized!.JobId); + Assert.Equal(progress.Stage, deserialized.Stage); + Assert.Equal(progress.Kind, deserialized.Kind); + Assert.Equal(progress.Sequence, deserialized.Sequence); + Assert.Equal(progress.Error?.Details, deserialized.Error?.Details); + } + + [Fact] + public void ScannerError_RoundTripMatchesGoldenFixture() + { + var error = CreateSampleError(); + + var json = JsonSerializer.Serialize(error, Options); + var expected = LoadFixture("scanner-error.json"); + Assert.Equal(expected, json); + + var deserialized = JsonSerializer.Deserialize(expected, Options); + Assert.NotNull(deserialized); + Assert.Equal(error.Code, deserialized!.Code); + Assert.Equal(error.Severity, deserialized.Severity); + Assert.Equal(error.Details, deserialized.Details); + } + + private static ScanJob CreateSampleJob() + { + var updatedAt = SampleCreatedAt.AddSeconds(5); + var correlationId = ScannerIdentifiers.CreateCorrelationId(SampleJobId, nameof(ScanStage.AnalyzeOperatingSystem)); + + return new ScanJob( + SampleJobId, + ScanJobStatus.Running, + "registry.example.com/stellaops/scanner:1.2.3", + "SHA256:ABCDEF", + SampleCreatedAt, + updatedAt, + correlationId, + "tenant-a", + new Dictionary + { + ["requestId"] = "req-1234", + ["source"] = "ci" + }, + CreateSampleError()); + } + + private static ScanProgressEvent CreateSampleProgressEvent() + { + return new ScanProgressEvent( + SampleJobId, + ScanStage.AnalyzeOperatingSystem, + ScanProgressEventKind.Warning, + sequence: 3, + timestamp: SampleCreatedAt.AddSeconds(1), + percentComplete: 42.5, + message: "OS analyzer reported missing packages", + attributes: new Dictionary + { + ["package"] = "openssl", + ["version"] = "1.1.1w" + }, + error: CreateSampleError()); + } + + private static ScannerError CreateSampleError() + { + return new ScannerError( + ScannerErrorCode.AnalyzerFailure, + ScannerErrorSeverity.Error, + "Analyzer failed to parse layer", + SampleCreatedAt, + retryable: false, + details: new Dictionary + { + ["layerDigest"] = "sha256:deadbeef", + ["attempt"] = "1" + }, + stage: nameof(ScanStage.AnalyzeOperatingSystem), + component: "os-analyzer"); + } + + private static string LoadFixture(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName); + return File.ReadAllText(path).Trim(); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-job.json b/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-job.json new file mode 100644 index 00000000..6d5e8b54 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-job.json @@ -0,0 +1 @@ +{"id":"8f4cc9c582454b9d9b4f5ae049631b7d","status":"running","imageReference":"registry.example.com/stellaops/scanner:1.2.3","imageDigest":"sha256:abcdef","createdAt":"2025-10-18T14:30:15.123456+00:00","updatedAt":"2025-10-18T14:30:20.123456+00:00","correlationId":"scan-analyzeoperatingsystem-8f4cc9c582454b9d9b4f5ae049631b7d","tenantId":"tenant-a","metadata":{"requestId":"req-1234","source":"ci"},"failure":{"code":"analyzerFailure","severity":"error","message":"Analyzer failed to parse layer","timestamp":"2025-10-18T14:30:15.123456+00:00","retryable":false,"stage":"AnalyzeOperatingSystem","component":"os-analyzer","details":{"layerDigest":"sha256:deadbeef","attempt":"1"}}} diff --git a/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-progress-event.json b/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-progress-event.json new file mode 100644 index 00000000..2ae678a4 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Fixtures/scan-progress-event.json @@ -0,0 +1 @@ +{"jobId":"8f4cc9c582454b9d9b4f5ae049631b7d","stage":"analyzeOperatingSystem","kind":"warning","sequence":3,"timestamp":"2025-10-18T14:30:16.123456+00:00","percentComplete":42.5,"message":"OS analyzer reported missing packages","attributes":{"package":"openssl","version":"1.1.1w"},"error":{"code":"analyzerFailure","severity":"error","message":"Analyzer failed to parse layer","timestamp":"2025-10-18T14:30:15.123456+00:00","retryable":false,"stage":"AnalyzeOperatingSystem","component":"os-analyzer","details":{"layerDigest":"sha256:deadbeef","attempt":"1"}}} diff --git a/src/StellaOps.Scanner.Core.Tests/Fixtures/scanner-error.json b/src/StellaOps.Scanner.Core.Tests/Fixtures/scanner-error.json new file mode 100644 index 00000000..388502d7 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Fixtures/scanner-error.json @@ -0,0 +1 @@ +{"code":"analyzerFailure","severity":"error","message":"Analyzer failed to parse layer","timestamp":"2025-10-18T14:30:15.123456+00:00","retryable":false,"stage":"AnalyzeOperatingSystem","component":"os-analyzer","details":{"layerDigest":"sha256:deadbeef","attempt":"1"}} diff --git a/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsPerformanceTests.cs b/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsPerformanceTests.cs new file mode 100644 index 00000000..8d3e5a89 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsPerformanceTests.cs @@ -0,0 +1,103 @@ +using System.Collections.Generic; +using System.Diagnostics; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Observability; +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Observability; + +public sealed class ScannerLogExtensionsPerformanceTests +{ + private const double ThresholdMicroseconds = 5.0; + private const int WarmupIterations = 5_000; + private const int MeasuredIterations = 200_000; + private static readonly DateTimeOffset Timestamp = ScannerTimestamps.Normalize(new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero)); + private static readonly string Stage = nameof(ScanStage.AnalyzeOperatingSystem); + private static readonly string Component = "os-analyzer"; + + [Fact] + public void BeginScanScope_CompletesWithinThreshold() + { + using var factory = LoggerFactory.Create(builder => builder.AddFilter(static _ => false)); + var logger = factory.CreateLogger("ScannerPerformance"); + var job = CreateScanJob(); + + var microseconds = Measure(() => logger.BeginScanScope(job, Stage, Component)); + + Assert.True(microseconds <= ThresholdMicroseconds, $"Expected BeginScanScope to stay ≤ {ThresholdMicroseconds} µs but measured {microseconds:F3} µs."); + } + + [Fact] + public void BeginProgressScope_CompletesWithinThreshold() + { + using var factory = LoggerFactory.Create(builder => builder.AddFilter(static _ => false)); + var logger = factory.CreateLogger("ScannerPerformance"); + var progress = CreateProgressEvent(); + + var microseconds = Measure(() => logger.BeginProgressScope(progress, Component)); + + Assert.True(microseconds <= ThresholdMicroseconds, $"Expected BeginProgressScope to stay ≤ {ThresholdMicroseconds} µs but measured {microseconds:F3} µs."); + } + + private static double Measure(Func scopeFactory) + { + for (var i = 0; i < WarmupIterations; i++) + { + using var scope = scopeFactory(); + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var stopwatch = Stopwatch.StartNew(); + for (var i = 0; i < MeasuredIterations; i++) + { + using var scope = scopeFactory(); + } + + stopwatch.Stop(); + + return stopwatch.Elapsed.TotalSeconds * 1_000_000 / MeasuredIterations; + } + + private static ScanJob CreateScanJob() + { + var jobId = ScannerIdentifiers.CreateJobId("registry.example.com/stellaops/scanner:1.2.3", "sha256:abcdef", "tenant-a", "perf"); + var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, Stage, Component); + + return new ScanJob( + jobId, + ScanJobStatus.Running, + "registry.example.com/stellaops/scanner:1.2.3", + "sha256:abcdef", + Timestamp, + Timestamp, + correlationId, + "tenant-a", + new Dictionary(StringComparer.Ordinal) + { + ["requestId"] = "req-perf" + }); + } + + private static ScanProgressEvent CreateProgressEvent() + { + var jobId = ScannerIdentifiers.CreateJobId("registry.example.com/stellaops/scanner:1.2.3", "sha256:abcdef", "tenant-a", "perf"); + + return new ScanProgressEvent( + jobId, + ScanStage.AnalyzeOperatingSystem, + ScanProgressEventKind.Progress, + sequence: 42, + Timestamp, + percentComplete: 10.5, + message: "performance check", + attributes: new Dictionary(StringComparer.Ordinal) + { + ["sample"] = "true" + }); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsTests.cs b/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsTests.cs new file mode 100644 index 00000000..02354c7a --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Observability/ScannerLogExtensionsTests.cs @@ -0,0 +1,39 @@ +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Observability; +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Observability; + +public sealed class ScannerLogExtensionsTests +{ + [Fact] + public void BeginScanScope_PopulatesCorrelationContext() + { + using var factory = LoggerFactory.Create(builder => builder.AddFilter(_ => true)); + var logger = factory.CreateLogger("test"); + + var jobId = ScannerIdentifiers.CreateJobId("example/scanner:1.0", "sha256:abc", null, null); + var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, "enqueue"); + var job = new ScanJob( + jobId, + ScanJobStatus.Pending, + "example/scanner:1.0", + "sha256:abc", + DateTimeOffset.UtcNow, + null, + correlationId, + null, + null, + null); + + using (logger.BeginScanScope(job, "enqueue")) + { + Assert.True(ScannerCorrelationContextAccessor.TryGetCorrelationId(out var current)); + Assert.Equal(correlationId, current); + } + + Assert.False(ScannerCorrelationContextAccessor.TryGetCorrelationId(out _)); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Security/AuthorityTokenSourceTests.cs b/src/StellaOps.Scanner.Core.Tests/Security/AuthorityTokenSourceTests.cs new file mode 100644 index 00000000..ae87541b --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Security/AuthorityTokenSourceTests.cs @@ -0,0 +1,89 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using Microsoft.IdentityModel.Tokens; +using StellaOps.Auth.Client; +using StellaOps.Scanner.Core.Security; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Security; + +public sealed class AuthorityTokenSourceTests +{ + [Fact] + public async Task GetAsync_ReusesCachedTokenUntilRefreshSkew() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var client = new FakeTokenClient(timeProvider); + var source = new AuthorityTokenSource(client, TimeSpan.FromSeconds(30), timeProvider, NullLogger.Instance); + + var token1 = await source.GetAsync("scanner", new[] { "scanner.read" }); + Assert.Equal(1, client.RequestCount); + + var token2 = await source.GetAsync("scanner", new[] { "scanner.read" }); + Assert.Equal(1, client.RequestCount); + Assert.Equal(token1.AccessToken, token2.AccessToken); + + timeProvider.Advance(TimeSpan.FromMinutes(3)); + var token3 = await source.GetAsync("scanner", new[] { "scanner.read" }); + Assert.Equal(2, client.RequestCount); + Assert.NotEqual(token1.AccessToken, token3.AccessToken); + } + + [Fact] + public async Task InvalidateAsync_RemovesCachedToken() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var client = new FakeTokenClient(timeProvider); + var source = new AuthorityTokenSource(client, TimeSpan.FromSeconds(30), timeProvider, NullLogger.Instance); + + _ = await source.GetAsync("scanner", new[] { "scanner.read" }); + Assert.Equal(1, client.RequestCount); + + await source.InvalidateAsync("scanner", new[] { "scanner.read" }); + _ = await source.GetAsync("scanner", new[] { "scanner.read" }); + + Assert.Equal(2, client.RequestCount); + } + + private sealed class FakeTokenClient : IStellaOpsTokenClient + { + private readonly FakeTimeProvider timeProvider; + private int counter; + + public FakeTokenClient(FakeTimeProvider timeProvider) + { + this.timeProvider = timeProvider; + } + + public int RequestCount => counter; + + public Task RequestClientCredentialsTokenAsync(string? scope = null, CancellationToken cancellationToken = default) + { + var access = $"token-{Interlocked.Increment(ref counter)}"; + var expires = timeProvider.GetUtcNow().AddMinutes(2); + var scopes = scope is null + ? Array.Empty() + : scope.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + + return Task.FromResult(new StellaOpsTokenResult(access, "Bearer", expires, scopes)); + } + + public Task RequestPasswordTokenAsync(string username, string password, string? scope = null, CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public Task GetJsonWebKeySetAsync(CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + + public ValueTask GetCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.FromResult(null); + + public ValueTask CacheTokenAsync(string key, StellaOpsTokenCacheEntry entry, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + + public ValueTask ClearCachedTokenAsync(string key, CancellationToken cancellationToken = default) + => ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Security/DpopProofValidatorTests.cs b/src/StellaOps.Scanner.Core.Tests/Security/DpopProofValidatorTests.cs new file mode 100644 index 00000000..041ed069 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Security/DpopProofValidatorTests.cs @@ -0,0 +1,117 @@ +using System.Collections.Generic; +using System.IdentityModel.Tokens.Jwt; +using System.Security.Cryptography; +using Microsoft.Extensions.Time.Testing; +using Microsoft.IdentityModel.Tokens; +using Microsoft.Extensions.Options; +using StellaOps.Auth.Security.Dpop; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Security; + +public sealed class DpopProofValidatorTests +{ + [Fact] + public async Task ValidateAsync_ReturnsSuccess_ForValidProof() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), new InMemoryDpopReplayCache(timeProvider), timeProvider); + using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") }; + + var proof = CreateProof(timeProvider, securityKey, "GET", new Uri("https://scanner.example.com/api/v1/scans")); + var result = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans")); + + Assert.True(result.IsValid); + Assert.NotNull(result.PublicKey); + Assert.NotNull(result.JwtId); + } + + [Fact] + public async Task ValidateAsync_Fails_OnNonceMismatch() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), new InMemoryDpopReplayCache(timeProvider), timeProvider); + using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") }; + + var proof = CreateProof(timeProvider, securityKey, "POST", new Uri("https://scanner.example.com/api/v1/scans"), nonce: "expected"); + var result = await validator.ValidateAsync(proof, "POST", new Uri("https://scanner.example.com/api/v1/scans"), nonce: "different"); + + Assert.False(result.IsValid); + Assert.Equal("invalid_token", result.ErrorCode); + } + + [Fact] + public async Task ValidateAsync_Fails_OnReplay() + { + var timeProvider = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var cache = new InMemoryDpopReplayCache(timeProvider); + var validator = new DpopProofValidator(Options.Create(new DpopValidationOptions()), cache, timeProvider); + using var key = ECDsa.Create(ECCurve.NamedCurves.nistP256); + var securityKey = new ECDsaSecurityKey(key) { KeyId = Guid.NewGuid().ToString("N") }; + var jti = Guid.NewGuid().ToString(); + + var proof = CreateProof(timeProvider, securityKey, "GET", new Uri("https://scanner.example.com/api/v1/scans"), jti: jti); + + var first = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans")); + Assert.True(first.IsValid); + + var second = await validator.ValidateAsync(proof, "GET", new Uri("https://scanner.example.com/api/v1/scans")); + Assert.False(second.IsValid); + Assert.Equal("replay", second.ErrorCode); + } + + private static string CreateProof(FakeTimeProvider timeProvider, ECDsaSecurityKey key, string method, Uri uri, string? nonce = null, string? jti = null) + { + var handler = new JwtSecurityTokenHandler(); + var signingCredentials = new SigningCredentials(key, SecurityAlgorithms.EcdsaSha256); + var jwk = JsonWebKeyConverter.ConvertFromECDsaSecurityKey(key); + + var header = new JwtHeader(signingCredentials) + { + ["typ"] = "dpop+jwt", + ["jwk"] = new Dictionary + { + ["kty"] = jwk.Kty, + ["crv"] = jwk.Crv, + ["x"] = jwk.X, + ["y"] = jwk.Y + } + }; + + var payload = new JwtPayload + { + ["htm"] = method.ToUpperInvariant(), + ["htu"] = Normalize(uri), + ["iat"] = timeProvider.GetUtcNow().ToUnixTimeSeconds(), + ["jti"] = jti ?? Guid.NewGuid().ToString() + }; + + if (nonce is not null) + { + payload["nonce"] = nonce; + } + + var token = new JwtSecurityToken(header, payload); + return handler.WriteToken(token); + } + + private static string Normalize(Uri uri) + { + var builder = new UriBuilder(uri) + { + Fragment = string.Empty + }; + + builder.Host = builder.Host.ToLowerInvariant(); + builder.Scheme = builder.Scheme.ToLowerInvariant(); + + if ((builder.Scheme == "http" && builder.Port == 80) || (builder.Scheme == "https" && builder.Port == 443)) + { + builder.Port = -1; + } + + return builder.Uri.GetComponents(UriComponents.SchemeAndServer | UriComponents.PathAndQuery, UriFormat.UriEscaped); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Security/RestartOnlyPluginGuardTests.cs b/src/StellaOps.Scanner.Core.Tests/Security/RestartOnlyPluginGuardTests.cs new file mode 100644 index 00000000..363c247a --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Security/RestartOnlyPluginGuardTests.cs @@ -0,0 +1,26 @@ +using System; +using StellaOps.Scanner.Core.Security; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Security; + +public sealed class RestartOnlyPluginGuardTests +{ + [Fact] + public void EnsureRegistrationAllowed_AllowsNewPluginsBeforeSeal() + { + var guard = new RestartOnlyPluginGuard(); + guard.EnsureRegistrationAllowed("./plugins/analyzer.dll"); + + Assert.Contains(guard.KnownPlugins, path => path.EndsWith("analyzer.dll", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void EnsureRegistrationAllowed_ThrowsAfterSeal() + { + var guard = new RestartOnlyPluginGuard(new[] { "./plugins/a.dll" }); + guard.Seal(); + + Assert.Throws(() => guard.EnsureRegistrationAllowed("./plugins/new.dll")); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj b/src/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj new file mode 100644 index 00000000..216382d9 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/StellaOps.Scanner.Core.Tests.csproj @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Core.Tests/Utility/ScannerIdentifiersTests.cs b/src/StellaOps.Scanner.Core.Tests/Utility/ScannerIdentifiersTests.cs new file mode 100644 index 00000000..224b5ed6 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Utility/ScannerIdentifiersTests.cs @@ -0,0 +1,33 @@ +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Utility; + +public sealed class ScannerIdentifiersTests +{ + [Fact] + public void CreateJobId_IsDeterministicAndCaseInsensitive() + { + var first = ScannerIdentifiers.CreateJobId("registry.example.com/repo:latest", "SHA256:ABC", "Tenant-A", "salt"); + var second = ScannerIdentifiers.CreateJobId("REGISTRY.EXAMPLE.COM/REPO:latest", "sha256:abc", "tenant-a", "salt"); + + Assert.Equal(first, second); + } + + [Fact] + public void CreateDeterministicHash_ProducesLowercaseHex() + { + var hash = ScannerIdentifiers.CreateDeterministicHash("scan", "abc", "123"); + + Assert.Matches("^[0-9a-f]{64}$", hash); + Assert.Equal(hash, hash.ToLowerInvariant()); + } + + [Fact] + public void NormalizeImageReference_LowercasesRegistryAndRepository() + { + var normalized = ScannerIdentifiers.NormalizeImageReference("Registry.Example.com/StellaOps/Scanner:1.0"); + + Assert.Equal("registry.example.com/stellaops/scanner:1.0", normalized); + } +} diff --git a/src/StellaOps.Scanner.Core.Tests/Utility/ScannerTimestampsTests.cs b/src/StellaOps.Scanner.Core.Tests/Utility/ScannerTimestampsTests.cs new file mode 100644 index 00000000..2cc61166 --- /dev/null +++ b/src/StellaOps.Scanner.Core.Tests/Utility/ScannerTimestampsTests.cs @@ -0,0 +1,26 @@ +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Core.Tests.Utility; + +public sealed class ScannerTimestampsTests +{ + [Fact] + public void Normalize_TrimsToMicroseconds() + { + var value = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.Zero).AddTicks(7); + var normalized = ScannerTimestamps.Normalize(value); + + var expectedTicks = value.UtcTicks - (value.UtcTicks % 10); + Assert.Equal(expectedTicks, normalized.UtcTicks); + } + + [Fact] + public void ToIso8601_ProducesUtcString() + { + var value = new DateTimeOffset(2025, 10, 18, 14, 30, 15, TimeSpan.FromHours(-4)); + var iso = ScannerTimestamps.ToIso8601(value); + + Assert.Equal("2025-10-18T18:30:15.000000Z", iso); + } +} diff --git a/src/StellaOps.Scanner.Core/AGENTS.md b/src/StellaOps.Scanner.Core/AGENTS.md new file mode 100644 index 00000000..4805f55e --- /dev/null +++ b/src/StellaOps.Scanner.Core/AGENTS.md @@ -0,0 +1,29 @@ +# AGENTS +## Role +Provide shared scanner contracts, observability primitives, and security utilities consumed by the WebService, Worker, analyzers, and downstream tooling. +## Scope +- Canonical DTOs for scan jobs, progress, outcomes, and error taxonomy shared across scanner services. +- Deterministic ID and timestamp helpers to guarantee reproducible job identifiers and ISO-8601 rendering. +- Observability helpers (logging scopes, correlation IDs, metric naming, activity sources) with negligible overhead. +- Authority/OpTok integrations, DPoP validation helpers, and restart-time plug-in guardrails for scanner components. +## Participants +- Scanner.WebService and Scanner.Worker depend on these primitives for request handling, queue interactions, and diagnostics. +- Policy/Signer integrations rely on deterministic identifiers and timestamps emitted here. +- DevOps/Offline kits bundle plug-in manifests validated via the guardrails defined in this module. +## Interfaces & contracts +- DTOs must round-trip via System.Text.Json with `JsonSerializerDefaults.Web` and preserve ordering. +- Deterministic helpers must not depend on ambient time/randomness; they derive IDs from explicit inputs and normalize timestamps to microsecond precision in UTC. +- Observability scopes expose `scanId`, `jobId`, `correlationId`, and `imageDigest` fields with `stellaops scanner` metric prefixing. +- Security helpers expose `IAuthorityTokenSource`, `IDPoPProofValidator`, and `IPluginCatalogGuard` abstractions with DI-friendly implementations. +## In/Out of scope +In: shared contracts, telemetry primitives, security utilities, plug-in manifest checks. +Out: queue implementations, analyzer logic, storage adapters, HTTP endpoints, UI wiring. +## Observability & security expectations +- No network calls except via registered Authority clients. +- Avoid allocations in hot paths; prefer struct enumerables/`ValueTask`. +- All logs structured, correlation IDs propagated, no secrets persisted. +- DPoP validation enforces algorithm allowlist (ES256/ES384) and ensures replay cache hooks. +## Tests +- `../StellaOps.Scanner.Core.Tests` owns unit coverage with deterministic fixtures. +- Golden JSON for DTO round-trips stored under `Fixtures/`. +- Security and observability helpers must include tests proving deterministic outputs and rejecting malformed proofs. diff --git a/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs b/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs new file mode 100644 index 00000000..29b5eb78 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ComponentGraph.cs @@ -0,0 +1,242 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Scanner.Core.Contracts; + +public sealed record ComponentGraph +{ + public required ImmutableArray Layers { get; init; } + + public required ImmutableArray Components { get; init; } + + public ImmutableDictionary ComponentMap { get; init; } = ImmutableDictionary.Empty; + + public bool TryGetComponent(string key, out AggregatedComponent component) + => ComponentMap.TryGetValue(key, out component!); +} + +public static class ComponentGraphBuilder +{ + public static ComponentGraph Build(IEnumerable fragments) + { + ArgumentNullException.ThrowIfNull(fragments); + + var orderedLayers = fragments + .Where(static fragment => !string.IsNullOrWhiteSpace(fragment.LayerDigest)) + .Select(NormalizeFragment) + .ToImmutableArray(); + + var accumulators = new Dictionary(StringComparer.Ordinal); + + foreach (var fragment in orderedLayers) + { + foreach (var component in fragment.Components) + { + var key = component.Identity.Key; + if (string.IsNullOrWhiteSpace(key)) + { + continue; + } + + if (!accumulators.TryGetValue(key, out var accumulator)) + { + accumulator = new ComponentAccumulator(component.Identity); + accumulators.Add(key, accumulator); + } + + accumulator.Include(component, fragment.LayerDigest); + } + } + + var components = accumulators.Values + .Select(static accumulator => accumulator.ToAggregatedComponent()) + .OrderBy(static component => component.Identity.Key, StringComparer.Ordinal) + .ToImmutableArray(); + + var map = components.ToImmutableDictionary(static component => component.Identity.Key, StringComparer.Ordinal); + + return new ComponentGraph + { + Layers = orderedLayers, + Components = components, + ComponentMap = map, + }; + } + + private static LayerComponentFragment NormalizeFragment(LayerComponentFragment fragment) + { + if (fragment.Components.All(component => string.Equals(component.LayerDigest, fragment.LayerDigest, StringComparison.Ordinal))) + { + return fragment; + } + + var normalizedComponents = fragment.Components + .Select(component => component.LayerDigest.Equals(fragment.LayerDigest, StringComparison.Ordinal) + ? component + : component with { LayerDigest = fragment.LayerDigest }) + .ToImmutableArray(); + + return fragment with { Components = normalizedComponents }; + } + + private sealed class ComponentAccumulator + { + private readonly ComponentIdentity _identity; + private readonly SortedSet _layers = new(StringComparer.Ordinal); + private readonly SortedSet _dependencies = new(StringComparer.Ordinal); + private readonly Dictionary _evidence = new(); + private ComponentUsage _usage = ComponentUsage.Unused; + private ComponentMetadata? _metadata; + private string? _firstLayer; + private string? _lastLayer; + + public ComponentAccumulator(ComponentIdentity identity) + { + _identity = identity; + } + + public void Include(ComponentRecord record, string layerDigest) + { + _layers.Add(layerDigest); + _dependencies.UnionWith(record.Dependencies); + + foreach (var evidence in record.Evidence) + { + var key = new EvidenceKey(evidence.Kind, evidence.Value, evidence.Source); + _evidence[key] = evidence; + } + + if (record.Metadata is not null) + { + _metadata = _metadata is null + ? record.Metadata + : MergeMetadata(_metadata, record.Metadata); + } + + if (record.Usage.UsedByEntrypoint || _usage.UsedByEntrypoint) + { + var entrypoints = record.Usage.EntryPointsOrEmpty(); + var existing = _usage.EntryPointsOrEmpty(); + var builder = ImmutableSortedSet.CreateBuilder(StringComparer.Ordinal); + builder.UnionWith(existing); + builder.UnionWith(entrypoints); + _usage = new ComponentUsage(true, builder.ToImmutableArray()); + } + else if (!_usage.UsedByEntrypoint && record.Usage.UsedByEntrypoint) + { + _usage = record.Usage; + } + else if (_usage is { UsedByEntrypoint: false } && record.Usage is { UsedByEntrypoint: false } && record.Usage.Entrypoints.Length > 0) + { + _usage = new ComponentUsage(false, record.Usage.Entrypoints); + } + + if (_firstLayer is null) + { + _firstLayer = layerDigest; + } + else if (!StringComparer.Ordinal.Equals(_firstLayer, layerDigest)) + { + _lastLayer = layerDigest; + } + } + + public AggregatedComponent ToAggregatedComponent() + { + return new AggregatedComponent + { + Identity = _identity, + FirstLayerDigest = _firstLayer ?? string.Empty, + LastLayerDigest = _lastLayer, + LayerDigests = _layers.ToImmutableArray(), + Evidence = _evidence.Values + .OrderBy(static evidence => evidence.Kind, StringComparer.Ordinal) + .ThenBy(static evidence => evidence.Value, StringComparer.Ordinal) + .ThenBy(static evidence => evidence.Source, StringComparer.Ordinal) + .ToImmutableArray(), + Dependencies = _dependencies.ToImmutableArray(), + Metadata = _metadata, + Usage = _usage, + }; + } + } + + private static ComponentMetadata MergeMetadata(ComponentMetadata existing, ComponentMetadata incoming) + { + var scope = existing.Scope ?? incoming.Scope; + + var licenses = MergeLists(existing.Licenses, incoming.Licenses); + var properties = MergeDictionary(existing.Properties, incoming.Properties); + + return new ComponentMetadata + { + Scope = scope, + Licenses = licenses, + Properties = properties, + }; + } + + private static IReadOnlyList? MergeLists(IReadOnlyList? left, IReadOnlyList? right) + { + if ((left is null || left.Count == 0) && (right is null || right.Count == 0)) + { + return null; + } + + var builder = ImmutableSortedSet.CreateBuilder(StringComparer.Ordinal); + if (left is not null) + { + builder.UnionWith(left.Where(static item => !string.IsNullOrWhiteSpace(item))); + } + + if (right is not null) + { + builder.UnionWith(right.Where(static item => !string.IsNullOrWhiteSpace(item))); + } + + return builder.ToImmutableArray(); + } + + private static IReadOnlyDictionary? MergeDictionary(IReadOnlyDictionary? left, IReadOnlyDictionary? right) + { + if ((left is null || left.Count == 0) && (right is null || right.Count == 0)) + { + return null; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + if (left is not null) + { + foreach (var (key, value) in left) + { + if (!string.IsNullOrWhiteSpace(key) && value is not null) + { + builder[key] = value; + } + } + } + + if (right is not null) + { + foreach (var (key, value) in right) + { + if (!string.IsNullOrWhiteSpace(key) && value is not null) + { + builder[key] = value; + } + } + } + + return builder.ToImmutable(); + } + + private readonly record struct EvidenceKey(string Kind, string Value, string? Source); +} + +internal static class ComponentUsageExtensions +{ + public static ImmutableArray EntryPointsOrEmpty(this ComponentUsage usage) + => usage.Entrypoints; +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs b/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs new file mode 100644 index 00000000..23d14a83 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ComponentModels.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Core.Contracts; + +/// +/// Canonical identifier for a component discovered during analysis. +/// +public sealed record ComponentIdentity +{ + [JsonPropertyName("key")] + public string Key { get; init; } = string.Empty; + + [JsonPropertyName("name")] + public string Name { get; init; } = string.Empty; + + [JsonPropertyName("version")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Version { get; init; } + = null; + + [JsonPropertyName("purl")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Purl { get; init; } + = null; + + [JsonPropertyName("type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ComponentType { get; init; } + = null; + + [JsonPropertyName("group")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Group { get; init; } + = null; + + public static ComponentIdentity Create(string key, string name, string? version = null, string? purl = null, string? componentType = null, string? group = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + key = key.Trim(); + name = name.Trim(); + version = version?.Trim(); + purl = purl?.Trim(); + componentType = componentType?.Trim(); + group = group?.Trim(); + + return new ComponentIdentity + { + Key = key, + Name = name, + Version = version, + Purl = purl, + ComponentType = componentType, + Group = group, + }; + } +} + +/// +/// Evidence associated with a component (e.g., file path, manifest origin). +/// +public sealed record ComponentEvidence +{ + [JsonPropertyName("kind")] + public string Kind { get; init; } = string.Empty; + + [JsonPropertyName("value")] + public string Value { get; init; } = string.Empty; + + [JsonPropertyName("source")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Source { get; init; } + = null; + + public static ComponentEvidence FromPath(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + return new ComponentEvidence { Kind = "file", Value = path }; + } +} + +/// +/// Optional metadata describing dependency relationships or classification. +/// +public sealed record ComponentMetadata +{ + [JsonPropertyName("scope")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Scope { get; init; } + = null; + + [JsonPropertyName("licenses")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyList? Licenses { get; init; } + = null; + + [JsonPropertyName("properties")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public IReadOnlyDictionary? Properties { get; init; } + = null; +} + +/// +/// Represents a single component discovered within a layer fragment. +/// +public sealed record ComponentRecord +{ + [JsonPropertyName("identity")] + public ComponentIdentity Identity { get; init; } = ComponentIdentity.Create("unknown", "unknown"); + + [JsonPropertyName("layerDigest")] + public string LayerDigest { get; init; } = string.Empty; + + [JsonPropertyName("evidence")] + public ImmutableArray Evidence { get; init; } = ImmutableArray.Empty; + + [JsonPropertyName("dependencies")] + public ImmutableArray Dependencies { get; init; } = ImmutableArray.Empty; + + [JsonPropertyName("metadata")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ComponentMetadata? Metadata { get; init; } + = null; + + [JsonPropertyName("usage")] + public ComponentUsage Usage { get; init; } = ComponentUsage.Unused; + + public ComponentRecord WithUsage(ComponentUsage usage) + => this with { Usage = usage }; + + public ComponentRecord WithLayer(string layerDigest) + => this with { LayerDigest = layerDigest }; +} + +/// +/// Usage annotations (derived from EntryTrace or other signals). +/// +public sealed record ComponentUsage +{ + public static ComponentUsage Unused { get; } = new(false, ImmutableArray.Empty); + + public ComponentUsage(bool usedByEntrypoint, ImmutableArray entrypoints) + { + UsedByEntrypoint = usedByEntrypoint; + Entrypoints = entrypoints.IsDefault ? ImmutableArray.Empty : entrypoints; + } + + [JsonPropertyName("usedByEntrypoint")] + public bool UsedByEntrypoint { get; init; } + = false; + + [JsonPropertyName("entrypoints")] + public ImmutableArray Entrypoints { get; init; } + = ImmutableArray.Empty; + + public static ComponentUsage Create(bool usedByEntrypoint, IEnumerable? entrypoints = null) + { + if (entrypoints is null) + { + return new ComponentUsage(usedByEntrypoint, ImmutableArray.Empty); + } + + var builder = ImmutableSortedSet.CreateBuilder(StringComparer.Ordinal); + foreach (var entry in entrypoints) + { + if (string.IsNullOrWhiteSpace(entry)) + { + continue; + } + + builder.Add(entry.Trim()); + } + + if (builder.Count == 0) + { + return new ComponentUsage(usedByEntrypoint, ImmutableArray.Empty); + } + + var arrayBuilder = ImmutableArray.CreateBuilder(builder.Count); + foreach (var entry in builder) + { + if (!string.IsNullOrEmpty(entry)) + { + arrayBuilder.Add(entry!); + } + } + + return new ComponentUsage(usedByEntrypoint, arrayBuilder.ToImmutable()); + } +} + +/// +/// Convenience helpers for component collections. +/// +public static class ComponentModelExtensions +{ + public static ImmutableArray Normalize(this IEnumerable? components) + { + if (components is null) + { + return ImmutableArray.Empty; + } + + return ImmutableArray.CreateRange(components); + } +} + +/// +/// Components introduced by a specific layer. +/// +public sealed record LayerComponentFragment +{ + [JsonPropertyName("layerDigest")] + public string LayerDigest { get; init; } = string.Empty; + + [JsonPropertyName("components")] + public ImmutableArray Components { get; init; } = ImmutableArray.Empty; + + public static LayerComponentFragment Create(string layerDigest, IEnumerable? components) + { + ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest); + var list = components is null + ? ImmutableArray.Empty + : ImmutableArray.CreateRange(components); + + if (!list.IsEmpty) + { + list = list + .OrderBy(component => component.Identity.Key, StringComparer.Ordinal) + .ToImmutableArray(); + } + + return new LayerComponentFragment + { + LayerDigest = layerDigest, + Components = list, + }; + } +} + +/// +/// Aggregated component spanning the complete image (all layers). +/// +public sealed record AggregatedComponent +{ + [JsonPropertyName("identity")] + public ComponentIdentity Identity { get; init; } = ComponentIdentity.Create("unknown", "unknown"); + + [JsonPropertyName("firstLayerDigest")] + public string FirstLayerDigest { get; init; } = string.Empty; + + [JsonPropertyName("lastLayerDigest")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? LastLayerDigest { get; init; } + = null; + + [JsonPropertyName("layerDigests")] + public ImmutableArray LayerDigests { get; init; } = ImmutableArray.Empty; + + [JsonPropertyName("evidence")] + public ImmutableArray Evidence { get; init; } = ImmutableArray.Empty; + + [JsonPropertyName("dependencies")] + public ImmutableArray Dependencies { get; init; } = ImmutableArray.Empty; + + [JsonPropertyName("metadata")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public ComponentMetadata? Metadata { get; init; } + = null; + + [JsonPropertyName("usage")] + public ComponentUsage Usage { get; init; } = ComponentUsage.Unused; +} diff --git a/src/StellaOps.Scanner.Core/Contracts/SbomView.cs b/src/StellaOps.Scanner.Core/Contracts/SbomView.cs new file mode 100644 index 00000000..9187ab31 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/SbomView.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Scanner.Core.Contracts; + +public enum SbomView +{ + Inventory, + Usage, +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs new file mode 100644 index 00000000..268bf424 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisKeys.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Scanner.Core.Contracts; + +public static class ScanAnalysisKeys +{ + public const string OsPackageAnalyzers = "analysis.os.packages"; + + public const string OsComponentFragments = "analysis.os.fragments"; + + public const string LayerComponentFragments = "analysis.layers.fragments"; +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStore.cs b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStore.cs new file mode 100644 index 00000000..b8b70fb3 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStore.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; + +namespace StellaOps.Scanner.Core.Contracts; + +public sealed class ScanAnalysisStore +{ + private readonly ConcurrentDictionary _items = new(StringComparer.OrdinalIgnoreCase); + + public void Set(string key, T value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentNullException.ThrowIfNull(value); + _items[key] = value!; + } + + public bool TryGet(string key, out T value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + if (_items.TryGetValue(key, out var stored) && stored is T typed) + { + value = typed; + return true; + } + + value = default!; + return false; + } + + public IReadOnlyDictionary Snapshot() + => new ReadOnlyDictionary(new Dictionary(_items, StringComparer.OrdinalIgnoreCase)); +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStoreExtensions.cs b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStoreExtensions.cs new file mode 100644 index 00000000..03b3480a --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanAnalysisStoreExtensions.cs @@ -0,0 +1,41 @@ +using System.Collections.Immutable; +using System.Linq; + +namespace StellaOps.Scanner.Core.Contracts; + +public static class ScanAnalysisStoreExtensions +{ + public static ImmutableArray GetLayerFragments(this ScanAnalysisStore store) + { + ArgumentNullException.ThrowIfNull(store); + + if (store.TryGet>(ScanAnalysisKeys.LayerComponentFragments, out var fragments) && !fragments.IsDefault) + { + return fragments; + } + + return ImmutableArray.Empty; + } + + public static ImmutableArray AppendLayerFragments(this ScanAnalysisStore store, IEnumerable fragments) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(fragments); + + var newFragments = fragments.ToImmutableArray(); + if (newFragments.IsDefaultOrEmpty) + { + return store.GetLayerFragments(); + } + + if (store.TryGet>(ScanAnalysisKeys.LayerComponentFragments, out var existing) && !existing.IsDefaultOrEmpty) + { + var combined = existing.AddRange(newFragments); + store.Set(ScanAnalysisKeys.LayerComponentFragments, combined); + return combined; + } + + store.Set(ScanAnalysisKeys.LayerComponentFragments, newFragments); + return newFragments; + } +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanJob.cs b/src/StellaOps.Scanner.Core/Contracts/ScanJob.cs new file mode 100644 index 00000000..3860bb27 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanJob.cs @@ -0,0 +1,173 @@ +using System.Collections.ObjectModel; +using System.Globalization; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Contracts; + +[JsonConverter(typeof(ScanJobIdJsonConverter))] +public readonly record struct ScanJobId(Guid Value) +{ + public static readonly ScanJobId Empty = new(Guid.Empty); + + public override string ToString() + => Value.ToString("n", CultureInfo.InvariantCulture); + + public static ScanJobId From(Guid value) + => new(value); + + public static bool TryParse(string? text, out ScanJobId id) + { + if (Guid.TryParse(text, out var guid)) + { + id = new ScanJobId(guid); + return true; + } + + id = Empty; + return false; + } +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ScanJobStatus +{ + Unknown = 0, + Pending, + Queued, + Running, + Succeeded, + Failed, + Cancelled +} + +public sealed class ScanJob +{ + private static readonly IReadOnlyDictionary EmptyMetadata = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)); + + [JsonConstructor] + public ScanJob( + ScanJobId id, + ScanJobStatus status, + string imageReference, + string? imageDigest, + DateTimeOffset createdAt, + DateTimeOffset? updatedAt, + string correlationId, + string? tenantId, + IReadOnlyDictionary? metadata = null, + ScannerError? failure = null) + { + if (string.IsNullOrWhiteSpace(imageReference)) + { + throw new ArgumentException("Image reference cannot be null or whitespace.", nameof(imageReference)); + } + + if (string.IsNullOrWhiteSpace(correlationId)) + { + throw new ArgumentException("Correlation identifier cannot be null or whitespace.", nameof(correlationId)); + } + + Id = id; + Status = status; + ImageReference = imageReference.Trim(); + ImageDigest = NormalizeDigest(imageDigest); + CreatedAt = ScannerTimestamps.Normalize(createdAt); + UpdatedAt = updatedAt is null ? null : ScannerTimestamps.Normalize(updatedAt.Value); + CorrelationId = correlationId; + TenantId = string.IsNullOrWhiteSpace(tenantId) ? null : tenantId.Trim(); + Metadata = metadata is null or { Count: 0 } + ? EmptyMetadata + : new ReadOnlyDictionary(new Dictionary(metadata, StringComparer.Ordinal)); + Failure = failure; + } + + [JsonPropertyName("id")] + [JsonPropertyOrder(0)] + public ScanJobId Id { get; } + + [JsonPropertyName("status")] + [JsonPropertyOrder(1)] + public ScanJobStatus Status { get; init; } + + [JsonPropertyName("imageReference")] + [JsonPropertyOrder(2)] + public string ImageReference { get; } + + [JsonPropertyName("imageDigest")] + [JsonPropertyOrder(3)] + public string? ImageDigest { get; } + + [JsonPropertyName("createdAt")] + [JsonPropertyOrder(4)] + public DateTimeOffset CreatedAt { get; } + + [JsonPropertyName("updatedAt")] + [JsonPropertyOrder(5)] + public DateTimeOffset? UpdatedAt { get; init; } + + [JsonPropertyName("correlationId")] + [JsonPropertyOrder(6)] + public string CorrelationId { get; } + + [JsonPropertyName("tenantId")] + [JsonPropertyOrder(7)] + public string? TenantId { get; } + + [JsonPropertyName("metadata")] + [JsonPropertyOrder(8)] + public IReadOnlyDictionary Metadata { get; } + + [JsonPropertyName("failure")] + [JsonPropertyOrder(9)] + public ScannerError? Failure { get; init; } + + public ScanJob WithStatus(ScanJobStatus status, DateTimeOffset? updatedAt = null) + => new( + Id, + status, + ImageReference, + ImageDigest, + CreatedAt, + updatedAt ?? UpdatedAt ?? CreatedAt, + CorrelationId, + TenantId, + Metadata, + Failure); + + public ScanJob WithFailure(ScannerError failure, DateTimeOffset? updatedAt = null, TimeProvider? timeProvider = null) + => new( + Id, + ScanJobStatus.Failed, + ImageReference, + ImageDigest, + CreatedAt, + updatedAt ?? ScannerTimestamps.UtcNow(timeProvider), + CorrelationId, + TenantId, + Metadata, + failure); + + private static string? NormalizeDigest(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var trimmed = digest.Trim(); + if (!trimmed.StartsWith("sha", StringComparison.OrdinalIgnoreCase)) + { + return trimmed; + } + + var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return trimmed.ToLowerInvariant(); + } + + return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}"; + } +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanJobIdJsonConverter.cs b/src/StellaOps.Scanner.Core/Contracts/ScanJobIdJsonConverter.cs new file mode 100644 index 00000000..81980df5 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanJobIdJsonConverter.cs @@ -0,0 +1,26 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Core.Contracts; + +internal sealed class ScanJobIdJsonConverter : JsonConverter +{ + public override ScanJobId Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.String) + { + throw new JsonException("Expected scan job identifier to be a string."); + } + + var value = reader.GetString(); + if (!ScanJobId.TryParse(value, out var id)) + { + throw new JsonException("Invalid scan job identifier."); + } + + return id; + } + + public override void Write(Utf8JsonWriter writer, ScanJobId value, JsonSerializerOptions options) + => writer.WriteStringValue(value.ToString()); +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanMetadataKeys.cs b/src/StellaOps.Scanner.Core/Contracts/ScanMetadataKeys.cs new file mode 100644 index 00000000..5628fc30 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanMetadataKeys.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Scanner.Core.Contracts; + +public static class ScanMetadataKeys +{ + public const string RootFilesystemPath = "scanner.rootfs.path"; + public const string WorkspacePath = "scanner.workspace.path"; +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScanProgressEvent.cs b/src/StellaOps.Scanner.Core/Contracts/ScanProgressEvent.cs new file mode 100644 index 00000000..9c5a71ae --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScanProgressEvent.cs @@ -0,0 +1,121 @@ +using System.Collections.ObjectModel; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Contracts; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ScanStage +{ + Unknown = 0, + ResolveImage, + FetchLayers, + MountLayers, + AnalyzeOperatingSystem, + AnalyzeLanguageEcosystems, + AnalyzeNativeArtifacts, + ComposeSbom, + BuildDiffs, + EmitArtifacts, + SignArtifacts, + Complete +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ScanProgressEventKind +{ + Progress = 0, + StageStarted, + StageCompleted, + Warning, + Error +} + +public sealed class ScanProgressEvent +{ + private static readonly IReadOnlyDictionary EmptyAttributes = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)); + + [JsonConstructor] + public ScanProgressEvent( + ScanJobId jobId, + ScanStage stage, + ScanProgressEventKind kind, + int sequence, + DateTimeOffset timestamp, + double? percentComplete = null, + string? message = null, + IReadOnlyDictionary? attributes = null, + ScannerError? error = null) + { + if (sequence < 0) + { + throw new ArgumentOutOfRangeException(nameof(sequence), sequence, "Sequence cannot be negative."); + } + + JobId = jobId; + Stage = stage; + Kind = kind; + Sequence = sequence; + Timestamp = ScannerTimestamps.Normalize(timestamp); + PercentComplete = percentComplete is < 0 or > 100 ? null : percentComplete; + Message = message is { Length: > 0 } ? message.Trim() : null; + Attributes = attributes is null or { Count: 0 } + ? EmptyAttributes + : new ReadOnlyDictionary(new Dictionary(attributes, StringComparer.Ordinal)); + Error = error; + } + + [JsonPropertyName("jobId")] + [JsonPropertyOrder(0)] + public ScanJobId JobId { get; } + + [JsonPropertyName("stage")] + [JsonPropertyOrder(1)] + public ScanStage Stage { get; } + + [JsonPropertyName("kind")] + [JsonPropertyOrder(2)] + public ScanProgressEventKind Kind { get; } + + [JsonPropertyName("sequence")] + [JsonPropertyOrder(3)] + public int Sequence { get; } + + [JsonPropertyName("timestamp")] + [JsonPropertyOrder(4)] + public DateTimeOffset Timestamp { get; } + + [JsonPropertyName("percentComplete")] + [JsonPropertyOrder(5)] + public double? PercentComplete { get; } + + [JsonPropertyName("message")] + [JsonPropertyOrder(6)] + public string? Message { get; } + + [JsonPropertyName("attributes")] + [JsonPropertyOrder(7)] + public IReadOnlyDictionary Attributes { get; } + + [JsonPropertyName("error")] + [JsonPropertyOrder(8)] + public ScannerError? Error { get; } + + public ScanProgressEvent With( + ScanProgressEventKind? kind = null, + double? percentComplete = null, + string? message = null, + IReadOnlyDictionary? attributes = null, + ScannerError? error = null) + => new( + JobId, + Stage, + kind ?? Kind, + Sequence, + Timestamp, + percentComplete ?? PercentComplete, + message ?? Message, + attributes ?? Attributes, + error ?? Error); +} diff --git a/src/StellaOps.Scanner.Core/Contracts/ScannerError.cs b/src/StellaOps.Scanner.Core/Contracts/ScannerError.cs new file mode 100644 index 00000000..2729bd44 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Contracts/ScannerError.cs @@ -0,0 +1,110 @@ +using System.Collections.ObjectModel; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Contracts; + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ScannerErrorCode +{ + Unknown = 0, + InvalidImageReference, + ImageNotFound, + AuthorizationFailed, + QueueUnavailable, + StorageUnavailable, + AnalyzerFailure, + ExportFailure, + SigningFailure, + RuntimeFailure, + Timeout, + Cancelled, + PluginViolation +} + +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum ScannerErrorSeverity +{ + Warning = 0, + Error, + Fatal +} + +public sealed class ScannerError +{ + private static readonly IReadOnlyDictionary EmptyDetails = + new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)); + + [JsonConstructor] + public ScannerError( + ScannerErrorCode code, + ScannerErrorSeverity severity, + string message, + DateTimeOffset timestamp, + bool retryable, + IReadOnlyDictionary? details = null, + string? stage = null, + string? component = null) + { + if (string.IsNullOrWhiteSpace(message)) + { + throw new ArgumentException("Error message cannot be null or whitespace.", nameof(message)); + } + + Code = code; + Severity = severity; + Message = message.Trim(); + Timestamp = ScannerTimestamps.Normalize(timestamp); + Retryable = retryable; + Stage = stage; + Component = component; + Details = details is null or { Count: 0 } + ? EmptyDetails + : new ReadOnlyDictionary(new Dictionary(details, StringComparer.Ordinal)); + } + + [JsonPropertyName("code")] + [JsonPropertyOrder(0)] + public ScannerErrorCode Code { get; } + + [JsonPropertyName("severity")] + [JsonPropertyOrder(1)] + public ScannerErrorSeverity Severity { get; } + + [JsonPropertyName("message")] + [JsonPropertyOrder(2)] + public string Message { get; } + + [JsonPropertyName("timestamp")] + [JsonPropertyOrder(3)] + public DateTimeOffset Timestamp { get; } + + [JsonPropertyName("retryable")] + [JsonPropertyOrder(4)] + public bool Retryable { get; } + + [JsonPropertyName("stage")] + [JsonPropertyOrder(5)] + public string? Stage { get; } + + [JsonPropertyName("component")] + [JsonPropertyOrder(6)] + public string? Component { get; } + + [JsonPropertyName("details")] + [JsonPropertyOrder(7)] + public IReadOnlyDictionary Details { get; } + + public ScannerError WithDetail(string key, string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(key); + ArgumentException.ThrowIfNullOrWhiteSpace(value); + + var mutable = new Dictionary(Details, StringComparer.Ordinal) + { + [key] = value + }; + + return new ScannerError(Code, Severity, Message, Timestamp, Retryable, mutable, Stage, Component); + } +} diff --git a/src/StellaOps.Scanner.Core/Observability/ScannerCorrelationContext.cs b/src/StellaOps.Scanner.Core/Observability/ScannerCorrelationContext.cs new file mode 100644 index 00000000..dbd937a9 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Observability/ScannerCorrelationContext.cs @@ -0,0 +1,80 @@ +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Observability; + +public readonly record struct ScannerCorrelationContext( + ScanJobId JobId, + string CorrelationId, + string? Stage, + string? Component, + string? Audience = null) +{ + public static ScannerCorrelationContext Create( + ScanJobId jobId, + string? stage = null, + string? component = null, + string? audience = null) + { + var correlationId = ScannerIdentifiers.CreateCorrelationId(jobId, stage, component); + return new ScannerCorrelationContext(jobId, correlationId, stage, component, audience); + } + + public string DeterministicHash() + => ScannerIdentifiers.CreateDeterministicHash( + JobId.ToString(), + Stage ?? string.Empty, + Component ?? string.Empty, + Audience ?? string.Empty); +} + +public static class ScannerCorrelationContextAccessor +{ + private static readonly AsyncLocal CurrentContext = new(); + + public static ScannerCorrelationContext? Current => CurrentContext.Value; + + public static IDisposable Push(in ScannerCorrelationContext context) + { + var previous = CurrentContext.Value; + CurrentContext.Value = context; + return new DisposableScope(() => CurrentContext.Value = previous); + } + + public static bool TryGetCorrelationId([NotNullWhen(true)] out string? correlationId) + { + var context = CurrentContext.Value; + if (context.HasValue) + { + correlationId = context.Value.CorrelationId; + return true; + } + + correlationId = null; + return false; + } + + private sealed class DisposableScope : IDisposable + { + private readonly Action release; + private bool disposed; + + public DisposableScope(Action release) + { + this.release = release ?? throw new ArgumentNullException(nameof(release)); + } + + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + release(); + } + } +} diff --git a/src/StellaOps.Scanner.Core/Observability/ScannerDiagnostics.cs b/src/StellaOps.Scanner.Core/Observability/ScannerDiagnostics.cs new file mode 100644 index 00000000..f36cda40 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Observability/ScannerDiagnostics.cs @@ -0,0 +1,55 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Observability; + +public static class ScannerDiagnostics +{ + public const string ActivitySourceName = "StellaOps.Scanner"; + public const string ActivityVersion = "1.0.0"; + public const string MeterName = "stellaops.scanner"; + public const string MeterVersion = "1.0.0"; + + public static ActivitySource ActivitySource { get; } = new(ActivitySourceName, ActivityVersion); + public static Meter Meter { get; } = new(MeterName, MeterVersion); + + public static Activity? StartActivity( + string name, + ScanJobId jobId, + string? stage = null, + string? component = null, + ActivityKind kind = ActivityKind.Internal, + IEnumerable>? tags = null) + { + var activity = ActivitySource.StartActivity(name, kind); + if (activity is null) + { + return null; + } + + activity.SetTag("stellaops.scanner.job_id", jobId.ToString()); + activity.SetTag("stellaops.scanner.correlation_id", ScannerIdentifiers.CreateCorrelationId(jobId, stage, component)); + + if (!string.IsNullOrWhiteSpace(stage)) + { + activity.SetTag("stellaops.scanner.stage", stage); + } + + if (!string.IsNullOrWhiteSpace(component)) + { + activity.SetTag("stellaops.scanner.component", component); + } + + if (tags is not null) + { + foreach (var tag in tags) + { + activity?.SetTag(tag.Key, tag.Value); + } + } + + return activity; + } +} diff --git a/src/StellaOps.Scanner.Core/Observability/ScannerLogExtensions.cs b/src/StellaOps.Scanner.Core/Observability/ScannerLogExtensions.cs new file mode 100644 index 00000000..a852c51f --- /dev/null +++ b/src/StellaOps.Scanner.Core/Observability/ScannerLogExtensions.cs @@ -0,0 +1,115 @@ +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Observability; + +public static class ScannerLogExtensions +{ + private sealed class NoopScope : IDisposable + { + public static NoopScope Instance { get; } = new(); + + public void Dispose() + { + } + } + + private sealed class CompositeScope : IDisposable + { + private readonly IDisposable first; + private readonly IDisposable second; + private bool disposed; + + public CompositeScope(IDisposable first, IDisposable second) + { + this.first = first; + this.second = second; + } + + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + second.Dispose(); + first.Dispose(); + } + } + + public static IDisposable BeginScanScope(this ILogger? logger, ScanJob job, string? stage = null, string? component = null) + { + var correlation = ScannerCorrelationContext.Create(job.Id, stage, component); + var logScope = logger is null + ? NoopScope.Instance + : logger.BeginScope(CreateScopeState( + job.Id, + job.CorrelationId, + stage, + component, + job.TenantId, + job.ImageDigest)) ?? NoopScope.Instance; + + var correlationScope = ScannerCorrelationContextAccessor.Push(correlation); + return new CompositeScope(logScope, correlationScope); + } + + public static IDisposable BeginProgressScope(this ILogger? logger, ScanProgressEvent progress, string? component = null) + { + var correlationId = ScannerIdentifiers.CreateCorrelationId(progress.JobId, progress.Stage.ToString(), component); + var correlation = new ScannerCorrelationContext(progress.JobId, correlationId, progress.Stage.ToString(), component); + + var logScope = logger is null + ? NoopScope.Instance + : logger.BeginScope(new Dictionary(6, StringComparer.Ordinal) + { + ["scanId"] = progress.JobId.ToString(), + ["stage"] = progress.Stage.ToString(), + ["sequence"] = progress.Sequence, + ["kind"] = progress.Kind.ToString(), + ["correlationId"] = correlationId, + ["component"] = component ?? string.Empty + }) ?? NoopScope.Instance; + + var correlationScope = ScannerCorrelationContextAccessor.Push(correlation); + return new CompositeScope(logScope, correlationScope); + } + + public static IDisposable BeginCorrelationScope(this ILogger? logger, ScannerCorrelationContext context) + { + var scope = logger is null + ? NoopScope.Instance + : logger.BeginScope(CreateScopeState(context.JobId, context.CorrelationId, context.Stage, context.Component, null, null)) ?? NoopScope.Instance; + + var correlationScope = ScannerCorrelationContextAccessor.Push(context); + return new CompositeScope(scope, correlationScope); + } + + private static Dictionary CreateScopeState( + ScanJobId jobId, + string correlationId, + string? stage, + string? component, + string? tenantId, + string? imageDigest) + { + var state = new Dictionary(6, StringComparer.Ordinal) + { + ["scanId"] = jobId.ToString(), + ["correlationId"] = correlationId, + ["stage"] = stage ?? string.Empty, + ["component"] = component ?? string.Empty, + ["tenantId"] = tenantId ?? string.Empty + }; + + if (!string.IsNullOrEmpty(imageDigest)) + { + state["imageDigest"] = imageDigest; + } + + return state; + } +} diff --git a/src/StellaOps.Scanner.Core/Observability/ScannerMetricNames.cs b/src/StellaOps.Scanner.Core/Observability/ScannerMetricNames.cs new file mode 100644 index 00000000..acb6c65b --- /dev/null +++ b/src/StellaOps.Scanner.Core/Observability/ScannerMetricNames.cs @@ -0,0 +1,55 @@ +using System.Collections.Frozen; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Observability; + +public static class ScannerMetricNames +{ + public const string Prefix = "stellaops.scanner"; + public const string QueueLatency = $"{Prefix}.queue.latency"; + public const string QueueDepth = $"{Prefix}.queue.depth"; + public const string StageDuration = $"{Prefix}.stage.duration"; + public const string StageProgress = $"{Prefix}.stage.progress"; + public const string JobCount = $"{Prefix}.jobs.count"; + public const string JobFailures = $"{Prefix}.jobs.failures"; + public const string ArtifactBytes = $"{Prefix}.artifacts.bytes"; + + public static FrozenDictionary BuildJobTags(ScanJob job, string? stage = null, string? component = null) + { + ArgumentNullException.ThrowIfNull(job); + + var builder = new Dictionary(6, StringComparer.Ordinal) + { + ["jobId"] = job.Id.ToString(), + ["stage"] = stage ?? string.Empty, + ["component"] = component ?? string.Empty, + ["tenantId"] = job.TenantId ?? string.Empty, + ["correlationId"] = job.CorrelationId, + ["status"] = job.Status.ToString() + }; + + if (!string.IsNullOrEmpty(job.ImageDigest)) + { + builder["imageDigest"] = job.ImageDigest; + } + + return builder.ToFrozenDictionary(StringComparer.Ordinal); + } + + public static FrozenDictionary BuildEventTags(ScanProgressEvent progress) + { + ArgumentNullException.ThrowIfNull(progress); + + var builder = new Dictionary(5, StringComparer.Ordinal) + { + ["jobId"] = progress.JobId.ToString(), + ["stage"] = progress.Stage.ToString(), + ["kind"] = progress.Kind.ToString(), + ["sequence"] = progress.Sequence, + ["correlationId"] = ScannerIdentifiers.CreateCorrelationId(progress.JobId, progress.Stage.ToString()) + }; + + return builder.ToFrozenDictionary(StringComparer.Ordinal); + } +} diff --git a/src/StellaOps.Scanner.Core/Security/AuthorityTokenSource.cs b/src/StellaOps.Scanner.Core/Security/AuthorityTokenSource.cs new file mode 100644 index 00000000..4a9bcf6d --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/AuthorityTokenSource.cs @@ -0,0 +1,128 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Client; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Core.Security; + +public sealed class AuthorityTokenSource : IAuthorityTokenSource +{ + private readonly IStellaOpsTokenClient tokenClient; + private readonly TimeProvider timeProvider; + private readonly TimeSpan refreshSkew; + private readonly ILogger? logger; + private readonly ConcurrentDictionary cache = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary locks = new(StringComparer.Ordinal); + + public AuthorityTokenSource( + IStellaOpsTokenClient tokenClient, + TimeSpan? refreshSkew = null, + TimeProvider? timeProvider = null, + ILogger? logger = null) + { + this.tokenClient = tokenClient ?? throw new ArgumentNullException(nameof(tokenClient)); + this.timeProvider = timeProvider ?? TimeProvider.System; + this.logger = logger; + this.refreshSkew = refreshSkew is { } value && value > TimeSpan.Zero ? value : TimeSpan.FromSeconds(30); + } + + public async ValueTask GetAsync(string audience, IEnumerable scopes, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + + var normalizedAudience = NormalizeAudience(audience); + var normalizedScopes = NormalizeScopes(scopes, normalizedAudience); + var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes); + + if (cache.TryGetValue(cacheKey, out var cached) && !cached.Token.IsExpired(timeProvider, refreshSkew)) + { + return cached.Token; + } + + var mutex = locks.GetOrAdd(cacheKey, static _ => new SemaphoreSlim(1, 1)); + await mutex.WaitAsync(cancellationToken).ConfigureAwait(false); + + try + { + if (cache.TryGetValue(cacheKey, out cached) && !cached.Token.IsExpired(timeProvider, refreshSkew)) + { + return cached.Token; + } + + var scopeString = string.Join(' ', normalizedScopes); + var tokenResult = await tokenClient.RequestClientCredentialsTokenAsync(scopeString, cancellationToken).ConfigureAwait(false); + + var token = ScannerOperationalToken.FromResult( + tokenResult.AccessToken, + tokenResult.TokenType, + tokenResult.ExpiresAtUtc, + tokenResult.Scopes); + + cache[cacheKey] = new CacheEntry(token); + logger?.LogDebug( + "Issued new scanner OpTok for audience {Audience} with scopes {Scopes}; expires at {ExpiresAt}.", + normalizedAudience, + scopeString, + token.ExpiresAt); + + return token; + } + finally + { + mutex.Release(); + } + } + + public ValueTask InvalidateAsync(string audience, IEnumerable scopes, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrWhiteSpace(audience); + + var normalizedAudience = NormalizeAudience(audience); + var normalizedScopes = NormalizeScopes(scopes, normalizedAudience); + var cacheKey = BuildCacheKey(normalizedAudience, normalizedScopes); + + cache.TryRemove(cacheKey, out _); + if (locks.TryRemove(cacheKey, out var mutex)) + { + mutex.Dispose(); + } + + logger?.LogDebug("Invalidated cached OpTok for {Audience} ({CacheKey}).", normalizedAudience, cacheKey); + return ValueTask.CompletedTask; + } + + private static string NormalizeAudience(string audience) + => audience.Trim().ToLowerInvariant(); + + private static IReadOnlyList NormalizeScopes(IEnumerable scopes, string audience) + { + var set = new SortedSet(StringComparer.Ordinal) + { + $"aud:{audience}" + }; + + if (scopes is not null) + { + foreach (var scope in scopes) + { + if (string.IsNullOrWhiteSpace(scope)) + { + continue; + } + + set.Add(scope.Trim()); + } + } + + return set.ToArray(); + } + + private static string BuildCacheKey(string audience, IReadOnlyList scopes) + => ScannerIdentifiers.CreateDeterministicHash(audience, string.Join(' ', scopes)); + + private readonly record struct CacheEntry(ScannerOperationalToken Token); +} diff --git a/src/StellaOps.Scanner.Core/Security/IAuthorityTokenSource.cs b/src/StellaOps.Scanner.Core/Security/IAuthorityTokenSource.cs new file mode 100644 index 00000000..4913050c --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/IAuthorityTokenSource.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Core.Security; + +public interface IAuthorityTokenSource +{ + ValueTask GetAsync(string audience, IEnumerable scopes, CancellationToken cancellationToken = default); + + ValueTask InvalidateAsync(string audience, IEnumerable scopes, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.Core/Security/IPluginCatalogGuard.cs b/src/StellaOps.Scanner.Core/Security/IPluginCatalogGuard.cs new file mode 100644 index 00000000..3b26d8bf --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/IPluginCatalogGuard.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Scanner.Core.Security; + +public interface IPluginCatalogGuard +{ + IReadOnlyCollection KnownPlugins { get; } + + bool IsSealed { get; } + + void EnsureRegistrationAllowed(string pluginPath); + + void Seal(); +} diff --git a/src/StellaOps.Scanner.Core/Security/RestartOnlyPluginGuard.cs b/src/StellaOps.Scanner.Core/Security/RestartOnlyPluginGuard.cs new file mode 100644 index 00000000..8faa25fc --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/RestartOnlyPluginGuard.cs @@ -0,0 +1,53 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; + +namespace StellaOps.Scanner.Core.Security; + +public sealed class RestartOnlyPluginGuard : IPluginCatalogGuard +{ + private readonly ConcurrentDictionary plugins = new(StringComparer.OrdinalIgnoreCase); + private bool sealedState; + + public RestartOnlyPluginGuard(IEnumerable? initialPlugins = null) + { + if (initialPlugins is not null) + { + foreach (var plugin in initialPlugins) + { + var normalized = Normalize(plugin); + plugins.TryAdd(normalized, 0); + } + } + } + + public IReadOnlyCollection KnownPlugins => plugins.Keys.ToArray(); + + public bool IsSealed => Volatile.Read(ref sealedState); + + public void EnsureRegistrationAllowed(string pluginPath) + { + ArgumentException.ThrowIfNullOrWhiteSpace(pluginPath); + + var normalized = Normalize(pluginPath); + if (IsSealed && !plugins.ContainsKey(normalized)) + { + throw new InvalidOperationException($"Plug-in '{pluginPath}' cannot be registered after startup. Restart required."); + } + + plugins.TryAdd(normalized, 0); + } + + public void Seal() + { + Volatile.Write(ref sealedState, true); + } + + private static string Normalize(string path) + { + var full = Path.GetFullPath(path); + return full.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + } +} diff --git a/src/StellaOps.Scanner.Core/Security/ScannerOperationalToken.cs b/src/StellaOps.Scanner.Core/Security/ScannerOperationalToken.cs new file mode 100644 index 00000000..3c7e609b --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/ScannerOperationalToken.cs @@ -0,0 +1,66 @@ +using System.Collections.ObjectModel; +using System.Linq; + +namespace StellaOps.Scanner.Core.Security; + +public readonly record struct ScannerOperationalToken( + string AccessToken, + string TokenType, + DateTimeOffset ExpiresAt, + IReadOnlyList Scopes) +{ + public bool IsExpired(TimeProvider timeProvider, TimeSpan refreshSkew) + { + ArgumentNullException.ThrowIfNull(timeProvider); + + var now = timeProvider.GetUtcNow(); + return now >= ExpiresAt - refreshSkew; + } + + public static ScannerOperationalToken FromResult( + string accessToken, + string tokenType, + DateTimeOffset expiresAt, + IEnumerable scopes) + { + ArgumentException.ThrowIfNullOrWhiteSpace(accessToken); + ArgumentException.ThrowIfNullOrWhiteSpace(tokenType); + + IReadOnlyList normalized = scopes switch + { + null => Array.Empty(), + IReadOnlyList readOnly => readOnly.Count == 0 ? Array.Empty() : readOnly, + ICollection collection => NormalizeCollection(collection), + _ => NormalizeEnumerable(scopes) + }; + + return new ScannerOperationalToken( + accessToken, + tokenType, + expiresAt, + normalized); + } + + private static IReadOnlyList NormalizeCollection(ICollection collection) + { + if (collection.Count == 0) + { + return Array.Empty(); + } + + if (collection is IReadOnlyList readOnly) + { + return readOnly; + } + + var buffer = new string[collection.Count]; + collection.CopyTo(buffer, 0); + return new ReadOnlyCollection(buffer); + } + + private static IReadOnlyList NormalizeEnumerable(IEnumerable scopes) + { + var buffer = scopes.ToArray(); + return buffer.Length == 0 ? Array.Empty() : new ReadOnlyCollection(buffer); + } +} diff --git a/src/StellaOps.Scanner.Core/Security/ServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Core/Security/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..b2947a63 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Security/ServiceCollectionExtensions.cs @@ -0,0 +1,36 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Auth.Client; +using StellaOps.Auth.Security.Dpop; + +namespace StellaOps.Scanner.Core.Security; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddScannerAuthorityCore( + this IServiceCollection services, + Action configureAuthority, + Action? configureDpop = null) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configureAuthority); + + services.AddStellaOpsAuthClient(configureAuthority); + + if (configureDpop is not null) + { + services.AddOptions().Configure(configureDpop).PostConfigure(static options => options.Validate()); + } + else + { + services.AddOptions().PostConfigure(static options => options.Validate()); + } + + services.TryAddSingleton(provider => new InMemoryDpopReplayCache(provider.GetService())); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + return services; + } +} diff --git a/src/StellaOps.Scanner.Core/Serialization/ScannerJsonOptions.cs b/src/StellaOps.Scanner.Core/Serialization/ScannerJsonOptions.cs new file mode 100644 index 00000000..437f8c23 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Serialization/ScannerJsonOptions.cs @@ -0,0 +1,21 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Core.Serialization; + +public static class ScannerJsonOptions +{ + public static JsonSerializerOptions Default { get; } = CreateDefault(); + + public static JsonSerializerOptions CreateDefault(bool indent = false) + { + var options = new JsonSerializerOptions(JsonSerializerDefaults.Web) + { + WriteIndented = indent + }; + + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)); + + return options; + } +} diff --git a/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj b/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj new file mode 100644 index 00000000..50818b33 --- /dev/null +++ b/src/StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Core/TASKS.md b/src/StellaOps.Scanner.Core/TASKS.md new file mode 100644 index 00000000..a4a90224 --- /dev/null +++ b/src/StellaOps.Scanner.Core/TASKS.md @@ -0,0 +1,7 @@ +# Scanner Core Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-CORE-09-501 | DONE (2025-10-19) | Scanner Core Guild | — | Define shared DTOs (ScanJob, ProgressEvent), error taxonomy, and deterministic ID/timestamp helpers aligning with `ARCHITECTURE_SCANNER.md` §3–§4.
          2025-10-19: Added golden fixtures + `ScannerCoreContractsTests` to lock canonical JSON.
          2025-10-19: Published canonical JSON snippet + acceptance notes in `docs/scanner-core-contracts.md`. | DTOs serialize deterministically, helpers produce reproducible IDs/timestamps, tests cover round-trips and hash derivation. | +| SCANNER-CORE-09-502 | DONE (2025-10-19) | Scanner Core Guild | SCANNER-CORE-09-501 | Observability helpers (correlation IDs, logging scopes, metric namespacing, deterministic hashes) consumed by WebService/Worker.
          2025-10-19: Verified progress scope serialisation via new fixtures/tests.
          2025-10-19: Added `ScannerLogExtensionsPerformanceTests` to enforce ≤ 5 µs scope overhead + documented micro-bench results. | Logging/metrics helpers allocate minimally, correlation IDs stable, ActivitySource emitted; tests assert determinism. | +| SCANNER-CORE-09-503 | DONE (2025-10-18) | Scanner Core Guild | SCANNER-CORE-09-501, SCANNER-CORE-09-502 | Security utilities: Authority client factory, OpTok caching, DPoP verifier, restart-time plug-in guardrails for scanner components. | Authority helpers cache tokens, DPoP validator rejects invalid proofs, plug-in guard prevents runtime additions; tests cover happy/error paths. | diff --git a/src/StellaOps.Scanner.Core/Utility/ScannerIdentifiers.cs b/src/StellaOps.Scanner.Core/Utility/ScannerIdentifiers.cs new file mode 100644 index 00000000..36bdd064 --- /dev/null +++ b/src/StellaOps.Scanner.Core/Utility/ScannerIdentifiers.cs @@ -0,0 +1,136 @@ +using System.Globalization; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Core.Utility; + +public static class ScannerIdentifiers +{ + private static readonly Guid ScanJobNamespace = new("d985aa76-8c2b-4cba-bac0-c98c90674f04"); + private static readonly Guid CorrelationNamespace = new("7cde18f5-729e-4ea1-be3d-46fda4c55e38"); + + public static ScanJobId CreateJobId( + string imageReference, + string? imageDigest = null, + string? tenantId = null, + string? salt = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(imageReference); + + var normalizedReference = NormalizeImageReference(imageReference); + var normalizedDigest = NormalizeDigest(imageDigest) ?? "none"; + var normalizedTenant = string.IsNullOrWhiteSpace(tenantId) ? "global" : tenantId.Trim().ToLowerInvariant(); + var normalizedSalt = (salt?.Trim() ?? string.Empty).ToLowerInvariant(); + + using var sha256 = SHA256.Create(); + var payload = $"{normalizedReference}|{normalizedDigest}|{normalizedTenant}|{normalizedSalt}"; + var hashed = sha256.ComputeHash(Encoding.UTF8.GetBytes(payload)); + return new ScanJobId(CreateGuidFromHash(ScanJobNamespace, hashed)); + } + + public static string CreateCorrelationId(ScanJobId jobId, string? stage = null, string? suffix = null) + { + var normalizedStage = string.IsNullOrWhiteSpace(stage) + ? "scan" + : stage.Trim().ToLowerInvariant().Replace(' ', '-'); + + var normalizedSuffix = string.IsNullOrWhiteSpace(suffix) + ? string.Empty + : "-" + suffix.Trim().ToLowerInvariant().Replace(' ', '-'); + + return $"scan-{normalizedStage}-{jobId}{normalizedSuffix}"; + } + + public static string CreateDeterministicHash(params string[] segments) + { + if (segments is null || segments.Length == 0) + { + throw new ArgumentException("At least one segment must be provided.", nameof(segments)); + } + + using var sha256 = SHA256.Create(); + var joined = string.Join('|', segments.Select(static s => s?.Trim() ?? string.Empty)); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(joined)); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + public static Guid CreateDeterministicGuid(Guid namespaceId, ReadOnlySpan nameBytes) + { + Span namespaceBytes = stackalloc byte[16]; + namespaceId.TryWriteBytes(namespaceBytes); + + Span buffer = stackalloc byte[namespaceBytes.Length + nameBytes.Length]; + namespaceBytes.CopyTo(buffer); + nameBytes.CopyTo(buffer[namespaceBytes.Length..]); + + Span hash = stackalloc byte[32]; + SHA256.TryHashData(buffer, hash, out _); + + Span guidBytes = stackalloc byte[16]; + hash[..16].CopyTo(guidBytes); + + guidBytes[6] = (byte)((guidBytes[6] & 0x0F) | 0x50); + guidBytes[8] = (byte)((guidBytes[8] & 0x3F) | 0x80); + + return new Guid(guidBytes); + } + + public static string NormalizeImageReference(string reference) + { + ArgumentException.ThrowIfNullOrWhiteSpace(reference); + var trimmed = reference.Trim(); + var atIndex = trimmed.IndexOf('@'); + if (atIndex > 0) + { + var prefix = trimmed[..atIndex].ToLowerInvariant(); + return $"{prefix}{trimmed[atIndex..]}"; + } + + var colonIndex = trimmed.IndexOf(':'); + if (colonIndex > 0) + { + var name = trimmed[..colonIndex].ToLowerInvariant(); + var tag = trimmed[(colonIndex + 1)..]; + return $"{name}:{tag}"; + } + + return trimmed.ToLowerInvariant(); + } + + public static string? NormalizeDigest(string? digest) + { + if (string.IsNullOrWhiteSpace(digest)) + { + return null; + } + + var trimmed = digest.Trim(); + var parts = trimmed.Split(':', 2, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + return trimmed.ToLowerInvariant(); + } + + return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}"; + } + + public static string CreateDeterministicCorrelation(string audience, ScanJobId jobId, string? component = null) + { + using var sha256 = SHA256.Create(); + var payload = $"{audience.Trim().ToLowerInvariant()}|{jobId}|{component?.Trim().ToLowerInvariant() ?? string.Empty}"; + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(payload)); + var guid = CreateGuidFromHash(CorrelationNamespace, hash); + return $"corr-{guid.ToString("n", CultureInfo.InvariantCulture)}"; + } + + private static Guid CreateGuidFromHash(Guid namespaceId, ReadOnlySpan hash) + { + Span guidBytes = stackalloc byte[16]; + hash[..16].CopyTo(guidBytes); + guidBytes[6] = (byte)((guidBytes[6] & 0x0F) | 0x50); + guidBytes[8] = (byte)((guidBytes[8] & 0x3F) | 0x80); + return new Guid(guidBytes); + } +} diff --git a/src/StellaOps.Scanner.Core/Utility/ScannerTimestamps.cs b/src/StellaOps.Scanner.Core/Utility/ScannerTimestamps.cs new file mode 100644 index 00000000..15b259ea --- /dev/null +++ b/src/StellaOps.Scanner.Core/Utility/ScannerTimestamps.cs @@ -0,0 +1,43 @@ +using System.Globalization; + +namespace StellaOps.Scanner.Core.Utility; + +public static class ScannerTimestamps +{ + private const long TicksPerMicrosecond = TimeSpan.TicksPerMillisecond / 1000; + + public static DateTimeOffset Normalize(DateTimeOffset value) + { + var utc = value.ToUniversalTime(); + var ticks = utc.Ticks - (utc.Ticks % TicksPerMicrosecond); + return new DateTimeOffset(ticks, TimeSpan.Zero); + } + + public static DateTimeOffset UtcNow(TimeProvider? provider = null) + => Normalize((provider ?? TimeProvider.System).GetUtcNow()); + + public static string ToIso8601(DateTimeOffset value) + => Normalize(value).ToString("yyyy-MM-dd'T'HH:mm:ss.ffffff'Z'", CultureInfo.InvariantCulture); + + public static bool TryParseIso8601(string? value, out DateTimeOffset timestamp) + { + if (string.IsNullOrWhiteSpace(value)) + { + timestamp = default; + return false; + } + + if (DateTimeOffset.TryParse( + value, + CultureInfo.InvariantCulture, + DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, + out var parsed)) + { + timestamp = Normalize(parsed); + return true; + } + + timestamp = default; + return false; + } +} diff --git a/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs b/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs new file mode 100644 index 00000000..843ef021 --- /dev/null +++ b/src/StellaOps.Scanner.Diff.Tests/ComponentDifferTests.cs @@ -0,0 +1,295 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Diff; +using StellaOps.Scanner.Core.Utility; +using Xunit; + +namespace StellaOps.Scanner.Diff.Tests; + +public sealed class ComponentDifferTests +{ + [Fact] + public void Compute_CapturesAddedRemovedAndChangedComponents() + { + var oldFragments = new[] + { + LayerComponentFragment.Create("sha256:layer1", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:layer1", + usage: ComponentUsage.Create(true, new[] { "/app/start.sh" }), + evidence: new[] { ComponentEvidence.FromPath("/app/package-lock.json") }), + CreateComponent("pkg:npm/b", version: "2.0.0", layer: "sha256:layer1", scope: "runtime"), + }), + LayerComponentFragment.Create("sha256:layer1b", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:layer1b", + usage: ComponentUsage.Create(true, new[] { "/app/start.sh" })), + CreateComponent("pkg:npm/d", version: "0.9.0", layer: "sha256:layer1b"), + }) + }; + + var newFragments = new[] + { + LayerComponentFragment.Create("sha256:layer2", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.1.0", + layer: "sha256:layer2", + usage: ComponentUsage.Create(true, new[] { "/app/start.sh" }), + evidence: new[] { ComponentEvidence.FromPath("/app/package-lock.json") }), + }), + LayerComponentFragment.Create("sha256:layer3", new[] + { + CreateComponent( + "pkg:npm/b", + version: "2.0.0", + layer: "sha256:layer3", + usage: ComponentUsage.Create(true, new[] { "/app/init.sh" }), + scope: "runtime"), + CreateComponent("pkg:npm/c", version: "3.0.0", layer: "sha256:layer3"), + }) + }; + + var oldGraph = ComponentGraphBuilder.Build(oldFragments); + var newGraph = ComponentGraphBuilder.Build(newFragments); + + var request = new ComponentDiffRequest + { + OldGraph = oldGraph, + NewGraph = newGraph, + GeneratedAt = new DateTimeOffset(2025, 10, 19, 10, 0, 0, TimeSpan.Zero), + View = SbomView.Inventory, + OldImageDigest = "sha256:old", + NewImageDigest = "sha256:new", + }; + + var differ = new ComponentDiffer(); + var document = differ.Compute(request); + + Assert.Equal(SbomView.Inventory, document.View); + Assert.Equal("sha256:old", document.OldImageDigest); + Assert.Equal("sha256:new", document.NewImageDigest); + Assert.Equal(1, document.Summary.Added); + Assert.Equal(1, document.Summary.Removed); + Assert.Equal(1, document.Summary.VersionChanged); + Assert.Equal(1, document.Summary.MetadataChanged); + + Assert.Equal(new[] { "sha256:layer2", "sha256:layer3", "sha256:layer1b" }, document.Layers.Select(layer => layer.LayerDigest)); + + var layerGroups = document.Layers.ToDictionary(layer => layer.LayerDigest); + Assert.True(layerGroups.ContainsKey("sha256:layer2"), "Expected layer2 group present"); + Assert.True(layerGroups.ContainsKey("sha256:layer3"), "Expected layer3 group present"); + Assert.True(layerGroups.ContainsKey("sha256:layer1b"), "Expected layer1b group present"); + + var addedChange = layerGroups["sha256:layer3"].Changes.Single(change => change.Kind == ComponentChangeKind.Added); + Assert.Equal("pkg:npm/c", addedChange.ComponentKey); + Assert.NotNull(addedChange.NewComponent); + + var versionChange = layerGroups["sha256:layer2"].Changes.Single(change => change.Kind == ComponentChangeKind.VersionChanged); + Assert.Equal("pkg:npm/a", versionChange.ComponentKey); + Assert.Equal("sha256:layer1b", versionChange.RemovingLayer); + Assert.Equal("sha256:layer2", versionChange.IntroducingLayer); + Assert.Equal("1.1.0", versionChange.NewComponent!.Identity.Version); + + var metadataChange = layerGroups["sha256:layer3"].Changes.Single(change => change.Kind == ComponentChangeKind.MetadataChanged); + Assert.True(metadataChange.NewComponent!.Usage.UsedByEntrypoint); + Assert.False(metadataChange.OldComponent!.Usage.UsedByEntrypoint); + Assert.Equal("sha256:layer3", metadataChange.IntroducingLayer); + Assert.Equal("sha256:layer1", metadataChange.RemovingLayer); + + var removedChange = layerGroups["sha256:layer1b"].Changes.Single(change => change.Kind == ComponentChangeKind.Removed); + Assert.Equal("pkg:npm/d", removedChange.ComponentKey); + Assert.Equal("sha256:layer1b", removedChange.RemovingLayer); + Assert.Null(removedChange.IntroducingLayer); + + var json = DiffJsonSerializer.Serialize(document); + using var parsed = JsonDocument.Parse(json); + var root = parsed.RootElement; + Assert.Equal("inventory", root.GetProperty("view").GetString()); + Assert.Equal("2025-10-19T10:00:00.000000Z", root.GetProperty("generatedAt").GetString()); + Assert.Equal("sha256:old", root.GetProperty("oldImageDigest").GetString()); + Assert.Equal("sha256:new", root.GetProperty("newImageDigest").GetString()); + + var summaryJson = root.GetProperty("summary"); + Assert.Equal(1, summaryJson.GetProperty("added").GetInt32()); + Assert.Equal(1, summaryJson.GetProperty("removed").GetInt32()); + Assert.Equal(1, summaryJson.GetProperty("versionChanged").GetInt32()); + Assert.Equal(1, summaryJson.GetProperty("metadataChanged").GetInt32()); + + var layersJson = root.GetProperty("layers"); + Assert.Equal(3, layersJson.GetArrayLength()); + + var layer2Json = layersJson[0]; + Assert.Equal("sha256:layer2", layer2Json.GetProperty("layerDigest").GetString()); + var layer2Changes = layer2Json.GetProperty("changes"); + Assert.Equal(1, layer2Changes.GetArrayLength()); + var versionChangeJson = layer2Changes.EnumerateArray().Single(); + Assert.Equal("versionChanged", versionChangeJson.GetProperty("kind").GetString()); + Assert.Equal("pkg:npm/a", versionChangeJson.GetProperty("componentKey").GetString()); + Assert.Equal("sha256:layer2", versionChangeJson.GetProperty("introducingLayer").GetString()); + Assert.Equal("sha256:layer1b", versionChangeJson.GetProperty("removingLayer").GetString()); + Assert.Equal("1.1.0", versionChangeJson.GetProperty("newComponent").GetProperty("identity").GetProperty("version").GetString()); + + var layer3Json = layersJson[1]; + Assert.Equal("sha256:layer3", layer3Json.GetProperty("layerDigest").GetString()); + var layer3Changes = layer3Json.GetProperty("changes"); + Assert.Equal(2, layer3Changes.GetArrayLength()); + var layer3ChangeArray = layer3Changes.EnumerateArray().ToArray(); + var metadataChangeJson = layer3ChangeArray[0]; + Assert.Equal("metadataChanged", metadataChangeJson.GetProperty("kind").GetString()); + Assert.Equal("pkg:npm/b", metadataChangeJson.GetProperty("componentKey").GetString()); + Assert.Equal("sha256:layer3", metadataChangeJson.GetProperty("introducingLayer").GetString()); + Assert.Equal("sha256:layer1", metadataChangeJson.GetProperty("removingLayer").GetString()); + Assert.True(metadataChangeJson.GetProperty("newComponent").GetProperty("usage").GetProperty("usedByEntrypoint").GetBoolean()); + Assert.False(metadataChangeJson.GetProperty("oldComponent").GetProperty("usage").GetProperty("usedByEntrypoint").GetBoolean()); + + var addedJson = layer3ChangeArray[1]; + Assert.Equal("added", addedJson.GetProperty("kind").GetString()); + Assert.Equal("pkg:npm/c", addedJson.GetProperty("componentKey").GetString()); + Assert.Equal("sha256:layer3", addedJson.GetProperty("introducingLayer").GetString()); + Assert.False(addedJson.TryGetProperty("removingLayer", out _)); + + var removedLayerJson = layersJson[2]; + Assert.Equal("sha256:layer1b", removedLayerJson.GetProperty("layerDigest").GetString()); + var removedChanges = removedLayerJson.GetProperty("changes"); + Assert.Equal(1, removedChanges.GetArrayLength()); + var removedJson = removedChanges.EnumerateArray().Single(); + Assert.Equal("removed", removedJson.GetProperty("kind").GetString()); + Assert.Equal("pkg:npm/d", removedJson.GetProperty("componentKey").GetString()); + Assert.Equal("sha256:layer1b", removedJson.GetProperty("removingLayer").GetString()); + Assert.False(removedJson.TryGetProperty("introducingLayer", out _)); + } + + [Fact] + public void Compute_UsageView_FiltersComponents() + { + var oldFragments = new[] + { + LayerComponentFragment.Create("sha256:base", new[] + { + CreateComponent("pkg:npm/a", "1", "sha256:base", usage: ComponentUsage.Create(false)), + }) + }; + + var newFragments = new[] + { + LayerComponentFragment.Create("sha256:new", new[] + { + CreateComponent("pkg:npm/a", "1", "sha256:new", usage: ComponentUsage.Create(false)), + CreateComponent("pkg:npm/b", "1", "sha256:new", usage: ComponentUsage.Create(true, new[] { "/entry" })), + }) + }; + + var request = new ComponentDiffRequest + { + OldGraph = ComponentGraphBuilder.Build(oldFragments), + NewGraph = ComponentGraphBuilder.Build(newFragments), + View = SbomView.Usage, + GeneratedAt = DateTimeOffset.UtcNow, + }; + + var differ = new ComponentDiffer(); + var document = differ.Compute(request); + + Assert.Single(document.Layers); + var layer = document.Layers[0]; + Assert.Single(layer.Changes); + Assert.Equal(ComponentChangeKind.Added, layer.Changes[0].Kind); + Assert.Equal("pkg:npm/b", layer.Changes[0].ComponentKey); + + var json = DiffJsonSerializer.Serialize(document); + using var parsed = JsonDocument.Parse(json); + Assert.Equal("usage", parsed.RootElement.GetProperty("view").GetString()); + Assert.Equal(1, parsed.RootElement.GetProperty("summary").GetProperty("added").GetInt32()); + Assert.False(parsed.RootElement.TryGetProperty("oldImageDigest", out _)); + Assert.False(parsed.RootElement.TryGetProperty("newImageDigest", out _)); + } + + [Fact] + public void Compute_MetadataChange_WhenEvidenceDiffers() + { + var oldFragments = new[] + { + LayerComponentFragment.Create("sha256:underlay", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:underlay", + usage: ComponentUsage.Create(false), + evidence: new[] { ComponentEvidence.FromPath("/workspace/package-lock.json") }), + }), + }; + + var newFragments = new[] + { + LayerComponentFragment.Create("sha256:overlay", new[] + { + CreateComponent( + "pkg:npm/a", + version: "1.0.0", + layer: "sha256:overlay", + usage: ComponentUsage.Create(false), + evidence: new[] + { + ComponentEvidence.FromPath("/workspace/package-lock.json"), + ComponentEvidence.FromPath("/workspace/yarn.lock"), + }), + }), + }; + + var request = new ComponentDiffRequest + { + OldGraph = ComponentGraphBuilder.Build(oldFragments), + NewGraph = ComponentGraphBuilder.Build(newFragments), + GeneratedAt = new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero), + }; + + var differ = new ComponentDiffer(); + var document = differ.Compute(request); + + Assert.Equal(0, document.Summary.Added); + Assert.Equal(0, document.Summary.Removed); + Assert.Equal(0, document.Summary.VersionChanged); + Assert.Equal(1, document.Summary.MetadataChanged); + + var layer = Assert.Single(document.Layers); + Assert.Equal("sha256:overlay", layer.LayerDigest); + + var change = Assert.Single(layer.Changes); + Assert.Equal(ComponentChangeKind.MetadataChanged, change.Kind); + Assert.Equal("sha256:overlay", change.IntroducingLayer); + Assert.Equal("sha256:underlay", change.RemovingLayer); + Assert.Equal(2, change.NewComponent!.Evidence.Length); + Assert.Equal(1, change.OldComponent!.Evidence.Length); + } + + private static ComponentRecord CreateComponent( + string key, + string version, + string layer, + ComponentUsage? usage = null, + string? scope = null, + IEnumerable? evidence = null) + { + return new ComponentRecord + { + Identity = ComponentIdentity.Create(key, key.Split('/', 2)[^1], version, purl: key, componentType: "library"), + LayerDigest = layer, + Usage = usage ?? ComponentUsage.Unused, + Metadata = scope is null ? null : new ComponentMetadata { Scope = scope }, + Evidence = evidence is null ? ImmutableArray.Empty : ImmutableArray.CreateRange(evidence), + }; + } +} diff --git a/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj b/src/StellaOps.Scanner.Diff.Tests/StellaOps.Scanner.Diff.Tests.csproj similarity index 69% rename from src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj rename to src/StellaOps.Scanner.Diff.Tests/StellaOps.Scanner.Diff.Tests.csproj index 9e7a8693..9b78c0fa 100644 --- a/src/StellaOps.Feedser.Core.Tests/StellaOps.Feedser.Core.Tests.csproj +++ b/src/StellaOps.Scanner.Diff.Tests/StellaOps.Scanner.Diff.Tests.csproj @@ -4,7 +4,8 @@ enable enable + - + diff --git a/src/StellaOps.Scanner.Diff/AGENTS.md b/src/StellaOps.Scanner.Diff/AGENTS.md new file mode 100644 index 00000000..0cf4ccca --- /dev/null +++ b/src/StellaOps.Scanner.Diff/AGENTS.md @@ -0,0 +1,20 @@ +# StellaOps.Scanner.Diff — Agent Charter + +## Mission +Deliver deterministic image-to-image component diffs grouped by layer with provenance signals that power policy previews, UI surfacing, and downstream scheduling. + +## Responsibilities +- Maintain diff computation pipelines for inventory and usage SBOM views. +- Ensure ordering, hashing, and serialization are stable across runs and hosts. +- Capture layer provenance, usage flags, and supporting evidence for every change. +- Provide JSON artifacts and helper APIs consumed by the Scanner WebService, Worker, CLI, and UI. + +## Interfaces & Dependencies +- Consumes normalized component fragments emitted by analyzers and usage signals from EntryTrace. +- Emits diff models used by `StellaOps.Scanner.WebService` and persisted by `StellaOps.Scanner.Storage`. +- Shares deterministic primitives from `StellaOps.Scanner.Core` once extended with component contracts. + +## Testing Expectations +- Golden diff fixtures for add/remove/version-change flows. +- Determinism checks comparing shuffled inputs. +- Layer attribution regression tests to guard provenance correctness. diff --git a/src/StellaOps.Scanner.Diff/ComponentDiffModels.cs b/src/StellaOps.Scanner.Diff/ComponentDiffModels.cs new file mode 100644 index 00000000..9a27374d --- /dev/null +++ b/src/StellaOps.Scanner.Diff/ComponentDiffModels.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Immutable; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Diff; + +public enum ComponentChangeKind +{ + Added, + Removed, + VersionChanged, + MetadataChanged, +} + +public sealed record ComponentDiffRequest +{ + public required ComponentGraph OldGraph { get; init; } + + public required ComponentGraph NewGraph { get; init; } + + public SbomView View { get; init; } = SbomView.Inventory; + + public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow; + + public string? OldImageDigest { get; init; } + = null; + + public string? NewImageDigest { get; init; } + = null; +} + +public sealed record ComponentChange +{ + [JsonPropertyName("kind")] + public ComponentChangeKind Kind { get; init; } + + [JsonPropertyName("componentKey")] + public string ComponentKey { get; init; } = string.Empty; + + [JsonPropertyName("introducingLayer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? IntroducingLayer { get; init; } + = null; + + [JsonPropertyName("removingLayer")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RemovingLayer { get; init; } + = null; + + [JsonPropertyName("oldComponent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public AggregatedComponent? OldComponent { get; init; } + = null; + + [JsonPropertyName("newComponent")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public AggregatedComponent? NewComponent { get; init; } + = null; +} + +public sealed record LayerDiff +{ + [JsonPropertyName("layerDigest")] + public string LayerDigest { get; init; } = string.Empty; + + [JsonPropertyName("changes")] + public ImmutableArray Changes { get; init; } = ImmutableArray.Empty; +} + +public sealed record DiffSummary +{ + [JsonPropertyName("added")] + public int Added { get; init; } + + [JsonPropertyName("removed")] + public int Removed { get; init; } + + [JsonPropertyName("versionChanged")] + public int VersionChanged { get; init; } + + [JsonPropertyName("metadataChanged")] + public int MetadataChanged { get; init; } +} + +public sealed record ComponentDiffDocument +{ + [JsonPropertyName("generatedAt")] + public DateTimeOffset GeneratedAt { get; init; } + + [JsonPropertyName("view")] + public SbomView View { get; init; } + + [JsonPropertyName("oldImageDigest")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? OldImageDigest { get; init; } + = null; + + [JsonPropertyName("newImageDigest")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? NewImageDigest { get; init; } + = null; + + [JsonPropertyName("summary")] + public DiffSummary Summary { get; init; } = new(); + + [JsonPropertyName("layers")] + public ImmutableArray Layers { get; init; } = ImmutableArray.Empty; +} diff --git a/src/StellaOps.Scanner.Diff/ComponentDiffer.cs b/src/StellaOps.Scanner.Diff/ComponentDiffer.cs new file mode 100644 index 00000000..a4777a24 --- /dev/null +++ b/src/StellaOps.Scanner.Diff/ComponentDiffer.cs @@ -0,0 +1,377 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Diff; + +public sealed class ComponentDiffer +{ + private static readonly StringComparer Ordinal = StringComparer.Ordinal; + private const string UnknownLayerKey = "(unknown)"; + + public ComponentDiffDocument Compute(ComponentDiffRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + var generatedAt = ScannerTimestamps.Normalize(request.GeneratedAt); + var oldComponents = ToDictionary(FilterComponents(request.OldGraph, request.View)); + var newComponents = ToDictionary(FilterComponents(request.NewGraph, request.View)); + var layerOrder = BuildLayerOrder(request.OldGraph, request.NewGraph); + + var changes = new List(); + var counters = new DiffCounters(); + + foreach (var (key, newComponent) in newComponents) + { + if (!oldComponents.TryGetValue(key, out var oldComponent)) + { + changes.Add(new ComponentChange + { + Kind = ComponentChangeKind.Added, + ComponentKey = key, + IntroducingLayer = GetIntroducingLayer(newComponent), + NewComponent = newComponent, + }); + counters.Added++; + continue; + } + + var change = CompareComponents(oldComponent, newComponent, key); + if (change is not null) + { + changes.Add(change); + counters.Register(change.Kind); + } + + oldComponents.Remove(key); + } + + foreach (var (key, oldComponent) in oldComponents) + { + changes.Add(new ComponentChange + { + Kind = ComponentChangeKind.Removed, + ComponentKey = key, + RemovingLayer = GetRemovingLayer(oldComponent), + OldComponent = oldComponent, + }); + counters.Removed++; + } + + var layerGroups = changes + .GroupBy(ResolveLayerKey, Ordinal) + .OrderBy(group => layerOrder.TryGetValue(group.Key, out var position) ? position : int.MaxValue) + .ThenBy(static group => group.Key, Ordinal) + .Select(group => new LayerDiff + { + LayerDigest = group.Key, + Changes = group + .OrderBy(change => change.ComponentKey, Ordinal) + .ThenBy(change => change.Kind) + .ThenBy(change => change.NewComponent?.Identity.Version ?? change.OldComponent?.Identity.Version ?? string.Empty, Ordinal) + .ToImmutableArray(), + }) + .ToImmutableArray(); + + var document = new ComponentDiffDocument + { + GeneratedAt = generatedAt, + View = request.View, + OldImageDigest = request.OldImageDigest, + NewImageDigest = request.NewImageDigest, + Summary = counters.ToSummary(), + Layers = layerGroups, + }; + + return document; + } + + private static ComponentChange? CompareComponents(AggregatedComponent oldComponent, AggregatedComponent newComponent, string key) + { + var versionChanged = !string.Equals(oldComponent.Identity.Version, newComponent.Identity.Version, StringComparison.Ordinal); + if (versionChanged) + { + return new ComponentChange + { + Kind = ComponentChangeKind.VersionChanged, + ComponentKey = key, + IntroducingLayer = GetIntroducingLayer(newComponent), + RemovingLayer = GetRemovingLayer(oldComponent), + OldComponent = oldComponent, + NewComponent = newComponent, + }; + } + + var metadataChanged = HasMetadataChanged(oldComponent, newComponent); + if (!metadataChanged) + { + return null; + } + + return new ComponentChange + { + Kind = ComponentChangeKind.MetadataChanged, + ComponentKey = key, + IntroducingLayer = GetIntroducingLayer(newComponent), + RemovingLayer = GetRemovingLayer(oldComponent), + OldComponent = oldComponent, + NewComponent = newComponent, + }; + } + + private static bool HasMetadataChanged(AggregatedComponent oldComponent, AggregatedComponent newComponent) + { + if (!string.Equals(oldComponent.Identity.Name, newComponent.Identity.Name, StringComparison.Ordinal)) + { + return true; + } + + if (!string.Equals(oldComponent.Identity.ComponentType, newComponent.Identity.ComponentType, StringComparison.Ordinal)) + { + return true; + } + + if (!string.Equals(oldComponent.Identity.Group, newComponent.Identity.Group, StringComparison.Ordinal)) + { + return true; + } + + if (!string.Equals(oldComponent.Identity.Purl, newComponent.Identity.Purl, StringComparison.Ordinal)) + { + return true; + } + + if (!oldComponent.Dependencies.SequenceEqual(newComponent.Dependencies, Ordinal)) + { + return true; + } + + if (!oldComponent.LayerDigests.SequenceEqual(newComponent.LayerDigests, Ordinal)) + { + return true; + } + + if (!oldComponent.Evidence.SequenceEqual(newComponent.Evidence)) + { + return true; + } + + if (UsageChanged(oldComponent.Usage, newComponent.Usage)) + { + return true; + } + + if (!MetadataEquals(oldComponent.Metadata, newComponent.Metadata)) + { + return true; + } + + return false; + } + + private static bool UsageChanged(ComponentUsage oldUsage, ComponentUsage newUsage) + { + if (oldUsage.UsedByEntrypoint != newUsage.UsedByEntrypoint) + { + return true; + } + + return !oldUsage.Entrypoints.SequenceEqual(newUsage.Entrypoints, Ordinal); + } + + private static bool MetadataEquals(ComponentMetadata? left, ComponentMetadata? right) + { + if (left is null && right is null) + { + return true; + } + + if (left is null || right is null) + { + return false; + } + + if (!string.Equals(left.Scope, right.Scope, StringComparison.Ordinal)) + { + return false; + } + + if (!SequenceEqual(left.Licenses, right.Licenses)) + { + return false; + } + + if (!DictionaryEqual(left.Properties, right.Properties)) + { + return false; + } + + return true; + } + + private static bool SequenceEqual(IReadOnlyList? left, IReadOnlyList? right) + { + if (left is null && right is null) + { + return true; + } + + if (left is null || right is null) + { + return false; + } + + if (left.Count != right.Count) + { + return false; + } + + for (var i = 0; i < left.Count; i++) + { + if (!string.Equals(left[i], right[i], StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static bool DictionaryEqual(IReadOnlyDictionary? left, IReadOnlyDictionary? right) + { + if (left is null && right is null) + { + return true; + } + + if (left is null || right is null) + { + return false; + } + + if (left.Count != right.Count) + { + return false; + } + + foreach (var (key, value) in left) + { + if (!right.TryGetValue(key, out var rightValue)) + { + return false; + } + + if (!string.Equals(value, rightValue, StringComparison.Ordinal)) + { + return false; + } + } + + return true; + } + + private static Dictionary ToDictionary(ImmutableArray components) + { + var dictionary = new Dictionary(components.Length, Ordinal); + foreach (var component in components) + { + dictionary[component.Identity.Key] = component; + } + + return dictionary; + } + + private static ImmutableArray FilterComponents(ComponentGraph graph, SbomView view) + { + if (view == SbomView.Usage) + { + return graph.Components.Where(static component => component.Usage.UsedByEntrypoint).ToImmutableArray(); + } + + return graph.Components; + } + + private static Dictionary BuildLayerOrder(ComponentGraph oldGraph, ComponentGraph newGraph) + { + var order = new Dictionary(Ordinal); + var index = 0; + + foreach (var layer in newGraph.Layers) + { + AddLayer(order, layer.LayerDigest, ref index); + } + + foreach (var layer in oldGraph.Layers) + { + AddLayer(order, layer.LayerDigest, ref index); + } + + return order; + } + + private static void AddLayer(IDictionary order, string? layerDigest, ref int index) + { + var normalized = NormalizeLayer(layerDigest); + if (normalized is null || order.ContainsKey(normalized)) + { + return; + } + + order[normalized] = index++; + } + + private static string ResolveLayerKey(ComponentChange change) + => NormalizeLayer(change.IntroducingLayer) ?? NormalizeLayer(change.RemovingLayer) ?? UnknownLayerKey; + + private static string? GetIntroducingLayer(AggregatedComponent component) + => NormalizeLayer(component.FirstLayerDigest); + + private static string? GetRemovingLayer(AggregatedComponent component) + { + var layer = component.LastLayerDigest ?? component.FirstLayerDigest; + return NormalizeLayer(layer); + } + + private static string? NormalizeLayer(string? layer) + { + if (string.IsNullOrWhiteSpace(layer)) + { + return null; + } + + return layer; + } + + private sealed class DiffCounters + { + public int Added; + public int Removed; + public int VersionChanged; + public int MetadataChanged; + + public void Register(ComponentChangeKind kind) + { + switch (kind) + { + case ComponentChangeKind.VersionChanged: + VersionChanged++; + break; + case ComponentChangeKind.MetadataChanged: + MetadataChanged++; + break; + } + } + + public DiffSummary ToSummary() + => new() + { + Added = Added, + Removed = Removed, + VersionChanged = VersionChanged, + MetadataChanged = MetadataChanged, + }; + } +} diff --git a/src/StellaOps.Scanner.Diff/DiffJsonSerializer.cs b/src/StellaOps.Scanner.Diff/DiffJsonSerializer.cs new file mode 100644 index 00000000..baae37b9 --- /dev/null +++ b/src/StellaOps.Scanner.Diff/DiffJsonSerializer.cs @@ -0,0 +1,10 @@ +using System.Text.Json; +using StellaOps.Scanner.Core.Serialization; + +namespace StellaOps.Scanner.Diff; + +public static class DiffJsonSerializer +{ + public static string Serialize(ComponentDiffDocument document) + => JsonSerializer.Serialize(document, ScannerJsonOptions.Default); +} diff --git a/src/StellaOps.Scanner.Diff/StellaOps.Scanner.Diff.csproj b/src/StellaOps.Scanner.Diff/StellaOps.Scanner.Diff.csproj new file mode 100644 index 00000000..7237dd23 --- /dev/null +++ b/src/StellaOps.Scanner.Diff/StellaOps.Scanner.Diff.csproj @@ -0,0 +1,12 @@ + + + net10.0 + enable + enable + true + + + + + + diff --git a/src/StellaOps.Scanner.Diff/TASKS.md b/src/StellaOps.Scanner.Diff/TASKS.md new file mode 100644 index 00000000..987cc228 --- /dev/null +++ b/src/StellaOps.Scanner.Diff/TASKS.md @@ -0,0 +1,7 @@ +# Scanner Diff Task Board (Sprint 10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-DIFF-10-501 | DONE (2025-10-19) | Diff Guild | SCANNER-CORE-09-501 | Build component differ tracking add/remove/version changes with deterministic ordering. | Diff engine produces deterministic results across runs; unit tests cover add/remove/version scenarios. | +| SCANNER-DIFF-10-502 | DONE (2025-10-19) | Diff Guild | SCANNER-DIFF-10-501 | Attribute diffs to introducing/removing layers including provenance evidence. | Layer attribution stored on every change; tests validate provenance with synthetic layer stacks. | +| SCANNER-DIFF-10-503 | DONE (2025-10-19) | Diff Guild | SCANNER-DIFF-10-502 | Produce JSON diff output for inventory vs usage views aligned with API contract. | JSON serializer emits stable ordering; golden fixture captured; API contract documented. | diff --git a/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs b/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs new file mode 100644 index 00000000..4f325250 --- /dev/null +++ b/src/StellaOps.Scanner.Emit.Tests/Composition/CycloneDxComposerTests.cs @@ -0,0 +1,148 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Emit.Composition; +using Xunit; + +namespace StellaOps.Scanner.Emit.Tests.Composition; + +public sealed class CycloneDxComposerTests +{ + [Fact] + public void Compose_ProducesInventoryAndUsageArtifacts() + { + var request = BuildRequest(); + var composer = new CycloneDxComposer(); + + var result = composer.Compose(request); + + Assert.NotNull(result.Inventory); + Assert.StartsWith("urn:uuid:", result.Inventory.SerialNumber, StringComparison.Ordinal); + Assert.Equal("application/vnd.cyclonedx+json; version=1.5", result.Inventory.JsonMediaType); + Assert.Equal("application/vnd.cyclonedx+protobuf; version=1.5", result.Inventory.ProtobufMediaType); + Assert.Equal(2, result.Inventory.Components.Length); + + Assert.NotNull(result.Usage); + Assert.Equal("application/vnd.cyclonedx+json; version=1.5; view=usage", result.Usage!.JsonMediaType); + Assert.Single(result.Usage.Components); + Assert.Equal("pkg:npm/a", result.Usage.Components[0].Identity.Key); + + ValidateJson(result.Inventory.JsonBytes, expectedComponentCount: 2, expectedView: "inventory"); + ValidateJson(result.Usage.JsonBytes, expectedComponentCount: 1, expectedView: "usage"); + } + + [Fact] + public void Compose_IsDeterministic() + { + var request = BuildRequest(); + var composer = new CycloneDxComposer(); + + var first = composer.Compose(request); + var second = composer.Compose(request); + + Assert.Equal(first.Inventory.JsonSha256, second.Inventory.JsonSha256); + Assert.Equal(first.Inventory.ProtobufSha256, second.Inventory.ProtobufSha256); + Assert.Equal(first.Inventory.SerialNumber, second.Inventory.SerialNumber); + + Assert.NotNull(first.Usage); + Assert.NotNull(second.Usage); + Assert.Equal(first.Usage!.JsonSha256, second.Usage!.JsonSha256); + Assert.Equal(first.Usage.ProtobufSha256, second.Usage.ProtobufSha256); + Assert.Equal(first.Usage.SerialNumber, second.Usage.SerialNumber); + } + + private static SbomCompositionRequest BuildRequest() + { + var fragments = new[] + { + LayerComponentFragment.Create("sha256:layer1", new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/a", "component-a", "1.0.0", "pkg:npm/a@1.0.0", "library"), + LayerDigest = "sha256:layer1", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/a/package.json")), + Dependencies = ImmutableArray.Create("pkg:npm/b"), + Usage = ComponentUsage.Create(true, new[] { "/app/start.sh" }), + Metadata = new ComponentMetadata + { + Scope = "runtime", + Licenses = new[] { "MIT" }, + Properties = new Dictionary + { + ["stellaops:source"] = "package-lock.json", + ["stellaops.os.analyzer"] = "apk", + ["stellaops.os.architecture"] = "x86_64", + }, + }, + } + }), + LayerComponentFragment.Create("sha256:layer2", new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:npm/b", "component-b", "2.0.0", "pkg:npm/b@2.0.0", "library"), + LayerDigest = "sha256:layer2", + Evidence = ImmutableArray.Create(ComponentEvidence.FromPath("/app/node_modules/b/package.json")), + Usage = ComponentUsage.Create(false), + Metadata = new ComponentMetadata + { + Scope = "development", + Properties = new Dictionary + { + ["stellaops.os.analyzer"] = "language-node", + }, + }, + } + }) + }; + + var image = new ImageArtifactDescriptor + { + ImageDigest = "sha256:1234567890abcdef", + ImageReference = "registry.example.com/app/service:1.2.3", + Repository = "registry.example.com/app/service", + Tag = "1.2.3", + Architecture = "amd64", + }; + + return SbomCompositionRequest.Create( + image, + fragments, + new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero), + generatorName: "StellaOps.Scanner", + generatorVersion: "0.10.0", + properties: new Dictionary + { + ["stellaops:scanId"] = "scan-1234", + }); + } + + private static void ValidateJson(byte[] data, int expectedComponentCount, string expectedView) + { + using var document = JsonDocument.Parse(data); + var root = document.RootElement; + + Assert.True(root.TryGetProperty("metadata", out var metadata), "metadata property missing"); + var properties = metadata.GetProperty("properties"); + var viewProperty = properties.EnumerateArray() + .Single(prop => prop.GetProperty("name").GetString() == "stellaops:sbom.view"); + Assert.Equal(expectedView, viewProperty.GetProperty("value").GetString()); + + var components = root.GetProperty("components").EnumerateArray().ToArray(); + Assert.Equal(expectedComponentCount, components.Length); + + var names = components.Select(component => component.GetProperty("name").GetString()).ToArray(); + Assert.Equal(names, names.OrderBy(n => n, StringComparer.Ordinal).ToArray()); + + var firstComponentProperties = components[0].GetProperty("properties").EnumerateArray().ToDictionary( + element => element.GetProperty("name").GetString(), + element => element.GetProperty("value").GetString()); + + Assert.Equal("apk", firstComponentProperties["stellaops.os.analyzer"]); + Assert.Equal("x86_64", firstComponentProperties["stellaops.os.architecture"]); + } +} diff --git a/src/StellaOps.Scanner.Emit.Tests/Composition/ScanAnalysisCompositionBuilderTests.cs b/src/StellaOps.Scanner.Emit.Tests/Composition/ScanAnalysisCompositionBuilderTests.cs new file mode 100644 index 00000000..4e6f4e59 --- /dev/null +++ b/src/StellaOps.Scanner.Emit.Tests/Composition/ScanAnalysisCompositionBuilderTests.cs @@ -0,0 +1,52 @@ +using System.Collections.Immutable; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Emit.Composition; +using Xunit; + +namespace StellaOps.Scanner.Emit.Tests.Composition; + +public class ScanAnalysisCompositionBuilderTests +{ + [Fact] + public void FromAnalysis_BuildsRequest_WhenFragmentsPresent() + { + var analysis = new ScanAnalysisStore(); + var fragment = LayerComponentFragment.Create( + "sha256:layer", + new[] + { + new ComponentRecord + { + Identity = ComponentIdentity.Create("pkg:test/a", "a", "1.0.0", "pkg:test/a@1.0.0", "library"), + LayerDigest = "sha256:layer", + Evidence = ImmutableArray.Empty, + Dependencies = ImmutableArray.Empty, + Metadata = null, + Usage = ComponentUsage.Unused, + } + }); + + analysis.AppendLayerFragments(new[] { fragment }); + + var request = ScanAnalysisCompositionBuilder.FromAnalysis( + analysis, + new ImageArtifactDescriptor { ImageDigest = "sha256:image" }, + DateTimeOffset.UtcNow, + generatorName: "test", + generatorVersion: "1.0.0"); + + Assert.Equal("sha256:image", request.Image.ImageDigest); + Assert.Single(request.LayerFragments); + Assert.Equal(fragment.LayerDigest, request.LayerFragments[0].LayerDigest); + } + + [Fact] + public void BuildComponentGraph_ReturnsEmpty_WhenNoFragments() + { + var analysis = new ScanAnalysisStore(); + var graph = ScanAnalysisCompositionBuilder.BuildComponentGraph(analysis); + + Assert.Empty(graph.Components); + Assert.Empty(graph.Layers); + } +} diff --git a/src/StellaOps.Scanner.Emit.Tests/Index/BomIndexBuilderTests.cs b/src/StellaOps.Scanner.Emit.Tests/Index/BomIndexBuilderTests.cs new file mode 100644 index 00000000..e3e0367d --- /dev/null +++ b/src/StellaOps.Scanner.Emit.Tests/Index/BomIndexBuilderTests.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using Collections.Special; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Emit.Index; + +namespace StellaOps.Scanner.Emit.Tests.Index; + +public sealed class BomIndexBuilderTests +{ + [Fact] + public void Build_GeneratesDeterministicBinaryIndex_WithUsageBitmaps() + { + var graph = ComponentGraphBuilder.Build(new[] + { + LayerComponentFragment.Create("sha256:layer1", new[] + { + CreateComponent("pkg:npm/a", "1.0.0", "sha256:layer1", usageEntrypoints: new[] { "/app/start.sh" }), + CreateComponent("pkg:npm/b", "2.0.0", "sha256:layer1"), + }), + LayerComponentFragment.Create("sha256:layer2", new[] + { + CreateComponent("pkg:npm/b", "2.0.0", "sha256:layer2"), + CreateComponent("pkg:npm/c", "3.1.0", "sha256:layer2", usageEntrypoints: new[] { "/app/init.sh" }), + }), + }); + + var request = new BomIndexBuildRequest + { + ImageDigest = "sha256:image", + Graph = graph, + GeneratedAt = new DateTimeOffset(2025, 10, 19, 9, 45, 0, TimeSpan.Zero), + }; + + var builder = new BomIndexBuilder(); + var artifact = builder.Build(request); + var second = builder.Build(request); + + Assert.Equal(artifact.Sha256, second.Sha256); + Assert.Equal(artifact.Bytes, second.Bytes); + Assert.Equal(2, artifact.LayerCount); + Assert.Equal(3, artifact.ComponentCount); + Assert.Equal(2, artifact.EntrypointCount); + + using var reader = new BinaryReader(new MemoryStream(artifact.Bytes), System.Text.Encoding.UTF8, leaveOpen: false); + ValidateHeader(reader, request); + var layers = ReadTable(reader, artifact.LayerCount); + Assert.Equal(new[] { "sha256:layer1", "sha256:layer2" }, layers); + + var purls = ReadTable(reader, artifact.ComponentCount); + Assert.Equal(new[] { "pkg:npm/a", "pkg:npm/b", "pkg:npm/c" }, purls); + + var componentBitmaps = ReadBitmaps(reader, artifact.ComponentCount); + Assert.Equal(new[] { new[] { 0 }, new[] { 0, 1 }, new[] { 1 } }, componentBitmaps); + + var entrypoints = ReadTable(reader, artifact.EntrypointCount); + Assert.Equal(new[] { "/app/init.sh", "/app/start.sh" }, entrypoints); + + var usageBitmaps = ReadBitmaps(reader, artifact.ComponentCount); + Assert.Equal(new[] { new[] { 1 }, Array.Empty(), new[] { 0 } }, usageBitmaps); + } + + private static void ValidateHeader(BinaryReader reader, BomIndexBuildRequest request) + { + var magic = reader.ReadBytes(7); + Assert.Equal("BOMIDX1", System.Text.Encoding.ASCII.GetString(magic)); + + var version = reader.ReadUInt16(); + Assert.Equal(1u, version); + + var flags = reader.ReadUInt16(); + Assert.Equal(0x1, flags); + + var digestLength = reader.ReadUInt16(); + var digestBytes = reader.ReadBytes(digestLength); + Assert.Equal(request.ImageDigest, System.Text.Encoding.UTF8.GetString(digestBytes)); + + var unixMicroseconds = reader.ReadInt64(); + var expectedMicroseconds = request.GeneratedAt.ToUniversalTime().ToUnixTimeMilliseconds() * 1000L; + expectedMicroseconds += request.GeneratedAt.ToUniversalTime().Ticks % TimeSpan.TicksPerMillisecond / 10; + Assert.Equal(expectedMicroseconds, unixMicroseconds); + + var layers = reader.ReadUInt32(); + var components = reader.ReadUInt32(); + var entrypoints = reader.ReadUInt32(); + + Assert.Equal(2u, layers); + Assert.Equal(3u, components); + Assert.Equal(2u, entrypoints); + } + + private static string[] ReadTable(BinaryReader reader, int count) + { + var values = new string[count]; + for (var i = 0; i < count; i++) + { + var length = reader.ReadUInt16(); + var bytes = reader.ReadBytes(length); + values[i] = System.Text.Encoding.UTF8.GetString(bytes); + } + + return values; + } + + private static int[][] ReadBitmaps(BinaryReader reader, int count) + { + var result = new int[count][]; + for (var i = 0; i < count; i++) + { + var length = reader.ReadUInt32(); + if (length == 0) + { + result[i] = Array.Empty(); + continue; + } + + var bytes = reader.ReadBytes((int)length); + using var ms = new MemoryStream(bytes, writable: false); + var bitmap = RoaringBitmap.Deserialize(ms); + result[i] = bitmap.ToArray(); + } + + return result; + } + + private static ComponentRecord CreateComponent(string key, string version, string layerDigest, string[]? usageEntrypoints = null) + { + var usage = usageEntrypoints is null + ? ComponentUsage.Unused + : ComponentUsage.Create(true, usageEntrypoints); + + return new ComponentRecord + { + Identity = ComponentIdentity.Create(key, key.Split('/', 2)[^1], version, key, "library"), + LayerDigest = layerDigest, + Usage = usage, + }; + } +} diff --git a/src/StellaOps.Scanner.Emit.Tests/Packaging/ScannerArtifactPackageBuilderTests.cs b/src/StellaOps.Scanner.Emit.Tests/Packaging/ScannerArtifactPackageBuilderTests.cs new file mode 100644 index 00000000..3e7b089d --- /dev/null +++ b/src/StellaOps.Scanner.Emit.Tests/Packaging/ScannerArtifactPackageBuilderTests.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Emit.Composition; +using StellaOps.Scanner.Emit.Index; +using StellaOps.Scanner.Emit.Packaging; + +namespace StellaOps.Scanner.Emit.Tests.Packaging; + +public sealed class ScannerArtifactPackageBuilderTests +{ + [Fact] + public void BuildPackage_ProducesDescriptorsAndManifest() + { + var fragments = new[] + { + LayerComponentFragment.Create("sha256:layer1", new[] + { + CreateComponent( + "pkg:npm/a", + "1.0.0", + "sha256:layer1", + usage: ComponentUsage.Create(true, new[] { "/app/start.sh" }), + metadata: new Dictionary + { + ["stellaops.os.analyzer"] = "apk", + ["stellaops.os.architecture"] = "x86_64", + }), + CreateComponent("pkg:npm/b", "2.0.0", "sha256:layer1"), + }), + LayerComponentFragment.Create("sha256:layer2", new[] + { + CreateComponent("pkg:npm/b", "2.0.0", "sha256:layer2"), + CreateComponent("pkg:npm/c", "3.0.0", "sha256:layer2", usage: ComponentUsage.Create(true, new[] { "/app/init.sh" })), + }) + }; + + var request = SbomCompositionRequest.Create( + new ImageArtifactDescriptor + { + ImageDigest = "sha256:image", + ImageReference = "registry.example/app:latest", + Repository = "registry.example/app", + Tag = "latest", + }, + fragments, + new DateTimeOffset(2025, 10, 19, 12, 30, 0, TimeSpan.Zero), + generatorName: "StellaOps.Scanner", + generatorVersion: "0.10.0"); + + var composer = new CycloneDxComposer(); + var composition = composer.Compose(request); + + var indexBuilder = new BomIndexBuilder(); + var bomIndex = indexBuilder.Build(new BomIndexBuildRequest + { + ImageDigest = request.Image.ImageDigest, + Graph = composition.Graph, + GeneratedAt = request.GeneratedAt, + }); + + var packageBuilder = new ScannerArtifactPackageBuilder(); + var package = packageBuilder.Build(request.Image.ImageDigest, request.GeneratedAt, composition, bomIndex); + + Assert.Equal(5, package.Artifacts.Length); // inventory JSON+PB, usage JSON+PB, index + + var kinds = package.Manifest.Artifacts.Select(entry => entry.Kind).ToArray(); + Assert.Equal(new[] { "bom-index", "sbom-inventory", "sbom-inventory", "sbom-usage", "sbom-usage" }, kinds); + + var manifestJson = package.Manifest.ToJsonBytes(); + using var document = JsonDocument.Parse(manifestJson); + var root = document.RootElement; + Assert.Equal("sha256:image", root.GetProperty("imageDigest").GetString()); + Assert.Equal(5, root.GetProperty("artifacts").GetArrayLength()); + + var usageEntry = root.GetProperty("artifacts").EnumerateArray().First(element => element.GetProperty("kind").GetString() == "sbom-usage"); + Assert.Equal("application/vnd.cyclonedx+json; version=1.5; view=usage", usageEntry.GetProperty("mediaType").GetString()); + } + + private static ComponentRecord CreateComponent(string key, string version, string layerDigest, ComponentUsage? usage = null, IReadOnlyDictionary? metadata = null) + { + return new ComponentRecord + { + Identity = ComponentIdentity.Create(key, key.Split('/', 2)[^1], version, key, "library"), + LayerDigest = layerDigest, + Usage = usage ?? ComponentUsage.Unused, + Metadata = metadata is null + ? null + : new ComponentMetadata + { + Properties = metadata, + }, + }; + } +} diff --git a/src/StellaOps.Scanner.Emit.Tests/StellaOps.Scanner.Emit.Tests.csproj b/src/StellaOps.Scanner.Emit.Tests/StellaOps.Scanner.Emit.Tests.csproj new file mode 100644 index 00000000..8f20afaf --- /dev/null +++ b/src/StellaOps.Scanner.Emit.Tests/StellaOps.Scanner.Emit.Tests.csproj @@ -0,0 +1,11 @@ + + + net10.0 + enable + enable + + + + + + diff --git a/src/StellaOps.Scanner.Emit/AGENTS.md b/src/StellaOps.Scanner.Emit/AGENTS.md new file mode 100644 index 00000000..9757097d --- /dev/null +++ b/src/StellaOps.Scanner.Emit/AGENTS.md @@ -0,0 +1,20 @@ +# StellaOps.Scanner.Emit — Agent Charter + +## Mission +Assemble deterministic SBOM artifacts (inventory, usage, BOM index) from analyzer fragments and usage telemetry, and prepare them for storage, signing, and distribution. + +## Responsibilities +- Merge per-layer/component fragments into CycloneDX JSON/Protobuf SBOMs. +- Generate BOM index sidecars with roaring bitmap acceleration and usage flags. +- Package artifacts with stable naming, hashing, and manifests for downstream storage and attestations. +- Surface helper APIs for Scanner Worker/WebService to request compositions and exports. + +## Interfaces & Dependencies +- Consumes analyzer outputs (OS, language, native) and EntryTrace usage annotations. +- Produces artifacts persisted via `StellaOps.Scanner.Storage` and referenced by policy/report pipelines. +- Relies on deterministic primitives from `StellaOps.Scanner.Core` for timestamps, hashing, and serialization defaults. + +## Testing Expectations +- Golden SBOM and BOM index fixtures with determinism checks. +- Schema validation for CycloneDX outputs and BOM index binary layout. +- Integration tests exercising packaging helpers with in-memory storage fakes. diff --git a/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs b/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs new file mode 100644 index 00000000..498127c1 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Composition/CycloneDxComposer.cs @@ -0,0 +1,405 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using CycloneDX; +using CycloneDX.Models; +using JsonSerializer = CycloneDX.Json.Serializer; +using ProtoSerializer = CycloneDX.Protobuf.Serializer; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Emit.Composition; + +public sealed class CycloneDxComposer +{ + private static readonly Guid SerialNamespace = new("0d3a422b-6e1b-4d9b-9c35-654b706c97e8"); + + private const string InventoryMediaTypeJson = "application/vnd.cyclonedx+json; version=1.5"; + private const string UsageMediaTypeJson = "application/vnd.cyclonedx+json; version=1.5; view=usage"; + private const string InventoryMediaTypeProtobuf = "application/vnd.cyclonedx+protobuf; version=1.5"; + private const string UsageMediaTypeProtobuf = "application/vnd.cyclonedx+protobuf; version=1.5; view=usage"; + + public SbomCompositionResult Compose(SbomCompositionRequest request) + { + ArgumentNullException.ThrowIfNull(request); + if (request.LayerFragments.IsDefaultOrEmpty) + { + throw new ArgumentException("At least one layer fragment is required.", nameof(request)); + } + + var graph = ComponentGraphBuilder.Build(request.LayerFragments); + var generatedAt = ScannerTimestamps.Normalize(request.GeneratedAt); + + var inventoryArtifact = BuildArtifact( + request, + graph, + SbomView.Inventory, + graph.Components, + generatedAt, + InventoryMediaTypeJson, + InventoryMediaTypeProtobuf); + + var usageComponents = graph.Components + .Where(static component => component.Usage.UsedByEntrypoint) + .ToImmutableArray(); + + CycloneDxArtifact? usageArtifact = null; + if (!usageComponents.IsEmpty) + { + usageArtifact = BuildArtifact( + request, + graph, + SbomView.Usage, + usageComponents, + generatedAt, + UsageMediaTypeJson, + UsageMediaTypeProtobuf); + } + + return new SbomCompositionResult + { + Inventory = inventoryArtifact, + Usage = usageArtifact, + Graph = graph, + }; + } + + private CycloneDxArtifact BuildArtifact( + SbomCompositionRequest request, + ComponentGraph graph, + SbomView view, + ImmutableArray components, + DateTimeOffset generatedAt, + string jsonMediaType, + string protobufMediaType) + { + var bom = BuildBom(request, view, components, generatedAt); + var json = JsonSerializer.Serialize(bom); + var jsonBytes = Encoding.UTF8.GetBytes(json); + var protobufBytes = ProtoSerializer.Serialize(bom); + + var jsonHash = ComputeSha256(jsonBytes); + var protobufHash = ComputeSha256(protobufBytes); + + return new CycloneDxArtifact + { + View = view, + SerialNumber = bom.SerialNumber ?? string.Empty, + GeneratedAt = generatedAt, + Components = components, + JsonBytes = jsonBytes, + JsonSha256 = jsonHash, + JsonMediaType = jsonMediaType, + ProtobufBytes = protobufBytes, + ProtobufSha256 = protobufHash, + ProtobufMediaType = protobufMediaType, + }; + } + + private Bom BuildBom( + SbomCompositionRequest request, + SbomView view, + ImmutableArray components, + DateTimeOffset generatedAt) + { + var bom = new Bom + { + SpecVersion = SpecificationVersion.v1_4, + Version = 1, + Metadata = BuildMetadata(request, view, generatedAt), + Components = BuildComponents(components), + Dependencies = BuildDependencies(components), + }; + + var serialPayload = $"{request.Image.ImageDigest}|{view}|{ScannerTimestamps.ToIso8601(generatedAt)}"; + bom.SerialNumber = $"urn:uuid:{ScannerIdentifiers.CreateDeterministicGuid(SerialNamespace, Encoding.UTF8.GetBytes(serialPayload)).ToString("d", CultureInfo.InvariantCulture)}"; + + return bom; + } + + private static Metadata BuildMetadata(SbomCompositionRequest request, SbomView view, DateTimeOffset generatedAt) + { + var metadata = new Metadata + { + Timestamp = generatedAt.UtcDateTime, + Component = BuildMetadataComponent(request.Image), + }; + + if (!string.IsNullOrWhiteSpace(request.GeneratorName)) + { + metadata.Tools = new List + { + new() + { + Name = request.GeneratorName, + Version = request.GeneratorVersion, + } + }; + } + + if (request.AdditionalProperties is not null && request.AdditionalProperties.Count > 0) + { + metadata.Properties = request.AdditionalProperties + .Where(static pair => !string.IsNullOrWhiteSpace(pair.Key) && pair.Value is not null) + .OrderBy(static pair => pair.Key, StringComparer.Ordinal) + .Select(pair => new Property + { + Name = pair.Key, + Value = pair.Value, + }) + .ToList(); + } + + if (metadata.Properties is null) + { + metadata.Properties = new List(); + } + + metadata.Properties.Add(new Property + { + Name = "stellaops:sbom.view", + Value = view.ToString().ToLowerInvariant(), + }); + + return metadata; + } + + private static Component BuildMetadataComponent(ImageArtifactDescriptor image) + { + var digest = image.ImageDigest; + var digestValue = digest.Split(':', 2, StringSplitOptions.TrimEntries)[^1]; + var bomRef = $"image:{digestValue}"; + + var name = image.ImageReference ?? image.Repository ?? digest; + var component = new Component + { + BomRef = bomRef, + Type = Component.Classification.Container, + Name = name, + Version = digestValue, + Purl = BuildImagePurl(image), + Properties = new List + { + new() { Name = "stellaops:image.digest", Value = image.ImageDigest }, + }, + }; + + if (!string.IsNullOrWhiteSpace(image.ImageReference)) + { + component.Properties.Add(new Property { Name = "stellaops:image.reference", Value = image.ImageReference }); + } + + if (!string.IsNullOrWhiteSpace(image.Repository)) + { + component.Properties.Add(new Property { Name = "stellaops:image.repository", Value = image.Repository }); + } + + if (!string.IsNullOrWhiteSpace(image.Tag)) + { + component.Properties.Add(new Property { Name = "stellaops:image.tag", Value = image.Tag }); + } + + if (!string.IsNullOrWhiteSpace(image.Architecture)) + { + component.Properties.Add(new Property { Name = "stellaops:image.architecture", Value = image.Architecture }); + } + + return component; + } + + private static string? BuildImagePurl(ImageArtifactDescriptor image) + { + if (string.IsNullOrWhiteSpace(image.Repository)) + { + return null; + } + + var repo = image.Repository.Trim(); + var tag = string.IsNullOrWhiteSpace(image.Tag) ? null : image.Tag.Trim(); + var digest = image.ImageDigest.Trim(); + + var purlBuilder = new StringBuilder("pkg:oci/"); + purlBuilder.Append(repo.Replace("/", "%2F", StringComparison.Ordinal)); + if (!string.IsNullOrWhiteSpace(tag)) + { + purlBuilder.Append('@').Append(tag); + } + + purlBuilder.Append("?digest=").Append(Uri.EscapeDataString(digest)); + + if (!string.IsNullOrWhiteSpace(image.Architecture)) + { + purlBuilder.Append("&arch=").Append(Uri.EscapeDataString(image.Architecture.Trim())); + } + + return purlBuilder.ToString(); + } + + private static List BuildComponents(ImmutableArray components) + { + var result = new List(components.Length); + foreach (var component in components) + { + var model = new Component + { + BomRef = component.Identity.Key, + Name = component.Identity.Name, + Version = component.Identity.Version, + Purl = component.Identity.Purl, + Group = component.Identity.Group, + Type = MapClassification(component.Identity.ComponentType), + Scope = MapScope(component.Metadata?.Scope), + Properties = BuildProperties(component), + }; + + result.Add(model); + } + + return result; + } + + private static List? BuildProperties(AggregatedComponent component) + { + var properties = new List(); + + if (component.Metadata?.Properties is not null) + { + foreach (var property in component.Metadata.Properties.OrderBy(static pair => pair.Key, StringComparer.Ordinal)) + { + properties.Add(new Property + { + Name = property.Key, + Value = property.Value, + }); + } + } + + properties.Add(new Property { Name = "stellaops:firstLayerDigest", Value = component.FirstLayerDigest }); + if (component.LastLayerDigest is not null) + { + properties.Add(new Property { Name = "stellaops:lastLayerDigest", Value = component.LastLayerDigest }); + } + + if (!component.LayerDigests.IsDefaultOrEmpty) + { + properties.Add(new Property + { + Name = "stellaops:layerDigests", + Value = string.Join(",", component.LayerDigests), + }); + } + + if (component.Usage.UsedByEntrypoint) + { + properties.Add(new Property { Name = "stellaops:usage.usedByEntrypoint", Value = "true" }); + } + + if (!component.Usage.Entrypoints.IsDefaultOrEmpty && component.Usage.Entrypoints.Length > 0) + { + for (var index = 0; index < component.Usage.Entrypoints.Length; index++) + { + properties.Add(new Property + { + Name = $"stellaops:usage.entrypoint[{index}]", + Value = component.Usage.Entrypoints[index], + }); + } + } + + for (var index = 0; index < component.Evidence.Length; index++) + { + var evidence = component.Evidence[index]; + var builder = new StringBuilder(evidence.Kind); + builder.Append(':').Append(evidence.Value); + if (!string.IsNullOrWhiteSpace(evidence.Source)) + { + builder.Append('@').Append(evidence.Source); + } + + properties.Add(new Property + { + Name = $"stellaops:evidence[{index}]", + Value = builder.ToString(), + }); + } + + return properties; + } + + private static List? BuildDependencies(ImmutableArray components) + { + var componentKeys = components.Select(static component => component.Identity.Key).ToImmutableHashSet(StringComparer.Ordinal); + var dependencies = new List(); + + foreach (var component in components) + { + if (component.Dependencies.IsDefaultOrEmpty || component.Dependencies.Length == 0) + { + continue; + } + + var filtered = component.Dependencies.Where(componentKeys.Contains).ToArray(); + if (filtered.Length == 0) + { + continue; + } + + dependencies.Add(new Dependency + { + Ref = component.Identity.Key, + Dependencies = filtered + .Select(dependencyKey => new Dependency { Ref = dependencyKey }) + .ToList(), + }); + } + + return dependencies.Count == 0 ? null : dependencies; + } + + private static Component.Classification MapClassification(string? type) + { + if (string.IsNullOrWhiteSpace(type)) + { + return Component.Classification.Library; + } + + return type.Trim().ToLowerInvariant() switch + { + "application" => Component.Classification.Application, + "framework" => Component.Classification.Framework, + "container" => Component.Classification.Container, + "operating-system" or "os" => Component.Classification.OperationSystem, + "device" => Component.Classification.Device, + "firmware" => Component.Classification.Firmware, + "file" => Component.Classification.File, + _ => Component.Classification.Library, + }; + } + + private static Component.ComponentScope? MapScope(string? scope) + { + if (string.IsNullOrWhiteSpace(scope)) + { + return null; + } + + return scope.Trim().ToLowerInvariant() switch + { + "runtime" or "required" => Component.ComponentScope.Required, + "development" or "optional" => Component.ComponentScope.Optional, + "excluded" => Component.ComponentScope.Excluded, + _ => null, + }; + } + + private static string ComputeSha256(byte[] bytes) + { + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Emit/Composition/SbomCompositionRequest.cs b/src/StellaOps.Scanner.Emit/Composition/SbomCompositionRequest.cs new file mode 100644 index 00000000..c5fb0bd8 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Composition/SbomCompositionRequest.cs @@ -0,0 +1,85 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Utility; + +namespace StellaOps.Scanner.Emit.Composition; + +public sealed record ImageArtifactDescriptor +{ + public string ImageDigest { get; init; } = string.Empty; + + public string? ImageReference { get; init; } + = null; + + public string? Repository { get; init; } + = null; + + public string? Tag { get; init; } + = null; + + public string? Architecture { get; init; } + = null; +} + +public sealed record SbomCompositionRequest +{ + public required ImageArtifactDescriptor Image { get; init; } + + public required ImmutableArray LayerFragments { get; init; } + + public DateTimeOffset GeneratedAt { get; init; } + = ScannerTimestamps.UtcNow(); + + public string? GeneratorName { get; init; } + = null; + + public string? GeneratorVersion { get; init; } + = null; + + public IReadOnlyDictionary? AdditionalProperties { get; init; } + = null; + + public static SbomCompositionRequest Create( + ImageArtifactDescriptor image, + IEnumerable fragments, + DateTimeOffset generatedAt, + string? generatorName = null, + string? generatorVersion = null, + IReadOnlyDictionary? properties = null) + { + ArgumentNullException.ThrowIfNull(image); + ArgumentNullException.ThrowIfNull(fragments); + + var normalizedImage = new ImageArtifactDescriptor + { + ImageDigest = ScannerIdentifiers.NormalizeDigest(image.ImageDigest) ?? throw new ArgumentException("Image digest is required.", nameof(image)), + ImageReference = Normalize(image.ImageReference), + Repository = Normalize(image.Repository), + Tag = Normalize(image.Tag), + Architecture = Normalize(image.Architecture), + }; + + return new SbomCompositionRequest + { + Image = normalizedImage, + LayerFragments = fragments.ToImmutableArray(), + GeneratedAt = ScannerTimestamps.Normalize(generatedAt), + GeneratorName = Normalize(generatorName), + GeneratorVersion = Normalize(generatorVersion), + AdditionalProperties = properties, + }; + } + + private static string? Normalize(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value.Trim(); + } +} diff --git a/src/StellaOps.Scanner.Emit/Composition/SbomCompositionResult.cs b/src/StellaOps.Scanner.Emit/Composition/SbomCompositionResult.cs new file mode 100644 index 00000000..d4afe004 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Composition/SbomCompositionResult.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Immutable; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Emit.Composition; + +public sealed record CycloneDxArtifact +{ + public required SbomView View { get; init; } + + public required string SerialNumber { get; init; } + + public required DateTimeOffset GeneratedAt { get; init; } + + public required ImmutableArray Components { get; init; } + + public required byte[] JsonBytes { get; init; } + + public required string JsonSha256 { get; init; } + + public required string JsonMediaType { get; init; } + + public required byte[] ProtobufBytes { get; init; } + + public required string ProtobufSha256 { get; init; } + + public required string ProtobufMediaType { get; init; } +} + +public sealed record SbomCompositionResult +{ + public required CycloneDxArtifact Inventory { get; init; } + + public CycloneDxArtifact? Usage { get; init; } + + public required ComponentGraph Graph { get; init; } +} diff --git a/src/StellaOps.Scanner.Emit/Composition/ScanAnalysisCompositionBuilder.cs b/src/StellaOps.Scanner.Emit/Composition/ScanAnalysisCompositionBuilder.cs new file mode 100644 index 00000000..645e4257 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Composition/ScanAnalysisCompositionBuilder.cs @@ -0,0 +1,53 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Emit.Composition; + +public static class ScanAnalysisCompositionBuilder +{ + public static SbomCompositionRequest FromAnalysis( + ScanAnalysisStore analysis, + ImageArtifactDescriptor image, + DateTimeOffset generatedAt, + string? generatorName = null, + string? generatorVersion = null, + IReadOnlyDictionary? properties = null) + { + ArgumentNullException.ThrowIfNull(analysis); + ArgumentNullException.ThrowIfNull(image); + + var fragments = analysis.GetLayerFragments(); + if (fragments.IsDefaultOrEmpty) + { + throw new InvalidOperationException("No layer fragments recorded in analysis."); + } + + return SbomCompositionRequest.Create( + image, + fragments, + generatedAt, + generatorName, + generatorVersion, + properties); + } + + public static ComponentGraph BuildComponentGraph(ScanAnalysisStore analysis) + { + ArgumentNullException.ThrowIfNull(analysis); + + var fragments = analysis.GetLayerFragments(); + if (fragments.IsDefaultOrEmpty) + { + return new ComponentGraph + { + Layers = ImmutableArray.Empty, + Components = ImmutableArray.Empty, + ComponentMap = ImmutableDictionary.Empty, + }; + } + + return ComponentGraphBuilder.Build(fragments); + } +} diff --git a/src/StellaOps.Scanner.Emit/Index/BomIndexBuilder.cs b/src/StellaOps.Scanner.Emit/Index/BomIndexBuilder.cs new file mode 100644 index 00000000..ab4744d6 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Index/BomIndexBuilder.cs @@ -0,0 +1,239 @@ +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using Collections.Special; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Emit.Index; + +public sealed record BomIndexBuildRequest +{ + public required string ImageDigest { get; init; } + + public required ComponentGraph Graph { get; init; } + + public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow; +} + +public sealed record BomIndexArtifact +{ + public required byte[] Bytes { get; init; } + + public required string Sha256 { get; init; } + + public required int LayerCount { get; init; } + + public required int ComponentCount { get; init; } + + public required int EntrypointCount { get; init; } + + public string MediaType { get; init; } = "application/vnd.stellaops.bom-index.v1+binary"; +} + +public sealed class BomIndexBuilder +{ + private static readonly byte[] Magic = Encoding.ASCII.GetBytes("BOMIDX1"); + + public BomIndexArtifact Build(BomIndexBuildRequest request) + { + ArgumentNullException.ThrowIfNull(request); + if (string.IsNullOrWhiteSpace(request.ImageDigest)) + { + throw new ArgumentException("Image digest is required.", nameof(request)); + } + + var normalizedDigest = request.ImageDigest.Trim(); + var graph = request.Graph ?? throw new ArgumentNullException(nameof(request.Graph)); + var layers = graph.Layers.Select(layer => layer.LayerDigest).ToImmutableArray(); + var components = graph.Components; + + var layerIndex = new Dictionary(layers.Length, StringComparer.Ordinal); + for (var i = 0; i < layers.Length; i++) + { + layerIndex[layers[i]] = i; + } + + var entrypointSet = new SortedSet(StringComparer.Ordinal); + foreach (var component in components) + { + if (!component.Usage.Entrypoints.IsDefaultOrEmpty) + { + foreach (var entry in component.Usage.Entrypoints) + { + if (!string.IsNullOrWhiteSpace(entry)) + { + entrypointSet.Add(entry); + } + } + } + } + + var entrypoints = entrypointSet.ToImmutableArray(); + var entrypointIndex = new Dictionary(entrypoints.Length, StringComparer.Ordinal); + for (var i = 0; i < entrypoints.Length; i++) + { + entrypointIndex[entrypoints[i]] = i; + } + + using var buffer = new MemoryStream(); + using var writer = new BinaryWriter(buffer, Encoding.UTF8, leaveOpen: true); + + WriteHeader(writer, normalizedDigest, request.GeneratedAt, layers.Length, components.Length, entrypoints.Length); + WriteLayerTable(writer, layers); + WriteComponentTable(writer, components); + WriteComponentBitmaps(writer, components, layerIndex); + + if (entrypoints.Length > 0) + { + WriteEntrypointTable(writer, entrypoints); + WriteEntrypointBitmaps(writer, components, entrypointIndex); + } + + writer.Flush(); + var bytes = buffer.ToArray(); + var sha256 = ComputeSha256(bytes); + + return new BomIndexArtifact + { + Bytes = bytes, + Sha256 = sha256, + LayerCount = layers.Length, + ComponentCount = components.Length, + EntrypointCount = entrypoints.Length, + }; + } + + private static void WriteHeader(BinaryWriter writer, string imageDigest, DateTimeOffset generatedAt, int layerCount, int componentCount, int entrypointCount) + { + writer.Write(Magic); + writer.Write((ushort)1); // version + + var flags = (ushort)0; + if (entrypointCount > 0) + { + flags |= 0x1; + } + + writer.Write(flags); + + var digestBytes = Encoding.UTF8.GetBytes(imageDigest); + if (digestBytes.Length > ushort.MaxValue) + { + throw new InvalidOperationException("Image digest exceeds maximum length."); + } + + writer.Write((ushort)digestBytes.Length); + writer.Write(digestBytes); + + var unixMicroseconds = ToUnixMicroseconds(generatedAt); + writer.Write(unixMicroseconds); + + writer.Write((uint)layerCount); + writer.Write((uint)componentCount); + writer.Write((uint)entrypointCount); + } + + private static void WriteLayerTable(BinaryWriter writer, ImmutableArray layers) + { + foreach (var layer in layers) + { + WriteUtf8String(writer, layer); + } + } + + private static void WriteComponentTable(BinaryWriter writer, ImmutableArray components) + { + foreach (var component in components) + { + var key = component.Identity.Purl ?? component.Identity.Key; + WriteUtf8String(writer, key); + } + } + + private static void WriteComponentBitmaps(BinaryWriter writer, ImmutableArray components, IReadOnlyDictionary layerIndex) + { + foreach (var component in components) + { + var indices = component.LayerDigests + .Select(digest => layerIndex.TryGetValue(digest, out var index) ? index : -1) + .Where(index => index >= 0) + .Distinct() + .OrderBy(index => index) + .ToArray(); + + var bitmap = RoaringBitmap.Create(indices).Optimize(); + WriteBitmap(writer, bitmap); + } + } + + private static void WriteEntrypointTable(BinaryWriter writer, ImmutableArray entrypoints) + { + foreach (var entry in entrypoints) + { + WriteUtf8String(writer, entry); + } + } + + private static void WriteEntrypointBitmaps(BinaryWriter writer, ImmutableArray components, IReadOnlyDictionary entrypointIndex) + { + foreach (var component in components) + { + var indices = component.Usage.Entrypoints + .Where(entrypointIndex.ContainsKey) + .Select(entry => entrypointIndex[entry]) + .Distinct() + .OrderBy(index => index) + .ToArray(); + + if (indices.Length == 0) + { + writer.Write((uint)0); + continue; + } + + var bitmap = RoaringBitmap.Create(indices).Optimize(); + WriteBitmap(writer, bitmap); + } + } + + private static void WriteBitmap(BinaryWriter writer, RoaringBitmap bitmap) + { + using var ms = new MemoryStream(); + RoaringBitmap.Serialize(bitmap, ms); + var data = ms.ToArray(); + writer.Write((uint)data.Length); + writer.Write(data); + } + + private static void WriteUtf8String(BinaryWriter writer, string value) + { + var bytes = Encoding.UTF8.GetBytes(value ?? string.Empty); + if (bytes.Length > ushort.MaxValue) + { + throw new InvalidOperationException("String value exceeds maximum length supported by BOM index."); + } + + writer.Write((ushort)bytes.Length); + writer.Write(bytes); + } + + private static long ToUnixMicroseconds(DateTimeOffset timestamp) + { + var normalized = timestamp.ToUniversalTime(); + var microseconds = normalized.ToUnixTimeMilliseconds() * 1000L; + microseconds += normalized.Ticks % TimeSpan.TicksPerMillisecond / 10; + return microseconds; + } + + private static string ComputeSha256(byte[] data) + { + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(data); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Emit/Packaging/ScannerArtifactPackageBuilder.cs b/src/StellaOps.Scanner.Emit/Packaging/ScannerArtifactPackageBuilder.cs new file mode 100644 index 00000000..86cd6419 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/Packaging/ScannerArtifactPackageBuilder.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Core.Serialization; +using StellaOps.Scanner.Emit.Composition; +using StellaOps.Scanner.Emit.Index; +using StellaOps.Scanner.Storage.Catalog; + +namespace StellaOps.Scanner.Emit.Packaging; + +public sealed record ScannerArtifactDescriptor +{ + public required ArtifactDocumentType Type { get; init; } + + public required ArtifactDocumentFormat Format { get; init; } + + public required string MediaType { get; init; } + + public required ReadOnlyMemory Content { get; init; } + + public required string Sha256 { get; init; } + + public SbomView? View { get; init; } + + public long Size => Content.Length; +} + +public sealed record ScannerArtifactManifestEntry +{ + public required string Kind { get; init; } + + public required ArtifactDocumentType Type { get; init; } + + public required ArtifactDocumentFormat Format { get; init; } + + public required string MediaType { get; init; } + + public required string Sha256 { get; init; } + + public required long Size { get; init; } + + public SbomView? View { get; init; } +} + +public sealed record ScannerArtifactManifest +{ + public required string ImageDigest { get; init; } + + public required DateTimeOffset GeneratedAt { get; init; } + + public required ImmutableArray Artifacts { get; init; } + + public byte[] ToJsonBytes() + => JsonSerializer.SerializeToUtf8Bytes(this, ScannerJsonOptions.Default); +} + +public sealed record ScannerArtifactPackage +{ + public required ImmutableArray Artifacts { get; init; } + + public required ScannerArtifactManifest Manifest { get; init; } +} + +public sealed class ScannerArtifactPackageBuilder +{ + public ScannerArtifactPackage Build( + string imageDigest, + DateTimeOffset generatedAt, + SbomCompositionResult composition, + BomIndexArtifact bomIndex) + { + if (string.IsNullOrWhiteSpace(imageDigest)) + { + throw new ArgumentException("Image digest is required.", nameof(imageDigest)); + } + + var descriptors = new List(); + + descriptors.Add(CreateDescriptor(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxJson, composition.Inventory.JsonMediaType, composition.Inventory.JsonBytes, composition.Inventory.JsonSha256, SbomView.Inventory)); + descriptors.Add(CreateDescriptor(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxProtobuf, composition.Inventory.ProtobufMediaType, composition.Inventory.ProtobufBytes, composition.Inventory.ProtobufSha256, SbomView.Inventory)); + + if (composition.Usage is not null) + { + descriptors.Add(CreateDescriptor(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxJson, composition.Usage.JsonMediaType, composition.Usage.JsonBytes, composition.Usage.JsonSha256, SbomView.Usage)); + descriptors.Add(CreateDescriptor(ArtifactDocumentType.ImageBom, ArtifactDocumentFormat.CycloneDxProtobuf, composition.Usage.ProtobufMediaType, composition.Usage.ProtobufBytes, composition.Usage.ProtobufSha256, SbomView.Usage)); + } + + descriptors.Add(CreateDescriptor(ArtifactDocumentType.Index, ArtifactDocumentFormat.BomIndex, "application/vnd.stellaops.bom-index.v1+binary", bomIndex.Bytes, bomIndex.Sha256, null)); + + var manifest = new ScannerArtifactManifest + { + ImageDigest = imageDigest.Trim(), + GeneratedAt = generatedAt, + Artifacts = descriptors + .Select(ToManifestEntry) + .OrderBy(entry => entry.Kind, StringComparer.Ordinal) + .ThenBy(entry => entry.Format) + .ToImmutableArray(), + }; + + return new ScannerArtifactPackage + { + Artifacts = descriptors.ToImmutableArray(), + Manifest = manifest, + }; + } + + private static ScannerArtifactDescriptor CreateDescriptor( + ArtifactDocumentType type, + ArtifactDocumentFormat format, + string mediaType, + ReadOnlyMemory content, + string sha256, + SbomView? view) + { + return new ScannerArtifactDescriptor + { + Type = type, + Format = format, + MediaType = mediaType, + Content = content, + Sha256 = sha256, + View = view, + }; + } + + private static ScannerArtifactManifestEntry ToManifestEntry(ScannerArtifactDescriptor descriptor) + { + var kind = descriptor.Type switch + { + ArtifactDocumentType.Index => "bom-index", + ArtifactDocumentType.ImageBom when descriptor.View == SbomView.Usage => "sbom-usage", + ArtifactDocumentType.ImageBom => "sbom-inventory", + ArtifactDocumentType.LayerBom => "layer-sbom", + ArtifactDocumentType.Diff => "diff", + ArtifactDocumentType.Attestation => "attestation", + _ => descriptor.Type.ToString().ToLowerInvariant(), + }; + + return new ScannerArtifactManifestEntry + { + Kind = kind, + Type = descriptor.Type, + Format = descriptor.Format, + MediaType = descriptor.MediaType, + Sha256 = descriptor.Sha256, + Size = descriptor.Size, + View = descriptor.View, + }; + } +} diff --git a/src/StellaOps.Scanner.Emit/StellaOps.Scanner.Emit.csproj b/src/StellaOps.Scanner.Emit/StellaOps.Scanner.Emit.csproj new file mode 100644 index 00000000..911c78f5 --- /dev/null +++ b/src/StellaOps.Scanner.Emit/StellaOps.Scanner.Emit.csproj @@ -0,0 +1,18 @@ + + + net10.0 + enable + enable + true + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Emit/TASKS.md b/src/StellaOps.Scanner.Emit/TASKS.md new file mode 100644 index 00000000..4da81e1b --- /dev/null +++ b/src/StellaOps.Scanner.Emit/TASKS.md @@ -0,0 +1,12 @@ +# Scanner Emit Task Board (Sprint 10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-EMIT-10-601 | DOING (2025-10-19) | Emit Guild | SCANNER-CACHE-10-101 | Compose inventory SBOM (CycloneDX JSON/Protobuf) from layer fragments with deterministic ordering. | Inventory SBOM validated against schema; fixtures confirm deterministic output. | +| SCANNER-EMIT-10-602 | DOING (2025-10-19) | Emit Guild | SCANNER-EMIT-10-601 | Compose usage SBOM leveraging EntryTrace to flag actual usage; ensure separate view toggles. | Usage SBOM tests confirm correct subset; API contract documented. | +| SCANNER-EMIT-10-603 | DOING (2025-10-19) | Emit Guild | SCANNER-EMIT-10-601 | Generate BOM index sidecar (purl table + roaring bitmap + usedByEntrypoint flag). | Index format validated; query helpers proven; stored artifacts hashed deterministically. | +| SCANNER-EMIT-10-604 | DOING (2025-10-19) | Emit Guild | SCANNER-EMIT-10-602 | Package artifacts for export + attestation (naming, compression, manifests). | Export pipeline produces deterministic file paths/hashes; integration test with storage passes. | +| SCANNER-EMIT-10-605 | DOING (2025-10-19) | Emit Guild | SCANNER-EMIT-10-603 | Emit BOM-Index sidecar schema/fixtures (`bom-index@1`) and note CRITICAL PATH for Scheduler. | Schema + fixtures in docs/artifacts/bom-index; tests `BOMIndexGoldenIsStable` green. | +| SCANNER-EMIT-10-606 | DOING (2025-10-19) | Emit Guild | SCANNER-EMIT-10-605 | Integrate EntryTrace usage flags into BOM-Index; document semantics. | Usage bits present in sidecar; integration tests with EntryTrace fixtures pass. | +| SCANNER-EMIT-17-701 | TODO | Emit Guild, Native Analyzer Guild | SCANNER-EMIT-10-602 | Record GNU build-id for ELF components and surface it in inventory/usage SBOM plus diff payloads with deterministic ordering. | Native analyzer emits buildId for every ELF executable/library, SBOM/diff fixtures updated with canonical `buildId` field, regression tests prove stability, docs call out debug-symbol lookup flow. | +| SCANNER-EMIT-10-607 | TODO | Emit Guild | SCANNER-EMIT-10-604, POLICY-CORE-09-005 | Embed scoring inputs, confidence band, and `quietedBy` provenance into CycloneDX 1.6 and DSSE predicates; verify deterministic serialization. | SBOM/attestation fixtures include score, inputs, configVersion, quiet metadata; golden tests confirm canonical output. | diff --git a/src/StellaOps.Scanner.EntryTrace.Tests/EntryTraceAnalyzerTests.cs b/src/StellaOps.Scanner.EntryTrace.Tests/EntryTraceAnalyzerTests.cs new file mode 100644 index 00000000..c7a4c8c2 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace.Tests/EntryTraceAnalyzerTests.cs @@ -0,0 +1,136 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.EntryTrace.Diagnostics; +using Xunit; + +namespace StellaOps.Scanner.EntryTrace.Tests; + +public sealed class EntryTraceAnalyzerTests +{ + private static EntryTraceAnalyzer CreateAnalyzer() + { + var options = Options.Create(new EntryTraceAnalyzerOptions + { + MaxDepth = 32, + FollowRunParts = true + }); + return new EntryTraceAnalyzer(options, new EntryTraceMetrics(), NullLogger.Instance); + } + + [Fact] + public async Task ResolveAsync_FollowsShellIncludeAndPythonModule() + { + var fs = new TestRootFileSystem(); + fs.AddFile("/entrypoint.sh", """ + #!/bin/sh + source /opt/setup.sh + exec python -m app.main --flag + """); + fs.AddFile("/opt/setup.sh", """ + #!/bin/sh + run-parts /opt/setup.d + """); + fs.AddDirectory("/opt/setup.d"); + fs.AddFile("/opt/setup.d/001-node.sh", """ + #!/bin/sh + exec node /app/server.js + """); + fs.AddFile("/opt/setup.d/010-java.sh", """ + #!/bin/sh + java -jar /app/app.jar + """); + fs.AddFile("/usr/bin/python", "#!/usr/bin/env python3\n", executable: true); + fs.AddFile("/usr/bin/node", "#!/usr/bin/env node\n", executable: true); + fs.AddFile("/usr/bin/java", "", executable: true); + fs.AddFile("/app/server.js", "console.log('hello');", executable: true); + fs.AddFile("/app/app.jar", string.Empty, executable: true); + + var analyzer = CreateAnalyzer(); + var context = new EntryTraceContext( + fs, + ImmutableDictionary.Empty, + ImmutableArray.Create("/usr/bin", "/usr/local/bin"), + "/", + "sha256:image", + "scan-entrytrace-1", + NullLogger.Instance); + + var spec = EntrypointSpecification.FromExecForm(new[] { "/entrypoint.sh" }, Array.Empty()); + var result = await analyzer.ResolveAsync(spec, context); + + Assert.Equal(EntryTraceOutcome.Resolved, result.Outcome); + Assert.Empty(result.Diagnostics); + + var nodeNames = result.Nodes.Select(n => (n.Kind, n.DisplayName)).ToArray(); + Assert.Contains((EntryTraceNodeKind.Command, "/entrypoint.sh"), nodeNames); + Assert.Contains((EntryTraceNodeKind.Include, "/opt/setup.sh"), nodeNames); + Assert.Contains(nodeNames, tuple => tuple.Kind == EntryTraceNodeKind.Command && tuple.DisplayName == "python"); + Assert.Contains(nodeNames, tuple => tuple.Kind == EntryTraceNodeKind.Command && tuple.DisplayName == "node"); + Assert.Contains(nodeNames, tuple => tuple.Kind == EntryTraceNodeKind.Command && tuple.DisplayName == "java"); + Assert.Contains(nodeNames, tuple => tuple.Kind == EntryTraceNodeKind.RunPartsDirectory && tuple.DisplayName == "/opt/setup.d"); + + Assert.Contains(result.Edges, edge => edge.Relationship == "python-module" && edge.Metadata is { } metadata && metadata.TryGetValue("module", out var module) && module == "app.main"); + } + + [Fact] + public async Task ResolveAsync_RecordsDiagnosticsForMissingInclude() + { + var fs = new TestRootFileSystem(); + fs.AddFile("/entrypoint.sh", """ + #!/bin/sh + source /missing/setup.sh + exec /bin/true + """); + fs.AddFile("/bin/true", string.Empty, executable: true); + + var analyzer = CreateAnalyzer(); + var context = new EntryTraceContext( + fs, + ImmutableDictionary.Empty, + ImmutableArray.Create("/bin"), + "/", + "sha256:image", + "scan-entrytrace-2", + NullLogger.Instance); + + var spec = EntrypointSpecification.FromExecForm(new[] { "/entrypoint.sh" }, Array.Empty()); + var result = await analyzer.ResolveAsync(spec, context); + + Assert.Equal(EntryTraceOutcome.PartiallyResolved, result.Outcome); + Assert.Single(result.Diagnostics); + Assert.Equal(EntryTraceUnknownReason.MissingFile, result.Diagnostics[0].Reason); + } + + [Fact] + public async Task ResolveAsync_IsDeterministic() + { + var fs = new TestRootFileSystem(); + fs.AddFile("/entrypoint.sh", """ + #!/bin/sh + exec node /app/index.js + """); + fs.AddFile("/usr/bin/node", string.Empty, executable: true); + fs.AddFile("/app/index.js", "console.log('deterministic');", executable: true); + + var analyzer = CreateAnalyzer(); + var context = new EntryTraceContext( + fs, + ImmutableDictionary.Empty, + ImmutableArray.Create("/usr/bin"), + "/", + "sha256:image", + "scan-entrytrace-3", + NullLogger.Instance); + + var spec = EntrypointSpecification.FromExecForm(new[] { "/entrypoint.sh" }, Array.Empty()); + var first = await analyzer.ResolveAsync(spec, context); + var second = await analyzer.ResolveAsync(spec, context); + + Assert.Equal(first.Outcome, second.Outcome); + Assert.Equal(first.Diagnostics, second.Diagnostics); + Assert.Equal(first.Nodes.Select(n => (n.Kind, n.DisplayName)).ToArray(), second.Nodes.Select(n => (n.Kind, n.DisplayName)).ToArray()); + Assert.Equal(first.Edges.Select(e => (e.FromNodeId, e.ToNodeId, e.Relationship)).ToArray(), + second.Edges.Select(e => (e.FromNodeId, e.ToNodeId, e.Relationship)).ToArray()); + } +} diff --git a/src/StellaOps.Scanner.EntryTrace.Tests/ShellParserTests.cs b/src/StellaOps.Scanner.EntryTrace.Tests/ShellParserTests.cs new file mode 100644 index 00000000..f8920c49 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace.Tests/ShellParserTests.cs @@ -0,0 +1,33 @@ +using StellaOps.Scanner.EntryTrace.Parsing; +using Xunit; + +namespace StellaOps.Scanner.EntryTrace.Tests; + +public sealed class ShellParserTests +{ + [Fact] + public void Parse_ProducesDeterministicNodes() + { + const string script = """ + #!/bin/sh + source /opt/init.sh + if [ -f /etc/profile ]; then + . /etc/profile + fi + + run-parts /etc/entry.d + exec python -m app.main --flag + """; + + var first = ShellParser.Parse(script); + var second = ShellParser.Parse(script); + + Assert.Equal(first.Nodes.Length, second.Nodes.Length); + var actual = first.Nodes.Select(n => n.GetType().Name).ToArray(); + var expected = new[] { nameof(ShellIncludeNode), nameof(ShellIfNode), nameof(ShellRunPartsNode), nameof(ShellExecNode) }; + Assert.Equal(expected, actual); + + var actualSecond = second.Nodes.Select(n => n.GetType().Name).ToArray(); + Assert.Equal(expected, actualSecond); + } +} diff --git a/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj b/src/StellaOps.Scanner.EntryTrace.Tests/StellaOps.Scanner.EntryTrace.Tests.csproj similarity index 51% rename from src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj rename to src/StellaOps.Scanner.EntryTrace.Tests/StellaOps.Scanner.EntryTrace.Tests.csproj index 0c41a48f..e0257e8c 100644 --- a/src/StellaOps.Feedser.Normalization.Tests/StellaOps.Feedser.Normalization.Tests.csproj +++ b/src/StellaOps.Scanner.EntryTrace.Tests/StellaOps.Scanner.EntryTrace.Tests.csproj @@ -5,7 +5,9 @@ enable - - + + + + diff --git a/src/StellaOps.Scanner.EntryTrace.Tests/TestRootFileSystem.cs b/src/StellaOps.Scanner.EntryTrace.Tests/TestRootFileSystem.cs new file mode 100644 index 00000000..93a29e8d --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace.Tests/TestRootFileSystem.cs @@ -0,0 +1,180 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using StellaOps.Scanner.EntryTrace; + +namespace StellaOps.Scanner.EntryTrace.Tests; + +internal sealed class TestRootFileSystem : IRootFileSystem +{ + private readonly Dictionary _entries = new(StringComparer.Ordinal); + private readonly HashSet _directories = new(StringComparer.Ordinal); + + public TestRootFileSystem() + { + _directories.Add("/"); + } + + public void AddFile(string path, string content, bool executable = true, string? layer = "sha256:layer-a") + { + var normalized = Normalize(path); + var directory = Path.GetDirectoryName(normalized); + if (!string.IsNullOrEmpty(directory)) + { + _directories.Add(directory!); + } + + _entries[normalized] = new FileEntry(normalized, content, executable, layer, IsDirectory: false); + } + + public void AddDirectory(string path) + { + var normalized = Normalize(path); + _directories.Add(normalized); + } + + public bool TryResolveExecutable(string name, IReadOnlyList searchPaths, out RootFileDescriptor descriptor) + { + if (name.Contains('/', StringComparison.Ordinal)) + { + var normalized = Normalize(name); + if (_entries.TryGetValue(normalized, out var file) && file.IsExecutable) + { + descriptor = file.ToDescriptor(); + return true; + } + + descriptor = null!; + return false; + } + + foreach (var prefix in searchPaths) + { + var candidate = Combine(prefix, name); + if (_entries.TryGetValue(candidate, out var file) && file.IsExecutable) + { + descriptor = file.ToDescriptor(); + return true; + } + } + + descriptor = null!; + return false; + } + + public bool TryReadAllText(string path, out RootFileDescriptor descriptor, out string content) + { + var normalized = Normalize(path); + if (_entries.TryGetValue(normalized, out var file)) + { + descriptor = file.ToDescriptor(); + content = file.Content; + return true; + } + + descriptor = null!; + content = string.Empty; + return false; + } + + public ImmutableArray EnumerateDirectory(string path) + { + var normalized = Normalize(path); + var builder = ImmutableArray.CreateBuilder(); + + foreach (var file in _entries.Values) + { + var directory = Normalize(Path.GetDirectoryName(file.Path) ?? "/"); + if (string.Equals(directory, normalized, StringComparison.Ordinal)) + { + builder.Add(file.ToDescriptor()); + } + } + + return builder.ToImmutable(); + } + + public bool DirectoryExists(string path) + { + var normalized = Normalize(path); + return _directories.Contains(normalized); + } + + private static string Combine(string prefix, string name) + { + var normalizedPrefix = Normalize(prefix); + if (normalizedPrefix == "/") + { + return Normalize("/" + name); + } + + return Normalize($"{normalizedPrefix}/{name}"); + } + + private static string Normalize(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return "/"; + } + + var text = path.Replace('\\', '/').Trim(); + if (!text.StartsWith("/", StringComparison.Ordinal)) + { + text = "/" + text; + } + + var parts = new List(); + foreach (var part in text.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (part == ".") + { + continue; + } + + if (part == "..") + { + if (parts.Count > 0) + { + parts.RemoveAt(parts.Count - 1); + } + continue; + } + + parts.Add(part); + } + + return "/" + string.Join('/', parts); + } + + private sealed record FileEntry(string Path, string Content, bool IsExecutable, string? Layer, bool IsDirectory) + { + public RootFileDescriptor ToDescriptor() + { + var shebang = ExtractShebang(Content); + return new RootFileDescriptor(Path, Layer, IsExecutable, IsDirectory, shebang); + } + } + + private static string? ExtractShebang(string content) + { + if (string.IsNullOrEmpty(content)) + { + return null; + } + + using var reader = new StringReader(content); + var firstLine = reader.ReadLine(); + if (firstLine is null) + { + return null; + } + + if (!firstLine.StartsWith("#!", StringComparison.Ordinal)) + { + return null; + } + + return firstLine[2..].Trim(); + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/AGENTS.md b/src/StellaOps.Scanner.EntryTrace/AGENTS.md new file mode 100644 index 00000000..3ad8f323 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/AGENTS.md @@ -0,0 +1,32 @@ +# StellaOps.Scanner.EntryTrace — Agent Charter + +## Mission +Resolve container `ENTRYPOINT`/`CMD` chains into deterministic call graphs that fuel usage-aware SBOMs, policy explainability, and runtime drift detection. Implement the EntryTrace analyzers and expose them as restart-time plug-ins for the Scanner Worker. + +## Scope +- Parse POSIX/Bourne shell constructs (exec, command, case, if, source/run-parts) with deterministic AST output. +- Walk layered root filesystems to resolve PATH lookups, interpreter hand-offs (Python/Node/Java), and record evidence. +- Surface explainable diagnostics for unresolved branches (env indirection, missing files, unsupported syntax) and emit metrics. +- Package analyzers as signed plug-ins under `plugins/scanner/entrytrace/`, guarded by restart-only policy. + +## Out of Scope +- SBOM emission/diffing (owned by `Scanner.Emit`/`Scanner.Diff`). +- Runtime enforcement or live drift reconciliation (owned by Zastava). +- Registry/network fetchers beyond file lookups inside extracted layers. + +## Interfaces & Contracts +- Primary entry point: `IEntryTraceAnalyzer.ResolveAsync` returning a deterministic `EntryTraceGraph`. +- Graph nodes must include file path, line span, interpreter classification, evidence source, and follow `Scanner.Core` timestamp/ID helpers when emitting events. +- Diagnostics must enumerate unknown reasons from fixed enum; metrics tagged `entrytrace.*`. +- Plug-ins register via `IEntryTraceAnalyzerFactory` and must validate against `IPluginCatalogGuard`. + +## Observability & Security +- No dynamic assembly loading beyond restart-time plug-in catalog. +- Structured logs include `scanId`, `imageDigest`, `layerDigest`, `command`, `reason`. +- Metrics counters: `entrytrace_resolutions_total{result}`, `entrytrace_unresolved_total{reason}`. +- Deny `source` directives outside image root; sandbox file IO via provided `IRootFileSystem`. + +## Testing +- Unit tests live in `../StellaOps.Scanner.EntryTrace.Tests` with golden fixtures under `Fixtures/`. +- Determinism harness: same inputs produce byte-identical serialized graphs. +- Parser fuzz seeds captured for regression; interpreter tracers validated with sample scripts for Python, Node, Java launchers. diff --git a/src/StellaOps.Scanner.EntryTrace/Diagnostics/EntryTraceMetrics.cs b/src/StellaOps.Scanner.EntryTrace/Diagnostics/EntryTraceMetrics.cs new file mode 100644 index 00000000..bb37b4c1 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/Diagnostics/EntryTraceMetrics.cs @@ -0,0 +1,51 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Scanner.EntryTrace.Diagnostics; + +public static class EntryTraceInstrumentation +{ + public static readonly Meter Meter = new("stellaops.scanner.entrytrace", "1.0.0"); +} + +public sealed class EntryTraceMetrics +{ + private readonly Counter _resolutions; + private readonly Counter _unresolved; + + public EntryTraceMetrics() + { + _resolutions = EntryTraceInstrumentation.Meter.CreateCounter( + "entrytrace_resolutions_total", + description: "Number of entry trace attempts by outcome."); + _unresolved = EntryTraceInstrumentation.Meter.CreateCounter( + "entrytrace_unresolved_total", + description: "Number of unresolved entry trace hops by reason."); + } + + public void RecordOutcome(string imageDigest, string scanId, EntryTraceOutcome outcome) + { + _resolutions.Add(1, CreateTags(imageDigest, scanId, ("outcome", outcome.ToString().ToLowerInvariant()))); + } + + public void RecordUnknown(string imageDigest, string scanId, EntryTraceUnknownReason reason) + { + _unresolved.Add(1, CreateTags(imageDigest, scanId, ("reason", reason.ToString().ToLowerInvariant()))); + } + + private static KeyValuePair[] CreateTags(string imageDigest, string scanId, params (string Key, object? Value)[] extras) + { + var tags = new List>(2 + extras.Length) + { + new("image", imageDigest), + new("scan.id", scanId) + }; + + foreach (var extra in extras) + { + tags.Add(new KeyValuePair(extra.Key, extra.Value)); + } + + return tags.ToArray(); + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzer.cs b/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzer.cs new file mode 100644 index 00000000..0bf38977 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzer.cs @@ -0,0 +1,963 @@ +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.EntryTrace.Diagnostics; +using StellaOps.Scanner.EntryTrace.Parsing; + +namespace StellaOps.Scanner.EntryTrace; + +public sealed class EntryTraceAnalyzer : IEntryTraceAnalyzer +{ + private readonly EntryTraceAnalyzerOptions _options; + private readonly EntryTraceMetrics _metrics; + private readonly ILogger _logger; + + public EntryTraceAnalyzer( + IOptions options, + EntryTraceMetrics metrics, + ILogger logger) + { + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + if (_options.MaxDepth <= 0) + { + _options.MaxDepth = 32; + } + + if (string.IsNullOrWhiteSpace(_options.DefaultPath)) + { + _options.DefaultPath = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + } + } + + public ValueTask ResolveAsync( + EntrypointSpecification entrypoint, + EntryTraceContext context, + CancellationToken cancellationToken = default) + { + if (entrypoint is null) + { + throw new ArgumentNullException(nameof(entrypoint)); + } + + if (context is null) + { + throw new ArgumentNullException(nameof(context)); + } + + cancellationToken.ThrowIfCancellationRequested(); + + var builder = new Builder( + entrypoint, + context, + _options, + _metrics, + _logger); + + var graph = builder.BuildGraph(); + _metrics.RecordOutcome(context.ImageDigest, context.ScanId, graph.Outcome); + foreach (var diagnostic in graph.Diagnostics) + { + _metrics.RecordUnknown(context.ImageDigest, context.ScanId, diagnostic.Reason); + } + + return ValueTask.FromResult(graph); + } + + private sealed class Builder + { + private readonly EntrypointSpecification _entrypoint; + private readonly EntryTraceContext _context; + private readonly EntryTraceAnalyzerOptions _options; + private readonly EntryTraceMetrics _metrics; + private readonly ILogger _logger; + private readonly ImmutableArray _pathEntries; + private readonly List _nodes = new(); + private readonly List _edges = new(); + private readonly List _diagnostics = new(); + private readonly HashSet _visitedScripts = new(StringComparer.Ordinal); + private readonly HashSet _visitedCommands = new(StringComparer.Ordinal); + private int _nextNodeId = 1; + + public Builder( + EntrypointSpecification entrypoint, + EntryTraceContext context, + EntryTraceAnalyzerOptions options, + EntryTraceMetrics metrics, + ILogger logger) + { + _entrypoint = entrypoint; + _context = context; + _options = options; + _metrics = metrics; + _logger = logger; + _pathEntries = DeterminePath(context); + } + + private static ImmutableArray DeterminePath(EntryTraceContext context) + { + if (context.Path.Length > 0) + { + return context.Path; + } + + if (context.Environment.TryGetValue("PATH", out var raw) && !string.IsNullOrWhiteSpace(raw)) + { + return raw.Split(':').Select(p => p.Trim()).Where(p => p.Length > 0).ToImmutableArray(); + } + + return ImmutableArray.Empty; + } + + public EntryTraceGraph BuildGraph() + { + var initialArgs = ComposeInitialCommand(_entrypoint); + if (initialArgs.Length == 0) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Error, + EntryTraceUnknownReason.CommandNotFound, + "ENTRYPOINT/CMD yielded no executable command.", + Span: null, + RelatedPath: null)); + return ToGraph(EntryTraceOutcome.Unresolved); + } + + ResolveCommand(initialArgs, parent: null, originSpan: null, depth: 0, relationship: "entrypoint"); + + var outcome = DetermineOutcome(); + return ToGraph(outcome); + } + + private EntryTraceOutcome DetermineOutcome() + { + if (_diagnostics.Count == 0) + { + return EntryTraceOutcome.Resolved; + } + + return _diagnostics.Any(d => d.Severity == EntryTraceDiagnosticSeverity.Error) + ? EntryTraceOutcome.Unresolved + : EntryTraceOutcome.PartiallyResolved; + } + + private EntryTraceGraph ToGraph(EntryTraceOutcome outcome) + { + return new EntryTraceGraph( + outcome, + _nodes.ToImmutableArray(), + _edges.ToImmutableArray(), + _diagnostics.ToImmutableArray()); + } + + private ImmutableArray ComposeInitialCommand(EntrypointSpecification specification) + { + if (specification.Entrypoint.Length > 0) + { + if (specification.Command.Length > 0) + { + return specification.Entrypoint.Concat(specification.Command).ToImmutableArray(); + } + + return specification.Entrypoint; + } + + if (specification.Command.Length > 0) + { + return specification.Command; + } + + if (!string.IsNullOrWhiteSpace(specification.EntrypointShell)) + { + return ImmutableArray.Create("/bin/sh", "-c", specification.EntrypointShell!); + } + + if (!string.IsNullOrWhiteSpace(specification.CommandShell)) + { + return ImmutableArray.Create("/bin/sh", "-c", specification.CommandShell!); + } + + return ImmutableArray.Empty; + } + + private void ResolveCommand( + ImmutableArray arguments, + EntryTraceNode? parent, + EntryTraceSpan? originSpan, + int depth, + string relationship) + { + if (arguments.Length == 0) + { + return; + } + + if (depth >= _options.MaxDepth) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.RecursionLimitReached, + $"Recursion depth limit {_options.MaxDepth} reached while resolving '{arguments[0]}'.", + originSpan, + RelatedPath: null)); + return; + } + + var commandName = arguments[0]; + var evidence = default(EntryTraceEvidence?); + var descriptor = default(RootFileDescriptor); + + if (!TryResolveExecutable(commandName, out descriptor, out evidence)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.CommandNotFound, + $"Command '{commandName}' not found in PATH.", + originSpan, + RelatedPath: null)); + return; + } + + var node = AddNode( + EntryTraceNodeKind.Command, + commandName, + arguments, + DetermineInterpreterKind(descriptor), + evidence, + originSpan); + + if (parent is not null) + { + _edges.Add(new EntryTraceEdge(parent.Id, node.Id, relationship, Metadata: null)); + } + + if (!_visitedCommands.Add(descriptor.Path)) + { + // Prevent infinite loops when scripts call themselves recursively. + return; + } + + if (TryFollowInterpreter(node, descriptor, arguments, depth)) + { + return; + } + + if (TryFollowShell(node, descriptor, arguments, depth)) + { + return; + } + + // Terminal executable. + } + + private bool TryResolveExecutable( + string commandName, + out RootFileDescriptor descriptor, + out EntryTraceEvidence? evidence) + { + evidence = null; + + if (commandName.Contains('/', StringComparison.Ordinal)) + { + if (_context.FileSystem.TryReadAllText(commandName, out descriptor, out _)) + { + evidence = new EntryTraceEvidence(commandName, descriptor.LayerDigest, "path", null); + return true; + } + + if (_context.FileSystem.TryResolveExecutable(commandName, Array.Empty(), out descriptor)) + { + evidence = new EntryTraceEvidence(descriptor.Path, descriptor.LayerDigest, "path", null); + return true; + } + } + + if (_context.FileSystem.TryResolveExecutable(commandName, _pathEntries, out descriptor)) + { + evidence = new EntryTraceEvidence(descriptor.Path, descriptor.LayerDigest, "path-search", new Dictionary + { + ["command"] = commandName + }); + return true; + } + + descriptor = null!; + return false; + } + + private bool TryFollowInterpreter( + EntryTraceNode node, + RootFileDescriptor descriptor, + ImmutableArray arguments, + int depth) + { + var interpreter = DetermineInterpreterKind(descriptor); + if (interpreter == EntryTraceInterpreterKind.None) + { + interpreter = DetectInterpreterFromCommand(arguments); + } + + if (interpreter == EntryTraceInterpreterKind.None) + { + return false; + } + + switch (interpreter) + { + case EntryTraceInterpreterKind.Python: + return HandlePython(node, arguments, descriptor, depth); + case EntryTraceInterpreterKind.Node: + return HandleNode(node, arguments, descriptor, depth); + case EntryTraceInterpreterKind.Java: + return HandleJava(node, arguments, descriptor, depth); + default: + return false; + } + } + + private EntryTraceInterpreterKind DetermineInterpreterKind(RootFileDescriptor descriptor) + { + if (descriptor.ShebangInterpreter is null) + { + return EntryTraceInterpreterKind.None; + } + + var shebang = descriptor.ShebangInterpreter.ToLowerInvariant(); + if (shebang.Contains("python", StringComparison.Ordinal)) + { + return EntryTraceInterpreterKind.Python; + } + + if (shebang.Contains("node", StringComparison.Ordinal)) + { + return EntryTraceInterpreterKind.Node; + } + + if (shebang.Contains("java", StringComparison.Ordinal)) + { + return EntryTraceInterpreterKind.Java; + } + + if (shebang.Contains("sh", StringComparison.Ordinal) || shebang.Contains("bash", StringComparison.Ordinal)) + { + return EntryTraceInterpreterKind.None; + } + + return EntryTraceInterpreterKind.None; + } + + private EntryTraceInterpreterKind DetectInterpreterFromCommand(ImmutableArray arguments) + { + if (arguments.Length == 0) + { + return EntryTraceInterpreterKind.None; + } + + var command = arguments[0]; + if (command.Equals("python", StringComparison.OrdinalIgnoreCase) || + command.StartsWith("python", StringComparison.OrdinalIgnoreCase)) + { + return EntryTraceInterpreterKind.Python; + } + + if (command.Equals("node", StringComparison.OrdinalIgnoreCase) || + command.Equals("nodejs", StringComparison.OrdinalIgnoreCase)) + { + return EntryTraceInterpreterKind.Node; + } + + if (command.Equals("java", StringComparison.OrdinalIgnoreCase)) + { + return EntryTraceInterpreterKind.Java; + } + + return EntryTraceInterpreterKind.None; + } + + private bool HandlePython( + EntryTraceNode node, + ImmutableArray arguments, + RootFileDescriptor descriptor, + int depth) + { + if (arguments.Length < 2) + { + return false; + } + + var argIndex = 1; + var moduleMode = false; + string? moduleName = null; + string? scriptPath = null; + + while (argIndex < arguments.Length) + { + var current = arguments[argIndex]; + if (current == "-m" && argIndex + 1 < arguments.Length) + { + moduleMode = true; + moduleName = arguments[argIndex + 1]; + break; + } + + if (!current.StartsWith("-", StringComparison.Ordinal)) + { + scriptPath = current; + break; + } + + argIndex++; + } + + if (moduleMode && moduleName is not null) + { + _edges.Add(new EntryTraceEdge(node.Id, node.Id, "python-module", new Dictionary + { + ["module"] = moduleName + })); + return true; + } + + if (scriptPath is null) + { + return false; + } + + if (!_context.FileSystem.TryReadAllText(scriptPath, out var scriptDescriptor, out var content)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.MissingFile, + $"Python script '{scriptPath}' was not found.", + Span: null, + RelatedPath: scriptPath)); + return true; + } + + var scriptNode = AddNode( + EntryTraceNodeKind.Script, + scriptPath, + ImmutableArray.Empty, + EntryTraceInterpreterKind.Python, + new EntryTraceEvidence(scriptDescriptor.Path, scriptDescriptor.LayerDigest, "script", null), + null); + + _edges.Add(new EntryTraceEdge(node.Id, scriptNode.Id, "executes", null)); + + if (IsLikelyShell(content)) + { + ResolveShellScript(content, scriptDescriptor.Path, scriptNode, depth + 1); + } + + return true; + } + + private bool HandleNode( + EntryTraceNode node, + ImmutableArray arguments, + RootFileDescriptor descriptor, + int depth) + { + if (arguments.Length < 2) + { + return false; + } + + var scriptArg = arguments.Skip(1).FirstOrDefault(a => !a.StartsWith("-", StringComparison.Ordinal)); + if (string.IsNullOrWhiteSpace(scriptArg)) + { + return false; + } + + if (!_context.FileSystem.TryReadAllText(scriptArg, out var scriptDescriptor, out var content)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.MissingFile, + $"Node script '{scriptArg}' was not found.", + Span: null, + RelatedPath: scriptArg)); + return true; + } + + var scriptNode = AddNode( + EntryTraceNodeKind.Script, + scriptArg, + ImmutableArray.Empty, + EntryTraceInterpreterKind.Node, + new EntryTraceEvidence(scriptDescriptor.Path, scriptDescriptor.LayerDigest, "script", null), + null); + + _edges.Add(new EntryTraceEdge(node.Id, scriptNode.Id, "executes", null)); + return true; + } + + private bool HandleJava( + EntryTraceNode node, + ImmutableArray arguments, + RootFileDescriptor descriptor, + int depth) + { + if (arguments.Length < 2) + { + return false; + } + + string? jar = null; + string? mainClass = null; + + for (var i = 1; i < arguments.Length; i++) + { + var arg = arguments[i]; + if (arg == "-jar" && i + 1 < arguments.Length) + { + jar = arguments[i + 1]; + break; + } + + if (!arg.StartsWith("-", StringComparison.Ordinal) && mainClass is null) + { + mainClass = arg; + } + } + + if (jar is not null) + { + if (!_context.FileSystem.TryResolveExecutable(jar, Array.Empty(), out var jarDescriptor)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.JarNotFound, + $"Java JAR '{jar}' not found.", + Span: null, + RelatedPath: jar)); + } + else + { + var jarNode = AddNode( + EntryTraceNodeKind.Executable, + jarDescriptor.Path, + ImmutableArray.Empty, + EntryTraceInterpreterKind.Java, + new EntryTraceEvidence(jarDescriptor.Path, jarDescriptor.LayerDigest, "jar", null), + null); + _edges.Add(new EntryTraceEdge(node.Id, jarNode.Id, "executes", null)); + } + + return true; + } + + if (mainClass is not null) + { + _edges.Add(new EntryTraceEdge(node.Id, node.Id, "java-main", new Dictionary + { + ["class"] = mainClass + })); + return true; + } + + return false; + } + + private bool TryFollowShell( + EntryTraceNode node, + RootFileDescriptor descriptor, + ImmutableArray arguments, + int depth) + { + if (!IsShellExecutable(descriptor, arguments)) + { + return false; + } + + if (arguments.Length >= 2 && arguments[1] == "-c" && arguments.Length >= 3) + { + var scriptText = arguments[2]; + ResolveShellScript(scriptText, descriptor.Path, node, depth + 1); + return true; + } + + if (arguments.Length >= 2) + { + var candidate = arguments[1]; + if (_context.FileSystem.TryReadAllText(candidate, out var scriptDescriptor, out var content)) + { + var scriptNode = AddNode( + EntryTraceNodeKind.Script, + candidate, + ImmutableArray.Empty, + EntryTraceInterpreterKind.None, + new EntryTraceEvidence(scriptDescriptor.Path, scriptDescriptor.LayerDigest, "script", null), + null); + _edges.Add(new EntryTraceEdge(node.Id, scriptNode.Id, "executes", null)); + ResolveShellScript(content, scriptDescriptor.Path, scriptNode, depth + 1); + return true; + } + } + + if (arguments.Length == 1) + { + if (_context.FileSystem.TryReadAllText(descriptor.Path, out var scriptDescriptor, out var content)) + { + var scriptNode = AddNode( + EntryTraceNodeKind.Script, + descriptor.Path, + ImmutableArray.Empty, + EntryTraceInterpreterKind.None, + new EntryTraceEvidence(scriptDescriptor.Path, scriptDescriptor.LayerDigest, "script", null), + null); + _edges.Add(new EntryTraceEdge(node.Id, scriptNode.Id, "executes", null)); + ResolveShellScript(content, scriptDescriptor.Path, scriptNode, depth + 1); + return true; + } + } + + return false; + } + + private static bool IsShellExecutable(RootFileDescriptor descriptor, ImmutableArray arguments) + { + if (descriptor.ShebangInterpreter is not null && + (descriptor.ShebangInterpreter.Contains("sh", StringComparison.OrdinalIgnoreCase) || + descriptor.ShebangInterpreter.Contains("bash", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + var command = arguments[0]; + return command is "/bin/sh" or "sh" or "bash" or "/bin/bash"; + } + + private void ResolveShellScript( + string scriptContent, + string scriptPath, + EntryTraceNode parent, + int depth) + { + if (_visitedScripts.Contains(scriptPath)) + { + return; + } + + _visitedScripts.Add(scriptPath); + + ShellScript ast; + try + { + ast = ShellParser.Parse(scriptContent); + } + catch (Exception ex) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.UnsupportedSyntax, + $"Failed to parse shell script '{scriptPath}': {ex.Message}", + Span: null, + RelatedPath: scriptPath)); + return; + } + + foreach (var node in ast.Nodes) + { + HandleShellNode(node, parent, scriptPath, depth); + } + } + + private void HandleShellNode( + ShellNode node, + EntryTraceNode parent, + string scriptPath, + int depth) + { + switch (node) + { + case ShellExecNode execNode: + { + var args = MaterializeArguments(execNode.Arguments); + if (args.Length <= 1) + { + break; + } + + var execArgs = args.RemoveAt(0); + ResolveCommand(execArgs, parent, ToEntryTraceSpan(execNode.Span, scriptPath), depth + 1, "executes"); + break; + } + case ShellIncludeNode includeNode: + { + var includeArg = includeNode.PathExpression; + var includePath = ResolveScriptPath(scriptPath, includeArg); + if (!_context.FileSystem.TryReadAllText(includePath, out var descriptor, out var content)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.MissingFile, + $"Included script '{includePath}' not found.", + ToEntryTraceSpan(includeNode.Span, scriptPath), + includePath)); + break; + } + + var includeTraceNode = AddNode( + EntryTraceNodeKind.Include, + includePath, + ImmutableArray.Empty, + EntryTraceInterpreterKind.None, + new EntryTraceEvidence(descriptor.Path, descriptor.LayerDigest, "include", null), + ToEntryTraceSpan(includeNode.Span, scriptPath)); + + _edges.Add(new EntryTraceEdge(parent.Id, includeTraceNode.Id, "includes", null)); + ResolveShellScript(content, descriptor.Path, includeTraceNode, depth + 1); + break; + } + case ShellRunPartsNode runPartsNode when _options.FollowRunParts: + { + var directory = ResolveScriptPath(scriptPath, runPartsNode.DirectoryExpression); + if (!_context.FileSystem.DirectoryExists(directory)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.MissingFile, + $"run-parts directory '{directory}' not found.", + ToEntryTraceSpan(runPartsNode.Span, scriptPath), + directory)); + break; + } + + var entries = _context.FileSystem.EnumerateDirectory(directory) + .Where(e => !e.IsDirectory && e.IsExecutable) + .OrderBy(e => e.Path, StringComparer.Ordinal) + .Take(_options.RunPartsLimit) + .ToList(); + + if (entries.Count == 0) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Info, + EntryTraceUnknownReason.RunPartsEmpty, + $"run-parts directory '{directory}' contained no executable files.", + ToEntryTraceSpan(runPartsNode.Span, scriptPath), + directory)); + break; + } + + var dirNode = AddNode( + EntryTraceNodeKind.RunPartsDirectory, + directory, + ImmutableArray.Empty, + EntryTraceInterpreterKind.None, + new EntryTraceEvidence(directory, null, "run-parts", null), + ToEntryTraceSpan(runPartsNode.Span, scriptPath)); + _edges.Add(new EntryTraceEdge(parent.Id, dirNode.Id, "run-parts", null)); + + foreach (var entry in entries) + { + var childNode = AddNode( + EntryTraceNodeKind.RunPartsScript, + entry.Path, + ImmutableArray.Empty, + EntryTraceInterpreterKind.None, + new EntryTraceEvidence(entry.Path, entry.LayerDigest, "run-parts", null), + null); + _edges.Add(new EntryTraceEdge(dirNode.Id, childNode.Id, "executes", null)); + + if (_context.FileSystem.TryReadAllText(entry.Path, out var childDescriptor, out var content)) + { + ResolveShellScript(content, childDescriptor.Path, childNode, depth + 1); + } + } + + break; + } + case ShellIfNode ifNode: + { + foreach (var branch in ifNode.Branches) + { + foreach (var inner in branch.Body) + { + HandleShellNode(inner, parent, scriptPath, depth + 1); + } + } + + break; + } + case ShellCaseNode caseNode: + { + foreach (var arm in caseNode.Arms) + { + foreach (var inner in arm.Body) + { + HandleShellNode(inner, parent, scriptPath, depth + 1); + } + } + + break; + } + case ShellCommandNode commandNode: + { + var args = MaterializeArguments(commandNode.Arguments); + if (args.Length == 0) + { + break; + } + + // Skip shell built-in wrappers. + if (args[0] is "command" or "env") + { + var sliced = args.Skip(1).ToImmutableArray(); + ResolveCommand(sliced, parent, ToEntryTraceSpan(commandNode.Span, scriptPath), depth + 1, "calls"); + } + else + { + ResolveCommand(args, parent, ToEntryTraceSpan(commandNode.Span, scriptPath), depth + 1, "calls"); + } + + break; + } + default: + break; + } + } + + private static EntryTraceSpan? ToEntryTraceSpan(ShellSpan span, string path) + => new(path, span.StartLine, span.StartColumn, span.EndLine, span.EndColumn); + + private static ImmutableArray MaterializeArguments(ImmutableArray tokens) + { + var builder = ImmutableArray.CreateBuilder(tokens.Length); + foreach (var token in tokens) + { + builder.Add(token.Value); + } + + return builder.ToImmutable(); + } + + private string ResolveScriptPath(string currentScript, string candidate) + { + if (string.IsNullOrWhiteSpace(candidate)) + { + return candidate; + } + + if (candidate.StartsWith("/", StringComparison.Ordinal)) + { + return NormalizeUnixPath(candidate); + } + + if (candidate.StartsWith("$", StringComparison.Ordinal)) + { + _diagnostics.Add(new EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity.Warning, + EntryTraceUnknownReason.DynamicEnvironmentReference, + $"Path '{candidate}' depends on environment variable expansion and cannot be resolved statically.", + Span: null, + RelatedPath: candidate)); + return candidate; + } + + var normalizedScript = NormalizeUnixPath(currentScript); + var lastSlash = normalizedScript.LastIndexOf('/'); + var baseDirectory = lastSlash <= 0 ? "/" : normalizedScript[..lastSlash]; + return CombineUnixPath(baseDirectory, candidate); + } + + private static bool IsLikelyShell(string content) + { + if (string.IsNullOrEmpty(content)) + { + return false; + } + + if (content.StartsWith("#!", StringComparison.Ordinal)) + { + return content.Contains("sh", StringComparison.OrdinalIgnoreCase); + } + + return content.Contains("#!/bin/sh", StringComparison.Ordinal); + } + + private EntryTraceNode AddNode( + EntryTraceNodeKind kind, + string displayName, + ImmutableArray arguments, + EntryTraceInterpreterKind interpreterKind, + EntryTraceEvidence? evidence, + EntryTraceSpan? span) + { + var node = new EntryTraceNode( + _nextNodeId++, + kind, + displayName, + arguments, + interpreterKind, + evidence, + span); + _nodes.Add(node); + return node; + } + + private static string CombineUnixPath(string baseDirectory, string relative) + { + var normalizedBase = NormalizeUnixPath(baseDirectory); + var trimmedRelative = relative.Replace('\\', '/').Trim(); + if (string.IsNullOrEmpty(trimmedRelative)) + { + return normalizedBase; + } + + if (trimmedRelative.StartsWith('/')) + { + return NormalizeUnixPath(trimmedRelative); + } + + if (!normalizedBase.EndsWith('/')) + { + normalizedBase += "/"; + } + + return NormalizeUnixPath(normalizedBase + trimmedRelative); + } + + private static string NormalizeUnixPath(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + return "/"; + } + + var text = path.Replace('\\', '/').Trim(); + if (!text.StartsWith('/')) + { + text = "/" + text; + } + + var segments = new List(); + foreach (var part in text.Split('/', StringSplitOptions.RemoveEmptyEntries)) + { + if (part == ".") + { + continue; + } + + if (part == "..") + { + if (segments.Count > 0) + { + segments.RemoveAt(segments.Count - 1); + } + continue; + } + + segments.Add(part); + } + + return segments.Count == 0 ? "/" : "/" + string.Join('/', segments); + } + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzerOptions.cs b/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzerOptions.cs new file mode 100644 index 00000000..3987c3a9 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/EntryTraceAnalyzerOptions.cs @@ -0,0 +1,26 @@ +namespace StellaOps.Scanner.EntryTrace; + +public sealed class EntryTraceAnalyzerOptions +{ + public const string SectionName = "Scanner:Analyzers:EntryTrace"; + + /// + /// Maximum recursion depth while following includes/run-parts/interpreters. + /// + public int MaxDepth { get; set; } = 64; + + /// + /// Enables traversal of run-parts directories. + /// + public bool FollowRunParts { get; set; } = true; + + /// + /// Colon-separated default PATH string used when the environment omits PATH. + /// + public string DefaultPath { get; set; } = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + + /// + /// Maximum number of scripts considered per run-parts directory to prevent explosion. + /// + public int RunPartsLimit { get; set; } = 64; +} diff --git a/src/StellaOps.Scanner.EntryTrace/EntryTraceContext.cs b/src/StellaOps.Scanner.EntryTrace/EntryTraceContext.cs new file mode 100644 index 00000000..3975d086 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/EntryTraceContext.cs @@ -0,0 +1,16 @@ +using System.Collections.Immutable; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.EntryTrace; + +/// +/// Provides runtime context for entry trace analysis. +/// +public sealed record EntryTraceContext( + IRootFileSystem FileSystem, + ImmutableDictionary Environment, + ImmutableArray Path, + string WorkingDirectory, + string ImageDigest, + string ScanId, + ILogger? Logger); diff --git a/src/StellaOps.Scanner.EntryTrace/EntryTraceTypes.cs b/src/StellaOps.Scanner.EntryTrace/EntryTraceTypes.cs new file mode 100644 index 00000000..89c978e5 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/EntryTraceTypes.cs @@ -0,0 +1,125 @@ +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace StellaOps.Scanner.EntryTrace; + +/// +/// Outcome classification for entrypoint resolution attempts. +/// +public enum EntryTraceOutcome +{ + Resolved, + PartiallyResolved, + Unresolved +} + +/// +/// Logical classification for nodes in the entry trace graph. +/// +public enum EntryTraceNodeKind +{ + Command, + Script, + Include, + Interpreter, + Executable, + RunPartsDirectory, + RunPartsScript +} + +/// +/// Interpreter categories supported by the analyzer. +/// +public enum EntryTraceInterpreterKind +{ + None, + Python, + Node, + Java +} + +/// +/// Diagnostic severity levels emitted by the analyzer. +/// +public enum EntryTraceDiagnosticSeverity +{ + Info, + Warning, + Error +} + +/// +/// Enumerates the canonical reasons for unresolved edges. +/// +public enum EntryTraceUnknownReason +{ + CommandNotFound, + MissingFile, + DynamicEnvironmentReference, + UnsupportedSyntax, + RecursionLimitReached, + InterpreterNotSupported, + ModuleNotFound, + JarNotFound, + RunPartsEmpty, + PermissionDenied +} + +/// +/// Represents a span within a script file. +/// +public readonly record struct EntryTraceSpan( + string? Path, + int StartLine, + int StartColumn, + int EndLine, + int EndColumn); + +/// +/// Evidence describing where a node originated from within the image. +/// +public sealed record EntryTraceEvidence( + string Path, + string? LayerDigest, + string Source, + IReadOnlyDictionary? Metadata); + +/// +/// Represents a node in the entry trace graph. +/// +public sealed record EntryTraceNode( + int Id, + EntryTraceNodeKind Kind, + string DisplayName, + ImmutableArray Arguments, + EntryTraceInterpreterKind InterpreterKind, + EntryTraceEvidence? Evidence, + EntryTraceSpan? Span); + +/// +/// Represents a directed edge in the entry trace graph. +/// +public sealed record EntryTraceEdge( + int FromNodeId, + int ToNodeId, + string Relationship, + IReadOnlyDictionary? Metadata); + +/// +/// Captures diagnostic information regarding resolution gaps. +/// +public sealed record EntryTraceDiagnostic( + EntryTraceDiagnosticSeverity Severity, + EntryTraceUnknownReason Reason, + string Message, + EntryTraceSpan? Span, + string? RelatedPath); + +/// +/// Final graph output produced by the analyzer. +/// +public sealed record EntryTraceGraph( + EntryTraceOutcome Outcome, + ImmutableArray Nodes, + ImmutableArray Edges, + ImmutableArray Diagnostics); diff --git a/src/StellaOps.Scanner.EntryTrace/EntrypointSpecification.cs b/src/StellaOps.Scanner.EntryTrace/EntrypointSpecification.cs new file mode 100644 index 00000000..91389e59 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/EntrypointSpecification.cs @@ -0,0 +1,71 @@ +using System.Collections.Immutable; + +namespace StellaOps.Scanner.EntryTrace; + +/// +/// Represents the combined Docker ENTRYPOINT/CMD contract provided to the analyzer. +/// +public sealed record EntrypointSpecification +{ + private EntrypointSpecification( + ImmutableArray entrypoint, + ImmutableArray command, + string? entrypointShell, + string? commandShell) + { + Entrypoint = entrypoint; + Command = command; + EntrypointShell = string.IsNullOrWhiteSpace(entrypointShell) ? null : entrypointShell; + CommandShell = string.IsNullOrWhiteSpace(commandShell) ? null : commandShell; + } + + /// + /// Exec-form ENTRYPOINT arguments. + /// + public ImmutableArray Entrypoint { get; } + + /// + /// Exec-form CMD arguments. + /// + public ImmutableArray Command { get; } + + /// + /// Shell-form ENTRYPOINT (if provided). + /// + public string? EntrypointShell { get; } + + /// + /// Shell-form CMD (if provided). + /// + public string? CommandShell { get; } + + public static EntrypointSpecification FromExecForm( + IEnumerable? entrypoint, + IEnumerable? command) + => new( + entrypoint is null ? ImmutableArray.Empty : entrypoint.ToImmutableArray(), + command is null ? ImmutableArray.Empty : command.ToImmutableArray(), + entrypointShell: null, + commandShell: null); + + public static EntrypointSpecification FromShellForm( + string? entrypoint, + string? command) + => new( + ImmutableArray.Empty, + ImmutableArray.Empty, + entrypoint, + command); + + public EntrypointSpecification WithCommand(IEnumerable? command) + => new(Entrypoint, command?.ToImmutableArray() ?? ImmutableArray.Empty, EntrypointShell, CommandShell); + + public EntrypointSpecification WithCommandShell(string? commandShell) + => new(Entrypoint, Command, EntrypointShell, commandShell); + + public EntrypointSpecification WithEntrypoint(IEnumerable? entrypoint) + => new(entrypoint?.ToImmutableArray() ?? ImmutableArray.Empty, Command, EntrypointShell, CommandShell); + + public EntrypointSpecification WithEntrypointShell(string? entrypointShell) + => new(Entrypoint, Command, entrypointShell, CommandShell); +} diff --git a/src/StellaOps.Scanner.EntryTrace/FileSystem/IRootFileSystem.cs b/src/StellaOps.Scanner.EntryTrace/FileSystem/IRootFileSystem.cs new file mode 100644 index 00000000..319e0643 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/FileSystem/IRootFileSystem.cs @@ -0,0 +1,39 @@ +using System.Collections.Immutable; + +namespace StellaOps.Scanner.EntryTrace; + +/// +/// Represents a layered read-only filesystem snapshot built from container layers. +/// +public interface IRootFileSystem +{ + /// + /// Attempts to resolve an executable by name using the provided PATH entries. + /// + bool TryResolveExecutable(string name, IReadOnlyList searchPaths, out RootFileDescriptor descriptor); + + /// + /// Attempts to read the contents of a file as UTF-8 text. + /// + bool TryReadAllText(string path, out RootFileDescriptor descriptor, out string content); + + /// + /// Returns descriptors for entries contained within a directory. + /// + ImmutableArray EnumerateDirectory(string path); + + /// + /// Checks whether a directory exists. + /// + bool DirectoryExists(string path); +} + +/// +/// Describes a file discovered within the layered filesystem. +/// +public sealed record RootFileDescriptor( + string Path, + string? LayerDigest, + bool IsExecutable, + bool IsDirectory, + string? ShebangInterpreter); diff --git a/src/StellaOps.Scanner.EntryTrace/IEntryTraceAnalyzer.cs b/src/StellaOps.Scanner.EntryTrace/IEntryTraceAnalyzer.cs new file mode 100644 index 00000000..459e9dbf --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/IEntryTraceAnalyzer.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Scanner.EntryTrace; + +public interface IEntryTraceAnalyzer +{ + ValueTask ResolveAsync( + EntrypointSpecification entrypoint, + EntryTraceContext context, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.EntryTrace/Parsing/ShellNodes.cs b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellNodes.cs new file mode 100644 index 00000000..57fc335b --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellNodes.cs @@ -0,0 +1,54 @@ +using System.Collections.Immutable; + +namespace StellaOps.Scanner.EntryTrace.Parsing; + +public abstract record ShellNode(ShellSpan Span); + +public sealed record ShellScript(ImmutableArray Nodes); + +public sealed record ShellSpan(int StartLine, int StartColumn, int EndLine, int EndColumn); + +public sealed record ShellCommandNode( + string Command, + ImmutableArray Arguments, + ShellSpan Span) : ShellNode(Span); + +public sealed record ShellIncludeNode( + string PathExpression, + ImmutableArray Arguments, + ShellSpan Span) : ShellNode(Span); + +public sealed record ShellExecNode( + ImmutableArray Arguments, + ShellSpan Span) : ShellNode(Span); + +public sealed record ShellIfNode( + ImmutableArray Branches, + ShellSpan Span) : ShellNode(Span); + +public sealed record ShellConditionalBranch( + ShellConditionalKind Kind, + ImmutableArray Body, + ShellSpan Span, + string? PredicateSummary); + +public enum ShellConditionalKind +{ + If, + Elif, + Else +} + +public sealed record ShellCaseNode( + ImmutableArray Arms, + ShellSpan Span) : ShellNode(Span); + +public sealed record ShellCaseArm( + ImmutableArray Patterns, + ImmutableArray Body, + ShellSpan Span); + +public sealed record ShellRunPartsNode( + string DirectoryExpression, + ImmutableArray Arguments, + ShellSpan Span) : ShellNode(Span); diff --git a/src/StellaOps.Scanner.EntryTrace/Parsing/ShellParser.cs b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellParser.cs new file mode 100644 index 00000000..c97e6192 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellParser.cs @@ -0,0 +1,485 @@ +using System.Collections.Immutable; +using System.Globalization; +using System.Linq; +using System.Text; + +namespace StellaOps.Scanner.EntryTrace.Parsing; + +/// +/// Deterministic parser producing a lightweight AST for Bourne shell constructs needed by EntryTrace. +/// Supports: simple commands, exec, source/dot, run-parts, if/elif/else/fi, case/esac. +/// +public sealed class ShellParser +{ + private readonly IReadOnlyList _tokens; + private int _index; + + private ShellParser(IReadOnlyList tokens) + { + _tokens = tokens; + } + + public static ShellScript Parse(string source) + { + var tokenizer = new ShellTokenizer(); + var tokens = tokenizer.Tokenize(source); + var parser = new ShellParser(tokens); + var nodes = parser.ParseNodes(untilKeywords: null); + return new ShellScript(nodes.ToImmutableArray()); + } + + private List ParseNodes(HashSet? untilKeywords) + { + var nodes = new List(); + + while (true) + { + SkipNewLines(); + var token = Peek(); + + if (token.Kind == ShellTokenKind.EndOfFile) + { + break; + } + + if (token.Kind == ShellTokenKind.Word && untilKeywords is not null && untilKeywords.Contains(token.Value)) + { + break; + } + + ShellNode? node = token.Kind switch + { + ShellTokenKind.Word when token.Value == "if" => ParseIf(), + ShellTokenKind.Word when token.Value == "case" => ParseCase(), + _ => ParseCommandLike() + }; + + if (node is not null) + { + nodes.Add(node); + } + + SkipCommandSeparators(); + } + + return nodes; + } + + private ShellNode ParseCommandLike() + { + var start = Peek(); + var tokens = ReadUntilTerminator(); + + if (tokens.Count == 0) + { + return new ShellCommandNode(string.Empty, ImmutableArray.Empty, CreateSpan(start, start)); + } + + var normalizedName = ExtractCommandName(tokens); + var immutableTokens = tokens.ToImmutableArray(); + var span = CreateSpan(tokens[0], tokens[^1]); + + return normalizedName switch + { + "exec" => new ShellExecNode(immutableTokens, span), + "source" or "." => new ShellIncludeNode( + ExtractPrimaryArgument(immutableTokens), + immutableTokens, + span), + "run-parts" => new ShellRunPartsNode( + ExtractPrimaryArgument(immutableTokens), + immutableTokens, + span), + _ => new ShellCommandNode(normalizedName, immutableTokens, span) + }; + } + + private ShellIfNode ParseIf() + { + var start = Expect(ShellTokenKind.Word, "if"); + var predicateTokens = ReadUntilKeyword("then"); + Expect(ShellTokenKind.Word, "then"); + + var branches = new List(); + var predicateSummary = JoinTokens(predicateTokens); + var thenNodes = ParseNodes(new HashSet(StringComparer.Ordinal) + { + "elif", + "else", + "fi" + }); + + branches.Add(new ShellConditionalBranch( + ShellConditionalKind.If, + thenNodes.ToImmutableArray(), + CreateSpan(start, thenNodes.LastOrDefault()?.Span ?? CreateSpan(start, start)), + predicateSummary)); + + while (true) + { + SkipNewLines(); + var next = Peek(); + if (next.Kind != ShellTokenKind.Word) + { + break; + } + + if (next.Value == "elif") + { + var elifStart = Advance(); + var elifPredicate = ReadUntilKeyword("then"); + Expect(ShellTokenKind.Word, "then"); + var elifBody = ParseNodes(new HashSet(StringComparer.Ordinal) + { + "elif", + "else", + "fi" + }); + var span = elifBody.Count > 0 + ? CreateSpan(elifStart, elifBody[^1].Span) + : CreateSpan(elifStart, elifStart); + + branches.Add(new ShellConditionalBranch( + ShellConditionalKind.Elif, + elifBody.ToImmutableArray(), + span, + JoinTokens(elifPredicate))); + continue; + } + + if (next.Value == "else") + { + var elseStart = Advance(); + var elseBody = ParseNodes(new HashSet(StringComparer.Ordinal) + { + "fi" + }); + branches.Add(new ShellConditionalBranch( + ShellConditionalKind.Else, + elseBody.ToImmutableArray(), + elseBody.Count > 0 ? CreateSpan(elseStart, elseBody[^1].Span) : CreateSpan(elseStart, elseStart), + null)); + break; + } + + break; + } + + Expect(ShellTokenKind.Word, "fi"); + var end = Previous(); + return new ShellIfNode(branches.ToImmutableArray(), CreateSpan(start, end)); + } + + private ShellCaseNode ParseCase() + { + var start = Expect(ShellTokenKind.Word, "case"); + var selectorTokens = ReadUntilKeyword("in"); + Expect(ShellTokenKind.Word, "in"); + + var arms = new List(); + while (true) + { + SkipNewLines(); + var token = Peek(); + if (token.Kind == ShellTokenKind.Word && token.Value == "esac") + { + break; + } + + if (token.Kind == ShellTokenKind.EndOfFile) + { + throw new FormatException("Unexpected end of file while parsing case arms."); + } + + var patterns = ReadPatterns(); + Expect(ShellTokenKind.Operator, ")"); + + var body = ParseNodes(new HashSet(StringComparer.Ordinal) + { + ";;", + "esac" + }); + + ShellSpan span; + if (body.Count > 0) + { + span = CreateSpan(patterns.FirstToken ?? token, body[^1].Span); + } + else + { + span = CreateSpan(patterns.FirstToken ?? token, token); + } + + arms.Add(new ShellCaseArm( + patterns.Values.ToImmutableArray(), + body.ToImmutableArray(), + span)); + + SkipNewLines(); + var separator = Peek(); + if (separator.Kind == ShellTokenKind.Operator && separator.Value == ";;") + { + Advance(); + continue; + } + + if (separator.Kind == ShellTokenKind.Word && separator.Value == "esac") + { + break; + } + } + + Expect(ShellTokenKind.Word, "esac"); + return new ShellCaseNode(arms.ToImmutableArray(), CreateSpan(start, Previous())); + + (List Values, ShellToken? FirstToken) ReadPatterns() + { + var values = new List(); + ShellToken? first = null; + var sb = new StringBuilder(); + + while (true) + { + var current = Peek(); + if (current.Kind is ShellTokenKind.Operator && current.Value is ")" or "|") + { + if (sb.Length > 0) + { + values.Add(sb.ToString()); + sb.Clear(); + } + + if (current.Value == "|") + { + Advance(); + continue; + } + + break; + } + + if (current.Kind == ShellTokenKind.EndOfFile) + { + throw new FormatException("Unexpected EOF in case arm pattern."); + } + + if (first is null) + { + first = current; + } + + sb.Append(current.Value); + Advance(); + } + + if (values.Count == 0 && sb.Length > 0) + { + values.Add(sb.ToString()); + } + + return (values, first); + } + } + + private List ReadUntilTerminator() + { + var tokens = new List(); + while (true) + { + var token = Peek(); + if (token.Kind is ShellTokenKind.EndOfFile or ShellTokenKind.NewLine) + { + break; + } + + if (token.Kind == ShellTokenKind.Operator && token.Value is ";" or "&&" or "||") + { + break; + } + + tokens.Add(Advance()); + } + + return tokens; + } + + private ImmutableArray ReadUntilKeyword(string keyword) + { + var tokens = new List(); + while (true) + { + var token = Peek(); + if (token.Kind == ShellTokenKind.EndOfFile) + { + throw new FormatException($"Unexpected EOF while looking for keyword '{keyword}'."); + } + + if (token.Kind == ShellTokenKind.Word && token.Value == keyword) + { + break; + } + + tokens.Add(Advance()); + } + + return tokens.ToImmutableArray(); + } + + private static string ExtractCommandName(IReadOnlyList tokens) + { + foreach (var token in tokens) + { + if (token.Kind is not ShellTokenKind.Word and not ShellTokenKind.SingleQuoted and not ShellTokenKind.DoubleQuoted) + { + continue; + } + + if (token.Value.Contains('=', StringComparison.Ordinal)) + { + // Skip environment assignments e.g. FOO=bar exec /app + var eqIndex = token.Value.IndexOf('=', StringComparison.Ordinal); + if (eqIndex > 0 && token.Value[..eqIndex].All(IsIdentifierChar)) + { + continue; + } + } + + return NormalizeCommandName(token.Value); + } + + return string.Empty; + + static bool IsIdentifierChar(char c) => char.IsLetterOrDigit(c) || c == '_'; + } + + private static string NormalizeCommandName(string value) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + return value switch + { + "." => ".", + _ => value.Trim() + }; + } + + private void SkipCommandSeparators() + { + while (true) + { + var token = Peek(); + if (token.Kind == ShellTokenKind.NewLine) + { + Advance(); + continue; + } + + if (token.Kind == ShellTokenKind.Operator && (token.Value == ";" || token.Value == "&")) + { + Advance(); + continue; + } + + break; + } + } + + private void SkipNewLines() + { + while (Peek().Kind == ShellTokenKind.NewLine) + { + Advance(); + } + } + + private ShellToken Expect(ShellTokenKind kind, string? value = null) + { + var token = Peek(); + if (token.Kind != kind || (value is not null && token.Value != value)) + { + throw new FormatException($"Unexpected token '{token.Value}' at line {token.Line}, expected {value ?? kind.ToString()}."); + } + + return Advance(); + } + + private ShellToken Advance() + { + if (_index >= _tokens.Count) + { + return _tokens[^1]; + } + + return _tokens[_index++]; + } + + private ShellToken Peek() + { + if (_index >= _tokens.Count) + { + return _tokens[^1]; + } + + return _tokens[_index]; + } + + private ShellToken Previous() + { + if (_index == 0) + { + return _tokens[0]; + } + + return _tokens[_index - 1]; + } + + private static ShellSpan CreateSpan(ShellToken start, ShellToken end) + { + return new ShellSpan(start.Line, start.Column, end.Line, end.Column + end.Value.Length); + } + + private static ShellSpan CreateSpan(ShellToken start, ShellSpan end) + { + return new ShellSpan(start.Line, start.Column, end.EndLine, end.EndColumn); + } + + private static string JoinTokens(IEnumerable tokens) + { + var builder = new StringBuilder(); + var first = true; + foreach (var token in tokens) + { + if (!first) + { + builder.Append(' '); + } + + builder.Append(token.Value); + first = false; + } + + return builder.ToString(); + } + + private static string ExtractPrimaryArgument(ImmutableArray tokens) + { + if (tokens.Length <= 1) + { + return string.Empty; + } + + for (var i = 1; i < tokens.Length; i++) + { + var token = tokens[i]; + if (token.Kind is ShellTokenKind.Word or ShellTokenKind.SingleQuoted or ShellTokenKind.DoubleQuoted) + { + return token.Value; + } + } + + return string.Empty; + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/Parsing/ShellToken.cs b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellToken.cs new file mode 100644 index 00000000..bc3d1950 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellToken.cs @@ -0,0 +1,16 @@ +namespace StellaOps.Scanner.EntryTrace.Parsing; + +/// +/// Token produced by the shell lexer. +/// +public readonly record struct ShellToken(ShellTokenKind Kind, string Value, int Line, int Column); + +public enum ShellTokenKind +{ + Word, + SingleQuoted, + DoubleQuoted, + Operator, + NewLine, + EndOfFile +} diff --git a/src/StellaOps.Scanner.EntryTrace/Parsing/ShellTokenizer.cs b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellTokenizer.cs new file mode 100644 index 00000000..9b976246 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/Parsing/ShellTokenizer.cs @@ -0,0 +1,200 @@ +using System.Globalization; +using System.Text; + +namespace StellaOps.Scanner.EntryTrace.Parsing; + +/// +/// Lightweight Bourne shell tokenizer sufficient for ENTRYPOINT scripts. +/// Deterministic: emits tokens in source order without normalization. +/// +public sealed class ShellTokenizer +{ + public IReadOnlyList Tokenize(string source) + { + if (source is null) + { + throw new ArgumentNullException(nameof(source)); + } + + var tokens = new List(); + var line = 1; + var column = 1; + var index = 0; + + while (index < source.Length) + { + var ch = source[index]; + + if (ch == '\r') + { + index++; + continue; + } + + if (ch == '\n') + { + tokens.Add(new ShellToken(ShellTokenKind.NewLine, "\n", line, column)); + index++; + line++; + column = 1; + continue; + } + + if (char.IsWhiteSpace(ch)) + { + index++; + column++; + continue; + } + + if (ch == '#') + { + // Comment: skip until newline. + while (index < source.Length && source[index] != '\n') + { + index++; + } + continue; + } + + if (IsOperatorStart(ch)) + { + var opStartColumn = column; + var op = ConsumeOperator(source, ref index, ref column); + tokens.Add(new ShellToken(ShellTokenKind.Operator, op, line, opStartColumn)); + continue; + } + + if (ch == '\'') + { + var (value, length) = ConsumeSingleQuoted(source, index + 1); + tokens.Add(new ShellToken(ShellTokenKind.SingleQuoted, value, line, column)); + index += length + 2; + column += length + 2; + continue; + } + + if (ch == '"') + { + var (value, length) = ConsumeDoubleQuoted(source, index + 1); + tokens.Add(new ShellToken(ShellTokenKind.DoubleQuoted, value, line, column)); + index += length + 2; + column += length + 2; + continue; + } + + var (word, consumed) = ConsumeWord(source, index); + tokens.Add(new ShellToken(ShellTokenKind.Word, word, line, column)); + index += consumed; + column += consumed; + } + + tokens.Add(new ShellToken(ShellTokenKind.EndOfFile, string.Empty, line, column)); + return tokens; + } + + private static bool IsOperatorStart(char ch) => ch switch + { + ';' or '&' or '|' or '(' or ')' => true, + _ => false + }; + + private static string ConsumeOperator(string source, ref int index, ref int column) + { + var start = index; + var ch = source[index]; + index++; + column++; + + if (index < source.Length) + { + var next = source[index]; + if ((ch == '&' && next == '&') || + (ch == '|' && next == '|') || + (ch == ';' && next == ';')) + { + index++; + column++; + } + } + + return source[start..index]; + } + + private static (string Value, int Length) ConsumeSingleQuoted(string source, int startIndex) + { + var end = startIndex; + while (end < source.Length && source[end] != '\'') + { + end++; + } + + if (end >= source.Length) + { + throw new FormatException("Unterminated single-quoted string in entrypoint script."); + } + + return (source[startIndex..end], end - startIndex); + } + + private static (string Value, int Length) ConsumeDoubleQuoted(string source, int startIndex) + { + var builder = new StringBuilder(); + var index = startIndex; + + while (index < source.Length) + { + var ch = source[index]; + if (ch == '"') + { + return (builder.ToString(), index - startIndex); + } + + if (ch == '\\' && index + 1 < source.Length) + { + var next = source[index + 1]; + if (next is '"' or '\\' or '$' or '`' or '\n') + { + builder.Append(next); + index += 2; + continue; + } + } + + builder.Append(ch); + index++; + } + + throw new FormatException("Unterminated double-quoted string in entrypoint script."); + } + + private static (string Value, int Length) ConsumeWord(string source, int startIndex) + { + var index = startIndex; + while (index < source.Length) + { + var ch = source[index]; + if (char.IsWhiteSpace(ch) || ch == '\n' || ch == '\r' || IsOperatorStart(ch) || ch == '#' ) + { + break; + } + + if (ch == '\\' && index + 1 < source.Length && source[index + 1] == '\n') + { + // Line continuation. + index += 2; + continue; + } + + index++; + } + + if (index == startIndex) + { + throw new InvalidOperationException("Tokenizer failed to advance while consuming word."); + } + + var text = source[startIndex..index]; + return (text, index - startIndex); + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/ServiceCollectionExtensions.cs b/src/StellaOps.Scanner.EntryTrace/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..b38fd37d --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/ServiceCollectionExtensions.cs @@ -0,0 +1,29 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.EntryTrace.Diagnostics; + +namespace StellaOps.Scanner.EntryTrace; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddEntryTraceAnalyzer(this IServiceCollection services, Action? configure = null) + { + if (services is null) + { + throw new ArgumentNullException(nameof(services)); + } + + services.AddOptions() + .BindConfiguration(EntryTraceAnalyzerOptions.SectionName); + + if (configure is not null) + { + services.Configure(configure); + } + + services.TryAddSingleton(); + services.TryAddSingleton(); + return services; + } +} diff --git a/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj b/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj new file mode 100644 index 00000000..f3cb0889 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.EntryTrace/TASKS.md b/src/StellaOps.Scanner.EntryTrace/TASKS.md new file mode 100644 index 00000000..bd445fa9 --- /dev/null +++ b/src/StellaOps.Scanner.EntryTrace/TASKS.md @@ -0,0 +1,16 @@ +# EntryTrace Analyzer Task Board (Sprint 10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-ENTRYTRACE-10-401 | DONE (2025-10-19) | EntryTrace Guild | Scanner Core contracts | Implement deterministic POSIX shell AST parser covering exec/command/source/run-parts/case/if used by ENTRYPOINT scripts. | Parser emits stable AST and serialization tests prove determinism for representative fixtures; see `ShellParserTests`. | +| SCANNER-ENTRYTRACE-10-402 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401 | Resolve commands across layered rootfs, tracking evidence per hop (PATH hit, layer origin, shebang). | Resolver returns terminal program path with layer attribution for fixtures; deterministic traversal asserted in `EntryTraceAnalyzerTests.ResolveAsync_IsDeterministic`. | +| SCANNER-ENTRYTRACE-10-403 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Follow interpreter wrappers (shell → Python/Node/Java launchers) to terminal target, including module/jar detection. | Interpreter tracer reports correct module/script for language launchers; tests cover Python/Node/Java wrappers. | +| SCANNER-ENTRYTRACE-10-404 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Build Python entry analyzer detecting venv shebangs, module invocations, `-m` usage and record usage flag. | Python fixtures produce expected module metadata (`python-module` edge) and diagnostics for missing scripts. | +| SCANNER-ENTRYTRACE-10-405 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-403 | Implement Node/Java launcher analyzer capturing script/jar targets including npm lifecycle wrappers. | Node/Java fixtures resolved with evidence chain; `RunParts` coverage ensures child scripts traced. | +| SCANNER-ENTRYTRACE-10-406 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-402 | Surface explainability + diagnostics for unresolved constructs and emit metrics counters. | Diagnostics catalog enumerates unknown reasons; metrics wired via `EntryTraceMetrics`; explainability doc updated. | +| SCANNER-ENTRYTRACE-10-407 | DONE (2025-10-19) | EntryTrace Guild | SCANNER-ENTRYTRACE-10-401..406 | Package EntryTrace analyzers as restart-time plug-ins with manifest + host registration. | Plug-in manifest under `plugins/scanner/entrytrace/`; restart-only policy documented; DI extension exposes `AddEntryTraceAnalyzer`. | + +## Status Review — 2025-10-19 + +- Confirmed Wave 0 instructions for EntryTrace Guild; SCANNER-ENTRYTRACE-10-401..407 already marked complete. +- No outstanding prerequisites identified during review; readiness noted for any follow-on work. diff --git a/src/StellaOps.Scanner.Queue.Tests/QueueLeaseIntegrationTests.cs b/src/StellaOps.Scanner.Queue.Tests/QueueLeaseIntegrationTests.cs new file mode 100644 index 00000000..78c5ce45 --- /dev/null +++ b/src/StellaOps.Scanner.Queue.Tests/QueueLeaseIntegrationTests.cs @@ -0,0 +1,390 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Queue; +using Xunit; + +namespace StellaOps.Scanner.Queue.Tests; + +public sealed class QueueLeaseIntegrationTests +{ + private readonly ScannerQueueOptions _options = new() + { + MaxDeliveryAttempts = 3, + RetryInitialBackoff = TimeSpan.FromMilliseconds(1), + RetryMaxBackoff = TimeSpan.FromMilliseconds(5), + DefaultLeaseDuration = TimeSpan.FromSeconds(5) + }; + + [Fact] + public async Task Enqueue_ShouldDeduplicate_ByIdempotencyKey() + { + var clock = new FakeTimeProvider(); + var queue = new InMemoryScanQueue(_options, clock); + + var payload = new byte[] { 1, 2, 3 }; + var message = new ScanQueueMessage("job-1", payload) + { + IdempotencyKey = "idem-1" + }; + + var first = await queue.EnqueueAsync(message); + first.Deduplicated.Should().BeFalse(); + + var second = await queue.EnqueueAsync(message); + second.Deduplicated.Should().BeTrue(); + } + + [Fact] + public async Task Lease_ShouldExposeTraceId_FromQueuedMessage() + { + var clock = new FakeTimeProvider(); + var queue = new InMemoryScanQueue(_options, clock); + + var payload = new byte[] { 9 }; + var message = new ScanQueueMessage("job-trace", payload) + { + TraceId = "trace-123" + }; + + await queue.EnqueueAsync(message); + + var lease = await LeaseSingleAsync(queue, consumer: "worker-trace"); + lease.Should().NotBeNull(); + lease!.TraceId.Should().Be("trace-123"); + } + + [Fact] + public async Task Lease_Acknowledge_ShouldRemoveFromQueue() + { + var clock = new FakeTimeProvider(); + var queue = new InMemoryScanQueue(_options, clock); + + var message = new ScanQueueMessage("job-ack", new byte[] { 42 }); + await queue.EnqueueAsync(message); + + var lease = await LeaseSingleAsync(queue, consumer: "worker-1"); + lease.Should().NotBeNull(); + + await lease!.AcknowledgeAsync(); + + var afterAck = await queue.LeaseAsync(new QueueLeaseRequest("worker-1", 1, TimeSpan.FromSeconds(1))); + afterAck.Should().BeEmpty(); + } + + [Fact] + public async Task Release_WithRetry_ShouldDeadLetterAfterMaxAttempts() + { + var clock = new FakeTimeProvider(); + var queue = new InMemoryScanQueue(_options, clock); + + var message = new ScanQueueMessage("job-retry", new byte[] { 5 }); + await queue.EnqueueAsync(message); + + for (var attempt = 1; attempt <= _options.MaxDeliveryAttempts; attempt++) + { + var lease = await LeaseSingleAsync(queue, consumer: $"worker-{attempt}"); + lease.Should().NotBeNull(); + + await lease!.ReleaseAsync(QueueReleaseDisposition.Retry); + } + + queue.DeadLetters.Should().ContainSingle(dead => dead.JobId == "job-retry"); + } + + [Fact] + public async Task Retry_ShouldIncreaseAttemptOnNextLease() + { + var clock = new FakeTimeProvider(); + var queue = new InMemoryScanQueue(_options, clock); + + await queue.EnqueueAsync(new ScanQueueMessage("job-retry-attempt", new byte[] { 77 })); + + var firstLease = await LeaseSingleAsync(queue, "worker-retry"); + firstLease.Should().NotBeNull(); + firstLease!.Attempt.Should().Be(1); + + await firstLease.ReleaseAsync(QueueReleaseDisposition.Retry); + + var secondLease = await LeaseSingleAsync(queue, "worker-retry"); + secondLease.Should().NotBeNull(); + secondLease!.Attempt.Should().Be(2); + } + + private static async Task LeaseSingleAsync(InMemoryScanQueue queue, string consumer) + { + var leases = await queue.LeaseAsync(new QueueLeaseRequest(consumer, 1, TimeSpan.FromSeconds(1))); + return leases.FirstOrDefault(); + } + + private sealed class InMemoryScanQueue : IScanQueue + { + private readonly ScannerQueueOptions _options; + private readonly TimeProvider _timeProvider; + private readonly ConcurrentQueue _ready = new(); + private readonly ConcurrentDictionary _idempotency = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _inFlight = new(StringComparer.Ordinal); + private readonly List _deadLetters = new(); + private long _sequence; + + public InMemoryScanQueue(ScannerQueueOptions options, TimeProvider timeProvider) + { + _options = options; + _timeProvider = timeProvider; + } + + public IReadOnlyList DeadLetters => _deadLetters; + + public ValueTask EnqueueAsync(ScanQueueMessage message, CancellationToken cancellationToken = default) + { + var token = message.IdempotencyKey ?? message.JobId; + if (_idempotency.TryGetValue(token, out var existing)) + { + return ValueTask.FromResult(new QueueEnqueueResult(existing.SequenceId, true)); + } + + var entry = new QueueEntry( + sequenceId: Interlocked.Increment(ref _sequence).ToString(), + jobId: message.JobId, + payload: message.Payload.ToArray(), + idempotencyKey: token, + attempt: 1, + enqueuedAt: _timeProvider.GetUtcNow(), + traceId: message.TraceId, + attributes: message.Attributes is null + ? new ReadOnlyDictionary(new Dictionary(0, StringComparer.Ordinal)) + : new ReadOnlyDictionary(new Dictionary(message.Attributes, StringComparer.Ordinal))); + + _idempotency[token] = entry; + _ready.Enqueue(entry); + return ValueTask.FromResult(new QueueEnqueueResult(entry.SequenceId, false)); + } + + public ValueTask> LeaseAsync(QueueLeaseRequest request, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var leases = new List(request.BatchSize); + + while (leases.Count < request.BatchSize && _ready.TryDequeue(out var entry)) + { + entry.Attempt = Math.Max(entry.Attempt, entry.Deliveries + 1); + entry.Deliveries = entry.Attempt; + entry.LastLeaseAt = now; + _inFlight[entry.SequenceId] = entry; + + var lease = new InMemoryLease( + this, + entry, + request.Consumer, + now, + request.LeaseDuration); + leases.Add(lease); + } + + return ValueTask.FromResult>(leases); + } + + public ValueTask> ClaimExpiredLeasesAsync(QueueClaimOptions options, CancellationToken cancellationToken = default) + { + var now = _timeProvider.GetUtcNow(); + var leases = _inFlight.Values + .Where(entry => now - entry.LastLeaseAt >= options.MinIdleTime) + .Take(options.BatchSize) + .Select(entry => new InMemoryLease(this, entry, options.ClaimantConsumer, now, _options.DefaultLeaseDuration)) + .Cast() + .ToList(); + + return ValueTask.FromResult>(leases); + } + + internal Task AcknowledgeAsync(QueueEntry entry) + { + _inFlight.TryRemove(entry.SequenceId, out _); + _idempotency.TryRemove(entry.IdempotencyKey, out _); + return Task.CompletedTask; + } + + internal Task RenewAsync(QueueEntry entry, TimeSpan leaseDuration) + { + var expires = _timeProvider.GetUtcNow().Add(leaseDuration); + entry.LeaseExpiresAt = expires; + return Task.FromResult(expires); + } + + internal Task ReleaseAsync(QueueEntry entry, QueueReleaseDisposition disposition) + { + if (disposition == QueueReleaseDisposition.Retry && entry.Attempt >= _options.MaxDeliveryAttempts) + { + return DeadLetterAsync(entry, $"max-delivery-attempts:{entry.Attempt}"); + } + + if (disposition == QueueReleaseDisposition.Retry) + { + entry.Attempt++; + _ready.Enqueue(entry); + } + else + { + _idempotency.TryRemove(entry.IdempotencyKey, out _); + } + + _inFlight.TryRemove(entry.SequenceId, out _); + return Task.CompletedTask; + } + + internal Task DeadLetterAsync(QueueEntry entry, string reason) + { + entry.DeadLetterReason = reason; + _inFlight.TryRemove(entry.SequenceId, out _); + _idempotency.TryRemove(entry.IdempotencyKey, out _); + _deadLetters.Add(entry); + return Task.CompletedTask; + } + + private sealed class InMemoryLease : IScanQueueLease + { + private readonly InMemoryScanQueue _owner; + private readonly QueueEntry _entry; + private int _completed; + + public InMemoryLease( + InMemoryScanQueue owner, + QueueEntry entry, + string consumer, + DateTimeOffset now, + TimeSpan leaseDuration) + { + _owner = owner; + _entry = entry; + Consumer = consumer; + MessageId = entry.SequenceId; + JobId = entry.JobId; + Payload = entry.Payload; + Attempt = entry.Attempt; + EnqueuedAt = entry.EnqueuedAt; + LeaseExpiresAt = now.Add(leaseDuration); + IdempotencyKey = entry.IdempotencyKey; + TraceId = entry.TraceId; + Attributes = entry.Attributes; + } + + public string MessageId { get; } + + public string JobId { get; } + + public ReadOnlyMemory Payload { get; } + + public int Attempt { get; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; private set; } + + public string Consumer { get; } + + public string? IdempotencyKey { get; } + + public string? TraceId { get; } + + public IReadOnlyDictionary Attributes { get; } + + public Task AcknowledgeAsync(CancellationToken cancellationToken = default) + { + if (TryComplete()) + { + return _owner.AcknowledgeAsync(_entry); + } + + return Task.CompletedTask; + } + + public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default) + { + return RenewInternalAsync(leaseDuration); + } + + public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default) + { + if (TryComplete()) + { + return _owner.ReleaseAsync(_entry, disposition); + } + + return Task.CompletedTask; + } + + public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default) + { + if (TryComplete()) + { + return _owner.DeadLetterAsync(_entry, reason); + } + + return Task.CompletedTask; + } + + private async Task RenewInternalAsync(TimeSpan leaseDuration) + { + var expires = await _owner.RenewAsync(_entry, leaseDuration).ConfigureAwait(false); + LeaseExpiresAt = expires; + } + + private bool TryComplete() + => Interlocked.CompareExchange(ref _completed, 1, 0) == 0; + } + + internal sealed class QueueEntry + { + public QueueEntry( + string sequenceId, + string jobId, + byte[] payload, + string idempotencyKey, + int attempt, + DateTimeOffset enqueuedAt, + string? traceId, + IReadOnlyDictionary attributes) + { + SequenceId = sequenceId; + JobId = jobId; + Payload = payload; + IdempotencyKey = idempotencyKey; + Attempt = attempt; + EnqueuedAt = enqueuedAt; + LastLeaseAt = enqueuedAt; + TraceId = traceId; + Attributes = attributes; + } + + public string SequenceId { get; } + + public string JobId { get; } + + public byte[] Payload { get; } + + public string IdempotencyKey { get; } + + public int Attempt { get; set; } + + public int Deliveries { get; set; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; set; } + + public DateTimeOffset LastLeaseAt { get; set; } + + public string? TraceId { get; } + + public IReadOnlyDictionary Attributes { get; } + + public string? DeadLetterReason { get; set; } + } + } +} diff --git a/src/StellaOps.Scanner.Queue.Tests/StellaOps.Scanner.Queue.Tests.csproj b/src/StellaOps.Scanner.Queue.Tests/StellaOps.Scanner.Queue.Tests.csproj new file mode 100644 index 00000000..5dac25ed --- /dev/null +++ b/src/StellaOps.Scanner.Queue.Tests/StellaOps.Scanner.Queue.Tests.csproj @@ -0,0 +1,14 @@ + + + net10.0 + enable + enable + false + + + + + + + + diff --git a/src/StellaOps.Scanner.Queue/AGENTS.md b/src/StellaOps.Scanner.Queue/AGENTS.md new file mode 100644 index 00000000..b518ef76 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/AGENTS.md @@ -0,0 +1,15 @@ +# StellaOps.Scanner.Queue — Agent Charter + +## Mission +Deliver the scanner job queue backbone defined in `docs/ARCHITECTURE_SCANNER.md`, providing deterministic, offline-friendly leasing semantics for WebService producers and Worker consumers. + +## Responsibilities +- Define queue abstractions with idempotent enqueue tokens, acknowledgement, lease renewal, and claim support. +- Ship first-party adapters for Redis Streams and NATS JetStream, respecting offline deployments and allow-listed hosts. +- Surface health probes, structured diagnostics, and metrics needed by Scanner WebService/Worker. +- Document operational expectations and configuration binding hooks. + +## Interfaces & Dependencies +- Consumes shared configuration primitives from `StellaOps.Configuration`. +- Exposes dependency injection extensions for `StellaOps.DependencyInjection`. +- Targets `net10.0` (preview) and aligns with scanner DTOs once `StellaOps.Scanner.Core` lands. diff --git a/src/StellaOps.Scanner.Queue/IScanQueue.cs b/src/StellaOps.Scanner.Queue/IScanQueue.cs new file mode 100644 index 00000000..1fda053d --- /dev/null +++ b/src/StellaOps.Scanner.Queue/IScanQueue.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Queue; + +public interface IScanQueue +{ + ValueTask EnqueueAsync( + ScanQueueMessage message, + CancellationToken cancellationToken = default); + + ValueTask> LeaseAsync( + QueueLeaseRequest request, + CancellationToken cancellationToken = default); + + ValueTask> ClaimExpiredLeasesAsync( + QueueClaimOptions options, + CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.Queue/IScanQueueLease.cs b/src/StellaOps.Scanner.Queue/IScanQueueLease.cs new file mode 100644 index 00000000..a39324b7 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/IScanQueueLease.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Queue; + +public interface IScanQueueLease +{ + string MessageId { get; } + + string JobId { get; } + + ReadOnlyMemory Payload { get; } + + int Attempt { get; } + + DateTimeOffset EnqueuedAt { get; } + + DateTimeOffset LeaseExpiresAt { get; } + + string Consumer { get; } + + string? IdempotencyKey { get; } + + string? TraceId { get; } + + IReadOnlyDictionary Attributes { get; } + + Task AcknowledgeAsync(CancellationToken cancellationToken = default); + + Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default); + + Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default); + + Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.Queue/Nats/NatsScanQueue.cs b/src/StellaOps.Scanner.Queue/Nats/NatsScanQueue.cs new file mode 100644 index 00000000..06169597 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/Nats/NatsScanQueue.cs @@ -0,0 +1,654 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using NATS.Client.Core; +using NATS.Client.JetStream; +using NATS.Client.JetStream.Models; + +namespace StellaOps.Scanner.Queue.Nats; + +internal sealed class NatsScanQueue : IScanQueue, IAsyncDisposable +{ + private const string TransportName = "nats"; + + private static readonly INatsSerializer PayloadSerializer = NatsRawSerializer.Default; + + private readonly ScannerQueueOptions _queueOptions; + private readonly NatsQueueOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _connectionGate = new(1, 1); + private readonly Func> _connectionFactory; + + private NatsConnection? _connection; + private NatsJSContext? _jsContext; + private INatsJSConsumer? _consumer; + private bool _disposed; + + public NatsScanQueue( + ScannerQueueOptions queueOptions, + NatsQueueOptions options, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + { + _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _connectionFactory = connectionFactory ?? ((opts, cancellationToken) => new ValueTask(new NatsConnection(opts))); + + if (string.IsNullOrWhiteSpace(_options.Url)) + { + throw new InvalidOperationException("NATS connection URL must be configured for the scanner queue."); + } + } + + public async ValueTask EnqueueAsync( + ScanQueueMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var idempotencyKey = message.IdempotencyKey ?? message.JobId; + var headers = BuildHeaders(message, idempotencyKey); + var publishOpts = new NatsJSPubOpts + { + MsgId = idempotencyKey, + RetryAttempts = 0 + }; + + var ack = await js.PublishAsync( + _options.Subject, + message.Payload.ToArray(), + PayloadSerializer, + publishOpts, + headers, + cancellationToken) + .ConfigureAwait(false); + + if (ack.Duplicate) + { + _logger.LogDebug( + "Duplicate NATS enqueue detected for job {JobId} (token {Token}).", + message.JobId, + idempotencyKey); + + QueueMetrics.RecordDeduplicated(TransportName); + return new QueueEnqueueResult(ack.Seq.ToString(), true); + } + + QueueMetrics.RecordEnqueued(TransportName); + _logger.LogDebug( + "Enqueued job {JobId} into NATS stream {Stream} with sequence {Sequence}.", + message.JobId, + ack.Stream, + ack.Seq); + + return new QueueEnqueueResult(ack.Seq.ToString(), false); + } + + public async ValueTask> LeaseAsync( + QueueLeaseRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var fetchOpts = new NatsJSFetchOpts + { + MaxMsgs = request.BatchSize, + Expires = request.LeaseDuration, + IdleHeartbeat = _options.IdleHeartbeat + }; + + var now = _timeProvider.GetUtcNow(); + var leases = new List(capacity: request.BatchSize); + + await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false)) + { + var lease = CreateLease(msg, request.Consumer, now, request.LeaseDuration); + if (lease is not null) + { + leases.Add(lease); + } + } + + return leases; + } + + public async ValueTask> ClaimExpiredLeasesAsync( + QueueClaimOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + var consumer = await EnsureStreamAndConsumerAsync(js, cancellationToken).ConfigureAwait(false); + + var fetchOpts = new NatsJSFetchOpts + { + MaxMsgs = options.BatchSize, + Expires = options.MinIdleTime, + IdleHeartbeat = _options.IdleHeartbeat + }; + + var now = _timeProvider.GetUtcNow(); + var leases = new List(options.BatchSize); + + await foreach (var msg in consumer.FetchAsync(PayloadSerializer, fetchOpts, cancellationToken).ConfigureAwait(false)) + { + var deliveries = (int)(msg.Metadata?.NumDelivered ?? 1); + if (deliveries <= 1) + { + // Fresh message; surface back to queue and continue. + await msg.NakAsync(new AckOpts(), TimeSpan.Zero, cancellationToken).ConfigureAwait(false); + continue; + } + + var lease = CreateLease(msg, options.ClaimantConsumer, now, _queueOptions.DefaultLeaseDuration); + if (lease is not null) + { + leases.Add(lease); + } + } + + return leases; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_connection is not null) + { + await _connection.DisposeAsync().ConfigureAwait(false); + } + + _connectionGate.Dispose(); + GC.SuppressFinalize(this); + } + + internal async Task AcknowledgeAsync( + NatsScanQueueLease lease, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + await lease.Message.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + + QueueMetrics.RecordAck(TransportName); + _logger.LogDebug( + "Acknowledged job {JobId} (seq {Seq}).", + lease.JobId, + lease.MessageId); + } + + internal async Task RenewLeaseAsync( + NatsScanQueueLease lease, + TimeSpan leaseDuration, + CancellationToken cancellationToken) + { + await lease.Message.AckProgressAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + var expires = _timeProvider.GetUtcNow().Add(leaseDuration); + lease.RefreshLease(expires); + + _logger.LogDebug( + "Renewed NATS lease for job {JobId} until {Expires:u}.", + lease.JobId, + expires); + } + + internal async Task ReleaseAsync( + NatsScanQueueLease lease, + QueueReleaseDisposition disposition, + CancellationToken cancellationToken) + { + if (disposition == QueueReleaseDisposition.Retry + && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) + { + _logger.LogWarning( + "Job {JobId} reached max delivery attempts ({Attempts}); shipping to dead-letter stream.", + lease.JobId, + lease.Attempt); + + await DeadLetterAsync( + lease, + $"max-delivery-attempts:{lease.Attempt}", + cancellationToken).ConfigureAwait(false); + return; + } + + if (!lease.TryBeginCompletion()) + { + return; + } + + if (disposition == QueueReleaseDisposition.Retry) + { + QueueMetrics.RecordRetry(TransportName); + + var delay = CalculateBackoff(lease.Attempt); + await lease.Message.NakAsync(new AckOpts(), delay, cancellationToken).ConfigureAwait(false); + + _logger.LogWarning( + "Rescheduled job {JobId} via NATS NAK with delay {Delay} (attempt {Attempt}).", + lease.JobId, + delay, + lease.Attempt); + } + else + { + await lease.Message.AckTerminateAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + QueueMetrics.RecordAck(TransportName); + + _logger.LogInformation( + "Abandoned job {JobId} after {Attempt} attempt(s).", + lease.JobId, + lease.Attempt); + } + } + + internal async Task DeadLetterAsync( + NatsScanQueueLease lease, + string reason, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + await lease.Message.AckAsync(new AckOpts(), cancellationToken).ConfigureAwait(false); + + var js = await GetJetStreamAsync(cancellationToken).ConfigureAwait(false); + await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false); + + var headers = BuildDeadLetterHeaders(lease, reason); + await js.PublishAsync( + _options.DeadLetterSubject, + lease.Payload.ToArray(), + PayloadSerializer, + new NatsJSPubOpts(), + headers, + cancellationToken) + .ConfigureAwait(false); + + QueueMetrics.RecordDeadLetter(TransportName); + _logger.LogError( + "Dead-lettered job {JobId} (attempt {Attempt}): {Reason}", + lease.JobId, + lease.Attempt, + reason); + } + + private async Task GetJetStreamAsync(CancellationToken cancellationToken) + { + if (_jsContext is not null) + { + return _jsContext; + } + + var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false); + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + _jsContext ??= new NatsJSContext(connection); + return _jsContext; + } + finally + { + _connectionGate.Release(); + } + } + + private async ValueTask EnsureStreamAndConsumerAsync( + NatsJSContext js, + CancellationToken cancellationToken) + { + if (_consumer is not null) + { + return _consumer; + } + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_consumer is not null) + { + return _consumer; + } + + await EnsureStreamAsync(js, cancellationToken).ConfigureAwait(false); + await EnsureDeadLetterStreamAsync(js, cancellationToken).ConfigureAwait(false); + + var consumerConfig = new ConsumerConfig + { + DurableName = _options.DurableConsumer, + AckPolicy = ConsumerConfigAckPolicy.Explicit, + ReplayPolicy = ConsumerConfigReplayPolicy.Instant, + DeliverPolicy = ConsumerConfigDeliverPolicy.All, + AckWait = ToNanoseconds(_options.AckWait), + MaxAckPending = _options.MaxInFlight, + MaxDeliver = Math.Max(1, _queueOptions.MaxDeliveryAttempts), + FilterSubjects = new[] { _options.Subject } + }; + + try + { + _consumer = await js.CreateConsumerAsync( + _options.Stream, + consumerConfig, + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException apiEx) + { + _logger.LogDebug(apiEx, + "CreateConsumerAsync failed with code {Code}; attempting to fetch existing durable consumer {Durable}.", + apiEx.Error?.Code, + _options.DurableConsumer); + + _consumer = await js.GetConsumerAsync( + _options.Stream, + _options.DurableConsumer, + cancellationToken) + .ConfigureAwait(false); + } + + return _consumer; + } + finally + { + _connectionGate.Release(); + } + } + + private async Task EnsureConnectionAsync(CancellationToken cancellationToken) + { + if (_connection is not null) + { + return _connection; + } + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is not null) + { + return _connection; + } + + var opts = new NatsOpts + { + Url = _options.Url!, + Name = "stellaops-scanner-queue", + CommandTimeout = TimeSpan.FromSeconds(10), + RequestTimeout = TimeSpan.FromSeconds(20), + PingInterval = TimeSpan.FromSeconds(30) + }; + + _connection = await _connectionFactory(opts, cancellationToken).ConfigureAwait(false); + await _connection.ConnectAsync().ConfigureAwait(false); + return _connection; + } + finally + { + _connectionGate.Release(); + } + } + + private async Task EnsureStreamAsync(NatsJSContext js, CancellationToken cancellationToken) + { + try + { + await js.GetStreamAsync( + _options.Stream, + new StreamInfoRequest(), + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException) + { + var config = new StreamConfig( + name: _options.Stream, + subjects: new[] { _options.Subject }) + { + Retention = StreamConfigRetention.Workqueue, + Storage = StreamConfigStorage.File, + MaxConsumers = -1, + MaxMsgs = -1, + MaxBytes = -1, + MaxAge = 0 + }; + + await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Created NATS JetStream stream {Stream} ({Subject}).", _options.Stream, _options.Subject); + } + } + + private async Task EnsureDeadLetterStreamAsync(NatsJSContext js, CancellationToken cancellationToken) + { + try + { + await js.GetStreamAsync( + _options.DeadLetterStream, + new StreamInfoRequest(), + cancellationToken) + .ConfigureAwait(false); + } + catch (NatsJSApiException) + { + var config = new StreamConfig( + name: _options.DeadLetterStream, + subjects: new[] { _options.DeadLetterSubject }) + { + Retention = StreamConfigRetention.Workqueue, + Storage = StreamConfigStorage.File, + MaxConsumers = -1, + MaxMsgs = -1, + MaxBytes = -1, + MaxAge = ToNanoseconds(_queueOptions.DeadLetter.Retention) + }; + + await js.CreateStreamAsync(config, cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Created NATS dead-letter stream {Stream} ({Subject}).", _options.DeadLetterStream, _options.DeadLetterSubject); + } + } + + internal async ValueTask PingAsync(CancellationToken cancellationToken) + { + var connection = await EnsureConnectionAsync(cancellationToken).ConfigureAwait(false); + await connection.PingAsync(cancellationToken).ConfigureAwait(false); + } + + private NatsScanQueueLease? CreateLease( + NatsJSMsg message, + string consumer, + DateTimeOffset now, + TimeSpan leaseDuration) + { + var headers = message.Headers; + if (headers is null) + { + return null; + } + + if (!headers.TryGetValue(QueueEnvelopeFields.JobId, out var jobIdValues) || jobIdValues.Count == 0) + { + return null; + } + + var jobId = jobIdValues[0]!; + var idempotencyKey = headers.TryGetValue(QueueEnvelopeFields.IdempotencyKey, out var idemValues) && idemValues.Count > 0 + ? idemValues[0] + : null; + + var traceId = headers.TryGetValue(QueueEnvelopeFields.TraceId, out var traceValues) && traceValues.Count > 0 + ? string.IsNullOrWhiteSpace(traceValues[0]) ? null : traceValues[0] + : null; + + var enqueuedAt = headers.TryGetValue(QueueEnvelopeFields.EnqueuedAt, out var enqueuedValues) && enqueuedValues.Count > 0 + && long.TryParse(enqueuedValues[0], out var unix) + ? DateTimeOffset.FromUnixTimeMilliseconds(unix) + : now; + + var attempt = headers.TryGetValue(QueueEnvelopeFields.Attempt, out var attemptValues) && attemptValues.Count > 0 + && int.TryParse(attemptValues[0], out var parsedAttempt) + ? parsedAttempt + : 1; + + if (message.Metadata?.NumDelivered is ulong delivered && delivered > 0) + { + var deliveredInt = delivered > int.MaxValue ? int.MaxValue : (int)delivered; + if (deliveredInt > attempt) + { + attempt = deliveredInt; + } + } + + var leaseExpires = now.Add(leaseDuration); + var attributes = ExtractAttributes(headers); + + var messageId = message.Metadata?.Sequence.Stream.ToString() ?? Guid.NewGuid().ToString("n"); + return new NatsScanQueueLease( + this, + message, + messageId, + jobId, + message.Data ?? Array.Empty(), + attempt, + enqueuedAt, + leaseExpires, + consumer, + idempotencyKey, + traceId, + attributes); + } + + private static IReadOnlyDictionary ExtractAttributes(NatsHeaders headers) + { + var attributes = new Dictionary(StringComparer.Ordinal); + + foreach (var key in headers.Keys) + { + if (!key.StartsWith(QueueEnvelopeFields.AttributePrefix, StringComparison.Ordinal)) + { + continue; + } + + if (headers.TryGetValue(key, out var values) && values.Count > 0) + { + attributes[key[QueueEnvelopeFields.AttributePrefix.Length..]] = values[0]!; + } + } + + return attributes.Count == 0 + ? EmptyReadOnlyDictionary.Instance + : new ReadOnlyDictionary(attributes); + } + + private NatsHeaders BuildHeaders(ScanQueueMessage message, string idempotencyKey) + { + var headers = new NatsHeaders + { + { QueueEnvelopeFields.JobId, message.JobId }, + { QueueEnvelopeFields.IdempotencyKey, idempotencyKey }, + { QueueEnvelopeFields.Attempt, "1" }, + { QueueEnvelopeFields.EnqueuedAt, _timeProvider.GetUtcNow().ToUnixTimeMilliseconds().ToString() } + }; + + if (!string.IsNullOrEmpty(message.TraceId)) + { + headers.Add(QueueEnvelopeFields.TraceId, message.TraceId!); + } + + if (message.Attributes is not null) + { + foreach (var kvp in message.Attributes) + { + headers.Add(QueueEnvelopeFields.AttributePrefix + kvp.Key, kvp.Value); + } + } + + return headers; + } + + private NatsHeaders BuildDeadLetterHeaders(NatsScanQueueLease lease, string reason) + { + var headers = new NatsHeaders + { + { QueueEnvelopeFields.JobId, lease.JobId }, + { QueueEnvelopeFields.IdempotencyKey, lease.IdempotencyKey ?? lease.JobId }, + { QueueEnvelopeFields.Attempt, lease.Attempt.ToString() }, + { QueueEnvelopeFields.EnqueuedAt, lease.EnqueuedAt.ToUnixTimeMilliseconds().ToString() }, + { "deadletter-reason", reason } + }; + + if (!string.IsNullOrWhiteSpace(lease.TraceId)) + { + headers.Add(QueueEnvelopeFields.TraceId, lease.TraceId!); + } + + foreach (var kvp in lease.Attributes) + { + headers.Add(QueueEnvelopeFields.AttributePrefix + kvp.Key, kvp.Value); + } + + return headers; + } + + private TimeSpan CalculateBackoff(int attempt) + { + var configuredInitial = _options.RetryDelay > TimeSpan.Zero + ? _options.RetryDelay + : _queueOptions.RetryInitialBackoff; + + if (configuredInitial <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + if (attempt <= 1) + { + return configuredInitial; + } + + var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero + ? _queueOptions.RetryMaxBackoff + : configuredInitial; + + var exponent = attempt - 1; + var scaledTicks = configuredInitial.Ticks * Math.Pow(2, exponent - 1); + var cappedTicks = Math.Min(max.Ticks, scaledTicks); + var resultTicks = Math.Max(configuredInitial.Ticks, (long)cappedTicks); + return TimeSpan.FromTicks(resultTicks); + } + + private static long ToNanoseconds(TimeSpan timeSpan) + => timeSpan <= TimeSpan.Zero ? 0 : timeSpan.Ticks * 100L; + + private static class EmptyReadOnlyDictionary + where TKey : notnull + { + public static readonly IReadOnlyDictionary Instance = + new ReadOnlyDictionary(new Dictionary(0, EqualityComparer.Default)); + } +} diff --git a/src/StellaOps.Scanner.Queue/Nats/NatsScanQueueLease.cs b/src/StellaOps.Scanner.Queue/Nats/NatsScanQueueLease.cs new file mode 100644 index 00000000..a8493f9a --- /dev/null +++ b/src/StellaOps.Scanner.Queue/Nats/NatsScanQueueLease.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using NATS.Client.JetStream; + +namespace StellaOps.Scanner.Queue.Nats; + +internal sealed class NatsScanQueueLease : IScanQueueLease +{ + private readonly NatsScanQueue _queue; + private readonly NatsJSMsg _message; + private int _completed; + + internal NatsScanQueueLease( + NatsScanQueue queue, + NatsJSMsg message, + string messageId, + string jobId, + byte[] payload, + int attempt, + DateTimeOffset enqueuedAt, + DateTimeOffset leaseExpiresAt, + string consumer, + string? idempotencyKey, + string? traceId, + IReadOnlyDictionary attributes) + { + _queue = queue; + _message = message; + MessageId = messageId; + JobId = jobId; + Payload = payload; + Attempt = attempt; + EnqueuedAt = enqueuedAt; + LeaseExpiresAt = leaseExpiresAt; + Consumer = consumer; + IdempotencyKey = idempotencyKey; + TraceId = traceId; + Attributes = attributes; + } + + public string MessageId { get; } + + public string JobId { get; } + + public ReadOnlyMemory Payload { get; } + + public int Attempt { get; internal set; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; private set; } + + public string Consumer { get; } + + public string? IdempotencyKey { get; } + + public string? TraceId { get; } + + public IReadOnlyDictionary Attributes { get; } + + internal NatsJSMsg Message => _message; + + public Task AcknowledgeAsync(CancellationToken cancellationToken = default) + => _queue.AcknowledgeAsync(this, cancellationToken); + + public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken); + + public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default) + => _queue.ReleaseAsync(this, disposition, cancellationToken); + + public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default) + => _queue.DeadLetterAsync(this, reason, cancellationToken); + + internal bool TryBeginCompletion() + => Interlocked.CompareExchange(ref _completed, 1, 0) == 0; + + internal void RefreshLease(DateTimeOffset expiresAt) + => LeaseExpiresAt = expiresAt; +} diff --git a/src/StellaOps.Scanner.Queue/QueueEnvelopeFields.cs b/src/StellaOps.Scanner.Queue/QueueEnvelopeFields.cs new file mode 100644 index 00000000..2e755bdc --- /dev/null +++ b/src/StellaOps.Scanner.Queue/QueueEnvelopeFields.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Scanner.Queue; + +internal static class QueueEnvelopeFields +{ + public const string Payload = "payload"; + public const string JobId = "jobId"; + public const string IdempotencyKey = "idempotency"; + public const string Attempt = "attempt"; + public const string EnqueuedAt = "enqueuedAt"; + public const string TraceId = "traceId"; + public const string AttributePrefix = "attr:"; +} diff --git a/src/StellaOps.Scanner.Queue/QueueMetrics.cs b/src/StellaOps.Scanner.Queue/QueueMetrics.cs new file mode 100644 index 00000000..fb3d5080 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/QueueMetrics.cs @@ -0,0 +1,28 @@ +using System.Diagnostics.Metrics; + +namespace StellaOps.Scanner.Queue; + +internal static class QueueMetrics +{ + private const string TransportTagName = "transport"; + + private static readonly Meter Meter = new("StellaOps.Scanner.Queue"); + private static readonly Counter EnqueuedCounter = Meter.CreateCounter("scanner_queue_enqueued_total"); + private static readonly Counter DeduplicatedCounter = Meter.CreateCounter("scanner_queue_deduplicated_total"); + private static readonly Counter AckCounter = Meter.CreateCounter("scanner_queue_ack_total"); + private static readonly Counter RetryCounter = Meter.CreateCounter("scanner_queue_retry_total"); + private static readonly Counter DeadLetterCounter = Meter.CreateCounter("scanner_queue_deadletter_total"); + + public static void RecordEnqueued(string transport) => EnqueuedCounter.Add(1, BuildTags(transport)); + + public static void RecordDeduplicated(string transport) => DeduplicatedCounter.Add(1, BuildTags(transport)); + + public static void RecordAck(string transport) => AckCounter.Add(1, BuildTags(transport)); + + public static void RecordRetry(string transport) => RetryCounter.Add(1, BuildTags(transport)); + + public static void RecordDeadLetter(string transport) => DeadLetterCounter.Add(1, BuildTags(transport)); + + private static KeyValuePair[] BuildTags(string transport) + => new[] { new KeyValuePair(TransportTagName, transport) }; +} diff --git a/src/StellaOps.Scanner.Queue/QueueTransportKind.cs b/src/StellaOps.Scanner.Queue/QueueTransportKind.cs new file mode 100644 index 00000000..4646d3f1 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/QueueTransportKind.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Scanner.Queue; + +public enum QueueTransportKind +{ + Redis, + Nats +} diff --git a/src/StellaOps.Scanner.Queue/Redis/RedisScanQueue.cs b/src/StellaOps.Scanner.Queue/Redis/RedisScanQueue.cs new file mode 100644 index 00000000..17ef23b2 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/Redis/RedisScanQueue.cs @@ -0,0 +1,766 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace StellaOps.Scanner.Queue.Redis; + +internal sealed class RedisScanQueue : IScanQueue, IAsyncDisposable +{ + private const string TransportName = "redis"; + + private readonly ScannerQueueOptions _queueOptions; + private readonly RedisQueueOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly SemaphoreSlim _connectionLock = new(1, 1); + private readonly SemaphoreSlim _groupInitLock = new(1, 1); + private readonly Func> _connectionFactory; + private IConnectionMultiplexer? _connection; + private volatile bool _groupInitialized; + private bool _disposed; + + private string BuildIdempotencyKey(string key) + => string.Concat(_options.IdempotencyKeyPrefix, key); + + public RedisScanQueue( + ScannerQueueOptions queueOptions, + RedisQueueOptions options, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + { + _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _connectionFactory = connectionFactory ?? (config => Task.FromResult(ConnectionMultiplexer.Connect(config))); + + if (string.IsNullOrWhiteSpace(_options.ConnectionString)) + { + throw new InvalidOperationException("Redis connection string must be configured for the scanner queue."); + } + } + + public async ValueTask EnqueueAsync( + ScanQueueMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + cancellationToken.ThrowIfCancellationRequested(); + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + var now = _timeProvider.GetUtcNow(); + await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false); + + var attempt = 1; + var entries = BuildEntries(message, now, attempt); + var messageId = await AddToStreamAsync( + db, + _options.StreamName, + entries, + _options.ApproximateMaxLength, + _options.ApproximateMaxLength is not null) + .ConfigureAwait(false); + + var idempotencyToken = message.IdempotencyKey ?? message.JobId; + var idempotencyKey = BuildIdempotencyKey(idempotencyToken); + + var stored = await db.StringSetAsync( + key: idempotencyKey, + value: messageId, + when: When.NotExists, + expiry: _options.IdempotencyWindow) + .ConfigureAwait(false); + + if (!stored) + { + // Duplicate enqueue – delete the freshly added entry and surface cached ID. + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { messageId }) + .ConfigureAwait(false); + + var existing = await db.StringGetAsync(idempotencyKey).ConfigureAwait(false); + var duplicateId = existing.IsNullOrEmpty ? messageId : existing; + + _logger.LogDebug( + "Duplicate queue enqueue detected for job {JobId} (token {Token}), returning existing stream id {StreamId}.", + message.JobId, + idempotencyToken, + duplicateId.ToString()); + + QueueMetrics.RecordDeduplicated(TransportName); + return new QueueEnqueueResult(duplicateId.ToString()!, true); + } + + _logger.LogDebug( + "Enqueued job {JobId} into stream {Stream} with id {StreamId}.", + message.JobId, + _options.StreamName, + messageId.ToString()); + + QueueMetrics.RecordEnqueued(TransportName); + return new QueueEnqueueResult(messageId.ToString()!, false); + } + + public async ValueTask> LeaseAsync( + QueueLeaseRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + cancellationToken.ThrowIfCancellationRequested(); + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false); + + var entries = await db.StreamReadGroupAsync( + _options.StreamName, + _options.ConsumerGroup, + request.Consumer, + position: ">", + count: request.BatchSize, + flags: CommandFlags.None) + .ConfigureAwait(false); + + if (entries is null || entries.Length == 0) + { + return Array.Empty(); + } + + var now = _timeProvider.GetUtcNow(); + var leases = new List(entries.Length); + + foreach (var entry in entries) + { + var lease = TryMapLease( + entry, + request.Consumer, + now, + request.LeaseDuration, + default); + + if (lease is null) + { + _logger.LogWarning( + "Stream entry {StreamId} is missing required metadata; acknowledging to avoid poison message.", + entry.Id.ToString()); + await db.StreamAcknowledgeAsync( + _options.StreamName, + _options.ConsumerGroup, + new RedisValue[] { entry.Id }) + .ConfigureAwait(false); + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { entry.Id }) + .ConfigureAwait(false); + continue; + } + + leases.Add(lease); + } + + return leases; + } + + public async ValueTask> ClaimExpiredLeasesAsync( + QueueClaimOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await EnsureConsumerGroupAsync(db, cancellationToken).ConfigureAwait(false); + + var pending = await db.StreamPendingMessagesAsync( + _options.StreamName, + _options.ConsumerGroup, + options.BatchSize, + RedisValue.Null, + (long)options.MinIdleTime.TotalMilliseconds) + .ConfigureAwait(false); + + if (pending is null || pending.Length == 0) + { + return Array.Empty(); + } + + var eligible = pending + .Where(p => p.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds) + .ToArray(); + + if (eligible.Length == 0) + { + return Array.Empty(); + } + + var messageIds = eligible + .Select(static p => (RedisValue)p.MessageId) + .ToArray(); + + var entries = await db.StreamClaimAsync( + _options.StreamName, + _options.ConsumerGroup, + options.ClaimantConsumer, + 0, + messageIds, + CommandFlags.None) + .ConfigureAwait(false); + + if (entries is null || entries.Length == 0) + { + return Array.Empty(); + } + + var now = _timeProvider.GetUtcNow(); + var pendingById = Enumerable.ToDictionary( + eligible, + static p => p.MessageId.IsNullOrEmpty ? string.Empty : p.MessageId.ToString(), + static p => p, + StringComparer.Ordinal); + + var leases = new List(entries.Length); + foreach (var entry in entries) + { + var entryIdValue = entry.Id; + var entryId = entryIdValue.IsNullOrEmpty ? string.Empty : entryIdValue.ToString(); + var hasPending = pendingById.TryGetValue(entryId, out var pendingInfo); + var attempt = hasPending + ? (int)Math.Max(1, pendingInfo.DeliveryCount) + : 1; + + var lease = TryMapLease( + entry, + options.ClaimantConsumer, + now, + _queueOptions.DefaultLeaseDuration, + attempt); + + if (lease is null) + { + _logger.LogWarning( + "Unable to map claimed stream entry {StreamId}; acknowledging to unblock queue.", + entry.Id.ToString()); + await db.StreamAcknowledgeAsync( + _options.StreamName, + _options.ConsumerGroup, + new RedisValue[] { entry.Id }) + .ConfigureAwait(false); + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { entry.Id }) + .ConfigureAwait(false); + continue; + } + + leases.Add(lease); + } + + return leases; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + if (_connection is not null) + { + await _connection.CloseAsync(); + _connection.Dispose(); + } + + _connectionLock.Dispose(); + _groupInitLock.Dispose(); + GC.SuppressFinalize(this); + } + + internal async Task AcknowledgeAsync( + RedisScanQueueLease lease, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await db.StreamAcknowledgeAsync( + _options.StreamName, + _options.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + _logger.LogDebug( + "Acknowledged job {JobId} ({MessageId}) on consumer {Consumer}.", + lease.JobId, + lease.MessageId, + lease.Consumer); + + QueueMetrics.RecordAck(TransportName); + } + + internal async Task RenewLeaseAsync( + RedisScanQueueLease lease, + TimeSpan leaseDuration, + CancellationToken cancellationToken) + { + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await db.StreamClaimAsync( + _options.StreamName, + _options.ConsumerGroup, + lease.Consumer, + 0, + new RedisValue[] { lease.MessageId }, + CommandFlags.None) + .ConfigureAwait(false); + + var expires = _timeProvider.GetUtcNow().Add(leaseDuration); + lease.RefreshLease(expires); + + _logger.LogDebug( + "Renewed lease for job {JobId} until {LeaseExpiry:u}.", + lease.JobId, + expires); + } + + internal async Task ReleaseAsync( + RedisScanQueueLease lease, + QueueReleaseDisposition disposition, + CancellationToken cancellationToken) + { + if (disposition == QueueReleaseDisposition.Retry + && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) + { + _logger.LogWarning( + "Job {JobId} reached max delivery attempts ({Attempts}); moving to dead-letter.", + lease.JobId, + lease.Attempt); + + await DeadLetterAsync( + lease, + $"max-delivery-attempts:{lease.Attempt}", + cancellationToken).ConfigureAwait(false); + return; + } + + if (!lease.TryBeginCompletion()) + { + return; + } + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await db.StreamAcknowledgeAsync( + _options.StreamName, + _options.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + QueueMetrics.RecordAck(TransportName); + + if (disposition == QueueReleaseDisposition.Retry) + { + QueueMetrics.RecordRetry(TransportName); + + var delay = CalculateBackoff(lease.Attempt); + if (delay > TimeSpan.Zero) + { + _logger.LogDebug( + "Delaying retry for job {JobId} by {Delay} (attempt {Attempt}).", + lease.JobId, + delay, + lease.Attempt); + + try + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + return; + } + } + + var requeueMessage = new ScanQueueMessage(lease.JobId, lease.Payload) + { + IdempotencyKey = lease.IdempotencyKey, + Attributes = lease.Attributes, + TraceId = lease.TraceId + }; + + var now = _timeProvider.GetUtcNow(); + var entries = BuildEntries(requeueMessage, now, lease.Attempt + 1); + + await AddToStreamAsync( + db, + _options.StreamName, + entries, + _options.ApproximateMaxLength, + _options.ApproximateMaxLength is not null) + .ConfigureAwait(false); + + QueueMetrics.RecordEnqueued(TransportName); + _logger.LogWarning( + "Released job {JobId} for retry (attempt {Attempt}).", + lease.JobId, + lease.Attempt + 1); + } + else + { + _logger.LogInformation( + "Abandoned job {JobId} after {Attempt} attempt(s).", + lease.JobId, + lease.Attempt); + } + } + + internal async Task DeadLetterAsync( + RedisScanQueueLease lease, + string reason, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await db.StreamAcknowledgeAsync( + _options.StreamName, + _options.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await db.StreamDeleteAsync( + _options.StreamName, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + var now = _timeProvider.GetUtcNow(); + var entries = BuildEntries( + new ScanQueueMessage(lease.JobId, lease.Payload) + { + IdempotencyKey = lease.IdempotencyKey, + Attributes = lease.Attributes, + TraceId = lease.TraceId + }, + now, + lease.Attempt); + + await AddToStreamAsync( + db, + _queueOptions.DeadLetter.StreamName, + entries, + null, + false) + .ConfigureAwait(false); + + _logger.LogError( + "Dead-lettered job {JobId} (attempt {Attempt}): {Reason}", + lease.JobId, + lease.Attempt, + reason); + + QueueMetrics.RecordDeadLetter(TransportName); + } + + private async ValueTask GetDatabaseAsync(CancellationToken cancellationToken) + { + if (_connection is not null) + { + return _connection.GetDatabase(_options.Database ?? -1); + } + + await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is null) + { + var config = ConfigurationOptions.Parse(_options.ConnectionString!); + config.AbortOnConnectFail = false; + config.ConnectTimeout = (int)_options.InitializationTimeout.TotalMilliseconds; + config.ConnectRetry = 3; + if (_options.Database is not null) + { + config.DefaultDatabase = _options.Database; + } + + _connection = await _connectionFactory(config).ConfigureAwait(false); + } + + return _connection.GetDatabase(_options.Database ?? -1); + } + finally + { + _connectionLock.Release(); + } + } + + private async Task EnsureConsumerGroupAsync( + IDatabase database, + CancellationToken cancellationToken) + { + if (_groupInitialized) + { + return; + } + + await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_groupInitialized) + { + return; + } + + try + { + await database.StreamCreateConsumerGroupAsync( + _options.StreamName, + _options.ConsumerGroup, + StreamPosition.Beginning, + createStream: true) + .ConfigureAwait(false); + } + catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase)) + { + // Already exists. + } + + _groupInitialized = true; + } + finally + { + _groupInitLock.Release(); + } + } + + private NameValueEntry[] BuildEntries( + ScanQueueMessage message, + DateTimeOffset enqueuedAt, + int attempt) + { + var attributeCount = message.Attributes?.Count ?? 0; + var entries = ArrayPool.Shared.Rent(6 + attributeCount); + var index = 0; + + entries[index++] = new NameValueEntry(QueueEnvelopeFields.JobId, message.JobId); + entries[index++] = new NameValueEntry(QueueEnvelopeFields.Attempt, attempt); + entries[index++] = new NameValueEntry(QueueEnvelopeFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds()); + entries[index++] = new NameValueEntry( + QueueEnvelopeFields.IdempotencyKey, + message.IdempotencyKey ?? message.JobId); + entries[index++] = new NameValueEntry( + QueueEnvelopeFields.Payload, + message.Payload.ToArray()); + entries[index++] = new NameValueEntry( + QueueEnvelopeFields.TraceId, + message.TraceId ?? string.Empty); + + if (attributeCount > 0) + { + foreach (var kvp in message.Attributes!) + { + entries[index++] = new NameValueEntry( + QueueEnvelopeFields.AttributePrefix + kvp.Key, + kvp.Value); + } + } + + var result = entries.AsSpan(0, index).ToArray(); + ArrayPool.Shared.Return(entries, clearArray: true); + return result; + } + + private RedisScanQueueLease? TryMapLease( + StreamEntry entry, + string consumer, + DateTimeOffset now, + TimeSpan leaseDuration, + int? attemptOverride) + { + if (entry.Values is null || entry.Values.Length == 0) + { + return null; + } + + string? jobId = null; + string? idempotency = null; + long? enqueuedAtUnix = null; + byte[]? payload = null; + string? traceId = null; + var attributes = new Dictionary(StringComparer.Ordinal); + var attempt = attemptOverride ?? 1; + + foreach (var field in entry.Values) + { + var name = field.Name.ToString(); + if (name.Equals(QueueEnvelopeFields.JobId, StringComparison.Ordinal)) + { + jobId = field.Value.ToString(); + } + else if (name.Equals(QueueEnvelopeFields.IdempotencyKey, StringComparison.Ordinal)) + { + idempotency = field.Value.ToString(); + } + else if (name.Equals(QueueEnvelopeFields.EnqueuedAt, StringComparison.Ordinal)) + { + if (long.TryParse(field.Value.ToString(), out var unix)) + { + enqueuedAtUnix = unix; + } + } + else if (name.Equals(QueueEnvelopeFields.Payload, StringComparison.Ordinal)) + { + payload = (byte[]?)field.Value ?? Array.Empty(); + } + else if (name.Equals(QueueEnvelopeFields.Attempt, StringComparison.Ordinal)) + { + if (int.TryParse(field.Value.ToString(), out var parsedAttempt)) + { + attempt = Math.Max(parsedAttempt, attempt); + } + } + else if (name.Equals(QueueEnvelopeFields.TraceId, StringComparison.Ordinal)) + { + var value = field.Value.ToString(); + traceId = string.IsNullOrWhiteSpace(value) ? null : value; + } + else if (name.StartsWith(QueueEnvelopeFields.AttributePrefix, StringComparison.Ordinal)) + { + attributes[name[QueueEnvelopeFields.AttributePrefix.Length..]] = field.Value.ToString(); + } + } + + if (jobId is null || payload is null || enqueuedAtUnix is null) + { + return null; + } + + var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value); + var leaseExpires = now.Add(leaseDuration); + + var attributeView = attributes.Count == 0 + ? EmptyReadOnlyDictionary.Instance + : new ReadOnlyDictionary(attributes); + + return new RedisScanQueueLease( + this, + entry.Id.ToString(), + jobId, + payload, + attempt, + enqueuedAt, + leaseExpires, + consumer, + idempotency, + traceId, + attributeView); + } + + private TimeSpan CalculateBackoff(int attempt) + { + var configuredInitial = _options.RetryInitialBackoff > TimeSpan.Zero + ? _options.RetryInitialBackoff + : _queueOptions.RetryInitialBackoff; + + var initial = configuredInitial > TimeSpan.Zero + ? configuredInitial + : TimeSpan.Zero; + + if (initial <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + if (attempt <= 1) + { + return initial; + } + + var configuredMax = _queueOptions.RetryMaxBackoff > TimeSpan.Zero + ? _queueOptions.RetryMaxBackoff + : initial; + + var max = configuredMax <= TimeSpan.Zero + ? initial + : configuredMax; + + var exponent = attempt - 1; + var scale = Math.Pow(2, exponent - 1); + var scaledTicks = initial.Ticks * scale; + var cappedTicks = Math.Min(max.Ticks, scaledTicks); + + var resultTicks = Math.Max(initial.Ticks, (long)cappedTicks); + return TimeSpan.FromTicks(resultTicks); + } + + private async Task AddToStreamAsync( + IDatabase database, + RedisKey stream, + NameValueEntry[] entries, + int? maxLength, + bool useApproximateLength) + { + var capacity = 4 + (entries.Length * 2); + var args = new List(capacity) + { + stream + }; + + if (maxLength.HasValue) + { + args.Add("MAXLEN"); + if (useApproximateLength) + { + args.Add("~"); + } + + args.Add(maxLength.Value); + } + + args.Add("*"); + for (var i = 0; i < entries.Length; i++) + { + args.Add(entries[i].Name); + args.Add(entries[i].Value); + } + + var result = await database.ExecuteAsync("XADD", args.ToArray()).ConfigureAwait(false); + return (RedisValue)result!; + } + + private static class EmptyReadOnlyDictionary + where TKey : notnull + { + public static readonly IReadOnlyDictionary Instance = + new ReadOnlyDictionary(new Dictionary(0, EqualityComparer.Default)); + } + + internal async ValueTask PingAsync(CancellationToken cancellationToken) + { + var db = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await db.ExecuteAsync("PING").ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scanner.Queue/Redis/RedisScanQueueLease.cs b/src/StellaOps.Scanner.Queue/Redis/RedisScanQueueLease.cs new file mode 100644 index 00000000..4fd4d734 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/Redis/RedisScanQueueLease.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Queue.Redis; + +internal sealed class RedisScanQueueLease : IScanQueueLease +{ + private readonly RedisScanQueue _queue; + private int _completed; + + internal RedisScanQueueLease( + RedisScanQueue queue, + string messageId, + string jobId, + byte[] payload, + int attempt, + DateTimeOffset enqueuedAt, + DateTimeOffset leaseExpiresAt, + string consumer, + string? idempotencyKey, + string? traceId, + IReadOnlyDictionary attributes) + { + _queue = queue; + MessageId = messageId; + JobId = jobId; + Payload = payload; + Attempt = attempt; + EnqueuedAt = enqueuedAt; + LeaseExpiresAt = leaseExpiresAt; + Consumer = consumer; + IdempotencyKey = idempotencyKey; + TraceId = traceId; + Attributes = attributes; + } + + public string MessageId { get; } + + public string JobId { get; } + + public ReadOnlyMemory Payload { get; } + + public int Attempt { get; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; private set; } + + public string Consumer { get; } + + public string? IdempotencyKey { get; } + + public string? TraceId { get; } + + public IReadOnlyDictionary Attributes { get; } + + public Task AcknowledgeAsync(CancellationToken cancellationToken = default) + => _queue.AcknowledgeAsync(this, cancellationToken); + + public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken); + + public Task ReleaseAsync(QueueReleaseDisposition disposition, CancellationToken cancellationToken = default) + => _queue.ReleaseAsync(this, disposition, cancellationToken); + + public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default) + => _queue.DeadLetterAsync(this, reason, cancellationToken); + + internal bool TryBeginCompletion() + => Interlocked.CompareExchange(ref _completed, 1, 0) == 0; + + internal void RefreshLease(DateTimeOffset expiresAt) + => LeaseExpiresAt = expiresAt; +} diff --git a/src/StellaOps.Scanner.Queue/ScanQueueContracts.cs b/src/StellaOps.Scanner.Queue/ScanQueueContracts.cs new file mode 100644 index 00000000..e8317c1d --- /dev/null +++ b/src/StellaOps.Scanner.Queue/ScanQueueContracts.cs @@ -0,0 +1,115 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Scanner.Queue; + +public sealed class ScanQueueMessage +{ + private readonly byte[] _payload; + + public ScanQueueMessage(string jobId, ReadOnlyMemory payload) + { + if (string.IsNullOrWhiteSpace(jobId)) + { + throw new ArgumentException("Job identifier must be provided.", nameof(jobId)); + } + + JobId = jobId; + _payload = CopyPayload(payload); + } + + public string JobId { get; } + + public string? IdempotencyKey { get; init; } + + public string? TraceId { get; init; } + + public IReadOnlyDictionary? Attributes { get; init; } + + public ReadOnlyMemory Payload => _payload; + + private static byte[] CopyPayload(ReadOnlyMemory payload) + { + if (payload.Length == 0) + { + return Array.Empty(); + } + + var copy = new byte[payload.Length]; + payload.Span.CopyTo(copy); + return copy; + } +} + +public readonly record struct QueueEnqueueResult(string MessageId, bool Deduplicated); + +public sealed class QueueLeaseRequest +{ + public QueueLeaseRequest(string consumer, int batchSize, TimeSpan leaseDuration) + { + if (string.IsNullOrWhiteSpace(consumer)) + { + throw new ArgumentException("Consumer name must be provided.", nameof(consumer)); + } + + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive."); + } + + if (leaseDuration <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(leaseDuration), leaseDuration, "Lease duration must be positive."); + } + + Consumer = consumer; + BatchSize = batchSize; + LeaseDuration = leaseDuration; + } + + public string Consumer { get; } + + public int BatchSize { get; } + + public TimeSpan LeaseDuration { get; } +} + +public sealed class QueueClaimOptions +{ + public QueueClaimOptions( + string claimantConsumer, + int batchSize, + TimeSpan minIdleTime) + { + if (string.IsNullOrWhiteSpace(claimantConsumer)) + { + throw new ArgumentException("Consumer must be provided.", nameof(claimantConsumer)); + } + + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive."); + } + + if (minIdleTime < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(minIdleTime), minIdleTime, "Idle time cannot be negative."); + } + + ClaimantConsumer = claimantConsumer; + BatchSize = batchSize; + MinIdleTime = minIdleTime; + } + + public string ClaimantConsumer { get; } + + public int BatchSize { get; } + + public TimeSpan MinIdleTime { get; } +} + +public enum QueueReleaseDisposition +{ + Retry, + Abandon +} diff --git a/src/StellaOps.Scanner.Queue/ScannerQueueHealthCheck.cs b/src/StellaOps.Scanner.Queue/ScannerQueueHealthCheck.cs new file mode 100644 index 00000000..65b23281 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/ScannerQueueHealthCheck.cs @@ -0,0 +1,55 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Queue.Nats; +using StellaOps.Scanner.Queue.Redis; + +namespace StellaOps.Scanner.Queue; + +public sealed class ScannerQueueHealthCheck : IHealthCheck +{ + private readonly IScanQueue _queue; + private readonly ILogger _logger; + + public ScannerQueueHealthCheck( + IScanQueue queue, + ILogger logger) + { + _queue = queue ?? throw new ArgumentNullException(nameof(queue)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task CheckHealthAsync( + HealthCheckContext context, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + switch (_queue) + { + case RedisScanQueue redisQueue: + await redisQueue.PingAsync(cancellationToken).ConfigureAwait(false); + return HealthCheckResult.Healthy("Redis queue reachable."); + + case NatsScanQueue natsQueue: + await natsQueue.PingAsync(cancellationToken).ConfigureAwait(false); + return HealthCheckResult.Healthy("NATS queue reachable."); + + default: + return HealthCheckResult.Healthy("Queue transport without dedicated ping returned healthy."); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "Scanner queue health check failed."); + return new HealthCheckResult( + context.Registration.FailureStatus, + "Queue transport unreachable.", + ex); + } + } +} diff --git a/src/StellaOps.Scanner.Queue/ScannerQueueOptions.cs b/src/StellaOps.Scanner.Queue/ScannerQueueOptions.cs new file mode 100644 index 00000000..4dd05160 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/ScannerQueueOptions.cs @@ -0,0 +1,92 @@ +using System; + +namespace StellaOps.Scanner.Queue; + +public sealed class ScannerQueueOptions +{ + public QueueTransportKind Kind { get; set; } = QueueTransportKind.Redis; + + public RedisQueueOptions Redis { get; set; } = new(); + + public NatsQueueOptions Nats { get; set; } = new(); + + /// + /// Default lease duration applied when callers do not override the visibility timeout. + /// + public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum number of times a message may be delivered before it is shunted to the dead-letter queue. + /// + public int MaxDeliveryAttempts { get; set; } = 5; + + /// + /// Options controlling retry/backoff/dead-letter handling. + /// + public DeadLetterQueueOptions DeadLetter { get; set; } = new(); + + /// + /// Initial backoff applied when a job is retried after failure. + /// + public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// Maximum backoff window applied for exponential retry. + /// + public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(2); +} + +public sealed class RedisQueueOptions +{ + public string? ConnectionString { get; set; } + + public int? Database { get; set; } + + public string StreamName { get; set; } = "scanner:jobs"; + + public string ConsumerGroup { get; set; } = "scanner-workers"; + + public string IdempotencyKeyPrefix { get; set; } = "scanner:jobs:idemp:"; + + public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(12); + + public int? ApproximateMaxLength { get; set; } + + public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public TimeSpan ClaimIdleThreshold { get; set; } = TimeSpan.FromMinutes(10); + + public TimeSpan PendingScanWindow { get; set; } = TimeSpan.FromMinutes(30); + + public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5); +} + +public sealed class NatsQueueOptions +{ + public string? Url { get; set; } + + public string Stream { get; set; } = "SCANNER_JOBS"; + + public string Subject { get; set; } = "scanner.jobs"; + + public string DurableConsumer { get; set; } = "scanner-workers"; + + public int MaxInFlight { get; set; } = 64; + + public TimeSpan AckWait { get; set; } = TimeSpan.FromMinutes(5); + + public string DeadLetterStream { get; set; } = "SCANNER_JOBS_DEAD"; + + public string DeadLetterSubject { get; set; } = "scanner.jobs.dead"; + + public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(10); + + public TimeSpan IdleHeartbeat { get; set; } = TimeSpan.FromSeconds(30); +} + +public sealed class DeadLetterQueueOptions +{ + public string StreamName { get; set; } = "scanner:jobs:dead"; + + public TimeSpan Retention { get; set; } = TimeSpan.FromDays(7); +} diff --git a/src/StellaOps.Scanner.Queue/ScannerQueueServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Queue/ScannerQueueServiceCollectionExtensions.cs new file mode 100644 index 00000000..c93cc9d7 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/ScannerQueueServiceCollectionExtensions.cs @@ -0,0 +1,67 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Queue.Nats; +using StellaOps.Scanner.Queue.Redis; + +namespace StellaOps.Scanner.Queue; + +public static class ScannerQueueServiceCollectionExtensions +{ + public static IServiceCollection AddScannerQueue( + this IServiceCollection services, + IConfiguration configuration, + string sectionName = "scanner:queue") + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + var options = new ScannerQueueOptions(); + configuration.GetSection(sectionName).Bind(options); + + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(options); + + services.AddSingleton(sp => + { + var loggerFactory = sp.GetRequiredService(); + var timeProvider = sp.GetService() ?? TimeProvider.System; + + return options.Kind switch + { + QueueTransportKind.Redis => new RedisScanQueue( + options, + options.Redis, + loggerFactory.CreateLogger(), + timeProvider), + QueueTransportKind.Nats => new NatsScanQueue( + options, + options.Nats, + loggerFactory.CreateLogger(), + timeProvider), + _ => throw new InvalidOperationException($"Unsupported queue transport kind '{options.Kind}'.") + }; + }); + + services.AddSingleton(); + + return services; + } + + public static IHealthChecksBuilder AddScannerQueueHealthCheck( + this IHealthChecksBuilder builder) + { + ArgumentNullException.ThrowIfNull(builder); + + builder.Services.TryAddSingleton(); + builder.AddCheck( + name: "scanner-queue", + failureStatus: HealthStatus.Unhealthy, + tags: new[] { "scanner", "queue" }); + + return builder; + } +} diff --git a/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj b/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj new file mode 100644 index 00000000..d58c88d0 --- /dev/null +++ b/src/StellaOps.Scanner.Queue/StellaOps.Scanner.Queue.csproj @@ -0,0 +1,21 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Queue/TASKS.md b/src/StellaOps.Scanner.Queue/TASKS.md new file mode 100644 index 00000000..c675440b --- /dev/null +++ b/src/StellaOps.Scanner.Queue/TASKS.md @@ -0,0 +1,7 @@ +# Scanner Queue Task Board (Sprint 9) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-QUEUE-09-401 | DONE (2025-10-19) | Scanner Queue Guild | — | Implement queue abstraction + Redis Streams adapter with ack/lease semantics, idempotency tokens, and deterministic job IDs. | Interfaces finalized; Redis adapter passes enqueue/dequeue/ack/claim lease tests; structured logs exercised. | +| SCANNER-QUEUE-09-402 | DONE (2025-10-19) | Scanner Queue Guild | SCANNER-QUEUE-09-401 | Add pluggable backend support (Redis, NATS) with configuration binding, health probes, failover documentation. | NATS adapter + DI bindings delivered; health checks documented; configuration tests green. | +| SCANNER-QUEUE-09-403 | DONE (2025-10-19) | Scanner Queue Guild | SCANNER-QUEUE-09-401 | Implement retry and dead-letter flow with structured metrics/logs for offline deployments. | Retry policy configurable; dead-letter queue persisted; metrics counters validated in integration tests. | diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Attestation/AttestorClientTests.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Attestation/AttestorClientTests.cs new file mode 100644 index 00000000..7f8b5bb9 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Attestation/AttestorClientTests.cs @@ -0,0 +1,82 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Sbomer.BuildXPlugin; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; +using Xunit; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Attestation; + +public sealed class AttestorClientTests +{ + [Fact] + public async Task SendPlaceholderAsync_PostsJsonPayload() + { + var handler = new RecordingHandler(new HttpResponseMessage(HttpStatusCode.Accepted)); + using var httpClient = new HttpClient(handler); + var client = new AttestorClient(httpClient); + + var document = BuildDescriptorDocument(); + var attestorUri = new Uri("https://attestor.example.com/api/v1/provenance"); + + await client.SendPlaceholderAsync(attestorUri, document, CancellationToken.None); + + Assert.NotNull(handler.CapturedRequest); + Assert.Equal(HttpMethod.Post, handler.CapturedRequest!.Method); + Assert.Equal(attestorUri, handler.CapturedRequest.RequestUri); + + var content = await handler.CapturedRequest.Content!.ReadAsStringAsync(); + var json = JsonDocument.Parse(content); + Assert.Equal(document.Subject.Digest, json.RootElement.GetProperty("imageDigest").GetString()); + Assert.Equal(document.Artifact.Digest, json.RootElement.GetProperty("sbomDigest").GetString()); + Assert.Equal(document.Provenance.ExpectedDsseSha256, json.RootElement.GetProperty("expectedDsseSha256").GetString()); + } + + [Fact] + public async Task SendPlaceholderAsync_ThrowsOnFailure() + { + var handler = new RecordingHandler(new HttpResponseMessage(HttpStatusCode.BadRequest) + { + Content = new StringContent("invalid") + }); + using var httpClient = new HttpClient(handler); + var client = new AttestorClient(httpClient); + + var document = BuildDescriptorDocument(); + var attestorUri = new Uri("https://attestor.example.com/api/v1/provenance"); + + await Assert.ThrowsAsync(() => client.SendPlaceholderAsync(attestorUri, document, CancellationToken.None)); + } + + private static DescriptorDocument BuildDescriptorDocument() + { + var subject = new DescriptorSubject("application/vnd.oci.image.manifest.v1+json", "sha256:img"); + var artifact = new DescriptorArtifact("application/vnd.cyclonedx+json", "sha256:sbom", 42, new System.Collections.Generic.Dictionary()); + var provenance = new DescriptorProvenance("pending", "sha256:dsse", "nonce", "https://attestor.example.com/api/v1/provenance", "https://slsa.dev/provenance/v1"); + var generatorMetadata = new DescriptorGeneratorMetadata("generator", "1.0.0"); + var metadata = new System.Collections.Generic.Dictionary(); + return new DescriptorDocument("schema", DateTimeOffset.UtcNow, generatorMetadata, subject, artifact, provenance, metadata); + } + + private sealed class RecordingHandler : HttpMessageHandler + { + private readonly HttpResponseMessage response; + + public RecordingHandler(HttpResponseMessage response) + { + this.response = response; + } + + public HttpRequestMessage? CapturedRequest { get; private set; } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + CapturedRequest = request; + return Task.FromResult(response); + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Cas/LocalCasClientTests.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Cas/LocalCasClientTests.cs new file mode 100644 index 00000000..3e0f541b --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Cas/LocalCasClientTests.cs @@ -0,0 +1,34 @@ +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Cas; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities; +using Xunit; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Cas; + +public sealed class LocalCasClientTests +{ + [Fact] + public async Task VerifyWriteAsync_WritesProbeObject() + { + await using var temp = new TempDirectory(); + var client = new LocalCasClient(new LocalCasOptions + { + RootDirectory = temp.Path, + Algorithm = "sha256" + }); + + var result = await client.VerifyWriteAsync(CancellationToken.None); + + Assert.Equal("sha256", result.Algorithm); + Assert.True(File.Exists(result.Path)); + + var bytes = await File.ReadAllBytesAsync(result.Path); + Assert.Equal("stellaops-buildx-probe"u8.ToArray(), bytes); + + var expectedDigest = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + Assert.Equal(expectedDigest, result.Digest); + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGeneratorTests.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGeneratorTests.cs new file mode 100644 index 00000000..1c334364 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGeneratorTests.cs @@ -0,0 +1,150 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities; +using Xunit; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Descriptor; + +public sealed class DescriptorGeneratorTests +{ + [Fact] + public async Task CreateAsync_BuildsDeterministicDescriptor() + { + await using var temp = new TempDirectory(); + var sbomPath = Path.Combine(temp.Path, "sample.cdx.json"); + await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.5\"}"); + + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var generator = new DescriptorGenerator(fakeTime); + + var request = new DescriptorRequest + { + ImageDigest = "sha256:0123456789abcdef", + SbomPath = sbomPath, + SbomMediaType = "application/vnd.cyclonedx+json", + SbomFormat = "cyclonedx-json", + SbomKind = "inventory", + SbomArtifactType = "application/vnd.stellaops.sbom.layer+json", + SubjectMediaType = "application/vnd.oci.image.manifest.v1+json", + GeneratorVersion = "1.2.3", + GeneratorName = "StellaOps.Scanner.Sbomer.BuildXPlugin", + LicenseId = "lic-123", + SbomName = "sample.cdx.json", + Repository = "git.stella-ops.org/stellaops", + BuildRef = "refs/heads/main", + AttestorUri = "https://attestor.local/api/v1/provenance" + }.Validate(); + + var document = await generator.CreateAsync(request, CancellationToken.None); + + Assert.Equal(DescriptorGenerator.Schema, document.Schema); + Assert.Equal(fakeTime.GetUtcNow(), document.GeneratedAt); + Assert.Equal(request.ImageDigest, document.Subject.Digest); + Assert.Equal(request.SbomMediaType, document.Artifact.MediaType); + Assert.Equal(request.SbomName, document.Artifact.Annotations["org.opencontainers.image.title"]); + Assert.Equal("pending", document.Provenance.Status); + Assert.Equal(request.AttestorUri, document.Provenance.AttestorUri); + Assert.Equal(request.PredicateType, document.Provenance.PredicateType); + + var expectedSbomDigest = ComputeSha256File(sbomPath); + Assert.Equal(expectedSbomDigest, document.Artifact.Digest); + Assert.Equal(expectedSbomDigest, document.Metadata["sbomDigest"]); + + var expectedDsse = ComputeExpectedDsse(request.ImageDigest, expectedSbomDigest, document.Provenance.Nonce); + Assert.Equal(expectedDsse, document.Provenance.ExpectedDsseSha256); + Assert.Equal(expectedDsse, document.Artifact.Annotations["org.stellaops.provenance.dsse.sha256"]); + Assert.Equal(document.Provenance.Nonce, document.Artifact.Annotations["org.stellaops.provenance.nonce"]); + } + + [Fact] + public async Task CreateAsync_RepeatedInvocationsReuseDeterministicNonce() + { + await using var temp = new TempDirectory(); + var sbomPath = Path.Combine(temp.Path, "sample.cdx.json"); + await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.5\"}"); + + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var generator = new DescriptorGenerator(fakeTime); + + var request = new DescriptorRequest + { + ImageDigest = "sha256:0123456789abcdef", + SbomPath = sbomPath, + SbomMediaType = "application/vnd.cyclonedx+json", + SbomFormat = "cyclonedx-json", + SbomKind = "inventory", + SbomArtifactType = "application/vnd.stellaops.sbom.layer+json", + SubjectMediaType = "application/vnd.oci.image.manifest.v1+json", + GeneratorVersion = "1.2.3", + GeneratorName = "StellaOps.Scanner.Sbomer.BuildXPlugin", + LicenseId = "lic-123", + SbomName = "sample.cdx.json", + Repository = "git.stella-ops.org/stellaops", + BuildRef = "refs/heads/main", + AttestorUri = "https://attestor.local/api/v1/provenance" + }.Validate(); + + var first = await generator.CreateAsync(request, CancellationToken.None); + var second = await generator.CreateAsync(request, CancellationToken.None); + + Assert.Equal(first.Provenance.Nonce, second.Provenance.Nonce); + Assert.Equal(first.Provenance.ExpectedDsseSha256, second.Provenance.ExpectedDsseSha256); + Assert.Equal(first.Artifact.Annotations["org.stellaops.provenance.nonce"], second.Artifact.Annotations["org.stellaops.provenance.nonce"]); + Assert.Equal(first.Artifact.Annotations["org.stellaops.provenance.dsse.sha256"], second.Artifact.Annotations["org.stellaops.provenance.dsse.sha256"]); + } + + [Fact] + public async Task CreateAsync_MetadataDifferencesYieldDistinctNonce() + { + await using var temp = new TempDirectory(); + var sbomPath = Path.Combine(temp.Path, "sample.cdx.json"); + await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.5\"}"); + + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var generator = new DescriptorGenerator(fakeTime); + + var baseline = new DescriptorRequest + { + ImageDigest = "sha256:0123456789abcdef", + SbomPath = sbomPath, + Repository = "git.stella-ops.org/stellaops", + BuildRef = "refs/heads/main" + }.Validate(); + + var variant = baseline with + { + BuildRef = "refs/heads/feature", + Repository = "git.stella-ops.org/stellaops/feature" + }; + variant = variant.Validate(); + + var baselineDocument = await generator.CreateAsync(baseline, CancellationToken.None); + var variantDocument = await generator.CreateAsync(variant, CancellationToken.None); + + Assert.NotEqual(baselineDocument.Provenance.Nonce, variantDocument.Provenance.Nonce); + Assert.NotEqual(baselineDocument.Provenance.ExpectedDsseSha256, variantDocument.Provenance.ExpectedDsseSha256); + } + + private static string ComputeSha256File(string path) + { + using var stream = File.OpenRead(path); + var hash = SHA256.HashData(stream); + return $"sha256:{Convert.ToHexString(hash).ToLower(CultureInfo.InvariantCulture)}"; + } + + private static string ComputeExpectedDsse(string imageDigest, string sbomDigest, string nonce) + { + var payload = $"{imageDigest}\n{sbomDigest}\n{nonce}"; + Span hash = stackalloc byte[32]; + SHA256.HashData(Encoding.UTF8.GetBytes(payload), hash); + return $"sha256:{Convert.ToHexString(hash).ToLower(CultureInfo.InvariantCulture)}"; + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGoldenTests.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGoldenTests.cs new file mode 100644 index 00000000..b5ce49b7 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Descriptor/DescriptorGoldenTests.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities; +using Xunit; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Descriptor; + +public sealed class DescriptorGoldenTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + [Fact] + public async Task DescriptorMatchesBaselineFixture() + { + await using var temp = new TempDirectory(); + var sbomPath = Path.Combine(temp.Path, "sample.cdx.json"); + await File.WriteAllTextAsync(sbomPath, "{\"bomFormat\":\"CycloneDX\",\"specVersion\":\"1.5\"}"); + + var request = new DescriptorRequest + { + ImageDigest = "sha256:0123456789abcdef", + SbomPath = sbomPath, + SbomMediaType = "application/vnd.cyclonedx+json", + SbomFormat = "cyclonedx-json", + SbomKind = "inventory", + SbomArtifactType = "application/vnd.stellaops.sbom.layer+json", + SubjectMediaType = "application/vnd.oci.image.manifest.v1+json", + GeneratorVersion = "1.2.3", + GeneratorName = "StellaOps.Scanner.Sbomer.BuildXPlugin", + LicenseId = "lic-123", + SbomName = "sample.cdx.json", + Repository = "git.stella-ops.org/stellaops", + BuildRef = "refs/heads/main", + AttestorUri = "https://attestor.local/api/v1/provenance" + }.Validate(); + + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 18, 12, 0, 0, TimeSpan.Zero)); + var generator = new DescriptorGenerator(fakeTime); + var document = await generator.CreateAsync(request, CancellationToken.None); + var actualJson = JsonSerializer.Serialize(document, SerializerOptions); + var normalizedJson = NormalizeDescriptorJson(actualJson, Path.GetFileName(sbomPath)); + + var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..")); + var fixturePath = Path.Combine(projectRoot, "Fixtures", "descriptor.baseline.json"); + var updateRequested = string.Equals(Environment.GetEnvironmentVariable("UPDATE_BUILDX_FIXTURES"), "1", StringComparison.OrdinalIgnoreCase); + + if (updateRequested) + { + Directory.CreateDirectory(Path.GetDirectoryName(fixturePath)!); + await File.WriteAllTextAsync(fixturePath, normalizedJson); + return; + } + + if (!File.Exists(fixturePath)) + { + throw new InvalidOperationException($"Baseline fixture '{fixturePath}' is missing. Set UPDATE_BUILDX_FIXTURES=1 and re-run the tests to generate it."); + } + + var baselineJson = await File.ReadAllTextAsync(fixturePath); + + using var baselineDoc = JsonDocument.Parse(baselineJson); + using var actualDoc = JsonDocument.Parse(normalizedJson); + + AssertJsonEquivalent(baselineDoc.RootElement, actualDoc.RootElement); + } + + private static string NormalizeDescriptorJson(string json, string sbomFileName) + { + var node = JsonNode.Parse(json)?.AsObject() + ?? throw new InvalidOperationException("Failed to parse descriptor JSON for normalization."); + + if (node["metadata"] is JsonObject metadata) + { + metadata["sbomPath"] = sbomFileName; + } + + return node.ToJsonString(SerializerOptions); + } + + private static void AssertJsonEquivalent(JsonElement expected, JsonElement actual) + { + if (expected.ValueKind != actual.ValueKind) + { + throw new Xunit.Sdk.XunitException($"Value kind mismatch. Expected '{expected.ValueKind}' but found '{actual.ValueKind}'."); + } + + switch (expected.ValueKind) + { + case JsonValueKind.Object: + var expectedProperties = expected.EnumerateObject().ToDictionary(p => p.Name, p => p.Value, StringComparer.Ordinal); + var actualProperties = actual.EnumerateObject().ToDictionary(p => p.Name, p => p.Value, StringComparer.Ordinal); + + Assert.Equal( + expectedProperties.Keys.OrderBy(static name => name).ToArray(), + actualProperties.Keys.OrderBy(static name => name).ToArray()); + + foreach (var propertyName in expectedProperties.Keys) + { + AssertJsonEquivalent(expectedProperties[propertyName], actualProperties[propertyName]); + } + + break; + case JsonValueKind.Array: + var expectedItems = expected.EnumerateArray().ToArray(); + var actualItems = actual.EnumerateArray().ToArray(); + + Assert.Equal(expectedItems.Length, actualItems.Length); + for (var i = 0; i < expectedItems.Length; i++) + { + AssertJsonEquivalent(expectedItems[i], actualItems[i]); + } + + break; + default: + Assert.Equal(expected.ToString(), actual.ToString()); + break; + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json new file mode 100644 index 00000000..4a707a28 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Fixtures/descriptor.baseline.json @@ -0,0 +1,45 @@ +{ + "schema": "stellaops.buildx.descriptor.v1", + "generatedAt": "2025-10-18T12:00:00\u002B00:00", + "generator": { + "name": "StellaOps.Scanner.Sbomer.BuildXPlugin", + "version": "1.2.3" + }, + "subject": { + "mediaType": "application/vnd.oci.image.manifest.v1\u002Bjson", + "digest": "sha256:0123456789abcdef" + }, + "artifact": { + "mediaType": "application/vnd.cyclonedx\u002Bjson", + "digest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c", + "size": 45, + "annotations": { + "org.opencontainers.artifact.type": "application/vnd.stellaops.sbom.layer\u002Bjson", + "org.stellaops.scanner.version": "1.2.3", + "org.stellaops.sbom.kind": "inventory", + "org.stellaops.sbom.format": "cyclonedx-json", + "org.stellaops.provenance.status": "pending", + "org.stellaops.provenance.dsse.sha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d", + "org.stellaops.provenance.nonce": "a608acf859cd58a8389816b8d9eb2a07", + "org.stellaops.license.id": "lic-123", + "org.opencontainers.image.title": "sample.cdx.json", + "org.stellaops.repository": "git.stella-ops.org/stellaops" + } + }, + "provenance": { + "status": "pending", + "expectedDsseSha256": "sha256:1b364a6b888d580feb8565f7b6195b24535ca8201b4bcac58da063b32c47220d", + "nonce": "a608acf859cd58a8389816b8d9eb2a07", + "attestorUri": "https://attestor.local/api/v1/provenance", + "predicateType": "https://slsa.dev/provenance/v1" + }, + "metadata": { + "sbomDigest": "sha256:d07d06ae82e1789a5b505731f3ec3add106e23a55395213c9a881c7e816c695c", + "sbomPath": "sample.cdx.json", + "sbomMediaType": "application/vnd.cyclonedx\u002Bjson", + "subjectMediaType": "application/vnd.oci.image.manifest.v1\u002Bjson", + "repository": "git.stella-ops.org/stellaops", + "buildRef": "refs/heads/main", + "attestorUri": "https://attestor.local/api/v1/provenance" + } +} \ No newline at end of file diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Manifest/BuildxPluginManifestLoaderTests.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Manifest/BuildxPluginManifestLoaderTests.cs new file mode 100644 index 00000000..2297f41a --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/Manifest/BuildxPluginManifestLoaderTests.cs @@ -0,0 +1,80 @@ +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Sbomer.BuildXPlugin; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities; +using Xunit; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.Manifest; + +public sealed class BuildxPluginManifestLoaderTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true + }; + + [Fact] + public async Task LoadAsync_ReturnsManifestWithSourceInformation() + { + await using var temp = new TempDirectory(); + var manifestPath = System.IO.Path.Combine(temp.Path, "stellaops.manifest.json"); + await File.WriteAllTextAsync(manifestPath, BuildSampleManifestJson("stellaops.sbom-indexer")); + + var loader = new BuildxPluginManifestLoader(temp.Path); + var manifests = await loader.LoadAsync(CancellationToken.None); + + var manifest = Assert.Single(manifests); + Assert.Equal("stellaops.sbom-indexer", manifest.Id); + Assert.Equal("0.1.0", manifest.Version); + Assert.Equal(manifestPath, manifest.SourcePath); + Assert.Equal(Path.GetDirectoryName(manifestPath), manifest.SourceDirectory); + } + + [Fact] + public async Task LoadDefaultAsync_ThrowsWhenNoManifests() + { + await using var temp = new TempDirectory(); + var loader = new BuildxPluginManifestLoader(temp.Path); + + await Assert.ThrowsAsync(() => loader.LoadDefaultAsync(CancellationToken.None)); + } + + [Fact] + public async Task LoadAsync_ThrowsWhenRestartRequiredMissing() + { + await using var temp = new TempDirectory(); + var manifestPath = Path.Combine(temp.Path, "failure.manifest.json"); + await File.WriteAllTextAsync(manifestPath, BuildSampleManifestJson("stellaops.failure", requiresRestart: false)); + + var loader = new BuildxPluginManifestLoader(temp.Path); + + await Assert.ThrowsAsync(() => loader.LoadAsync(CancellationToken.None)); + } + + private static string BuildSampleManifestJson(string id, bool requiresRestart = true) + { + var manifest = new BuildxPluginManifest + { + SchemaVersion = BuildxPluginManifest.CurrentSchemaVersion, + Id = id, + DisplayName = "Sample", + Version = "0.1.0", + RequiresRestart = requiresRestart, + EntryPoint = new BuildxPluginEntryPoint + { + Type = "dotnet", + Executable = "StellaOps.Scanner.Sbomer.BuildXPlugin.dll" + }, + Cas = new BuildxPluginCas + { + Protocol = "filesystem", + DefaultRoot = "cas" + } + }; + + return JsonSerializer.Serialize(manifest, SerializerOptions); + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.csproj b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.csproj new file mode 100644 index 00000000..55c965cb --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/TestUtilities/TempDirectory.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/TestUtilities/TempDirectory.cs new file mode 100644 index 00000000..cd29e142 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin.Tests/TestUtilities/TempDirectory.cs @@ -0,0 +1,44 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Tests.TestUtilities; + +internal sealed class TempDirectory : IDisposable, IAsyncDisposable +{ + public string Path { get; } + + public TempDirectory() + { + Path = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"stellaops-buildx-{Guid.NewGuid():N}"); + Directory.CreateDirectory(Path); + } + + public void Dispose() + { + Cleanup(); + GC.SuppressFinalize(this); + } + + public ValueTask DisposeAsync() + { + Cleanup(); + GC.SuppressFinalize(this); + return ValueTask.CompletedTask; + } + + private void Cleanup() + { + try + { + if (Directory.Exists(Path)) + { + Directory.Delete(Path, recursive: true); + } + } + catch + { + // Best effort cleanup only. + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/AGENTS.md b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/AGENTS.md new file mode 100644 index 00000000..7b119d2c --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/AGENTS.md @@ -0,0 +1,12 @@ +# StellaOps.Scanner.Sbomer.BuildXPlugin — Agent Charter + +## Mission +Implement the build-time SBOM generator described in `docs/ARCHITECTURE_SCANNER.md` and new buildx dossier requirements: +- Provide a deterministic BuildKit/Buildx generator that produces layer SBOM fragments and uploads them to local CAS. +- Emit OCI annotations (+provenance) compatible with Scanner.Emit and Attestor hand-offs. +- Respect restart-time plug-in policy (`plugins/scanner/buildx/` manifests) and keep CI overhead ≤300 ms per layer. + +## Expectations +- Read architecture + upcoming Buildx addendum before coding. +- Ensure graceful fallback to post-build scan when generator unavailable. +- Provide integration tests with mock BuildKit, and update `TASKS.md` as states change. diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorClient.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorClient.cs new file mode 100644 index 00000000..05c09ab5 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorClient.cs @@ -0,0 +1,49 @@ +using System; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation; + +/// +/// Sends provenance placeholders to the Attestor service for asynchronous DSSE signing. +/// +public sealed class AttestorClient +{ + private readonly HttpClient httpClient; + + public AttestorClient(HttpClient httpClient) + { + this.httpClient = httpClient ?? throw new ArgumentNullException(nameof(httpClient)); + } + + public async Task SendPlaceholderAsync(Uri attestorUri, DescriptorDocument document, CancellationToken cancellationToken) + { + if (attestorUri is null) + { + throw new ArgumentNullException(nameof(attestorUri)); + } + + if (document is null) + { + throw new ArgumentNullException(nameof(document)); + } + + var payload = new AttestorProvenanceRequest( + ImageDigest: document.Subject.Digest, + SbomDigest: document.Artifact.Digest, + ExpectedDsseSha256: document.Provenance.ExpectedDsseSha256, + Nonce: document.Provenance.Nonce, + PredicateType: document.Provenance.PredicateType, + Schema: document.Schema); + + using var response = await httpClient.PostAsJsonAsync(attestorUri, payload, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + throw new BuildxPluginException($"Attestor rejected provenance placeholder ({(int)response.StatusCode}): {body}"); + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorProvenanceRequest.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorProvenanceRequest.cs new file mode 100644 index 00000000..601a0d71 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Attestation/AttestorProvenanceRequest.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation; + +public sealed record AttestorProvenanceRequest( + [property: JsonPropertyName("imageDigest")] string ImageDigest, + [property: JsonPropertyName("sbomDigest")] string SbomDigest, + [property: JsonPropertyName("expectedDsseSha256")] string ExpectedDsseSha256, + [property: JsonPropertyName("nonce")] string Nonce, + [property: JsonPropertyName("predicateType")] string PredicateType, + [property: JsonPropertyName("schema")] string Schema); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/BuildxPluginException.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/BuildxPluginException.cs new file mode 100644 index 00000000..edf1beeb --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/BuildxPluginException.cs @@ -0,0 +1,19 @@ +using System; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin; + +/// +/// Represents user-facing errors raised by the BuildX plug-in. +/// +public sealed class BuildxPluginException : Exception +{ + public BuildxPluginException(string message) + : base(message) + { + } + + public BuildxPluginException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/CasWriteResult.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/CasWriteResult.cs new file mode 100644 index 00000000..2aa9b2cf --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/CasWriteResult.cs @@ -0,0 +1,6 @@ +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas; + +/// +/// Result of persisting bytes into the local CAS. +/// +public sealed record CasWriteResult(string Algorithm, string Digest, string Path); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasClient.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasClient.cs new file mode 100644 index 00000000..0bb157a2 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasClient.cs @@ -0,0 +1,74 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas; + +/// +/// Minimal filesystem-backed CAS used when the BuildX generator runs inside CI. +/// +public sealed class LocalCasClient +{ + private readonly string rootDirectory; + private readonly string algorithm; + + public LocalCasClient(LocalCasOptions options) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + algorithm = options.Algorithm.ToLowerInvariant(); + if (!string.Equals(algorithm, "sha256", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("Only the sha256 algorithm is supported.", nameof(options)); + } + + rootDirectory = Path.GetFullPath(options.RootDirectory); + } + + public Task VerifyWriteAsync(CancellationToken cancellationToken) + { + ReadOnlyMemory probe = "stellaops-buildx-probe"u8.ToArray(); + return WriteAsync(probe, cancellationToken); + } + + public async Task WriteAsync(ReadOnlyMemory content, CancellationToken cancellationToken) + { + var digest = ComputeDigest(content.Span); + var path = BuildObjectPath(digest); + + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + + await using var stream = new FileStream( + path, + FileMode.Create, + FileAccess.Write, + FileShare.Read, + bufferSize: 16 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + + await stream.WriteAsync(content, cancellationToken).ConfigureAwait(false); + await stream.FlushAsync(cancellationToken).ConfigureAwait(false); + + return new CasWriteResult(algorithm, digest, path); + } + + private string BuildObjectPath(string digest) + { + // Layout: ///.bin + var prefix = digest.Substring(0, 2); + var suffix = digest[2..]; + return Path.Combine(rootDirectory, algorithm, prefix, $"{suffix}.bin"); + } + + private static string ComputeDigest(ReadOnlySpan content) + { + Span buffer = stackalloc byte[32]; + SHA256.HashData(content, buffer); + return Convert.ToHexString(buffer).ToLowerInvariant(); + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasOptions.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasOptions.cs new file mode 100644 index 00000000..0f6ed9c4 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Cas/LocalCasOptions.cs @@ -0,0 +1,40 @@ +using System; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Cas; + +/// +/// Configuration for the on-disk content-addressable store used during CI. +/// +public sealed record LocalCasOptions +{ + private string rootDirectory = string.Empty; + private string algorithm = "sha256"; + + public string RootDirectory + { + get => rootDirectory; + init + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Root directory must be provided.", nameof(value)); + } + + rootDirectory = value; + } + } + + public string Algorithm + { + get => algorithm; + init + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Algorithm must be provided.", nameof(value)); + } + + algorithm = value; + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorArtifact.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorArtifact.cs new file mode 100644 index 00000000..03945b91 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorArtifact.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +/// +/// Represents an OCI artifact descriptor emitted by the BuildX generator. +/// +public sealed record DescriptorArtifact( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("digest")] string Digest, + [property: JsonPropertyName("size")] long Size, + [property: JsonPropertyName("annotations")] IReadOnlyDictionary Annotations); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorDocument.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorDocument.cs new file mode 100644 index 00000000..b220832a --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorDocument.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +/// +/// Root payload describing BuildX generator output with provenance placeholders. +/// +public sealed record DescriptorDocument( + [property: JsonPropertyName("schema")] string Schema, + [property: JsonPropertyName("generatedAt")] DateTimeOffset GeneratedAt, + [property: JsonPropertyName("generator")] DescriptorGeneratorMetadata Generator, + [property: JsonPropertyName("subject")] DescriptorSubject Subject, + [property: JsonPropertyName("artifact")] DescriptorArtifact Artifact, + [property: JsonPropertyName("provenance")] DescriptorProvenance Provenance, + [property: JsonPropertyName("metadata")] IReadOnlyDictionary Metadata); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGenerator.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGenerator.cs new file mode 100644 index 00000000..d0d4abd7 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGenerator.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +/// +/// Builds immutable OCI descriptors enriched with provenance placeholders. +/// +public sealed class DescriptorGenerator +{ + public const string Schema = "stellaops.buildx.descriptor.v1"; + + private readonly TimeProvider timeProvider; + + public DescriptorGenerator(TimeProvider timeProvider) + { + timeProvider ??= TimeProvider.System; + this.timeProvider = timeProvider; + } + + public async Task CreateAsync(DescriptorRequest request, CancellationToken cancellationToken) + { + if (request is null) + { + throw new ArgumentNullException(nameof(request)); + } + + if (string.IsNullOrWhiteSpace(request.ImageDigest)) + { + throw new BuildxPluginException("Image digest must be provided."); + } + + if (string.IsNullOrWhiteSpace(request.SbomPath)) + { + throw new BuildxPluginException("SBOM path must be provided."); + } + + var sbomFile = new FileInfo(request.SbomPath); + if (!sbomFile.Exists) + { + throw new BuildxPluginException($"SBOM file '{request.SbomPath}' was not found."); + } + + var sbomDigest = await ComputeFileDigestAsync(sbomFile, cancellationToken).ConfigureAwait(false); + var nonce = ComputeDeterministicNonce(request, sbomFile, sbomDigest); + var expectedDsseSha = ComputeExpectedDsseDigest(request.ImageDigest, sbomDigest, nonce); + + var artifactAnnotations = BuildArtifactAnnotations(request, nonce, expectedDsseSha); + + var subject = new DescriptorSubject( + MediaType: request.SubjectMediaType, + Digest: request.ImageDigest); + + var artifact = new DescriptorArtifact( + MediaType: request.SbomMediaType, + Digest: sbomDigest, + Size: sbomFile.Length, + Annotations: artifactAnnotations); + + var provenance = new DescriptorProvenance( + Status: "pending", + ExpectedDsseSha256: expectedDsseSha, + Nonce: nonce, + AttestorUri: request.AttestorUri, + PredicateType: request.PredicateType); + + var generatorMetadata = new DescriptorGeneratorMetadata( + Name: request.GeneratorName ?? "StellaOps.Scanner.Sbomer.BuildXPlugin", + Version: request.GeneratorVersion); + + var metadata = BuildDocumentMetadata(request, sbomFile, sbomDigest); + + return new DescriptorDocument( + Schema: Schema, + GeneratedAt: timeProvider.GetUtcNow(), + Generator: generatorMetadata, + Subject: subject, + Artifact: artifact, + Provenance: provenance, + Metadata: metadata); + } + + private static string ComputeDeterministicNonce(DescriptorRequest request, FileInfo sbomFile, string sbomDigest) + { + var builder = new StringBuilder(); + builder.AppendLine("stellaops.buildx.nonce.v1"); + builder.AppendLine(request.ImageDigest); + builder.AppendLine(sbomDigest); + builder.AppendLine(sbomFile.Length.ToString(CultureInfo.InvariantCulture)); + builder.AppendLine(request.SbomMediaType); + builder.AppendLine(request.SbomFormat); + builder.AppendLine(request.SbomKind); + builder.AppendLine(request.SbomArtifactType); + builder.AppendLine(request.SubjectMediaType); + builder.AppendLine(request.GeneratorVersion); + builder.AppendLine(request.GeneratorName ?? string.Empty); + builder.AppendLine(request.LicenseId ?? string.Empty); + builder.AppendLine(request.SbomName ?? string.Empty); + builder.AppendLine(request.Repository ?? string.Empty); + builder.AppendLine(request.BuildRef ?? string.Empty); + builder.AppendLine(request.AttestorUri ?? string.Empty); + builder.AppendLine(request.PredicateType); + + var payload = Encoding.UTF8.GetBytes(builder.ToString()); + Span hash = stackalloc byte[32]; + SHA256.HashData(payload, hash); + + Span nonceBytes = stackalloc byte[16]; + hash[..16].CopyTo(nonceBytes); + return Convert.ToHexString(nonceBytes).ToLowerInvariant(); + } + + private static async Task ComputeFileDigestAsync(FileInfo file, CancellationToken cancellationToken) + { + await using var stream = new FileStream( + file.FullName, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 128 * 1024, + FileOptions.Asynchronous | FileOptions.SequentialScan); + + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + + var buffer = new byte[128 * 1024]; + int bytesRead; + while ((bytesRead = await stream.ReadAsync(buffer.AsMemory(0, buffer.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + hash.AppendData(buffer, 0, bytesRead); + } + + var digest = hash.GetHashAndReset(); + return $"sha256:{Convert.ToHexString(digest).ToLowerInvariant()}"; + } + + private static string ComputeExpectedDsseDigest(string imageDigest, string sbomDigest, string nonce) + { + var payload = $"{imageDigest}\n{sbomDigest}\n{nonce}"; + var bytes = System.Text.Encoding.UTF8.GetBytes(payload); + Span hash = stackalloc byte[32]; + SHA256.HashData(bytes, hash); + return $"sha256:{Convert.ToHexString(hash).ToLowerInvariant()}"; + } + + private static IReadOnlyDictionary BuildArtifactAnnotations(DescriptorRequest request, string nonce, string expectedDsse) + { + var annotations = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["org.opencontainers.artifact.type"] = request.SbomArtifactType, + ["org.stellaops.scanner.version"] = request.GeneratorVersion, + ["org.stellaops.sbom.kind"] = request.SbomKind, + ["org.stellaops.sbom.format"] = request.SbomFormat, + ["org.stellaops.provenance.status"] = "pending", + ["org.stellaops.provenance.dsse.sha256"] = expectedDsse, + ["org.stellaops.provenance.nonce"] = nonce + }; + + if (!string.IsNullOrWhiteSpace(request.LicenseId)) + { + annotations["org.stellaops.license.id"] = request.LicenseId!; + } + + if (!string.IsNullOrWhiteSpace(request.SbomName)) + { + annotations["org.opencontainers.image.title"] = request.SbomName!; + } + + if (!string.IsNullOrWhiteSpace(request.Repository)) + { + annotations["org.stellaops.repository"] = request.Repository!; + } + + return annotations; + } + + private static IReadOnlyDictionary BuildDocumentMetadata(DescriptorRequest request, FileInfo fileInfo, string sbomDigest) + { + var metadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["sbomDigest"] = sbomDigest, + ["sbomPath"] = fileInfo.FullName, + ["sbomMediaType"] = request.SbomMediaType, + ["subjectMediaType"] = request.SubjectMediaType + }; + + if (!string.IsNullOrWhiteSpace(request.Repository)) + { + metadata["repository"] = request.Repository!; + } + + if (!string.IsNullOrWhiteSpace(request.BuildRef)) + { + metadata["buildRef"] = request.BuildRef!; + } + + if (!string.IsNullOrWhiteSpace(request.AttestorUri)) + { + metadata["attestorUri"] = request.AttestorUri!; + } + + return metadata; + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGeneratorMetadata.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGeneratorMetadata.cs new file mode 100644 index 00000000..35bdd293 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorGeneratorMetadata.cs @@ -0,0 +1,7 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +public sealed record DescriptorGeneratorMetadata( + [property: JsonPropertyName("name")] string Name, + [property: JsonPropertyName("version")] string Version); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorProvenance.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorProvenance.cs new file mode 100644 index 00000000..610794f3 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorProvenance.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +/// +/// Provenance placeholders that the Attestor will fulfil post-build. +/// +public sealed record DescriptorProvenance( + [property: JsonPropertyName("status")] string Status, + [property: JsonPropertyName("expectedDsseSha256")] string ExpectedDsseSha256, + [property: JsonPropertyName("nonce")] string Nonce, + [property: JsonPropertyName("attestorUri")] string? AttestorUri, + [property: JsonPropertyName("predicateType")] string PredicateType); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorRequest.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorRequest.cs new file mode 100644 index 00000000..041cafc3 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorRequest.cs @@ -0,0 +1,45 @@ +using System; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +/// +/// Request for generating BuildX descriptor artifacts. +/// +public sealed record DescriptorRequest +{ + public string ImageDigest { get; init; } = string.Empty; + public string SbomPath { get; init; } = string.Empty; + public string SbomMediaType { get; init; } = "application/vnd.cyclonedx+json"; + public string SbomFormat { get; init; } = "cyclonedx-json"; + public string SbomArtifactType { get; init; } = "application/vnd.stellaops.sbom.layer+json"; + public string SbomKind { get; init; } = "inventory"; + public string SubjectMediaType { get; init; } = "application/vnd.oci.image.manifest.v1+json"; + public string GeneratorVersion { get; init; } = "0.0.0"; + public string? GeneratorName { get; init; } + public string? LicenseId { get; init; } + public string? SbomName { get; init; } + public string? Repository { get; init; } + public string? BuildRef { get; init; } + public string? AttestorUri { get; init; } + public string PredicateType { get; init; } = "https://slsa.dev/provenance/v1"; + + public DescriptorRequest Validate() + { + if (string.IsNullOrWhiteSpace(ImageDigest)) + { + throw new BuildxPluginException("Image digest is required."); + } + + if (!ImageDigest.Contains(':', StringComparison.Ordinal)) + { + throw new BuildxPluginException("Image digest must include the algorithm prefix, e.g. 'sha256:...'."); + } + + if (string.IsNullOrWhiteSpace(SbomPath)) + { + throw new BuildxPluginException("SBOM path is required."); + } + + return this; + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorSubject.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorSubject.cs new file mode 100644 index 00000000..adbf000f --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Descriptor/DescriptorSubject.cs @@ -0,0 +1,7 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; + +public sealed record DescriptorSubject( + [property: JsonPropertyName("mediaType")] string MediaType, + [property: JsonPropertyName("digest")] string Digest); diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginCas.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginCas.cs new file mode 100644 index 00000000..fbdacb16 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginCas.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +/// +/// Describes default Content Addressable Storage configuration for the plug-in. +/// +public sealed record BuildxPluginCas +{ + [JsonPropertyName("protocol")] + public string Protocol { get; init; } = "filesystem"; + + [JsonPropertyName("defaultRoot")] + public string DefaultRoot { get; init; } = "cas"; + + [JsonPropertyName("compression")] + public string Compression { get; init; } = "zstd"; +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginEntryPoint.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginEntryPoint.cs new file mode 100644 index 00000000..7217364d --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginEntryPoint.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +/// +/// Describes how the buildx plug-in executable should be invoked. +/// +public sealed record BuildxPluginEntryPoint +{ + [JsonPropertyName("type")] + public string Type { get; init; } = "dotnet"; + + [JsonPropertyName("executable")] + public string Executable { get; init; } = string.Empty; + + [JsonPropertyName("arguments")] + public IReadOnlyList Arguments { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginImage.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginImage.cs new file mode 100644 index 00000000..f6052e6a --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginImage.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +/// +/// Provides distribution information for the container image form-factor. +/// +public sealed record BuildxPluginImage +{ + [JsonPropertyName("name")] + public string Name { get; init; } = string.Empty; + + [JsonPropertyName("digest")] + public string? Digest { get; init; } + + [JsonPropertyName("platforms")] + public IReadOnlyList Platforms { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifest.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifest.cs new file mode 100644 index 00000000..3e717788 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifest.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +/// +/// Canonical manifest describing a buildx generator plug-in. +/// +public sealed record BuildxPluginManifest +{ + public const string CurrentSchemaVersion = "1.0"; + + [JsonPropertyName("schemaVersion")] + public string SchemaVersion { get; init; } = CurrentSchemaVersion; + + [JsonPropertyName("id")] + public string Id { get; init; } = string.Empty; + + [JsonPropertyName("displayName")] + public string DisplayName { get; init; } = string.Empty; + + [JsonPropertyName("version")] + public string Version { get; init; } = string.Empty; + + [JsonPropertyName("entryPoint")] + public BuildxPluginEntryPoint EntryPoint { get; init; } = new(); + + [JsonPropertyName("requiresRestart")] + public bool RequiresRestart { get; init; } = true; + + [JsonPropertyName("capabilities")] + public IReadOnlyList Capabilities { get; init; } = Array.Empty(); + + [JsonPropertyName("cas")] + public BuildxPluginCas Cas { get; init; } = new(); + + [JsonPropertyName("image")] + public BuildxPluginImage? Image { get; init; } + + [JsonPropertyName("metadata")] + public IReadOnlyDictionary? Metadata { get; init; } + + [JsonIgnore] + public string? SourcePath { get; init; } + + [JsonIgnore] + public string? SourceDirectory { get; init; } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifestLoader.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifestLoader.cs new file mode 100644 index 00000000..22d8b212 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Manifest/BuildxPluginManifestLoader.cs @@ -0,0 +1,189 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +/// +/// Loads buildx plug-in manifests from the restart-time plug-in directory. +/// +public sealed class BuildxPluginManifestLoader +{ + public const string DefaultSearchPattern = "*.manifest.json"; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + AllowTrailingCommas = true, + ReadCommentHandling = JsonCommentHandling.Skip, + PropertyNameCaseInsensitive = true + }; + + private readonly string manifestDirectory; + private readonly string searchPattern; + + public BuildxPluginManifestLoader(string manifestDirectory, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(manifestDirectory)) + { + throw new ArgumentException("Manifest directory is required.", nameof(manifestDirectory)); + } + + this.manifestDirectory = Path.GetFullPath(manifestDirectory); + this.searchPattern = string.IsNullOrWhiteSpace(searchPattern) + ? DefaultSearchPattern + : searchPattern; + } + + /// + /// Loads all manifests in the configured directory. + /// + public async Task> LoadAsync(CancellationToken cancellationToken) + { + if (!Directory.Exists(manifestDirectory)) + { + return Array.Empty(); + } + + var manifests = new List(); + + foreach (var file in Directory.EnumerateFiles(manifestDirectory, searchPattern, SearchOption.TopDirectoryOnly)) + { + if (IsHiddenPath(file)) + { + continue; + } + + var manifest = await DeserializeManifestAsync(file, cancellationToken).ConfigureAwait(false); + manifests.Add(manifest); + } + + return manifests + .OrderBy(static m => m.Id, StringComparer.OrdinalIgnoreCase) + .ThenBy(static m => m.Version, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + /// + /// Loads the manifest with the specified identifier. + /// + public async Task LoadByIdAsync(string manifestId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(manifestId)) + { + throw new ArgumentException("Manifest identifier is required.", nameof(manifestId)); + } + + var manifests = await LoadAsync(cancellationToken).ConfigureAwait(false); + var manifest = manifests.FirstOrDefault(m => string.Equals(m.Id, manifestId, StringComparison.OrdinalIgnoreCase)); + if (manifest is null) + { + throw new BuildxPluginException($"Buildx plug-in manifest '{manifestId}' was not found in '{manifestDirectory}'."); + } + + return manifest; + } + + /// + /// Loads the first available manifest. + /// + public async Task LoadDefaultAsync(CancellationToken cancellationToken) + { + var manifests = await LoadAsync(cancellationToken).ConfigureAwait(false); + if (manifests.Count == 0) + { + throw new BuildxPluginException($"No buildx plug-in manifests were discovered under '{manifestDirectory}'."); + } + + return manifests[0]; + } + + private static bool IsHiddenPath(string path) + { + var directory = Path.GetDirectoryName(path); + while (!string.IsNullOrEmpty(directory)) + { + var segment = Path.GetFileName(directory); + if (segment.StartsWith(".", StringComparison.Ordinal)) + { + return true; + } + + directory = Path.GetDirectoryName(directory); + } + + return false; + } + + private static async Task DeserializeManifestAsync(string file, CancellationToken cancellationToken) + { + await using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous); + BuildxPluginManifest? manifest; + + try + { + manifest = await JsonSerializer.DeserializeAsync(stream, SerializerOptions, cancellationToken) + .ConfigureAwait(false); + } + catch (JsonException ex) + { + throw new BuildxPluginException($"Failed to parse manifest '{file}'.", ex); + } + + if (manifest is null) + { + throw new BuildxPluginException($"Manifest '{file}' is empty or invalid."); + } + + ValidateManifest(manifest, file); + + var directory = Path.GetDirectoryName(file); + return manifest with + { + SourcePath = file, + SourceDirectory = directory + }; + } + + private static void ValidateManifest(BuildxPluginManifest manifest, string file) + { + if (!string.Equals(manifest.SchemaVersion, BuildxPluginManifest.CurrentSchemaVersion, StringComparison.OrdinalIgnoreCase)) + { + throw new BuildxPluginException( + $"Manifest '{file}' uses unsupported schema version '{manifest.SchemaVersion}'. Expected '{BuildxPluginManifest.CurrentSchemaVersion}'."); + } + + if (string.IsNullOrWhiteSpace(manifest.Id)) + { + throw new BuildxPluginException($"Manifest '{file}' must specify a non-empty 'id'."); + } + + if (manifest.EntryPoint is null) + { + throw new BuildxPluginException($"Manifest '{file}' must specify an 'entryPoint'."); + } + + if (string.IsNullOrWhiteSpace(manifest.EntryPoint.Executable)) + { + throw new BuildxPluginException($"Manifest '{file}' must specify an executable entry point."); + } + + if (!manifest.RequiresRestart) + { + throw new BuildxPluginException($"Manifest '{file}' must enforce restart-required activation."); + } + + if (manifest.Cas is null) + { + throw new BuildxPluginException($"Manifest '{file}' must define CAS defaults."); + } + + if (string.IsNullOrWhiteSpace(manifest.Cas.DefaultRoot)) + { + throw new BuildxPluginException($"Manifest '{file}' must specify a CAS default root directory."); + } + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Program.cs b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Program.cs new file mode 100644 index 00000000..810aa6a5 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/Program.cs @@ -0,0 +1,327 @@ +using System; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json.Serialization; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Attestation; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Cas; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Descriptor; +using StellaOps.Scanner.Sbomer.BuildXPlugin.Manifest; + +namespace StellaOps.Scanner.Sbomer.BuildXPlugin; + +internal static class Program +{ + private static readonly JsonSerializerOptions ManifestPrintOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private static readonly JsonSerializerOptions DescriptorJsonOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private static async Task Main(string[] args) + { + using var cancellation = new CancellationTokenSource(); + Console.CancelKeyPress += (_, eventArgs) => + { + eventArgs.Cancel = true; + cancellation.Cancel(); + }; + + var command = args.Length > 0 ? args[0].ToLowerInvariant() : "handshake"; + var commandArgs = args.Skip(1).ToArray(); + + try + { + return command switch + { + "handshake" => await RunHandshakeAsync(commandArgs, cancellation.Token).ConfigureAwait(false), + "manifest" => await RunManifestAsync(commandArgs, cancellation.Token).ConfigureAwait(false), + "descriptor" or "annotate" => await RunDescriptorAsync(commandArgs, cancellation.Token).ConfigureAwait(false), + "version" => RunVersion(), + "help" or "--help" or "-h" => PrintHelp(), + _ => UnknownCommand(command) + }; + } + catch (OperationCanceledException) + { + Console.Error.WriteLine("Operation cancelled."); + return 130; + } + catch (BuildxPluginException ex) + { + Console.Error.WriteLine(ex.Message); + return 2; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Unhandled error: {ex}"); + return 1; + } + } + + private static async Task RunHandshakeAsync(string[] args, CancellationToken cancellationToken) + { + var manifestDirectory = ResolveManifestDirectory(args); + var loader = new BuildxPluginManifestLoader(manifestDirectory); + var manifest = await loader.LoadDefaultAsync(cancellationToken).ConfigureAwait(false); + + var casRoot = ResolveCasRoot(args, manifest); + var casClient = new LocalCasClient(new LocalCasOptions + { + RootDirectory = casRoot, + Algorithm = "sha256" + }); + + var result = await casClient.VerifyWriteAsync(cancellationToken).ConfigureAwait(false); + + Console.WriteLine($"handshake ok: {manifest.Id}@{manifest.Version} → {result.Algorithm}:{result.Digest}"); + Console.WriteLine(result.Path); + return 0; + } + + private static async Task RunManifestAsync(string[] args, CancellationToken cancellationToken) + { + var manifestDirectory = ResolveManifestDirectory(args); + var loader = new BuildxPluginManifestLoader(manifestDirectory); + var manifest = await loader.LoadDefaultAsync(cancellationToken).ConfigureAwait(false); + + var json = JsonSerializer.Serialize(manifest, ManifestPrintOptions); + Console.WriteLine(json); + return 0; + } + + private static int RunVersion() + { + var assembly = Assembly.GetExecutingAssembly(); + var version = assembly.GetCustomAttribute()?.InformationalVersion + ?? assembly.GetName().Version?.ToString() + ?? "unknown"; + Console.WriteLine(version); + return 0; + } + + private static int PrintHelp() + { + Console.WriteLine("StellaOps BuildX SBOM generator"); + Console.WriteLine("Usage:"); + Console.WriteLine(" stellaops-buildx [handshake|manifest|descriptor|version]"); + Console.WriteLine(); + Console.WriteLine("Commands:"); + Console.WriteLine(" handshake Probe the local CAS and ensure manifests are discoverable."); + Console.WriteLine(" manifest Print the resolved manifest JSON."); + Console.WriteLine(" descriptor Emit OCI descriptor + provenance placeholder for the provided SBOM."); + Console.WriteLine(" version Print the plug-in version."); + Console.WriteLine(); + Console.WriteLine("Options:"); + Console.WriteLine(" --manifest Override the manifest directory."); + Console.WriteLine(" --cas Override the CAS root directory."); + Console.WriteLine(" --image (descriptor) Image digest the SBOM belongs to."); + Console.WriteLine(" --sbom (descriptor) Path to the SBOM file to describe."); + Console.WriteLine(" --attestor (descriptor) Optional Attestor endpoint for provenance placeholders."); + Console.WriteLine(" --attestor-token Bearer token for Attestor requests (or STELLAOPS_ATTESTOR_TOKEN)."); + Console.WriteLine(" --attestor-insecure Skip TLS verification for Attestor requests (dev/test only)."); + return 0; + } + + private static int UnknownCommand(string command) + { + Console.Error.WriteLine($"Unknown command '{command}'. Use 'help' for usage."); + return 1; + } + + private static string ResolveManifestDirectory(string[] args) + { + var explicitPath = GetOption(args, "--manifest") + ?? Environment.GetEnvironmentVariable("STELLAOPS_BUILDX_MANIFEST_DIR"); + + if (!string.IsNullOrWhiteSpace(explicitPath)) + { + return Path.GetFullPath(explicitPath); + } + + var defaultDirectory = Path.Combine(AppContext.BaseDirectory, "plugins", "scanner", "buildx"); + if (Directory.Exists(defaultDirectory)) + { + return defaultDirectory; + } + + return AppContext.BaseDirectory; + } + + private static string ResolveCasRoot(string[] args, BuildxPluginManifest manifest) + { + var overrideValue = GetOption(args, "--cas") + ?? Environment.GetEnvironmentVariable("STELLAOPS_SCANNER_CAS_ROOT"); + + if (!string.IsNullOrWhiteSpace(overrideValue)) + { + return Path.GetFullPath(overrideValue); + } + + var manifestDefault = manifest.Cas.DefaultRoot; + if (!string.IsNullOrWhiteSpace(manifestDefault)) + { + if (Path.IsPathRooted(manifestDefault)) + { + return Path.GetFullPath(manifestDefault); + } + + var baseDirectory = manifest.SourceDirectory ?? AppContext.BaseDirectory; + return Path.GetFullPath(Path.Combine(baseDirectory, manifestDefault)); + } + + return Path.Combine(AppContext.BaseDirectory, "cas"); + } + + private static async Task RunDescriptorAsync(string[] args, CancellationToken cancellationToken) + { + var imageDigest = RequireOption(args, "--image"); + var sbomPath = RequireOption(args, "--sbom"); + + var sbomMediaType = GetOption(args, "--media-type") ?? "application/vnd.cyclonedx+json"; + var sbomFormat = GetOption(args, "--sbom-format") ?? "cyclonedx-json"; + var sbomKind = GetOption(args, "--sbom-kind") ?? "inventory"; + var artifactType = GetOption(args, "--artifact-type") ?? "application/vnd.stellaops.sbom.layer+json"; + var subjectMediaType = GetOption(args, "--subject-media-type") ?? "application/vnd.oci.image.manifest.v1+json"; + var predicateType = GetOption(args, "--predicate-type") ?? "https://slsa.dev/provenance/v1"; + var licenseId = GetOption(args, "--license-id") ?? Environment.GetEnvironmentVariable("STELLAOPS_LICENSE_ID"); + var repository = GetOption(args, "--repository"); + var buildRef = GetOption(args, "--build-ref"); + var sbomName = GetOption(args, "--sbom-name") ?? Path.GetFileName(sbomPath); + + var attestorUriText = GetOption(args, "--attestor") ?? Environment.GetEnvironmentVariable("STELLAOPS_ATTESTOR_URL"); + var attestorToken = GetOption(args, "--attestor-token") ?? Environment.GetEnvironmentVariable("STELLAOPS_ATTESTOR_TOKEN"); + var attestorInsecure = GetFlag(args, "--attestor-insecure") + || string.Equals(Environment.GetEnvironmentVariable("STELLAOPS_ATTESTOR_INSECURE"), "true", StringComparison.OrdinalIgnoreCase); + Uri? attestorUri = null; + if (!string.IsNullOrWhiteSpace(attestorUriText)) + { + attestorUri = new Uri(attestorUriText, UriKind.Absolute); + } + + var assembly = Assembly.GetExecutingAssembly(); + var version = assembly.GetCustomAttribute()?.InformationalVersion + ?? assembly.GetName().Version?.ToString() + ?? "0.0.0"; + + var request = new DescriptorRequest + { + ImageDigest = imageDigest, + SbomPath = sbomPath, + SbomMediaType = sbomMediaType, + SbomFormat = sbomFormat, + SbomKind = sbomKind, + SbomArtifactType = artifactType, + SubjectMediaType = subjectMediaType, + PredicateType = predicateType, + GeneratorVersion = version, + GeneratorName = assembly.GetName().Name, + LicenseId = licenseId, + SbomName = sbomName, + Repository = repository, + BuildRef = buildRef, + AttestorUri = attestorUri?.ToString() + }.Validate(); + + var generator = new DescriptorGenerator(TimeProvider.System); + var document = await generator.CreateAsync(request, cancellationToken).ConfigureAwait(false); + + if (attestorUri is not null) + { + using var httpClient = CreateAttestorHttpClient(attestorUri, attestorToken, attestorInsecure); + var attestorClient = new AttestorClient(httpClient); + await attestorClient.SendPlaceholderAsync(attestorUri, document, cancellationToken).ConfigureAwait(false); + } + + var json = JsonSerializer.Serialize(document, DescriptorJsonOptions); + Console.WriteLine(json); + return 0; + } + + private static string? GetOption(string[] args, string optionName) + { + for (var i = 0; i < args.Length; i++) + { + var argument = args[i]; + if (string.Equals(argument, optionName, StringComparison.OrdinalIgnoreCase)) + { + if (i + 1 >= args.Length) + { + throw new BuildxPluginException($"Option '{optionName}' requires a value."); + } + + return args[i + 1]; + } + + if (argument.StartsWith(optionName + "=", StringComparison.OrdinalIgnoreCase)) + { + return argument[(optionName.Length + 1)..]; + } + } + + return null; + } + + private static bool GetFlag(string[] args, string optionName) + { + foreach (var argument in args) + { + if (string.Equals(argument, optionName, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + + private static string RequireOption(string[] args, string optionName) + { + var value = GetOption(args, optionName); + if (string.IsNullOrWhiteSpace(value)) + { + throw new BuildxPluginException($"Option '{optionName}' is required."); + } + + return value; + } + + private static HttpClient CreateAttestorHttpClient(Uri attestorUri, string? bearerToken, bool insecure) + { + var handler = new HttpClientHandler + { + CheckCertificateRevocationList = true, + }; + + if (insecure && string.Equals(attestorUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { +#pragma warning disable S4830 // Explicitly gated by --attestor-insecure flag/env for dev/test usage. + handler.ServerCertificateCustomValidationCallback = (_, _, _, _) => true; +#pragma warning restore S4830 + } + + var client = new HttpClient(handler, disposeHandler: true) + { + Timeout = TimeSpan.FromSeconds(30) + }; + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + if (!string.IsNullOrWhiteSpace(bearerToken)) + { + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); + } + + return client; + } +} diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj new file mode 100644 index 00000000..b5c66a0d --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/StellaOps.Scanner.Sbomer.BuildXPlugin.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + Exe + StellaOps.Scanner.Sbomer.BuildXPlugin + StellaOps.Scanner.Sbomer.BuildXPlugin + 0.1.0-alpha + 0.1.0.0 + 0.1.0.0 + 0.1.0-alpha + + + + + PreserveNewest + + + diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md new file mode 100644 index 00000000..5d93f607 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/TASKS.md @@ -0,0 +1,9 @@ +# BuildX Plugin Task Board (Sprint 9) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SP9-BLDX-09-001 | DONE | BuildX Guild | SCANNER-EMIT-10-601 (awareness) | Scaffold buildx driver, manifest, local CAS handshake; ensure plugin loads from `plugins/scanner/buildx/`. | Plugin manifest + loader tests; local CAS writes succeed; restart required to activate. | +| SP9-BLDX-09-002 | DONE | BuildX Guild | SP9-BLDX-09-001 | Emit OCI annotations + provenance metadata for Attestor handoff (image + SBOM). | OCI descriptors include DSSE/provenance placeholders; Attestor mock accepts payload. | +| SP9-BLDX-09-003 | DONE | BuildX Guild | SP9-BLDX-09-002 | CI demo pipeline: build sample image, produce SBOM, verify backend report wiring. | GitHub/CI job runs sample build within 5 s overhead; artifacts saved; documentation updated. | +| SP9-BLDX-09-004 | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-002 | Stabilize descriptor nonce derivation so repeated builds emit deterministic placeholders. | Repeated descriptor runs with fixed inputs yield identical JSON; regression tests cover nonce determinism. | +| SP9-BLDX-09-005 | DONE (2025-10-19) | BuildX Guild | SP9-BLDX-09-004 | Integrate determinism check in GitHub/Gitea workflows and capture sample artifacts. | Determinism step runs in `.gitea/workflows/build-test-deploy.yml` and `samples/ci/buildx-demo`, producing matching descriptors + archived artifacts. | diff --git a/src/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json new file mode 100644 index 00000000..e3ceac65 --- /dev/null +++ b/src/StellaOps.Scanner.Sbomer.BuildXPlugin/stellaops.sbom-indexer.manifest.json @@ -0,0 +1,35 @@ +{ + "schemaVersion": "1.0", + "id": "stellaops.sbom-indexer", + "displayName": "StellaOps SBOM BuildX Generator", + "version": "0.1.0-alpha", + "requiresRestart": true, + "entryPoint": { + "type": "dotnet", + "executable": "StellaOps.Scanner.Sbomer.BuildXPlugin.dll", + "arguments": [ + "handshake" + ] + }, + "capabilities": [ + "generator", + "sbom" + ], + "cas": { + "protocol": "filesystem", + "defaultRoot": "cas", + "compression": "zstd" + }, + "image": { + "name": "stellaops/sbom-indexer", + "digest": null, + "platforms": [ + "linux/amd64", + "linux/arm64" + ] + }, + "metadata": { + "org.stellaops.plugin.kind": "buildx-generator", + "org.stellaops.restart.required": "true" + } +} diff --git a/src/StellaOps.Scanner.Storage.Tests/InMemoryArtifactObjectStore.cs b/src/StellaOps.Scanner.Storage.Tests/InMemoryArtifactObjectStore.cs new file mode 100644 index 00000000..33177ec0 --- /dev/null +++ b/src/StellaOps.Scanner.Storage.Tests/InMemoryArtifactObjectStore.cs @@ -0,0 +1,34 @@ +using System.Collections.Concurrent; +using StellaOps.Scanner.Storage.ObjectStore; + +namespace StellaOps.Scanner.Storage.Tests; + +internal sealed class InMemoryArtifactObjectStore : IArtifactObjectStore +{ + private readonly ConcurrentDictionary<(string Bucket, string Key), byte[]> _objects = new(); + + public IReadOnlyDictionary<(string Bucket, string Key), byte[]> Objects => _objects; + + public Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + _objects.TryRemove((descriptor.Bucket, descriptor.Key), out _); + return Task.CompletedTask; + } + + public Task GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + if (_objects.TryGetValue((descriptor.Bucket, descriptor.Key), out var bytes)) + { + return Task.FromResult(new MemoryStream(bytes, writable: false)); + } + + return Task.FromResult(null); + } + + public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken) + { + using var buffer = new MemoryStream(); + await content.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + _objects[(descriptor.Bucket, descriptor.Key)] = buffer.ToArray(); + } +} diff --git a/src/StellaOps.Scanner.Storage.Tests/ScannerMongoFixture.cs b/src/StellaOps.Scanner.Storage.Tests/ScannerMongoFixture.cs new file mode 100644 index 00000000..0efefdc1 --- /dev/null +++ b/src/StellaOps.Scanner.Storage.Tests/ScannerMongoFixture.cs @@ -0,0 +1,26 @@ +using Mongo2Go; +using MongoDB.Driver; +using Xunit; + +namespace StellaOps.Scanner.Storage.Tests; + +public sealed class ScannerMongoFixture : IAsyncLifetime +{ + public MongoDbRunner Runner { get; private set; } = null!; + public IMongoClient Client { get; private set; } = null!; + public IMongoDatabase Database { get; private set; } = null!; + + public Task InitializeAsync() + { + Runner = MongoDbRunner.Start(singleNodeReplSet: true); + Client = new MongoClient(Runner.ConnectionString); + Database = Client.GetDatabase($"scanner-tests-{Guid.NewGuid():N}"); + return Task.CompletedTask; + } + + public Task DisposeAsync() + { + Runner.Dispose(); + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scanner.Storage.Tests/StellaOps.Scanner.Storage.Tests.csproj b/src/StellaOps.Scanner.Storage.Tests/StellaOps.Scanner.Storage.Tests.csproj new file mode 100644 index 00000000..2b133e25 --- /dev/null +++ b/src/StellaOps.Scanner.Storage.Tests/StellaOps.Scanner.Storage.Tests.csproj @@ -0,0 +1,10 @@ + + + net10.0 + enable + enable + + + + + diff --git a/src/StellaOps.Scanner.Storage.Tests/StorageDualWriteFixture.cs b/src/StellaOps.Scanner.Storage.Tests/StorageDualWriteFixture.cs new file mode 100644 index 00000000..b93a99db --- /dev/null +++ b/src/StellaOps.Scanner.Storage.Tests/StorageDualWriteFixture.cs @@ -0,0 +1,149 @@ +using System.Security.Cryptography; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scanner.Storage; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Migrations; +using StellaOps.Scanner.Storage.Mongo; +using StellaOps.Scanner.Storage.ObjectStore; +using StellaOps.Scanner.Storage.Repositories; +using StellaOps.Scanner.Storage.Services; +using Xunit; +using Microsoft.Extensions.Time.Testing; + +namespace StellaOps.Scanner.Storage.Tests; + +[CollectionDefinition("scanner-mongo-fixture")] +public sealed class ScannerMongoCollection : ICollectionFixture +{ +} + +[Collection("scanner-mongo-fixture")] +public sealed class StorageDualWriteFixture +{ + private readonly ScannerMongoFixture _fixture; + + public StorageDualWriteFixture(ScannerMongoFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task StoreArtifactAsync_DualWrite_WritesToMirrorAndCatalog() + { + var options = BuildOptions(dualWrite: true, mirrorBucket: "mirror-bucket"); + var objectStore = new InMemoryArtifactObjectStore(); + var fakeTime = new FakeTimeProvider(new DateTimeOffset(2025, 10, 19, 12, 0, 0, TimeSpan.Zero)); + + await InitializeMongoAsync(options); + var provider = new MongoCollectionProvider(_fixture.Database, Options.Create(options)); + var artifactRepository = new ArtifactRepository(provider, fakeTime); + var lifecycleRepository = new LifecycleRuleRepository(provider, fakeTime); + var service = new ArtifactStorageService( + artifactRepository, + lifecycleRepository, + objectStore, + Options.Create(options), + NullLogger.Instance, + fakeTime); + + var bytes = System.Text.Encoding.UTF8.GetBytes("test artifact payload"); + using var stream = new MemoryStream(bytes); + var expiresAt = DateTime.UtcNow.AddHours(6); + var expectedTimestamp = fakeTime.GetUtcNow().UtcDateTime; + + var document = await service.StoreArtifactAsync( + ArtifactDocumentType.LayerBom, + ArtifactDocumentFormat.CycloneDxJson, + mediaType: "application/vnd.cyclonedx+json", + content: stream, + immutable: true, + ttlClass: "compliance", + expiresAtUtc: expiresAt, + cancellationToken: CancellationToken.None); + + var digest = Convert.ToHexString(SHA256.HashData(bytes)).ToLowerInvariant(); + var expectedKey = $"{options.ObjectStore.RootPrefix.TrimEnd('/')}/layers/{digest}/sbom.cdx.json"; + Assert.Contains(objectStore.Objects.Keys, key => key.Bucket == options.ObjectStore.BucketName && key.Key == expectedKey); + Assert.Contains(objectStore.Objects.Keys, key => key.Bucket == options.DualWrite.MirrorBucket && key.Key == expectedKey); + + var artifact = await artifactRepository.GetAsync(document.Id, CancellationToken.None); + Assert.NotNull(artifact); + Assert.Equal($"sha256:{digest}", artifact!.BytesSha256); + Assert.Equal(1, artifact.RefCount); + Assert.Equal("compliance", artifact.TtlClass); + Assert.True(artifact.Immutable); + Assert.Equal(expectedTimestamp, artifact.CreatedAtUtc); + Assert.Equal(expectedTimestamp, artifact.UpdatedAtUtc); + + var lifecycleCollection = _fixture.Database.GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + var lifecycle = await lifecycleCollection.Find(x => x.ArtifactId == document.Id).FirstOrDefaultAsync(); + Assert.NotNull(lifecycle); + Assert.Equal("compliance", lifecycle!.Class); + Assert.True(lifecycle.ExpiresAtUtc.HasValue); + Assert.True(lifecycle.ExpiresAtUtc.Value <= expiresAt.AddSeconds(5)); + Assert.Equal(expectedTimestamp, lifecycle.CreatedAtUtc); + } + + [Fact] + public async Task Bootstrapper_CreatesLifecycleTtlIndex() + { + var options = BuildOptions(dualWrite: false, mirrorBucket: null); + await InitializeMongoAsync(options); + + var collection = _fixture.Database.GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + var cursor = await collection.Indexes.ListAsync(); + var indexes = await cursor.ToListAsync(); + var ttlIndex = indexes.SingleOrDefault(x => string.Equals(x["name"].AsString, "lifecycle_expiresAt", StringComparison.Ordinal)); + + Assert.NotNull(ttlIndex); + Assert.True(ttlIndex!.TryGetValue("expireAfterSeconds", out var expireValue)); + Assert.Equal(0, expireValue.ToInt64()); + + var uniqueIndex = indexes.SingleOrDefault(x => string.Equals(x["name"].AsString, "lifecycle_artifact_class", StringComparison.Ordinal)); + Assert.NotNull(uniqueIndex); + Assert.True(uniqueIndex!["unique"].AsBoolean); + } + + private ScannerStorageOptions BuildOptions(bool dualWrite, string? mirrorBucket) + { + var options = new ScannerStorageOptions + { + Mongo = + { + ConnectionString = _fixture.Runner.ConnectionString, + DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName, + }, + ObjectStore = + { + BucketName = "primary-bucket", + RootPrefix = "scanner", + EnableObjectLock = true, + }, + }; + + options.DualWrite.Enabled = dualWrite; + options.DualWrite.MirrorBucket = mirrorBucket; + return options; + } + + private async Task InitializeMongoAsync(ScannerStorageOptions options) + { + await _fixture.Client.DropDatabaseAsync(options.Mongo.DatabaseName); + var migrations = new IMongoMigration[] { new EnsureLifecycleRuleTtlMigration() }; + var runner = new MongoMigrationRunner( + _fixture.Database, + migrations, + NullLogger.Instance, + TimeProvider.System); + var bootstrapper = new MongoBootstrapper( + _fixture.Database, + Options.Create(options), + NullLogger.Instance, + runner); + + await bootstrapper.InitializeAsync(CancellationToken.None); + } +} diff --git a/src/StellaOps.Scanner.Storage/AGENTS.md b/src/StellaOps.Scanner.Storage/AGENTS.md new file mode 100644 index 00000000..76bc40e5 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS +## Role +Provide durable catalog and artifact storage for the Scanner plane, spanning Mongo catalog collections and MinIO object storage. Expose repositories and services used by WebService and Worker components to persist job state, image metadata, and exported artefacts deterministically. +## Scope +- Mongo collections: artifacts, images, layers, links, jobs, lifecycle_rules, migrations. +- Metadata documents: enforce majority write/read concerns, UTC timestamps, deterministic identifiers (SHA-256 digests, ULIDs for jobs). +- Bootstrapper: create collections + indexes (unique digests, compound references, TTL on lifecycle rules, sparse lookup helpers) and run schema migrations. +- Object storage (MinIO/S3): manage bucket layout (layers/, images/, indexes/, attest/), immutability policies, deterministic paths, and retention classes. +- Services: coordinate dual-write between Mongo metadata and MinIO blobs, compute digests, manage reference counts, and expose typed repositories for WebService/Worker interactions. +## Participants +- Scanner.WebService binds configuration, runs bootstrapper during startup, and uses repositories to enqueue scans, look up catalog entries, and manage lifecycle policies. +- Scanner.Worker writes job progress, uploads SBOM artefacts, and updates artefact reference counts. +- Policy / Notify consumers resolve artefact metadata for reports via catalog APIs once exposed. +## Interfaces & contracts +- Options configured via `ScannerStorageOptions` (Mongo + object store). `EnsureValid` rejects incomplete/unsafe configuration. +- Mongo access uses `IMongoDatabase` scoped with majority `ReadConcern`/`WriteConcern` and cancellation tokens. +- Object store abstraction (`IArtifactObjectStore`) encapsulates MinIO (S3) operations with server-side checksum validation and optional object-lock retain-until. +- Service APIs follow deterministic naming: digests normalized (`sha256:`), ULIDs sortable, timestamps ISO-8601 UTC. +## In/Out of scope +In: persistence models, bootstrap/migrations, catalog repositories, object storage client, retention helpers, dual-write coordination, deterministic digests. +Out: HTTP endpoints, queue processing, analyzer logic, SBOM composition, policy decisions, UI contracts. +## Observability & security expectations +- Emit structured logs for catalog/object-store writes including correlation IDs and digests. +- Guard against double writes; idempotent operations keyed by digests. +- Do not log credentials; redact connection strings. Honour cancellation tokens. +- Metrics hooks (pending) must expose duration counters for Mongo and MinIO operations. +## Tests +- Integration tests with ephemeral Mongo/MinIO stubs covering bootstrapper indexes, TTL enforcement, dual-write coordination, digest determinism, and majority read/write concerns. diff --git a/src/StellaOps.Scanner.Storage/Catalog/ArtifactDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/ArtifactDocument.cs new file mode 100644 index 00000000..cbe44e3d --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/ArtifactDocument.cs @@ -0,0 +1,85 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +public enum ArtifactDocumentType +{ + LayerBom, + ImageBom, + Diff, + Index, + Attestation, +} + +public enum ArtifactDocumentFormat +{ + CycloneDxJson, + CycloneDxProtobuf, + SpdxJson, + BomIndex, + DsseJson, +} + +[BsonIgnoreExtraElements] +public sealed class ArtifactDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("type")] + public ArtifactDocumentType Type { get; set; } + = ArtifactDocumentType.ImageBom; + + [BsonElement("format")] + public ArtifactDocumentFormat Format { get; set; } + = ArtifactDocumentFormat.CycloneDxJson; + + [BsonElement("mediaType")] + public string MediaType { get; set; } = string.Empty; + + [BsonElement("bytesSha256")] + public string BytesSha256 { get; set; } = string.Empty; + + [BsonElement("sizeBytes")] + public long SizeBytes { get; set; } + = 0; + + [BsonElement("immutable")] + public bool Immutable { get; set; } + = false; + + [BsonElement("refCount")] + public long RefCount { get; set; } + = 0; + + [BsonElement("rekor")] + [BsonIgnoreIfNull] + public RekorReference? Rekor { get; set; } + = null; + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; + + [BsonElement("updatedAt")] + public DateTime UpdatedAtUtc { get; set; } + = DateTime.UtcNow; + + [BsonElement("ttlClass")] + public string TtlClass { get; set; } = "default"; +} + +public sealed class RekorReference +{ + [BsonElement("uuid")] + public string? Uuid { get; set; } + = null; + + [BsonElement("index")] + public long? Index { get; set; } + = null; + + [BsonElement("url")] + public string? Url { get; set; } + = null; +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/CatalogIdFactory.cs b/src/StellaOps.Scanner.Storage/Catalog/CatalogIdFactory.cs new file mode 100644 index 00000000..53ea538b --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/CatalogIdFactory.cs @@ -0,0 +1,43 @@ +using System.Security.Cryptography; +using System.Text; + +namespace StellaOps.Scanner.Storage.Catalog; + +public static class CatalogIdFactory +{ + public static string CreateArtifactId(ArtifactDocumentType type, string digest) + { + ArgumentException.ThrowIfNullOrWhiteSpace(digest); + return $"{type.ToString().ToLowerInvariant()}::{NormalizeDigest(digest)}"; + } + + public static string CreateLinkId(LinkSourceType type, string fromDigest, string artifactId) + { + ArgumentException.ThrowIfNullOrWhiteSpace(fromDigest); + ArgumentException.ThrowIfNullOrWhiteSpace(artifactId); + + var input = Encoding.UTF8.GetBytes($"{type}:{NormalizeDigest(fromDigest)}:{artifactId}"); + var hash = SHA256.HashData(input); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + public static string CreateLifecycleRuleId(string artifactId, string @class) + { + ArgumentException.ThrowIfNullOrWhiteSpace(artifactId); + var normalizedClass = string.IsNullOrWhiteSpace(@class) ? "default" : @class.Trim().ToLowerInvariant(); + var payload = Encoding.UTF8.GetBytes($"{artifactId}:{normalizedClass}"); + var hash = SHA256.HashData(payload); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static string NormalizeDigest(string digest) + { + if (!digest.Contains(':', StringComparison.Ordinal)) + { + return $"sha256:{digest.Trim().ToLowerInvariant()}"; + } + + var parts = digest.Split(':', 2, StringSplitOptions.TrimEntries); + return $"{parts[0].ToLowerInvariant()}:{parts[1].ToLowerInvariant()}"; + } +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/ImageDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/ImageDocument.cs new file mode 100644 index 00000000..6d3971ab --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/ImageDocument.cs @@ -0,0 +1,29 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +[BsonIgnoreExtraElements] +public sealed class ImageDocument +{ + [BsonId] + public string ImageDigest { get; set; } = string.Empty; + + [BsonElement("repository")] + public string Repository { get; set; } = string.Empty; + + [BsonElement("tag")] + [BsonIgnoreIfNull] + public string? Tag { get; set; } + = null; + + [BsonElement("architecture")] + public string Architecture { get; set; } = string.Empty; + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; + + [BsonElement("lastSeenAt")] + public DateTime LastSeenAtUtc { get; set; } + = DateTime.UtcNow; +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/JobDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/JobDocument.cs new file mode 100644 index 00000000..e4c4e675 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/JobDocument.cs @@ -0,0 +1,54 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +public enum JobState +{ + Pending, + Running, + Succeeded, + Failed, + Cancelled, +} + +[BsonIgnoreExtraElements] +public sealed class JobDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("kind")] + public string Kind { get; set; } = string.Empty; + + [BsonElement("state")] + public JobState State { get; set; } = JobState.Pending; + + [BsonElement("args")] + public BsonDocument Arguments { get; set; } + = new(); + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; + + [BsonElement("startedAt")] + [BsonIgnoreIfNull] + public DateTime? StartedAtUtc { get; set; } + = null; + + [BsonElement("completedAt")] + [BsonIgnoreIfNull] + public DateTime? CompletedAtUtc { get; set; } + = null; + + [BsonElement("heartbeatAt")] + [BsonIgnoreIfNull] + public DateTime? HeartbeatAtUtc { get; set; } + = null; + + [BsonElement("error")] + [BsonIgnoreIfNull] + public string? Error { get; set; } + = null; +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/LayerDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/LayerDocument.cs new file mode 100644 index 00000000..4d30e3c4 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/LayerDocument.cs @@ -0,0 +1,25 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +[BsonIgnoreExtraElements] +public sealed class LayerDocument +{ + [BsonId] + public string LayerDigest { get; set; } = string.Empty; + + [BsonElement("mediaType")] + public string MediaType { get; set; } = string.Empty; + + [BsonElement("sizeBytes")] + public long SizeBytes { get; set; } + = 0; + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; + + [BsonElement("lastSeenAt")] + public DateTime LastSeenAtUtc { get; set; } + = DateTime.UtcNow; +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/LifecycleRuleDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/LifecycleRuleDocument.cs new file mode 100644 index 00000000..f7615d87 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/LifecycleRuleDocument.cs @@ -0,0 +1,25 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +[BsonIgnoreExtraElements] +public sealed class LifecycleRuleDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("artifactId")] + public string ArtifactId { get; set; } = string.Empty; + + [BsonElement("class")] + public string Class { get; set; } = "default"; + + [BsonElement("expiresAt")] + [BsonIgnoreIfNull] + public DateTime? ExpiresAtUtc { get; set; } + = null; + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; +} diff --git a/src/StellaOps.Scanner.Storage/Catalog/LinkDocument.cs b/src/StellaOps.Scanner.Storage/Catalog/LinkDocument.cs new file mode 100644 index 00000000..a9d8b3c5 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Catalog/LinkDocument.cs @@ -0,0 +1,30 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Catalog; + +public enum LinkSourceType +{ + Image, + Layer, +} + +[BsonIgnoreExtraElements] +public sealed class LinkDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("fromType")] + public LinkSourceType FromType { get; set; } + = LinkSourceType.Image; + + [BsonElement("fromDigest")] + public string FromDigest { get; set; } = string.Empty; + + [BsonElement("artifactId")] + public string ArtifactId { get; set; } = string.Empty; + + [BsonElement("createdAt")] + public DateTime CreatedAtUtc { get; set; } + = DateTime.UtcNow; +} diff --git a/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..06e0d823 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Extensions/ServiceCollectionExtensions.cs @@ -0,0 +1,113 @@ +using Amazon; +using Amazon.S3; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Migrations; +using StellaOps.Scanner.Storage.Mongo; +using StellaOps.Scanner.Storage.ObjectStore; +using StellaOps.Scanner.Storage.Repositories; +using StellaOps.Scanner.Storage.Services; + +namespace StellaOps.Scanner.Storage.Extensions; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddScannerStorage(this IServiceCollection services, Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + + services.AddOptions().Configure(configure).PostConfigure(options => options.EnsureValid()); + RegisterScannerStorageServices(services); + return services; + } + + public static IServiceCollection AddScannerStorage(this IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(configuration); + + services.AddOptions() + .Bind(configuration) + .PostConfigure(options => options.EnsureValid()); + + RegisterScannerStorageServices(services); + return services; + } + + private static void RegisterScannerStorageServices(IServiceCollection services) + { + services.TryAddSingleton(TimeProvider.System); + services.TryAddSingleton(CreateMongoClient); + services.TryAddSingleton(CreateMongoDatabase); + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddSingleton(provider => + { + var migrations = provider.GetServices(); + return new MongoMigrationRunner( + provider.GetRequiredService(), + migrations, + provider.GetRequiredService>(), + TimeProvider.System); + }); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.TryAddSingleton(CreateAmazonS3Client); + services.TryAddSingleton(); + services.TryAddSingleton(); + } + + private static IMongoClient CreateMongoClient(IServiceProvider provider) + { + var options = provider.GetRequiredService>().Value; + options.EnsureValid(); + + var settings = MongoClientSettings.FromConnectionString(options.Mongo.ConnectionString); + settings.RetryReads = true; + settings.RetryWrites = true; + settings.DirectConnection = false; + settings.ReadPreference = ReadPreference.PrimaryPreferred; + settings.ServerSelectionTimeout = options.Mongo.CommandTimeout; + settings.ConnectTimeout = options.Mongo.CommandTimeout; + settings.SocketTimeout = options.Mongo.CommandTimeout; + settings.ReadConcern = options.Mongo.UseMajorityReadConcern ? ReadConcern.Majority : ReadConcern.Local; + settings.WriteConcern = options.Mongo.UseMajorityWriteConcern ? WriteConcern.WMajority : WriteConcern.W1; + + return new MongoClient(settings); + } + + private static IMongoDatabase CreateMongoDatabase(IServiceProvider provider) + { + var options = provider.GetRequiredService>().Value; + var client = provider.GetRequiredService(); + var databaseName = options.Mongo.ResolveDatabaseName(); + return client.GetDatabase(databaseName); + } + + private static IAmazonS3 CreateAmazonS3Client(IServiceProvider provider) + { + var options = provider.GetRequiredService>().Value.ObjectStore; + var config = new AmazonS3Config + { + RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region), + ForcePathStyle = options.ForcePathStyle, + }; + + if (!string.IsNullOrWhiteSpace(options.ServiceUrl)) + { + config.ServiceURL = options.ServiceUrl; + } + + return new AmazonS3Client(config); + } +} diff --git a/src/StellaOps.Scanner.Storage/Migrations/EnsureLifecycleRuleTtlMigration.cs b/src/StellaOps.Scanner.Storage/Migrations/EnsureLifecycleRuleTtlMigration.cs new file mode 100644 index 00000000..c2e9b9dc --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Migrations/EnsureLifecycleRuleTtlMigration.cs @@ -0,0 +1,30 @@ +using System.Linq; +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; + +namespace StellaOps.Scanner.Storage.Migrations; + +public sealed class EnsureLifecycleRuleTtlMigration : IMongoMigration +{ + public string Id => "20251018-lifecycle-ttl"; + + public string Description => "Ensure lifecycle_rules expiresAt TTL index exists."; + + public async Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + var collection = database.GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + var indexes = await collection.Indexes.ListAsync(cancellationToken).ConfigureAwait(false); + var existing = await indexes.ToListAsync(cancellationToken).ConfigureAwait(false); + + if (existing.Any(x => string.Equals(x["name"].AsString, "lifecycle_expiresAt", StringComparison.Ordinal))) + { + return; + } + + var model = new CreateIndexModel( + Builders.IndexKeys.Ascending(x => x.ExpiresAtUtc), + new CreateIndexOptions { Name = "lifecycle_expiresAt", ExpireAfter = TimeSpan.Zero }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scanner.Storage/Migrations/IMongoMigration.cs b/src/StellaOps.Scanner.Storage/Migrations/IMongoMigration.cs new file mode 100644 index 00000000..4e44edf7 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Migrations/IMongoMigration.cs @@ -0,0 +1,12 @@ +using MongoDB.Driver; + +namespace StellaOps.Scanner.Storage.Migrations; + +public interface IMongoMigration +{ + string Id { get; } + + string Description { get; } + + Task ApplyAsync(IMongoDatabase database, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationDocument.cs b/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationDocument.cs new file mode 100644 index 00000000..9b24b73a --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationDocument.cs @@ -0,0 +1,19 @@ +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scanner.Storage.Migrations; + +[BsonIgnoreExtraElements] +internal sealed class MongoMigrationDocument +{ + [BsonId] + public string Id { get; set; } = string.Empty; + + [BsonElement("description")] + [BsonIgnoreIfNull] + public string? Description { get; set; } + = null; + + [BsonElement("appliedAt")] + public DateTime AppliedAtUtc { get; set; } + = DateTime.UtcNow; +} diff --git a/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationRunner.cs b/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationRunner.cs new file mode 100644 index 00000000..94f4bfcb --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Migrations/MongoMigrationRunner.cs @@ -0,0 +1,94 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; + +namespace StellaOps.Scanner.Storage.Migrations; + +public sealed class MongoMigrationRunner +{ + private readonly IMongoDatabase _database; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public MongoMigrationRunner( + IMongoDatabase database, + IEnumerable migrations, + ILogger logger, + TimeProvider? timeProvider = null) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? TimeProvider.System; + _migrations = (migrations ?? throw new ArgumentNullException(nameof(migrations))) + .OrderBy(m => m.Id, StringComparer.Ordinal) + .ToArray(); + } + + public async Task RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + await EnsureCollectionExistsAsync(_database, cancellationToken).ConfigureAwait(false); + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Migrations); + var applied = await LoadAppliedMigrationIdsAsync(collection, cancellationToken).ConfigureAwait(false); + + foreach (var migration in _migrations) + { + if (applied.Contains(migration.Id, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Applying scanner Mongo migration {MigrationId}: {Description}", migration.Id, migration.Description); + try + { + await migration.ApplyAsync(_database, cancellationToken).ConfigureAwait(false); + var document = new MongoMigrationDocument + { + Id = migration.Id, + Description = string.IsNullOrWhiteSpace(migration.Description) ? null : migration.Description, + AppliedAtUtc = _timeProvider.GetUtcNow().UtcDateTime, + }; + + await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Scanner Mongo migration {MigrationId} applied", migration.Id); + } + catch (Exception ex) + { + _logger.LogError(ex, "Scanner Mongo migration {MigrationId} failed", migration.Id); + throw; + } + } + } + + private static async Task EnsureCollectionExistsAsync(IMongoDatabase database, CancellationToken cancellationToken) + { + using var cursor = await database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var names = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + if (!names.Contains(ScannerStorageDefaults.Collections.Migrations, StringComparer.Ordinal)) + { + await database.CreateCollectionAsync(ScannerStorageDefaults.Collections.Migrations, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + private static async Task> LoadAppliedMigrationIdsAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + using var cursor = await collection.FindAsync(FilterDefinition.Empty, cancellationToken: cancellationToken).ConfigureAwait(false); + var documents = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + var ids = new HashSet(StringComparer.Ordinal); + foreach (var doc in documents) + { + if (!string.IsNullOrWhiteSpace(doc.Id)) + { + ids.Add(doc.Id); + } + } + + return ids; + } +} diff --git a/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs b/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs new file mode 100644 index 00000000..3470eb6b --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Mongo/MongoBootstrapper.cs @@ -0,0 +1,181 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Migrations; + +namespace StellaOps.Scanner.Storage.Mongo; + +public sealed class MongoBootstrapper +{ + private readonly IMongoDatabase _database; + private readonly ScannerStorageOptions _options; + private readonly ILogger _logger; + private readonly MongoMigrationRunner _migrationRunner; + + public MongoBootstrapper( + IMongoDatabase database, + IOptions options, + ILogger logger, + MongoMigrationRunner migrationRunner) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value; + } + + public async Task InitializeAsync(CancellationToken cancellationToken) + { + _options.EnsureValid(); + + await EnsureCollectionsAsync(cancellationToken).ConfigureAwait(false); + await EnsureIndexesAsync(cancellationToken).ConfigureAwait(false); + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + } + + private async Task EnsureCollectionsAsync(CancellationToken cancellationToken) + { + var targetCollections = new[] + { + ScannerStorageDefaults.Collections.Artifacts, + ScannerStorageDefaults.Collections.Images, + ScannerStorageDefaults.Collections.Layers, + ScannerStorageDefaults.Collections.Links, + ScannerStorageDefaults.Collections.Jobs, + ScannerStorageDefaults.Collections.LifecycleRules, + ScannerStorageDefaults.Collections.Migrations, + }; + + using var cursor = await _database.ListCollectionNamesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var existing = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + foreach (var name in targetCollections) + { + if (existing.Contains(name, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Creating Mongo collection {Collection}", name); + await _database.CreateCollectionAsync(name, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } + + private async Task EnsureIndexesAsync(CancellationToken cancellationToken) + { + await EnsureArtifactIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureImageIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureLayerIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureLinkIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureJobIndexesAsync(cancellationToken).ConfigureAwait(false); + await EnsureLifecycleIndexesAsync(cancellationToken).ConfigureAwait(false); + } + + private Task EnsureArtifactIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Artifacts); + var models = new List> + { + new( + Builders.IndexKeys + .Ascending(x => x.Type) + .Ascending(x => x.BytesSha256), + new CreateIndexOptions { Name = "artifact_type_bytesSha256", Unique = true }), + new( + Builders.IndexKeys.Ascending(x => x.RefCount), + new CreateIndexOptions { Name = "artifact_refCount" }), + new( + Builders.IndexKeys.Ascending(x => x.CreatedAtUtc), + new CreateIndexOptions { Name = "artifact_createdAt" }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } + + private Task EnsureImageIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Images); + var models = new List> + { + new( + Builders.IndexKeys + .Ascending(x => x.Repository) + .Ascending(x => x.Tag), + new CreateIndexOptions { Name = "image_repo_tag" }), + new( + Builders.IndexKeys.Ascending(x => x.LastSeenAtUtc), + new CreateIndexOptions { Name = "image_lastSeen" }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } + + private Task EnsureLayerIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Layers); + var models = new List> + { + new( + Builders.IndexKeys.Ascending(x => x.LastSeenAtUtc), + new CreateIndexOptions { Name = "layer_lastSeen" }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } + + private Task EnsureLinkIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Links); + var models = new List> + { + new( + Builders.IndexKeys + .Ascending(x => x.FromType) + .Ascending(x => x.FromDigest) + .Ascending(x => x.ArtifactId), + new CreateIndexOptions { Name = "link_from_artifact", Unique = true }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } + + private Task EnsureJobIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.Jobs); + var models = new List> + { + new( + Builders.IndexKeys + .Ascending(x => x.State) + .Ascending(x => x.CreatedAtUtc), + new CreateIndexOptions { Name = "job_state_createdAt" }), + new( + Builders.IndexKeys.Ascending(x => x.HeartbeatAtUtc), + new CreateIndexOptions { Name = "job_heartbeat" }) + }; + + return collection.Indexes.CreateManyAsync(models, cancellationToken); + } + + private Task EnsureLifecycleIndexesAsync(CancellationToken cancellationToken) + { + var collection = _database.GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + var expiresIndex = new CreateIndexModel( + Builders.IndexKeys.Ascending(x => x.ExpiresAtUtc), + new CreateIndexOptions + { + Name = "lifecycle_expiresAt", + ExpireAfter = TimeSpan.Zero, + }); + + var artifactIndex = new CreateIndexModel( + Builders.IndexKeys + .Ascending(x => x.ArtifactId) + .Ascending(x => x.Class), + new CreateIndexOptions { Name = "lifecycle_artifact_class", Unique = true }); + + return collection.Indexes.CreateManyAsync(new[] { expiresIndex, artifactIndex }, cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs b/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs new file mode 100644 index 00000000..c87eea5e --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Mongo/MongoCollectionProvider.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; + +namespace StellaOps.Scanner.Storage.Mongo; + +public sealed class MongoCollectionProvider +{ + private readonly IMongoDatabase _database; + private readonly MongoOptions _options; + + public MongoCollectionProvider(IMongoDatabase database, IOptions options) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value.Mongo; + } + + public IMongoCollection Artifacts => GetCollection(ScannerStorageDefaults.Collections.Artifacts); + public IMongoCollection Images => GetCollection(ScannerStorageDefaults.Collections.Images); + public IMongoCollection Layers => GetCollection(ScannerStorageDefaults.Collections.Layers); + public IMongoCollection Links => GetCollection(ScannerStorageDefaults.Collections.Links); + public IMongoCollection Jobs => GetCollection(ScannerStorageDefaults.Collections.Jobs); + public IMongoCollection LifecycleRules => GetCollection(ScannerStorageDefaults.Collections.LifecycleRules); + + private IMongoCollection GetCollection(string name) + { + var database = _database; + + if (_options.UseMajorityReadConcern) + { + database = database.WithReadConcern(ReadConcern.Majority); + } + + if (_options.UseMajorityWriteConcern) + { + database = database.WithWriteConcern(WriteConcern.WMajority); + } + + return database.GetCollection(name); + } +} diff --git a/src/StellaOps.Scanner.Storage/ObjectStore/IArtifactObjectStore.cs b/src/StellaOps.Scanner.Storage/ObjectStore/IArtifactObjectStore.cs new file mode 100644 index 00000000..1df31011 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/ObjectStore/IArtifactObjectStore.cs @@ -0,0 +1,12 @@ +namespace StellaOps.Scanner.Storage.ObjectStore; + +public interface IArtifactObjectStore +{ + Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken); + + Task GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken); + + Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken); +} + +public sealed record ArtifactObjectDescriptor(string Bucket, string Key, bool Immutable, TimeSpan? RetainFor = null); diff --git a/src/StellaOps.Scanner.Storage/ObjectStore/S3ArtifactObjectStore.cs b/src/StellaOps.Scanner.Storage/ObjectStore/S3ArtifactObjectStore.cs new file mode 100644 index 00000000..99fb1b7f --- /dev/null +++ b/src/StellaOps.Scanner.Storage/ObjectStore/S3ArtifactObjectStore.cs @@ -0,0 +1,75 @@ +using Amazon.S3; +using Amazon.S3.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scanner.Storage.ObjectStore; + +public sealed class S3ArtifactObjectStore : IArtifactObjectStore +{ + private readonly IAmazonS3 _s3; + private readonly ObjectStoreOptions _options; + private readonly ILogger _logger; + + public S3ArtifactObjectStore(IAmazonS3 s3, IOptions options, ILogger logger) + { + _s3 = s3 ?? throw new ArgumentNullException(nameof(s3)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value.ObjectStore; + } + + public async Task PutAsync(ArtifactObjectDescriptor descriptor, Stream content, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + ArgumentNullException.ThrowIfNull(content); + + var request = new PutObjectRequest + { + BucketName = descriptor.Bucket, + Key = descriptor.Key, + InputStream = content, + AutoCloseStream = false, + }; + + if (descriptor.Immutable && _options.EnableObjectLock) + { + request.ObjectLockMode = ObjectLockMode.Compliance; + if (descriptor.RetainFor is { } retention && retention > TimeSpan.Zero) + { + request.ObjectLockRetainUntilDate = DateTime.UtcNow + retention; + } + else if (_options.ComplianceRetention is { } defaultRetention && defaultRetention > TimeSpan.Zero) + { + request.ObjectLockRetainUntilDate = DateTime.UtcNow + defaultRetention; + } + } + + await _s3.PutObjectAsync(request, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Uploaded scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key); + } + + public async Task GetAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + try + { + var response = await _s3.GetObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false); + var buffer = new MemoryStream(); + await response.ResponseStream.CopyToAsync(buffer, cancellationToken).ConfigureAwait(false); + buffer.Position = 0; + return buffer; + } + catch (AmazonS3Exception ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) + { + _logger.LogDebug("Scanner object {Bucket}/{Key} not found", descriptor.Bucket, descriptor.Key); + return null; + } + } + + public async Task DeleteAsync(ArtifactObjectDescriptor descriptor, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(descriptor); + await _s3.DeleteObjectAsync(descriptor.Bucket, descriptor.Key, cancellationToken).ConfigureAwait(false); + _logger.LogDebug("Deleted scanner object {Bucket}/{Key}", descriptor.Bucket, descriptor.Key); + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/ArtifactRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/ArtifactRepository.cs new file mode 100644 index 00000000..5600015f --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/ArtifactRepository.cs @@ -0,0 +1,73 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class ArtifactRepository +{ + private readonly MongoCollectionProvider _collections; + private readonly TimeProvider _timeProvider; + + public ArtifactRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task GetAsync(string artifactId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(artifactId); + return await _collections.Artifacts + .Find(x => x.Id == artifactId) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public async Task UpsertAsync(ArtifactDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + var now = _timeProvider.GetUtcNow().UtcDateTime; + document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc; + document.UpdatedAtUtc = now; + var options = new ReplaceOptions { IsUpsert = true }; + await _collections.Artifacts + .ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken) + .ConfigureAwait(false); + } + + public async Task UpdateRekorAsync(string artifactId, RekorReference reference, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(artifactId); + ArgumentNullException.ThrowIfNull(reference); + + var now = _timeProvider.GetUtcNow().UtcDateTime; + var update = Builders.Update + .Set(x => x.Rekor, reference) + .Set(x => x.UpdatedAtUtc, now); + + await _collections.Artifacts.UpdateOneAsync(x => x.Id == artifactId, update, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + public async Task IncrementRefCountAsync(string artifactId, long delta, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(artifactId); + + var now = _timeProvider.GetUtcNow().UtcDateTime; + var update = Builders.Update + .Inc(x => x.RefCount, delta) + .Set(x => x.UpdatedAtUtc, now); + + var options = new FindOneAndUpdateOptions + { + ReturnDocument = ReturnDocument.After, + IsUpsert = false, + }; + + var result = await _collections.Artifacts + .FindOneAndUpdateAsync(x => x.Id == artifactId, update, options, cancellationToken) + .ConfigureAwait(false); + + return result?.RefCount ?? 0; + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/ImageRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/ImageRepository.cs new file mode 100644 index 00000000..d271f03f --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/ImageRepository.cs @@ -0,0 +1,36 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class ImageRepository +{ + private readonly MongoCollectionProvider _collections; + private readonly TimeProvider _timeProvider; + + public ImageRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task UpsertAsync(ImageDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + document.LastSeenAtUtc = _timeProvider.GetUtcNow().UtcDateTime; + var updateOptions = new ReplaceOptions { IsUpsert = true }; + await _collections.Images + .ReplaceOneAsync(x => x.ImageDigest == document.ImageDigest, document, updateOptions, cancellationToken) + .ConfigureAwait(false); + } + + public async Task GetAsync(string imageDigest, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(imageDigest); + return await _collections.Images + .Find(x => x.ImageDigest == imageDigest) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/JobRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/JobRepository.cs new file mode 100644 index 00000000..667caf57 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/JobRepository.cs @@ -0,0 +1,78 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class JobRepository +{ + private readonly MongoCollectionProvider _collections; + private readonly TimeProvider _timeProvider; + + public JobRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task InsertAsync(JobDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + document.CreatedAtUtc = _timeProvider.GetUtcNow().UtcDateTime; + await _collections.Jobs.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false); + return document; + } + + public async Task TryTransitionAsync(string jobId, JobState expected, JobState next, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jobId); + var now = _timeProvider.GetUtcNow().UtcDateTime; + var update = Builders.Update + .Set(x => x.State, next) + .Set(x => x.HeartbeatAtUtc, now); + + if (next == JobState.Running) + { + update = update.Set(x => x.StartedAtUtc, now); + } + + if (next is JobState.Succeeded or JobState.Failed or JobState.Cancelled) + { + update = update.Set(x => x.CompletedAtUtc, now); + } + + var result = await _collections.Jobs.UpdateOneAsync( + Builders.Filter.And( + Builders.Filter.Eq(x => x.Id, jobId), + Builders.Filter.Eq(x => x.State, expected)), + update, + cancellationToken: cancellationToken).ConfigureAwait(false); + + return result.ModifiedCount == 1; + } + + public async Task GetAsync(string jobId, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(jobId); + return await _collections.Jobs + .Find(x => x.Id == jobId) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } + + public Task> ListStaleAsync(TimeSpan heartbeatThreshold, CancellationToken cancellationToken) + { + if (heartbeatThreshold <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(heartbeatThreshold)); + } + + var cutoff = _timeProvider.GetUtcNow().UtcDateTime - heartbeatThreshold; + var filter = Builders.Filter.And( + Builders.Filter.Eq(x => x.State, JobState.Running), + Builders.Filter.Lt(x => x.HeartbeatAtUtc, cutoff)); + + return _collections.Jobs.Find(filter).ToListAsync(cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/LayerRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/LayerRepository.cs new file mode 100644 index 00000000..389de79a --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/LayerRepository.cs @@ -0,0 +1,36 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class LayerRepository +{ + private readonly MongoCollectionProvider _collections; + private readonly TimeProvider _timeProvider; + + public LayerRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task UpsertAsync(LayerDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + document.LastSeenAtUtc = _timeProvider.GetUtcNow().UtcDateTime; + var options = new ReplaceOptions { IsUpsert = true }; + await _collections.Layers + .ReplaceOneAsync(x => x.LayerDigest == document.LayerDigest, document, options, cancellationToken) + .ConfigureAwait(false); + } + + public async Task GetAsync(string layerDigest, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(layerDigest); + return await _collections.Layers + .Find(x => x.LayerDigest == layerDigest) + .FirstOrDefaultAsync(cancellationToken) + .ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/LifecycleRuleRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/LifecycleRuleRepository.cs new file mode 100644 index 00000000..5363e629 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/LifecycleRuleRepository.cs @@ -0,0 +1,36 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class LifecycleRuleRepository +{ + private readonly MongoCollectionProvider _collections; + private readonly TimeProvider _timeProvider; + + public LifecycleRuleRepository(MongoCollectionProvider collections, TimeProvider? timeProvider = null) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task UpsertAsync(LifecycleRuleDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + var now = _timeProvider.GetUtcNow().UtcDateTime; + document.CreatedAtUtc = document.CreatedAtUtc == default ? now : document.CreatedAtUtc; + var options = new ReplaceOptions { IsUpsert = true }; + await _collections.LifecycleRules + .ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken) + .ConfigureAwait(false); + } + + public Task> ListExpiredAsync(DateTime utcNow, CancellationToken cancellationToken) + { + var filter = Builders.Filter.Lt(x => x.ExpiresAtUtc, utcNow); + return _collections.LifecycleRules + .Find(filter) + .ToListAsync(cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Storage/Repositories/LinkRepository.cs b/src/StellaOps.Scanner.Storage/Repositories/LinkRepository.cs new file mode 100644 index 00000000..0c823c64 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Repositories/LinkRepository.cs @@ -0,0 +1,32 @@ +using MongoDB.Driver; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.Mongo; + +namespace StellaOps.Scanner.Storage.Repositories; + +public sealed class LinkRepository +{ + private readonly MongoCollectionProvider _collections; + + public LinkRepository(MongoCollectionProvider collections) + { + _collections = collections ?? throw new ArgumentNullException(nameof(collections)); + } + + public async Task UpsertAsync(LinkDocument document, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(document); + var options = new ReplaceOptions { IsUpsert = true }; + await _collections.Links + .ReplaceOneAsync(x => x.Id == document.Id, document, options, cancellationToken) + .ConfigureAwait(false); + } + + public Task> ListBySourceAsync(LinkSourceType type, string digest, CancellationToken cancellationToken) + { + ArgumentException.ThrowIfNullOrWhiteSpace(digest); + return _collections.Links + .Find(x => x.FromType == type && x.FromDigest == digest) + .ToListAsync(cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs new file mode 100644 index 00000000..c52950de --- /dev/null +++ b/src/StellaOps.Scanner.Storage/ScannerStorageDefaults.cs @@ -0,0 +1,27 @@ +namespace StellaOps.Scanner.Storage; + +public static class ScannerStorageDefaults +{ + public const string DefaultDatabaseName = "scanner"; + public const string DefaultBucketName = "stellaops"; + public const string DefaultRootPrefix = "scanner"; + + public static class Collections + { + public const string Artifacts = "artifacts"; + public const string Images = "images"; + public const string Layers = "layers"; + public const string Links = "links"; + public const string Jobs = "jobs"; + public const string LifecycleRules = "lifecycle_rules"; + public const string Migrations = "schema_migrations"; + } + + public static class ObjectPrefixes + { + public const string Layers = "layers"; + public const string Images = "images"; + public const string Indexes = "indexes"; + public const string Attestations = "attest"; + } +} diff --git a/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs b/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs new file mode 100644 index 00000000..374acec1 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/ScannerStorageOptions.cs @@ -0,0 +1,124 @@ +using MongoDB.Driver; + +namespace StellaOps.Scanner.Storage; + +public sealed class ScannerStorageOptions +{ + public MongoOptions Mongo { get; set; } = new(); + + public ObjectStoreOptions ObjectStore { get; set; } = new(); + + public DualWriteOptions DualWrite { get; set; } = new(); + + public void EnsureValid() + { + Mongo.EnsureValid(); + ObjectStore.EnsureValid(); + DualWrite.EnsureValid(); + } +} + +public sealed class MongoOptions +{ + public string ConnectionString { get; set; } = string.Empty; + + public string? DatabaseName { get; set; } + = null; + + public TimeSpan CommandTimeout { get; set; } + = TimeSpan.FromSeconds(30); + + public bool UseMajorityReadConcern { get; set; } + = true; + + public bool UseMajorityWriteConcern { get; set; } + = true; + + public string ResolveDatabaseName() + { + if (!string.IsNullOrWhiteSpace(DatabaseName)) + { + return DatabaseName.Trim(); + } + + if (!string.IsNullOrWhiteSpace(ConnectionString)) + { + var url = MongoUrl.Create(ConnectionString); + if (!string.IsNullOrWhiteSpace(url.DatabaseName)) + { + return url.DatabaseName; + } + } + + return ScannerStorageDefaults.DefaultDatabaseName; + } + + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + { + throw new InvalidOperationException("Scanner storage Mongo connection string is not configured."); + } + + if (CommandTimeout <= TimeSpan.Zero) + { + throw new InvalidOperationException("Scanner storage Mongo command timeout must be positive."); + } + + _ = ResolveDatabaseName(); + } +} + +public sealed class ObjectStoreOptions +{ + public string Region { get; set; } = "us-east-1"; + + public string? ServiceUrl { get; set; } + = null; + + public string BucketName { get; set; } = ScannerStorageDefaults.DefaultBucketName; + + public string RootPrefix { get; set; } = ScannerStorageDefaults.DefaultRootPrefix; + + public bool ForcePathStyle { get; set; } = true; + + public bool EnableObjectLock { get; set; } = false; + + public TimeSpan? ComplianceRetention { get; set; } + = TimeSpan.FromDays(90); + + public void EnsureValid() + { + if (string.IsNullOrWhiteSpace(BucketName)) + { + throw new InvalidOperationException("Scanner storage bucket name cannot be empty."); + } + + if (string.IsNullOrWhiteSpace(RootPrefix)) + { + throw new InvalidOperationException("Scanner storage root prefix cannot be empty."); + } + + if (ComplianceRetention is { } retention && retention <= TimeSpan.Zero) + { + throw new InvalidOperationException("Compliance retention must be positive when specified."); + } + } +} + +public sealed class DualWriteOptions +{ + public bool Enabled { get; set; } + = false; + + public string? MirrorBucket { get; set; } + = null; + + public void EnsureValid() + { + if (Enabled && string.IsNullOrWhiteSpace(MirrorBucket)) + { + throw new InvalidOperationException("Dual-write mirror bucket must be configured when enabled."); + } + } +} diff --git a/src/StellaOps.Scanner.Storage/Services/ArtifactStorageService.cs b/src/StellaOps.Scanner.Storage/Services/ArtifactStorageService.cs new file mode 100644 index 00000000..75f4e23b --- /dev/null +++ b/src/StellaOps.Scanner.Storage/Services/ArtifactStorageService.cs @@ -0,0 +1,181 @@ +using System.Buffers; +using System.Security.Cryptography; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Storage.Catalog; +using StellaOps.Scanner.Storage.ObjectStore; +using StellaOps.Scanner.Storage.Repositories; + +namespace StellaOps.Scanner.Storage.Services; + +public sealed class ArtifactStorageService +{ + private readonly ArtifactRepository _artifactRepository; + private readonly LifecycleRuleRepository _lifecycleRuleRepository; + private readonly IArtifactObjectStore _objectStore; + private readonly ScannerStorageOptions _options; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + + public ArtifactStorageService( + ArtifactRepository artifactRepository, + LifecycleRuleRepository lifecycleRuleRepository, + IArtifactObjectStore objectStore, + IOptions options, + ILogger logger, + TimeProvider? timeProvider = null) + { + _artifactRepository = artifactRepository ?? throw new ArgumentNullException(nameof(artifactRepository)); + _lifecycleRuleRepository = lifecycleRuleRepository ?? throw new ArgumentNullException(nameof(lifecycleRuleRepository)); + _objectStore = objectStore ?? throw new ArgumentNullException(nameof(objectStore)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value; + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task StoreArtifactAsync( + ArtifactDocumentType type, + ArtifactDocumentFormat format, + string mediaType, + Stream content, + bool immutable, + string ttlClass, + DateTime? expiresAtUtc, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(content); + ArgumentException.ThrowIfNullOrWhiteSpace(mediaType); + + var (buffer, size, digestHex) = await BufferAndHashAsync(content, cancellationToken).ConfigureAwait(false); + try + { + var normalizedDigest = $"sha256:{digestHex}"; + var artifactId = CatalogIdFactory.CreateArtifactId(type, normalizedDigest); + var key = BuildObjectKey(type, format, normalizedDigest); + var descriptor = new ArtifactObjectDescriptor( + _options.ObjectStore.BucketName, + key, + immutable, + _options.ObjectStore.ComplianceRetention); + + buffer.Position = 0; + await _objectStore.PutAsync(descriptor, buffer, cancellationToken).ConfigureAwait(false); + + if (_options.DualWrite.Enabled) + { + buffer.Position = 0; + var mirrorDescriptor = descriptor with { Bucket = _options.DualWrite.MirrorBucket! }; + await _objectStore.PutAsync(mirrorDescriptor, buffer, cancellationToken).ConfigureAwait(false); + } + + var now = _timeProvider.GetUtcNow().UtcDateTime; + var document = new ArtifactDocument + { + Id = artifactId, + Type = type, + Format = format, + MediaType = mediaType, + BytesSha256 = normalizedDigest, + SizeBytes = size, + Immutable = immutable, + RefCount = 1, + CreatedAtUtc = now, + UpdatedAtUtc = now, + TtlClass = ttlClass, + }; + + await _artifactRepository.UpsertAsync(document, cancellationToken).ConfigureAwait(false); + + if (expiresAtUtc.HasValue) + { + var lifecycle = new LifecycleRuleDocument + { + Id = CatalogIdFactory.CreateLifecycleRuleId(document.Id, ttlClass), + ArtifactId = document.Id, + Class = ttlClass, + ExpiresAtUtc = expiresAtUtc, + CreatedAtUtc = now, + }; + + await _lifecycleRuleRepository.UpsertAsync(lifecycle, cancellationToken).ConfigureAwait(false); + } + + _logger.LogInformation("Stored scanner artifact {ArtifactId} ({SizeBytes} bytes, digest {Digest})", document.Id, size, normalizedDigest); + return document; + } + finally + { + await buffer.DisposeAsync().ConfigureAwait(false); + } + } + + private static async Task<(MemoryStream Buffer, long Size, string DigestHex)> BufferAndHashAsync(Stream content, CancellationToken cancellationToken) + { + var bufferStream = new MemoryStream(); + var hasher = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var rented = ArrayPool.Shared.Rent(81920); + long total = 0; + + try + { + int read; + while ((read = await content.ReadAsync(rented.AsMemory(0, rented.Length), cancellationToken).ConfigureAwait(false)) > 0) + { + hasher.AppendData(rented, 0, read); + await bufferStream.WriteAsync(rented.AsMemory(0, read), cancellationToken).ConfigureAwait(false); + total += read; + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + + bufferStream.Position = 0; + var digest = hasher.GetCurrentHash(); + var digestHex = Convert.ToHexString(digest).ToLowerInvariant(); + return (bufferStream, total, digestHex); + } + + private string BuildObjectKey(ArtifactDocumentType type, ArtifactDocumentFormat format, string digest) + { + var normalizedDigest = digest.Split(':', 2, StringSplitOptions.TrimEntries)[^1]; + var prefix = type switch + { + ArtifactDocumentType.LayerBom => ScannerStorageDefaults.ObjectPrefixes.Layers, + ArtifactDocumentType.ImageBom => ScannerStorageDefaults.ObjectPrefixes.Images, + ArtifactDocumentType.Diff => "diffs", + ArtifactDocumentType.Index => ScannerStorageDefaults.ObjectPrefixes.Indexes, + ArtifactDocumentType.Attestation => ScannerStorageDefaults.ObjectPrefixes.Attestations, + _ => ScannerStorageDefaults.ObjectPrefixes.Images, + }; + + var extension = format switch + { + ArtifactDocumentFormat.CycloneDxJson => "sbom.cdx.json", + ArtifactDocumentFormat.CycloneDxProtobuf => "sbom.cdx.pb", + ArtifactDocumentFormat.SpdxJson => "sbom.spdx.json", + ArtifactDocumentFormat.BomIndex => "bom-index.bin", + ArtifactDocumentFormat.DsseJson => "artifact.dsse.json", + _ => "artifact.bin", + }; + + var rootPrefix = _options.ObjectStore.RootPrefix; + if (string.IsNullOrWhiteSpace(rootPrefix)) + { + return $"{prefix}/{normalizedDigest}/{extension}"; + } + + return $"{TrimTrailingSlash(rootPrefix)}/{prefix}/{normalizedDigest}/{extension}"; + } + + private static string TrimTrailingSlash(string prefix) + { + if (string.IsNullOrWhiteSpace(prefix)) + { + return string.Empty; + } + + return prefix.TrimEnd('/'); + } +} diff --git a/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj new file mode 100644 index 00000000..2acd5ed0 --- /dev/null +++ b/src/StellaOps.Scanner.Storage/StellaOps.Scanner.Storage.csproj @@ -0,0 +1,17 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Storage/TASKS.md b/src/StellaOps.Scanner.Storage/TASKS.md new file mode 100644 index 00000000..e3e6bd2e --- /dev/null +++ b/src/StellaOps.Scanner.Storage/TASKS.md @@ -0,0 +1,8 @@ +# Scanner Storage Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-STORAGE-09-301 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-CORE-09-501 | Mongo catalog schemas/indexes for images, layers, artifacts, jobs, lifecycle rules plus migrations. | Collections created via bootstrapper; migrations recorded; indexes enforce uniqueness + TTL; majority read/write configured. | +| SCANNER-STORAGE-09-302 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301 | MinIO layout, immutability policies, client abstraction, and configuration binding. | S3 client abstraction configurable via options; bucket/prefix defaults documented; immutability flags enforced with tests; config binding validated. | +| SCANNER-STORAGE-09-303 | DONE (2025-10-18) | Scanner Storage Guild | SCANNER-STORAGE-09-301, SCANNER-STORAGE-09-302 | Repositories/services with dual-write feature flag, deterministic digests, TTL enforcement tests. | Dual-write service writes metadata + objects atomically; digest determinism covered by tests; TTL enforcement fixture passing. | +| SCANNER-STORAGE-09-304 | DONE (2025-10-19) | Scanner Storage Guild | SCANNER-STORAGE-09-303 | Adopt `TimeProvider` across storage timestamps for determinism. | Storage services/repositories use injected `TimeProvider`; tests cover timestamp determinism. | diff --git a/src/StellaOps.Scanner.WebService.Tests/AuthorizationTests.cs b/src/StellaOps.Scanner.WebService.Tests/AuthorizationTests.cs new file mode 100644 index 00000000..c63fd287 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/AuthorizationTests.cs @@ -0,0 +1,25 @@ +using System.Net; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class AuthorizationTests +{ + [Fact] + public async Task ApiRoutesRequireAuthenticationWhenAuthorityEnabled() + { + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:authority:enabled"] = "true"; + configuration["scanner:authority:allowAnonymousFallback"] = "false"; + configuration["scanner:authority:issuer"] = "https://authority.local"; + configuration["scanner:authority:audiences:0"] = "scanner-api"; + configuration["scanner:authority:clientId"] = "scanner-web"; + configuration["scanner:authority:clientSecret"] = "secret"; + }); + + using var client = factory.CreateClient(); + var response = await client.GetAsync("/api/v1/__auth-probe"); + + Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode); + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/HealthEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/HealthEndpointsTests.cs new file mode 100644 index 00000000..9a3cad38 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/HealthEndpointsTests.cs @@ -0,0 +1,49 @@ +using System.Net.Http.Json; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class HealthEndpointsTests +{ + [Fact] + public async Task HealthAndReadyEndpointsRespond() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var healthResponse = await client.GetAsync("/healthz"); + Assert.True(healthResponse.IsSuccessStatusCode, $"Expected 200 from /healthz, received {(int)healthResponse.StatusCode}."); + + var readyResponse = await client.GetAsync("/readyz"); + Assert.True(readyResponse.IsSuccessStatusCode, $"Expected 200 from /readyz, received {(int)readyResponse.StatusCode}."); + + var healthDocument = await healthResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(healthDocument); + Assert.Equal("healthy", healthDocument!.Status); + Assert.True(healthDocument.UptimeSeconds >= 0); + Assert.NotNull(healthDocument.Telemetry); + + var readyDocument = await readyResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(readyDocument); + Assert.Equal("ready", readyDocument!.Status); + Assert.Null(readyDocument.Error); + } + + private sealed record HealthDocument( + string Status, + DateTimeOffset StartedAt, + DateTimeOffset CapturedAt, + double UptimeSeconds, + TelemetryDocument Telemetry); + + private sealed record TelemetryDocument( + bool Enabled, + bool Logging, + bool Metrics, + bool Tracing); + + private sealed record ReadyDocument( + string Status, + DateTimeOffset CheckedAt, + double? LatencyMs, + string? Error); +} diff --git a/src/StellaOps.Scanner.WebService.Tests/PlatformEventPublisherRegistrationTests.cs b/src/StellaOps.Scanner.WebService.Tests/PlatformEventPublisherRegistrationTests.cs new file mode 100644 index 00000000..62347cfe --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/PlatformEventPublisherRegistrationTests.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class PlatformEventPublisherRegistrationTests +{ + [Fact] + public void NullPublisherRegisteredWhenEventsDisabled() + { + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:events:enabled"] = "false"; + configuration["scanner:events:dsn"] = string.Empty; + }); + using var scope = factory.Services.CreateScope(); + + var publisher = scope.ServiceProvider.GetRequiredService(); + Assert.IsType(publisher); + } + + [Fact] + public void RedisPublisherRegisteredWhenEventsEnabled() + { + var originalEnabled = Environment.GetEnvironmentVariable("SCANNER__EVENTS__ENABLED"); + var originalDriver = Environment.GetEnvironmentVariable("SCANNER__EVENTS__DRIVER"); + var originalDsn = Environment.GetEnvironmentVariable("SCANNER__EVENTS__DSN"); + var originalStream = Environment.GetEnvironmentVariable("SCANNER__EVENTS__STREAM"); + var originalTimeout = Environment.GetEnvironmentVariable("SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS"); + var originalMax = Environment.GetEnvironmentVariable("SCANNER__EVENTS__MAXSTREAMLENGTH"); + + Environment.SetEnvironmentVariable("SCANNER__EVENTS__ENABLED", "true"); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__DRIVER", "redis"); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__DSN", "localhost:6379"); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__STREAM", "stella.events.tests"); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS", "1"); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__MAXSTREAMLENGTH", "100"); + + try + { + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:events:enabled"] = "true"; + configuration["scanner:events:driver"] = "redis"; + configuration["scanner:events:dsn"] = "localhost:6379"; + configuration["scanner:events:stream"] = "stella.events.tests"; + configuration["scanner:events:publishTimeoutSeconds"] = "1"; + configuration["scanner:events:maxStreamLength"] = "100"; + }); + using var scope = factory.Services.CreateScope(); + + var options = scope.ServiceProvider.GetRequiredService>().Value; + Assert.True(options.Events.Enabled); + Assert.Equal("redis", options.Events.Driver); + + var publisher = scope.ServiceProvider.GetRequiredService(); + Assert.IsType(publisher); + } + finally + { + Environment.SetEnvironmentVariable("SCANNER__EVENTS__ENABLED", originalEnabled); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__DRIVER", originalDriver); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__DSN", originalDsn); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__STREAM", originalStream); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS", originalTimeout); + Environment.SetEnvironmentVariable("SCANNER__EVENTS__MAXSTREAMLENGTH", originalMax); + } + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs b/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs new file mode 100644 index 00000000..97f403eb --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/PlatformEventSamplesTests.cs @@ -0,0 +1,71 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using StellaOps.Notify.Models; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class PlatformEventSamplesTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter() } + }; + + [Theory] + [InlineData("scanner.report.ready@1.sample.json", NotifyEventKinds.ScannerReportReady)] + [InlineData("scanner.scan.completed@1.sample.json", NotifyEventKinds.ScannerScanCompleted)] + public void PlatformEventSamplesStayCanonical(string fileName, string expectedKind) + { + var json = LoadSample(fileName); + var notifyEvent = JsonSerializer.Deserialize(json, SerializerOptions); + + Assert.NotNull(notifyEvent); + Assert.Equal(expectedKind, notifyEvent!.Kind); + Assert.NotEqual(Guid.Empty, notifyEvent.EventId); + Assert.NotNull(notifyEvent.Payload); + + AssertCanonical(json, notifyEvent); + AssertReportConsistency(notifyEvent.Payload); + } + + private static void AssertCanonical(string originalJson, NotifyEvent notifyEvent) + { + var canonicalJson = NotifyCanonicalJsonSerializer.Serialize(notifyEvent); + var originalNode = JsonNode.Parse(originalJson) ?? throw new InvalidOperationException("Sample JSON must not be null."); + var canonicalNode = JsonNode.Parse(canonicalJson) ?? throw new InvalidOperationException("Canonical JSON must not be null."); + + Assert.True(JsonNode.DeepEquals(originalNode, canonicalNode), "Platform event sample must remain canonical."); + } + + private static void AssertReportConsistency(JsonNode? payloadNode) + { + var payload = Assert.IsType(payloadNode); + + var reportNode = Assert.IsType(payload["report"]); + var report = reportNode.Deserialize(SerializerOptions); + Assert.NotNull(report); + + var dsseNode = Assert.IsType(payload["dsse"]); + var payloadValueNode = Assert.IsAssignableFrom(dsseNode["payload"]); + var base64Payload = payloadValueNode.GetValue(); + + var canonicalReportBytes = JsonSerializer.SerializeToUtf8Bytes(report!, SerializerOptions); + var expectedPayload = Convert.ToBase64String(canonicalReportBytes); + Assert.Equal(expectedPayload, base64Payload); + + var reportIdReference = Assert.IsAssignableFrom(payload["reportId"]).GetValue(); + Assert.Equal(report!.ReportId, reportIdReference); + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + Assert.True(File.Exists(path), $"Sample file not found at '{path}'."); + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/PolicyEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/PolicyEndpointsTests.cs new file mode 100644 index 00000000..20773aef --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/PolicyEndpointsTests.cs @@ -0,0 +1,108 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Threading.Tasks; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class PolicyEndpointsTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public async Task PolicySchemaReturnsEmbeddedSchema() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var response = await client.GetAsync("/api/v1/policy/schema"); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/schema+json", response.Content.Headers.ContentType?.MediaType); + + var payload = await response.Content.ReadAsStringAsync(); + Assert.Contains("\"$schema\"", payload); + Assert.Contains("\"properties\"", payload); + } + + [Fact] + public async Task PolicyDiagnosticsReturnsRecommendations() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new PolicyDiagnosticsRequestDto + { + Policy = new PolicyPreviewPolicyDto + { + Content = "version: \"1.0\"\nrules: []\n", + Format = "yaml", + Actor = "tester", + Description = "empty ruleset" + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/policy/diagnostics", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var diagnostics = await response.Content.ReadFromJsonAsync(SerializerOptions); + Assert.NotNull(diagnostics); + Assert.False(diagnostics!.Success); + Assert.True(diagnostics.ErrorCount >= 0); + Assert.NotEmpty(diagnostics.Recommendations); + } + + [Fact] + public async Task PolicyPreviewUsesProposedPolicy() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + const string policyYaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + + var request = new PolicyPreviewRequestDto + { + ImageDigest = "sha256:abc123", + Findings = new[] + { + new PolicyPreviewFindingDto + { + Id = "finding-1", + Severity = "Critical", + Source = "NVD", + Tags = new[] { "reachability:runtime" } + } + }, + Policy = new PolicyPreviewPolicyDto + { + Content = policyYaml, + Format = "yaml", + Actor = "preview", + Description = "test policy" + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/policy/preview", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var preview = await response.Content.ReadFromJsonAsync(SerializerOptions); + Assert.NotNull(preview); + Assert.True(preview!.Success); + Assert.Equal(1, preview.Changed); + var diff = Assert.Single(preview.Diffs); + Assert.Equal("finding-1", diff.Projected?.FindingId); + Assert.Equal("Blocked", diff.Projected?.Status); + Assert.Equal(PolicyScoringConfig.Default.Version, diff.Projected?.ConfigVersion); + Assert.NotNull(diff.Projected?.Inputs); + Assert.True(diff.Projected!.Inputs!.ContainsKey("severityWeight")); + Assert.Equal("NVD", diff.Projected.SourceTrust); + Assert.Equal("runtime", diff.Projected.Reachability); + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs new file mode 100644 index 00000000..c2aa4f03 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/ReportEventDispatcherTests.cs @@ -0,0 +1,156 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using System.Security.Claims; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Auth.Abstractions; +using StellaOps.Notify.Models; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class ReportEventDispatcherTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + [Fact] + public async Task PublishAsync_EmitsReportReadyAndScanCompleted() + { + var publisher = new RecordingEventPublisher(); + var dispatcher = new ReportEventDispatcher(publisher, TimeProvider.System, NullLogger.Instance); + var cancellationToken = CancellationToken.None; + + var request = new ReportRequestDto + { + ImageDigest = "sha256:feedface", + Findings = new[] + { + new PolicyPreviewFindingDto + { + Id = "finding-1", + Severity = "Critical", + Repository = "acme/edge/api", + Cve = "CVE-2024-9999", + Tags = new[] { "reachability:runtime", "kev:CVE-2024-9999" } + } + } + }; + + var baseline = new PolicyVerdict("finding-1", PolicyVerdictStatus.Pass, ConfigVersion: "1.0"); + var projected = new PolicyVerdict( + "finding-1", + PolicyVerdictStatus.Blocked, + Score: 47.5, + ConfigVersion: "1.0", + SourceTrust: "NVD", + Reachability: "runtime"); + + var preview = new PolicyPreviewResponse( + Success: true, + PolicyDigest: "digest-123", + RevisionId: "rev-42", + Issues: ImmutableArray.Empty, + Diffs: ImmutableArray.Create(new PolicyVerdictDiff(baseline, projected)), + ChangedCount: 1); + + var document = new ReportDocumentDto + { + ReportId = "report-abc", + ImageDigest = "sha256:feedface", + GeneratedAt = DateTimeOffset.Parse("2025-10-19T12:34:56Z"), + Verdict = "blocked", + Policy = new ReportPolicyDto + { + RevisionId = "rev-42", + Digest = "digest-123" + }, + Summary = new ReportSummaryDto + { + Total = 1, + Blocked = 1, + Warned = 0, + Ignored = 0, + Quieted = 0 + }, + Verdicts = new[] + { + new PolicyPreviewVerdictDto + { + FindingId = "finding-1", + Status = "Blocked", + Score = 47.5, + SourceTrust = "NVD", + Reachability = "runtime" + } + } + }; + + var envelope = new DsseEnvelopeDto + { + PayloadType = "application/vnd.stellaops.report+json", + Payload = Convert.ToBase64String(JsonSerializer.SerializeToUtf8Bytes(document, SerializerOptions)), + Signatures = new[] + { + new DsseSignatureDto { KeyId = "test-key", Algorithm = "hs256", Signature = "signature-value" } + } + }; + + var context = new DefaultHttpContext(); + context.User = new ClaimsPrincipal(new ClaimsIdentity(new[] + { + new Claim(StellaOpsClaimTypes.Tenant, "tenant-alpha") + })); + context.Request.Scheme = "https"; + context.Request.Host = new HostString("scanner.example"); + + await dispatcher.PublishAsync(request, preview, document, envelope, context, cancellationToken); + + Assert.Equal(2, publisher.Events.Count); + + var readyEvent = Assert.Single(publisher.Events, evt => evt.Kind == NotifyEventKinds.ScannerReportReady); + Assert.Equal("tenant-alpha", readyEvent.Tenant); + Assert.Equal("api", readyEvent.Scope?.Repo); + Assert.Equal("acme/edge", readyEvent.Scope?.Namespace); + Assert.Equal("sha256:feedface", readyEvent.Scope?.Digest); + Assert.NotNull(readyEvent.Payload); + Assert.Equal("fail", readyEvent.Payload?["verdict"]?.GetValue()); + Assert.Equal("report-abc", readyEvent.Payload?["reportId"]?.GetValue()); + Assert.Equal("signature-value", readyEvent.Payload?["dsse"]?["signatures"]?[0]?["signature"]?.GetValue()); + Assert.Equal(envelope.Payload, readyEvent.Payload?["dsse"]?["payload"]?.GetValue()); + Assert.Equal(1, readyEvent.Payload?["delta"]?["newCritical"]?.GetValue()); + Assert.Equal("CVE-2024-9999", readyEvent.Payload?["delta"]?["kev"]?[0]?.GetValue()); + Assert.Equal("https://scanner.example/ui/reports/report-abc", readyEvent.Payload?["links"]?["ui"]?.GetValue()); + var scanEvent = Assert.Single(publisher.Events, evt => evt.Kind == NotifyEventKinds.ScannerScanCompleted); + Assert.Equal("fail", scanEvent.Payload?["verdict"]?.GetValue()); + Assert.Equal("report-abc", scanEvent.Payload?["reportId"]?.GetValue()); + Assert.Equal("sha256:feedface", scanEvent.Payload?["digest"]?.GetValue()); + Assert.Equal("runtime", scanEvent.Payload?["findings"]?[0]?["reachability"]?.GetValue()); + Assert.Equal("report-abc", scanEvent.Payload?["report"]?["reportId"]?.GetValue()); + Assert.Equal("blocked", scanEvent.Payload?["report"]?["verdict"]?.GetValue()); + Assert.Equal(envelope.Payload, scanEvent.Payload?["dsse"]?["payload"]?.GetValue()); + Assert.Equal(NotifyEventKinds.ScannerScanCompleted, scanEvent.Kind); + } + + private sealed class RecordingEventPublisher : IPlatformEventPublisher + { + public List Events { get; } = new(); + + public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) + { + Events.Add(@event); + return Task.CompletedTask; + } + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportSamplesTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportSamplesTests.cs new file mode 100644 index 00000000..07685c26 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/ReportSamplesTests.cs @@ -0,0 +1,35 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class ReportSamplesTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter() } + }; + + [Fact] + public async Task ReportSampleEnvelope_RemainsCanonical() + { + var baseDirectory = AppContext.BaseDirectory; + var repoRoot = Path.GetFullPath(Path.Combine(baseDirectory, "..", "..", "..", "..", "..")); + var path = Path.Combine(repoRoot, "samples", "api", "reports", "report-sample.dsse.json"); + Assert.True(File.Exists(path), $"Sample file not found at {path}."); + await using var stream = File.OpenRead(path); + var response = await JsonSerializer.DeserializeAsync(stream, SerializerOptions); + Assert.NotNull(response); + Assert.NotNull(response!.Report); + Assert.NotNull(response.Dsse); + + var reportBytes = JsonSerializer.SerializeToUtf8Bytes(response.Report, SerializerOptions); + var expectedPayload = Convert.ToBase64String(reportBytes); + Assert.Equal(expectedPayload, response.Dsse!.Payload); + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs new file mode 100644 index 00000000..e71594b5 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/ReportsEndpointsTests.cs @@ -0,0 +1,136 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class ReportsEndpointsTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + [Fact] + public async Task ReportsEndpointReturnsSignedEnvelope() + { + const string policyYaml = """ +version: "1.0" +rules: + - name: Block Critical + severity: [Critical] + action: block +"""; + + var hmacKey = Convert.ToBase64String(Encoding.UTF8.GetBytes("scanner-report-hmac-key-2025!")); + + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:signing:enabled"] = "true"; + configuration["scanner:signing:keyId"] = "scanner-report-signing"; + configuration["scanner:signing:algorithm"] = "hs256"; + configuration["scanner:signing:keyPem"] = hmacKey; + configuration["scanner:features:enableSignedReports"] = "true"; + }); + + var store = factory.Services.GetRequiredService(); + await store.SaveAsync( + new PolicySnapshotContent(policyYaml, PolicyDocumentFormat.Yaml, "tester", "seed", "initial"), + CancellationToken.None); + + using var client = factory.CreateClient(); + + var request = new ReportRequestDto + { + ImageDigest = "sha256:deadbeef", + Findings = new[] + { + new PolicyPreviewFindingDto + { + Id = "finding-1", + Severity = "Critical", + Source = "NVD", + Tags = new[] { "reachability:runtime" } + } + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/reports", request); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var raw = await response.Content.ReadAsStringAsync(); + Assert.False(string.IsNullOrWhiteSpace(raw), raw); + var payload = JsonSerializer.Deserialize(raw, SerializerOptions); + Assert.NotNull(payload); + Assert.NotNull(payload!.Report); + Assert.NotNull(payload.Dsse); + Assert.StartsWith("report-", payload.Report.ReportId, StringComparison.Ordinal); + Assert.Equal("blocked", payload.Report.Verdict); + + var dsse = payload.Dsse!; + Assert.Equal("application/vnd.stellaops.report+json", dsse.PayloadType); + var decodedPayload = Convert.FromBase64String(dsse.Payload); + var canonicalPayload = JsonSerializer.SerializeToUtf8Bytes(payload.Report, SerializerOptions); + var expectedBase64 = Convert.ToBase64String(canonicalPayload); + Assert.Equal(expectedBase64, dsse.Payload); + + var reportVerdict = Assert.Single(payload.Report.Verdicts); + Assert.Equal("NVD", reportVerdict.SourceTrust); + Assert.Equal("runtime", reportVerdict.Reachability); + Assert.NotNull(reportVerdict.Inputs); + Assert.True(reportVerdict.Inputs!.ContainsKey("severityWeight")); + Assert.Equal(PolicyScoringConfig.Default.Version, reportVerdict.ConfigVersion); + + var signature = Assert.Single(dsse.Signatures); + Assert.Equal("scanner-report-signing", signature.KeyId); + Assert.Equal("hs256", signature.Algorithm, ignoreCase: true); + + using var hmac = new System.Security.Cryptography.HMACSHA256(Convert.FromBase64String(hmacKey)); + var expectedSig = Convert.ToBase64String(hmac.ComputeHash(decodedPayload)); + var actualSig = signature.Signature; + Assert.True(expectedSig == actualSig, $"expected:{expectedSig}, actual:{actualSig}"); + } + + [Fact] + public async Task ReportsEndpointValidatesDigest() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ReportRequestDto + { + ImageDigest = "", + Findings = Array.Empty() + }; + + var response = await client.PostAsJsonAsync("/api/v1/reports", request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task ReportsEndpointReturnsServiceUnavailableWhenPolicyMissing() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ReportRequestDto + { + ImageDigest = "sha256:feedface", + Findings = new[] + { + new PolicyPreviewFindingDto { Id = "finding-1", Severity = "High" } + } + }; + + var response = await client.PostAsJsonAsync("/api/v1/reports", request); + Assert.Equal((HttpStatusCode)StatusCodes.Status503ServiceUnavailable, response.StatusCode); + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs b/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs new file mode 100644 index 00000000..b6368503 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/ScannerApplicationFactory.cs @@ -0,0 +1,167 @@ +using System.Collections.Generic; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Mongo2Go; + +namespace StellaOps.Scanner.WebService.Tests; + +internal sealed class ScannerApplicationFactory : WebApplicationFactory +{ + private readonly MongoDbRunner mongoRunner; + private readonly Dictionary configuration = new() + { + ["scanner:storage:driver"] = "mongo", + ["scanner:storage:dsn"] = string.Empty, + ["scanner:queue:driver"] = "redis", + ["scanner:queue:dsn"] = "redis://localhost:6379", + ["scanner:artifactStore:driver"] = "minio", + ["scanner:artifactStore:endpoint"] = "https://minio.local", + ["scanner:artifactStore:accessKey"] = "test-access", + ["scanner:artifactStore:secretKey"] = "test-secret", + ["scanner:artifactStore:bucket"] = "scanner-artifacts", + ["scanner:telemetry:minimumLogLevel"] = "Information", + ["scanner:telemetry:enableRequestLogging"] = "false", + ["scanner:events:enabled"] = "false", + ["scanner:features:enableSignedReports"] = "false" + }; + + private readonly Action>? configureConfiguration; + private readonly Action? configureServices; + + public ScannerApplicationFactory( + Action>? configureConfiguration = null, + Action? configureServices = null) + { + EnsureMongo2GoEnvironment(); + mongoRunner = MongoDbRunner.Start(singleNodeReplSet: true); + configuration["scanner:storage:dsn"] = mongoRunner.ConnectionString; + this.configureConfiguration = configureConfiguration; + this.configureServices = configureServices; + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + configureConfiguration?.Invoke(configuration); + + builder.UseEnvironment("Testing"); + + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ENABLED", null); + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ALLOWANONYMOUSFALLBACK", null); + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ISSUER", null); + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__AUDIENCES__0", null); + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTID", null); + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTSECRET", null); + Environment.SetEnvironmentVariable("SCANNER__STORAGE__DSN", configuration["scanner:storage:dsn"]); + Environment.SetEnvironmentVariable("SCANNER__QUEUE__DSN", configuration["scanner:queue:dsn"]); + Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ENDPOINT", configuration["scanner:artifactStore:endpoint"]); + Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__ACCESSKEY", configuration["scanner:artifactStore:accessKey"]); + Environment.SetEnvironmentVariable("SCANNER__ARTIFACTSTORE__SECRETKEY", configuration["scanner:artifactStore:secretKey"]); + if (configuration.TryGetValue("scanner:events:enabled", out var eventsEnabled)) + { + Environment.SetEnvironmentVariable("SCANNER__EVENTS__ENABLED", eventsEnabled); + } + + if (configuration.TryGetValue("scanner:authority:enabled", out var authorityEnabled)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ENABLED", authorityEnabled); + } + + if (configuration.TryGetValue("scanner:authority:allowAnonymousFallback", out var allowAnonymous)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ALLOWANONYMOUSFALLBACK", allowAnonymous); + } + + if (configuration.TryGetValue("scanner:authority:issuer", out var authorityIssuer)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__ISSUER", authorityIssuer); + } + + if (configuration.TryGetValue("scanner:authority:audiences:0", out var primaryAudience)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__AUDIENCES__0", primaryAudience); + } + + if (configuration.TryGetValue("scanner:authority:clientId", out var clientId)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTID", clientId); + } + + if (configuration.TryGetValue("scanner:authority:clientSecret", out var clientSecret)) + { + Environment.SetEnvironmentVariable("SCANNER__AUTHORITY__CLIENTSECRET", clientSecret); + } + + builder.ConfigureAppConfiguration((_, configBuilder) => + { + configBuilder.AddInMemoryCollection(configuration); + }); + + builder.ConfigureTestServices(services => + { + configureServices?.Invoke(services); + }); + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (disposing) + { + mongoRunner.Dispose(); + } + } + + private static void EnsureMongo2GoEnvironment() + { + if (!OperatingSystem.IsLinux()) + { + return; + } + + var libraryPath = ResolveOpenSslLibraryPath(); + if (libraryPath is null) + { + return; + } + + var existing = Environment.GetEnvironmentVariable("LD_LIBRARY_PATH"); + if (string.IsNullOrEmpty(existing)) + { + Environment.SetEnvironmentVariable("LD_LIBRARY_PATH", libraryPath); + return; + } + + var segments = existing.Split(':', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (Array.IndexOf(segments, libraryPath) < 0) + { + Environment.SetEnvironmentVariable("LD_LIBRARY_PATH", string.Join(':', new[] { libraryPath }.Concat(segments))); + } + } + + private static string? ResolveOpenSslLibraryPath() + { + var current = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(current)) + { + var candidate = Path.Combine(current, "tools", "openssl", "linux-x64"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + var parent = Directory.GetParent(current); + if (parent is null) + { + break; + } + + current = parent.FullName; + } + + return null; + } +} diff --git a/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs b/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs new file mode 100644 index 00000000..40a8e9be --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/ScansEndpointsTests.cs @@ -0,0 +1,303 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Domain; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Tests; + +public sealed class ScansEndpointsTests +{ + [Fact] + public async Task SubmitScanReturnsAcceptedAndStatusRetrievable() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "ghcr.io/demo/app:1.0.0" }, + Force = false + }; + + var response = await client.PostAsJsonAsync("/api/v1/scans", request); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + var payload = await response.Content.ReadFromJsonAsync(); + Assert.NotNull(payload); + Assert.False(string.IsNullOrWhiteSpace(payload!.ScanId)); + Assert.Equal("Pending", payload.Status); + Assert.True(payload.Created); + Assert.False(string.IsNullOrWhiteSpace(payload.Location)); + + var statusResponse = await client.GetAsync(payload.Location); + Assert.Equal(HttpStatusCode.OK, statusResponse.StatusCode); + + var status = await statusResponse.Content.ReadFromJsonAsync(); + Assert.NotNull(status); + Assert.Equal(payload.ScanId, status!.ScanId); + Assert.Equal("Pending", status.Status); + Assert.Equal("ghcr.io/demo/app:1.0.0", status.Image.Reference); + } + + [Fact] + public async Task SubmitScanIsDeterministicForIdenticalPayloads() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "registry.example.com/acme/app:latest" }, + Force = false, + ClientRequestId = "client-123", + Metadata = new Dictionary { ["origin"] = "unit-test" } + }; + + var first = await client.PostAsJsonAsync("/api/v1/scans", request); + var firstPayload = await first.Content.ReadFromJsonAsync(); + + var second = await client.PostAsJsonAsync("/api/v1/scans", request); + var secondPayload = await second.Content.ReadFromJsonAsync(); + + Assert.NotNull(firstPayload); + Assert.NotNull(secondPayload); + Assert.Equal(firstPayload!.ScanId, secondPayload!.ScanId); + Assert.True(firstPayload.Created); + Assert.False(secondPayload.Created); + } + + [Fact] + public async Task SubmitScanValidatesImageDescriptor() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new + { + image = new { reference = "", digest = "" } + }; + + var response = await client.PostAsJsonAsync("/api/v1/scans", request); + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + } + + [Fact] + public async Task SubmitScanPropagatesRequestAbortedToken() + { + RecordingCoordinator coordinator = null!; + using var factory = new ScannerApplicationFactory(configuration => + { + configuration["scanner:authority:enabled"] = "false"; + }, services => + { + services.AddSingleton(sp => + { + coordinator = new RecordingCoordinator( + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService()); + return coordinator; + }); + }); + + using var client = factory.CreateClient(new WebApplicationFactoryClientOptions + { + AllowAutoRedirect = false + }); + + var cts = new CancellationTokenSource(); + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "example.com/demo:1.0" } + }; + + var response = await client.PostAsJsonAsync("/api/v1/scans", request, cts.Token); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + + Assert.NotNull(coordinator); + Assert.True(coordinator.TokenMatched); + Assert.True(coordinator.LastToken.CanBeCanceled); + } + + private sealed class RecordingCoordinator : IScanCoordinator + { + private readonly IHttpContextAccessor accessor; + private readonly InMemoryScanCoordinator inner; + + public RecordingCoordinator(IHttpContextAccessor accessor, TimeProvider timeProvider, IScanProgressPublisher publisher) + { + this.accessor = accessor; + inner = new InMemoryScanCoordinator(timeProvider, publisher); + } + + public CancellationToken LastToken { get; private set; } + + public bool TokenMatched { get; private set; } + + public async ValueTask SubmitAsync(ScanSubmission submission, CancellationToken cancellationToken) + { + LastToken = cancellationToken; + TokenMatched = accessor.HttpContext?.RequestAborted.Equals(cancellationToken) ?? false; + return await inner.SubmitAsync(submission, cancellationToken); + } + + public ValueTask GetAsync(ScanId scanId, CancellationToken cancellationToken) + => inner.GetAsync(scanId, cancellationToken); + } + + [Fact] + public async Task ProgressStreamReturnsInitialPendingEvent() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "ghcr.io/demo/app:2.0.0" } + }; + + var submit = await client.PostAsJsonAsync("/api/v1/scans", request); + var submitPayload = await submit.Content.ReadFromJsonAsync(); + Assert.NotNull(submitPayload); + + var response = await client.GetAsync($"/api/v1/scans/{submitPayload!.ScanId}/events?format=jsonl", HttpCompletionOption.ResponseHeadersRead); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/x-ndjson", response.Content.Headers.ContentType?.MediaType); + + await using var stream = await response.Content.ReadAsStreamAsync(); + using var reader = new StreamReader(stream); + var line = await reader.ReadLineAsync(); + Assert.False(string.IsNullOrWhiteSpace(line)); + + var envelope = JsonSerializer.Deserialize(line!, SerializerOptions); + Assert.NotNull(envelope); + Assert.Equal(submitPayload.ScanId, envelope!.ScanId); + Assert.Equal("Pending", envelope.State); + Assert.Equal(1, envelope.Sequence); + Assert.NotEqual(default, envelope.Timestamp); + } + + [Fact] + public async Task ProgressStreamYieldsSubsequentEvents() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "registry.example.com/acme/app:stream" } + }; + + var submit = await client.PostAsJsonAsync("/api/v1/scans", request); + var submitPayload = await submit.Content.ReadFromJsonAsync(); + Assert.NotNull(submitPayload); + + var publisher = factory.Services.GetRequiredService(); + + var response = await client.GetAsync($"/api/v1/scans/{submitPayload!.ScanId}/events?format=jsonl", HttpCompletionOption.ResponseHeadersRead); + await using var stream = await response.Content.ReadAsStreamAsync(); + using var reader = new StreamReader(stream); + + var firstLine = await reader.ReadLineAsync(); + Assert.NotNull(firstLine); + var firstEnvelope = JsonSerializer.Deserialize(firstLine!, SerializerOptions); + Assert.NotNull(firstEnvelope); + Assert.Equal("Pending", firstEnvelope!.State); + + _ = Task.Run(async () => + { + await Task.Delay(50); + publisher.Publish(new ScanId(submitPayload.ScanId), "Running", "worker-started", new Dictionary + { + ["stage"] = "download" + }); + }); + + ProgressEnvelope? envelope = null; + string? line; + do + { + line = await reader.ReadLineAsync(); + if (line is null) + { + break; + } + + if (line.Length == 0) + { + continue; + } + + envelope = JsonSerializer.Deserialize(line, SerializerOptions); + } + while (envelope is not null && envelope.State == "Pending"); + + Assert.NotNull(envelope); + Assert.Equal("Running", envelope!.State); + Assert.True(envelope.Sequence >= 2); + Assert.Contains(envelope.Data.Keys, key => key == "stage"); + } + + [Fact] + public async Task ProgressStreamSupportsServerSentEvents() + { + using var factory = new ScannerApplicationFactory(); + using var client = factory.CreateClient(); + + var request = new ScanSubmitRequest + { + Image = new ScanImageDescriptor { Reference = "ghcr.io/demo/app:3.0.0" } + }; + + var submit = await client.PostAsJsonAsync("/api/v1/scans", request); + var submitPayload = await submit.Content.ReadFromJsonAsync(); + Assert.NotNull(submitPayload); + + var response = await client.GetAsync($"/api/v1/scans/{submitPayload!.ScanId}/events", HttpCompletionOption.ResponseHeadersRead); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/event-stream", response.Content.Headers.ContentType?.MediaType); + + await using var stream = await response.Content.ReadAsStreamAsync(); + using var reader = new StreamReader(stream); + + var idLine = await reader.ReadLineAsync(); + var eventLine = await reader.ReadLineAsync(); + var dataLine = await reader.ReadLineAsync(); + var separator = await reader.ReadLineAsync(); + + Assert.Equal("id: 1", idLine); + Assert.Equal("event: pending", eventLine); + Assert.NotNull(dataLine); + Assert.StartsWith("data: ", dataLine, StringComparison.Ordinal); + Assert.Equal(string.Empty, separator); + + var json = dataLine!["data: ".Length..]; + var envelope = JsonSerializer.Deserialize(json, SerializerOptions); + Assert.NotNull(envelope); + Assert.Equal(submitPayload.ScanId, envelope!.ScanId); + Assert.Equal("Pending", envelope.State); + Assert.Equal(1, envelope.Sequence); + Assert.True(envelope.Timestamp.UtcDateTime <= DateTime.UtcNow); + } + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private sealed record ProgressEnvelope( + string ScanId, + int Sequence, + string State, + string? Message, + DateTimeOffset Timestamp, + string CorrelationId, + Dictionary Data); +} diff --git a/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj new file mode 100644 index 00000000..6d931cc0 --- /dev/null +++ b/src/StellaOps.Scanner.WebService.Tests/StellaOps.Scanner.WebService.Tests.csproj @@ -0,0 +1,20 @@ + + + net10.0 + enable + enable + false + StellaOps.Scanner.WebService.Tests + + + + + + + Always + + + Always + + + diff --git a/src/StellaOps.Scanner.WebService/AssemblyInfo.cs b/src/StellaOps.Scanner.WebService/AssemblyInfo.cs new file mode 100644 index 00000000..56e7e675 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scanner.WebService.Tests")] diff --git a/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs b/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs new file mode 100644 index 00000000..ec951f54 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Constants/ProblemTypes.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Scanner.WebService.Constants; + +internal static class ProblemTypes +{ + public const string Validation = "https://stellaops.org/problems/validation"; + public const string Conflict = "https://stellaops.org/problems/conflict"; + public const string NotFound = "https://stellaops.org/problems/not-found"; + public const string InternalError = "https://stellaops.org/problems/internal-error"; +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/PolicyDiagnosticsContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/PolicyDiagnosticsContracts.cs new file mode 100644 index 00000000..03a8dc03 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/PolicyDiagnosticsContracts.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record PolicyDiagnosticsRequestDto +{ + [JsonPropertyName("policy")] + public PolicyPreviewPolicyDto? Policy { get; init; } +} + +public sealed record PolicyDiagnosticsResponseDto +{ + [JsonPropertyName("success")] + public bool Success { get; init; } + + [JsonPropertyName("version")] + public string Version { get; init; } = string.Empty; + + [JsonPropertyName("ruleCount")] + public int RuleCount { get; init; } + + [JsonPropertyName("errorCount")] + public int ErrorCount { get; init; } + + [JsonPropertyName("warningCount")] + public int WarningCount { get; init; } + + [JsonPropertyName("generatedAt")] + public DateTimeOffset GeneratedAt { get; init; } + + [JsonPropertyName("issues")] + public IReadOnlyList Issues { get; init; } = Array.Empty(); + + [JsonPropertyName("recommendations")] + public IReadOnlyList Recommendations { get; init; } = Array.Empty(); +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/PolicyPreviewContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/PolicyPreviewContracts.cs new file mode 100644 index 00000000..c1971dee --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/PolicyPreviewContracts.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record PolicyPreviewRequestDto +{ + [JsonPropertyName("imageDigest")] + public string? ImageDigest { get; init; } + + [JsonPropertyName("findings")] + public IReadOnlyList? Findings { get; init; } + + [JsonPropertyName("baseline")] + public IReadOnlyList? Baseline { get; init; } + + [JsonPropertyName("policy")] + public PolicyPreviewPolicyDto? Policy { get; init; } +} + +public sealed record PolicyPreviewFindingDto +{ + [JsonPropertyName("id")] + public string? Id { get; init; } + + [JsonPropertyName("severity")] + public string? Severity { get; init; } + + [JsonPropertyName("environment")] + public string? Environment { get; init; } + + [JsonPropertyName("source")] + public string? Source { get; init; } + + [JsonPropertyName("vendor")] + public string? Vendor { get; init; } + + [JsonPropertyName("license")] + public string? License { get; init; } + + [JsonPropertyName("image")] + public string? Image { get; init; } + + [JsonPropertyName("repository")] + public string? Repository { get; init; } + + [JsonPropertyName("package")] + public string? Package { get; init; } + + [JsonPropertyName("purl")] + public string? Purl { get; init; } + + [JsonPropertyName("cve")] + public string? Cve { get; init; } + + [JsonPropertyName("path")] + public string? Path { get; init; } + + [JsonPropertyName("layerDigest")] + public string? LayerDigest { get; init; } + + [JsonPropertyName("tags")] + public IReadOnlyList? Tags { get; init; } +} + +public sealed record PolicyPreviewVerdictDto +{ + [JsonPropertyName("findingId")] + public string? FindingId { get; init; } + + [JsonPropertyName("status")] + public string? Status { get; init; } + + [JsonPropertyName("ruleName")] + public string? RuleName { get; init; } + + [JsonPropertyName("ruleAction")] + public string? RuleAction { get; init; } + + [JsonPropertyName("notes")] + public string? Notes { get; init; } + + [JsonPropertyName("score")] + public double? Score { get; init; } + + [JsonPropertyName("configVersion")] + public string? ConfigVersion { get; init; } + + [JsonPropertyName("inputs")] + public IReadOnlyDictionary? Inputs { get; init; } + + [JsonPropertyName("quietedBy")] + public string? QuietedBy { get; init; } + + [JsonPropertyName("quiet")] + public bool? Quiet { get; init; } + + [JsonPropertyName("unknownConfidence")] + public double? UnknownConfidence { get; init; } + + [JsonPropertyName("confidenceBand")] + public string? ConfidenceBand { get; init; } + + [JsonPropertyName("unknownAgeDays")] + public double? UnknownAgeDays { get; init; } + + [JsonPropertyName("sourceTrust")] + public string? SourceTrust { get; init; } + + [JsonPropertyName("reachability")] + public string? Reachability { get; init; } + +} + +public sealed record PolicyPreviewPolicyDto +{ + [JsonPropertyName("content")] + public string? Content { get; init; } + + [JsonPropertyName("format")] + public string? Format { get; init; } + + [JsonPropertyName("actor")] + public string? Actor { get; init; } + + [JsonPropertyName("description")] + public string? Description { get; init; } +} + +public sealed record PolicyPreviewResponseDto +{ + [JsonPropertyName("success")] + public bool Success { get; init; } + + [JsonPropertyName("policyDigest")] + public string? PolicyDigest { get; init; } + + [JsonPropertyName("revisionId")] + public string? RevisionId { get; init; } + + [JsonPropertyName("changed")] + public int Changed { get; init; } + + [JsonPropertyName("diffs")] + public IReadOnlyList Diffs { get; init; } = Array.Empty(); + + [JsonPropertyName("issues")] + public IReadOnlyList Issues { get; init; } = Array.Empty(); +} + +public sealed record PolicyPreviewDiffDto +{ + [JsonPropertyName("findingId")] + public string? FindingId { get; init; } + + [JsonPropertyName("baseline")] + public PolicyPreviewVerdictDto? Baseline { get; init; } + + [JsonPropertyName("projected")] + public PolicyPreviewVerdictDto? Projected { get; init; } + + [JsonPropertyName("changed")] + public bool Changed { get; init; } +} + +public sealed record PolicyPreviewIssueDto +{ + [JsonPropertyName("code")] + public string Code { get; init; } = string.Empty; + + [JsonPropertyName("message")] + public string Message { get; init; } = string.Empty; + + [JsonPropertyName("severity")] + public string Severity { get; init; } = string.Empty; + + [JsonPropertyName("path")] + public string Path { get; init; } = string.Empty; +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/ReportContracts.cs b/src/StellaOps.Scanner.WebService/Contracts/ReportContracts.cs new file mode 100644 index 00000000..f64e1a2f --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/ReportContracts.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record ReportRequestDto +{ + [JsonPropertyName("imageDigest")] + public string? ImageDigest { get; init; } + + [JsonPropertyName("findings")] + public IReadOnlyList? Findings { get; init; } + + [JsonPropertyName("baseline")] + public IReadOnlyList? Baseline { get; init; } +} + +public sealed record ReportResponseDto +{ + [JsonPropertyName("report")] + public ReportDocumentDto Report { get; init; } = new(); + + [JsonPropertyName("dsse")] + public DsseEnvelopeDto? Dsse { get; init; } +} + +public sealed record ReportDocumentDto +{ + [JsonPropertyName("reportId")] + [JsonPropertyOrder(0)] + public string ReportId { get; init; } = string.Empty; + + [JsonPropertyName("imageDigest")] + [JsonPropertyOrder(1)] + public string ImageDigest { get; init; } = string.Empty; + + [JsonPropertyName("generatedAt")] + [JsonPropertyOrder(2)] + public DateTimeOffset GeneratedAt { get; init; } + + [JsonPropertyName("verdict")] + [JsonPropertyOrder(3)] + public string Verdict { get; init; } = string.Empty; + + [JsonPropertyName("policy")] + [JsonPropertyOrder(4)] + public ReportPolicyDto Policy { get; init; } = new(); + + [JsonPropertyName("summary")] + [JsonPropertyOrder(5)] + public ReportSummaryDto Summary { get; init; } = new(); + + [JsonPropertyName("verdicts")] + [JsonPropertyOrder(6)] + public IReadOnlyList Verdicts { get; init; } = Array.Empty(); + + [JsonPropertyName("issues")] + [JsonPropertyOrder(7)] + public IReadOnlyList Issues { get; init; } = Array.Empty(); +} + +public sealed record ReportPolicyDto +{ + [JsonPropertyName("revisionId")] + [JsonPropertyOrder(0)] + public string? RevisionId { get; init; } + + [JsonPropertyName("digest")] + [JsonPropertyOrder(1)] + public string? Digest { get; init; } +} + +public sealed record ReportSummaryDto +{ + [JsonPropertyName("total")] + [JsonPropertyOrder(0)] + public int Total { get; init; } + + [JsonPropertyName("blocked")] + [JsonPropertyOrder(1)] + public int Blocked { get; init; } + + [JsonPropertyName("warned")] + [JsonPropertyOrder(2)] + public int Warned { get; init; } + + [JsonPropertyName("ignored")] + [JsonPropertyOrder(3)] + public int Ignored { get; init; } + + [JsonPropertyName("quieted")] + [JsonPropertyOrder(4)] + public int Quieted { get; init; } +} + +public sealed record DsseEnvelopeDto +{ + [JsonPropertyName("payloadType")] + [JsonPropertyOrder(0)] + public string PayloadType { get; init; } = string.Empty; + + [JsonPropertyName("payload")] + [JsonPropertyOrder(1)] + public string Payload { get; init; } = string.Empty; + + [JsonPropertyName("signatures")] + [JsonPropertyOrder(2)] + public IReadOnlyList Signatures { get; init; } = Array.Empty(); +} + +public sealed record DsseSignatureDto +{ + [JsonPropertyName("keyId")] + public string KeyId { get; init; } = string.Empty; + + [JsonPropertyName("algorithm")] + public string Algorithm { get; init; } = string.Empty; + + [JsonPropertyName("signature")] + public string Signature { get; init; } = string.Empty; +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/ScanStatusResponse.cs b/src/StellaOps.Scanner.WebService/Contracts/ScanStatusResponse.cs new file mode 100644 index 00000000..9eb05279 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/ScanStatusResponse.cs @@ -0,0 +1,13 @@ +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record ScanStatusResponse( + string ScanId, + string Status, + ScanStatusTarget Image, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + string? FailureReason); + +public sealed record ScanStatusTarget( + string? Reference, + string? Digest); diff --git a/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitRequest.cs b/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitRequest.cs new file mode 100644 index 00000000..725b7e14 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitRequest.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; + +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record ScanSubmitRequest +{ + public required ScanImageDescriptor Image { get; init; } = new(); + + public bool Force { get; init; } + + public string? ClientRequestId { get; init; } + + public IDictionary Metadata { get; init; } = new Dictionary(StringComparer.OrdinalIgnoreCase); +} + +public sealed record ScanImageDescriptor +{ + public string? Reference { get; init; } + + public string? Digest { get; init; } +} diff --git a/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitResponse.cs b/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitResponse.cs new file mode 100644 index 00000000..d486fd42 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Contracts/ScanSubmitResponse.cs @@ -0,0 +1,7 @@ +namespace StellaOps.Scanner.WebService.Contracts; + +public sealed record ScanSubmitResponse( + string ScanId, + string Status, + string? Location, + bool Created); diff --git a/src/StellaOps.Scanner.WebService/Diagnostics/ServiceStatus.cs b/src/StellaOps.Scanner.WebService/Diagnostics/ServiceStatus.cs new file mode 100644 index 00000000..8b978fa6 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Diagnostics/ServiceStatus.cs @@ -0,0 +1,47 @@ +using System; + +namespace StellaOps.Scanner.WebService.Diagnostics; + +/// +/// Tracks runtime health snapshots for the Scanner WebService. +/// +public sealed class ServiceStatus +{ + private readonly TimeProvider timeProvider; + private readonly DateTimeOffset startedAt; + private ReadySnapshot readySnapshot; + + public ServiceStatus(TimeProvider timeProvider) + { + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + startedAt = timeProvider.GetUtcNow(); + readySnapshot = ReadySnapshot.CreateInitial(startedAt); + } + + public ServiceSnapshot CreateSnapshot() + { + var now = timeProvider.GetUtcNow(); + return new ServiceSnapshot(startedAt, now, readySnapshot); + } + + public void RecordReadyCheck(bool success, TimeSpan latency, string? error) + { + var now = timeProvider.GetUtcNow(); + readySnapshot = new ReadySnapshot(now, latency, success, success ? null : error); + } + + public readonly record struct ServiceSnapshot( + DateTimeOffset StartedAt, + DateTimeOffset CapturedAt, + ReadySnapshot Ready); + + public readonly record struct ReadySnapshot( + DateTimeOffset CheckedAt, + TimeSpan? Latency, + bool IsReady, + string? Error) + { + public static ReadySnapshot CreateInitial(DateTimeOffset timestamp) + => new ReadySnapshot(timestamp, null, true, null); + } +} diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanId.cs b/src/StellaOps.Scanner.WebService/Domain/ScanId.cs new file mode 100644 index 00000000..c80ed9fb --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanId.cs @@ -0,0 +1,18 @@ +namespace StellaOps.Scanner.WebService.Domain; + +public readonly record struct ScanId(string Value) +{ + public override string ToString() => Value; + + public static bool TryParse(string? value, out ScanId scanId) + { + if (!string.IsNullOrWhiteSpace(value)) + { + scanId = new ScanId(value.Trim()); + return true; + } + + scanId = default; + return false; + } +} diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanProgressEvent.cs b/src/StellaOps.Scanner.WebService/Domain/ScanProgressEvent.cs new file mode 100644 index 00000000..fae0693e --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanProgressEvent.cs @@ -0,0 +1,12 @@ +using System.Collections.Generic; + +namespace StellaOps.Scanner.WebService.Domain; + +public sealed record ScanProgressEvent( + ScanId ScanId, + int Sequence, + DateTimeOffset Timestamp, + string State, + string? Message, + string CorrelationId, + IReadOnlyDictionary Data); diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanSnapshot.cs b/src/StellaOps.Scanner.WebService/Domain/ScanSnapshot.cs new file mode 100644 index 00000000..6b5cec3d --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanSnapshot.cs @@ -0,0 +1,9 @@ +namespace StellaOps.Scanner.WebService.Domain; + +public sealed record ScanSnapshot( + ScanId ScanId, + ScanTarget Target, + ScanStatus Status, + DateTimeOffset CreatedAt, + DateTimeOffset UpdatedAt, + string? FailureReason); diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanStatus.cs b/src/StellaOps.Scanner.WebService/Domain/ScanStatus.cs new file mode 100644 index 00000000..6dc31b82 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanStatus.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Scanner.WebService.Domain; + +public enum ScanStatus +{ + Pending, + Running, + Succeeded, + Failed, + Cancelled +} diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanSubmission.cs b/src/StellaOps.Scanner.WebService/Domain/ScanSubmission.cs new file mode 100644 index 00000000..331e356e --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanSubmission.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace StellaOps.Scanner.WebService.Domain; + +public sealed record ScanSubmission( + ScanTarget Target, + bool Force, + string? ClientRequestId, + IReadOnlyDictionary Metadata); + +public sealed record ScanSubmissionResult( + ScanSnapshot Snapshot, + bool Created); diff --git a/src/StellaOps.Scanner.WebService/Domain/ScanTarget.cs b/src/StellaOps.Scanner.WebService/Domain/ScanTarget.cs new file mode 100644 index 00000000..2c6ef753 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Domain/ScanTarget.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Scanner.WebService.Domain; + +public sealed record ScanTarget(string? Reference, string? Digest) +{ + public ScanTarget Normalize() + { + var normalizedReference = string.IsNullOrWhiteSpace(Reference) ? null : Reference.Trim(); + var normalizedDigest = string.IsNullOrWhiteSpace(Digest) ? null : Digest.Trim().ToLowerInvariant(); + return new ScanTarget(normalizedReference, normalizedDigest); + } +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/HealthEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/HealthEndpoints.cs new file mode 100644 index 00000000..0268f2e9 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Endpoints/HealthEndpoints.cs @@ -0,0 +1,112 @@ +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.WebService.Diagnostics; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Endpoints; + +internal static class HealthEndpoints +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); + + public static void MapHealthEndpoints(this IEndpointRouteBuilder endpoints) + { + ArgumentNullException.ThrowIfNull(endpoints); + + var group = endpoints.MapGroup("/"); + group.MapGet("/healthz", HandleHealth) + .WithName("scanner.health") + .Produces(StatusCodes.Status200OK) + .AllowAnonymous(); + + group.MapGet("/readyz", HandleReady) + .WithName("scanner.ready") + .Produces(StatusCodes.Status200OK) + .AllowAnonymous(); + } + + private static IResult HandleHealth( + ServiceStatus status, + IOptions options, + HttpContext context) + { + ApplyNoCache(context.Response); + + var snapshot = status.CreateSnapshot(); + var uptimeSeconds = Math.Max((snapshot.CapturedAt - snapshot.StartedAt).TotalSeconds, 0d); + + var telemetry = new TelemetrySnapshot( + Enabled: options.Value.Telemetry.Enabled, + Logging: options.Value.Telemetry.EnableLogging, + Metrics: options.Value.Telemetry.EnableMetrics, + Tracing: options.Value.Telemetry.EnableTracing); + + var document = new HealthDocument( + Status: "healthy", + StartedAt: snapshot.StartedAt, + CapturedAt: snapshot.CapturedAt, + UptimeSeconds: uptimeSeconds, + Telemetry: telemetry); + + return Json(document, StatusCodes.Status200OK); + } + + private static async Task HandleReady( + ServiceStatus status, + HttpContext context, + CancellationToken cancellationToken) + { + ApplyNoCache(context.Response); + + await Task.CompletedTask; + + status.RecordReadyCheck(success: true, latency: TimeSpan.Zero, error: null); + var snapshot = status.CreateSnapshot(); + var ready = snapshot.Ready; + + var document = new ReadyDocument( + Status: ready.IsReady ? "ready" : "unready", + CheckedAt: ready.CheckedAt, + LatencyMs: ready.Latency?.TotalMilliseconds, + Error: ready.Error); + + return Json(document, StatusCodes.Status200OK); + } + + private static void ApplyNoCache(HttpResponse response) + { + response.Headers.CacheControl = "no-store, no-cache, max-age=0, must-revalidate"; + response.Headers.Pragma = "no-cache"; + response.Headers["Expires"] = "0"; + } + + private static IResult Json(T value, int statusCode) + { + var payload = JsonSerializer.Serialize(value, JsonOptions); + return Results.Content(payload, "application/json", Encoding.UTF8, statusCode); + } + + internal sealed record TelemetrySnapshot( + bool Enabled, + bool Logging, + bool Metrics, + bool Tracing); + + internal sealed record HealthDocument( + string Status, + DateTimeOffset StartedAt, + DateTimeOffset CapturedAt, + double UptimeSeconds, + TelemetrySnapshot Telemetry); + + internal sealed record ReadyDocument( + string Status, + DateTimeOffset CheckedAt, + double? LatencyMs, + string? Error); +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs new file mode 100644 index 00000000..06708c0a --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Endpoints/PolicyEndpoints.cs @@ -0,0 +1,175 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Constants; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Infrastructure; +using StellaOps.Scanner.WebService.Security; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Endpoints; + +internal static class PolicyEndpoints +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + public static void MapPolicyEndpoints(this RouteGroupBuilder apiGroup, string policySegment) + { + ArgumentNullException.ThrowIfNull(apiGroup); + + var policyGroup = apiGroup + .MapGroup(NormalizeSegment(policySegment)) + .WithTags("Policy"); + + policyGroup.MapGet("/schema", HandleSchemaAsync) + .WithName("scanner.policy.schema") + .Produces(StatusCodes.Status200OK) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Retrieve the embedded policy JSON schema."; + operation.Description = "Returns the policy schema (`policy-schema@1`) used to validate YAML or JSON rulesets."; + return operation; + }); + + policyGroup.MapPost("/diagnostics", HandleDiagnosticsAsync) + .WithName("scanner.policy.diagnostics") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Run policy diagnostics."; + operation.Description = "Accepts YAML or JSON policy content and returns normalization issues plus recommendations (ignore rules, VEX include/exclude, vendor precedence)."; + return operation; + }); + + policyGroup.MapPost("/preview", HandlePreviewAsync) + .WithName("scanner.policy.preview") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Preview policy impact against findings."; + operation.Description = "Evaluates the supplied findings against the active or proposed policy, returning diffs, quieted verdicts, and actionable validation messages."; + return operation; + }); + } + + private static IResult HandleSchemaAsync(HttpContext context) + { + var schema = PolicySchemaResource.ReadSchemaJson(); + return Results.Text(schema, "application/schema+json", Encoding.UTF8); + } + + private static IResult HandleDiagnosticsAsync( + PolicyDiagnosticsRequestDto request, + TimeProvider timeProvider, + HttpContext context) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(timeProvider); + + if (request.Policy is null || string.IsNullOrWhiteSpace(request.Policy.Content)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid policy diagnostics request", + StatusCodes.Status400BadRequest, + detail: "Policy content is required for diagnostics."); + } + + var format = PolicyDtoMapper.ParsePolicyFormat(request.Policy.Format); + var binding = PolicyBinder.Bind(request.Policy.Content, format); + var diagnostics = PolicyDiagnostics.Create(binding, timeProvider); + + var response = new PolicyDiagnosticsResponseDto + { + Success = diagnostics.ErrorCount == 0, + Version = diagnostics.Version, + RuleCount = diagnostics.RuleCount, + ErrorCount = diagnostics.ErrorCount, + WarningCount = diagnostics.WarningCount, + GeneratedAt = diagnostics.GeneratedAt, + Issues = diagnostics.Issues.Select(PolicyDtoMapper.ToIssueDto).ToImmutableArray(), + Recommendations = diagnostics.Recommendations + }; + + return Json(response); + } + + private static async Task HandlePreviewAsync( + PolicyPreviewRequestDto request, + PolicyPreviewService previewService, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(previewService); + + if (string.IsNullOrWhiteSpace(request.ImageDigest)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid policy preview request", + StatusCodes.Status400BadRequest, + detail: "imageDigest is required."); + } + + if (!request.ImageDigest.Contains(':', StringComparison.Ordinal)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid policy preview request", + StatusCodes.Status400BadRequest, + detail: "imageDigest must include algorithm prefix (e.g. sha256:...)."); + } + + if (request.Findings is not null) + { + var missingIds = request.Findings.Any(f => string.IsNullOrWhiteSpace(f.Id)); + if (missingIds) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid policy preview request", + StatusCodes.Status400BadRequest, + detail: "All findings must include an id value."); + } + } + + var domainRequest = PolicyDtoMapper.ToDomain(request); + var response = await previewService.PreviewAsync(domainRequest, cancellationToken).ConfigureAwait(false); + var payload = PolicyDtoMapper.ToDto(response); + return Json(payload); + } + + private static string NormalizeSegment(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return "/policy"; + } + + var trimmed = segment.Trim('/'); + return "/" + trimmed; + } + + private static IResult Json(T value) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Content(payload, "application/json", Encoding.UTF8); + } +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/ReportEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/ReportEndpoints.cs new file mode 100644 index 00000000..fcc84e1b --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Endpoints/ReportEndpoints.cs @@ -0,0 +1,266 @@ +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Constants; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Infrastructure; +using StellaOps.Scanner.WebService.Security; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Endpoints; + +internal static class ReportEndpoints +{ + private const string PayloadType = "application/vnd.stellaops.report+json"; + + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + Converters = { new JsonStringEnumConverter() } + }; + + public static void MapReportEndpoints(this RouteGroupBuilder apiGroup, string reportsSegment) + { + ArgumentNullException.ThrowIfNull(apiGroup); + + var reports = apiGroup + .MapGroup(NormalizeSegment(reportsSegment)) + .WithTags("Reports"); + + reports.MapPost("/", HandleCreateReportAsync) + .WithName("scanner.reports.create") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status503ServiceUnavailable) + .RequireAuthorization(ScannerPolicies.Reports) + .WithOpenApi(operation => + { + operation.Summary = "Assemble a signed scan report."; + operation.Description = "Aggregates latest findings with the active policy snapshot, returning verdicts plus an optional DSSE envelope."; + return operation; + }); + } + + private static async Task HandleCreateReportAsync( + ReportRequestDto request, + PolicyPreviewService previewService, + IReportSigner signer, + TimeProvider timeProvider, + IReportEventDispatcher eventDispatcher, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(previewService); + ArgumentNullException.ThrowIfNull(signer); + ArgumentNullException.ThrowIfNull(timeProvider); + ArgumentNullException.ThrowIfNull(eventDispatcher); + + if (string.IsNullOrWhiteSpace(request.ImageDigest)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid report request", + StatusCodes.Status400BadRequest, + detail: "imageDigest is required."); + } + + if (!request.ImageDigest.Contains(':', StringComparison.Ordinal)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid report request", + StatusCodes.Status400BadRequest, + detail: "imageDigest must include algorithm prefix (e.g. sha256:...)."); + } + + if (request.Findings is not null && request.Findings.Any(f => string.IsNullOrWhiteSpace(f.Id))) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid report request", + StatusCodes.Status400BadRequest, + detail: "All findings must include an id value."); + } + + var previewDto = new PolicyPreviewRequestDto + { + ImageDigest = request.ImageDigest, + Findings = request.Findings, + Baseline = request.Baseline, + Policy = null + }; + + var domainRequest = PolicyDtoMapper.ToDomain(previewDto) with { ProposedPolicy = null }; + var preview = await previewService.PreviewAsync(domainRequest, cancellationToken).ConfigureAwait(false); + + if (!preview.Success) + { + var issues = preview.Issues.Select(PolicyDtoMapper.ToIssueDto).ToArray(); + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["issues"] = issues + }; + + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Unable to assemble report", + StatusCodes.Status503ServiceUnavailable, + detail: "No policy snapshot is available or validation failed.", + extensions: extensions); + } + + var projectedVerdicts = preview.Diffs + .Select(diff => PolicyDtoMapper.ToVerdictDto(diff.Projected)) + .ToArray(); + + var issuesDto = preview.Issues.Select(PolicyDtoMapper.ToIssueDto).ToArray(); + var summary = BuildSummary(projectedVerdicts); + var verdict = ComputeVerdict(projectedVerdicts); + var reportId = CreateReportId(request.ImageDigest!, preview.PolicyDigest); + var generatedAt = timeProvider.GetUtcNow(); + + var document = new ReportDocumentDto + { + ReportId = reportId, + ImageDigest = request.ImageDigest!, + GeneratedAt = generatedAt, + Verdict = verdict, + Policy = new ReportPolicyDto + { + RevisionId = preview.RevisionId, + Digest = preview.PolicyDigest + }, + Summary = summary, + Verdicts = projectedVerdicts, + Issues = issuesDto + }; + + var payloadBytes = JsonSerializer.SerializeToUtf8Bytes(document, SerializerOptions); + var signature = signer.Sign(payloadBytes); + DsseEnvelopeDto? envelope = null; + if (signature is not null) + { + envelope = new DsseEnvelopeDto + { + PayloadType = PayloadType, + Payload = Convert.ToBase64String(payloadBytes), + Signatures = new[] + { + new DsseSignatureDto + { + KeyId = signature.KeyId, + Algorithm = signature.Algorithm, + Signature = signature.Signature + } + } + }; + } + + var response = new ReportResponseDto + { + Report = document, + Dsse = envelope + }; + + await eventDispatcher + .PublishAsync(request, preview, document, envelope, context, cancellationToken) + .ConfigureAwait(false); + + return Json(response); + } + + private static ReportSummaryDto BuildSummary(IReadOnlyList verdicts) + { + if (verdicts.Count == 0) + { + return new ReportSummaryDto { Total = 0 }; + } + + var blocked = verdicts.Count(v => string.Equals(v.Status, nameof(PolicyVerdictStatus.Blocked), StringComparison.OrdinalIgnoreCase)); + var warned = verdicts.Count(v => + string.Equals(v.Status, nameof(PolicyVerdictStatus.Warned), StringComparison.OrdinalIgnoreCase) + || string.Equals(v.Status, nameof(PolicyVerdictStatus.Deferred), StringComparison.OrdinalIgnoreCase) + || string.Equals(v.Status, nameof(PolicyVerdictStatus.RequiresVex), StringComparison.OrdinalIgnoreCase) + || string.Equals(v.Status, nameof(PolicyVerdictStatus.Escalated), StringComparison.OrdinalIgnoreCase)); + var ignored = verdicts.Count(v => string.Equals(v.Status, nameof(PolicyVerdictStatus.Ignored), StringComparison.OrdinalIgnoreCase)); + var quieted = verdicts.Count(v => v.Quiet is true); + + return new ReportSummaryDto + { + Total = verdicts.Count, + Blocked = blocked, + Warned = warned, + Ignored = ignored, + Quieted = quieted + }; + } + + private static string ComputeVerdict(IReadOnlyList verdicts) + { + if (verdicts.Count == 0) + { + return "unknown"; + } + + if (verdicts.Any(v => string.Equals(v.Status, nameof(PolicyVerdictStatus.Blocked), StringComparison.OrdinalIgnoreCase))) + { + return "blocked"; + } + + if (verdicts.Any(v => string.Equals(v.Status, nameof(PolicyVerdictStatus.Escalated), StringComparison.OrdinalIgnoreCase))) + { + return "escalated"; + } + + if (verdicts.Any(v => + string.Equals(v.Status, nameof(PolicyVerdictStatus.Warned), StringComparison.OrdinalIgnoreCase) + || string.Equals(v.Status, nameof(PolicyVerdictStatus.Deferred), StringComparison.OrdinalIgnoreCase) + || string.Equals(v.Status, nameof(PolicyVerdictStatus.RequiresVex), StringComparison.OrdinalIgnoreCase))) + { + return "warn"; + } + + return "pass"; + } + + private static string CreateReportId(string imageDigest, string policyDigest) + { + var builder = new StringBuilder(); + builder.Append(imageDigest.Trim()); + builder.Append('|'); + builder.Append(policyDigest ?? string.Empty); + + using var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(builder.ToString())); + var hex = Convert.ToHexString(hash.AsSpan(0, 10)).ToLowerInvariant(); + return $"report-{hex}"; + } + + private static string NormalizeSegment(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return "/reports"; + } + + var trimmed = segment.Trim('/'); + return "/" + trimmed; + } + + private static IResult Json(T value) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Content(payload, "application/json", Encoding.UTF8); + } +} diff --git a/src/StellaOps.Scanner.WebService/Endpoints/ScanEndpoints.cs b/src/StellaOps.Scanner.WebService/Endpoints/ScanEndpoints.cs new file mode 100644 index 00000000..06e5f416 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Endpoints/ScanEndpoints.cs @@ -0,0 +1,309 @@ +using System.Collections.Generic; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using System.Text; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using StellaOps.Scanner.WebService.Constants; +using StellaOps.Scanner.WebService.Contracts; +using StellaOps.Scanner.WebService.Domain; +using StellaOps.Scanner.WebService.Infrastructure; +using StellaOps.Scanner.WebService.Security; +using StellaOps.Scanner.WebService.Services; + +namespace StellaOps.Scanner.WebService.Endpoints; + +internal static class ScanEndpoints +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + Converters = { new JsonStringEnumConverter() } + }; + + public static void MapScanEndpoints(this RouteGroupBuilder apiGroup, string scansSegment) + { + ArgumentNullException.ThrowIfNull(apiGroup); + + var scans = apiGroup.MapGroup(NormalizeSegment(scansSegment)); + + scans.MapPost("/", HandleSubmitAsync) + .WithName("scanner.scans.submit") + .Produces(StatusCodes.Status202Accepted) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status409Conflict) + .RequireAuthorization(ScannerPolicies.ScansEnqueue); + + scans.MapGet("/{scanId}", HandleStatusAsync) + .WithName("scanner.scans.status") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(ScannerPolicies.ScansRead); + + scans.MapGet("/{scanId}/events", HandleProgressStreamAsync) + .WithName("scanner.scans.events") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status404NotFound) + .RequireAuthorization(ScannerPolicies.ScansRead); + } + + private static async Task HandleSubmitAsync( + ScanSubmitRequest request, + IScanCoordinator coordinator, + LinkGenerator links, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(coordinator); + ArgumentNullException.ThrowIfNull(links); + + if (request.Image is null) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid scan submission", + StatusCodes.Status400BadRequest, + detail: "Request image descriptor is required."); + } + + var reference = request.Image.Reference; + var digest = request.Image.Digest; + if (string.IsNullOrWhiteSpace(reference) && string.IsNullOrWhiteSpace(digest)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid scan submission", + StatusCodes.Status400BadRequest, + detail: "Either image.reference or image.digest must be provided."); + } + + if (!string.IsNullOrWhiteSpace(digest) && !digest.Contains(':', StringComparison.Ordinal)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid scan submission", + StatusCodes.Status400BadRequest, + detail: "Image digest must include algorithm prefix (e.g. sha256:...)."); + } + + var target = new ScanTarget(reference, digest).Normalize(); + var metadata = NormalizeMetadata(request.Metadata); + var submission = new ScanSubmission( + Target: target, + Force: request.Force, + ClientRequestId: request.ClientRequestId?.Trim(), + Metadata: metadata); + + ScanSubmissionResult result; + try + { + result = await coordinator.SubmitAsync(submission, context.RequestAborted).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + + var statusText = result.Snapshot.Status.ToString(); + var location = links.GetPathByName( + httpContext: context, + endpointName: "scanner.scans.status", + values: new { scanId = result.Snapshot.ScanId.Value }); + + if (!string.IsNullOrWhiteSpace(location)) + { + context.Response.Headers.Location = location; + } + + var response = new ScanSubmitResponse( + ScanId: result.Snapshot.ScanId.Value, + Status: statusText, + Location: location, + Created: result.Created); + + return Json(response, StatusCodes.Status202Accepted); + } + + private static async Task HandleStatusAsync( + string scanId, + IScanCoordinator coordinator, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(coordinator); + + if (!ScanId.TryParse(scanId, out var parsed)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid scan identifier", + StatusCodes.Status400BadRequest, + detail: "Scan identifier is required."); + } + + var snapshot = await coordinator.GetAsync(parsed, context.RequestAborted).ConfigureAwait(false); + if (snapshot is null) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.NotFound, + "Scan not found", + StatusCodes.Status404NotFound, + detail: "Requested scan could not be located."); + } + + var response = new ScanStatusResponse( + ScanId: snapshot.ScanId.Value, + Status: snapshot.Status.ToString(), + Image: new ScanStatusTarget(snapshot.Target.Reference, snapshot.Target.Digest), + CreatedAt: snapshot.CreatedAt, + UpdatedAt: snapshot.UpdatedAt, + FailureReason: snapshot.FailureReason); + + return Json(response, StatusCodes.Status200OK); + } + + private static async Task HandleProgressStreamAsync( + string scanId, + string? format, + IScanProgressReader progressReader, + HttpContext context, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(progressReader); + + if (!ScanId.TryParse(scanId, out var parsed)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.Validation, + "Invalid scan identifier", + StatusCodes.Status400BadRequest, + detail: "Scan identifier is required."); + } + + if (!progressReader.Exists(parsed)) + { + return ProblemResultFactory.Create( + context, + ProblemTypes.NotFound, + "Scan not found", + StatusCodes.Status404NotFound, + detail: "Requested scan could not be located."); + } + + var streamFormat = string.Equals(format, "jsonl", StringComparison.OrdinalIgnoreCase) + ? "jsonl" + : "sse"; + + context.Response.StatusCode = StatusCodes.Status200OK; + context.Response.Headers.CacheControl = "no-store"; + context.Response.Headers["X-Accel-Buffering"] = "no"; + context.Response.Headers["Connection"] = "keep-alive"; + + if (streamFormat == "jsonl") + { + context.Response.ContentType = "application/x-ndjson"; + } + else + { + context.Response.ContentType = "text/event-stream"; + } + + await foreach (var progressEvent in progressReader.SubscribeAsync(parsed, context.RequestAborted).WithCancellation(context.RequestAborted)) + { + var payload = new + { + scanId = progressEvent.ScanId.Value, + sequence = progressEvent.Sequence, + state = progressEvent.State, + message = progressEvent.Message, + timestamp = progressEvent.Timestamp, + correlationId = progressEvent.CorrelationId, + data = progressEvent.Data + }; + + if (streamFormat == "jsonl") + { + await WriteJsonLineAsync(context.Response.BodyWriter, payload, cancellationToken).ConfigureAwait(false); + } + else + { + await WriteSseAsync(context.Response.BodyWriter, payload, progressEvent, cancellationToken).ConfigureAwait(false); + } + + await context.Response.BodyWriter.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + return Results.Empty; + } + + private static IReadOnlyDictionary NormalizeMetadata(IDictionary metadata) + { + if (metadata is null || metadata.Count == 0) + { + return new Dictionary(); + } + + var normalized = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var pair in metadata) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + var key = pair.Key.Trim(); + var value = pair.Value?.Trim() ?? string.Empty; + normalized[key] = value; + } + + return normalized; + } + + private static async Task WriteJsonLineAsync(PipeWriter writer, object payload, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(payload, SerializerOptions); + var jsonBytes = Encoding.UTF8.GetBytes(json); + await writer.WriteAsync(jsonBytes, cancellationToken).ConfigureAwait(false); + await writer.WriteAsync(new[] { (byte)'\n' }, cancellationToken).ConfigureAwait(false); + } + + private static async Task WriteSseAsync(PipeWriter writer, object payload, ScanProgressEvent progressEvent, CancellationToken cancellationToken) + { + var json = JsonSerializer.Serialize(payload, SerializerOptions); + var eventName = progressEvent.State.ToLowerInvariant(); + var builder = new StringBuilder(); + builder.Append("id: ").Append(progressEvent.Sequence).Append('\n'); + builder.Append("event: ").Append(eventName).Append('\n'); + builder.Append("data: ").Append(json).Append('\n'); + builder.Append('\n'); + + var bytes = Encoding.UTF8.GetBytes(builder.ToString()); + await writer.WriteAsync(bytes, cancellationToken).ConfigureAwait(false); + } + + private static IResult Json(T value, int statusCode) + { + var payload = JsonSerializer.Serialize(value, SerializerOptions); + return Results.Content(payload, "application/json", System.Text.Encoding.UTF8, statusCode); + } + + private static string NormalizeSegment(string segment) + { + if (string.IsNullOrWhiteSpace(segment)) + { + return "/scans"; + } + + var trimmed = segment.Trim('/'); + return "/" + trimmed; + } +} diff --git a/src/StellaOps.Scanner.WebService/Extensions/ConfigurationExtensions.cs b/src/StellaOps.Scanner.WebService/Extensions/ConfigurationExtensions.cs new file mode 100644 index 00000000..2d1503fc --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Extensions/ConfigurationExtensions.cs @@ -0,0 +1,38 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Configuration; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace StellaOps.Scanner.WebService.Extensions; + +/// +/// Scanner-specific configuration helpers. +/// +public static class ConfigurationExtensions +{ + public static IConfigurationBuilder AddScannerYaml(this IConfigurationBuilder builder, string path) + { + ArgumentNullException.ThrowIfNull(builder); + + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + { + return builder; + } + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + using var reader = File.OpenText(path); + var yamlObject = deserializer.Deserialize(reader); + if (yamlObject is null) + { + return builder; + } + + var payload = JsonSerializer.Serialize(yamlObject); + var stream = new MemoryStream(Encoding.UTF8.GetBytes(payload)); + return builder.AddJsonStream(stream); + } +} diff --git a/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs b/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs new file mode 100644 index 00000000..5359e73c --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Extensions/OpenApiRegistrationExtensions.cs @@ -0,0 +1,58 @@ +using System.Linq; +using System.Reflection; +using Microsoft.AspNetCore.Builder; +using Microsoft.Extensions.DependencyInjection; + +namespace StellaOps.Scanner.WebService.Extensions; + +internal static class OpenApiRegistrationExtensions +{ + public static IServiceCollection AddOpenApiIfAvailable(this IServiceCollection services) + { + ArgumentNullException.ThrowIfNull(services); + + var extensionType = Type.GetType("Microsoft.Extensions.DependencyInjection.OpenApiServiceCollectionExtensions, Microsoft.AspNetCore.OpenApi"); + if (extensionType is not null) + { + var method = extensionType + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + string.Equals(m.Name, "AddOpenApi", StringComparison.Ordinal) && + m.GetParameters().Length == 2); + + if (method is not null) + { + var result = method.Invoke(null, new object?[] { services, null }); + if (result is IServiceCollection collection) + { + return collection; + } + } + } + + services.AddEndpointsApiExplorer(); + return services; + } + + public static WebApplication MapOpenApiIfAvailable(this WebApplication app) + { + ArgumentNullException.ThrowIfNull(app); + + var extensionType = Type.GetType("Microsoft.AspNetCore.Builder.OpenApiApplicationBuilderExtensions, Microsoft.AspNetCore.OpenApi"); + if (extensionType is not null) + { + var method = extensionType + .GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + string.Equals(m.Name, "MapOpenApi", StringComparison.Ordinal) && + m.GetParameters().Length == 1); + + if (method is not null) + { + method.Invoke(null, new object?[] { app }); + } + } + + return app; + } +} diff --git a/src/StellaOps.Scanner.WebService/Hosting/ScannerPluginHostFactory.cs b/src/StellaOps.Scanner.WebService/Hosting/ScannerPluginHostFactory.cs new file mode 100644 index 00000000..0249b7a6 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Hosting/ScannerPluginHostFactory.cs @@ -0,0 +1,55 @@ +using System; +using System.IO; +using StellaOps.Plugin.Hosting; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Hosting; + +internal static class ScannerPluginHostFactory +{ + public static PluginHostOptions Build(ScannerWebServiceOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(contentRootPath); + + var baseDirectory = options.Plugins.BaseDirectory; + if (string.IsNullOrWhiteSpace(baseDirectory)) + { + baseDirectory = Path.Combine(contentRootPath, ".."); + } + else if (!Path.IsPathRooted(baseDirectory)) + { + baseDirectory = Path.GetFullPath(Path.Combine(contentRootPath, baseDirectory)); + } + + var pluginsDirectory = options.Plugins.Directory; + if (string.IsNullOrWhiteSpace(pluginsDirectory)) + { + pluginsDirectory = Path.Combine("plugins", "scanner"); + } + + if (!Path.IsPathRooted(pluginsDirectory)) + { + pluginsDirectory = Path.Combine(baseDirectory, pluginsDirectory); + } + + var hostOptions = new PluginHostOptions + { + BaseDirectory = baseDirectory, + PluginsDirectory = pluginsDirectory, + PrimaryPrefix = "StellaOps.Scanner" + }; + + foreach (var additionalPrefix in options.Plugins.OrderedPlugins) + { + hostOptions.PluginOrder.Add(additionalPrefix); + } + + foreach (var pattern in options.Plugins.SearchPatterns) + { + hostOptions.SearchPatterns.Add(pattern); + } + + return hostOptions; + } +} diff --git a/src/StellaOps.Scanner.WebService/Infrastructure/ProblemResultFactory.cs b/src/StellaOps.Scanner.WebService/Infrastructure/ProblemResultFactory.cs new file mode 100644 index 00000000..c9a0a67c --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Infrastructure/ProblemResultFactory.cs @@ -0,0 +1,53 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace StellaOps.Scanner.WebService.Infrastructure; + +internal static class ProblemResultFactory +{ + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + public static IResult Create( + HttpContext context, + string type, + string title, + int statusCode, + string? detail = null, + IDictionary? extensions = null) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentException.ThrowIfNullOrWhiteSpace(type); + ArgumentException.ThrowIfNullOrWhiteSpace(title); + + var traceId = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier; + + var problem = new ProblemDetails + { + Type = type, + Title = title, + Detail = detail, + Status = statusCode, + Instance = context.Request.Path + }; + + problem.Extensions["traceId"] = traceId; + if (extensions is not null) + { + foreach (var entry in extensions) + { + problem.Extensions[entry.Key] = entry.Value; + } + } + + var payload = JsonSerializer.Serialize(problem, JsonOptions); + return Results.Content(payload, "application/problem+json", Encoding.UTF8, statusCode); + } +} diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs new file mode 100644 index 00000000..29b732ab --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptions.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Scanner.WebService.Options; + +/// +/// Strongly typed configuration for the Scanner WebService host. +/// +public sealed class ScannerWebServiceOptions +{ + public const string SectionName = "scanner"; + + /// + /// Schema version for configuration consumers to coordinate breaking changes. + /// + public int SchemaVersion { get; set; } = 1; + + /// + /// Mongo storage configuration used for catalog and job state. + /// + public StorageOptions Storage { get; set; } = new(); + + /// + /// Queue configuration used to enqueue scan jobs. + /// + public QueueOptions Queue { get; set; } = new(); + + /// + /// Object store configuration for SBOM artefacts. + /// + public ArtifactStoreOptions ArtifactStore { get; set; } = new(); + + /// + /// Feature flags toggling optional behaviours. + /// + public FeatureFlagOptions Features { get; set; } = new(); + + /// + /// Plug-in loader configuration. + /// + public PluginOptions Plugins { get; set; } = new(); + + /// + /// Telemetry configuration for logs, metrics, traces. + /// + public TelemetryOptions Telemetry { get; set; } = new(); + + /// + /// Authority / authentication configuration. + /// + public AuthorityOptions Authority { get; set; } = new(); + + /// + /// Signing configuration for report envelopes and attestations. + /// + public SigningOptions Signing { get; set; } = new(); + + /// + /// API-specific settings such as base path. + /// + public ApiOptions Api { get; set; } = new(); + + /// + /// Platform event emission settings. + /// + public EventsOptions Events { get; set; } = new(); + + public sealed class StorageOptions + { + public string Driver { get; set; } = "mongo"; + + public string Dsn { get; set; } = string.Empty; + + public string? Database { get; set; } + + public int CommandTimeoutSeconds { get; set; } = 30; + + public int HealthCheckTimeoutSeconds { get; set; } = 5; + + public IList Migrations { get; set; } = new List(); + } + + public sealed class QueueOptions + { + public string Driver { get; set; } = "redis"; + + public string Dsn { get; set; } = string.Empty; + + public string Namespace { get; set; } = "scanner"; + + public int VisibilityTimeoutSeconds { get; set; } = 300; + + public int LeaseHeartbeatSeconds { get; set; } = 30; + + public int MaxDeliveryAttempts { get; set; } = 5; + + public IDictionary DriverSettings { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class ArtifactStoreOptions + { + public string Driver { get; set; } = "minio"; + + public string Endpoint { get; set; } = string.Empty; + + public bool UseTls { get; set; } = true; + + public string AccessKey { get; set; } = string.Empty; + + public string SecretKey { get; set; } = string.Empty; + + public string? SecretKeyFile { get; set; } + + public string Bucket { get; set; } = "scanner-artifacts"; + + public string? Region { get; set; } + + public bool EnableObjectLock { get; set; } = true; + + public int ObjectLockRetentionDays { get; set; } = 30; + + public IDictionary Headers { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class FeatureFlagOptions + { + public bool AllowAnonymousScanSubmission { get; set; } + + public bool EnableSignedReports { get; set; } = true; + + public bool EnablePolicyPreview { get; set; } = true; + + public IDictionary Experimental { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class PluginOptions + { + public string? BaseDirectory { get; set; } + + public string? Directory { get; set; } + + public IList SearchPatterns { get; set; } = new List(); + + public IList OrderedPlugins { get; set; } = new List(); + } + + public sealed class TelemetryOptions + { + public bool Enabled { get; set; } = true; + + public bool EnableTracing { get; set; } = true; + + public bool EnableMetrics { get; set; } = true; + + public bool EnableLogging { get; set; } = true; + + public bool EnableRequestLogging { get; set; } = true; + + public string MinimumLogLevel { get; set; } = "Information"; + + public string? ServiceName { get; set; } + + public string? OtlpEndpoint { get; set; } + + public IDictionary OtlpHeaders { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public IDictionary ResourceAttributes { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class AuthorityOptions + { + public bool Enabled { get; set; } + + public bool AllowAnonymousFallback { get; set; } = true; + + public string Issuer { get; set; } = string.Empty; + + public string? MetadataAddress { get; set; } + + public bool RequireHttpsMetadata { get; set; } = true; + + public int BackchannelTimeoutSeconds { get; set; } = 30; + + public int TokenClockSkewSeconds { get; set; } = 60; + + public IList Audiences { get; set; } = new List(); + + public IList RequiredScopes { get; set; } = new List(); + + public IList BypassNetworks { get; set; } = new List(); + + public string? ClientId { get; set; } + + public string? ClientSecret { get; set; } + + public string? ClientSecretFile { get; set; } + + public IList ClientScopes { get; set; } = new List(); + + public ResilienceOptions Resilience { get; set; } = new(); + + public sealed class ResilienceOptions + { + public bool? EnableRetries { get; set; } + + public IList RetryDelays { get; set; } = new List(); + + public bool? AllowOfflineCacheFallback { get; set; } + + public TimeSpan? OfflineCacheTolerance { get; set; } + } + } + + public sealed class SigningOptions + { + public bool Enabled { get; set; } = false; + + public string KeyId { get; set; } = string.Empty; + + public string Algorithm { get; set; } = "ed25519"; + + public string? Provider { get; set; } + + public string? KeyPem { get; set; } + + public string? KeyPemFile { get; set; } + + public string? CertificatePem { get; set; } + + public string? CertificatePemFile { get; set; } + + public string? CertificateChainPem { get; set; } + + public string? CertificateChainPemFile { get; set; } + + public int EnvelopeTtlSeconds { get; set; } = 600; + } + + public sealed class ApiOptions + { + public string BasePath { get; set; } = "/api/v1"; + + public string ScansSegment { get; set; } = "scans"; + + public string ReportsSegment { get; set; } = "reports"; + + public string PolicySegment { get; set; } = "policy"; + } + + public sealed class EventsOptions + { + public bool Enabled { get; set; } + + public string Driver { get; set; } = "redis"; + + public string Dsn { get; set; } = string.Empty; + + public string Stream { get; set; } = "stella.events"; + + public double PublishTimeoutSeconds { get; set; } = 5; + + public long MaxStreamLength { get; set; } = 10000; + + public IDictionary DriverSettings { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + } +} diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs new file mode 100644 index 00000000..d2169dd0 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsPostConfigure.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace StellaOps.Scanner.WebService.Options; + +/// +/// Post-configuration helpers for . +/// +public static class ScannerWebServiceOptionsPostConfigure +{ + public static void Apply(ScannerWebServiceOptions options, string contentRootPath) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(contentRootPath); + + options.Plugins ??= new ScannerWebServiceOptions.PluginOptions(); + if (string.IsNullOrWhiteSpace(options.Plugins.Directory)) + { + options.Plugins.Directory = Path.Combine("plugins", "scanner"); + } + + options.Authority ??= new ScannerWebServiceOptions.AuthorityOptions(); + var authority = options.Authority; + if (string.IsNullOrWhiteSpace(authority.ClientSecret) + && !string.IsNullOrWhiteSpace(authority.ClientSecretFile)) + { + authority.ClientSecret = ReadSecretFile(authority.ClientSecretFile!, contentRootPath); + } + + options.ArtifactStore ??= new ScannerWebServiceOptions.ArtifactStoreOptions(); + var artifactStore = options.ArtifactStore; + if (string.IsNullOrWhiteSpace(artifactStore.SecretKey) + && !string.IsNullOrWhiteSpace(artifactStore.SecretKeyFile)) + { + artifactStore.SecretKey = ReadSecretFile(artifactStore.SecretKeyFile!, contentRootPath); + } + + options.Signing ??= new ScannerWebServiceOptions.SigningOptions(); + var signing = options.Signing; + if (string.IsNullOrWhiteSpace(signing.KeyPem) + && !string.IsNullOrWhiteSpace(signing.KeyPemFile)) + { + signing.KeyPem = ReadAllText(signing.KeyPemFile!, contentRootPath); + } + + if (string.IsNullOrWhiteSpace(signing.CertificatePem) + && !string.IsNullOrWhiteSpace(signing.CertificatePemFile)) + { + signing.CertificatePem = ReadAllText(signing.CertificatePemFile!, contentRootPath); + } + + if (string.IsNullOrWhiteSpace(signing.CertificateChainPem) + && !string.IsNullOrWhiteSpace(signing.CertificateChainPemFile)) + { + signing.CertificateChainPem = ReadAllText(signing.CertificateChainPemFile!, contentRootPath); + } + + options.Events ??= new ScannerWebServiceOptions.EventsOptions(); + var eventsOptions = options.Events; + eventsOptions.DriverSettings ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (string.IsNullOrWhiteSpace(eventsOptions.Driver)) + { + eventsOptions.Driver = "redis"; + } + + if (string.IsNullOrWhiteSpace(eventsOptions.Stream)) + { + eventsOptions.Stream = "stella.events"; + } + + if (string.IsNullOrWhiteSpace(eventsOptions.Dsn) + && string.Equals(options.Queue?.Driver, "redis", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(options.Queue?.Dsn)) + { + eventsOptions.Dsn = options.Queue!.Dsn; + } + + } + + private static string ReadSecretFile(string path, string contentRootPath) + { + var resolvedPath = ResolvePath(path, contentRootPath); + if (!File.Exists(resolvedPath)) + { + throw new InvalidOperationException($"Secret file '{resolvedPath}' was not found."); + } + + var secret = File.ReadAllText(resolvedPath).Trim(); + if (string.IsNullOrEmpty(secret)) + { + throw new InvalidOperationException($"Secret file '{resolvedPath}' is empty."); + } + + return secret; + } + + private static string ReadAllText(string path, string contentRootPath) + { + var resolvedPath = ResolvePath(path, contentRootPath); + if (!File.Exists(resolvedPath)) + { + throw new InvalidOperationException($"File '{resolvedPath}' was not found."); + } + + return File.ReadAllText(resolvedPath); + } + + private static string ResolvePath(string path, string contentRootPath) + => Path.IsPathRooted(path) + ? path + : Path.GetFullPath(Path.Combine(contentRootPath, path)); +} diff --git a/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs new file mode 100644 index 00000000..10132358 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Options/ScannerWebServiceOptionsValidator.cs @@ -0,0 +1,388 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.WebService.Security; + +namespace StellaOps.Scanner.WebService.Options; + +/// +/// Validation helpers for . +/// +public static class ScannerWebServiceOptionsValidator +{ + private static readonly HashSet SupportedStorageDrivers = new(StringComparer.OrdinalIgnoreCase) + { + "mongo" + }; + + private static readonly HashSet SupportedQueueDrivers = new(StringComparer.OrdinalIgnoreCase) + { + "redis", + "nats", + "rabbitmq" + }; + + private static readonly HashSet SupportedArtifactDrivers = new(StringComparer.OrdinalIgnoreCase) + { + "minio" + }; + + private static readonly HashSet SupportedEventDrivers = new(StringComparer.OrdinalIgnoreCase) + { + "redis" + }; + + public static void Validate(ScannerWebServiceOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + if (options.SchemaVersion <= 0) + { + throw new InvalidOperationException("Scanner configuration requires a positive schemaVersion."); + } + + options.Storage ??= new ScannerWebServiceOptions.StorageOptions(); + ValidateStorage(options.Storage); + + options.Queue ??= new ScannerWebServiceOptions.QueueOptions(); + ValidateQueue(options.Queue); + + options.ArtifactStore ??= new ScannerWebServiceOptions.ArtifactStoreOptions(); + ValidateArtifactStore(options.ArtifactStore); + + options.Features ??= new ScannerWebServiceOptions.FeatureFlagOptions(); + options.Plugins ??= new ScannerWebServiceOptions.PluginOptions(); + options.Telemetry ??= new ScannerWebServiceOptions.TelemetryOptions(); + ValidateTelemetry(options.Telemetry); + + options.Authority ??= new ScannerWebServiceOptions.AuthorityOptions(); + ValidateAuthority(options.Authority); + + options.Signing ??= new ScannerWebServiceOptions.SigningOptions(); + ValidateSigning(options.Signing); + + options.Api ??= new ScannerWebServiceOptions.ApiOptions(); + if (string.IsNullOrWhiteSpace(options.Api.BasePath)) + { + throw new InvalidOperationException("API basePath must be configured."); + } + + if (string.IsNullOrWhiteSpace(options.Api.ScansSegment)) + { + throw new InvalidOperationException("API scansSegment must be configured."); + } + + if (string.IsNullOrWhiteSpace(options.Api.ReportsSegment)) + { + throw new InvalidOperationException("API reportsSegment must be configured."); + } + + if (string.IsNullOrWhiteSpace(options.Api.PolicySegment)) + { + throw new InvalidOperationException("API policySegment must be configured."); + } + + options.Events ??= new ScannerWebServiceOptions.EventsOptions(); + ValidateEvents(options.Events); + } + + private static void ValidateStorage(ScannerWebServiceOptions.StorageOptions storage) + { + if (!SupportedStorageDrivers.Contains(storage.Driver)) + { + throw new InvalidOperationException($"Unsupported storage driver '{storage.Driver}'. Supported drivers: mongo."); + } + + if (string.IsNullOrWhiteSpace(storage.Dsn)) + { + throw new InvalidOperationException("Storage DSN must be configured."); + } + + if (storage.CommandTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Storage commandTimeoutSeconds must be greater than zero."); + } + + if (storage.HealthCheckTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Storage healthCheckTimeoutSeconds must be greater than zero."); + } + } + + private static void ValidateQueue(ScannerWebServiceOptions.QueueOptions queue) + { + if (!SupportedQueueDrivers.Contains(queue.Driver)) + { + throw new InvalidOperationException($"Unsupported queue driver '{queue.Driver}'. Supported drivers: redis, nats, rabbitmq."); + } + + if (string.IsNullOrWhiteSpace(queue.Dsn)) + { + throw new InvalidOperationException("Queue DSN must be configured."); + } + + if (string.IsNullOrWhiteSpace(queue.Namespace)) + { + throw new InvalidOperationException("Queue namespace must be configured."); + } + + if (queue.VisibilityTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Queue visibilityTimeoutSeconds must be greater than zero."); + } + + if (queue.LeaseHeartbeatSeconds <= 0) + { + throw new InvalidOperationException("Queue leaseHeartbeatSeconds must be greater than zero."); + } + + if (queue.MaxDeliveryAttempts <= 0) + { + throw new InvalidOperationException("Queue maxDeliveryAttempts must be greater than zero."); + } + } + + private static void ValidateArtifactStore(ScannerWebServiceOptions.ArtifactStoreOptions artifactStore) + { + if (!SupportedArtifactDrivers.Contains(artifactStore.Driver)) + { + throw new InvalidOperationException($"Unsupported artifact store driver '{artifactStore.Driver}'. Supported drivers: minio."); + } + + if (string.IsNullOrWhiteSpace(artifactStore.Endpoint)) + { + throw new InvalidOperationException("Artifact store endpoint must be configured."); + } + + if (string.IsNullOrWhiteSpace(artifactStore.Bucket)) + { + throw new InvalidOperationException("Artifact store bucket must be configured."); + } + + if (artifactStore.EnableObjectLock && artifactStore.ObjectLockRetentionDays <= 0) + { + throw new InvalidOperationException("Artifact store objectLockRetentionDays must be greater than zero when object lock is enabled."); + } + } + + private static void ValidateEvents(ScannerWebServiceOptions.EventsOptions eventsOptions) + { + if (!eventsOptions.Enabled) + { + return; + } + + if (!SupportedEventDrivers.Contains(eventsOptions.Driver)) + { + throw new InvalidOperationException($"Unsupported events driver '{eventsOptions.Driver}'. Supported drivers: redis."); + } + + if (string.IsNullOrWhiteSpace(eventsOptions.Dsn)) + { + throw new InvalidOperationException("Events DSN must be configured when event emission is enabled."); + } + + if (string.IsNullOrWhiteSpace(eventsOptions.Stream)) + { + throw new InvalidOperationException("Events stream must be configured when event emission is enabled."); + } + + if (eventsOptions.PublishTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Events publishTimeoutSeconds must be greater than zero."); + } + + if (eventsOptions.MaxStreamLength < 0) + { + throw new InvalidOperationException("Events maxStreamLength must be zero or greater."); + } + } + + private static void ValidateTelemetry(ScannerWebServiceOptions.TelemetryOptions telemetry) + { + if (string.IsNullOrWhiteSpace(telemetry.MinimumLogLevel)) + { + throw new InvalidOperationException("Telemetry minimumLogLevel must be configured."); + } + + if (!Enum.TryParse(telemetry.MinimumLogLevel, ignoreCase: true, out LogLevel _)) + { + throw new InvalidOperationException($"Telemetry minimumLogLevel '{telemetry.MinimumLogLevel}' is invalid."); + } + + if (!string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint) && !Uri.TryCreate(telemetry.OtlpEndpoint, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Telemetry OTLP endpoint must be an absolute URI when specified."); + } + + foreach (var attribute in telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(attribute.Key)) + { + throw new InvalidOperationException("Telemetry resource attribute keys must be non-empty."); + } + } + + foreach (var header in telemetry.OtlpHeaders) + { + if (string.IsNullOrWhiteSpace(header.Key)) + { + throw new InvalidOperationException("Telemetry OTLP header keys must be non-empty."); + } + } + } + + private static void ValidateAuthority(ScannerWebServiceOptions.AuthorityOptions authority) + { + authority.Resilience ??= new ScannerWebServiceOptions.AuthorityOptions.ResilienceOptions(); + NormalizeList(authority.Audiences, toLower: false); + NormalizeList(authority.RequiredScopes, toLower: true); + NormalizeList(authority.BypassNetworks, toLower: false); + NormalizeList(authority.ClientScopes, toLower: true); + NormalizeResilience(authority.Resilience); + + if (authority.RequiredScopes.Count == 0) + { + authority.RequiredScopes.Add(ScannerAuthorityScopes.ScansEnqueue); + } + + if (authority.ClientScopes.Count == 0) + { + foreach (var scope in authority.RequiredScopes) + { + authority.ClientScopes.Add(scope); + } + } + + if (authority.BackchannelTimeoutSeconds <= 0) + { + throw new InvalidOperationException("Authority backchannelTimeoutSeconds must be greater than zero."); + } + + if (authority.TokenClockSkewSeconds < 0 || authority.TokenClockSkewSeconds > 300) + { + throw new InvalidOperationException("Authority tokenClockSkewSeconds must be between 0 and 300 seconds."); + } + + if (!authority.Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(authority.Issuer)) + { + throw new InvalidOperationException("Authority issuer must be configured when authority is enabled."); + } + + if (!Uri.TryCreate(authority.Issuer, UriKind.Absolute, out var issuerUri)) + { + throw new InvalidOperationException("Authority issuer must be an absolute URI."); + } + + if (authority.RequireHttpsMetadata && !issuerUri.IsLoopback && !string.Equals(issuerUri.Scheme, Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException("Authority issuer must use HTTPS when requireHttpsMetadata is enabled."); + } + + if (!string.IsNullOrWhiteSpace(authority.MetadataAddress) && !Uri.TryCreate(authority.MetadataAddress, UriKind.Absolute, out _)) + { + throw new InvalidOperationException("Authority metadataAddress must be an absolute URI when specified."); + } + + if (authority.Audiences.Count == 0) + { + throw new InvalidOperationException("Authority audiences must include at least one entry when authority is enabled."); + } + + if (!authority.AllowAnonymousFallback) + { + if (string.IsNullOrWhiteSpace(authority.ClientId)) + { + throw new InvalidOperationException("Authority clientId must be configured when anonymous fallback is disabled."); + } + + if (string.IsNullOrWhiteSpace(authority.ClientSecret)) + { + throw new InvalidOperationException("Authority clientSecret must be configured when anonymous fallback is disabled."); + } + } + } + + private static void ValidateSigning(ScannerWebServiceOptions.SigningOptions signing) + { + if (signing.EnvelopeTtlSeconds <= 0) + { + throw new InvalidOperationException("Signing envelopeTtlSeconds must be greater than zero."); + } + + if (!signing.Enabled) + { + return; + } + + if (string.IsNullOrWhiteSpace(signing.KeyId)) + { + throw new InvalidOperationException("Signing keyId must be configured when signing is enabled."); + } + + if (string.IsNullOrWhiteSpace(signing.Algorithm)) + { + throw new InvalidOperationException("Signing algorithm must be configured when signing is enabled."); + } + + if (string.IsNullOrWhiteSpace(signing.KeyPem) && string.IsNullOrWhiteSpace(signing.KeyPemFile)) + { + throw new InvalidOperationException("Signing requires keyPem or keyPemFile when enabled."); + } + } + + private static void NormalizeList(IList values, bool toLower) + { + if (values is null || values.Count == 0) + { + return; + } + + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = values.Count - 1; i >= 0; i--) + { + var entry = values[i]; + if (string.IsNullOrWhiteSpace(entry)) + { + values.RemoveAt(i); + continue; + } + + var normalized = toLower ? entry.Trim().ToLowerInvariant() : entry.Trim(); + if (!seen.Add(normalized)) + { + values.RemoveAt(i); + continue; + } + + values[i] = normalized; + } + } + + private static void NormalizeResilience(ScannerWebServiceOptions.AuthorityOptions.ResilienceOptions resilience) + { + if (resilience.RetryDelays is null) + { + return; + } + + foreach (var delay in resilience.RetryDelays.ToArray()) + { + if (delay <= TimeSpan.Zero) + { + throw new InvalidOperationException("Authority resilience retryDelays must be greater than zero."); + } + } + + if (resilience.OfflineCacheTolerance.HasValue && resilience.OfflineCacheTolerance.Value < TimeSpan.Zero) + { + throw new InvalidOperationException("Authority resilience offlineCacheTolerance must be greater than or equal to zero."); + } + } +} diff --git a/src/StellaOps.Scanner.WebService/Program.cs b/src/StellaOps.Scanner.WebService/Program.cs new file mode 100644 index 00000000..227ffd52 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Program.cs @@ -0,0 +1,276 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Options; +using Serilog; +using Serilog.Events; +using StellaOps.Auth.Client; +using StellaOps.Auth.ServerIntegration; +using StellaOps.Configuration; +using StellaOps.Plugin.DependencyInjection; +using StellaOps.Cryptography.DependencyInjection; +using StellaOps.Cryptography.Plugin.BouncyCastle; +using StellaOps.Policy; +using StellaOps.Scanner.Cache; +using StellaOps.Scanner.WebService.Diagnostics; +using StellaOps.Scanner.WebService.Endpoints; +using StellaOps.Scanner.WebService.Extensions; +using StellaOps.Scanner.WebService.Hosting; +using StellaOps.Scanner.WebService.Options; +using StellaOps.Scanner.WebService.Services; +using StellaOps.Scanner.WebService.Security; + +var builder = WebApplication.CreateBuilder(args); + +builder.Configuration.AddStellaOpsDefaults(options => +{ + options.BasePath = builder.Environment.ContentRootPath; + options.EnvironmentPrefix = "SCANNER_"; + options.ConfigureBuilder = configurationBuilder => + { + configurationBuilder.AddScannerYaml(Path.Combine(builder.Environment.ContentRootPath, "../etc/scanner.yaml")); + }; +}); + +var contentRoot = builder.Environment.ContentRootPath; + +var bootstrapOptions = builder.Configuration.BindOptions( + ScannerWebServiceOptions.SectionName, + (opts, _) => + { + ScannerWebServiceOptionsPostConfigure.Apply(opts, contentRoot); + ScannerWebServiceOptionsValidator.Validate(opts); + }); + +builder.Services.AddOptions() + .Bind(builder.Configuration.GetSection(ScannerWebServiceOptions.SectionName)) + .PostConfigure(options => + { + ScannerWebServiceOptionsPostConfigure.Apply(options, contentRoot); + ScannerWebServiceOptionsValidator.Validate(options); + }) + .ValidateOnStart(); + +builder.Host.UseSerilog((context, services, loggerConfiguration) => +{ + loggerConfiguration + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) + .Enrich.FromLogContext() + .WriteTo.Console(); +}); + +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddScannerCache(builder.Configuration); +builder.Services.AddSingleton(); +builder.Services.AddHttpContextAccessor(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddStellaOpsCrypto(); +builder.Services.AddBouncyCastleEd25519Provider(); +builder.Services.AddSingleton(); +if (bootstrapOptions.Events is { Enabled: true } eventsOptions + && string.Equals(eventsOptions.Driver, "redis", StringComparison.OrdinalIgnoreCase)) +{ + builder.Services.AddSingleton(); +} +else +{ + builder.Services.AddSingleton(); +} +builder.Services.AddSingleton(); + +var pluginHostOptions = ScannerPluginHostFactory.Build(bootstrapOptions, contentRoot); +builder.Services.RegisterPluginRoutines(builder.Configuration, pluginHostOptions); + +builder.Services.AddOpenApiIfAvailable(); + +if (bootstrapOptions.Authority.Enabled) +{ + builder.Services.AddStellaOpsAuthClient(clientOptions => + { + clientOptions.Authority = bootstrapOptions.Authority.Issuer; + clientOptions.ClientId = bootstrapOptions.Authority.ClientId ?? string.Empty; + clientOptions.ClientSecret = bootstrapOptions.Authority.ClientSecret; + clientOptions.HttpTimeout = TimeSpan.FromSeconds(bootstrapOptions.Authority.BackchannelTimeoutSeconds); + + clientOptions.DefaultScopes.Clear(); + foreach (var scope in bootstrapOptions.Authority.ClientScopes) + { + clientOptions.DefaultScopes.Add(scope); + } + + var resilience = bootstrapOptions.Authority.Resilience ?? new ScannerWebServiceOptions.AuthorityOptions.ResilienceOptions(); + if (resilience.EnableRetries.HasValue) + { + clientOptions.EnableRetries = resilience.EnableRetries.Value; + } + + if (resilience.RetryDelays is { Count: > 0 }) + { + clientOptions.RetryDelays.Clear(); + foreach (var delay in resilience.RetryDelays) + { + clientOptions.RetryDelays.Add(delay); + } + } + + if (resilience.AllowOfflineCacheFallback.HasValue) + { + clientOptions.AllowOfflineCacheFallback = resilience.AllowOfflineCacheFallback.Value; + } + + if (resilience.OfflineCacheTolerance.HasValue) + { + clientOptions.OfflineCacheTolerance = resilience.OfflineCacheTolerance.Value; + } + }); + + builder.Services.AddStellaOpsResourceServerAuthentication( + builder.Configuration, + configurationSection: null, + configure: resourceOptions => + { + resourceOptions.Authority = bootstrapOptions.Authority.Issuer; + resourceOptions.RequireHttpsMetadata = bootstrapOptions.Authority.RequireHttpsMetadata; + resourceOptions.MetadataAddress = bootstrapOptions.Authority.MetadataAddress; + resourceOptions.BackchannelTimeout = TimeSpan.FromSeconds(bootstrapOptions.Authority.BackchannelTimeoutSeconds); + resourceOptions.TokenClockSkew = TimeSpan.FromSeconds(bootstrapOptions.Authority.TokenClockSkewSeconds); + + resourceOptions.Audiences.Clear(); + foreach (var audience in bootstrapOptions.Authority.Audiences) + { + resourceOptions.Audiences.Add(audience); + } + + resourceOptions.RequiredScopes.Clear(); + foreach (var scope in bootstrapOptions.Authority.RequiredScopes) + { + resourceOptions.RequiredScopes.Add(scope); + } + + resourceOptions.BypassNetworks.Clear(); + foreach (var network in bootstrapOptions.Authority.BypassNetworks) + { + resourceOptions.BypassNetworks.Add(network); + } + }); + + builder.Services.AddAuthorization(options => + { + options.AddStellaOpsScopePolicy(ScannerPolicies.ScansEnqueue, bootstrapOptions.Authority.RequiredScopes.ToArray()); + options.AddStellaOpsScopePolicy(ScannerPolicies.ScansRead, ScannerAuthorityScopes.ScansRead); + options.AddStellaOpsScopePolicy(ScannerPolicies.Reports, ScannerAuthorityScopes.ReportsRead); + }); +} +else +{ + builder.Services.AddAuthentication(options => + { + options.DefaultAuthenticateScheme = "Anonymous"; + options.DefaultChallengeScheme = "Anonymous"; + }) + .AddScheme("Anonymous", _ => { }); + + builder.Services.AddAuthorization(options => + { + options.AddPolicy(ScannerPolicies.ScansEnqueue, policy => policy.RequireAssertion(_ => true)); + options.AddPolicy(ScannerPolicies.ScansRead, policy => policy.RequireAssertion(_ => true)); + options.AddPolicy(ScannerPolicies.Reports, policy => policy.RequireAssertion(_ => true)); + }); +} + +var app = builder.Build(); + +var resolvedOptions = app.Services.GetRequiredService>().Value; +var authorityConfigured = resolvedOptions.Authority.Enabled; +if (authorityConfigured && resolvedOptions.Authority.AllowAnonymousFallback) +{ + app.Logger.LogWarning( + "Scanner authority authentication is enabled but anonymous fallback remains allowed. Disable fallback before production rollout."); +} + +if (resolvedOptions.Telemetry.EnableLogging && resolvedOptions.Telemetry.EnableRequestLogging) +{ + app.UseSerilogRequestLogging(options => + { + options.GetLevel = (httpContext, elapsed, exception) => + exception is null ? LogEventLevel.Information : LogEventLevel.Error; + options.EnrichDiagnosticContext = (diagnosticContext, httpContext) => + { + diagnosticContext.Set("RequestId", httpContext.TraceIdentifier); + diagnosticContext.Set("UserAgent", httpContext.Request.Headers.UserAgent.ToString()); + if (Activity.Current is { TraceId: var traceId } && traceId != default) + { + diagnosticContext.Set("TraceId", traceId.ToString()); + } + }; + }); +} + +app.UseExceptionHandler(errorApp => +{ + errorApp.Run(async context => + { + context.Response.ContentType = "application/problem+json"; + var feature = context.Features.Get(); + var error = feature?.Error; + + var extensions = new Dictionary(StringComparer.Ordinal) + { + ["traceId"] = Activity.Current?.TraceId.ToString() ?? context.TraceIdentifier, + }; + + var problem = Results.Problem( + detail: error?.Message, + instance: context.Request.Path, + statusCode: StatusCodes.Status500InternalServerError, + title: "Unexpected server error", + type: "https://stellaops.org/problems/internal-error", + extensions: extensions); + + await problem.ExecuteAsync(context).ConfigureAwait(false); + }); +}); + +if (authorityConfigured) +{ + app.UseAuthentication(); + app.UseAuthorization(); +} + +app.MapHealthEndpoints(); + +var apiGroup = app.MapGroup(resolvedOptions.Api.BasePath); + +if (app.Environment.IsEnvironment("Testing")) +{ + apiGroup.MapGet("/__auth-probe", () => Results.Ok("ok")) + .RequireAuthorization(ScannerPolicies.ScansEnqueue) + .WithName("scanner.auth-probe"); +} + +apiGroup.MapScanEndpoints(resolvedOptions.Api.ScansSegment); + +if (resolvedOptions.Features.EnablePolicyPreview) +{ + apiGroup.MapPolicyEndpoints(resolvedOptions.Api.PolicySegment); +} + +apiGroup.MapReportEndpoints(resolvedOptions.Api.ReportsSegment); + +app.MapOpenApiIfAvailable(); +await app.RunAsync().ConfigureAwait(false); diff --git a/src/StellaOps.Scanner.WebService/Security/AnonymousAuthenticationHandler.cs b/src/StellaOps.Scanner.WebService/Security/AnonymousAuthenticationHandler.cs new file mode 100644 index 00000000..ed695549 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Security/AnonymousAuthenticationHandler.cs @@ -0,0 +1,26 @@ +using System.Security.Claims; +using System.Text.Encodings.Web; +using Microsoft.AspNetCore.Authentication; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scanner.WebService.Security; + +internal sealed class AnonymousAuthenticationHandler : AuthenticationHandler +{ + public AnonymousAuthenticationHandler( + IOptionsMonitor options, + ILoggerFactory logger, + UrlEncoder encoder) + : base(options, logger, encoder) + { + } + + protected override Task HandleAuthenticateAsync() + { + var identity = new ClaimsIdentity(authenticationType: Scheme.Name); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, Scheme.Name); + return Task.FromResult(AuthenticateResult.Success(ticket)); + } +} diff --git a/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs b/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs new file mode 100644 index 00000000..d640c911 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Security/ScannerAuthorityScopes.cs @@ -0,0 +1,11 @@ +namespace StellaOps.Scanner.WebService.Security; + +/// +/// Canonical scope names consumed by the Scanner WebService. +/// +internal static class ScannerAuthorityScopes +{ + public const string ScansEnqueue = "scanner.scans.enqueue"; + public const string ScansRead = "scanner.scans.read"; + public const string ReportsRead = "scanner.reports.read"; +} diff --git a/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs b/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs new file mode 100644 index 00000000..ee3c31e3 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Security/ScannerPolicies.cs @@ -0,0 +1,8 @@ +namespace StellaOps.Scanner.WebService.Security; + +internal static class ScannerPolicies +{ + public const string ScansEnqueue = "scanner.api"; + public const string ScansRead = "scanner.scans.read"; + public const string Reports = "scanner.reports"; +} diff --git a/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs new file mode 100644 index 00000000..eca26ba3 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/IPlatformEventPublisher.cs @@ -0,0 +1,16 @@ +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Notify.Models; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// Publishes platform events to the internal bus consumed by downstream services (Notify, UI, etc.). +/// +public interface IPlatformEventPublisher +{ + /// + /// Publishes the supplied event envelope. + /// + Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default); +} diff --git a/src/StellaOps.Scanner.WebService/Services/IReportEventDispatcher.cs b/src/StellaOps.Scanner.WebService/Services/IReportEventDispatcher.cs new file mode 100644 index 00000000..6fd473dd --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/IReportEventDispatcher.cs @@ -0,0 +1,21 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// Coordinates generation and publication of scanner-related platform events. +/// +public interface IReportEventDispatcher +{ + Task PublishAsync( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + HttpContext httpContext, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.WebService/Services/IScanCoordinator.cs b/src/StellaOps.Scanner.WebService/Services/IScanCoordinator.cs new file mode 100644 index 00000000..7ed43c5d --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/IScanCoordinator.cs @@ -0,0 +1,10 @@ +using StellaOps.Scanner.WebService.Domain; + +namespace StellaOps.Scanner.WebService.Services; + +public interface IScanCoordinator +{ + ValueTask SubmitAsync(ScanSubmission submission, CancellationToken cancellationToken); + + ValueTask GetAsync(ScanId scanId, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.WebService/Services/InMemoryScanCoordinator.cs b/src/StellaOps.Scanner.WebService/Services/InMemoryScanCoordinator.cs new file mode 100644 index 00000000..6bd6bebe --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/InMemoryScanCoordinator.cs @@ -0,0 +1,80 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using StellaOps.Scanner.WebService.Domain; +using StellaOps.Scanner.WebService.Utilities; + +namespace StellaOps.Scanner.WebService.Services; + +public sealed class InMemoryScanCoordinator : IScanCoordinator +{ + private sealed record ScanEntry(ScanSnapshot Snapshot); + + private readonly ConcurrentDictionary scans = new(StringComparer.OrdinalIgnoreCase); + private readonly TimeProvider timeProvider; + private readonly IScanProgressPublisher progressPublisher; + + public InMemoryScanCoordinator(TimeProvider timeProvider, IScanProgressPublisher progressPublisher) + { + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + this.progressPublisher = progressPublisher ?? throw new ArgumentNullException(nameof(progressPublisher)); + } + + public ValueTask SubmitAsync(ScanSubmission submission, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(submission); + + var normalizedTarget = submission.Target.Normalize(); + var metadata = submission.Metadata ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + var scanId = ScanIdGenerator.Create(normalizedTarget, submission.Force, submission.ClientRequestId, metadata); + var now = timeProvider.GetUtcNow(); + + var eventData = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["force"] = submission.Force, + }; + foreach (var pair in metadata) + { + eventData[$"meta.{pair.Key}"] = pair.Value; + } + + ScanEntry entry = scans.AddOrUpdate( + scanId.Value, + _ => new ScanEntry(new ScanSnapshot( + scanId, + normalizedTarget, + ScanStatus.Pending, + now, + now, + null)), + (_, existing) => + { + if (submission.Force) + { + var snapshot = existing.Snapshot with + { + Status = ScanStatus.Pending, + UpdatedAt = now, + FailureReason = null + }; + return new ScanEntry(snapshot); + } + + return existing; + }); + + var created = entry.Snapshot.CreatedAt == now; + var state = entry.Snapshot.Status.ToString(); + progressPublisher.Publish(scanId, state, created ? "queued" : "requeued", eventData); + return ValueTask.FromResult(new ScanSubmissionResult(entry.Snapshot, created)); + } + + public ValueTask GetAsync(ScanId scanId, CancellationToken cancellationToken) + { + if (scans.TryGetValue(scanId.Value, out var entry)) + { + return ValueTask.FromResult(entry.Snapshot); + } + + return ValueTask.FromResult(null); + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs new file mode 100644 index 00000000..a12797a2 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/NullPlatformEventPublisher.cs @@ -0,0 +1,34 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Notify.Models; + +namespace StellaOps.Scanner.WebService.Services; + +/// +/// No-op fallback publisher used until queue adapters register a concrete implementation. +/// +internal sealed class NullPlatformEventPublisher : IPlatformEventPublisher +{ + private readonly ILogger _logger; + + public NullPlatformEventPublisher(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) + { + if (@event is null) + { + throw new ArgumentNullException(nameof(@event)); + } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + _logger.LogDebug("Suppressing publish for event {EventKind} (tenant {Tenant}).", @event.Kind, @event.Tenant); + } + + return Task.CompletedTask; + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/PolicyDtoMapper.cs b/src/StellaOps.Scanner.WebService/Services/PolicyDtoMapper.cs new file mode 100644 index 00000000..6da25c08 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/PolicyDtoMapper.cs @@ -0,0 +1,356 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +internal static class PolicyDtoMapper +{ + private static readonly StringComparer OrdinalIgnoreCase = StringComparer.OrdinalIgnoreCase; + + public static PolicyPreviewRequest ToDomain(PolicyPreviewRequestDto request) + { + ArgumentNullException.ThrowIfNull(request); + + var findings = BuildFindings(request.Findings); + var baseline = BuildBaseline(request.Baseline); + var proposedPolicy = ToSnapshotContent(request.Policy); + + return new PolicyPreviewRequest( + request.ImageDigest!.Trim(), + findings, + baseline, + SnapshotOverride: null, + ProposedPolicy: proposedPolicy); + } + + public static PolicyPreviewResponseDto ToDto(PolicyPreviewResponse response) + { + ArgumentNullException.ThrowIfNull(response); + + var diffs = response.Diffs.Select(ToDiffDto).ToImmutableArray(); + var issues = response.Issues.Select(ToIssueDto).ToImmutableArray(); + + return new PolicyPreviewResponseDto + { + Success = response.Success, + PolicyDigest = response.PolicyDigest, + RevisionId = response.RevisionId, + Changed = response.ChangedCount, + Diffs = diffs, + Issues = issues + }; + } + + public static PolicyPreviewIssueDto ToIssueDto(PolicyIssue issue) + { + ArgumentNullException.ThrowIfNull(issue); + + return new PolicyPreviewIssueDto + { + Code = issue.Code, + Message = issue.Message, + Severity = issue.Severity.ToString(), + Path = issue.Path + }; + } + + public static PolicyDocumentFormat ParsePolicyFormat(string? format) + => string.Equals(format, "json", StringComparison.OrdinalIgnoreCase) + ? PolicyDocumentFormat.Json + : PolicyDocumentFormat.Yaml; + + private static ImmutableArray BuildFindings(IReadOnlyList? findings) + { + if (findings is null || findings.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(findings.Count); + foreach (var finding in findings) + { + if (finding is null) + { + continue; + } + + var tags = finding.Tags is { Count: > 0 } + ? finding.Tags.Where(tag => !string.IsNullOrWhiteSpace(tag)) + .Select(tag => tag.Trim()) + .ToImmutableArray() + : ImmutableArray.Empty; + + var severity = ParseSeverity(finding.Severity); + var candidate = PolicyFinding.Create( + finding.Id!.Trim(), + severity, + environment: Normalize(finding.Environment), + source: Normalize(finding.Source), + vendor: Normalize(finding.Vendor), + license: Normalize(finding.License), + image: Normalize(finding.Image), + repository: Normalize(finding.Repository), + package: Normalize(finding.Package), + purl: Normalize(finding.Purl), + cve: Normalize(finding.Cve), + path: Normalize(finding.Path), + layerDigest: Normalize(finding.LayerDigest), + tags: tags); + + builder.Add(candidate); + } + + return builder.ToImmutable(); + } + + private static ImmutableArray BuildBaseline(IReadOnlyList? baseline) + { + if (baseline is null || baseline.Count == 0) + { + return ImmutableArray.Empty; + } + + var builder = ImmutableArray.CreateBuilder(baseline.Count); + foreach (var verdict in baseline) + { + if (verdict is null || string.IsNullOrWhiteSpace(verdict.FindingId)) + { + continue; + } + + var inputs = verdict.Inputs is { Count: > 0 } + ? CreateImmutableDeterministicDictionary(verdict.Inputs) + : ImmutableDictionary.Empty; + + var status = ParseVerdictStatus(verdict.Status); + builder.Add(new PolicyVerdict( + verdict.FindingId!.Trim(), + status, + verdict.RuleName, + verdict.RuleAction, + verdict.Notes, + verdict.Score ?? 0, + verdict.ConfigVersion ?? PolicyScoringConfig.Default.Version, + inputs, + verdict.QuietedBy, + verdict.Quiet ?? false, + verdict.UnknownConfidence, + verdict.ConfidenceBand, + verdict.UnknownAgeDays, + verdict.SourceTrust, + verdict.Reachability)); + } + + return builder.ToImmutable(); + } + + private static PolicyPreviewDiffDto ToDiffDto(PolicyVerdictDiff diff) + { + ArgumentNullException.ThrowIfNull(diff); + + return new PolicyPreviewDiffDto + { + FindingId = diff.Projected.FindingId, + Baseline = ToVerdictDto(diff.Baseline), + Projected = ToVerdictDto(diff.Projected), + Changed = diff.Changed + }; + } + + internal static PolicyPreviewVerdictDto ToVerdictDto(PolicyVerdict verdict) + { + ArgumentNullException.ThrowIfNull(verdict); + + IReadOnlyDictionary? inputs = null; + var verdictInputs = verdict.GetInputs(); + if (verdictInputs.Count > 0) + { + inputs = CreateDeterministicInputs(verdictInputs); + } + + var sourceTrust = verdict.SourceTrust; + if (string.IsNullOrWhiteSpace(sourceTrust)) + { + sourceTrust = ExtractSuffix(verdictInputs, "trustWeight."); + } + + var reachability = verdict.Reachability; + if (string.IsNullOrWhiteSpace(reachability)) + { + reachability = ExtractSuffix(verdictInputs, "reachability."); + } + + return new PolicyPreviewVerdictDto + { + FindingId = verdict.FindingId, + Status = verdict.Status.ToString(), + RuleName = verdict.RuleName, + RuleAction = verdict.RuleAction, + Notes = verdict.Notes, + Score = verdict.Score, + ConfigVersion = verdict.ConfigVersion, + Inputs = inputs, + QuietedBy = verdict.QuietedBy, + Quiet = verdict.Quiet, + UnknownConfidence = verdict.UnknownConfidence, + ConfidenceBand = verdict.ConfidenceBand, + UnknownAgeDays = verdict.UnknownAgeDays, + SourceTrust = sourceTrust, + Reachability = reachability + }; + } + + private static ImmutableDictionary CreateImmutableDeterministicDictionary(IEnumerable> inputs) + { + var sorted = CreateDeterministicInputs(inputs); + var builder = ImmutableDictionary.CreateBuilder(OrdinalIgnoreCase); + foreach (var pair in sorted) + { + builder[pair.Key] = pair.Value; + } + + return builder.ToImmutable(); + } + + private static IReadOnlyDictionary CreateDeterministicInputs(IEnumerable> inputs) + { + ArgumentNullException.ThrowIfNull(inputs); + + var dictionary = new SortedDictionary(InputKeyComparer.Instance); + foreach (var pair in inputs) + { + if (string.IsNullOrWhiteSpace(pair.Key)) + { + continue; + } + + var key = pair.Key.Trim(); + dictionary[key] = pair.Value; + } + + return dictionary; + } + + private sealed class InputKeyComparer : IComparer + { + public static InputKeyComparer Instance { get; } = new(); + + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) + { + return 0; + } + + if (x is null) + { + return -1; + } + + if (y is null) + { + return 1; + } + + var px = GetPriority(x); + var py = GetPriority(y); + if (px != py) + { + return px.CompareTo(py); + } + + return string.Compare(x, y, StringComparison.Ordinal); + } + + private static int GetPriority(string key) + { + if (string.Equals(key, "reachabilityWeight", StringComparison.OrdinalIgnoreCase)) + { + return 0; + } + + if (string.Equals(key, "baseScore", StringComparison.OrdinalIgnoreCase)) + { + return 1; + } + + if (string.Equals(key, "severityWeight", StringComparison.OrdinalIgnoreCase)) + { + return 2; + } + + if (string.Equals(key, "trustWeight", StringComparison.OrdinalIgnoreCase)) + { + return 3; + } + + if (key.StartsWith("trustWeight.", StringComparison.OrdinalIgnoreCase)) + { + return 4; + } + + if (key.StartsWith("reachability.", StringComparison.OrdinalIgnoreCase)) + { + return 5; + } + + return 6; + } + } + + private static PolicySnapshotContent? ToSnapshotContent(PolicyPreviewPolicyDto? policy) + { + if (policy is null || string.IsNullOrWhiteSpace(policy.Content)) + { + return null; + } + + var format = ParsePolicyFormat(policy.Format); + return new PolicySnapshotContent( + policy.Content, + format, + policy.Actor, + Source: null, + policy.Description); + } + + private static PolicySeverity ParseSeverity(string? value) + { + if (Enum.TryParse(value, true, out var severity)) + { + return severity; + } + + return PolicySeverity.Unknown; + } + + private static PolicyVerdictStatus ParseVerdictStatus(string? value) + { + if (Enum.TryParse(value, true, out var status)) + { + return status; + } + + return PolicyVerdictStatus.Pass; + } + + private static string? Normalize(string? value) + => string.IsNullOrWhiteSpace(value) ? null : value.Trim(); + + private static string? ExtractSuffix(ImmutableDictionary inputs, string prefix) + { + foreach (var key in inputs.Keys) + { + if (key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) && key.Length > prefix.Length) + { + return key.Substring(prefix.Length); + } + } + + return null; + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs new file mode 100644 index 00000000..20ea85da --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/RedisPlatformEventPublisher.cs @@ -0,0 +1,148 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StackExchange.Redis; +using StellaOps.Notify.Models; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Services; + +internal sealed class RedisPlatformEventPublisher : IPlatformEventPublisher, IAsyncDisposable +{ + private readonly ScannerWebServiceOptions.EventsOptions _options; + private readonly ILogger _logger; + private readonly TimeSpan _publishTimeout; + private readonly string _streamKey; + private readonly long? _maxStreamLength; + + private readonly SemaphoreSlim _connectionGate = new(1, 1); + private IConnectionMultiplexer? _connection; + private bool _disposed; + + public RedisPlatformEventPublisher( + IOptions options, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + + _options = options.Value.Events ?? throw new InvalidOperationException("Events options are required when redis publisher is registered."); + if (!_options.Enabled) + { + throw new InvalidOperationException("RedisPlatformEventPublisher requires events emission to be enabled."); + } + + if (!string.Equals(_options.Driver, "redis", StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException($"RedisPlatformEventPublisher cannot be used with driver '{_options.Driver}'."); + } + + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _streamKey = string.IsNullOrWhiteSpace(_options.Stream) ? "stella.events" : _options.Stream; + _publishTimeout = TimeSpan.FromSeconds(_options.PublishTimeoutSeconds <= 0 ? 5 : _options.PublishTimeoutSeconds); + _maxStreamLength = _options.MaxStreamLength > 0 ? _options.MaxStreamLength : null; + } + + public async Task PublishAsync(NotifyEvent @event, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(@event); + cancellationToken.ThrowIfCancellationRequested(); + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + var payload = NotifyCanonicalJsonSerializer.Serialize(@event); + + var entries = new NameValueEntry[] + { + new("event", payload), + new("kind", @event.Kind), + new("tenant", @event.Tenant), + new("ts", @event.Ts.ToString("O")) + }; + + int? maxLength = null; + if (_maxStreamLength.HasValue) + { + var clamped = Math.Min(_maxStreamLength.Value, int.MaxValue); + maxLength = (int)clamped; + } + + var publishTask = maxLength.HasValue + ? database.StreamAddAsync(_streamKey, entries, maxLength: maxLength, useApproximateMaxLength: true) + : database.StreamAddAsync(_streamKey, entries); + + if (_publishTimeout > TimeSpan.Zero) + { + await publishTask.WaitAsync(_publishTimeout, cancellationToken).ConfigureAwait(false); + } + else + { + await publishTask.ConfigureAwait(false); + } + } + + private async Task GetDatabaseAsync(CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (_connection is not null && _connection.IsConnected) + { + return _connection.GetDatabase(); + } + + await _connectionGate.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is null || !_connection.IsConnected) + { + var config = ConfigurationOptions.Parse(_options.Dsn); + config.AbortOnConnectFail = false; + + if (_options.DriverSettings.TryGetValue("clientName", out var clientName) && !string.IsNullOrWhiteSpace(clientName)) + { + config.ClientName = clientName; + } + + if (_options.DriverSettings.TryGetValue("ssl", out var sslValue) && bool.TryParse(sslValue, out var ssl)) + { + config.Ssl = ssl; + } + + _connection = await ConnectionMultiplexer.ConnectAsync(config).WaitAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Connected Redis platform event publisher to stream {Stream}.", _streamKey); + } + } + finally + { + _connectionGate.Release(); + } + + return _connection!.GetDatabase(); + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_connection is not null) + { + try + { + await _connection.CloseAsync(); + } + catch (Exception ex) + { + _logger.LogDebug(ex, "Error while closing Redis platform event publisher connection."); + } + + _connection.Dispose(); + } + + _connectionGate.Dispose(); + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs b/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs new file mode 100644 index 00000000..7ad4f8b4 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/ReportEventDispatcher.cs @@ -0,0 +1,520 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Security.Claims; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; +using StellaOps.Notify.Models; +using StellaOps.Policy; +using StellaOps.Scanner.WebService.Contracts; + +namespace StellaOps.Scanner.WebService.Services; + +internal sealed class ReportEventDispatcher : IReportEventDispatcher +{ + private const string DefaultTenant = "default"; + private const string Actor = "scanner.webservice"; + + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web) + { + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull + }; + + private readonly IPlatformEventPublisher _publisher; + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public ReportEventDispatcher( + IPlatformEventPublisher publisher, + TimeProvider timeProvider, + ILogger logger) + { + _publisher = publisher ?? throw new ArgumentNullException(nameof(publisher)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task PublishAsync( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + HttpContext httpContext, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(preview); + ArgumentNullException.ThrowIfNull(document); + ArgumentNullException.ThrowIfNull(httpContext); + + cancellationToken.ThrowIfCancellationRequested(); + + var now = _timeProvider.GetUtcNow(); + var tenant = ResolveTenant(httpContext); + var scope = BuildScope(request, document); + var attributes = BuildAttributes(document); + + var reportPayload = BuildReportReadyPayload(request, preview, document, envelope, httpContext); + var reportEvent = NotifyEvent.Create( + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerReportReady, + tenant: tenant, + ts: document.GeneratedAt == default ? now : document.GeneratedAt, + payload: reportPayload, + scope: scope, + actor: Actor, + attributes: attributes); + + await PublishSafelyAsync(reportEvent, document.ReportId, cancellationToken).ConfigureAwait(false); + + var scanPayload = BuildScanCompletedPayload(request, preview, document, envelope); + var scanEvent = NotifyEvent.Create( + eventId: Guid.NewGuid(), + kind: NotifyEventKinds.ScannerScanCompleted, + tenant: tenant, + ts: document.GeneratedAt == default ? now : document.GeneratedAt, + payload: scanPayload, + scope: scope, + actor: Actor, + attributes: attributes); + + await PublishSafelyAsync(scanEvent, document.ReportId, cancellationToken).ConfigureAwait(false); + } + + private async Task PublishSafelyAsync(NotifyEvent @event, string reportId, CancellationToken cancellationToken) + { + try + { + await _publisher.PublishAsync(@event, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogError( + ex, + "Failed to publish event {EventKind} for report {ReportId}.", + @event.Kind, + reportId); + } + } + + private static string ResolveTenant(HttpContext context) + { + var tenant = context.User?.FindFirstValue(StellaOpsClaimTypes.Tenant); + if (!string.IsNullOrWhiteSpace(tenant)) + { + return tenant.Trim(); + } + + if (context.Request.Headers.TryGetValue("X-Stella-Tenant", out var headerTenant)) + { + var headerValue = headerTenant.ToString(); + if (!string.IsNullOrWhiteSpace(headerValue)) + { + return headerValue.Trim(); + } + } + + return DefaultTenant; + } + + private static NotifyEventScope BuildScope(ReportRequestDto request, ReportDocumentDto document) + { + var repository = ResolveRepository(request); + var (ns, repo) = SplitRepository(repository); + + var digest = string.IsNullOrWhiteSpace(document.ImageDigest) + ? request.ImageDigest ?? string.Empty + : document.ImageDigest; + + return NotifyEventScope.Create( + @namespace: ns, + repo: string.IsNullOrWhiteSpace(repo) ? "(unknown)" : repo, + digest: string.IsNullOrWhiteSpace(digest) ? "(unknown)" : digest); + } + + private static string ResolveRepository(ReportRequestDto request) + { + if (request.Findings is { Count: > 0 }) + { + foreach (var finding in request.Findings) + { + if (!string.IsNullOrWhiteSpace(finding.Repository)) + { + return finding.Repository!.Trim(); + } + + if (!string.IsNullOrWhiteSpace(finding.Image)) + { + return finding.Image!.Trim(); + } + } + } + + return string.Empty; + } + + private static (string? Namespace, string Repo) SplitRepository(string repository) + { + if (string.IsNullOrWhiteSpace(repository)) + { + return (null, string.Empty); + } + + var normalized = repository.Trim(); + var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + if (segments.Length == 0) + { + return (null, normalized); + } + + if (segments.Length == 1) + { + return (null, segments[0]); + } + + var repo = segments[^1]; + var ns = string.Join('/', segments[..^1]); + return (ns, repo); + } + + private static IEnumerable> BuildAttributes(ReportDocumentDto document) + { + var attributes = new List>(capacity: 4) + { + new("reportId", document.ReportId) + }; + + if (!string.IsNullOrWhiteSpace(document.Policy.RevisionId)) + { + attributes.Add(new("policyRevisionId", document.Policy.RevisionId!)); + } + + if (!string.IsNullOrWhiteSpace(document.Policy.Digest)) + { + attributes.Add(new("policyDigest", document.Policy.Digest!)); + } + + attributes.Add(new("verdict", document.Verdict)); + return attributes; + } + + private static JsonObject BuildReportReadyPayload( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope, + HttpContext context) + { + var payload = new JsonObject + { + ["reportId"] = document.ReportId, + ["generatedAt"] = document.GeneratedAt == default + ? null + : JsonValue.Create(document.GeneratedAt), + ["verdict"] = MapVerdict(document.Verdict), + ["summary"] = JsonSerializer.SerializeToNode(document.Summary, JsonOptions), + ["delta"] = BuildDelta(preview, request), + ["links"] = BuildLinks(context, document), + ["quietedFindingCount"] = document.Summary.Quieted + }; + + payload.RemoveNulls(); + + if (envelope is not null) + { + payload["dsse"] = JsonSerializer.SerializeToNode(envelope, JsonOptions); + } + + payload["report"] = JsonSerializer.SerializeToNode(document, JsonOptions); + return payload; + } + + private static JsonObject BuildScanCompletedPayload( + ReportRequestDto request, + PolicyPreviewResponse preview, + ReportDocumentDto document, + DsseEnvelopeDto? envelope) + { + var payload = new JsonObject + { + ["reportId"] = document.ReportId, + ["digest"] = document.ImageDigest, + ["summary"] = JsonSerializer.SerializeToNode(document.Summary, JsonOptions), + ["verdict"] = MapVerdict(document.Verdict), + ["policy"] = JsonSerializer.SerializeToNode(document.Policy, JsonOptions), + ["delta"] = BuildDelta(preview, request), + ["report"] = JsonSerializer.SerializeToNode(document, JsonOptions) + }; + + if (envelope is not null) + { + payload["dsse"] = JsonSerializer.SerializeToNode(envelope, JsonOptions); + } + + payload["findings"] = BuildFindingSummaries(request); + payload.RemoveNulls(); + return payload; + } + + private static JsonArray BuildFindingSummaries(ReportRequestDto request) + { + var array = new JsonArray(); + if (request.Findings is { Count: > 0 }) + { + foreach (var finding in request.Findings) + { + if (string.IsNullOrWhiteSpace(finding.Id)) + { + continue; + } + + var summary = new JsonObject + { + ["id"] = finding.Id, + ["severity"] = finding.Severity, + ["cve"] = finding.Cve, + ["purl"] = finding.Purl, + ["reachability"] = ResolveReachability(finding.Tags) + }; + + summary.RemoveNulls(); + array.Add(summary); + } + } + + return array; + } + + private static string? ResolveReachability(IReadOnlyList? tags) + { + if (tags is null) + { + return null; + } + + foreach (var tag in tags) + { + if (string.IsNullOrWhiteSpace(tag)) + { + continue; + } + + if (tag.StartsWith("reachability:", StringComparison.OrdinalIgnoreCase)) + { + return tag["reachability:".Length..]; + } + } + + return null; + } + + private static JsonObject BuildDelta(PolicyPreviewResponse preview, ReportRequestDto request) + { + var delta = new JsonObject(); + if (preview.Diffs.IsDefaultOrEmpty) + { + return delta; + } + + var findings = BuildFindingsIndex(request.Findings); + var kevIds = new SortedSet(StringComparer.OrdinalIgnoreCase); + var newCritical = 0; + var newHigh = 0; + + foreach (var diff in preview.Diffs) + { + var projected = diff.Projected; + if (projected is null || string.IsNullOrWhiteSpace(projected.FindingId)) + { + continue; + } + + if (!findings.TryGetValue(projected.FindingId, out var finding)) + { + finding = null; + } + + if (IsNewlyImportant(diff)) + { + var severity = finding?.Severity; + if (string.Equals(severity, "Critical", StringComparison.OrdinalIgnoreCase)) + { + newCritical++; + } + else if (string.Equals(severity, "High", StringComparison.OrdinalIgnoreCase)) + { + newHigh++; + } + + var kevId = ResolveKevIdentifier(finding); + if (!string.IsNullOrWhiteSpace(kevId)) + { + kevIds.Add(kevId); + } + } + } + + if (newCritical > 0) + { + delta["newCritical"] = newCritical; + } + + if (newHigh > 0) + { + delta["newHigh"] = newHigh; + } + + if (kevIds.Count > 0) + { + var kev = new JsonArray(); + foreach (var id in kevIds) + { + kev.Add(id); + } + + delta["kev"] = kev; + } + + return delta; + } + + private static ImmutableDictionary BuildFindingsIndex( + IReadOnlyList? findings) + { + if (findings is null || findings.Count == 0) + { + return ImmutableDictionary.Empty; + } + + var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var finding in findings) + { + if (string.IsNullOrWhiteSpace(finding.Id)) + { + continue; + } + + if (!builder.ContainsKey(finding.Id)) + { + builder.Add(finding.Id, finding); + } + } + + return builder.ToImmutable(); + } + + private static bool IsNewlyImportant(PolicyVerdictDiff diff) + { + var projected = diff.Projected.Status; + var baseline = diff.Baseline.Status; + + return projected switch + { + PolicyVerdictStatus.Blocked or PolicyVerdictStatus.Escalated + => baseline != PolicyVerdictStatus.Blocked && baseline != PolicyVerdictStatus.Escalated, + PolicyVerdictStatus.Warned or PolicyVerdictStatus.Deferred or PolicyVerdictStatus.RequiresVex + => baseline != PolicyVerdictStatus.Warned + && baseline != PolicyVerdictStatus.Deferred + && baseline != PolicyVerdictStatus.RequiresVex + && baseline != PolicyVerdictStatus.Blocked + && baseline != PolicyVerdictStatus.Escalated, + _ => false + }; + } + + private static string? ResolveKevIdentifier(PolicyPreviewFindingDto? finding) + { + if (finding is null) + { + return null; + } + + var tags = finding.Tags; + if (tags is not null) + { + foreach (var tag in tags) + { + if (string.IsNullOrWhiteSpace(tag)) + { + continue; + } + + if (string.Equals(tag, "kev", StringComparison.OrdinalIgnoreCase)) + { + return finding.Cve; + } + + if (tag.StartsWith("kev:", StringComparison.OrdinalIgnoreCase)) + { + var value = tag["kev:".Length..]; + if (!string.IsNullOrWhiteSpace(value)) + { + return value.Trim(); + } + } + } + } + + return finding.Cve; + } + + private static JsonObject BuildLinks(HttpContext context, ReportDocumentDto document) + { + var links = new JsonObject(); + + if (context.Request.Host.HasValue) + { + var scheme = string.IsNullOrWhiteSpace(context.Request.Scheme) ? "https" : context.Request.Scheme; + var builder = new UriBuilder(scheme, context.Request.Host.Host) + { + Port = context.Request.Host.Port ?? -1, + Path = $"/ui/reports/{Uri.EscapeDataString(document.ReportId)}" + }; + links["ui"] = builder.Uri.ToString(); + } + + return links; + } + + private static string MapVerdict(string verdict) + => verdict.ToLowerInvariant() switch + { + "blocked" or "fail" => "fail", + "escalated" => "fail", + "warn" or "warned" or "deferred" or "requiresvex" => "warn", + _ => "pass" + }; +} + +internal static class ReportEventDispatcherExtensions +{ + public static void RemoveNulls(this JsonObject jsonObject) + { + if (jsonObject is null) + { + return; + } + + var keysToRemove = new List(); + foreach (var pair in jsonObject) + { + if (pair.Value is null || pair.Value.GetValueKind() == JsonValueKind.Null) + { + keysToRemove.Add(pair.Key); + } + } + + foreach (var key in keysToRemove) + { + jsonObject.Remove(key); + } + } +} diff --git a/src/StellaOps.Scanner.WebService/Services/ReportSigner.cs b/src/StellaOps.Scanner.WebService/Services/ReportSigner.cs new file mode 100644 index 00000000..21433a48 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/ReportSigner.cs @@ -0,0 +1,263 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Cryptography; +using StellaOps.Scanner.WebService.Options; + +namespace StellaOps.Scanner.WebService.Services; + +public interface IReportSigner : IDisposable +{ + ReportSignature? Sign(ReadOnlySpan payload); +} + +public sealed class ReportSigner : IReportSigner +{ + private enum SigningMode + { + Disabled, + Provider, + Hs256 + } + + private readonly SigningMode mode; + private readonly string keyId = string.Empty; + private readonly string algorithmName = string.Empty; + private readonly ILogger logger; + private readonly ICryptoProviderRegistry cryptoRegistry; + private readonly ICryptoProvider? provider; + private readonly CryptoKeyReference? keyReference; + private readonly CryptoSignerResolution? signerResolution; + private readonly byte[]? hmacKey; + + public ReportSigner( + IOptions options, + ICryptoProviderRegistry cryptoRegistry, + ILogger logger) + { + ArgumentNullException.ThrowIfNull(options); + this.cryptoRegistry = cryptoRegistry ?? throw new ArgumentNullException(nameof(cryptoRegistry)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var value = options.Value ?? new ScannerWebServiceOptions(); + var features = value.Features ?? new ScannerWebServiceOptions.FeatureFlagOptions(); + var signing = value.Signing ?? new ScannerWebServiceOptions.SigningOptions(); + + if (!features.EnableSignedReports || !signing.Enabled) + { + mode = SigningMode.Disabled; + logger.LogInformation("Report signing disabled (feature flag or signing.enabled=false)."); + return; + } + + if (string.IsNullOrWhiteSpace(signing.KeyId)) + { + throw new InvalidOperationException("Signing keyId must be configured when signing is enabled."); + } + + var keyPem = ResolveKeyMaterial(signing); + keyId = signing.KeyId.Trim(); + + var resolvedMode = ResolveSigningMode(signing.Algorithm, out var canonicalAlgorithm, out var joseAlgorithm); + algorithmName = joseAlgorithm; + + switch (resolvedMode) + { + case SigningMode.Provider: + { + provider = ResolveProvider(signing.Provider, canonicalAlgorithm); + + var privateKey = DecodeKey(keyPem); + var reference = new CryptoKeyReference(keyId, provider.Name); + var signingKeyDescriptor = new CryptoSigningKey( + reference, + canonicalAlgorithm, + privateKey, + createdAt: DateTimeOffset.UtcNow); + + provider.UpsertSigningKey(signingKeyDescriptor); + + signerResolution = cryptoRegistry.ResolveSigner( + CryptoCapability.Signing, + canonicalAlgorithm, + reference, + provider.Name); + + keyReference = reference; + mode = SigningMode.Provider; + break; + } + case SigningMode.Hs256: + { + hmacKey = DecodeKey(keyPem); + mode = SigningMode.Hs256; + break; + } + default: + mode = SigningMode.Disabled; + break; + } + } + + public ReportSignature? Sign(ReadOnlySpan payload) + { + if (mode == SigningMode.Disabled) + { + return null; + } + + if (payload.IsEmpty) + { + throw new ArgumentException("Payload must be non-empty.", nameof(payload)); + } + + return mode switch + { + SigningMode.Provider => SignWithProvider(payload), + SigningMode.Hs256 => SignHs256(payload), + _ => null + }; + } + + private ReportSignature SignWithProvider(ReadOnlySpan payload) + { + var resolution = signerResolution ?? throw new InvalidOperationException("Signing provider has not been initialised."); + + var signature = resolution.Signer + .SignAsync(payload.ToArray()) + .ConfigureAwait(false) + .GetAwaiter() + .GetResult(); + + return new ReportSignature(keyId, algorithmName, Convert.ToBase64String(signature)); + } + + private ReportSignature SignHs256(ReadOnlySpan payload) + { + if (hmacKey is null) + { + throw new InvalidOperationException("HMAC signing has not been initialised."); + } + + using var hmac = new HMACSHA256(hmacKey); + var signature = hmac.ComputeHash(payload.ToArray()); + return new ReportSignature(keyId, algorithmName, Convert.ToBase64String(signature)); + } + + public void Dispose() + { + if (provider is not null && keyReference is not null) + { + provider.RemoveSigningKey(keyReference.KeyId); + } + } + + private ICryptoProvider ResolveProvider(string? configuredProvider, string canonicalAlgorithm) + { + if (!string.IsNullOrWhiteSpace(configuredProvider)) + { + if (!cryptoRegistry.TryResolve(configuredProvider.Trim(), out var hinted)) + { + throw new InvalidOperationException($"Configured signing provider '{configuredProvider}' is not registered."); + } + + if (!hinted.Supports(CryptoCapability.Signing, canonicalAlgorithm)) + { + throw new InvalidOperationException($"Provider '{configuredProvider}' does not support algorithm '{canonicalAlgorithm}'."); + } + + return hinted; + } + + return cryptoRegistry.ResolveOrThrow(CryptoCapability.Signing, canonicalAlgorithm); + } + + private static SigningMode ResolveSigningMode(string? algorithm, out string canonicalAlgorithm, out string joseAlgorithm) + { + if (string.IsNullOrWhiteSpace(algorithm)) + { + throw new InvalidOperationException("Signing algorithm must be specified when signing is enabled."); + } + + switch (algorithm.Trim().ToLowerInvariant()) + { + case "ed25519": + case "eddsa": + canonicalAlgorithm = SignatureAlgorithms.Ed25519; + joseAlgorithm = SignatureAlgorithms.EdDsa; + return SigningMode.Provider; + case "hs256": + canonicalAlgorithm = "HS256"; + joseAlgorithm = "HS256"; + return SigningMode.Hs256; + default: + throw new InvalidOperationException($"Unsupported signing algorithm '{algorithm}'."); + } + } + + private static string ResolveKeyMaterial(ScannerWebServiceOptions.SigningOptions signing) + { + if (!string.IsNullOrWhiteSpace(signing.KeyPem)) + { + return signing.KeyPem; + } + + if (!string.IsNullOrWhiteSpace(signing.KeyPemFile)) + { + try + { + return File.ReadAllText(signing.KeyPemFile); + } + catch (Exception ex) + { + throw new InvalidOperationException($"Unable to read signing key file '{signing.KeyPemFile}'.", ex); + } + } + + throw new InvalidOperationException("Signing keyPem must be configured when signing is enabled."); + } + + private static byte[] DecodeKey(string keyMaterial) + { + if (string.IsNullOrWhiteSpace(keyMaterial)) + { + throw new InvalidOperationException("Signing key material is empty."); + } + + var segments = keyMaterial.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var builder = new StringBuilder(); + var hadPemMarkers = false; + foreach (var segment in segments) + { + var trimmed = segment.Trim(); + if (trimmed.Length == 0) + { + continue; + } + + if (trimmed.StartsWith("-----", StringComparison.Ordinal)) + { + hadPemMarkers = true; + continue; + } + + builder.Append(trimmed); + } + + var base64 = hadPemMarkers ? builder.ToString() : keyMaterial.Trim(); + try + { + return Convert.FromBase64String(base64); + } + catch (FormatException ex) + { + throw new InvalidOperationException("Signing key must be Base64 encoded.", ex); + } + } +} + +public sealed record ReportSignature(string KeyId, string Algorithm, string Signature); diff --git a/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs b/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs new file mode 100644 index 00000000..a4f67e84 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Services/ScanProgressStream.cs @@ -0,0 +1,136 @@ +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Channels; +using StellaOps.Scanner.WebService.Domain; + +namespace StellaOps.Scanner.WebService.Services; + +public interface IScanProgressPublisher +{ + ScanProgressEvent Publish( + ScanId scanId, + string state, + string? message = null, + IReadOnlyDictionary? data = null, + string? correlationId = null); +} + +public interface IScanProgressReader +{ + bool Exists(ScanId scanId); + + IAsyncEnumerable SubscribeAsync(ScanId scanId, CancellationToken cancellationToken); +} + +public sealed class ScanProgressStream : IScanProgressPublisher, IScanProgressReader +{ + private sealed class ProgressChannel + { + private readonly List history = new(); + private readonly Channel channel = Channel.CreateUnbounded(new UnboundedChannelOptions + { + AllowSynchronousContinuations = true, + SingleReader = false, + SingleWriter = false + }); + + public int Sequence { get; private set; } + + public ScanProgressEvent Append(ScanProgressEvent progressEvent) + { + history.Add(progressEvent); + channel.Writer.TryWrite(progressEvent); + return progressEvent; + } + + public IReadOnlyList Snapshot() + { + return history.Count == 0 + ? Array.Empty() + : history.ToArray(); + } + + public ChannelReader Reader => channel.Reader; + + public int NextSequence() => ++Sequence; + } + + private static readonly IReadOnlyDictionary EmptyData = new ReadOnlyDictionary(new Dictionary(StringComparer.OrdinalIgnoreCase)); + + private readonly ConcurrentDictionary channels = new(StringComparer.OrdinalIgnoreCase); + private readonly TimeProvider timeProvider; + + public ScanProgressStream(TimeProvider timeProvider) + { + this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + } + + public bool Exists(ScanId scanId) + => channels.ContainsKey(scanId.Value); + + public ScanProgressEvent Publish( + ScanId scanId, + string state, + string? message = null, + IReadOnlyDictionary? data = null, + string? correlationId = null) + { + var channel = channels.GetOrAdd(scanId.Value, _ => new ProgressChannel()); + + ScanProgressEvent progressEvent; + lock (channel) + { + var sequence = channel.NextSequence(); + var correlation = correlationId ?? $"{scanId.Value}:{sequence:D4}"; + var payload = data is null || data.Count == 0 + ? EmptyData + : new ReadOnlyDictionary(new Dictionary(data, StringComparer.OrdinalIgnoreCase)); + + progressEvent = new ScanProgressEvent( + scanId, + sequence, + timeProvider.GetUtcNow(), + state, + message, + correlation, + payload); + + channel.Append(progressEvent); + } + + return progressEvent; + } + + public async IAsyncEnumerable SubscribeAsync( + ScanId scanId, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + if (!channels.TryGetValue(scanId.Value, out var channel)) + { + yield break; + } + + IReadOnlyList snapshot; + lock (channel) + { + snapshot = channel.Snapshot(); + } + + foreach (var progressEvent in snapshot) + { + yield return progressEvent; + } + + var reader = channel.Reader; + while (await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false)) + { + while (reader.TryRead(out var progressEvent)) + { + yield return progressEvent; + } + } + } +} diff --git a/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj new file mode 100644 index 00000000..2cf5041b --- /dev/null +++ b/src/StellaOps.Scanner.WebService/StellaOps.Scanner.WebService.csproj @@ -0,0 +1,31 @@ + + + net10.0 + preview + enable + enable + true + StellaOps.Scanner.WebService + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.WebService/TASKS.md b/src/StellaOps.Scanner.WebService/TASKS.md new file mode 100644 index 00000000..20a5be5b --- /dev/null +++ b/src/StellaOps.Scanner.WebService/TASKS.md @@ -0,0 +1,16 @@ +# Scanner WebService Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-WEB-09-101 | DONE (2025-10-18) | Scanner WebService Guild | SCANNER-CORE-09-501 | Stand up minimal API host with Authority OpTok + DPoP enforcement, health/ready endpoints, and restart-time plug-in loader per architecture §1, §4. | Host boots with configuration validation, `/healthz` and `/readyz` return 200, Authority middleware enforced in integration tests. | +| SCANNER-WEB-09-102 | DONE (2025-10-18) | Scanner WebService Guild | SCANNER-WEB-09-101, SCANNER-QUEUE-09-401 | Implement `/api/v1/scans` submission/status endpoints with deterministic IDs, validation, and cancellation tokens. | Contract documented, e2e test posts scan request and retrieves status, cancellation token honoured. | +| SCANNER-WEB-09-103 | DONE (2025-10-19) | Scanner WebService Guild | SCANNER-WEB-09-102, SCANNER-CORE-09-502 | Emit scan progress via SSE/JSONL with correlation IDs and deterministic timestamps; document API reference. | Streaming endpoint verified in tests, timestamps formatted ISO-8601 UTC, docs updated in `docs/09_API_CLI_REFERENCE.md`. | +| SCANNER-WEB-09-104 | DONE (2025-10-19) | Scanner WebService Guild | SCANNER-STORAGE-09-301, SCANNER-QUEUE-09-401 | Bind configuration for Mongo, MinIO, queue, feature flags; add startup diagnostics and fail-fast policy for missing deps. | Misconfiguration fails fast with actionable errors, configuration bound tests pass, diagnostics logged with correlation IDs. | +| SCANNER-POLICY-09-105 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-001 | Integrate policy schema loader + diagnostics + OpenAPI (YAML ignore rules, VEX include/exclude, vendor precedence). | Policy endpoints documented; validation surfaces actionable errors; OpenAPI schema published. | +| SCANNER-POLICY-09-106 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-002, SCANNER-POLICY-09-105 | `/reports` verdict assembly (Feedser/Vexer/Policy merge) + signed response envelope. | Aggregated report includes policy metadata; integration test verifies signed response; docs updated. | +| SCANNER-POLICY-09-107 | DONE (2025-10-19) | Scanner WebService Guild | POLICY-CORE-09-005, SCANNER-POLICY-09-106 | Surface score inputs, config version, and `quietedBy` provenance in `/reports` response and signed payload; document schema changes. | `/reports` JSON + DSSE contain score, reachability, sourceTrust, confidenceBand, quiet provenance; contract tests updated; docs refreshed. | +| SCANNER-WEB-10-201 | DONE (2025-10-19) | Scanner WebService Guild | SCANNER-CACHE-10-101 | Register scanner cache services and maintenance loop within WebService host. | `AddScannerCache` wired for configuration binding; maintenance service skips when disabled; project references updated. | +| SCANNER-RUNTIME-12-301 | TODO | Scanner WebService Guild | ZASTAVA-CORE-12-201 | Implement `/runtime/events` ingestion endpoint with validation, batching, and storage hooks per Zastava contract. | Observer fixtures POST events, data persisted and acked; invalid payloads rejected with deterministic errors. | +| SCANNER-RUNTIME-12-302 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-CORE-12-201 | Implement `/policy/runtime` endpoint joining SBOM baseline + policy verdict, returning admission guidance. Coordinate with CLI (`CLI-RUNTIME-13-008`) before GA to lock response field names/metadata. | Webhook integration test passes; responses include verdict, TTL, reasons; metrics/logging added; CLI contract review signed off. | +| SCANNER-EVENTS-15-201 | DOING (2025-10-19) | Scanner WebService Guild | NOTIFY-QUEUE-15-401 | Emit `scanner.report.ready` and `scanner.scan.completed` events (bus adapters + tests). | Event envelopes published to queue with schemas; fixtures committed; Notify consumption test passes. | +| SCANNER-RUNTIME-17-401 | TODO | Scanner WebService Guild | SCANNER-RUNTIME-12-301, ZASTAVA-OBS-17-005, SCANNER-EMIT-17-701, POLICY-RUNTIME-17-201 | Persist runtime build-id observations and expose them via `/runtime/events` + policy joins for debug-symbol correlation. | Mongo schema stores optional `buildId`, API/SDK responses document field, integration test resolves debug-store path using stored build-id, docs updated accordingly. | diff --git a/src/StellaOps.Scanner.WebService/Utilities/ScanIdGenerator.cs b/src/StellaOps.Scanner.WebService/Utilities/ScanIdGenerator.cs new file mode 100644 index 00000000..3055c073 --- /dev/null +++ b/src/StellaOps.Scanner.WebService/Utilities/ScanIdGenerator.cs @@ -0,0 +1,48 @@ +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using StellaOps.Scanner.WebService.Domain; + +namespace StellaOps.Scanner.WebService.Utilities; + +internal static class ScanIdGenerator +{ + public static ScanId Create( + ScanTarget target, + bool force, + string? clientRequestId, + IReadOnlyDictionary? metadata) + { + ArgumentNullException.ThrowIfNull(target); + + var builder = new StringBuilder(); + builder.Append('|'); + builder.Append(target.Reference?.Trim().ToLowerInvariant() ?? string.Empty); + builder.Append('|'); + builder.Append(target.Digest?.Trim().ToLowerInvariant() ?? string.Empty); + builder.Append("|force:"); + builder.Append(force ? '1' : '0'); + builder.Append("|client:"); + builder.Append(clientRequestId?.Trim().ToLowerInvariant() ?? string.Empty); + + if (metadata is not null && metadata.Count > 0) + { + foreach (var pair in metadata.OrderBy(static entry => entry.Key, StringComparer.OrdinalIgnoreCase)) + { + var key = pair.Key?.Trim().ToLowerInvariant() ?? string.Empty; + var value = pair.Value?.Trim() ?? string.Empty; + builder.Append('|'); + builder.Append(key); + builder.Append('='); + builder.Append(value); + } + } + + var canonical = builder.ToString(); + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(canonical)); + var hex = Convert.ToHexString(hash).ToLowerInvariant(); + var trimmed = hex.Length > 40 ? hex[..40] : hex; + return new ScanId(trimmed); + } +} diff --git a/src/StellaOps.Scanner.Worker.Tests/LeaseHeartbeatServiceTests.cs b/src/StellaOps.Scanner.Worker.Tests/LeaseHeartbeatServiceTests.cs new file mode 100644 index 00000000..ce8fa19a --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/LeaseHeartbeatServiceTests.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Worker.Options; +using StellaOps.Scanner.Worker.Processing; +using Xunit; + +namespace StellaOps.Scanner.Worker.Tests; + +public sealed class LeaseHeartbeatServiceTests +{ + [Fact] + public async Task RunAsync_RespectsSafetyFactorBudget() + { + var options = new ScannerWorkerOptions + { + MaxConcurrentJobs = 1, + }; + options.Queue.HeartbeatSafetyFactor = 3.0; + options.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(5); + options.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(60); + options.Queue.SetHeartbeatRetryDelays(Array.Empty()); + options.Queue.MaxHeartbeatJitterMilliseconds = 750; + + var optionsMonitor = new StaticOptionsMonitor(options); + using var cts = new CancellationTokenSource(); + var scheduler = new RecordingDelayScheduler(cts); + var lease = new TestJobLease(TimeSpan.FromSeconds(90)); + + var service = new LeaseHeartbeatService(TimeProvider.System, scheduler, optionsMonitor, NullLogger.Instance); + + await service.RunAsync(lease, cts.Token); + + var delay = Assert.Single(scheduler.Delays); + var expectedMax = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / Math.Max(3.0, options.Queue.HeartbeatSafetyFactor))); + Assert.True(delay <= expectedMax, $"Heartbeat delay {delay} should stay within safety factor budget {expectedMax}."); + Assert.True(delay >= options.Queue.MinHeartbeatInterval, $"Heartbeat delay {delay} should respect minimum interval {options.Queue.MinHeartbeatInterval}."); + } + + private sealed class RecordingDelayScheduler : IDelayScheduler + { + private readonly CancellationTokenSource _cts; + + public RecordingDelayScheduler(CancellationTokenSource cts) + { + _cts = cts ?? throw new ArgumentNullException(nameof(cts)); + } + + public List Delays { get; } = new(); + + public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + Delays.Add(delay); + _cts.Cancel(); + return Task.CompletedTask; + } + } + + private sealed class TestJobLease : IScanJobLease + { + public TestJobLease(TimeSpan leaseDuration) + { + LeaseDuration = leaseDuration; + EnqueuedAtUtc = DateTimeOffset.UtcNow - leaseDuration; + LeasedAtUtc = DateTimeOffset.UtcNow; + } + + public string JobId { get; } = Guid.NewGuid().ToString("n"); + + public string ScanId { get; } = $"scan-{Guid.NewGuid():n}"; + + public int Attempt { get; } = 1; + + public DateTimeOffset EnqueuedAtUtc { get; } + + public DateTimeOffset LeasedAtUtc { get; } + + public TimeSpan LeaseDuration { get; } + + public IReadOnlyDictionary Metadata { get; } = new Dictionary(); + + public ValueTask RenewAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask CompleteAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) => ValueTask.CompletedTask; + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + private sealed class StaticOptionsMonitor : IOptionsMonitor + where TOptions : class + { + private readonly TOptions _value; + + public StaticOptionsMonitor(TOptions value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + public TOptions CurrentValue => _value; + + public TOptions Get(string? name) => _value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + + public void Dispose() + { + } + } + } +} diff --git a/src/StellaOps.Scanner.Worker.Tests/RedisWorkerSmokeTests.cs b/src/StellaOps.Scanner.Worker.Tests/RedisWorkerSmokeTests.cs new file mode 100644 index 00000000..7cc5750d --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/RedisWorkerSmokeTests.cs @@ -0,0 +1,245 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Queue; +using StellaOps.Scanner.Worker.Diagnostics; +using StellaOps.Scanner.Worker.Hosting; +using StellaOps.Scanner.Worker.Options; +using StellaOps.Scanner.Worker.Processing; +using StellaOps.Scanner.Worker.Tests.TestInfrastructure; +using Xunit; + +namespace StellaOps.Scanner.Worker.Tests; + +public sealed class RedisWorkerSmokeTests +{ + [Fact] + public async Task Worker_CompletesJob_ViaRedisQueue() + { + var flag = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_SMOKE"); + if (string.IsNullOrWhiteSpace(flag)) + { + return; + } + + var redisConnection = Environment.GetEnvironmentVariable("STELLAOPS_REDIS_CONNECTION") ?? "localhost:6379"; + var streamName = $"scanner:jobs:{Guid.NewGuid():n}"; + var consumerGroup = $"worker-smoke-{Guid.NewGuid():n}"; + var configuration = BuildQueueConfiguration(redisConnection, streamName, consumerGroup); + + var queueOptions = new ScannerQueueOptions(); + configuration.GetSection("scanner:queue").Bind(queueOptions); + + var workerOptions = new ScannerWorkerOptions + { + MaxConcurrentJobs = 1, + }; + workerOptions.Queue.HeartbeatSafetyFactor = 3.0; + workerOptions.Queue.MinHeartbeatInterval = TimeSpan.FromSeconds(2); + workerOptions.Queue.MaxHeartbeatInterval = TimeSpan.FromSeconds(8); + workerOptions.Queue.SetHeartbeatRetryDelays(new[] + { + TimeSpan.FromMilliseconds(200), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(1), + }); + + var services = new ServiceCollection(); + services.AddLogging(builder => + { + builder.SetMinimumLevel(LogLevel.Debug); + builder.AddConsole(); + }); + services.AddSingleton(TimeProvider.System); + services.AddScannerQueue(configuration, "scanner:queue"); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(queueOptions); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton>(new StaticOptionsMonitor(workerOptions)); + services.AddSingleton(); + + using var provider = services.BuildServiceProvider(); + var queue = provider.GetRequiredService(); + + var jobId = $"job-{Guid.NewGuid():n}"; + var scanId = $"scan-{Guid.NewGuid():n}"; + await queue.EnqueueAsync(new ScanQueueMessage(jobId, Encoding.UTF8.GetBytes("smoke")) + { + Attributes = new Dictionary(StringComparer.Ordinal) + { + ["scanId"] = scanId, + ["queue"] = "redis", + } + }); + + var hostedService = provider.GetRequiredService(); + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + + await hostedService.StartAsync(cts.Token); + + var smokeObserver = provider.GetRequiredService(); + await smokeObserver.JobCompleted.Task.WaitAsync(TimeSpan.FromSeconds(20)); + + await hostedService.StopAsync(CancellationToken.None); + } + + private static IConfiguration BuildQueueConfiguration(string connection, string stream, string consumerGroup) + { + return new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary + { + ["scanner:queue:kind"] = "redis", + ["scanner:queue:defaultLeaseDuration"] = "00:00:30", + ["scanner:queue:redis:connectionString"] = connection, + ["scanner:queue:redis:streamName"] = stream, + ["scanner:queue:redis:consumerGroup"] = consumerGroup, + ["scanner:queue:redis:idempotencyKeyPrefix"] = $"{stream}:idemp:", + ["scanner:queue:redis:initializationTimeout"] = "00:00:10", + }) + .Build(); + } + + private sealed class SmokeAnalyzerDispatcher : IScanAnalyzerDispatcher + { + public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + return ValueTask.CompletedTask; + } + } + + private sealed class QueueBackedScanJobSourceDependencies + { + public QueueBackedScanJobSourceDependencies() + { + JobCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public TaskCompletionSource JobCompleted { get; } + } + + private sealed class QueueBackedScanJobSource : IScanJobSource + { + private readonly IScanQueue _queue; + private readonly ScannerQueueOptions _queueOptions; + private readonly QueueBackedScanJobSourceDependencies _deps; + private readonly TimeProvider _timeProvider; + private readonly string _consumerName = $"worker-smoke-{Guid.NewGuid():n}"; + + public QueueBackedScanJobSource( + IScanQueue queue, + ScannerQueueOptions queueOptions, + QueueBackedScanJobSourceDependencies deps, + TimeProvider timeProvider) + { + _queue = queue ?? throw new ArgumentNullException(nameof(queue)); + _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); + _deps = deps ?? throw new ArgumentNullException(nameof(deps)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async Task TryAcquireAsync(CancellationToken cancellationToken) + { + var request = new QueueLeaseRequest(_consumerName, 1, _queueOptions.DefaultLeaseDuration); + var leases = await _queue.LeaseAsync(request, cancellationToken).ConfigureAwait(false); + if (leases.Count == 0) + { + return null; + } + + return new QueueBackedScanJobLease( + leases[0], + _queueOptions, + _deps, + _timeProvider.GetUtcNow()); + } + } + + private sealed class QueueBackedScanJobLease : IScanJobLease + { + private readonly IScanQueueLease _lease; + private readonly ScannerQueueOptions _options; + private readonly QueueBackedScanJobSourceDependencies _deps; + private readonly DateTimeOffset _leasedAt; + private readonly IReadOnlyDictionary _metadata; + + public QueueBackedScanJobLease( + IScanQueueLease lease, + ScannerQueueOptions options, + QueueBackedScanJobSourceDependencies deps, + DateTimeOffset leasedAt) + { + _lease = lease ?? throw new ArgumentNullException(nameof(lease)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _deps = deps ?? throw new ArgumentNullException(nameof(deps)); + _leasedAt = leasedAt; + + var metadata = new Dictionary(StringComparer.Ordinal) + { + ["queue"] = _options.Kind.ToString(), + ["queue.consumer"] = lease.Consumer, + }; + + if (!string.IsNullOrWhiteSpace(lease.IdempotencyKey)) + { + metadata["job.idempotency"] = lease.IdempotencyKey; + } + + foreach (var attribute in lease.Attributes) + { + metadata[attribute.Key] = attribute.Value; + } + + _metadata = metadata; + } + + public string JobId => _lease.JobId; + + public string ScanId => _metadata.TryGetValue("scanId", out var scanId) ? scanId : _lease.JobId; + + public int Attempt => _lease.Attempt; + + public DateTimeOffset EnqueuedAtUtc => _lease.EnqueuedAt; + + public DateTimeOffset LeasedAtUtc => _leasedAt; + + public TimeSpan LeaseDuration => _lease.LeaseExpiresAt - _leasedAt; + + public IReadOnlyDictionary Metadata => _metadata; + + public async ValueTask RenewAsync(CancellationToken cancellationToken) + { + await _lease.RenewAsync(_options.DefaultLeaseDuration, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask CompleteAsync(CancellationToken cancellationToken) + { + await _lease.AcknowledgeAsync(cancellationToken).ConfigureAwait(false); + _deps.JobCompleted.TrySetResult(); + } + + public async ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) + { + await _lease.ReleaseAsync(QueueReleaseDisposition.Retry, cancellationToken).ConfigureAwait(false); + } + + public async ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) + { + await _lease.DeadLetterAsync(reason, cancellationToken).ConfigureAwait(false); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } +} diff --git a/src/StellaOps.Scanner.Worker.Tests/ScannerWorkerOptionsValidatorTests.cs b/src/StellaOps.Scanner.Worker.Tests/ScannerWorkerOptionsValidatorTests.cs new file mode 100644 index 00000000..5d56c22d --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/ScannerWorkerOptionsValidatorTests.cs @@ -0,0 +1,34 @@ +using System; +using System.Linq; +using StellaOps.Scanner.Worker.Options; +using Xunit; + +namespace StellaOps.Scanner.Worker.Tests; + +public sealed class ScannerWorkerOptionsValidatorTests +{ + [Fact] + public void Validate_Fails_WhenHeartbeatSafetyFactorBelowThree() + { + var options = new ScannerWorkerOptions(); + options.Queue.HeartbeatSafetyFactor = 2.5; + + var validator = new ScannerWorkerOptionsValidator(); + var result = validator.Validate(string.Empty, options); + + Assert.True(result.Failed, "Validation should fail when HeartbeatSafetyFactor < 3."); + Assert.Contains(result.Failures, failure => failure.Contains("HeartbeatSafetyFactor", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void Validate_Succeeds_WhenHeartbeatSafetyFactorAtLeastThree() + { + var options = new ScannerWorkerOptions(); + options.Queue.HeartbeatSafetyFactor = 3.5; + + var validator = new ScannerWorkerOptionsValidator(); + var result = validator.Validate(string.Empty, options); + + Assert.True(result.Succeeded, "Validation should succeed when HeartbeatSafetyFactor >= 3."); + } +} diff --git a/src/StellaOps.Scanner.Worker.Tests/StellaOps.Scanner.Worker.Tests.csproj b/src/StellaOps.Scanner.Worker.Tests/StellaOps.Scanner.Worker.Tests.csproj new file mode 100644 index 00000000..265caf0c --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/StellaOps.Scanner.Worker.Tests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + preview + enable + enable + false + + + + + + diff --git a/src/StellaOps.Scanner.Worker.Tests/TestInfrastructure/StaticOptionsMonitor.cs b/src/StellaOps.Scanner.Worker.Tests/TestInfrastructure/StaticOptionsMonitor.cs new file mode 100644 index 00000000..1dbd7ea6 --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/TestInfrastructure/StaticOptionsMonitor.cs @@ -0,0 +1,29 @@ +using System; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scanner.Worker.Tests.TestInfrastructure; + +public sealed class StaticOptionsMonitor : IOptionsMonitor + where TOptions : class +{ + private readonly TOptions _value; + + public StaticOptionsMonitor(TOptions value) + { + _value = value ?? throw new ArgumentNullException(nameof(value)); + } + + public TOptions CurrentValue => _value; + + public TOptions Get(string? name) => _value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + public void Dispose() + { + } + } +} diff --git a/src/StellaOps.Scanner.Worker.Tests/WorkerBasicScanScenarioTests.cs b/src/StellaOps.Scanner.Worker.Tests/WorkerBasicScanScenarioTests.cs new file mode 100644 index 00000000..a0b49712 --- /dev/null +++ b/src/StellaOps.Scanner.Worker.Tests/WorkerBasicScanScenarioTests.cs @@ -0,0 +1,450 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; +using StellaOps.Scanner.Worker.Diagnostics; +using StellaOps.Scanner.Worker.Hosting; +using StellaOps.Scanner.Worker.Options; +using StellaOps.Scanner.Worker.Processing; +using Xunit; + +namespace StellaOps.Scanner.Worker.Tests; + +public sealed class WorkerBasicScanScenarioTests +{ + [Fact] + public async Task DelayAsync_CompletesAfterTimeAdvance() + { + var scheduler = new ControlledDelayScheduler(); + var delayTask = scheduler.DelayAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + scheduler.AdvanceBy(TimeSpan.FromSeconds(5)); + await delayTask.WaitAsync(TimeSpan.FromSeconds(1)); + } + + [Fact] + public async Task Worker_CompletesJob_RecordsTelemetry_And_Heartbeats() + { + var fakeTime = new FakeTimeProvider(); + fakeTime.SetUtcNow(DateTimeOffset.UtcNow); + + var options = new ScannerWorkerOptions + { + MaxConcurrentJobs = 1, + }; + options.Telemetry.EnableTelemetry = false; + options.Telemetry.EnableMetrics = true; + + var optionsMonitor = new StaticOptionsMonitor(options); + var testLoggerProvider = new TestLoggerProvider(); + var lease = new TestJobLease(fakeTime); + var jobSource = new TestJobSource(lease); + var scheduler = new ControlledDelayScheduler(); + var analyzer = new TestAnalyzerDispatcher(scheduler); + + using var listener = new WorkerMetricsListener(); + listener.Start(); + + using var services = new ServiceCollection() + .AddLogging(builder => + { + builder.ClearProviders(); + builder.AddProvider(testLoggerProvider); + builder.SetMinimumLevel(LogLevel.Debug); + }) + .AddSingleton(fakeTime) + .AddSingleton(fakeTime) + .AddSingleton>(optionsMonitor) + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton() + .AddSingleton(scheduler) + .AddSingleton(_ => jobSource) + .AddSingleton(analyzer) + .AddSingleton() + .AddSingleton() + .BuildServiceProvider(); + + var worker = services.GetRequiredService(); + + await worker.StartAsync(CancellationToken.None); + + await jobSource.LeaseIssued.Task.WaitAsync(TimeSpan.FromSeconds(5)); + await Task.Yield(); + + var spin = 0; + while (!lease.Completed.Task.IsCompleted && spin++ < 24) + { + fakeTime.Advance(TimeSpan.FromSeconds(15)); + scheduler.AdvanceBy(TimeSpan.FromSeconds(15)); + await Task.Delay(1); + } + + try + { + await lease.Completed.Task.WaitAsync(TimeSpan.FromSeconds(30)); + } + catch (TimeoutException ex) + { + var stageLogs = string.Join(Environment.NewLine, testLoggerProvider + .GetEntriesForCategory(typeof(ScanProgressReporter).FullName!) + .Select(entry => entry.ToFormattedString())); + + throw new TimeoutException($"Worker did not complete within timeout. Logs:{Environment.NewLine}{stageLogs}", ex); + } + + await worker.StopAsync(CancellationToken.None); + + Assert.True(lease.Completed.Task.IsCompletedSuccessfully, "Job should complete successfully."); + Assert.Single(analyzer.Executions); + Assert.True(lease.RenewalCount >= 1, "Lease should have been renewed at least once."); + + var stageOrder = testLoggerProvider + .GetEntriesForCategory(typeof(ScanProgressReporter).FullName!) + .Where(entry => entry.EventId.Id == 1000) + .Select(entry => entry.GetScopeProperty("Stage")) + .Where(stage => stage is not null) + .Cast() + .ToArray(); + + Assert.Equal(ScanStageNames.Ordered, stageOrder); + + var queueLatency = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_queue_latency_ms").ToArray(); + Assert.Single(queueLatency); + Assert.True(queueLatency[0].Value > 0, "Queue latency should be positive."); + + var jobDuration = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_job_duration_ms").ToArray(); + Assert.Single(jobDuration); + Assert.True(jobDuration[0].Value > 0, "Job duration should be positive."); + + var stageDurations = listener.Measurements.Where(m => m.InstrumentName == "scanner_worker_stage_duration_ms").ToArray(); + Assert.Contains(stageDurations, m => m.Tags.TryGetValue("stage", out var stage) && Equals(stage, ScanStageNames.ExecuteAnalyzers)); + } + + private sealed class TestJobSource : IScanJobSource + { + private readonly TestJobLease _lease; + private int _delivered; + + public TestJobSource(TestJobLease lease) + { + _lease = lease; + } + + public TaskCompletionSource LeaseIssued { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task TryAcquireAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _delivered, 1) == 0) + { + LeaseIssued.TrySetResult(); + return Task.FromResult(_lease); + } + + return Task.FromResult(null); + } + } + + private sealed class TestJobLease : IScanJobLease + { + private readonly FakeTimeProvider _timeProvider; + private readonly Dictionary _metadata = new() + { + { "queue", "tests" }, + { "job.kind", "basic" }, + }; + + public TestJobLease(FakeTimeProvider timeProvider) + { + _timeProvider = timeProvider; + EnqueuedAtUtc = _timeProvider.GetUtcNow() - TimeSpan.FromSeconds(5); + LeasedAtUtc = _timeProvider.GetUtcNow(); + } + + public string JobId { get; } = Guid.NewGuid().ToString("n"); + + public string ScanId { get; } = $"scan-{Guid.NewGuid():n}"; + + public int Attempt { get; } = 1; + + public DateTimeOffset EnqueuedAtUtc { get; } + + public DateTimeOffset LeasedAtUtc { get; } + + public TimeSpan LeaseDuration { get; } = TimeSpan.FromSeconds(90); + + public IReadOnlyDictionary Metadata => _metadata; + + public TaskCompletionSource Completed { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int RenewalCount => _renewalCount; + + public ValueTask RenewAsync(CancellationToken cancellationToken) + { + Interlocked.Increment(ref _renewalCount); + return ValueTask.CompletedTask; + } + + public ValueTask CompleteAsync(CancellationToken cancellationToken) + { + Completed.TrySetResult(); + return ValueTask.CompletedTask; + } + + public ValueTask AbandonAsync(string reason, CancellationToken cancellationToken) + { + Completed.TrySetException(new InvalidOperationException($"Abandoned: {reason}")); + return ValueTask.CompletedTask; + } + + public ValueTask PoisonAsync(string reason, CancellationToken cancellationToken) + { + Completed.TrySetException(new InvalidOperationException($"Poisoned: {reason}")); + return ValueTask.CompletedTask; + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private int _renewalCount; + } + + private sealed class TestAnalyzerDispatcher : IScanAnalyzerDispatcher + { + private readonly IDelayScheduler _scheduler; + + public TestAnalyzerDispatcher(IDelayScheduler scheduler) + { + _scheduler = scheduler; + } + + public List Executions { get; } = new(); + + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + Executions.Add(context.JobId); + await _scheduler.DelayAsync(TimeSpan.FromSeconds(45), cancellationToken); + } + } + + private sealed class ControlledDelayScheduler : IDelayScheduler + { + private readonly object _lock = new(); + private readonly SortedDictionary> _scheduled = new(); + private double _currentMilliseconds; + + public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + if (delay <= TimeSpan.Zero) + { + return Task.CompletedTask; + } + + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var scheduled = new ScheduledDelay(tcs, cancellationToken); + lock (_lock) + { + var due = _currentMilliseconds + delay.TotalMilliseconds; + if (!_scheduled.TryGetValue(due, out var list)) + { + list = new List(); + _scheduled.Add(due, list); + } + + list.Add(scheduled); + } + + return scheduled.Task; + } + + public void AdvanceBy(TimeSpan delta) + { + lock (_lock) + { + _currentMilliseconds += delta.TotalMilliseconds; + var dueKeys = _scheduled.Keys.Where(key => key <= _currentMilliseconds).ToList(); + foreach (var due in dueKeys) + { + foreach (var scheduled in _scheduled[due]) + { + scheduled.Complete(); + } + + _scheduled.Remove(due); + } + } + } + + private sealed class ScheduledDelay + { + private readonly TaskCompletionSource _tcs; + private readonly CancellationTokenRegistration _registration; + + public ScheduledDelay(TaskCompletionSource tcs, CancellationToken cancellationToken) + { + _tcs = tcs; + if (cancellationToken.CanBeCanceled) + { + _registration = cancellationToken.Register(state => + { + var source = (TaskCompletionSource)state!; + source.TrySetCanceled(cancellationToken); + }, tcs); + } + } + + public Task Task => _tcs.Task; + + public void Complete() + { + _registration.Dispose(); + _tcs.TrySetResult(null); + } + } + } + + private sealed class StaticOptionsMonitor : IOptionsMonitor + where TOptions : class + { + private readonly TOptions _value; + + public StaticOptionsMonitor(TOptions value) + { + _value = value; + } + + public TOptions CurrentValue => _value; + + public TOptions Get(string? name) => _value; + + public IDisposable OnChange(Action listener) => NullDisposable.Instance; + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + public void Dispose() + { + } + } + } + + private sealed class WorkerMetricsListener : IDisposable + { + private readonly MeterListener _listener; + public ConcurrentBag Measurements { get; } = new(); + + public WorkerMetricsListener() + { + _listener = new MeterListener + { + InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == ScannerWorkerInstrumentation.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + } + }; + + _listener.SetMeasurementEventCallback((instrument, measurement, tags, state) => + { + var tagDictionary = new Dictionary(tags.Length, StringComparer.Ordinal); + foreach (var tag in tags) + { + tagDictionary[tag.Key] = tag.Value; + } + + Measurements.Add(new Measurement(instrument.Name, measurement, tagDictionary)); + }); + } + + public void Start() => _listener.Start(); + + public void Dispose() => _listener.Dispose(); + } + + public sealed record Measurement(string InstrumentName, double Value, IReadOnlyDictionary Tags) + { + public object? this[string name] => Tags.TryGetValue(name, out var value) ? value : null; + } + + private sealed class TestLoggerProvider : ILoggerProvider + { + private readonly ConcurrentQueue _entries = new(); + + public ILogger CreateLogger(string categoryName) => new TestLogger(categoryName, _entries); + + public void Dispose() + { + } + + public IEnumerable GetEntriesForCategory(string categoryName) + => _entries.Where(entry => entry.Category == categoryName); + + private sealed class TestLogger : ILogger + { + private readonly string _category; + private readonly ConcurrentQueue _entries; + + public TestLogger(string category, ConcurrentQueue entries) + { + _category = category; + _entries = entries; + } + + public IDisposable? BeginScope(TState state) where TState : notnull => NullDisposable.Instance; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) + { + _entries.Enqueue(new TestLogEntry(_category, logLevel, eventId, state, exception)); + } + } + + private sealed class NullDisposable : IDisposable + { + public static readonly NullDisposable Instance = new(); + public void Dispose() + { + } + } + } + + public sealed record TestLogEntry(string Category, LogLevel Level, EventId EventId, object? State, Exception? Exception) + { + public T? GetScopeProperty(string name) + { + if (State is not IEnumerable> state) + { + return default; + } + + foreach (var kvp in state) + { + if (string.Equals(kvp.Key, name, StringComparison.OrdinalIgnoreCase) && kvp.Value is T value) + { + return value; + } + } + + return default; + } + + public string ToFormattedString() + { + var properties = State is IEnumerable> kvps + ? string.Join(", ", kvps.Select(kvp => $"{kvp.Key}={kvp.Value}")) + : State?.ToString() ?? string.Empty; + + var exceptionPart = Exception is null ? string.Empty : $" Exception={Exception.GetType().Name}: {Exception.Message}"; + return $"[{Level}] {Category} ({EventId.Id}) {properties}{exceptionPart}"; + } + } +} diff --git a/src/StellaOps.Scanner.Worker/AGENTS.md b/src/StellaOps.Scanner.Worker/AGENTS.md new file mode 100644 index 00000000..ae5d9277 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/AGENTS.md @@ -0,0 +1,26 @@ +# AGENTS +## Role +Scanner.Worker engineers own the queue-driven execution host that turns scan jobs into SBOM artefacts with deterministic progress reporting. +## Scope +- Host bootstrap: configuration binding, Authority client wiring, graceful shutdown, restart-time plug-in discovery hooks. +- Job acquisition & lease renewal semantics backed by the Scanner queue abstraction. +- Analyzer orchestration skeleton: stage pipeline, cancellation awareness, deterministic progress emissions. +- Telemetry: structured logging, OpenTelemetry metrics/traces, health counters for offline diagnostics. +## Participants +- Consumes jobs from `StellaOps.Scanner.Queue`. +- Persists progress/artifacts via `StellaOps.Scanner.Storage` once those modules land. +- Emits metrics and structured logs consumed by Observability stack & WebService status endpoints. +## Interfaces & contracts +- Queue lease abstraction (`IScanJobLease`, `IScanJobSource`) with deterministic identifiers and attempt counters. +- Analyzer dispatcher contracts for OS/lang/native analyzers and emitters. +- Telemetry resource attributes shared with Scanner.WebService and Scheduler. +## In/Out of scope +In scope: worker host, concurrency orchestration, lease renewal, cancellation wiring, deterministic logging/metrics. +Out of scope: queue provider implementations, analyzer business logic, Mongo/object-store repositories. +## Observability expectations +- Meter `StellaOps.Scanner.Worker` with queue latency, stage duration, failure counters. +- Activity source `StellaOps.Scanner.Worker.Job` for per-job tracing. +- Log correlation IDs (`jobId`, `leaseId`, `scanId`) with structured payloads; avoid dumping secrets or full manifests. +## Tests +- Integration fixture `WorkerBasicScanScenario` verifying acquisition → heartbeat → analyzer stages → completion. +- Unit tests around retry/jitter calculators as they are introduced. diff --git a/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerInstrumentation.cs b/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerInstrumentation.cs new file mode 100644 index 00000000..aeecaef9 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerInstrumentation.cs @@ -0,0 +1,15 @@ +using System.Diagnostics; +using System.Diagnostics.Metrics; + +namespace StellaOps.Scanner.Worker.Diagnostics; + +public static class ScannerWorkerInstrumentation +{ + public const string ActivitySourceName = "StellaOps.Scanner.Worker.Job"; + + public const string MeterName = "StellaOps.Scanner.Worker"; + + public static ActivitySource ActivitySource { get; } = new(ActivitySourceName); + + public static Meter Meter { get; } = new(MeterName, version: "1.0.0"); +} diff --git a/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs b/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs new file mode 100644 index 00000000..ff6005f5 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Diagnostics/ScannerWorkerMetrics.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using StellaOps.Scanner.Worker.Processing; + +namespace StellaOps.Scanner.Worker.Diagnostics; + +public sealed class ScannerWorkerMetrics +{ + private readonly Histogram _queueLatencyMs; + private readonly Histogram _jobDurationMs; + private readonly Histogram _stageDurationMs; + private readonly Counter _jobsCompleted; + private readonly Counter _jobsFailed; + + public ScannerWorkerMetrics() + { + _queueLatencyMs = ScannerWorkerInstrumentation.Meter.CreateHistogram( + "scanner_worker_queue_latency_ms", + unit: "ms", + description: "Time from job enqueue to lease acquisition."); + _jobDurationMs = ScannerWorkerInstrumentation.Meter.CreateHistogram( + "scanner_worker_job_duration_ms", + unit: "ms", + description: "Total processing duration per job."); + _stageDurationMs = ScannerWorkerInstrumentation.Meter.CreateHistogram( + "scanner_worker_stage_duration_ms", + unit: "ms", + description: "Stage execution duration per job."); + _jobsCompleted = ScannerWorkerInstrumentation.Meter.CreateCounter( + "scanner_worker_jobs_completed_total", + description: "Number of successfully completed scan jobs."); + _jobsFailed = ScannerWorkerInstrumentation.Meter.CreateCounter( + "scanner_worker_jobs_failed_total", + description: "Number of scan jobs that failed permanently."); + } + + public void RecordQueueLatency(ScanJobContext context, TimeSpan latency) + { + if (latency <= TimeSpan.Zero) + { + return; + } + + _queueLatencyMs.Record(latency.TotalMilliseconds, CreateTags(context)); + } + + public void RecordJobDuration(ScanJobContext context, TimeSpan duration) + { + if (duration <= TimeSpan.Zero) + { + return; + } + + _jobDurationMs.Record(duration.TotalMilliseconds, CreateTags(context)); + } + + public void RecordStageDuration(ScanJobContext context, string stage, TimeSpan duration) + { + if (duration <= TimeSpan.Zero) + { + return; + } + + _stageDurationMs.Record(duration.TotalMilliseconds, CreateTags(context, stage: stage)); + } + + public void IncrementJobCompleted(ScanJobContext context) + { + _jobsCompleted.Add(1, CreateTags(context)); + } + + public void IncrementJobFailed(ScanJobContext context, string failureReason) + { + _jobsFailed.Add(1, CreateTags(context, failureReason: failureReason)); + } + + private static KeyValuePair[] CreateTags(ScanJobContext context, string? stage = null, string? failureReason = null) + { + var tags = new List>(stage is null ? 5 : 6) + { + new("job.id", context.JobId), + new("scan.id", context.ScanId), + new("attempt", context.Lease.Attempt), + }; + + if (context.Lease.Metadata.TryGetValue("queue", out var queueName) && !string.IsNullOrWhiteSpace(queueName)) + { + tags.Add(new KeyValuePair("queue", queueName)); + } + + if (context.Lease.Metadata.TryGetValue("job.kind", out var jobKind) && !string.IsNullOrWhiteSpace(jobKind)) + { + tags.Add(new KeyValuePair("job.kind", jobKind)); + } + + if (!string.IsNullOrWhiteSpace(stage)) + { + tags.Add(new KeyValuePair("stage", stage)); + } + + if (!string.IsNullOrWhiteSpace(failureReason)) + { + tags.Add(new KeyValuePair("reason", failureReason)); + } + + return tags.ToArray(); + } +} diff --git a/src/StellaOps.Scanner.Worker/Diagnostics/TelemetryExtensions.cs b/src/StellaOps.Scanner.Worker/Diagnostics/TelemetryExtensions.cs new file mode 100644 index 00000000..2b085dd0 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Diagnostics/TelemetryExtensions.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Reflection; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; +using StellaOps.Scanner.Worker.Options; + +namespace StellaOps.Scanner.Worker.Diagnostics; + +public static class TelemetryExtensions +{ + public static void ConfigureScannerWorkerTelemetry(this IHostApplicationBuilder builder, ScannerWorkerOptions options) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentNullException.ThrowIfNull(options); + + var telemetry = options.Telemetry; + if (!telemetry.EnableTelemetry) + { + return; + } + + var openTelemetry = builder.Services.AddOpenTelemetry(); + + openTelemetry.ConfigureResource(resource => + { + var version = Assembly.GetExecutingAssembly().GetName().Version?.ToString() ?? "unknown"; + resource.AddService(telemetry.ServiceName, serviceVersion: version, serviceInstanceId: Environment.MachineName); + resource.AddAttributes(new[] + { + new KeyValuePair("deployment.environment", builder.Environment.EnvironmentName), + }); + + foreach (var kvp in telemetry.ResourceAttributes) + { + if (string.IsNullOrWhiteSpace(kvp.Key) || kvp.Value is null) + { + continue; + } + + resource.AddAttributes(new[] { new KeyValuePair(kvp.Key, kvp.Value) }); + } + }); + + if (telemetry.EnableTracing) + { + openTelemetry.WithTracing(tracing => + { + tracing.AddSource(ScannerWorkerInstrumentation.ActivitySourceName); + ConfigureExporter(tracing, telemetry); + }); + } + + if (telemetry.EnableMetrics) + { + openTelemetry.WithMetrics(metrics => + { + metrics + .AddMeter( + ScannerWorkerInstrumentation.MeterName, + "StellaOps.Scanner.Analyzers.Lang.Node") + .AddRuntimeInstrumentation() + .AddProcessInstrumentation(); + + ConfigureExporter(metrics, telemetry); + }); + } + } + + private static void ConfigureExporter(TracerProviderBuilder tracing, ScannerWorkerOptions.TelemetryOptions telemetry) + { + if (!string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + tracing.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + }); + } + + if (telemetry.ExportConsole || string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + tracing.AddConsoleExporter(); + } + } + + private static void ConfigureExporter(MeterProviderBuilder metrics, ScannerWorkerOptions.TelemetryOptions telemetry) + { + if (!string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + metrics.AddOtlpExporter(options => + { + options.Endpoint = new Uri(telemetry.OtlpEndpoint); + }); + } + + if (telemetry.ExportConsole || string.IsNullOrWhiteSpace(telemetry.OtlpEndpoint)) + { + metrics.AddConsoleExporter(); + } + } +} diff --git a/src/StellaOps.Scanner.Worker/Hosting/ScannerWorkerHostedService.cs b/src/StellaOps.Scanner.Worker/Hosting/ScannerWorkerHostedService.cs new file mode 100644 index 00000000..5b2bdcf9 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Hosting/ScannerWorkerHostedService.cs @@ -0,0 +1,202 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Worker.Diagnostics; +using StellaOps.Scanner.Worker.Options; +using StellaOps.Scanner.Worker.Processing; + +namespace StellaOps.Scanner.Worker.Hosting; + +public sealed partial class ScannerWorkerHostedService : BackgroundService +{ + private readonly IScanJobSource _jobSource; + private readonly ScanJobProcessor _processor; + private readonly LeaseHeartbeatService _heartbeatService; + private readonly ScannerWorkerMetrics _metrics; + private readonly TimeProvider _timeProvider; + private readonly IOptionsMonitor _options; + private readonly ILogger _logger; + private readonly IDelayScheduler _delayScheduler; + + public ScannerWorkerHostedService( + IScanJobSource jobSource, + ScanJobProcessor processor, + LeaseHeartbeatService heartbeatService, + ScannerWorkerMetrics metrics, + TimeProvider timeProvider, + IDelayScheduler delayScheduler, + IOptionsMonitor options, + ILogger logger) + { + _jobSource = jobSource ?? throw new ArgumentNullException(nameof(jobSource)); + _processor = processor ?? throw new ArgumentNullException(nameof(processor)); + _heartbeatService = heartbeatService ?? throw new ArgumentNullException(nameof(heartbeatService)); + _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + var runningJobs = new HashSet(); + var delayStrategy = new PollDelayStrategy(_options.CurrentValue.Polling); + + WorkerStarted(_logger); + + while (!stoppingToken.IsCancellationRequested) + { + runningJobs.RemoveWhere(static task => task.IsCompleted); + + var options = _options.CurrentValue; + if (runningJobs.Count >= options.MaxConcurrentJobs) + { + var completed = await Task.WhenAny(runningJobs).ConfigureAwait(false); + runningJobs.Remove(completed); + continue; + } + + IScanJobLease? lease = null; + try + { + lease = await _jobSource.TryAcquireAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.LogError(ex, "Scanner worker failed to acquire job lease; backing off."); + } + + if (lease is null) + { + var delay = delayStrategy.NextDelay(); + await _delayScheduler.DelayAsync(delay, stoppingToken).ConfigureAwait(false); + continue; + } + + delayStrategy.Reset(); + runningJobs.Add(RunJobAsync(lease, stoppingToken)); + } + + if (runningJobs.Count > 0) + { + await Task.WhenAll(runningJobs).ConfigureAwait(false); + } + + WorkerStopping(_logger); + } + + private async Task RunJobAsync(IScanJobLease lease, CancellationToken stoppingToken) + { + var options = _options.CurrentValue; + var jobStart = _timeProvider.GetUtcNow(); + var queueLatency = jobStart - lease.EnqueuedAtUtc; + var jobCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + var jobToken = jobCts.Token; + var context = new ScanJobContext(lease, _timeProvider, jobStart, jobToken); + + _metrics.RecordQueueLatency(context, queueLatency); + JobAcquired(_logger, lease.JobId, lease.ScanId, lease.Attempt, queueLatency.TotalMilliseconds); + + var processingTask = _processor.ExecuteAsync(context, jobToken).AsTask(); + var heartbeatTask = _heartbeatService.RunAsync(lease, jobToken); + Exception? processingException = null; + + try + { + await processingTask.ConfigureAwait(false); + jobCts.Cancel(); + await heartbeatTask.ConfigureAwait(false); + await lease.CompleteAsync(stoppingToken).ConfigureAwait(false); + var duration = _timeProvider.GetUtcNow() - jobStart; + _metrics.RecordJobDuration(context, duration); + _metrics.IncrementJobCompleted(context); + JobCompleted(_logger, lease.JobId, lease.ScanId, duration.TotalMilliseconds); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + processingException = null; + await lease.AbandonAsync("host-stopping", CancellationToken.None).ConfigureAwait(false); + JobAbandoned(_logger, lease.JobId, lease.ScanId); + } + catch (Exception ex) + { + processingException = ex; + var duration = _timeProvider.GetUtcNow() - jobStart; + _metrics.RecordJobDuration(context, duration); + + var reason = ex.GetType().Name; + var maxAttempts = options.Queue.MaxAttempts; + if (lease.Attempt >= maxAttempts) + { + await lease.PoisonAsync(reason, CancellationToken.None).ConfigureAwait(false); + _metrics.IncrementJobFailed(context, reason); + JobPoisoned(_logger, lease.JobId, lease.ScanId, lease.Attempt, maxAttempts, ex); + } + else + { + await lease.AbandonAsync(reason, CancellationToken.None).ConfigureAwait(false); + JobAbandonedWithError(_logger, lease.JobId, lease.ScanId, lease.Attempt, maxAttempts, ex); + } + } + finally + { + jobCts.Cancel(); + try + { + await heartbeatTask.ConfigureAwait(false); + } + catch (Exception ex) when (processingException is null && ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Heartbeat loop ended with an exception for job {JobId}.", lease.JobId); + } + + await lease.DisposeAsync().ConfigureAwait(false); + jobCts.Dispose(); + } + } + + [LoggerMessage(EventId = 2000, Level = LogLevel.Information, Message = "Scanner worker host started.")] + private static partial void WorkerStarted(ILogger logger); + + [LoggerMessage(EventId = 2001, Level = LogLevel.Information, Message = "Scanner worker host stopping.")] + private static partial void WorkerStopping(ILogger logger); + + [LoggerMessage( + EventId = 2002, + Level = LogLevel.Information, + Message = "Leased job {JobId} (scan {ScanId}) attempt {Attempt}; queue latency {LatencyMs:F0} ms.")] + private static partial void JobAcquired(ILogger logger, string jobId, string scanId, int attempt, double latencyMs); + + [LoggerMessage( + EventId = 2003, + Level = LogLevel.Information, + Message = "Job {JobId} (scan {ScanId}) completed in {DurationMs:F0} ms.")] + private static partial void JobCompleted(ILogger logger, string jobId, string scanId, double durationMs); + + [LoggerMessage( + EventId = 2004, + Level = LogLevel.Warning, + Message = "Job {JobId} (scan {ScanId}) abandoned due to host shutdown.")] + private static partial void JobAbandoned(ILogger logger, string jobId, string scanId); + + [LoggerMessage( + EventId = 2005, + Level = LogLevel.Warning, + Message = "Job {JobId} (scan {ScanId}) attempt {Attempt}/{MaxAttempts} abandoned after failure; job will be retried.")] + private static partial void JobAbandonedWithError(ILogger logger, string jobId, string scanId, int attempt, int maxAttempts, Exception exception); + + [LoggerMessage( + EventId = 2006, + Level = LogLevel.Error, + Message = "Job {JobId} (scan {ScanId}) attempt {Attempt}/{MaxAttempts} exceeded retry budget; quarantining job.")] + private static partial void JobPoisoned(ILogger logger, string jobId, string scanId, int attempt, int maxAttempts, Exception exception); +} diff --git a/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs b/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs new file mode 100644 index 00000000..464ee264 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptions.cs @@ -0,0 +1,163 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Worker.Options; + +public sealed class ScannerWorkerOptions +{ + public const string SectionName = "Scanner:Worker"; + + public int MaxConcurrentJobs { get; set; } = 2; + + public QueueOptions Queue { get; } = new(); + + public PollingOptions Polling { get; } = new(); + + public AuthorityOptions Authority { get; } = new(); + + public TelemetryOptions Telemetry { get; } = new(); + + public ShutdownOptions Shutdown { get; } = new(); + + public AnalyzerOptions Analyzers { get; } = new(); + + public sealed class QueueOptions + { + public int MaxAttempts { get; set; } = 5; + + public double HeartbeatSafetyFactor { get; set; } = 3.0; + + public int MaxHeartbeatJitterMilliseconds { get; set; } = 750; + + public IReadOnlyList HeartbeatRetryDelays => _heartbeatRetryDelays; + + public TimeSpan MinHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(10); + + public TimeSpan MaxHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(30); + + public void SetHeartbeatRetryDelays(IEnumerable delays) + { + _heartbeatRetryDelays = NormalizeDelays(delays); + } + + internal IReadOnlyList NormalizedHeartbeatRetryDelays => _heartbeatRetryDelays; + + private static IReadOnlyList NormalizeDelays(IEnumerable delays) + { + var buffer = new List(); + foreach (var delay in delays) + { + if (delay <= TimeSpan.Zero) + { + continue; + } + + buffer.Add(delay); + } + + buffer.Sort(); + return new ReadOnlyCollection(buffer); + } + + private IReadOnlyList _heartbeatRetryDelays = new ReadOnlyCollection(new TimeSpan[] + { + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(10), + }); + } + + public sealed class PollingOptions + { + public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMilliseconds(200); + + public TimeSpan MaxDelay { get; set; } = TimeSpan.FromSeconds(5); + + public double JitterRatio { get; set; } = 0.2; + } + + public sealed class AuthorityOptions + { + public bool Enabled { get; set; } + + public string? Issuer { get; set; } + + public string? ClientId { get; set; } + + public string? ClientSecret { get; set; } + + public bool RequireHttpsMetadata { get; set; } = true; + + public string? MetadataAddress { get; set; } + + public int BackchannelTimeoutSeconds { get; set; } = 20; + + public int TokenClockSkewSeconds { get; set; } = 30; + + public IList Scopes { get; } = new List { "scanner.scan" }; + + public ResilienceOptions Resilience { get; } = new(); + } + + public sealed class ResilienceOptions + { + public bool? EnableRetries { get; set; } + + public IList RetryDelays { get; } = new List + { + TimeSpan.FromMilliseconds(250), + TimeSpan.FromMilliseconds(500), + TimeSpan.FromSeconds(1), + TimeSpan.FromSeconds(5), + }; + + public bool? AllowOfflineCacheFallback { get; set; } + + public TimeSpan? OfflineCacheTolerance { get; set; } + } + + public sealed class TelemetryOptions + { + public bool EnableLogging { get; set; } = true; + + public bool EnableTelemetry { get; set; } = true; + + public bool EnableTracing { get; set; } + + public bool EnableMetrics { get; set; } = true; + + public string ServiceName { get; set; } = "stellaops-scanner-worker"; + + public string? OtlpEndpoint { get; set; } + + public bool ExportConsole { get; set; } + + public IDictionary ResourceAttributes { get; } = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + } + + public sealed class ShutdownOptions + { + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30); + } + + public sealed class AnalyzerOptions + { + public AnalyzerOptions() + { + PluginDirectories = new List + { + Path.Combine("plugins", "scanner", "analyzers", "os"), + }; + } + + public IList PluginDirectories { get; } + + public string RootFilesystemMetadataKey { get; set; } = ScanMetadataKeys.RootFilesystemPath; + + public string WorkspaceMetadataKey { get; set; } = ScanMetadataKeys.WorkspacePath; + } +} diff --git a/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptionsValidator.cs b/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptionsValidator.cs new file mode 100644 index 00000000..66ac9879 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Options/ScannerWorkerOptionsValidator.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Options; + +namespace StellaOps.Scanner.Worker.Options; + +public sealed class ScannerWorkerOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, ScannerWorkerOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var failures = new List(); + + if (options.MaxConcurrentJobs <= 0) + { + failures.Add("Scanner.Worker:MaxConcurrentJobs must be greater than zero."); + } + + if (options.Queue.HeartbeatSafetyFactor < 3.0) + { + failures.Add("Scanner.Worker:Queue:HeartbeatSafetyFactor must be at least 3."); + } + + if (options.Queue.MaxAttempts <= 0) + { + failures.Add("Scanner.Worker:Queue:MaxAttempts must be greater than zero."); + } + + if (options.Queue.MinHeartbeatInterval <= TimeSpan.Zero) + { + failures.Add("Scanner.Worker:Queue:MinHeartbeatInterval must be greater than zero."); + } + + if (options.Queue.MaxHeartbeatInterval <= options.Queue.MinHeartbeatInterval) + { + failures.Add("Scanner.Worker:Queue:MaxHeartbeatInterval must be greater than MinHeartbeatInterval."); + } + + if (options.Polling.InitialDelay <= TimeSpan.Zero) + { + failures.Add("Scanner.Worker:Polling:InitialDelay must be greater than zero."); + } + + if (options.Polling.MaxDelay < options.Polling.InitialDelay) + { + failures.Add("Scanner.Worker:Polling:MaxDelay must be greater than or equal to InitialDelay."); + } + + if (options.Polling.JitterRatio is < 0 or > 1) + { + failures.Add("Scanner.Worker:Polling:JitterRatio must be between 0 and 1."); + } + + if (options.Authority.Enabled) + { + if (string.IsNullOrWhiteSpace(options.Authority.Issuer)) + { + failures.Add("Scanner.Worker:Authority requires Issuer when Enabled is true."); + } + + if (string.IsNullOrWhiteSpace(options.Authority.ClientId)) + { + failures.Add("Scanner.Worker:Authority requires ClientId when Enabled is true."); + } + + if (options.Authority.BackchannelTimeoutSeconds <= 0) + { + failures.Add("Scanner.Worker:Authority:BackchannelTimeoutSeconds must be greater than zero."); + } + + if (options.Authority.TokenClockSkewSeconds < 0) + { + failures.Add("Scanner.Worker:Authority:TokenClockSkewSeconds cannot be negative."); + } + + if (options.Authority.Resilience.RetryDelays.Any(delay => delay <= TimeSpan.Zero)) + { + failures.Add("Scanner.Worker:Authority:Resilience:RetryDelays must be positive durations."); + } + } + + if (options.Shutdown.Timeout < TimeSpan.FromSeconds(5)) + { + failures.Add("Scanner.Worker:Shutdown:Timeout must be at least 5 seconds to allow lease completion."); + } + + if (options.Telemetry.EnableTelemetry) + { + if (!options.Telemetry.EnableMetrics && !options.Telemetry.EnableTracing) + { + failures.Add("Scanner.Worker:Telemetry:EnableTelemetry requires metrics or tracing to be enabled."); + } + } + + if (string.IsNullOrWhiteSpace(options.Analyzers.RootFilesystemMetadataKey)) + { + failures.Add("Scanner.Worker:Analyzers:RootFilesystemMetadataKey must be provided."); + } + + return failures.Count == 0 ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failures); + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/AnalyzerStageExecutor.cs b/src/StellaOps.Scanner.Worker/Processing/AnalyzerStageExecutor.cs new file mode 100644 index 00000000..a3ed97f0 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/AnalyzerStageExecutor.cs @@ -0,0 +1,20 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class AnalyzerStageExecutor : IScanStageExecutor +{ + private readonly IScanAnalyzerDispatcher _dispatcher; + + public AnalyzerStageExecutor(IScanAnalyzerDispatcher dispatcher) + { + _dispatcher = dispatcher ?? throw new ArgumentNullException(nameof(dispatcher)); + } + + public string StageName => ScanStageNames.ExecuteAnalyzers; + + public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + => _dispatcher.ExecuteAsync(context, cancellationToken); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/IDelayScheduler.cs b/src/StellaOps.Scanner.Worker/Processing/IDelayScheduler.cs new file mode 100644 index 00000000..49870639 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/IDelayScheduler.cs @@ -0,0 +1,10 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public interface IDelayScheduler +{ + Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/IScanAnalyzerDispatcher.cs b/src/StellaOps.Scanner.Worker/Processing/IScanAnalyzerDispatcher.cs new file mode 100644 index 00000000..7f998b87 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/IScanAnalyzerDispatcher.cs @@ -0,0 +1,15 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public interface IScanAnalyzerDispatcher +{ + ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken); +} + +public sealed class NullScanAnalyzerDispatcher : IScanAnalyzerDispatcher +{ + public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} diff --git a/src/StellaOps.Scanner.Worker/Processing/IScanJobLease.cs b/src/StellaOps.Scanner.Worker/Processing/IScanJobLease.cs new file mode 100644 index 00000000..f7e2bafe --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/IScanJobLease.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public interface IScanJobLease : IAsyncDisposable +{ + string JobId { get; } + + string ScanId { get; } + + int Attempt { get; } + + DateTimeOffset EnqueuedAtUtc { get; } + + DateTimeOffset LeasedAtUtc { get; } + + TimeSpan LeaseDuration { get; } + + IReadOnlyDictionary Metadata { get; } + + ValueTask RenewAsync(CancellationToken cancellationToken); + + ValueTask CompleteAsync(CancellationToken cancellationToken); + + ValueTask AbandonAsync(string reason, CancellationToken cancellationToken); + + ValueTask PoisonAsync(string reason, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/IScanJobSource.cs b/src/StellaOps.Scanner.Worker/Processing/IScanJobSource.cs new file mode 100644 index 00000000..d11bf48f --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/IScanJobSource.cs @@ -0,0 +1,9 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public interface IScanJobSource +{ + Task TryAcquireAsync(CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/IScanStageExecutor.cs b/src/StellaOps.Scanner.Worker/Processing/IScanStageExecutor.cs new file mode 100644 index 00000000..f30f2fb0 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/IScanStageExecutor.cs @@ -0,0 +1,11 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public interface IScanStageExecutor +{ + string StageName { get; } + + ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/LeaseHeartbeatService.cs b/src/StellaOps.Scanner.Worker/Processing/LeaseHeartbeatService.cs new file mode 100644 index 00000000..0e5c30fe --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/LeaseHeartbeatService.cs @@ -0,0 +1,155 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Worker.Options; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class LeaseHeartbeatService +{ + private readonly TimeProvider _timeProvider; + private readonly IOptionsMonitor _options; + private readonly IDelayScheduler _delayScheduler; + private readonly ILogger _logger; + + public LeaseHeartbeatService(TimeProvider timeProvider, IDelayScheduler delayScheduler, IOptionsMonitor options, ILogger logger) + { + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _delayScheduler = delayScheduler ?? throw new ArgumentNullException(nameof(delayScheduler)); + _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task RunAsync(IScanJobLease lease, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(lease); + + await Task.Yield(); + + while (!cancellationToken.IsCancellationRequested) + { + var options = _options.CurrentValue; + var interval = ComputeInterval(options, lease); + var delay = ApplyJitter(interval, options.Queue); + try + { + await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + + if (cancellationToken.IsCancellationRequested) + { + break; + } + + if (await TryRenewAsync(options, lease, cancellationToken).ConfigureAwait(false)) + { + continue; + } + + _logger.LogError( + "Job {JobId} (scan {ScanId}) lease renewal exhausted retries; cancelling processing.", + lease.JobId, + lease.ScanId); + throw new InvalidOperationException("Lease renewal retries exhausted."); + } + } + + private static TimeSpan ComputeInterval(ScannerWorkerOptions options, IScanJobLease lease) + { + var divisor = options.Queue.HeartbeatSafetyFactor <= 0 ? 3.0 : options.Queue.HeartbeatSafetyFactor; + var safetyFactor = Math.Max(3.0, divisor); + var recommended = TimeSpan.FromTicks((long)(lease.LeaseDuration.Ticks / safetyFactor)); + if (recommended < options.Queue.MinHeartbeatInterval) + { + recommended = options.Queue.MinHeartbeatInterval; + } + else if (recommended > options.Queue.MaxHeartbeatInterval) + { + recommended = options.Queue.MaxHeartbeatInterval; + } + + return recommended; + } + + private static TimeSpan ApplyJitter(TimeSpan duration, ScannerWorkerOptions.QueueOptions queueOptions) + { + if (queueOptions.MaxHeartbeatJitterMilliseconds <= 0) + { + return duration; + } + + var offsetMs = Random.Shared.NextDouble() * queueOptions.MaxHeartbeatJitterMilliseconds; + var adjusted = duration - TimeSpan.FromMilliseconds(offsetMs); + if (adjusted < queueOptions.MinHeartbeatInterval) + { + return queueOptions.MinHeartbeatInterval; + } + + return adjusted > TimeSpan.Zero ? adjusted : queueOptions.MinHeartbeatInterval; + } + + private async Task TryRenewAsync(ScannerWorkerOptions options, IScanJobLease lease, CancellationToken cancellationToken) + { + try + { + await lease.RenewAsync(cancellationToken).ConfigureAwait(false); + return true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Job {JobId} (scan {ScanId}) heartbeat failed; retrying.", + lease.JobId, + lease.ScanId); + } + + foreach (var delay in options.Queue.NormalizedHeartbeatRetryDelays) + { + if (cancellationToken.IsCancellationRequested) + { + return false; + } + + try + { + await _delayScheduler.DelayAsync(delay, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + + try + { + await lease.RenewAsync(cancellationToken).ConfigureAwait(false); + return true; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Job {JobId} (scan {ScanId}) heartbeat retry failed; will retry after {Delay}.", + lease.JobId, + lease.ScanId, + delay); + } + } + + return false; + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/NoOpStageExecutor.cs b/src/StellaOps.Scanner.Worker/Processing/NoOpStageExecutor.cs new file mode 100644 index 00000000..0b27a2e0 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/NoOpStageExecutor.cs @@ -0,0 +1,18 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class NoOpStageExecutor : IScanStageExecutor +{ + public NoOpStageExecutor(string stageName) + { + StageName = stageName ?? throw new ArgumentNullException(nameof(stageName)); + } + + public string StageName { get; } + + public ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + => ValueTask.CompletedTask; +} diff --git a/src/StellaOps.Scanner.Worker/Processing/NullScanJobSource.cs b/src/StellaOps.Scanner.Worker/Processing/NullScanJobSource.cs new file mode 100644 index 00000000..4efc29e4 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/NullScanJobSource.cs @@ -0,0 +1,26 @@ +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class NullScanJobSource : IScanJobSource +{ + private readonly ILogger _logger; + private int _logged; + + public NullScanJobSource(ILogger logger) + { + _logger = logger; + } + + public Task TryAcquireAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _logged, 1) == 0) + { + _logger.LogWarning("No queue provider registered. Scanner worker will idle until a queue adapter is configured."); + } + + return Task.FromResult(null); + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/OsScanAnalyzerDispatcher.cs b/src/StellaOps.Scanner.Worker/Processing/OsScanAnalyzerDispatcher.cs new file mode 100644 index 00000000..725d7c47 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/OsScanAnalyzerDispatcher.cs @@ -0,0 +1,164 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Collections.Immutable; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using StellaOps.Scanner.Analyzers.OS; +using StellaOps.Scanner.Analyzers.OS.Abstractions; +using StellaOps.Scanner.Analyzers.OS.Mapping; +using StellaOps.Scanner.Analyzers.OS.Plugin; +using StellaOps.Scanner.Core.Contracts; +using StellaOps.Scanner.Worker.Options; + +namespace StellaOps.Scanner.Worker.Processing; + +internal sealed class OsScanAnalyzerDispatcher : IScanAnalyzerDispatcher +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly OsAnalyzerPluginCatalog _catalog; + private readonly ScannerWorkerOptions _options; + private readonly ILogger _logger; + private IReadOnlyList _pluginDirectories = Array.Empty(); + + public OsScanAnalyzerDispatcher( + IServiceScopeFactory scopeFactory, + OsAnalyzerPluginCatalog catalog, + IOptions options, + ILogger logger) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _catalog = catalog ?? throw new ArgumentNullException(nameof(catalog)); + _options = options?.Value ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + LoadPlugins(); + } + + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + using var scope = _scopeFactory.CreateScope(); + var services = scope.ServiceProvider; + var analyzers = _catalog.CreateAnalyzers(services); + if (analyzers.Count == 0) + { + _logger.LogWarning("No OS analyzers available; skipping analyzer stage for job {JobId}.", context.JobId); + return; + } + + var metadata = new Dictionary(context.Lease.Metadata, StringComparer.Ordinal); + var rootfsPath = ResolvePath(metadata, _options.Analyzers.RootFilesystemMetadataKey); + if (rootfsPath is null) + { + _logger.LogWarning( + "Metadata key '{MetadataKey}' missing for job {JobId}; unable to locate root filesystem. OS analyzers skipped.", + _options.Analyzers.RootFilesystemMetadataKey, + context.JobId); + return; + } + + var workspacePath = ResolvePath(metadata, _options.Analyzers.WorkspaceMetadataKey); + var loggerFactory = services.GetRequiredService(); + + var results = new List(analyzers.Count); + + foreach (var analyzer in analyzers) + { + cancellationToken.ThrowIfCancellationRequested(); + var analyzerLogger = loggerFactory.CreateLogger(analyzer.GetType()); + var analyzerContext = new OSPackageAnalyzerContext(rootfsPath, workspacePath, context.TimeProvider, analyzerLogger, metadata); + + try + { + var result = await analyzer.AnalyzeAsync(analyzerContext, cancellationToken).ConfigureAwait(false); + results.Add(result); + } + catch (Exception ex) + { + _logger.LogError(ex, "Analyzer {AnalyzerId} failed for job {JobId}.", analyzer.AnalyzerId, context.JobId); + } + } + + if (results.Count == 0) + { + return; + } + + var dictionary = results.ToDictionary(result => result.AnalyzerId, StringComparer.OrdinalIgnoreCase); + context.Analysis.Set(ScanAnalysisKeys.OsPackageAnalyzers, dictionary); + + var fragments = OsComponentMapper.ToLayerFragments(results); + if (!fragments.IsDefaultOrEmpty) + { + context.Analysis.AppendLayerFragments(fragments); + context.Analysis.Set(ScanAnalysisKeys.OsComponentFragments, fragments); + } + } + + private void LoadPlugins() + { + var directories = new List(); + foreach (var configured in _options.Analyzers.PluginDirectories) + { + if (string.IsNullOrWhiteSpace(configured)) + { + continue; + } + + var path = configured; + if (!Path.IsPathRooted(path)) + { + path = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, path)); + } + + directories.Add(path); + } + + if (directories.Count == 0) + { + directories.Add(Path.Combine(AppContext.BaseDirectory, "plugins", "scanner", "analyzers", "os")); + } + + _pluginDirectories = new ReadOnlyCollection(directories); + + for (var i = 0; i < _pluginDirectories.Count; i++) + { + var directory = _pluginDirectories[i]; + var seal = i == _pluginDirectories.Count - 1; + + try + { + _catalog.LoadFromDirectory(directory, seal); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to load analyzer plug-ins from {Directory}.", directory); + } + } + } + + private static string? ResolvePath(IReadOnlyDictionary metadata, string key) + { + if (string.IsNullOrWhiteSpace(key)) + { + return null; + } + + if (!metadata.TryGetValue(key, out var value) || string.IsNullOrWhiteSpace(value)) + { + return null; + } + + var trimmed = value.Trim(); + return Path.IsPathRooted(trimmed) + ? trimmed + : Path.GetFullPath(trimmed); + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/PollDelayStrategy.cs b/src/StellaOps.Scanner.Worker/Processing/PollDelayStrategy.cs new file mode 100644 index 00000000..48d3dc96 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/PollDelayStrategy.cs @@ -0,0 +1,49 @@ +using System; + +using StellaOps.Scanner.Worker.Options; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class PollDelayStrategy +{ + private readonly ScannerWorkerOptions.PollingOptions _options; + private TimeSpan _currentDelay; + + public PollDelayStrategy(ScannerWorkerOptions.PollingOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public TimeSpan NextDelay() + { + if (_currentDelay == TimeSpan.Zero) + { + _currentDelay = _options.InitialDelay; + return ApplyJitter(_currentDelay); + } + + var doubled = _currentDelay + _currentDelay; + _currentDelay = doubled < _options.MaxDelay ? doubled : _options.MaxDelay; + return ApplyJitter(_currentDelay); + } + + public void Reset() => _currentDelay = TimeSpan.Zero; + + private TimeSpan ApplyJitter(TimeSpan duration) + { + if (_options.JitterRatio <= 0) + { + return duration; + } + + var maxOffset = duration.TotalMilliseconds * _options.JitterRatio; + if (maxOffset <= 0) + { + return duration; + } + + var offset = (Random.Shared.NextDouble() * 2.0 - 1.0) * maxOffset; + var adjustedMs = Math.Max(0, duration.TotalMilliseconds + offset); + return TimeSpan.FromMilliseconds(adjustedMs); + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/ScanJobContext.cs b/src/StellaOps.Scanner.Worker/Processing/ScanJobContext.cs new file mode 100644 index 00000000..6bcd9e56 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/ScanJobContext.cs @@ -0,0 +1,31 @@ +using System; +using System.Threading; +using StellaOps.Scanner.Core.Contracts; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class ScanJobContext +{ + public ScanJobContext(IScanJobLease lease, TimeProvider timeProvider, DateTimeOffset startUtc, CancellationToken cancellationToken) + { + Lease = lease ?? throw new ArgumentNullException(nameof(lease)); + TimeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + StartUtc = startUtc; + CancellationToken = cancellationToken; + Analysis = new ScanAnalysisStore(); + } + + public IScanJobLease Lease { get; } + + public TimeProvider TimeProvider { get; } + + public DateTimeOffset StartUtc { get; } + + public CancellationToken CancellationToken { get; } + + public string JobId => Lease.JobId; + + public string ScanId => Lease.ScanId; + + public ScanAnalysisStore Analysis { get; } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs b/src/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs new file mode 100644 index 00000000..7fb48c5f --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/ScanJobProcessor.cs @@ -0,0 +1,65 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class ScanJobProcessor +{ + private readonly IReadOnlyDictionary _executors; + private readonly ScanProgressReporter _progressReporter; + private readonly ILogger _logger; + + public ScanJobProcessor(IEnumerable executors, ScanProgressReporter progressReporter, ILogger logger) + { + _progressReporter = progressReporter ?? throw new ArgumentNullException(nameof(progressReporter)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var executor in executors ?? Array.Empty()) + { + if (executor is null || string.IsNullOrWhiteSpace(executor.StageName)) + { + continue; + } + + map[executor.StageName] = executor; + } + + foreach (var stage in ScanStageNames.Ordered) + { + if (map.ContainsKey(stage)) + { + continue; + } + + map[stage] = new NoOpStageExecutor(stage); + _logger.LogDebug("No executor registered for stage {Stage}; using no-op placeholder.", stage); + } + + _executors = map; + } + + public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + foreach (var stage in ScanStageNames.Ordered) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_executors.TryGetValue(stage, out var executor)) + { + continue; + } + + await _progressReporter.ExecuteStageAsync( + context, + stage, + executor.ExecuteAsync, + cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Scanner.Worker/Processing/ScanProgressReporter.cs b/src/StellaOps.Scanner.Worker/Processing/ScanProgressReporter.cs new file mode 100644 index 00000000..a2cccc49 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/ScanProgressReporter.cs @@ -0,0 +1,86 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Scanner.Worker.Diagnostics; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed partial class ScanProgressReporter +{ + private readonly ScannerWorkerMetrics _metrics; + private readonly ILogger _logger; + + public ScanProgressReporter(ScannerWorkerMetrics metrics, ILogger logger) + { + _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask ExecuteStageAsync( + ScanJobContext context, + string stageName, + Func stageWork, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + ArgumentException.ThrowIfNullOrWhiteSpace(stageName); + ArgumentNullException.ThrowIfNull(stageWork); + + StageStarting(_logger, context.JobId, context.ScanId, stageName, context.Lease.Attempt); + + var start = context.TimeProvider.GetUtcNow(); + using var activity = ScannerWorkerInstrumentation.ActivitySource.StartActivity( + $"scanner.worker.{stageName}", + ActivityKind.Internal); + + activity?.SetTag("scanner.worker.job_id", context.JobId); + activity?.SetTag("scanner.worker.scan_id", context.ScanId); + activity?.SetTag("scanner.worker.stage", stageName); + + try + { + await stageWork(context, cancellationToken).ConfigureAwait(false); + var duration = context.TimeProvider.GetUtcNow() - start; + _metrics.RecordStageDuration(context, stageName, duration); + StageCompleted(_logger, context.JobId, context.ScanId, stageName, duration.TotalMilliseconds); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + StageCancelled(_logger, context.JobId, context.ScanId, stageName); + throw; + } + catch (Exception ex) + { + var duration = context.TimeProvider.GetUtcNow() - start; + _metrics.RecordStageDuration(context, stageName, duration); + StageFailed(_logger, context.JobId, context.ScanId, stageName, ex); + throw; + } + } + + [LoggerMessage( + EventId = 1000, + Level = LogLevel.Information, + Message = "Job {JobId} (scan {ScanId}) entering stage {Stage} (attempt {Attempt}).")] + private static partial void StageStarting(ILogger logger, string jobId, string scanId, string stage, int attempt); + + [LoggerMessage( + EventId = 1001, + Level = LogLevel.Information, + Message = "Job {JobId} (scan {ScanId}) finished stage {Stage} in {ElapsedMs:F0} ms.")] + private static partial void StageCompleted(ILogger logger, string jobId, string scanId, string stage, double elapsedMs); + + [LoggerMessage( + EventId = 1002, + Level = LogLevel.Warning, + Message = "Job {JobId} (scan {ScanId}) stage {Stage} cancelled by request.")] + private static partial void StageCancelled(ILogger logger, string jobId, string scanId, string stage); + + [LoggerMessage( + EventId = 1003, + Level = LogLevel.Error, + Message = "Job {JobId} (scan {ScanId}) stage {Stage} failed.")] + private static partial void StageFailed(ILogger logger, string jobId, string scanId, string stage, Exception exception); +} diff --git a/src/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs b/src/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs new file mode 100644 index 00000000..d1529ae0 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/ScanStageNames.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace StellaOps.Scanner.Worker.Processing; + +public static class ScanStageNames +{ + public const string ResolveImage = "resolve-image"; + public const string PullLayers = "pull-layers"; + public const string BuildFilesystem = "build-filesystem"; + public const string ExecuteAnalyzers = "execute-analyzers"; + public const string ComposeArtifacts = "compose-artifacts"; + public const string EmitReports = "emit-reports"; + + public static readonly IReadOnlyList Ordered = new[] + { + ResolveImage, + PullLayers, + BuildFilesystem, + ExecuteAnalyzers, + ComposeArtifacts, + EmitReports, + }; +} diff --git a/src/StellaOps.Scanner.Worker/Processing/SystemDelayScheduler.cs b/src/StellaOps.Scanner.Worker/Processing/SystemDelayScheduler.cs new file mode 100644 index 00000000..b167974c --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Processing/SystemDelayScheduler.cs @@ -0,0 +1,18 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scanner.Worker.Processing; + +public sealed class SystemDelayScheduler : IDelayScheduler +{ + public Task DelayAsync(TimeSpan delay, CancellationToken cancellationToken) + { + if (delay <= TimeSpan.Zero) + { + return Task.CompletedTask; + } + + return Task.Delay(delay, cancellationToken); + } +} diff --git a/src/StellaOps.Scanner.Worker/Program.cs b/src/StellaOps.Scanner.Worker/Program.cs new file mode 100644 index 00000000..ade9f109 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/Program.cs @@ -0,0 +1,105 @@ +using System.Diagnostics; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Auth.Client; +using StellaOps.Scanner.Cache; +using StellaOps.Scanner.Analyzers.OS.Plugin; +using StellaOps.Scanner.EntryTrace; +using StellaOps.Scanner.Worker.Diagnostics; +using StellaOps.Scanner.Worker.Hosting; +using StellaOps.Scanner.Worker.Options; +using StellaOps.Scanner.Worker.Processing; + +var builder = Host.CreateApplicationBuilder(args); + +builder.Services.AddOptions() + .BindConfiguration(ScannerWorkerOptions.SectionName) + .ValidateOnStart(); + +builder.Services.AddSingleton, ScannerWorkerOptionsValidator>(); +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddScannerCache(builder.Configuration); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +builder.Services.AddEntryTraceAnalyzer(); + +builder.Services.TryAddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(); + +builder.Services.AddSingleton(); +builder.Services.AddHostedService(sp => sp.GetRequiredService()); + +var workerOptions = builder.Configuration.GetSection(ScannerWorkerOptions.SectionName).Get() ?? new ScannerWorkerOptions(); + +builder.Services.Configure(options => +{ + options.ShutdownTimeout = workerOptions.Shutdown.Timeout; +}); + +builder.ConfigureScannerWorkerTelemetry(workerOptions); + +if (workerOptions.Authority.Enabled) +{ + builder.Services.AddStellaOpsAuthClient(clientOptions => + { + clientOptions.Authority = workerOptions.Authority.Issuer?.Trim() ?? string.Empty; + clientOptions.ClientId = workerOptions.Authority.ClientId?.Trim() ?? string.Empty; + clientOptions.ClientSecret = workerOptions.Authority.ClientSecret; + clientOptions.EnableRetries = workerOptions.Authority.Resilience.EnableRetries ?? true; + clientOptions.HttpTimeout = TimeSpan.FromSeconds(workerOptions.Authority.BackchannelTimeoutSeconds); + + clientOptions.DefaultScopes.Clear(); + foreach (var scope in workerOptions.Authority.Scopes) + { + if (string.IsNullOrWhiteSpace(scope)) + { + continue; + } + + clientOptions.DefaultScopes.Add(scope); + } + + clientOptions.RetryDelays.Clear(); + foreach (var delay in workerOptions.Authority.Resilience.RetryDelays) + { + if (delay <= TimeSpan.Zero) + { + continue; + } + + clientOptions.RetryDelays.Add(delay); + } + + if (workerOptions.Authority.Resilience.AllowOfflineCacheFallback is bool allowOffline) + { + clientOptions.AllowOfflineCacheFallback = allowOffline; + } + + if (workerOptions.Authority.Resilience.OfflineCacheTolerance is { } tolerance && tolerance > TimeSpan.Zero) + { + clientOptions.OfflineCacheTolerance = tolerance; + } + }); +} + +builder.Logging.Configure(options => +{ + options.ActivityTrackingOptions = ActivityTrackingOptions.SpanId + | ActivityTrackingOptions.TraceId + | ActivityTrackingOptions.ParentId; +}); + +var host = builder.Build(); + +await host.RunAsync(); + +public partial class Program; diff --git a/src/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj b/src/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj new file mode 100644 index 00000000..87d2aa51 --- /dev/null +++ b/src/StellaOps.Scanner.Worker/StellaOps.Scanner.Worker.csproj @@ -0,0 +1,23 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scanner.Worker/TASKS.md b/src/StellaOps.Scanner.Worker/TASKS.md new file mode 100644 index 00000000..a21d474d --- /dev/null +++ b/src/StellaOps.Scanner.Worker/TASKS.md @@ -0,0 +1,10 @@ +# Scanner Worker Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCANNER-WORKER-09-201 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-CORE-09-501 | Worker host bootstrap with Authority auth, hosted services, and graceful shutdown semantics. | `Program.cs` binds `Scanner:Worker` options, registers delay scheduler, configures telemetry + Authority client, and enforces shutdown timeout. | +| SCANNER-WORKER-09-202 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-WORKER-09-201, SCANNER-QUEUE-09-401 | Lease/heartbeat loop with retry+jitter, poison-job quarantine, structured logging. | `ScannerWorkerHostedService` + `LeaseHeartbeatService` manage concurrency, renewal margins, poison handling, and structured logs exercised by integration fixture. | +| SCANNER-WORKER-09-203 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-WORKER-09-202, SCANNER-STORAGE-09-301 | Analyzer dispatch skeleton emitting deterministic stage progress and honoring cancellation tokens. | Deterministic stage list + `ScanProgressReporter`; `WorkerBasicScanScenario` validates ordering and cancellation propagation. | +| SCANNER-WORKER-09-204 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-WORKER-09-203 | Worker metrics (queue latency, stage duration, failure counts) with OpenTelemetry resource wiring. | `ScannerWorkerMetrics` records queue/job/stage metrics; integration test asserts analyzer stage histogram entries. | +| SCANNER-WORKER-09-205 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-WORKER-09-202 | Harden heartbeat jitter so lease safety margin stays ≥3× and cover with regression tests. | `LeaseHeartbeatService` clamps jitter to safety window, validator enforces ≥3 safety factor, regression tests cover heartbeat scheduling and metrics. | +| SCANNER-WORKER-10-201 | DONE (2025-10-19) | Scanner Worker Guild | SCANNER-CACHE-10-101 | Wire scanner cache services and maintenance into worker host. | `AddScannerCache` registered with worker configuration; cache maintenance hosted service runs respecting enabled/auto-evict flags. | diff --git a/src/StellaOps.Scheduler.ImpactIndex/AGENTS.md b/src/StellaOps.Scheduler.ImpactIndex/AGENTS.md new file mode 100644 index 00000000..9a52955a --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.ImpactIndex — Agent Charter + +## Mission +Build the global impact index per `docs/ARCHITECTURE_SCHEDULER.md` (roaring bitmaps, selectors, snapshotting). diff --git a/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj new file mode 100644 index 00000000..6d665dea --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/StellaOps.Scheduler.ImpactIndex.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/src/StellaOps.Scheduler.ImpactIndex/TASKS.md b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md new file mode 100644 index 00000000..81d7fc4b --- /dev/null +++ b/src/StellaOps.Scheduler.ImpactIndex/TASKS.md @@ -0,0 +1,8 @@ +# Scheduler ImpactIndex Task Board (Sprint 16) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-IMPACT-16-300 | DOING | Scheduler ImpactIndex Guild | SAMPLES-10-001 | **STUB** ingest/query using fixtures to unblock Scheduler planning (remove by SP16 end). | Stub merges fixture BOM-Index, query API returns deterministic results, removal note tracked. | +| SCHED-IMPACT-16-301 | TODO | Scheduler ImpactIndex Guild | SCANNER-EMIT-10-605 | Implement ingestion of per-image BOM-Index sidecars into roaring bitmap store (contains/usedBy). | Ingestion tests process sample SBOM index; bitmaps persisted; deterministic IDs assigned. | +| SCHED-IMPACT-16-302 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Provide query APIs (ResolveByPurls, ResolveByVulns, ResolveAll, selectors) with tenant/namespace filters. | Query functions tested; performance benchmarks documented; selectors enforce filters. | +| SCHED-IMPACT-16-303 | TODO | Scheduler ImpactIndex Guild | SCHED-IMPACT-16-301 | Snapshot/compaction + invalidation for removed images; persistence to RocksDB/Redis per architecture. | Snapshot routine implemented; invalidation tests pass; docs describe recovery. | diff --git a/src/StellaOps.Scheduler.Models.Tests/AuditRecordTests.cs b/src/StellaOps.Scheduler.Models.Tests/AuditRecordTests.cs new file mode 100644 index 00000000..9ebfa773 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/AuditRecordTests.cs @@ -0,0 +1,39 @@ +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class AuditRecordTests +{ + [Fact] + public void AuditRecordNormalizesMetadataAndIdentifiers() + { + var actor = new AuditActor(actorId: "user_admin", displayName: "Cluster Admin", kind: "user"); + var metadata = new[] + { + new KeyValuePair("details", "schedule paused"), + new KeyValuePair("Details", "should be overridden"), // duplicate with different casing + new KeyValuePair("reason", "maintenance"), + }; + + var record = new AuditRecord( + id: "audit_001", + tenantId: "tenant-alpha", + category: "scheduler", + action: "pause", + occurredAt: DateTimeOffset.Parse("2025-10-18T05:00:00Z"), + actor: actor, + scheduleId: "sch_001", + runId: null, + correlationId: "corr-123", + metadata: metadata, + message: "Paused via API"); + + Assert.Equal("tenant-alpha", record.TenantId); + Assert.Equal("scheduler", record.Category); + Assert.Equal(2, record.Metadata.Count); + Assert.Equal("schedule paused", record.Metadata["details"]); + Assert.Equal("maintenance", record.Metadata["reason"]); + + var json = CanonicalJsonSerializer.Serialize(record); + Assert.Contains("\"category\":\"scheduler\"", json, StringComparison.Ordinal); + Assert.Contains("\"metadata\":{\"details\":\"schedule paused\",\"reason\":\"maintenance\"}", json, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/ImpactSetTests.cs b/src/StellaOps.Scheduler.Models.Tests/ImpactSetTests.cs new file mode 100644 index 00000000..6fad3443 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/ImpactSetTests.cs @@ -0,0 +1,55 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class ImpactSetTests +{ + [Fact] + public void ImpactSetSortsImagesByDigest() + { + var selector = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + var images = new[] + { + new ImpactImage( + imageDigest: "sha256:bbbb", + registry: "registry.internal", + repository: "app/api", + namespaces: new[] { "team-a" }, + tags: new[] { "prod", "latest" }, + usedByEntrypoint: true, + labels: new Dictionary + { + ["env"] = "prod", + }), + new ImpactImage( + imageDigest: "sha256:aaaa", + registry: "registry.internal", + repository: "app/api", + namespaces: new[] { "team-a" }, + tags: new[] { "prod" }, + usedByEntrypoint: false), + }; + + var impactSet = new ImpactSet( + selector, + images, + usageOnly: true, + generatedAt: DateTimeOffset.Parse("2025-10-18T05:04:03Z"), + total: 2, + snapshotId: "snap-001"); + + Assert.Equal(SchedulerSchemaVersions.ImpactSet, impactSet.SchemaVersion); + Assert.Equal(new[] { "sha256:aaaa", "sha256:bbbb" }, impactSet.Images.Select(i => i.ImageDigest)); + Assert.True(impactSet.UsageOnly); + Assert.Equal(2, impactSet.Total); + + var json = CanonicalJsonSerializer.Serialize(impactSet); + Assert.Contains("\"snapshotId\":\"snap-001\"", json, StringComparison.Ordinal); + } + + [Fact] + public void ImpactImageRejectsInvalidDigest() + { + Assert.Throws(() => new ImpactImage("sha1:not-supported", "registry", "repo")); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/RescanDeltaEventSampleTests.cs b/src/StellaOps.Scheduler.Models.Tests/RescanDeltaEventSampleTests.cs new file mode 100644 index 00000000..efd18178 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/RescanDeltaEventSampleTests.cs @@ -0,0 +1,59 @@ +using System; +using System.IO; +using System.Text.Json; +using System.Text.Json.Nodes; +using StellaOps.Notify.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class RescanDeltaEventSampleTests +{ + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + [Fact] + public void RescanDeltaEventSampleAlignsWithContracts() + { + const string fileName = "scheduler.rescan.delta@1.sample.json"; + var json = LoadSample(fileName); + var notifyEvent = JsonSerializer.Deserialize(json, SerializerOptions); + + Assert.NotNull(notifyEvent); + Assert.Equal(NotifyEventKinds.SchedulerRescanDelta, notifyEvent!.Kind); + Assert.NotEqual(Guid.Empty, notifyEvent.EventId); + Assert.NotNull(notifyEvent.Payload); + Assert.Null(notifyEvent.Scope); + + var payload = Assert.IsType(notifyEvent.Payload); + var scheduleId = Assert.IsAssignableFrom(payload["scheduleId"]).GetValue(); + Assert.Equal("rescan-weekly-critical", scheduleId); + + var digests = Assert.IsType(payload["impactedDigests"]); + Assert.Equal(2, digests.Count); + foreach (var digestNode in digests) + { + var digest = Assert.IsAssignableFrom(digestNode).GetValue(); + Assert.StartsWith("sha256:", digest, StringComparison.Ordinal); + } + + var summary = Assert.IsType(payload["summary"]); + Assert.Equal(0, summary["newCritical"]!.GetValue()); + Assert.Equal(1, summary["newHigh"]!.GetValue()); + Assert.Equal(4, summary["total"]!.GetValue()); + + var canonicalJson = NotifyCanonicalJsonSerializer.Serialize(notifyEvent); + var canonicalNode = JsonNode.Parse(canonicalJson) ?? throw new InvalidOperationException("Canonical JSON null."); + var sampleNode = JsonNode.Parse(json) ?? throw new InvalidOperationException("Sample JSON null."); + Assert.True(JsonNode.DeepEquals(sampleNode, canonicalNode), "Rescan delta event sample must remain canonical."); + } + + private static string LoadSample(string fileName) + { + var path = Path.Combine(AppContext.BaseDirectory, fileName); + if (!File.Exists(path)) + { + throw new FileNotFoundException($"Unable to locate sample '{fileName}'.", path); + } + + return File.ReadAllText(path); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/RunStateMachineTests.cs b/src/StellaOps.Scheduler.Models.Tests/RunStateMachineTests.cs new file mode 100644 index 00000000..ce2c92b3 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/RunStateMachineTests.cs @@ -0,0 +1,108 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class RunStateMachineTests +{ + [Fact] + public void EnsureTransition_FromQueuedToRunningSetsStartedAt() + { + var run = new Run( + id: "run-queued", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Queued, + stats: RunStats.Empty, + createdAt: DateTimeOffset.Parse("2025-10-18T03:00:00Z")); + + var transitionTime = DateTimeOffset.Parse("2025-10-18T03:05:00Z"); + + var updated = RunStateMachine.EnsureTransition( + run, + RunState.Running, + transitionTime, + mutateStats: builder => builder.SetQueued(1)); + + Assert.Equal(RunState.Running, updated.State); + Assert.Equal(transitionTime.ToUniversalTime(), updated.StartedAt); + Assert.Equal(1, updated.Stats.Queued); + Assert.Null(updated.Error); + } + + [Fact] + public void EnsureTransition_ToCompletedPopulatesFinishedAt() + { + var run = new Run( + id: "run-running", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Running, + stats: RunStats.Empty, + createdAt: DateTimeOffset.Parse("2025-10-18T03:00:00Z"), + startedAt: DateTimeOffset.Parse("2025-10-18T03:05:00Z")); + + var completedAt = DateTimeOffset.Parse("2025-10-18T03:10:00Z"); + + var updated = RunStateMachine.EnsureTransition( + run, + RunState.Completed, + completedAt, + mutateStats: builder => + { + builder.SetQueued(1); + builder.SetCompleted(1); + }); + + Assert.Equal(RunState.Completed, updated.State); + Assert.Equal(completedAt.ToUniversalTime(), updated.FinishedAt); + Assert.Equal(1, updated.Stats.Completed); + } + + [Fact] + public void EnsureTransition_ErrorRequiresMessage() + { + var run = new Run( + id: "run-running", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Running, + stats: RunStats.Empty, + createdAt: DateTimeOffset.Parse("2025-10-18T03:00:00Z"), + startedAt: DateTimeOffset.Parse("2025-10-18T03:05:00Z")); + + var timestamp = DateTimeOffset.Parse("2025-10-18T03:06:00Z"); + + var ex = Assert.Throws( + () => RunStateMachine.EnsureTransition(run, RunState.Error, timestamp)); + + Assert.Contains("requires a non-empty error message", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Validate_ThrowsWhenTerminalWithoutFinishedAt() + { + var run = new Run( + id: "run-bad", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Completed, + stats: RunStats.Empty, + createdAt: DateTimeOffset.Parse("2025-10-18T03:00:00Z"), + startedAt: DateTimeOffset.Parse("2025-10-18T03:05:00Z")); + + Assert.Throws(() => RunStateMachine.Validate(run)); + } + + [Fact] + public void RunReasonExtension_NormalizesImpactWindow() + { + var reason = new RunReason(manualReason: "delta"); + var from = DateTimeOffset.Parse("2025-10-18T01:00:00+02:00"); + var to = DateTimeOffset.Parse("2025-10-18T03:30:00+02:00"); + + var updated = reason.WithImpactWindow(from, to); + + Assert.Equal(from.ToUniversalTime().ToString("O"), updated.ImpactWindowFrom); + Assert.Equal(to.ToUniversalTime().ToString("O"), updated.ImpactWindowTo); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/RunValidationTests.cs b/src/StellaOps.Scheduler.Models.Tests/RunValidationTests.cs new file mode 100644 index 00000000..0157d2e7 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/RunValidationTests.cs @@ -0,0 +1,78 @@ +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class RunValidationTests +{ + [Fact] + public void RunStatsRejectsNegativeValues() + { + Assert.Throws(() => new RunStats(candidates: -1)); + Assert.Throws(() => new RunStats(deduped: -1)); + Assert.Throws(() => new RunStats(queued: -1)); + Assert.Throws(() => new RunStats(completed: -1)); + Assert.Throws(() => new RunStats(deltas: -1)); + Assert.Throws(() => new RunStats(newCriticals: -1)); + Assert.Throws(() => new RunStats(newHigh: -1)); + Assert.Throws(() => new RunStats(newMedium: -1)); + Assert.Throws(() => new RunStats(newLow: -1)); + } + + [Fact] + public void DeltaSummarySortsTopFindingsBySeverityThenId() + { + var summary = new DeltaSummary( + imageDigest: "sha256:0011", + newFindings: 3, + newCriticals: 1, + newHigh: 1, + newMedium: 1, + newLow: 0, + kevHits: new[] { "CVE-2025-0002", "CVE-2025-0001" }, + topFindings: new[] + { + new DeltaFinding("pkg:maven/b", "CVE-2025-0002", SeverityRank.High), + new DeltaFinding("pkg:maven/a", "CVE-2024-0001", SeverityRank.Critical), + new DeltaFinding("pkg:maven/c", "CVE-2025-0008", SeverityRank.Medium), + }, + reportUrl: "https://ui.example/reports/sha256:0011", + attestation: new DeltaAttestation(uuid: "rekor-1", verified: true), + detectedAt: DateTimeOffset.Parse("2025-10-18T00:01:02Z")); + + Assert.Equal(new[] { "pkg:maven/a", "pkg:maven/b", "pkg:maven/c" }, summary.TopFindings.Select(f => f.Purl)); + Assert.Equal(new[] { "CVE-2025-0001", "CVE-2025-0002" }, summary.KevHits); + } + + [Fact] + public void RunSerializationIncludesDeterministicOrdering() + { + var stats = new RunStats(candidates: 10, deduped: 8, queued: 8, completed: 5, deltas: 3, newCriticals: 2); + var run = new Run( + id: "run_001", + tenantId: "tenant-alpha", + trigger: RunTrigger.Feedser, + state: RunState.Running, + stats: stats, + reason: new RunReason(feedserExportId: "exp-123"), + scheduleId: "sch_001", + createdAt: DateTimeOffset.Parse("2025-10-18T01:00:00Z"), + startedAt: DateTimeOffset.Parse("2025-10-18T01:00:05Z"), + finishedAt: null, + error: null, + deltas: new[] + { + new DeltaSummary( + imageDigest: "sha256:aaa", + newFindings: 1, + newCriticals: 1, + newHigh: 0, + newMedium: 0, + newLow: 0) + }); + + var json = CanonicalJsonSerializer.Serialize(run); + Assert.Equal(SchedulerSchemaVersions.Run, run.SchemaVersion); + Assert.Contains("\"trigger\":\"feedser\"", json, StringComparison.Ordinal); + Assert.Contains("\"stats\":{\"candidates\":10,\"deduped\":8,\"queued\":8,\"completed\":5,\"deltas\":3,\"newCriticals\":2,\"newHigh\":0,\"newMedium\":0,\"newLow\":0}", json, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs b/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs new file mode 100644 index 00000000..284f83a0 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/SamplePayloadTests.cs @@ -0,0 +1,105 @@ +using System.Text.Json; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class SamplePayloadTests +{ + private static readonly string SamplesRoot = LocateSamplesRoot(); + + [Fact] + public void ScheduleSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("schedule.json"); + var schedule = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal("sch_20251018a", schedule.Id); + Assert.Equal("tenant-alpha", schedule.TenantId); + + var canonical = CanonicalJsonSerializer.Serialize(schedule); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void RunSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("run.json"); + var run = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal(RunState.Running, run.State); + Assert.Equal(42, run.Stats.Deltas); + + var canonical = CanonicalJsonSerializer.Serialize(run); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void ImpactSetSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("impact-set.json"); + var impact = CanonicalJsonSerializer.Deserialize(json); + + Assert.True(impact.UsageOnly); + Assert.Single(impact.Images); + + var canonical = CanonicalJsonSerializer.Serialize(impact); + AssertJsonEquivalent(json, canonical); + } + + [Fact] + public void AuditSample_RoundtripsThroughCanonicalSerializer() + { + var json = ReadSample("audit.json"); + var audit = CanonicalJsonSerializer.Deserialize(json); + + Assert.Equal("scheduler", audit.Category); + Assert.Equal("pause", audit.Action); + + var canonical = CanonicalJsonSerializer.Serialize(audit); + AssertJsonEquivalent(json, canonical); + } + + private static string ReadSample(string fileName) + { + var path = Path.Combine(SamplesRoot, fileName); + return File.ReadAllText(path); + } + + private static string LocateSamplesRoot() + { + var current = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(current)) + { + var candidate = Path.Combine(current, "samples", "api", "scheduler"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + var parent = Path.GetDirectoryName(current.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (string.Equals(parent, current, StringComparison.Ordinal)) + { + break; + } + + current = parent; + } + + throw new DirectoryNotFoundException("Unable to locate samples/api/scheduler in repository tree."); + } + + private static void AssertJsonEquivalent(string expected, string actual) + { + var normalizedExpected = NormalizeJson(expected); + var normalizedActual = NormalizeJson(actual); + Assert.Equal(normalizedExpected, normalizedActual); + } + + private static string NormalizeJson(string json) + { + using var document = JsonDocument.Parse(json); + return JsonSerializer.Serialize(document.RootElement, new JsonSerializerOptions + { + WriteIndented = false + }); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/ScheduleSerializationTests.cs b/src/StellaOps.Scheduler.Models.Tests/ScheduleSerializationTests.cs new file mode 100644 index 00000000..984764ce --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/ScheduleSerializationTests.cs @@ -0,0 +1,113 @@ +using System.Text.Json; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class ScheduleSerializationTests +{ + [Fact] + public void ScheduleSerialization_IsDeterministicRegardlessOfInputOrdering() + { + var selectionA = new Selector( + SelectorScope.ByNamespace, + tenantId: "tenant-alpha", + namespaces: new[] { "team-b", "team-a" }, + repositories: new[] { "app/service-api", "app/service-web" }, + digests: new[] { "sha256:bb", "sha256:aa" }, + includeTags: new[] { "prod", "canary" }, + labels: new[] + { + new LabelSelector("env", new[] { "prod", "staging" }), + new LabelSelector("app", new[] { "web", "api" }), + }, + resolvesTags: true); + + var selectionB = new Selector( + scope: SelectorScope.ByNamespace, + tenantId: "tenant-alpha", + namespaces: new[] { "team-a", "team-b" }, + repositories: new[] { "app/service-web", "app/service-api" }, + digests: new[] { "sha256:aa", "sha256:bb" }, + includeTags: new[] { "canary", "prod" }, + labels: new[] + { + new LabelSelector("app", new[] { "api", "web" }), + new LabelSelector("env", new[] { "staging", "prod" }), + }, + resolvesTags: true); + + var scheduleA = new Schedule( + id: "sch_001", + tenantId: "tenant-alpha", + name: "Nightly Prod", + enabled: true, + cronExpression: "0 2 * * *", + timezone: "UTC", + mode: ScheduleMode.AnalysisOnly, + selection: selectionA, + onlyIf: new ScheduleOnlyIf(lastReportOlderThanDays: 7, policyRevision: "policy@42"), + notify: new ScheduleNotify(onNewFindings: true, SeverityRank.High, includeKev: true), + limits: new ScheduleLimits(maxJobs: 1000, ratePerSecond: 25, parallelism: 4), + createdAt: DateTimeOffset.Parse("2025-10-18T23:00:00Z"), + createdBy: "svc_scheduler", + updatedAt: DateTimeOffset.Parse("2025-10-18T23:00:00Z"), + updatedBy: "svc_scheduler"); + + var scheduleB = new Schedule( + id: scheduleA.Id, + tenantId: scheduleA.TenantId, + name: scheduleA.Name, + enabled: scheduleA.Enabled, + cronExpression: scheduleA.CronExpression, + timezone: scheduleA.Timezone, + mode: scheduleA.Mode, + selection: selectionB, + onlyIf: scheduleA.OnlyIf, + notify: scheduleA.Notify, + limits: scheduleA.Limits, + createdAt: scheduleA.CreatedAt, + createdBy: scheduleA.CreatedBy, + updatedAt: scheduleA.UpdatedAt, + updatedBy: scheduleA.UpdatedBy, + subscribers: scheduleA.Subscribers); + + var jsonA = CanonicalJsonSerializer.Serialize(scheduleA); + var jsonB = CanonicalJsonSerializer.Serialize(scheduleB); + + Assert.Equal(jsonA, jsonB); + + using var doc = JsonDocument.Parse(jsonA); + var root = doc.RootElement; + Assert.Equal(SchedulerSchemaVersions.Schedule, root.GetProperty("schemaVersion").GetString()); + Assert.Equal("analysis-only", root.GetProperty("mode").GetString()); + Assert.Equal("tenant-alpha", root.GetProperty("tenantId").GetString()); + + var namespaces = root.GetProperty("selection").GetProperty("namespaces").EnumerateArray().Select(e => e.GetString()).ToArray(); + Assert.Equal(new[] { "team-a", "team-b" }, namespaces); + } + + [Theory] + [InlineData("")] + [InlineData("not-a-timezone")] + public void Schedule_ThrowsWhenTimezoneInvalid(string timezone) + { + var selection = new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"); + + Assert.ThrowsAny(() => new Schedule( + id: "sch_002", + tenantId: "tenant-alpha", + name: "Invalid timezone", + enabled: true, + cronExpression: "0 3 * * *", + timezone: timezone, + mode: ScheduleMode.AnalysisOnly, + selection: selection, + onlyIf: null, + notify: null, + limits: null, + createdAt: DateTimeOffset.UtcNow, + createdBy: "svc", + updatedAt: DateTimeOffset.UtcNow, + updatedBy: "svc")); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs b/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs new file mode 100644 index 00000000..809a4aec --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/SchedulerSchemaMigrationTests.cs @@ -0,0 +1,72 @@ +using System.Text.Json.Nodes; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Models.Tests; + +public sealed class SchedulerSchemaMigrationTests +{ + [Fact] + public void UpgradeSchedule_DefaultsSchemaVersionWhenMissing() + { + var schedule = new Schedule( + id: "sch-01", + tenantId: "tenant-alpha", + name: "Nightly", + enabled: true, + cronExpression: "0 2 * * *", + timezone: "UTC", + mode: ScheduleMode.AnalysisOnly, + selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), + onlyIf: null, + notify: null, + limits: null, + createdAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z"), + createdBy: "svc-scheduler", + updatedAt: DateTimeOffset.Parse("2025-10-18T00:00:00Z"), + updatedBy: "svc-scheduler"); + + var json = JsonNode.Parse(CanonicalJsonSerializer.Serialize(schedule))!.AsObject(); + json.Remove("schemaVersion"); + + var result = SchedulerSchemaMigration.UpgradeSchedule(json); + + Assert.Equal(SchedulerSchemaVersions.Schedule, result.Value.SchemaVersion); + Assert.Equal(SchedulerSchemaVersions.Schedule, result.ToVersion); + Assert.Empty(result.Warnings); + } + + [Fact] + public void UpgradeRun_StrictModeRemovesUnknownProperties() + { + var run = new Run( + id: "run-01", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Queued, + stats: RunStats.Empty, + createdAt: DateTimeOffset.Parse("2025-10-18T01:10:00Z")); + + var json = JsonNode.Parse(CanonicalJsonSerializer.Serialize(run))!.AsObject(); + json["extraField"] = "to-be-removed"; + + var result = SchedulerSchemaMigration.UpgradeRun(json, strict: true); + + Assert.Contains(result.Warnings, warning => warning.Contains("extraField", StringComparison.Ordinal)); + } + + [Fact] + public void UpgradeImpactSet_ThrowsForUnsupportedVersion() + { + var impactSet = new ImpactSet( + selector: new Selector(SelectorScope.AllImages, "tenant-alpha"), + images: Array.Empty(), + usageOnly: false, + generatedAt: DateTimeOffset.Parse("2025-10-18T02:00:00Z")); + + var json = JsonNode.Parse(CanonicalJsonSerializer.Serialize(impactSet))!.AsObject(); + json["schemaVersion"] = "scheduler.impact-set@99"; + + var ex = Assert.Throws(() => SchedulerSchemaMigration.UpgradeImpactSet(json)); + Assert.Contains("Unsupported scheduler schema version", ex.Message, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Scheduler.Models.Tests/StellaOps.Scheduler.Models.Tests.csproj b/src/StellaOps.Scheduler.Models.Tests/StellaOps.Scheduler.Models.Tests.csproj new file mode 100644 index 00000000..39536098 --- /dev/null +++ b/src/StellaOps.Scheduler.Models.Tests/StellaOps.Scheduler.Models.Tests.csproj @@ -0,0 +1,18 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + Always + + + diff --git a/src/StellaOps.Scheduler.Models/AGENTS.md b/src/StellaOps.Scheduler.Models/AGENTS.md new file mode 100644 index 00000000..14ccf784 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.Models — Agent Charter + +## Mission +Define Scheduler DTOs (Schedule, Run, ImpactSet, Selector, DeltaSummary) per `docs/ARCHITECTURE_SCHEDULER.md`. diff --git a/src/StellaOps.Scheduler.Models/AuditRecord.cs b/src/StellaOps.Scheduler.Models/AuditRecord.cs new file mode 100644 index 00000000..6f4deef8 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/AuditRecord.cs @@ -0,0 +1,120 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Audit log entry capturing schedule/run lifecycle events. +/// +public sealed record AuditRecord +{ + public AuditRecord( + string id, + string tenantId, + string category, + string action, + DateTimeOffset occurredAt, + AuditActor actor, + string? entityId = null, + string? scheduleId = null, + string? runId = null, + string? correlationId = null, + IEnumerable>? metadata = null, + string? message = null) + : this( + id, + tenantId, + Validation.EnsureSimpleIdentifier(category, nameof(category)), + Validation.EnsureSimpleIdentifier(action, nameof(action)), + Validation.NormalizeTimestamp(occurredAt), + actor, + Validation.TrimToNull(entityId), + Validation.TrimToNull(scheduleId), + Validation.TrimToNull(runId), + Validation.TrimToNull(correlationId), + Validation.NormalizeMetadata(metadata), + Validation.TrimToNull(message)) + { + } + + [JsonConstructor] + public AuditRecord( + string id, + string tenantId, + string category, + string action, + DateTimeOffset occurredAt, + AuditActor actor, + string? entityId, + string? scheduleId, + string? runId, + string? correlationId, + ImmutableSortedDictionary metadata, + string? message) + { + Id = Validation.EnsureId(id, nameof(id)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + Category = Validation.EnsureSimpleIdentifier(category, nameof(category)); + Action = Validation.EnsureSimpleIdentifier(action, nameof(action)); + OccurredAt = Validation.NormalizeTimestamp(occurredAt); + Actor = actor ?? throw new ArgumentNullException(nameof(actor)); + EntityId = Validation.TrimToNull(entityId); + ScheduleId = Validation.TrimToNull(scheduleId); + RunId = Validation.TrimToNull(runId); + CorrelationId = Validation.TrimToNull(correlationId); + var materializedMetadata = metadata ?? ImmutableSortedDictionary.Empty; + Metadata = materializedMetadata.Count > 0 + ? materializedMetadata.WithComparers(StringComparer.Ordinal) + : ImmutableSortedDictionary.Empty; + Message = Validation.TrimToNull(message); + } + + public string Id { get; } + + public string TenantId { get; } + + public string Category { get; } + + public string Action { get; } + + public DateTimeOffset OccurredAt { get; } + + public AuditActor Actor { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? EntityId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScheduleId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? RunId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? CorrelationId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Metadata { get; } = ImmutableSortedDictionary.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; } +} + +/// +/// Actor associated with an audit entry. +/// +public sealed record AuditActor +{ + public AuditActor(string actorId, string displayName, string kind) + { + ActorId = Validation.EnsureSimpleIdentifier(actorId, nameof(actorId)); + DisplayName = Validation.EnsureName(displayName, nameof(displayName)); + Kind = Validation.EnsureSimpleIdentifier(kind, nameof(kind)); + } + + public string ActorId { get; } + + public string DisplayName { get; } + + public string Kind { get; } +} diff --git a/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs b/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs new file mode 100644 index 00000000..722a2ef2 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/CanonicalJsonSerializer.cs @@ -0,0 +1,253 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace StellaOps.Scheduler.Models; + +/// +/// Deterministic serializer for scheduler DTOs. +/// +public static class CanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + private static readonly IReadOnlyDictionary PropertyOrder = new Dictionary + { + [typeof(Schedule)] = new[] + { + "schemaVersion", + "id", + "tenantId", + "name", + "enabled", + "cronExpression", + "timezone", + "mode", + "selection", + "onlyIf", + "notify", + "limits", + "subscribers", + "createdAt", + "createdBy", + "updatedAt", + "updatedBy", + }, + [typeof(Selector)] = new[] + { + "scope", + "tenantId", + "namespaces", + "repositories", + "digests", + "includeTags", + "labels", + "resolvesTags", + }, + [typeof(LabelSelector)] = new[] + { + "key", + "values", + }, + [typeof(ScheduleOnlyIf)] = new[] + { + "lastReportOlderThanDays", + "policyRevision", + }, + [typeof(ScheduleNotify)] = new[] + { + "onNewFindings", + "minSeverity", + "includeKev", + "includeQuietFindings", + }, + [typeof(ScheduleLimits)] = new[] + { + "maxJobs", + "ratePerSecond", + "parallelism", + "burst", + }, + [typeof(Run)] = new[] + { + "schemaVersion", + "id", + "tenantId", + "scheduleId", + "trigger", + "state", + "stats", + "reason", + "createdAt", + "startedAt", + "finishedAt", + "error", + "deltas", + }, + [typeof(RunStats)] = new[] + { + "candidates", + "deduped", + "queued", + "completed", + "deltas", + "newCriticals", + "newHigh", + "newMedium", + "newLow", + }, + [typeof(RunReason)] = new[] + { + "manualReason", + "feedserExportId", + "vexerExportId", + "cursor", + "impactWindowFrom", + "impactWindowTo", + }, + [typeof(DeltaSummary)] = new[] + { + "imageDigest", + "newFindings", + "newCriticals", + "newHigh", + "newMedium", + "newLow", + "kevHits", + "topFindings", + "reportUrl", + "attestation", + "detectedAt", + }, + [typeof(DeltaFinding)] = new[] + { + "purl", + "vulnerabilityId", + "severity", + "link", + }, + [typeof(ImpactSet)] = new[] + { + "schemaVersion", + "selector", + "images", + "usageOnly", + "generatedAt", + "total", + "snapshotId", + }, + [typeof(ImpactImage)] = new[] + { + "imageDigest", + "registry", + "repository", + "namespaces", + "tags", + "usedByEntrypoint", + "labels", + }, + [typeof(AuditRecord)] = new[] + { + "id", + "tenantId", + "category", + "action", + "occurredAt", + "actor", + "entityId", + "scheduleId", + "runId", + "correlationId", + "metadata", + "message", + }, + [typeof(AuditActor)] = new[] + { + "actorId", + "displayName", + "kind", + }, + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, PrettyOptions) + ?? throw new InvalidOperationException($"Unable to deserialize {typeof(T).Name}."); + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = writeIndented, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }; + + var resolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicResolver(resolver); + options.Converters.Add(new ScheduleModeConverter()); + options.Converters.Add(new SelectorScopeConverter()); + options.Converters.Add(new RunTriggerConverter()); + options.Converters.Add(new RunStateConverter()); + options.Converters.Add(new SeverityRankConverter()); + return options; + } + + private sealed class DeterministicResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver _inner; + + public DeterministicResolver(IJsonTypeInfoResolver inner) + { + _inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = _inner.GetTypeInfo(type, options); + if (info is null) + { + throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + } + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties.Count > 1) + { + var ordered = info.Properties + .OrderBy(property => ResolveOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int ResolveOrder(Type type, string propertyName) + { + if (PropertyOrder.TryGetValue(type, out var order)) + { + var index = Array.IndexOf(order, propertyName); + if (index >= 0) + { + return index; + } + } + + return int.MaxValue; + } + } +} diff --git a/src/StellaOps.Scheduler.Models/EnumConverters.cs b/src/StellaOps.Scheduler.Models/EnumConverters.cs new file mode 100644 index 00000000..e3d1d6d0 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/EnumConverters.cs @@ -0,0 +1,109 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +internal sealed class ScheduleModeConverter : HyphenatedEnumConverter +{ + protected override IReadOnlyDictionary Map { get; } = new Dictionary + { + [ScheduleMode.AnalysisOnly] = "analysis-only", + [ScheduleMode.ContentRefresh] = "content-refresh", + }; +} + +internal sealed class SelectorScopeConverter : HyphenatedEnumConverter +{ + protected override IReadOnlyDictionary Map { get; } = new Dictionary + { + [SelectorScope.AllImages] = "all-images", + [SelectorScope.ByNamespace] = "by-namespace", + [SelectorScope.ByRepository] = "by-repo", + [SelectorScope.ByDigest] = "by-digest", + [SelectorScope.ByLabels] = "by-labels", + }; +} + +internal sealed class RunTriggerConverter : LowerCaseEnumConverter +{ +} + +internal sealed class RunStateConverter : LowerCaseEnumConverter +{ +} + +internal sealed class SeverityRankConverter : LowerCaseEnumConverter +{ + protected override string ConvertToString(SeverityRank value) + => value switch + { + SeverityRank.None => "none", + SeverityRank.Info => "info", + SeverityRank.Low => "low", + SeverityRank.Medium => "medium", + SeverityRank.High => "high", + SeverityRank.Critical => "critical", + SeverityRank.Unknown => "unknown", + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null), + }; +} + +internal abstract class HyphenatedEnumConverter : JsonConverter + where TEnum : struct, Enum +{ + private readonly Dictionary _reverse; + + protected HyphenatedEnumConverter() + { + _reverse = Map.ToDictionary(static pair => pair.Value, static pair => pair.Key, StringComparer.OrdinalIgnoreCase); + } + + protected abstract IReadOnlyDictionary Map { get; } + + public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + if (value is not null && _reverse.TryGetValue(value, out var parsed)) + { + return parsed; + } + + throw new JsonException($"Value '{value}' is not a valid {typeof(TEnum).Name}."); + } + + public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) + { + if (Map.TryGetValue(value, out var text)) + { + writer.WriteStringValue(text); + return; + } + + throw new JsonException($"Unable to serialize {typeof(TEnum).Name} value '{value}'."); + } +} + +internal class LowerCaseEnumConverter : JsonConverter + where TEnum : struct, Enum +{ + private static readonly Dictionary Reverse = Enum + .GetValues() + .ToDictionary(static value => value.ToString().ToLowerInvariant(), static value => value, StringComparer.OrdinalIgnoreCase); + + public override TEnum Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var value = reader.GetString(); + if (value is not null && Reverse.TryGetValue(value, out var parsed)) + { + return parsed; + } + + throw new JsonException($"Value '{value}' is not a valid {typeof(TEnum).Name}."); + } + + public override void Write(Utf8JsonWriter writer, TEnum value, JsonSerializerOptions options) + => writer.WriteStringValue(ConvertToString(value)); + + protected virtual string ConvertToString(TEnum value) + => value.ToString().ToLowerInvariant(); +} diff --git a/src/StellaOps.Scheduler.Models/Enums.cs b/src/StellaOps.Scheduler.Models/Enums.cs new file mode 100644 index 00000000..dfdabe8c --- /dev/null +++ b/src/StellaOps.Scheduler.Models/Enums.cs @@ -0,0 +1,67 @@ +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Execution mode for a schedule. +/// +[JsonConverter(typeof(ScheduleModeConverter))] +public enum ScheduleMode +{ + AnalysisOnly, + ContentRefresh, +} + +/// +/// Selector scope determining which filters are applied. +/// +[JsonConverter(typeof(SelectorScopeConverter))] +public enum SelectorScope +{ + AllImages, + ByNamespace, + ByRepository, + ByDigest, + ByLabels, +} + +/// +/// Source that triggered a run. +/// +[JsonConverter(typeof(RunTriggerConverter))] +public enum RunTrigger +{ + Cron, + Feedser, + Vexer, + Manual, +} + +/// +/// Lifecycle state of a scheduler run. +/// +[JsonConverter(typeof(RunStateConverter))] +public enum RunState +{ + Planning, + Queued, + Running, + Completed, + Error, + Cancelled, +} + +/// +/// Severity rankings used in scheduler payloads. +/// +[JsonConverter(typeof(SeverityRankConverter))] +public enum SeverityRank +{ + None = 0, + Info = 1, + Low = 2, + Medium = 3, + High = 4, + Critical = 5, + Unknown = 6, +} diff --git a/src/StellaOps.Scheduler.Models/ImpactSet.cs b/src/StellaOps.Scheduler.Models/ImpactSet.cs new file mode 100644 index 00000000..c18e5bf3 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/ImpactSet.cs @@ -0,0 +1,138 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Result from resolving impacted images for a selector. +/// +public sealed record ImpactSet +{ + public ImpactSet( + Selector selector, + IEnumerable images, + bool usageOnly, + DateTimeOffset generatedAt, + int? total = null, + string? snapshotId = null, + string? schemaVersion = null) + : this( + selector, + NormalizeImages(images), + usageOnly, + Validation.NormalizeTimestamp(generatedAt), + total ?? images.Count(), + Validation.TrimToNull(snapshotId), + schemaVersion) + { + } + + [JsonConstructor] + public ImpactSet( + Selector selector, + ImmutableArray images, + bool usageOnly, + DateTimeOffset generatedAt, + int total, + string? snapshotId, + string? schemaVersion = null) + { + Selector = selector ?? throw new ArgumentNullException(nameof(selector)); + Images = images.IsDefault ? ImmutableArray.Empty : images; + UsageOnly = usageOnly; + GeneratedAt = Validation.NormalizeTimestamp(generatedAt); + Total = Validation.EnsureNonNegative(total, nameof(total)); + SnapshotId = Validation.TrimToNull(snapshotId); + SchemaVersion = SchedulerSchemaVersions.EnsureImpactSet(schemaVersion); + } + + public string SchemaVersion { get; } + + public Selector Selector { get; } + + public ImmutableArray Images { get; } = ImmutableArray.Empty; + + public bool UsageOnly { get; } + + public DateTimeOffset GeneratedAt { get; } + + public int Total { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? SnapshotId { get; } + + private static ImmutableArray NormalizeImages(IEnumerable images) + { + ArgumentNullException.ThrowIfNull(images); + + return images + .Where(static image => image is not null) + .Select(static image => image!) + .OrderBy(static image => image.ImageDigest, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +/// +/// Impacted image descriptor returned from the impact index. +/// +public sealed record ImpactImage +{ + public ImpactImage( + string imageDigest, + string registry, + string repository, + IEnumerable? namespaces = null, + IEnumerable? tags = null, + bool usedByEntrypoint = false, + IEnumerable>? labels = null) + : this( + Validation.EnsureDigestFormat(imageDigest, nameof(imageDigest)), + Validation.EnsureSimpleIdentifier(registry, nameof(registry)), + Validation.EnsureSimpleIdentifier(repository, nameof(repository)), + Validation.NormalizeStringSet(namespaces, nameof(namespaces)), + Validation.NormalizeTagPatterns(tags), + usedByEntrypoint, + Validation.NormalizeMetadata(labels)) + { + } + + [JsonConstructor] + public ImpactImage( + string imageDigest, + string registry, + string repository, + ImmutableArray namespaces, + ImmutableArray tags, + bool usedByEntrypoint, + ImmutableSortedDictionary labels) + { + ImageDigest = Validation.EnsureDigestFormat(imageDigest, nameof(imageDigest)); + Registry = Validation.EnsureSimpleIdentifier(registry, nameof(registry)); + Repository = Validation.EnsureSimpleIdentifier(repository, nameof(repository)); + Namespaces = namespaces.IsDefault ? ImmutableArray.Empty : namespaces; + Tags = tags.IsDefault ? ImmutableArray.Empty : tags; + UsedByEntrypoint = usedByEntrypoint; + var materializedLabels = labels ?? ImmutableSortedDictionary.Empty; + Labels = materializedLabels.Count > 0 + ? materializedLabels.WithComparers(StringComparer.Ordinal) + : ImmutableSortedDictionary.Empty; + } + + public string ImageDigest { get; } + + public string Registry { get; } + + public string Repository { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Namespaces { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Tags { get; } = ImmutableArray.Empty; + + public bool UsedByEntrypoint { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableSortedDictionary Labels { get; } = ImmutableSortedDictionary.Empty; +} diff --git a/src/StellaOps.Scheduler.Models/Run.cs b/src/StellaOps.Scheduler.Models/Run.cs new file mode 100644 index 00000000..be8798a0 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/Run.cs @@ -0,0 +1,378 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Execution record for a scheduler run. +/// +public sealed record Run +{ + public Run( + string id, + string tenantId, + RunTrigger trigger, + RunState state, + RunStats stats, + DateTimeOffset createdAt, + RunReason? reason = null, + string? scheduleId = null, + DateTimeOffset? startedAt = null, + DateTimeOffset? finishedAt = null, + string? error = null, + IEnumerable? deltas = null, + string? schemaVersion = null) + : this( + id, + tenantId, + trigger, + state, + stats, + reason ?? RunReason.Empty, + scheduleId, + Validation.NormalizeTimestamp(createdAt), + Validation.NormalizeTimestamp(startedAt), + Validation.NormalizeTimestamp(finishedAt), + Validation.TrimToNull(error), + NormalizeDeltas(deltas), + schemaVersion) + { + } + + [JsonConstructor] + public Run( + string id, + string tenantId, + RunTrigger trigger, + RunState state, + RunStats stats, + RunReason reason, + string? scheduleId, + DateTimeOffset createdAt, + DateTimeOffset? startedAt, + DateTimeOffset? finishedAt, + string? error, + ImmutableArray deltas, + string? schemaVersion = null) + { + Id = Validation.EnsureId(id, nameof(id)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + Trigger = trigger; + State = state; + Stats = stats ?? throw new ArgumentNullException(nameof(stats)); + Reason = reason ?? RunReason.Empty; + ScheduleId = Validation.TrimToNull(scheduleId); + CreatedAt = Validation.NormalizeTimestamp(createdAt); + StartedAt = Validation.NormalizeTimestamp(startedAt); + FinishedAt = Validation.NormalizeTimestamp(finishedAt); + Error = Validation.TrimToNull(error); + Deltas = deltas.IsDefault + ? ImmutableArray.Empty + : deltas.OrderBy(static delta => delta.ImageDigest, StringComparer.Ordinal).ToImmutableArray(); + SchemaVersion = SchedulerSchemaVersions.EnsureRun(schemaVersion); + } + + public string SchemaVersion { get; } + + public string Id { get; } + + public string TenantId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ScheduleId { get; } + + public RunTrigger Trigger { get; } + + public RunState State { get; init; } + + public RunStats Stats { get; init; } + + public RunReason Reason { get; } + + public DateTimeOffset CreatedAt { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? StartedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? FinishedAt { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Error { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Deltas { get; } = ImmutableArray.Empty; + + private static ImmutableArray NormalizeDeltas(IEnumerable? deltas) + { + if (deltas is null) + { + return ImmutableArray.Empty; + } + + return deltas + .Where(static delta => delta is not null) + .Select(static delta => delta!) + .OrderBy(static delta => delta.ImageDigest, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +/// +/// Context describing why a run executed. +/// +public sealed record RunReason +{ + public static RunReason Empty { get; } = new(); + + public RunReason( + string? manualReason = null, + string? feedserExportId = null, + string? vexerExportId = null, + string? cursor = null) + { + ManualReason = Validation.TrimToNull(manualReason); + FeedserExportId = Validation.TrimToNull(feedserExportId); + VexerExportId = Validation.TrimToNull(vexerExportId); + Cursor = Validation.TrimToNull(cursor); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ManualReason { get; } = null; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FeedserExportId { get; } = null; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? VexerExportId { get; } = null; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Cursor { get; } = null; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImpactWindowFrom { get; init; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ImpactWindowTo { get; init; } +} + +/// +/// Aggregated counters for a scheduler run. +/// +public sealed record RunStats +{ + public static RunStats Empty { get; } = new(); + + public RunStats( + int candidates = 0, + int deduped = 0, + int queued = 0, + int completed = 0, + int deltas = 0, + int newCriticals = 0, + int newHigh = 0, + int newMedium = 0, + int newLow = 0) + { + Candidates = Validation.EnsureNonNegative(candidates, nameof(candidates)); + Deduped = Validation.EnsureNonNegative(deduped, nameof(deduped)); + Queued = Validation.EnsureNonNegative(queued, nameof(queued)); + Completed = Validation.EnsureNonNegative(completed, nameof(completed)); + Deltas = Validation.EnsureNonNegative(deltas, nameof(deltas)); + NewCriticals = Validation.EnsureNonNegative(newCriticals, nameof(newCriticals)); + NewHigh = Validation.EnsureNonNegative(newHigh, nameof(newHigh)); + NewMedium = Validation.EnsureNonNegative(newMedium, nameof(newMedium)); + NewLow = Validation.EnsureNonNegative(newLow, nameof(newLow)); + } + + public int Candidates { get; } = 0; + + public int Deduped { get; } = 0; + + public int Queued { get; } = 0; + + public int Completed { get; } = 0; + + public int Deltas { get; } = 0; + + public int NewCriticals { get; } = 0; + + public int NewHigh { get; } = 0; + + public int NewMedium { get; } = 0; + + public int NewLow { get; } = 0; +} + +/// +/// Snapshot of delta impact for an image processed in a run. +/// +public sealed record DeltaSummary +{ + public DeltaSummary( + string imageDigest, + int newFindings, + int newCriticals, + int newHigh, + int newMedium, + int newLow, + IEnumerable? kevHits = null, + IEnumerable? topFindings = null, + string? reportUrl = null, + DeltaAttestation? attestation = null, + DateTimeOffset? detectedAt = null) + : this( + imageDigest, + Validation.EnsureNonNegative(newFindings, nameof(newFindings)), + Validation.EnsureNonNegative(newCriticals, nameof(newCriticals)), + Validation.EnsureNonNegative(newHigh, nameof(newHigh)), + Validation.EnsureNonNegative(newMedium, nameof(newMedium)), + Validation.EnsureNonNegative(newLow, nameof(newLow)), + NormalizeKevHits(kevHits), + NormalizeFindings(topFindings), + Validation.TrimToNull(reportUrl), + attestation, + Validation.NormalizeTimestamp(detectedAt)) + { + } + + [JsonConstructor] + public DeltaSummary( + string imageDigest, + int newFindings, + int newCriticals, + int newHigh, + int newMedium, + int newLow, + ImmutableArray kevHits, + ImmutableArray topFindings, + string? reportUrl, + DeltaAttestation? attestation, + DateTimeOffset? detectedAt) + { + ImageDigest = Validation.EnsureDigestFormat(imageDigest, nameof(imageDigest)); + NewFindings = Validation.EnsureNonNegative(newFindings, nameof(newFindings)); + NewCriticals = Validation.EnsureNonNegative(newCriticals, nameof(newCriticals)); + NewHigh = Validation.EnsureNonNegative(newHigh, nameof(newHigh)); + NewMedium = Validation.EnsureNonNegative(newMedium, nameof(newMedium)); + NewLow = Validation.EnsureNonNegative(newLow, nameof(newLow)); + KevHits = kevHits.IsDefault ? ImmutableArray.Empty : kevHits; + TopFindings = topFindings.IsDefault + ? ImmutableArray.Empty + : topFindings + .OrderBy(static finding => finding.Severity, SeverityRankComparer.Instance) + .ThenBy(static finding => finding.VulnerabilityId, StringComparer.Ordinal) + .ToImmutableArray(); + ReportUrl = Validation.TrimToNull(reportUrl); + Attestation = attestation; + DetectedAt = Validation.NormalizeTimestamp(detectedAt); + } + + public string ImageDigest { get; } + + public int NewFindings { get; } + + public int NewCriticals { get; } + + public int NewHigh { get; } + + public int NewMedium { get; } + + public int NewLow { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray KevHits { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray TopFindings { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ReportUrl { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DeltaAttestation? Attestation { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public DateTimeOffset? DetectedAt { get; } + + private static ImmutableArray NormalizeKevHits(IEnumerable? kevHits) + => Validation.NormalizeStringSet(kevHits, nameof(kevHits)); + + private static ImmutableArray NormalizeFindings(IEnumerable? findings) + { + if (findings is null) + { + return ImmutableArray.Empty; + } + + return findings + .Where(static finding => finding is not null) + .Select(static finding => finding!) + .OrderBy(static finding => finding.Severity, SeverityRankComparer.Instance) + .ThenBy(static finding => finding.VulnerabilityId, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +/// +/// Top finding entry included in delta summaries. +/// +public sealed record DeltaFinding +{ + public DeltaFinding(string purl, string vulnerabilityId, SeverityRank severity, string? link = null) + { + Purl = Validation.EnsureSimpleIdentifier(purl, nameof(purl)); + VulnerabilityId = Validation.EnsureSimpleIdentifier(vulnerabilityId, nameof(vulnerabilityId)); + Severity = severity; + Link = Validation.TrimToNull(link); + } + + public string Purl { get; } + + public string VulnerabilityId { get; } + + public SeverityRank Severity { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Link { get; } +} + +/// +/// Rekor/attestation information surfaced with a delta summary. +/// +public sealed record DeltaAttestation +{ + public DeltaAttestation(string? uuid, bool? verified = null) + { + Uuid = Validation.TrimToNull(uuid); + Verified = verified; + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Uuid { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public bool? Verified { get; } +} + +internal sealed class SeverityRankComparer : IComparer +{ + public static SeverityRankComparer Instance { get; } = new(); + + private static readonly Dictionary Order = new() + { + [SeverityRank.Critical] = 0, + [SeverityRank.High] = 1, + [SeverityRank.Unknown] = 2, + [SeverityRank.Medium] = 3, + [SeverityRank.Low] = 4, + [SeverityRank.Info] = 5, + [SeverityRank.None] = 6, + }; + + public int Compare(SeverityRank x, SeverityRank y) + => GetOrder(x).CompareTo(GetOrder(y)); + + private static int GetOrder(SeverityRank severity) + => Order.TryGetValue(severity, out var value) ? value : int.MaxValue; +} diff --git a/src/StellaOps.Scheduler.Models/RunReasonExtensions.cs b/src/StellaOps.Scheduler.Models/RunReasonExtensions.cs new file mode 100644 index 00000000..21f02fc5 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/RunReasonExtensions.cs @@ -0,0 +1,33 @@ +using System; +using System.Globalization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Convenience helpers for mutations. +/// +public static class RunReasonExtensions +{ + /// + /// Returns a copy of with impact window timestamps normalized to ISO-8601. + /// + public static RunReason WithImpactWindow( + this RunReason reason, + DateTimeOffset? from, + DateTimeOffset? to) + { + var normalizedFrom = Validation.NormalizeTimestamp(from); + var normalizedTo = Validation.NormalizeTimestamp(to); + + if (normalizedFrom.HasValue && normalizedTo.HasValue && normalizedFrom > normalizedTo) + { + throw new ArgumentException("Impact window start must be earlier than or equal to end."); + } + + return reason with + { + ImpactWindowFrom = normalizedFrom?.ToString("O", CultureInfo.InvariantCulture), + ImpactWindowTo = normalizedTo?.ToString("O", CultureInfo.InvariantCulture), + }; + } +} diff --git a/src/StellaOps.Scheduler.Models/RunStateMachine.cs b/src/StellaOps.Scheduler.Models/RunStateMachine.cs new file mode 100644 index 00000000..f9d0fcc6 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/RunStateMachine.cs @@ -0,0 +1,157 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace StellaOps.Scheduler.Models; + +/// +/// Encapsulates allowed transitions and invariants. +/// +public static class RunStateMachine +{ + private static readonly IReadOnlyDictionary Adjacency = new Dictionary + { + [RunState.Planning] = new[] { RunState.Planning, RunState.Queued, RunState.Cancelled }, + [RunState.Queued] = new[] { RunState.Queued, RunState.Running, RunState.Cancelled }, + [RunState.Running] = new[] { RunState.Running, RunState.Completed, RunState.Error, RunState.Cancelled }, + [RunState.Completed] = new[] { RunState.Completed }, + [RunState.Error] = new[] { RunState.Error }, + [RunState.Cancelled] = new[] { RunState.Cancelled }, + }; + + public static bool CanTransition(RunState from, RunState to) + { + if (!Adjacency.TryGetValue(from, out var allowed)) + { + return false; + } + + return allowed.Contains(to); + } + + public static bool IsTerminal(RunState state) + => state is RunState.Completed or RunState.Error or RunState.Cancelled; + + /// + /// Applies a state transition ensuring timestamps, stats, and error contracts stay consistent. + /// + public static Run EnsureTransition( + Run run, + RunState next, + DateTimeOffset timestamp, + Action? mutateStats = null, + string? errorMessage = null) + { + ArgumentNullException.ThrowIfNull(run); + + var normalizedTimestamp = Validation.NormalizeTimestamp(timestamp); + var current = run.State; + + if (!CanTransition(current, next)) + { + throw new InvalidOperationException($"Run state transition from '{current}' to '{next}' is not allowed."); + } + + var statsBuilder = new RunStatsBuilder(run.Stats); + mutateStats?.Invoke(statsBuilder); + var newStats = statsBuilder.Build(); + + var startedAt = run.StartedAt; + var finishedAt = run.FinishedAt; + + if (current != RunState.Running && next == RunState.Running && startedAt is null) + { + startedAt = normalizedTimestamp; + } + + if (IsTerminal(next)) + { + finishedAt ??= normalizedTimestamp; + } + + if (startedAt is { } start && start < run.CreatedAt) + { + throw new InvalidOperationException("Run started time cannot be earlier than created time."); + } + + if (finishedAt is { } finish) + { + if (startedAt is { } startTime && finish < startTime) + { + throw new InvalidOperationException("Run finished time cannot be earlier than start time."); + } + + if (!IsTerminal(next)) + { + throw new InvalidOperationException("Finished time present but next state is not terminal."); + } + } + + string? nextError = null; + if (next == RunState.Error) + { + var effectiveError = string.IsNullOrWhiteSpace(errorMessage) ? run.Error : errorMessage.Trim(); + if (string.IsNullOrWhiteSpace(effectiveError)) + { + throw new InvalidOperationException("Transitioning to Error requires a non-empty error message."); + } + + nextError = effectiveError; + } + else if (!string.IsNullOrWhiteSpace(errorMessage)) + { + throw new InvalidOperationException("Error message can only be provided when transitioning to Error state."); + } + + var updated = run with + { + State = next, + Stats = newStats, + StartedAt = startedAt, + FinishedAt = finishedAt, + Error = nextError, + }; + + Validate(updated); + return updated; + } + + public static void Validate(Run run) + { + ArgumentNullException.ThrowIfNull(run); + + if (run.StartedAt is { } started && started < run.CreatedAt) + { + throw new InvalidOperationException("Run.StartedAt cannot be earlier than CreatedAt."); + } + + if (run.FinishedAt is { } finished) + { + if (run.StartedAt is { } startedAt && finished < startedAt) + { + throw new InvalidOperationException("Run.FinishedAt cannot be earlier than StartedAt."); + } + + if (!IsTerminal(run.State)) + { + throw new InvalidOperationException("Run.FinishedAt set while state is not terminal."); + } + } + else if (IsTerminal(run.State)) + { + throw new InvalidOperationException("Terminal run states must include FinishedAt."); + } + + if (run.State == RunState.Error) + { + if (string.IsNullOrWhiteSpace(run.Error)) + { + throw new InvalidOperationException("Run.Error must be populated when state is Error."); + } + } + else if (!string.IsNullOrWhiteSpace(run.Error)) + { + throw new InvalidOperationException("Run.Error must be null for non-error states."); + } + } +} diff --git a/src/StellaOps.Scheduler.Models/RunStatsBuilder.cs b/src/StellaOps.Scheduler.Models/RunStatsBuilder.cs new file mode 100644 index 00000000..f07b793f --- /dev/null +++ b/src/StellaOps.Scheduler.Models/RunStatsBuilder.cs @@ -0,0 +1,92 @@ +using System; + +namespace StellaOps.Scheduler.Models; + +/// +/// Helper that enforces monotonic updates. +/// +public sealed class RunStatsBuilder +{ + private int _candidates; + private int _deduped; + private int _queued; + private int _completed; + private int _deltas; + private int _newCriticals; + private int _newHigh; + private int _newMedium; + private int _newLow; + + public RunStatsBuilder(RunStats? baseline = null) + { + baseline ??= RunStats.Empty; + _candidates = baseline.Candidates; + _deduped = baseline.Deduped; + _queued = baseline.Queued; + _completed = baseline.Completed; + _deltas = baseline.Deltas; + _newCriticals = baseline.NewCriticals; + _newHigh = baseline.NewHigh; + _newMedium = baseline.NewMedium; + _newLow = baseline.NewLow; + } + + public void SetCandidates(int value) => _candidates = EnsureMonotonic(value, _candidates, nameof(RunStats.Candidates)); + + public void IncrementCandidates(int value = 1) => SetCandidates(_candidates + value); + + public void SetDeduped(int value) => _deduped = EnsureMonotonic(value, _deduped, nameof(RunStats.Deduped)); + + public void IncrementDeduped(int value = 1) => SetDeduped(_deduped + value); + + public void SetQueued(int value) => _queued = EnsureMonotonic(value, _queued, nameof(RunStats.Queued)); + + public void IncrementQueued(int value = 1) => SetQueued(_queued + value); + + public void SetCompleted(int value) => _completed = EnsureMonotonic(value, _completed, nameof(RunStats.Completed)); + + public void IncrementCompleted(int value = 1) => SetCompleted(_completed + value); + + public void SetDeltas(int value) => _deltas = EnsureMonotonic(value, _deltas, nameof(RunStats.Deltas)); + + public void IncrementDeltas(int value = 1) => SetDeltas(_deltas + value); + + public void SetNewCriticals(int value) => _newCriticals = EnsureMonotonic(value, _newCriticals, nameof(RunStats.NewCriticals)); + + public void IncrementNewCriticals(int value = 1) => SetNewCriticals(_newCriticals + value); + + public void SetNewHigh(int value) => _newHigh = EnsureMonotonic(value, _newHigh, nameof(RunStats.NewHigh)); + + public void IncrementNewHigh(int value = 1) => SetNewHigh(_newHigh + value); + + public void SetNewMedium(int value) => _newMedium = EnsureMonotonic(value, _newMedium, nameof(RunStats.NewMedium)); + + public void IncrementNewMedium(int value = 1) => SetNewMedium(_newMedium + value); + + public void SetNewLow(int value) => _newLow = EnsureMonotonic(value, _newLow, nameof(RunStats.NewLow)); + + public void IncrementNewLow(int value = 1) => SetNewLow(_newLow + value); + + public RunStats Build() + => new( + candidates: _candidates, + deduped: _deduped, + queued: _queued, + completed: _completed, + deltas: _deltas, + newCriticals: _newCriticals, + newHigh: _newHigh, + newMedium: _newMedium, + newLow: _newLow); + + private static int EnsureMonotonic(int value, int current, string fieldName) + { + Validation.EnsureNonNegative(value, fieldName); + if (value < current) + { + throw new InvalidOperationException($"RunStats.{fieldName} cannot decrease (current: {current}, attempted: {value})."); + } + + return value; + } +} diff --git a/src/StellaOps.Scheduler.Models/Schedule.cs b/src/StellaOps.Scheduler.Models/Schedule.cs new file mode 100644 index 00000000..3f49414e --- /dev/null +++ b/src/StellaOps.Scheduler.Models/Schedule.cs @@ -0,0 +1,227 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Scheduler configuration entity persisted in Mongo. +/// +public sealed record Schedule +{ + public Schedule( + string id, + string tenantId, + string name, + bool enabled, + string cronExpression, + string timezone, + ScheduleMode mode, + Selector selection, + ScheduleOnlyIf? onlyIf, + ScheduleNotify? notify, + ScheduleLimits? limits, + DateTimeOffset createdAt, + string createdBy, + DateTimeOffset updatedAt, + string updatedBy, + ImmutableArray? subscribers = null, + string? schemaVersion = null) + : this( + id, + tenantId, + name, + enabled, + cronExpression, + timezone, + mode, + selection, + onlyIf ?? ScheduleOnlyIf.Default, + notify ?? ScheduleNotify.Default, + limits ?? ScheduleLimits.Default, + subscribers ?? ImmutableArray.Empty, + createdAt, + createdBy, + updatedAt, + updatedBy, + schemaVersion) + { + } + + [JsonConstructor] + public Schedule( + string id, + string tenantId, + string name, + bool enabled, + string cronExpression, + string timezone, + ScheduleMode mode, + Selector selection, + ScheduleOnlyIf onlyIf, + ScheduleNotify notify, + ScheduleLimits limits, + ImmutableArray subscribers, + DateTimeOffset createdAt, + string createdBy, + DateTimeOffset updatedAt, + string updatedBy, + string? schemaVersion = null) + { + Id = Validation.EnsureId(id, nameof(id)); + TenantId = Validation.EnsureTenantId(tenantId, nameof(tenantId)); + Name = Validation.EnsureName(name, nameof(name)); + Enabled = enabled; + CronExpression = Validation.EnsureCronExpression(cronExpression, nameof(cronExpression)); + Timezone = Validation.EnsureTimezone(timezone, nameof(timezone)); + Mode = mode; + Selection = selection ?? throw new ArgumentNullException(nameof(selection)); + OnlyIf = onlyIf ?? ScheduleOnlyIf.Default; + Notify = notify ?? ScheduleNotify.Default; + Limits = limits ?? ScheduleLimits.Default; + Subscribers = (subscribers.IsDefault ? ImmutableArray.Empty : subscribers) + .Select(static value => Validation.EnsureSimpleIdentifier(value, nameof(subscribers))) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + CreatedAt = Validation.NormalizeTimestamp(createdAt); + CreatedBy = Validation.EnsureSimpleIdentifier(createdBy, nameof(createdBy)); + UpdatedAt = Validation.NormalizeTimestamp(updatedAt); + UpdatedBy = Validation.EnsureSimpleIdentifier(updatedBy, nameof(updatedBy)); + SchemaVersion = SchedulerSchemaVersions.EnsureSchedule(schemaVersion); + + if (Selection.TenantId is not null && !string.Equals(Selection.TenantId, TenantId, StringComparison.Ordinal)) + { + throw new ArgumentException("Selection tenant must match schedule tenant.", nameof(selection)); + } + } + + public string SchemaVersion { get; } + + public string Id { get; } + + public string TenantId { get; } + + public string Name { get; } + + public bool Enabled { get; } + + public string CronExpression { get; } + + public string Timezone { get; } + + public ScheduleMode Mode { get; } + + public Selector Selection { get; } + + public ScheduleOnlyIf OnlyIf { get; } + + public ScheduleNotify Notify { get; } + + public ScheduleLimits Limits { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Subscribers { get; } = ImmutableArray.Empty; + + public DateTimeOffset CreatedAt { get; } + + public string CreatedBy { get; } + + public DateTimeOffset UpdatedAt { get; } + + public string UpdatedBy { get; } +} + +/// +/// Conditions that must hold before a schedule enqueues work. +/// +public sealed record ScheduleOnlyIf +{ + public static ScheduleOnlyIf Default { get; } = new(); + + [JsonConstructor] + public ScheduleOnlyIf(int? lastReportOlderThanDays = null, string? policyRevision = null) + { + LastReportOlderThanDays = Validation.EnsurePositiveOrNull(lastReportOlderThanDays, nameof(lastReportOlderThanDays)); + PolicyRevision = Validation.TrimToNull(policyRevision); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? LastReportOlderThanDays { get; } = null; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? PolicyRevision { get; } = null; +} + +/// +/// Notification preferences for schedule outcomes. +/// +public sealed record ScheduleNotify +{ + public static ScheduleNotify Default { get; } = new(onNewFindings: true, null, includeKev: true); + + public ScheduleNotify(bool onNewFindings, SeverityRank? minSeverity, bool includeKev) + { + OnNewFindings = onNewFindings; + if (minSeverity is SeverityRank.Unknown or SeverityRank.None) + { + MinSeverity = minSeverity == SeverityRank.Unknown ? SeverityRank.Unknown : SeverityRank.Low; + } + else + { + MinSeverity = minSeverity; + } + + IncludeKev = includeKev; + } + + [JsonConstructor] + public ScheduleNotify(bool onNewFindings, SeverityRank? minSeverity, bool includeKev, bool includeQuietFindings = false) + : this(onNewFindings, minSeverity, includeKev) + { + IncludeQuietFindings = includeQuietFindings; + } + + public bool OnNewFindings { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public SeverityRank? MinSeverity { get; } + + public bool IncludeKev { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool IncludeQuietFindings { get; } +} + +/// +/// Execution limits that bound scheduler throughput. +/// +public sealed record ScheduleLimits +{ + public static ScheduleLimits Default { get; } = new(); + + public ScheduleLimits(int? maxJobs = null, int? ratePerSecond = null, int? parallelism = null) + { + MaxJobs = Validation.EnsurePositiveOrNull(maxJobs, nameof(maxJobs)); + RatePerSecond = Validation.EnsurePositiveOrNull(ratePerSecond, nameof(ratePerSecond)); + Parallelism = Validation.EnsurePositiveOrNull(parallelism, nameof(parallelism)); + } + + [JsonConstructor] + public ScheduleLimits(int? maxJobs, int? ratePerSecond, int? parallelism, int? burst = null) + : this(maxJobs, ratePerSecond, parallelism) + { + Burst = Validation.EnsurePositiveOrNull(burst, nameof(burst)); + } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? MaxJobs { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? RatePerSecond { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Parallelism { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public int? Burst { get; } +} diff --git a/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs new file mode 100644 index 00000000..8b6e8f48 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigration.cs @@ -0,0 +1,172 @@ +using System.Collections.Immutable; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace StellaOps.Scheduler.Models; + +/// +/// Upgrades scheduler documents emitted by earlier schema revisions to the latest DTOs. +/// +public static class SchedulerSchemaMigration +{ + private static readonly ImmutableHashSet ScheduleProperties = ImmutableHashSet.Create( + StringComparer.Ordinal, + "schemaVersion", + "id", + "tenantId", + "name", + "enabled", + "cronExpression", + "timezone", + "mode", + "selection", + "onlyIf", + "notify", + "limits", + "subscribers", + "createdAt", + "createdBy", + "updatedAt", + "updatedBy"); + + private static readonly ImmutableHashSet RunProperties = ImmutableHashSet.Create( + StringComparer.Ordinal, + "schemaVersion", + "id", + "tenantId", + "scheduleId", + "trigger", + "state", + "stats", + "reason", + "createdAt", + "startedAt", + "finishedAt", + "error", + "deltas"); + + private static readonly ImmutableHashSet ImpactSetProperties = ImmutableHashSet.Create( + StringComparer.Ordinal, + "schemaVersion", + "selector", + "images", + "usageOnly", + "generatedAt", + "total", + "snapshotId"); + + public static SchedulerSchemaMigrationResult UpgradeSchedule(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.Schedule, + SchedulerSchemaVersions.EnsureSchedule, + ScheduleProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + strict); + + public static SchedulerSchemaMigrationResult UpgradeRun(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.Run, + SchedulerSchemaVersions.EnsureRun, + RunProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + strict); + + public static SchedulerSchemaMigrationResult UpgradeImpactSet(JsonNode document, bool strict = false) + => Upgrade( + document, + SchedulerSchemaVersions.ImpactSet, + SchedulerSchemaVersions.EnsureImpactSet, + ImpactSetProperties, + static json => CanonicalJsonSerializer.Deserialize(json), + strict); + + private static SchedulerSchemaMigrationResult Upgrade( + JsonNode document, + string latestVersion, + Func ensureVersion, + ImmutableHashSet knownProperties, + Func deserialize, + bool strict) + { + ArgumentNullException.ThrowIfNull(document); + + var (normalized, fromVersion) = Normalize(document, ensureVersion); + var warnings = ImmutableArray.CreateBuilder(); + + if (strict) + { + RemoveUnknownMembers(normalized, knownProperties, warnings, fromVersion); + } + + if (!string.Equals(fromVersion, latestVersion, StringComparison.Ordinal)) + { + // Placeholder for forward upgrades once schema@2 exists. + throw new NotSupportedException($"Unsupported scheduler schema version '{fromVersion}', expected '{latestVersion}'."); + } + + var canonicalJson = normalized.ToJsonString(new JsonSerializerOptions + { + WriteIndented = false, + }); + + var value = deserialize(canonicalJson); + return new SchedulerSchemaMigrationResult( + value, + fromVersion, + latestVersion, + warnings.ToImmutable()); + } + + private static (JsonObject Clone, string SchemaVersion) Normalize(JsonNode node, Func ensureVersion) + { + if (node is not JsonObject obj) + { + throw new ArgumentException("Document must be a JSON object.", nameof(node)); + } + + if (obj.DeepClone() is not JsonObject clone) + { + throw new InvalidOperationException("Unable to clone scheduler document."); + } + + string schemaVersion; + if (clone.TryGetPropertyValue("schemaVersion", out var value) && + value is JsonValue jsonValue && + jsonValue.TryGetValue(out string? rawVersion)) + { + schemaVersion = ensureVersion(rawVersion); + } + else + { + schemaVersion = ensureVersion(null); + clone["schemaVersion"] = schemaVersion; + } + + // Ensure schemaVersion is normalized in the clone. + clone["schemaVersion"] = schemaVersion; + + return (clone, schemaVersion); + } + + private static void RemoveUnknownMembers( + JsonObject json, + ImmutableHashSet knownProperties, + ImmutableArray.Builder warnings, + string schemaVersion) + { + var unknownKeys = json + .Where(static pair => pair.Key is not null) + .Select(pair => pair.Key!) + .Where(key => !knownProperties.Contains(key)) + .ToArray(); + + foreach (var key in unknownKeys) + { + json.Remove(key); + warnings.Add($"Removed unknown property '{key}' from scheduler document (schemaVersion={schemaVersion})."); + } + } +} diff --git a/src/StellaOps.Scheduler.Models/SchedulerSchemaMigrationResult.cs b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigrationResult.cs new file mode 100644 index 00000000..7f0907a3 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/SchedulerSchemaMigrationResult.cs @@ -0,0 +1,13 @@ +using System.Collections.Immutable; + +namespace StellaOps.Scheduler.Models; + +/// +/// Result from upgrading a scheduler document to the latest schema version. +/// +/// Target DTO type. +public sealed record SchedulerSchemaMigrationResult( + T Value, + string FromVersion, + string ToVersion, + ImmutableArray Warnings); diff --git a/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs b/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs new file mode 100644 index 00000000..2a3f725b --- /dev/null +++ b/src/StellaOps.Scheduler.Models/SchedulerSchemaVersions.cs @@ -0,0 +1,26 @@ +namespace StellaOps.Scheduler.Models; + +/// +/// Canonical schema version identifiers for scheduler documents. +/// +public static class SchedulerSchemaVersions +{ + public const string Schedule = "scheduler.schedule@1"; + public const string Run = "scheduler.run@1"; + public const string ImpactSet = "scheduler.impact-set@1"; + public const string ScheduleLegacy0 = "scheduler.schedule@0"; + public const string RunLegacy0 = "scheduler.run@0"; + public const string ImpactSetLegacy0 = "scheduler.impact-set@0"; + + public static string EnsureSchedule(string? value) + => Normalize(value, Schedule); + + public static string EnsureRun(string? value) + => Normalize(value, Run); + + public static string EnsureImpactSet(string? value) + => Normalize(value, ImpactSet); + + private static string Normalize(string? value, string fallback) + => string.IsNullOrWhiteSpace(value) ? fallback : value.Trim(); +} diff --git a/src/StellaOps.Scheduler.Models/Selector.cs b/src/StellaOps.Scheduler.Models/Selector.cs new file mode 100644 index 00000000..235b8d37 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/Selector.cs @@ -0,0 +1,134 @@ +using System.Collections.Immutable; +using System.Text.Json.Serialization; + +namespace StellaOps.Scheduler.Models; + +/// +/// Selector filters used to resolve impacted assets. +/// +public sealed record Selector +{ + public Selector( + SelectorScope scope, + string? tenantId = null, + IEnumerable? namespaces = null, + IEnumerable? repositories = null, + IEnumerable? digests = null, + IEnumerable? includeTags = null, + IEnumerable? labels = null, + bool resolvesTags = false) + : this( + scope, + tenantId, + Validation.NormalizeStringSet(namespaces, nameof(namespaces)), + Validation.NormalizeStringSet(repositories, nameof(repositories)), + Validation.NormalizeDigests(digests, nameof(digests)), + Validation.NormalizeTagPatterns(includeTags), + NormalizeLabels(labels), + resolvesTags) + { + } + + [JsonConstructor] + public Selector( + SelectorScope scope, + string? tenantId, + ImmutableArray namespaces, + ImmutableArray repositories, + ImmutableArray digests, + ImmutableArray includeTags, + ImmutableArray labels, + bool resolvesTags) + { + Scope = scope; + TenantId = tenantId is null ? null : Validation.EnsureTenantId(tenantId, nameof(tenantId)); + Namespaces = namespaces.IsDefault ? ImmutableArray.Empty : namespaces; + Repositories = repositories.IsDefault ? ImmutableArray.Empty : repositories; + Digests = digests.IsDefault ? ImmutableArray.Empty : digests; + IncludeTags = includeTags.IsDefault ? ImmutableArray.Empty : includeTags; + Labels = labels.IsDefault ? ImmutableArray.Empty : labels; + ResolvesTags = resolvesTags; + + if (Scope is SelectorScope.ByDigest && Digests.Length == 0) + { + throw new ArgumentException("At least one digest is required when scope is by-digest.", nameof(digests)); + } + + if (Scope is SelectorScope.ByNamespace && Namespaces.Length == 0) + { + throw new ArgumentException("Namespaces are required when scope is by-namespace.", nameof(namespaces)); + } + + if (Scope is SelectorScope.ByRepository && Repositories.Length == 0) + { + throw new ArgumentException("Repositories are required when scope is by-repo.", nameof(repositories)); + } + + if (Scope is SelectorScope.ByLabels && Labels.Length == 0) + { + throw new ArgumentException("Labels are required when scope is by-labels.", nameof(labels)); + } + } + + public SelectorScope Scope { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? TenantId { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Namespaces { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Repositories { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Digests { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray IncludeTags { get; } = ImmutableArray.Empty; + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Labels { get; } = ImmutableArray.Empty; + + public bool ResolvesTags { get; } + + private static ImmutableArray NormalizeLabels(IEnumerable? labels) + { + if (labels is null) + { + return ImmutableArray.Empty; + } + + return labels + .Where(static label => label is not null) + .Select(static label => label!) + .OrderBy(static label => label.Key, StringComparer.Ordinal) + .ToImmutableArray(); + } +} + +/// +/// Describes a label match (key and optional accepted values). +/// +public sealed record LabelSelector +{ + public LabelSelector(string key, IEnumerable? values = null) + : this(key, NormalizeValues(values)) + { + } + + [JsonConstructor] + public LabelSelector(string key, ImmutableArray values) + { + Key = Validation.EnsureSimpleIdentifier(key, nameof(key)); + Values = values.IsDefault ? ImmutableArray.Empty : values; + } + + public string Key { get; } + + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public ImmutableArray Values { get; } = ImmutableArray.Empty; + + private static ImmutableArray NormalizeValues(IEnumerable? values) + => Validation.NormalizeStringSet(values, nameof(values)); +} diff --git a/src/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj b/src/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj new file mode 100644 index 00000000..da8a44d8 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/StellaOps.Scheduler.Models.csproj @@ -0,0 +1,9 @@ + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Scheduler.Models/TASKS.md b/src/StellaOps.Scheduler.Models/TASKS.md new file mode 100644 index 00000000..fd871f78 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/TASKS.md @@ -0,0 +1,7 @@ +# Scheduler Models Task Board (Sprint 16) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-MODELS-16-101 | DONE (2025-10-19) | Scheduler Models Guild | — | Define DTOs (Schedule, Run, ImpactSet, Selector, DeltaSummary, AuditRecord) with validation + canonical JSON. | DTOs merged with tests; documentation snippet added; serialization deterministic. | +| SCHED-MODELS-16-102 | DONE (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Publish schema docs & sample payloads for UI/Notify integration. | Samples committed; docs referenced; contract tests pass. | +| SCHED-MODELS-16-103 | DOING (2025-10-19) | Scheduler Models Guild | SCHED-MODELS-16-101 | Versioning/migration helpers (schedule evolution, run state transitions). | Migration helpers implemented; tests cover upgrade/downgrade; guidelines documented. | diff --git a/src/StellaOps.Scheduler.Models/Validation.cs b/src/StellaOps.Scheduler.Models/Validation.cs new file mode 100644 index 00000000..d8628580 --- /dev/null +++ b/src/StellaOps.Scheduler.Models/Validation.cs @@ -0,0 +1,247 @@ +using System.Collections.Immutable; +using System.Text.RegularExpressions; + +namespace StellaOps.Scheduler.Models; + +/// +/// Lightweight validation helpers for scheduler DTO constructors. +/// +internal static partial class Validation +{ + private const int MaxIdentifierLength = 256; + private const int MaxNameLength = 200; + + public static string EnsureId(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName); + if (normalized.Length > MaxIdentifierLength) + { + throw new ArgumentException($"Value exceeds {MaxIdentifierLength} characters.", paramName); + } + + return normalized; + } + + public static string EnsureName(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName); + if (normalized.Length > MaxNameLength) + { + throw new ArgumentException($"Value exceeds {MaxNameLength} characters.", paramName); + } + + return normalized; + } + + public static string EnsureTenantId(string value, string paramName) + { + var normalized = EnsureId(value, paramName); + if (!TenantRegex().IsMatch(normalized)) + { + throw new ArgumentException("Tenant id must be alphanumeric with '-', '_' separators.", paramName); + } + + return normalized; + } + + public static string EnsureCronExpression(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName); + if (normalized.Length > 128 || normalized.Contains('\n', StringComparison.Ordinal) || normalized.Contains('\r', StringComparison.Ordinal)) + { + throw new ArgumentException("Cron expression too long or contains invalid characters.", paramName); + } + + if (!CronSegmentRegex().IsMatch(normalized)) + { + throw new ArgumentException("Cron expression contains unsupported characters.", paramName); + } + + return normalized; + } + + public static string EnsureTimezone(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName); + try + { + _ = TimeZoneInfo.FindSystemTimeZoneById(normalized); + } + catch (TimeZoneNotFoundException ex) + { + throw new ArgumentException($"Timezone '{normalized}' is not recognized on this host.", paramName, ex); + } + catch (InvalidTimeZoneException ex) + { + throw new ArgumentException($"Timezone '{normalized}' is invalid.", paramName, ex); + } + + return normalized; + } + + public static string? TrimToNull(string? value) + => string.IsNullOrWhiteSpace(value) + ? null + : value.Trim(); + + public static ImmutableArray NormalizeStringSet(IEnumerable? values, string paramName, bool allowWildcards = false) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + var result = values + .Select(static value => TrimToNull(value)) + .Where(static value => value is not null) + .Select(value => allowWildcards ? value! : EnsureSimpleIdentifier(value!, paramName)) + .Distinct(StringComparer.Ordinal) + .OrderBy(static value => value, StringComparer.Ordinal) + .ToImmutableArray(); + + return result; + } + + public static ImmutableArray NormalizeTagPatterns(IEnumerable? values) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + var result = values + .Select(static value => TrimToNull(value)) + .Where(static value => value is not null) + .Select(static value => value!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return result; + } + + public static ImmutableArray NormalizeDigests(IEnumerable? values, string paramName) + { + if (values is null) + { + return ImmutableArray.Empty; + } + + var result = values + .Select(static value => TrimToNull(value)) + .Where(static value => value is not null) + .Select(value => EnsureDigestFormat(value!, paramName)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(static value => value, StringComparer.OrdinalIgnoreCase) + .ToImmutableArray(); + + return result; + } + + public static int? EnsurePositiveOrNull(int? value, string paramName) + { + if (value is null) + { + return null; + } + + if (value <= 0) + { + throw new ArgumentOutOfRangeException(paramName, value, "Value must be greater than zero."); + } + + return value; + } + + public static int EnsureNonNegative(int value, string paramName) + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(paramName, value, "Value must be zero or greater."); + } + + return value; + } + + public static ImmutableSortedDictionary NormalizeMetadata(IEnumerable>? metadata) + { + if (metadata is null) + { + return ImmutableSortedDictionary.Empty; + } + + var builder = ImmutableSortedDictionary.CreateBuilder(StringComparer.Ordinal); + foreach (var pair in metadata) + { + var key = TrimToNull(pair.Key); + var value = TrimToNull(pair.Value); + if (key is null || value is null) + { + continue; + } + + var normalizedKey = key.ToLowerInvariant(); + if (!builder.ContainsKey(normalizedKey)) + { + builder[normalizedKey] = value; + } + } + + return builder.ToImmutable(); + } + + public static string EnsureSimpleIdentifier(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName); + if (!SimpleIdentifierRegex().IsMatch(normalized)) + { + throw new ArgumentException("Value must contain letters, digits, '-', '_', '.', or '/'.", paramName); + } + + return normalized; + } + + public static string EnsureDigestFormat(string value, string paramName) + { + var normalized = EnsureNotNullOrWhiteSpace(value, paramName).ToLowerInvariant(); + if (!normalized.StartsWith("sha256:", StringComparison.Ordinal) || normalized.Length <= 7) + { + throw new ArgumentException("Digest must start with 'sha256:' and contain a hex payload.", paramName); + } + + if (!HexRegex().IsMatch(normalized.AsSpan(7))) + { + throw new ArgumentException("Digest must be hexadecimal.", paramName); + } + + return normalized; + } + + public static string EnsureNotNullOrWhiteSpace(string value, string paramName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new ArgumentException("Value cannot be null or whitespace.", paramName); + } + + return value.Trim(); + } + + public static DateTimeOffset NormalizeTimestamp(DateTimeOffset value) + => value.ToUniversalTime(); + + public static DateTimeOffset? NormalizeTimestamp(DateTimeOffset? value) + => value?.ToUniversalTime(); + + [GeneratedRegex("^[A-Za-z0-9_-]+$")] + private static partial Regex TenantRegex(); + + [GeneratedRegex("^[A-Za-z0-9_./:@+\\-]+$")] + private static partial Regex SimpleIdentifierRegex(); + + [GeneratedRegex("^[A-Za-z0-9:*?/_.,\\- ]+$")] + private static partial Regex CronSegmentRegex(); + + [GeneratedRegex("^[a-f0-9]+$", RegexOptions.IgnoreCase)] + private static partial Regex HexRegex(); +} diff --git a/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md new file mode 100644 index 00000000..adae0c7a --- /dev/null +++ b/src/StellaOps.Scheduler.Models/docs/SCHED-MODELS-16-103-DESIGN.md @@ -0,0 +1,80 @@ +# SCHED-MODELS-16-103 — Scheduler Schema Versioning & Run State Helpers + +## Goals +- Track schema revisions for `Schedule` and `Run` documents so storage upgrades are deterministic across air-gapped installs. +- Provide reusable upgrade helpers that normalize Mongo snapshots (raw BSON → JSON) into the latest DTOs without mutating inputs. +- Formalize the allowed `RunState` graph and surface guard-rail helpers (timestamps, stats monotonicity) for planners/runners. + +## Non-goals +- Implementing the helpers (covered by the main task). +- Downgrading documents to legacy schema revisions (can be added if Offline Kit requires it). +- Persisted data backfills or data migration jobs; we focus on in-process upgrades during read. + +## Schema Version Strategy +- Introduce `SchedulerSchemaVersions` constants: + - `scheduler.schedule@1` (base record with subscribers, limits burst default). + - `scheduler.run@1` (run metadata + delta summaries). + - `scheduler.impact-set@1` (shared envelope used by planners). +- Expose `EnsureSchedule`, `EnsureRun`, `EnsureImpactSet` helpers mirroring the Notify model pattern to normalize missing/whitespace values. +- Extend `Schedule`, `Run`, and `ImpactSet` records with an optional `schemaVersion` constructor parameter defaulting through the `Ensure*` helpers. The canonical JSON serializer will list `schemaVersion` first so documents round-trip deterministically. +- Persisted Mongo documents will now always include `schemaVersion`; exporters/backups can rely on this when bundling Offline Kit snapshots. + +## Migration Helper Shape +- Add `SchedulerSchemaMigration` static class with: + - `Schedule UpgradeSchedule(JsonNode document)` + - `Run UpgradeRun(JsonNode document)` + - `ImpactSet UpgradeImpactSet(JsonNode document)` +- Each method clones the incoming node, normalizes `schemaVersion` (injecting default if missing), then applies an upgrade pipeline: + 1. `Normalize` — ensure object, strip unknown members when `strict` flag is set, coerce enums via converters. + 2. `ApplyLegacyFixups` — version-specific patches, e.g., backfill `subscribers`, migrate `limits.burst`, convert legacy trigger strings. + 3. `Deserialize` — use `CanonicalJsonSerializer.Deserialize` so property order/enum parsing stays centralized. +- Expose `SchedulerSchemaMigrationResult` record returning `(T Value, string FromVersion, string ToVersion, ImmutableArray Warnings)` to surface non-blocking issues to callers (web service, worker, storage). +- Helpers remain dependency-free so storage/web modules can reference them without circular dependencies. + +## Schedule Evolution Considerations +- **@1** fields: `mode`, `selection`, `onlyIf`, `notify`, `limits` (incl. `burst` default 0), `subscribers` (sorted unique), audit metadata. +- Future **@2** candidate changes to plan for in helpers: + - `limits`: splitting `parallelism` into planner/runner concurrency. + - `selection`: adding `impactWindow` semantics. + - `notify`: optional per-channel overrides. +- Upgrade pipeline will carry forward unknown fields in a `JsonNode` bag so future versions can opt-in to strict dropping while maintaining backwards compatibility for current release. + +## Run State Transition Helper +- Introduce `RunStateMachine` (static) encapsulating allowed transitions and invariants. + - Define adjacency map: + - `Planning → {Queued, Cancelled}` + - `Queued → {Running, Cancelled}` + - `Running → {Completed, Error, Cancelled}` + - `Completed`, `Error`, `Cancelled` are terminal. + - Provide `bool CanTransition(RunState from, RunState to)` and `Run EnsureTransition(Run run, RunState next, DateTimeOffset now, Action? mutateStats = null)`. +- `EnsureTransition` performs: + - Timestamp enforcement: `StartedAt` auto-populated on first entry into `Running`; `FinishedAt` set when entering any terminal state; ensures monotonic ordering (`CreatedAt ≤ StartedAt ≤ FinishedAt`). + - Stats guardrails: cumulative counters must not decrease; `RunStatsBuilder` wrapper ensures atomic updates. + - Error context: require `error` message when transitioning to `Error`; clear error for non-error entries. +- Provide `Validate(Run run)` to check invariants for documents loaded from storage before use (e.g., stale snapshots). +- Expose small helper to tag `RunReason.ImpactWindowFrom/To` automatically when set by planners (using normalized ISO-8601). + +## Interaction Points +- **WebService**: call `SchedulerSchemaMigration.UpgradeSchedule` when returning schedules from Mongo, so clients always see the newest DTO regardless of stored version. +- **Storage.Mongo**: wrap DTO round-trips; the migration helper acts during read, and the state machine ensures updates respect transition rules before writing. +- **Queue/Worker**: use `RunStateMachine.EnsureTransition` to guard planner/runner state updates (replace ad-hoc `with run` clones). +- **Offline Kit**: embed `schemaVersion` in exported JSON/Trivy artifacts; migrations ensure air-gapped upgrades flow without manual scripts. + +## Implementation Steps (for follow-up task) +1. Add `SchedulerSchemaVersions` + update DTO constructors/properties. +2. Implement `SchedulerSchemaMigration` helpers and shared `MigrationResult` envelope. +3. Introduce `RunStateMachine` with invariants + supporting `RunStatsBuilder`. +4. Update modules (Storage, WebService, Worker) to use new helpers; add logging around migrations/transitions. + +## Test Strategy +- **Migration happy-path**: load sample Mongo fixtures for `schedule@1` and `run@1`, assert `schemaVersion` normalization, deduplicated subscribers, limits defaults. Include snapshots without the version field to exercise defaulting logic. +- **Legacy upgrade cases**: craft synthetic `schedule@0` / `run@0` JSON fragments (missing new fields, using old enum names) and verify version-specific fixups produce the latest DTO while populating `MigrationResult.Warnings`. +- **Strict mode behavior**: attempt to upgrade documents with unexpected properties and ensure warnings/throws align with configuration. +- **Run state transitions**: unit-test `RunStateMachine` for every allowed edge, invalid transitions, and timestamp/error invariants (e.g., `FinishedAt` only set on terminal states). Provide parameterized tests to confirm stats monotonicity enforcement. +- **Serialization determinism**: round-trip upgraded DTOs via `CanonicalJsonSerializer` to confirm property order includes `schemaVersion` first and produces stable hashes. +- **Documentation snippets**: extend module README or API docs with example migrations/run-state usage; verify via doc samples test (if available) or include as part of CI doc linting. + +## Open Questions +- Do we need downgrade (`ToVersion`) helpers for Offline Kit exports? (Assumed no for now. Add backlog item if required.) +- Should `ImpactSet` migrations live here or in ImpactIndex module? (Lean towards here because DTO defined in Models; coordinate with ImpactIndex guild if they need specialized upgrades.) +- How do we surface migration warnings to telemetry? Proposal: caller logs `warning` with `MigrationResult.Warnings` immediately after calling helper. diff --git a/src/StellaOps.Scheduler.Queue.Tests/PlannerAndRunnerMessageTests.cs b/src/StellaOps.Scheduler.Queue.Tests/PlannerAndRunnerMessageTests.cs new file mode 100644 index 00000000..ffe8804d --- /dev/null +++ b/src/StellaOps.Scheduler.Queue.Tests/PlannerAndRunnerMessageTests.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using FluentAssertions; +using StellaOps.Scheduler.Models; +using Xunit; + +namespace StellaOps.Scheduler.Queue.Tests; + +public sealed class PlannerAndRunnerMessageTests +{ + [Fact] + public void PlannerMessage_CanonicalSerialization_RoundTrips() + { + var schedule = new Schedule( + id: "sch-tenant-nightly", + tenantId: "tenant-alpha", + name: "Nightly Deltas", + enabled: true, + cronExpression: "0 2 * * *", + timezone: "UTC", + mode: ScheduleMode.AnalysisOnly, + selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), + onlyIf: new ScheduleOnlyIf(lastReportOlderThanDays: 3), + notify: new ScheduleNotify(onNewFindings: true, SeverityRank.High, includeKev: true), + limits: new ScheduleLimits(maxJobs: 10, ratePerSecond: 5, parallelism: 3), + createdAt: DateTimeOffset.Parse("2025-10-01T02:00:00Z"), + createdBy: "system", + updatedAt: DateTimeOffset.Parse("2025-10-02T02:00:00Z"), + updatedBy: "system", + subscribers: ImmutableArray.Empty, + schemaVersion: "1.0.0"); + + var run = new Run( + id: "run-123", + tenantId: "tenant-alpha", + trigger: RunTrigger.Cron, + state: RunState.Planning, + stats: new RunStats(candidates: 5, deduped: 4, queued: 0, completed: 0, deltas: 0), + createdAt: DateTimeOffset.Parse("2025-10-02T02:05:00Z"), + reason: new RunReason(manualReason: null, feedserExportId: null, vexerExportId: null, cursor: null) + with { ImpactWindowFrom = "2025-10-01T00:00:00Z", ImpactWindowTo = "2025-10-02T00:00:00Z" }, + scheduleId: "sch-tenant-nightly"); + + var impactSet = new ImpactSet( + selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), + images: new[] + { + new ImpactImage( + imageDigest: "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + registry: "registry", + repository: "repo", + namespaces: new[] { "prod" }, + tags: new[] { "latest" }, + usedByEntrypoint: true, + labels: new[] { KeyValuePair.Create("team", "appsec") }) + }, + usageOnly: true, + generatedAt: DateTimeOffset.Parse("2025-10-02T02:06:00Z"), + total: 1, + snapshotId: "snap-001"); + + var message = new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-1"); + + var json = CanonicalJsonSerializer.Serialize(message); + var roundTrip = CanonicalJsonSerializer.Deserialize(json); + + roundTrip.Should().BeEquivalentTo(message, options => options.WithStrictOrdering()); + } + + [Fact] + public void RunnerSegmentMessage_RequiresAtLeastOneDigest() + { + var act = () => new RunnerSegmentQueueMessage( + segmentId: "segment-empty", + runId: "run-123", + tenantId: "tenant-alpha", + imageDigests: Array.Empty()); + + act.Should().Throw(); + } + + [Fact] + public void RunnerSegmentMessage_CanonicalSerialization_RoundTrips() + { + var message = new RunnerSegmentQueueMessage( + segmentId: "segment-01", + runId: "run-123", + tenantId: "tenant-alpha", + imageDigests: new[] + { + "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + scheduleId: "sch-tenant-nightly", + ratePerSecond: 25, + usageOnly: true, + attributes: new Dictionary + { + ["plannerShard"] = "0", + ["priority"] = "kev" + }, + correlationId: "corr-2"); + + var json = CanonicalJsonSerializer.Serialize(message); + var roundTrip = CanonicalJsonSerializer.Deserialize(json); + + roundTrip.Should().BeEquivalentTo(message, options => options.WithStrictOrdering()); + } +} diff --git a/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs b/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs new file mode 100644 index 00000000..a5f2302a --- /dev/null +++ b/src/StellaOps.Scheduler.Queue.Tests/RedisSchedulerQueueTests.cs @@ -0,0 +1,269 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Configurations; +using FluentAssertions; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using StellaOps.Scheduler.Models; +using StellaOps.Scheduler.Queue.Redis; +using Xunit; + +namespace StellaOps.Scheduler.Queue.Tests; + +public sealed class RedisSchedulerQueueTests : IAsyncLifetime +{ + private readonly RedisTestcontainer _redis; + private string? _skipReason; + + public RedisSchedulerQueueTests() + { + var configuration = new RedisTestcontainerConfiguration(); + + _redis = new TestcontainersBuilder() + .WithDatabase(configuration) + .Build(); + } + + public async Task InitializeAsync() + { + try + { + await _redis.StartAsync(); + } + catch (Exception ex) when (IsDockerUnavailable(ex)) + { + _skipReason = $"Docker engine is not available for Redis-backed tests: {ex.Message}"; + } + } + + public async Task DisposeAsync() + { + if (_skipReason is not null) + { + return; + } + + await _redis.DisposeAsync().AsTask(); + } + + [Fact] + public async Task PlannerQueue_EnqueueLeaseAck_RemovesMessage() + { + SkipIfUnavailable(); + + var options = CreateOptions(); + + await using var queue = new RedisSchedulerPlannerQueue( + options, + options.Redis, + NullLogger.Instance, + TimeProvider.System, + async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); + + var message = TestData.CreatePlannerMessage(); + + var enqueue = await queue.EnqueueAsync(message); + enqueue.Deduplicated.Should().BeFalse(); + + var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", batchSize: 5, options.DefaultLeaseDuration)); + leases.Should().HaveCount(1); + + var lease = leases[0]; + lease.Message.Run.Id.Should().Be(message.Run.Id); + lease.TenantId.Should().Be(message.TenantId); + lease.ScheduleId.Should().Be(message.ScheduleId); + + await lease.AcknowledgeAsync(); + + var afterAck = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-1", 5, options.DefaultLeaseDuration)); + afterAck.Should().BeEmpty(); + } + + [Fact] + public async Task RunnerQueue_Retry_IncrementsDeliveryAttempt() + { + SkipIfUnavailable(); + + var options = CreateOptions(); + options.RetryInitialBackoff = TimeSpan.Zero; + options.RetryMaxBackoff = TimeSpan.Zero; + + await using var queue = new RedisSchedulerRunnerQueue( + options, + options.Redis, + NullLogger.Instance, + TimeProvider.System, + async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); + + var message = TestData.CreateRunnerMessage(); + + await queue.EnqueueAsync(message); + + var firstLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration)); + firstLease.Should().ContainSingle(); + + var lease = firstLease[0]; + lease.Attempt.Should().Be(1); + + await lease.ReleaseAsync(SchedulerQueueReleaseDisposition.Retry); + + var secondLease = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("runner-1", batchSize: 1, options.DefaultLeaseDuration)); + secondLease.Should().ContainSingle(); + secondLease[0].Attempt.Should().Be(2); + } + + [Fact] + public async Task PlannerQueue_ClaimExpired_ReassignsLease() + { + SkipIfUnavailable(); + + var options = CreateOptions(); + + await using var queue = new RedisSchedulerPlannerQueue( + options, + options.Redis, + NullLogger.Instance, + TimeProvider.System, + async config => (IConnectionMultiplexer)await ConnectionMultiplexer.ConnectAsync(config).ConfigureAwait(false)); + + var message = TestData.CreatePlannerMessage(); + await queue.EnqueueAsync(message); + + var leases = await queue.LeaseAsync(new SchedulerQueueLeaseRequest("planner-a", 1, options.DefaultLeaseDuration)); + leases.Should().ContainSingle(); + + await Task.Delay(50); + + var reclaimed = await queue.ClaimExpiredAsync(new SchedulerQueueClaimOptions("planner-b", batchSize: 1, minIdleTime: TimeSpan.Zero)); + reclaimed.Should().ContainSingle(); + reclaimed[0].Consumer.Should().Be("planner-b"); + reclaimed[0].RunId.Should().Be(message.Run.Id); + + await reclaimed[0].AcknowledgeAsync(); + } + + private SchedulerQueueOptions CreateOptions() + { + var unique = Guid.NewGuid().ToString("N"); + + return new SchedulerQueueOptions + { + Kind = SchedulerQueueTransportKind.Redis, + DefaultLeaseDuration = TimeSpan.FromSeconds(2), + MaxDeliveryAttempts = 5, + RetryInitialBackoff = TimeSpan.FromMilliseconds(10), + RetryMaxBackoff = TimeSpan.FromMilliseconds(50), + Redis = new SchedulerRedisQueueOptions + { + ConnectionString = _redis.ConnectionString, + Database = 0, + InitializationTimeout = TimeSpan.FromSeconds(10), + Planner = new RedisSchedulerStreamOptions + { + Stream = $"scheduler:test:planner:{unique}", + ConsumerGroup = $"planner-consumers-{unique}", + DeadLetterStream = $"scheduler:test:planner:{unique}:dead", + IdempotencyKeyPrefix = $"scheduler:test:planner:{unique}:idemp:", + IdempotencyWindow = TimeSpan.FromMinutes(5) + }, + Runner = new RedisSchedulerStreamOptions + { + Stream = $"scheduler:test:runner:{unique}", + ConsumerGroup = $"runner-consumers-{unique}", + DeadLetterStream = $"scheduler:test:runner:{unique}:dead", + IdempotencyKeyPrefix = $"scheduler:test:runner:{unique}:idemp:", + IdempotencyWindow = TimeSpan.FromMinutes(5) + } + } + }; + } + + private void SkipIfUnavailable() + { + if (_skipReason is not null) + { + Skip.If(true, _skipReason); + } + } + + private static bool IsDockerUnavailable(Exception exception) + { + while (exception is AggregateException aggregate && aggregate.InnerException is not null) + { + exception = aggregate.InnerException; + } + + return exception is TimeoutException + || exception.GetType().Name.Contains("Docker", StringComparison.OrdinalIgnoreCase); + } + + private static class TestData + { + public static PlannerQueueMessage CreatePlannerMessage() + { + var schedule = new Schedule( + id: "sch-test", + tenantId: "tenant-alpha", + name: "Test", + enabled: true, + cronExpression: "0 0 * * *", + timezone: "UTC", + mode: ScheduleMode.AnalysisOnly, + selection: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), + onlyIf: ScheduleOnlyIf.Default, + notify: ScheduleNotify.Default, + limits: ScheduleLimits.Default, + createdAt: DateTimeOffset.UtcNow, + createdBy: "tests", + updatedAt: DateTimeOffset.UtcNow, + updatedBy: "tests"); + + var run = new Run( + id: "run-test", + tenantId: "tenant-alpha", + trigger: RunTrigger.Manual, + state: RunState.Planning, + stats: RunStats.Empty, + createdAt: DateTimeOffset.UtcNow, + reason: RunReason.Empty, + scheduleId: schedule.Id); + + var impactSet = new ImpactSet( + selector: new Selector(SelectorScope.AllImages, tenantId: "tenant-alpha"), + images: new[] + { + new ImpactImage( + imageDigest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + registry: "registry", + repository: "repo", + namespaces: new[] { "prod" }, + tags: new[] { "latest" }) + }, + usageOnly: true, + generatedAt: DateTimeOffset.UtcNow, + total: 1); + + return new PlannerQueueMessage(run, impactSet, schedule, correlationId: "corr-test"); + } + + public static RunnerSegmentQueueMessage CreateRunnerMessage() + { + return new RunnerSegmentQueueMessage( + segmentId: "segment-test", + runId: "run-test", + tenantId: "tenant-alpha", + imageDigests: new[] + { + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + scheduleId: "sch-test", + ratePerSecond: 10, + usageOnly: true, + attributes: new Dictionary { ["priority"] = "kev" }, + correlationId: "corr-runner"); + } + } +} diff --git a/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj new file mode 100644 index 00000000..84663c3f --- /dev/null +++ b/src/StellaOps.Scheduler.Queue.Tests/StellaOps.Scheduler.Queue.Tests.csproj @@ -0,0 +1,25 @@ + + + net10.0 + enable + enable + false + false + + + + + + + + all + + + all + + + + + + + diff --git a/src/StellaOps.Scheduler.Queue/AGENTS.md b/src/StellaOps.Scheduler.Queue/AGENTS.md new file mode 100644 index 00000000..75b36a81 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.Queue — Agent Charter + +## Mission +Provide queue abstraction (Redis Streams / NATS JetStream) for planner inputs and runner segments per `docs/ARCHITECTURE_SCHEDULER.md`. diff --git a/src/StellaOps.Scheduler.Queue/AssemblyInfo.cs b/src/StellaOps.Scheduler.Queue/AssemblyInfo.cs new file mode 100644 index 00000000..f9558736 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scheduler.Queue.Tests")] diff --git a/src/StellaOps.Scheduler.Queue/README.md b/src/StellaOps.Scheduler.Queue/README.md new file mode 100644 index 00000000..066c389c --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/README.md @@ -0,0 +1,16 @@ +# Scheduler Queue — Sprint 16 Coordination Notes + +Queue work now has concrete contracts from `StellaOps.Scheduler.Models`: + +* Planner inputs reference `Schedule` and `ImpactSet` samples (`samples/api/scheduler/`). +* Runner segment payloads should carry `runId`, `scheduleId?`, `tenantId`, and the impacted digest list (mirrors `Run.Deltas`). +* Notify fanout relies on the `DeltaSummary` shape already emitted by the model layer. + +## Action items for SCHED-QUEUE-16-401..403 + +1. Reference `StellaOps.Scheduler.Models` so adapters can serialise `Run`/`DeltaSummary` without bespoke DTOs. +2. Use the canonical serializer for queue messages to keep ordering consistent with API payloads. +3. Coverage: add fixture-driven tests that enqueue the sample payloads, then dequeue and re-serialise to verify byte-for-byte stability. +4. Expose queue depth/lease metrics with the identifiers provided by the models (`Run.Id`, `Schedule.Id`). + +These notes unblock the queue guild now that SCHED-MODELS-16-102 is complete. diff --git a/src/StellaOps.Scheduler.Queue/Redis/IRedisSchedulerQueuePayload.cs b/src/StellaOps.Scheduler.Queue/Redis/IRedisSchedulerQueuePayload.cs new file mode 100644 index 00000000..fb1dc8d5 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Redis/IRedisSchedulerQueuePayload.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +namespace StellaOps.Scheduler.Queue.Redis; + +internal interface IRedisSchedulerQueuePayload +{ + string QueueName { get; } + + string GetIdempotencyKey(TMessage message); + + string Serialize(TMessage message); + + TMessage Deserialize(string payload); + + string GetRunId(TMessage message); + + string GetTenantId(TMessage message); + + string? GetScheduleId(TMessage message); + + string? GetSegmentId(TMessage message); + + string? GetCorrelationId(TMessage message); + + IReadOnlyDictionary? GetAttributes(TMessage message); +} diff --git a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerPlannerQueue.cs b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerPlannerQueue.cs new file mode 100644 index 00000000..bf3ff9c9 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerPlannerQueue.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Queue.Redis; + +internal sealed class RedisSchedulerPlannerQueue + : RedisSchedulerQueueBase, ISchedulerPlannerQueue +{ + public RedisSchedulerPlannerQueue( + SchedulerQueueOptions queueOptions, + SchedulerRedisQueueOptions redisOptions, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + : base( + queueOptions, + redisOptions, + redisOptions.Planner, + PlannerPayload.Instance, + logger, + timeProvider, + connectionFactory) + { + } + + private sealed class PlannerPayload : IRedisSchedulerQueuePayload + { + public static PlannerPayload Instance { get; } = new(); + + public string QueueName => "planner"; + + public string GetIdempotencyKey(PlannerQueueMessage message) + => message.IdempotencyKey; + + public string Serialize(PlannerQueueMessage message) + => CanonicalJsonSerializer.Serialize(message); + + public PlannerQueueMessage Deserialize(string payload) + => CanonicalJsonSerializer.Deserialize(payload); + + public string GetRunId(PlannerQueueMessage message) + => message.Run.Id; + + public string GetTenantId(PlannerQueueMessage message) + => message.Run.TenantId; + + public string? GetScheduleId(PlannerQueueMessage message) + => message.ScheduleId; + + public string? GetSegmentId(PlannerQueueMessage message) + => null; + + public string? GetCorrelationId(PlannerQueueMessage message) + => message.CorrelationId; + + public IReadOnlyDictionary? GetAttributes(PlannerQueueMessage message) + => null; + } +} diff --git a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs new file mode 100644 index 00000000..e7220ddb --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueBase.cs @@ -0,0 +1,758 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; + +namespace StellaOps.Scheduler.Queue.Redis; + +internal abstract class RedisSchedulerQueueBase : ISchedulerQueue, IAsyncDisposable +{ + private const string TransportName = "redis"; + + private readonly SchedulerQueueOptions _queueOptions; + private readonly SchedulerRedisQueueOptions _redisOptions; + private readonly RedisSchedulerStreamOptions _streamOptions; + private readonly IRedisSchedulerQueuePayload _payload; + private readonly ILogger _logger; + private readonly TimeProvider _timeProvider; + private readonly Func> _connectionFactory; + private readonly SemaphoreSlim _connectionLock = new(1, 1); + private readonly SemaphoreSlim _groupInitLock = new(1, 1); + + private IConnectionMultiplexer? _connection; + private volatile bool _groupInitialized; + private bool _disposed; + + protected RedisSchedulerQueueBase( + SchedulerQueueOptions queueOptions, + SchedulerRedisQueueOptions redisOptions, + RedisSchedulerStreamOptions streamOptions, + IRedisSchedulerQueuePayload payload, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + { + _queueOptions = queueOptions ?? throw new ArgumentNullException(nameof(queueOptions)); + _redisOptions = redisOptions ?? throw new ArgumentNullException(nameof(redisOptions)); + _streamOptions = streamOptions ?? throw new ArgumentNullException(nameof(streamOptions)); + _payload = payload ?? throw new ArgumentNullException(nameof(payload)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + _connectionFactory = connectionFactory ?? (config => Task.FromResult(ConnectionMultiplexer.Connect(config))); + + if (string.IsNullOrWhiteSpace(_redisOptions.ConnectionString)) + { + throw new InvalidOperationException("Redis connection string must be configured for the scheduler queue."); + } + + if (string.IsNullOrWhiteSpace(_streamOptions.Stream)) + { + throw new InvalidOperationException("Redis stream name must be configured for the scheduler queue."); + } + + if (string.IsNullOrWhiteSpace(_streamOptions.ConsumerGroup)) + { + throw new InvalidOperationException("Redis consumer group must be configured for the scheduler queue."); + } + } + + public async ValueTask EnqueueAsync( + TMessage message, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(message); + cancellationToken.ThrowIfCancellationRequested(); + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false); + + var now = _timeProvider.GetUtcNow(); + var attempt = 1; + var entries = BuildEntries(message, now, attempt); + + var messageId = await AddToStreamAsync( + database, + _streamOptions.Stream, + entries, + _streamOptions.ApproximateMaxLength, + _streamOptions.ApproximateMaxLength is not null) + .ConfigureAwait(false); + + var idempotencyKey = BuildIdempotencyKey(_payload.GetIdempotencyKey(message)); + var stored = await database.StringSetAsync( + idempotencyKey, + messageId, + when: When.NotExists, + expiry: _streamOptions.IdempotencyWindow) + .ConfigureAwait(false); + + if (!stored) + { + await database.StreamDeleteAsync(_streamOptions.Stream, new RedisValue[] { messageId }).ConfigureAwait(false); + + var existing = await database.StringGetAsync(idempotencyKey).ConfigureAwait(false); + var reusable = existing.IsNullOrEmpty ? messageId : existing; + + SchedulerQueueMetrics.RecordDeduplicated(TransportName, _payload.QueueName); + _logger.LogDebug( + "Duplicate enqueue detected for scheduler queue {Queue} with key {Key}; returning existing stream id {StreamId}.", + _payload.QueueName, + idempotencyKey, + reusable.ToString()); + + return new SchedulerQueueEnqueueResult(reusable.ToString(), true); + } + + SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName); + _logger.LogDebug( + "Enqueued {Queue} message into {Stream} with id {StreamId}.", + _payload.QueueName, + _streamOptions.Stream, + messageId.ToString()); + + return new SchedulerQueueEnqueueResult(messageId.ToString(), false); + } + + public async ValueTask>> LeaseAsync( + SchedulerQueueLeaseRequest request, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(request); + cancellationToken.ThrowIfCancellationRequested(); + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false); + + var entries = await database.StreamReadGroupAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + request.Consumer, + position: ">", + count: request.BatchSize, + flags: CommandFlags.None) + .ConfigureAwait(false); + + if (entries is null || entries.Length == 0) + { + return Array.Empty>(); + } + + var now = _timeProvider.GetUtcNow(); + var leases = new List>(entries.Length); + + foreach (var entry in entries) + { + var lease = TryMapLease(entry, request.Consumer, now, request.LeaseDuration, attemptOverride: null); + if (lease is null) + { + await HandlePoisonEntryAsync(database, entry.Id).ConfigureAwait(false); + continue; + } + + leases.Add(lease); + } + + return leases; + } + + public async ValueTask>> ClaimExpiredAsync( + SchedulerQueueClaimOptions options, + CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(options); + cancellationToken.ThrowIfCancellationRequested(); + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await EnsureConsumerGroupAsync(database, cancellationToken).ConfigureAwait(false); + + var pending = await database.StreamPendingMessagesAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + options.BatchSize, + RedisValue.Null, + (long)options.MinIdleTime.TotalMilliseconds) + .ConfigureAwait(false); + + if (pending is null || pending.Length == 0) + { + return Array.Empty>(); + } + + var eligible = pending + .Where(info => info.IdleTimeInMilliseconds >= options.MinIdleTime.TotalMilliseconds) + .ToArray(); + + if (eligible.Length == 0) + { + return Array.Empty>(); + } + + var messageIds = eligible + .Select(info => (RedisValue)info.MessageId) + .ToArray(); + + var claimed = await database.StreamClaimAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + options.ClaimantConsumer, + 0, + messageIds, + CommandFlags.None) + .ConfigureAwait(false); + + if (claimed is null || claimed.Length == 0) + { + return Array.Empty>(); + } + + var now = _timeProvider.GetUtcNow(); + var attemptLookup = eligible.ToDictionary( + info => info.MessageId.IsNullOrEmpty ? string.Empty : info.MessageId.ToString(), + info => (int)Math.Max(1, info.DeliveryCount), + StringComparer.Ordinal); + + var leases = new List>(claimed.Length); + foreach (var entry in claimed) + { + var entryId = entry.Id.ToString(); + attemptLookup.TryGetValue(entryId, out var attempt); + + var lease = TryMapLease( + entry, + options.ClaimantConsumer, + now, + _queueOptions.DefaultLeaseDuration, + attemptOverride: attempt); + + if (lease is null) + { + await HandlePoisonEntryAsync(database, entry.Id).ConfigureAwait(false); + continue; + } + + leases.Add(lease); + } + + return leases; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (_connection is not null) + { + await _connection.CloseAsync(); + _connection.Dispose(); + } + + _connectionLock.Dispose(); + _groupInitLock.Dispose(); + GC.SuppressFinalize(this); + } + + internal async Task AcknowledgeAsync( + RedisSchedulerQueueLease lease, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await database.StreamAcknowledgeAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await database.StreamDeleteAsync( + _streamOptions.Stream, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); + } + + internal async Task RenewLeaseAsync( + RedisSchedulerQueueLease lease, + TimeSpan leaseDuration, + CancellationToken cancellationToken) + { + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await database.StreamClaimAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + lease.Consumer, + 0, + new RedisValue[] { lease.MessageId }, + CommandFlags.None) + .ConfigureAwait(false); + + var expires = _timeProvider.GetUtcNow().Add(leaseDuration); + lease.RefreshLease(expires); + } + + internal async Task ReleaseAsync( + RedisSchedulerQueueLease lease, + SchedulerQueueReleaseDisposition disposition, + CancellationToken cancellationToken) + { + if (disposition == SchedulerQueueReleaseDisposition.Retry + && lease.Attempt >= _queueOptions.MaxDeliveryAttempts) + { + await DeadLetterAsync( + lease, + $"max-delivery-attempts:{lease.Attempt}", + cancellationToken).ConfigureAwait(false); + return; + } + + if (!lease.TryBeginCompletion()) + { + return; + } + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await database.StreamAcknowledgeAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await database.StreamDeleteAsync( + _streamOptions.Stream, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordAck(TransportName, _payload.QueueName); + + if (disposition == SchedulerQueueReleaseDisposition.Retry) + { + SchedulerQueueMetrics.RecordRetry(TransportName, _payload.QueueName); + + lease.IncrementAttempt(); + + var backoff = CalculateBackoff(lease.Attempt); + if (backoff > TimeSpan.Zero) + { + try + { + await Task.Delay(backoff, cancellationToken).ConfigureAwait(false); + } + catch (TaskCanceledException) + { + return; + } + } + + var now = _timeProvider.GetUtcNow(); + var entries = BuildEntries(lease.Message, now, lease.Attempt); + + await AddToStreamAsync( + database, + _streamOptions.Stream, + entries, + _streamOptions.ApproximateMaxLength, + _streamOptions.ApproximateMaxLength is not null) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordEnqueued(TransportName, _payload.QueueName); + } + } + + internal async Task DeadLetterAsync( + RedisSchedulerQueueLease lease, + string reason, + CancellationToken cancellationToken) + { + if (!lease.TryBeginCompletion()) + { + return; + } + + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + + await database.StreamAcknowledgeAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + await database.StreamDeleteAsync( + _streamOptions.Stream, + new RedisValue[] { lease.MessageId }) + .ConfigureAwait(false); + + var now = _timeProvider.GetUtcNow(); + var entries = BuildEntries(lease.Message, now, lease.Attempt); + + await AddToStreamAsync( + database, + _streamOptions.DeadLetterStream, + entries, + null, + false) + .ConfigureAwait(false); + + SchedulerQueueMetrics.RecordDeadLetter(TransportName, _payload.QueueName); + _logger.LogError( + "Dead-lettered {Queue} message {MessageId} after {Attempt} attempt(s): {Reason}", + _payload.QueueName, + lease.MessageId, + lease.Attempt, + reason); + } + + internal async ValueTask PingAsync(CancellationToken cancellationToken) + { + var database = await GetDatabaseAsync(cancellationToken).ConfigureAwait(false); + await database.ExecuteAsync("PING").ConfigureAwait(false); + } + + private string BuildIdempotencyKey(string key) + => string.Concat(_streamOptions.IdempotencyKeyPrefix, key); + + private TimeSpan CalculateBackoff(int attempt) + { + if (attempt <= 1) + { + return _queueOptions.RetryInitialBackoff > TimeSpan.Zero + ? _queueOptions.RetryInitialBackoff + : TimeSpan.Zero; + } + + var initial = _queueOptions.RetryInitialBackoff > TimeSpan.Zero + ? _queueOptions.RetryInitialBackoff + : TimeSpan.Zero; + + if (initial <= TimeSpan.Zero) + { + return TimeSpan.Zero; + } + + var max = _queueOptions.RetryMaxBackoff > TimeSpan.Zero + ? _queueOptions.RetryMaxBackoff + : initial; + + var exponent = attempt - 1; + var scaledTicks = initial.Ticks * Math.Pow(2, exponent - 1); + var cappedTicks = Math.Min(max.Ticks, scaledTicks); + + return TimeSpan.FromTicks((long)Math.Max(initial.Ticks, cappedTicks)); + } + + private async ValueTask GetDatabaseAsync(CancellationToken cancellationToken) + { + if (_connection is not null) + { + return _connection.GetDatabase(_redisOptions.Database ?? -1); + } + + await _connectionLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_connection is null) + { + var config = ConfigurationOptions.Parse(_redisOptions.ConnectionString!); + config.AbortOnConnectFail = false; + config.ConnectTimeout = (int)_redisOptions.InitializationTimeout.TotalMilliseconds; + config.ConnectRetry = 3; + + if (_redisOptions.Database is not null) + { + config.DefaultDatabase = _redisOptions.Database; + } + + _connection = await _connectionFactory(config).ConfigureAwait(false); + } + } + finally + { + _connectionLock.Release(); + } + + return _connection.GetDatabase(_redisOptions.Database ?? -1); + } + + private async Task EnsureConsumerGroupAsync( + IDatabase database, + CancellationToken cancellationToken) + { + if (_groupInitialized) + { + return; + } + + await _groupInitLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_groupInitialized) + { + return; + } + + try + { + await database.StreamCreateConsumerGroupAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + StreamPosition.Beginning, + createStream: true) + .ConfigureAwait(false); + } + catch (RedisServerException ex) when (ex.Message.Contains("BUSYGROUP", StringComparison.OrdinalIgnoreCase)) + { + // Group already exists. + } + + _groupInitialized = true; + } + finally + { + _groupInitLock.Release(); + } + } + + private NameValueEntry[] BuildEntries( + TMessage message, + DateTimeOffset enqueuedAt, + int attempt) + { + var attributes = _payload.GetAttributes(message); + var attributeCount = attributes?.Count ?? 0; + var entries = ArrayPool.Shared.Rent(10 + attributeCount); + var index = 0; + + entries[index++] = new NameValueEntry(SchedulerQueueFields.QueueKind, _payload.QueueName); + entries[index++] = new NameValueEntry(SchedulerQueueFields.RunId, _payload.GetRunId(message)); + entries[index++] = new NameValueEntry(SchedulerQueueFields.TenantId, _payload.GetTenantId(message)); + + var scheduleId = _payload.GetScheduleId(message); + if (!string.IsNullOrWhiteSpace(scheduleId)) + { + entries[index++] = new NameValueEntry(SchedulerQueueFields.ScheduleId, scheduleId); + } + + var segmentId = _payload.GetSegmentId(message); + if (!string.IsNullOrWhiteSpace(segmentId)) + { + entries[index++] = new NameValueEntry(SchedulerQueueFields.SegmentId, segmentId); + } + + var correlationId = _payload.GetCorrelationId(message); + if (!string.IsNullOrWhiteSpace(correlationId)) + { + entries[index++] = new NameValueEntry(SchedulerQueueFields.CorrelationId, correlationId); + } + + entries[index++] = new NameValueEntry(SchedulerQueueFields.IdempotencyKey, _payload.GetIdempotencyKey(message)); + entries[index++] = new NameValueEntry(SchedulerQueueFields.Attempt, attempt); + entries[index++] = new NameValueEntry(SchedulerQueueFields.EnqueuedAt, enqueuedAt.ToUnixTimeMilliseconds()); + entries[index++] = new NameValueEntry(SchedulerQueueFields.Payload, _payload.Serialize(message)); + + if (attributeCount > 0 && attributes is not null) + { + foreach (var kvp in attributes) + { + entries[index++] = new NameValueEntry( + SchedulerQueueFields.AttributePrefix + kvp.Key, + kvp.Value); + } + } + + var result = entries.AsSpan(0, index).ToArray(); + ArrayPool.Shared.Return(entries, clearArray: true); + return result; + } + + private RedisSchedulerQueueLease? TryMapLease( + StreamEntry entry, + string consumer, + DateTimeOffset now, + TimeSpan leaseDuration, + int? attemptOverride) + { + if (entry.Values is null || entry.Values.Length == 0) + { + return null; + } + + string? payload = null; + string? runId = null; + string? tenantId = null; + string? scheduleId = null; + string? segmentId = null; + string? correlationId = null; + string? idempotencyKey = null; + long? enqueuedAtUnix = null; + var attempt = attemptOverride ?? 1; + var attributes = new Dictionary(StringComparer.Ordinal); + + foreach (var field in entry.Values) + { + var name = field.Name.ToString(); + var value = field.Value; + + if (name.Equals(SchedulerQueueFields.Payload, StringComparison.Ordinal)) + { + payload = value.ToString(); + } + else if (name.Equals(SchedulerQueueFields.RunId, StringComparison.Ordinal)) + { + runId = value.ToString(); + } + else if (name.Equals(SchedulerQueueFields.TenantId, StringComparison.Ordinal)) + { + tenantId = value.ToString(); + } + else if (name.Equals(SchedulerQueueFields.ScheduleId, StringComparison.Ordinal)) + { + scheduleId = NormalizeOptional(value.ToString()); + } + else if (name.Equals(SchedulerQueueFields.SegmentId, StringComparison.Ordinal)) + { + segmentId = NormalizeOptional(value.ToString()); + } + else if (name.Equals(SchedulerQueueFields.CorrelationId, StringComparison.Ordinal)) + { + correlationId = NormalizeOptional(value.ToString()); + } + else if (name.Equals(SchedulerQueueFields.IdempotencyKey, StringComparison.Ordinal)) + { + idempotencyKey = value.ToString(); + } + else if (name.Equals(SchedulerQueueFields.EnqueuedAt, StringComparison.Ordinal)) + { + if (long.TryParse(value.ToString(), out var unixMs)) + { + enqueuedAtUnix = unixMs; + } + } + else if (name.Equals(SchedulerQueueFields.Attempt, StringComparison.Ordinal)) + { + if (int.TryParse(value.ToString(), out var parsedAttempt)) + { + attempt = attemptOverride.HasValue + ? Math.Max(attemptOverride.Value, parsedAttempt) + : Math.Max(1, parsedAttempt); + } + } + else if (name.StartsWith(SchedulerQueueFields.AttributePrefix, StringComparison.Ordinal)) + { + var key = name[SchedulerQueueFields.AttributePrefix.Length..]; + attributes[key] = value.ToString(); + } + } + + if (payload is null || runId is null || tenantId is null || enqueuedAtUnix is null || idempotencyKey is null) + { + return null; + } + + var message = _payload.Deserialize(payload); + var enqueuedAt = DateTimeOffset.FromUnixTimeMilliseconds(enqueuedAtUnix.Value); + var leaseExpires = now.Add(leaseDuration); + + IReadOnlyDictionary attributeView = attributes.Count == 0 + ? EmptyReadOnlyDictionary.Instance + : new ReadOnlyDictionary(attributes); + + return new RedisSchedulerQueueLease( + this, + entry.Id.ToString(), + idempotencyKey, + runId, + tenantId, + scheduleId, + segmentId, + correlationId, + attributeView, + message, + attempt, + enqueuedAt, + leaseExpires, + consumer); + } + + private async Task HandlePoisonEntryAsync(IDatabase database, RedisValue entryId) + { + await database.StreamAcknowledgeAsync( + _streamOptions.Stream, + _streamOptions.ConsumerGroup, + new RedisValue[] { entryId }) + .ConfigureAwait(false); + + await database.StreamDeleteAsync( + _streamOptions.Stream, + new RedisValue[] { entryId }) + .ConfigureAwait(false); + } + + private async Task AddToStreamAsync( + IDatabase database, + RedisKey stream, + NameValueEntry[] entries, + int? maxLength, + bool useApproximateLength) + { + var capacity = 4 + (entries.Length * 2); + var args = new List(capacity) + { + stream + }; + + if (maxLength.HasValue) + { + args.Add("MAXLEN"); + if (useApproximateLength) + { + args.Add("~"); + } + + args.Add(maxLength.Value); + } + + args.Add("*"); + + for (var i = 0; i < entries.Length; i++) + { + args.Add(entries[i].Name); + args.Add(entries[i].Value); + } + + var result = await database.ExecuteAsync("XADD", args.ToArray()).ConfigureAwait(false); + return (RedisValue)result!; + } + + private static string? NormalizeOptional(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return value; + } + + private sealed class EmptyReadOnlyDictionary + where TKey : notnull + { + public static readonly IReadOnlyDictionary Instance = + new ReadOnlyDictionary(new Dictionary(0, EqualityComparer.Default)); + } +} diff --git a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueLease.cs b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueLease.cs new file mode 100644 index 00000000..7dc07c73 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerQueueLease.cs @@ -0,0 +1,91 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Scheduler.Queue.Redis; + +internal sealed class RedisSchedulerQueueLease : ISchedulerQueueLease +{ + private readonly RedisSchedulerQueueBase _queue; + private int _completed; + + internal RedisSchedulerQueueLease( + RedisSchedulerQueueBase queue, + string messageId, + string idempotencyKey, + string runId, + string tenantId, + string? scheduleId, + string? segmentId, + string? correlationId, + IReadOnlyDictionary attributes, + TMessage message, + int attempt, + DateTimeOffset enqueuedAt, + DateTimeOffset leaseExpiresAt, + string consumer) + { + _queue = queue; + MessageId = messageId; + IdempotencyKey = idempotencyKey; + RunId = runId; + TenantId = tenantId; + ScheduleId = scheduleId; + SegmentId = segmentId; + CorrelationId = correlationId; + Attributes = attributes; + Message = message; + Attempt = attempt; + EnqueuedAt = enqueuedAt; + LeaseExpiresAt = leaseExpiresAt; + Consumer = consumer; + } + + public string MessageId { get; } + + public string IdempotencyKey { get; } + + public string RunId { get; } + + public string TenantId { get; } + + public string? ScheduleId { get; } + + public string? SegmentId { get; } + + public string? CorrelationId { get; } + + public IReadOnlyDictionary Attributes { get; } + + public TMessage Message { get; } + + public int Attempt { get; private set; } + + public DateTimeOffset EnqueuedAt { get; } + + public DateTimeOffset LeaseExpiresAt { get; private set; } + + public string Consumer { get; } + + public Task AcknowledgeAsync(CancellationToken cancellationToken = default) + => _queue.AcknowledgeAsync(this, cancellationToken); + + public Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default) + => _queue.RenewLeaseAsync(this, leaseDuration, cancellationToken); + + public Task ReleaseAsync(SchedulerQueueReleaseDisposition disposition, CancellationToken cancellationToken = default) + => _queue.ReleaseAsync(this, disposition, cancellationToken); + + public Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default) + => _queue.DeadLetterAsync(this, reason, cancellationToken); + + internal bool TryBeginCompletion() + => Interlocked.CompareExchange(ref _completed, 1, 0) == 0; + + internal void RefreshLease(DateTimeOffset expiresAt) + => LeaseExpiresAt = expiresAt; + + internal void IncrementAttempt() + => Attempt++; +} diff --git a/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerRunnerQueue.cs b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerRunnerQueue.cs new file mode 100644 index 00000000..d717ee89 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/Redis/RedisSchedulerRunnerQueue.cs @@ -0,0 +1,90 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StackExchange.Redis; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Queue.Redis; + +internal sealed class RedisSchedulerRunnerQueue + : RedisSchedulerQueueBase, ISchedulerRunnerQueue +{ + public RedisSchedulerRunnerQueue( + SchedulerQueueOptions queueOptions, + SchedulerRedisQueueOptions redisOptions, + ILogger logger, + TimeProvider timeProvider, + Func>? connectionFactory = null) + : base( + queueOptions, + redisOptions, + redisOptions.Runner, + RunnerPayload.Instance, + logger, + timeProvider, + connectionFactory) + { + } + + private sealed class RunnerPayload : IRedisSchedulerQueuePayload + { + public static RunnerPayload Instance { get; } = new(); + + public string QueueName => "runner"; + + public string GetIdempotencyKey(RunnerSegmentQueueMessage message) + => message.IdempotencyKey; + + public string Serialize(RunnerSegmentQueueMessage message) + => CanonicalJsonSerializer.Serialize(message); + + public RunnerSegmentQueueMessage Deserialize(string payload) + => CanonicalJsonSerializer.Deserialize(payload); + + public string GetRunId(RunnerSegmentQueueMessage message) + => message.RunId; + + public string GetTenantId(RunnerSegmentQueueMessage message) + => message.TenantId; + + public string? GetScheduleId(RunnerSegmentQueueMessage message) + => message.ScheduleId; + + public string? GetSegmentId(RunnerSegmentQueueMessage message) + => message.SegmentId; + + public string? GetCorrelationId(RunnerSegmentQueueMessage message) + => message.CorrelationId; + + public IReadOnlyDictionary? GetAttributes(RunnerSegmentQueueMessage message) + { + if (message.Attributes.Count == 0 && message.ImageDigests.Count == 0) + { + return null; + } + + // Ensure digests remain accessible without deserializing the entire payload. + var map = new Dictionary(message.Attributes, StringComparer.Ordinal); + map["imageDigestCount"] = message.ImageDigests.Count.ToString(); + + // populate first few digests for quick inspection (bounded) + var take = Math.Min(message.ImageDigests.Count, 5); + for (var i = 0; i < take; i++) + { + map[$"digest{i}"] = message.ImageDigests[i]; + } + + if (message.RatePerSecond.HasValue) + { + map["ratePerSecond"] = message.RatePerSecond.Value.ToString(); + } + + map["usageOnly"] = message.UsageOnly ? "true" : "false"; + + return map; + } + } +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueContracts.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueContracts.cs new file mode 100644 index 00000000..c2c83f86 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueContracts.cs @@ -0,0 +1,274 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; +using StellaOps.Scheduler.Models; + +namespace StellaOps.Scheduler.Queue; + +public sealed class PlannerQueueMessage +{ + [JsonConstructor] + public PlannerQueueMessage( + Run run, + ImpactSet impactSet, + Schedule? schedule = null, + string? correlationId = null) + { + Run = run ?? throw new ArgumentNullException(nameof(run)); + ImpactSet = impactSet ?? throw new ArgumentNullException(nameof(impactSet)); + + if (schedule is not null && string.IsNullOrWhiteSpace(schedule.Id)) + { + throw new ArgumentException("Schedule must have a valid identifier.", nameof(schedule)); + } + + if (!string.IsNullOrWhiteSpace(correlationId)) + { + correlationId = correlationId!.Trim(); + } + + Schedule = schedule; + CorrelationId = string.IsNullOrWhiteSpace(correlationId) ? null : correlationId; + } + + public Run Run { get; } + + public ImpactSet ImpactSet { get; } + + public Schedule? Schedule { get; } + + public string? CorrelationId { get; } + + public string IdempotencyKey => Run.Id; + + public string TenantId => Run.TenantId; + + public string? ScheduleId => Run.ScheduleId; +} + +public sealed class RunnerSegmentQueueMessage +{ + private readonly ReadOnlyCollection _imageDigests; + private readonly IReadOnlyDictionary _attributes; + + [JsonConstructor] + public RunnerSegmentQueueMessage( + string segmentId, + string runId, + string tenantId, + IReadOnlyList imageDigests, + string? scheduleId = null, + int? ratePerSecond = null, + bool usageOnly = true, + IReadOnlyDictionary? attributes = null, + string? correlationId = null) + { + if (string.IsNullOrWhiteSpace(segmentId)) + { + throw new ArgumentException("Segment identifier must be provided.", nameof(segmentId)); + } + + if (string.IsNullOrWhiteSpace(runId)) + { + throw new ArgumentException("Run identifier must be provided.", nameof(runId)); + } + + if (string.IsNullOrWhiteSpace(tenantId)) + { + throw new ArgumentException("Tenant identifier must be provided.", nameof(tenantId)); + } + + SegmentId = segmentId; + RunId = runId; + TenantId = tenantId; + ScheduleId = string.IsNullOrWhiteSpace(scheduleId) ? null : scheduleId; + RatePerSecond = ratePerSecond; + UsageOnly = usageOnly; + CorrelationId = string.IsNullOrWhiteSpace(correlationId) ? null : correlationId; + + _imageDigests = new ReadOnlyCollection(NormalizeDigests(imageDigests)); + _attributes = attributes is null + ? EmptyReadOnlyDictionary.Instance + : new ReadOnlyDictionary(new Dictionary(attributes, StringComparer.Ordinal)); + } + + public string SegmentId { get; } + + public string RunId { get; } + + public string TenantId { get; } + + public string? ScheduleId { get; } + + public int? RatePerSecond { get; } + + public bool UsageOnly { get; } + + public string? CorrelationId { get; } + + public IReadOnlyList ImageDigests => _imageDigests; + + public IReadOnlyDictionary Attributes => _attributes; + + public string IdempotencyKey => SegmentId; + + private static List NormalizeDigests(IReadOnlyList digests) + { + if (digests is null) + { + throw new ArgumentNullException(nameof(digests)); + } + + var list = new List(); + foreach (var digest in digests) + { + if (string.IsNullOrWhiteSpace(digest)) + { + continue; + } + + list.Add(digest.Trim()); + } + + if (list.Count == 0) + { + throw new ArgumentException("At least one image digest must be provided.", nameof(digests)); + } + + return list; + } + + private sealed class EmptyReadOnlyDictionary + where TKey : notnull + { + public static readonly IReadOnlyDictionary Instance = + new ReadOnlyDictionary(new Dictionary(0, EqualityComparer.Default)); + } +} + +public readonly record struct SchedulerQueueEnqueueResult(string MessageId, bool Deduplicated); + +public sealed class SchedulerQueueLeaseRequest +{ + public SchedulerQueueLeaseRequest(string consumer, int batchSize, TimeSpan leaseDuration) + { + if (string.IsNullOrWhiteSpace(consumer)) + { + throw new ArgumentException("Consumer identifier must be provided.", nameof(consumer)); + } + + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive."); + } + + if (leaseDuration <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(leaseDuration), leaseDuration, "Lease duration must be positive."); + } + + Consumer = consumer; + BatchSize = batchSize; + LeaseDuration = leaseDuration; + } + + public string Consumer { get; } + + public int BatchSize { get; } + + public TimeSpan LeaseDuration { get; } +} + +public sealed class SchedulerQueueClaimOptions +{ + public SchedulerQueueClaimOptions(string claimantConsumer, int batchSize, TimeSpan minIdleTime) + { + if (string.IsNullOrWhiteSpace(claimantConsumer)) + { + throw new ArgumentException("Consumer identifier must be provided.", nameof(claimantConsumer)); + } + + if (batchSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be positive."); + } + + if (minIdleTime < TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(minIdleTime), minIdleTime, "Idle time cannot be negative."); + } + + ClaimantConsumer = claimantConsumer; + BatchSize = batchSize; + MinIdleTime = minIdleTime; + } + + public string ClaimantConsumer { get; } + + public int BatchSize { get; } + + public TimeSpan MinIdleTime { get; } +} + +public enum SchedulerQueueReleaseDisposition +{ + Retry, + Abandon +} + +public interface ISchedulerQueue +{ + ValueTask EnqueueAsync(TMessage message, CancellationToken cancellationToken = default); + + ValueTask>> LeaseAsync(SchedulerQueueLeaseRequest request, CancellationToken cancellationToken = default); + + ValueTask>> ClaimExpiredAsync(SchedulerQueueClaimOptions options, CancellationToken cancellationToken = default); +} + +public interface ISchedulerQueueLease +{ + string MessageId { get; } + + int Attempt { get; } + + DateTimeOffset EnqueuedAt { get; } + + DateTimeOffset LeaseExpiresAt { get; } + + string Consumer { get; } + + string TenantId { get; } + + string RunId { get; } + + string? ScheduleId { get; } + + string? SegmentId { get; } + + string? CorrelationId { get; } + + string IdempotencyKey { get; } + + IReadOnlyDictionary Attributes { get; } + + TMessage Message { get; } + + Task AcknowledgeAsync(CancellationToken cancellationToken = default); + + Task RenewAsync(TimeSpan leaseDuration, CancellationToken cancellationToken = default); + + Task ReleaseAsync(SchedulerQueueReleaseDisposition disposition, CancellationToken cancellationToken = default); + + Task DeadLetterAsync(string reason, CancellationToken cancellationToken = default); +} + +public interface ISchedulerPlannerQueue : ISchedulerQueue +{ +} + +public interface ISchedulerRunnerQueue : ISchedulerQueue +{ +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueFields.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueFields.cs new file mode 100644 index 00000000..0afe58aa --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueFields.cs @@ -0,0 +1,16 @@ +namespace StellaOps.Scheduler.Queue; + +internal static class SchedulerQueueFields +{ + public const string Payload = "payload"; + public const string Attempt = "attempt"; + public const string EnqueuedAt = "enqueuedAt"; + public const string IdempotencyKey = "idempotency"; + public const string RunId = "runId"; + public const string TenantId = "tenantId"; + public const string ScheduleId = "scheduleId"; + public const string SegmentId = "segmentId"; + public const string QueueKind = "queueKind"; + public const string CorrelationId = "correlationId"; + public const string AttributePrefix = "attr:"; +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs new file mode 100644 index 00000000..bcb90371 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueMetrics.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using System.Diagnostics.Metrics; + +namespace StellaOps.Scheduler.Queue; + +internal static class SchedulerQueueMetrics +{ + private const string TransportTagName = "transport"; + private const string QueueTagName = "queue"; + + private static readonly Meter Meter = new("StellaOps.Scheduler.Queue"); + private static readonly Counter EnqueuedCounter = Meter.CreateCounter("scheduler_queue_enqueued_total"); + private static readonly Counter DeduplicatedCounter = Meter.CreateCounter("scheduler_queue_deduplicated_total"); + private static readonly Counter AckCounter = Meter.CreateCounter("scheduler_queue_ack_total"); + private static readonly Counter RetryCounter = Meter.CreateCounter("scheduler_queue_retry_total"); + private static readonly Counter DeadLetterCounter = Meter.CreateCounter("scheduler_queue_deadletter_total"); + + public static void RecordEnqueued(string transport, string queue) + => EnqueuedCounter.Add(1, BuildTags(transport, queue)); + + public static void RecordDeduplicated(string transport, string queue) + => DeduplicatedCounter.Add(1, BuildTags(transport, queue)); + + public static void RecordAck(string transport, string queue) + => AckCounter.Add(1, BuildTags(transport, queue)); + + public static void RecordRetry(string transport, string queue) + => RetryCounter.Add(1, BuildTags(transport, queue)); + + public static void RecordDeadLetter(string transport, string queue) + => DeadLetterCounter.Add(1, BuildTags(transport, queue)); + + private static KeyValuePair[] BuildTags(string transport, string queue) + => new[] + { + new KeyValuePair(TransportTagName, transport), + new KeyValuePair(QueueTagName, queue) + }; +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs new file mode 100644 index 00000000..5e5073f9 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueOptions.cs @@ -0,0 +1,76 @@ +using System; + +namespace StellaOps.Scheduler.Queue; + +public sealed class SchedulerQueueOptions +{ + public SchedulerQueueTransportKind Kind { get; set; } = SchedulerQueueTransportKind.Redis; + + public SchedulerRedisQueueOptions Redis { get; set; } = new(); + + /// + /// Default lease/visibility window applied when callers do not override the duration. + /// + public TimeSpan DefaultLeaseDuration { get; set; } = TimeSpan.FromMinutes(5); + + /// + /// Maximum number of deliveries before a message is shunted to the dead-letter stream. + /// + public int MaxDeliveryAttempts { get; set; } = 5; + + /// + /// Base retry delay used when a message is released for retry. + /// + public TimeSpan RetryInitialBackoff { get; set; } = TimeSpan.FromSeconds(5); + + /// + /// Cap applied to the retry delay when exponential backoff is used. + /// + public TimeSpan RetryMaxBackoff { get; set; } = TimeSpan.FromMinutes(1); +} + +public sealed class SchedulerRedisQueueOptions +{ + public string? ConnectionString { get; set; } + + public int? Database { get; set; } + + public TimeSpan InitializationTimeout { get; set; } = TimeSpan.FromSeconds(30); + + public RedisSchedulerStreamOptions Planner { get; set; } = RedisSchedulerStreamOptions.ForPlanner(); + + public RedisSchedulerStreamOptions Runner { get; set; } = RedisSchedulerStreamOptions.ForRunner(); +} + +public sealed class RedisSchedulerStreamOptions +{ + public string Stream { get; set; } = string.Empty; + + public string ConsumerGroup { get; set; } = string.Empty; + + public string DeadLetterStream { get; set; } = string.Empty; + + public string IdempotencyKeyPrefix { get; set; } = string.Empty; + + public TimeSpan IdempotencyWindow { get; set; } = TimeSpan.FromHours(12); + + public int? ApproximateMaxLength { get; set; } + + public static RedisSchedulerStreamOptions ForPlanner() + => new() + { + Stream = "scheduler:planner", + ConsumerGroup = "scheduler-planners", + DeadLetterStream = "scheduler:planner:dead", + IdempotencyKeyPrefix = "scheduler:planner:idemp:" + }; + + public static RedisSchedulerStreamOptions ForRunner() + => new() + { + Stream = "scheduler:runner", + ConsumerGroup = "scheduler-runners", + DeadLetterStream = "scheduler:runner:dead", + IdempotencyKeyPrefix = "scheduler:runner:idemp:" + }; +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs new file mode 100644 index 00000000..45f78f7f --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueServiceCollectionExtensions.cs @@ -0,0 +1,60 @@ +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Queue.Redis; + +namespace StellaOps.Scheduler.Queue; + +public static class SchedulerQueueServiceCollectionExtensions +{ + public static IServiceCollection AddSchedulerQueues( + this IServiceCollection services, + IConfiguration configuration, + string sectionName = "scheduler:queue") + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + var options = new SchedulerQueueOptions(); + configuration.GetSection(sectionName).Bind(options); + + services.TryAddSingleton(TimeProvider.System); + services.AddSingleton(options); + + services.AddSingleton(sp => + { + var loggerFactory = sp.GetRequiredService(); + var timeProvider = sp.GetService() ?? TimeProvider.System; + + return options.Kind switch + { + SchedulerQueueTransportKind.Redis => new RedisSchedulerPlannerQueue( + options, + options.Redis, + loggerFactory.CreateLogger(), + timeProvider), + _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") + }; + }); + + services.AddSingleton(sp => + { + var loggerFactory = sp.GetRequiredService(); + var timeProvider = sp.GetService() ?? TimeProvider.System; + + return options.Kind switch + { + SchedulerQueueTransportKind.Redis => new RedisSchedulerRunnerQueue( + options, + options.Redis, + loggerFactory.CreateLogger(), + timeProvider), + _ => throw new InvalidOperationException($"Unsupported scheduler queue transport '{options.Kind}'.") + }; + }); + + return services; + } +} diff --git a/src/StellaOps.Scheduler.Queue/SchedulerQueueTransportKind.cs b/src/StellaOps.Scheduler.Queue/SchedulerQueueTransportKind.cs new file mode 100644 index 00000000..758bdf51 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/SchedulerQueueTransportKind.cs @@ -0,0 +1,10 @@ +namespace StellaOps.Scheduler.Queue; + +/// +/// Transport backends supported by the scheduler queue abstraction. +/// +public enum SchedulerQueueTransportKind +{ + Redis = 0, + Nats = 1, +} diff --git a/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj new file mode 100644 index 00000000..fef0b5b8 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/StellaOps.Scheduler.Queue.csproj @@ -0,0 +1,17 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.Queue/TASKS.md b/src/StellaOps.Scheduler.Queue/TASKS.md new file mode 100644 index 00000000..0d121b51 --- /dev/null +++ b/src/StellaOps.Scheduler.Queue/TASKS.md @@ -0,0 +1,9 @@ +# Scheduler Queue Task Board (Sprint 16) + +> **Status note (2025-10-19):** Scheduler DTOs and sample payloads are now available (SCHED-MODELS-16-102). Queue tasks remain pending on this board. + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-QUEUE-16-401 | DOING (2025-10-19) | Scheduler Queue Guild | SCHED-MODELS-16-101 | Implement queue abstraction + Redis Streams adapter (planner inputs, runner segments) with ack/lease semantics. | Integration tests cover enqueue/dequeue/ack; lease renewal implemented; ordering preserved. | +| SCHED-QUEUE-16-402 | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Add NATS JetStream adapter with configuration binding, health probes, failover. | Health endpoints verified; failover documented; adapter tested. | +| SCHED-QUEUE-16-403 | TODO | Scheduler Queue Guild | SCHED-QUEUE-16-401 | Dead-letter handling + metrics (queue depth, retry counts), configuration toggles. | Dead-letter policy tested; metrics exported; docs updated. | diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/GlobalUsings.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/GlobalUsings.cs new file mode 100644 index 00000000..a1599310 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/GlobalUsings.cs @@ -0,0 +1,12 @@ +global using System.Text.Json; +global using System.Text.Json.Nodes; +global using Microsoft.Extensions.Logging.Abstractions; +global using Microsoft.Extensions.Options; +global using Mongo2Go; +global using MongoDB.Bson; +global using MongoDB.Driver; +global using StellaOps.Scheduler.Models; +global using StellaOps.Scheduler.Storage.Mongo.Internal; +global using StellaOps.Scheduler.Storage.Mongo.Migrations; +global using StellaOps.Scheduler.Storage.Mongo.Options; +global using Xunit; diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Integration/SchedulerMongoRoundTripTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Integration/SchedulerMongoRoundTripTests.cs new file mode 100644 index 00000000..0315377e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Integration/SchedulerMongoRoundTripTests.cs @@ -0,0 +1,126 @@ +using System.Text.Json.Nodes; + +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Integration; + +public sealed class SchedulerMongoRoundTripTests : IDisposable +{ + private readonly MongoDbRunner _runner; + private readonly SchedulerMongoContext _context; + + public SchedulerMongoRoundTripTests() + { + _runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet"); + var options = new SchedulerMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = $"scheduler_roundtrip_{Guid.NewGuid():N}" + }; + + _context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger.Instance); + var migrations = new ISchedulerMongoMigration[] + { + new EnsureSchedulerCollectionsMigration(NullLogger.Instance), + new EnsureSchedulerIndexesMigration() + }; + var runner = new SchedulerMongoMigrationRunner(_context, migrations, NullLogger.Instance); + runner.RunAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + + [Fact] + public async Task SamplesRoundTripThroughMongoWithoutLosingCanonicalShape() + { + var samplesRoot = LocateSamplesRoot(); + + var scheduleJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "schedule.json"), CancellationToken.None); + await AssertRoundTripAsync( + scheduleJson, + _context.Options.SchedulesCollection, + CanonicalJsonSerializer.Deserialize, + schedule => schedule.Id); + + var runJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "run.json"), CancellationToken.None); + await AssertRoundTripAsync( + runJson, + _context.Options.RunsCollection, + CanonicalJsonSerializer.Deserialize, + run => run.Id); + + var impactJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "impact-set.json"), CancellationToken.None); + await AssertRoundTripAsync( + impactJson, + _context.Options.ImpactSnapshotsCollection, + CanonicalJsonSerializer.Deserialize, + _ => null); + + var auditJson = await File.ReadAllTextAsync(Path.Combine(samplesRoot, "audit.json"), CancellationToken.None); + await AssertRoundTripAsync( + auditJson, + _context.Options.AuditCollection, + CanonicalJsonSerializer.Deserialize, + audit => audit.Id); + } + + private async Task AssertRoundTripAsync( + string json, + string collectionName, + Func deserialize, + Func resolveId) + { + ArgumentNullException.ThrowIfNull(deserialize); + ArgumentNullException.ThrowIfNull(resolveId); + + var model = deserialize(json); + var canonical = CanonicalJsonSerializer.Serialize(model); + + var document = BsonDocument.Parse(canonical); + var identifier = resolveId(model); + if (!string.IsNullOrEmpty(identifier)) + { + document["_id"] = identifier; + } + + var collection = _context.Database.GetCollection(collectionName); + await collection.InsertOneAsync(document, cancellationToken: CancellationToken.None); + + var filter = identifier is null ? Builders.Filter.Empty : Builders.Filter.Eq("_id", identifier); + var stored = await collection.Find(filter).FirstOrDefaultAsync(); + Assert.NotNull(stored); + + var sanitized = stored!.DeepClone().AsBsonDocument; + sanitized.Remove("_id"); + + var storedJson = sanitized.ToJson(); + + var parsedExpected = JsonNode.Parse(canonical) ?? throw new InvalidOperationException("Canonical node null."); + var parsedActual = JsonNode.Parse(storedJson) ?? throw new InvalidOperationException("Stored node null."); + Assert.True(JsonNode.DeepEquals(parsedExpected, parsedActual), "Document changed shape after Mongo round-trip."); + } + + private static string LocateSamplesRoot() + { + var current = AppContext.BaseDirectory; + while (!string.IsNullOrEmpty(current)) + { + var candidate = Path.Combine(current, "samples", "api", "scheduler"); + if (Directory.Exists(candidate)) + { + return candidate; + } + + var parent = Path.GetDirectoryName(current.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (string.Equals(parent, current, StringComparison.Ordinal)) + { + break; + } + + current = parent; + } + + throw new DirectoryNotFoundException("Unable to locate samples/api/scheduler in repository tree."); + } + + public void Dispose() + { + _runner.Dispose(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/Migrations/SchedulerMongoMigrationTests.cs b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Migrations/SchedulerMongoMigrationTests.cs new file mode 100644 index 00000000..4b5d5a00 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/Migrations/SchedulerMongoMigrationTests.cs @@ -0,0 +1,106 @@ +namespace StellaOps.Scheduler.Storage.Mongo.Tests.Migrations; + +public sealed class SchedulerMongoMigrationTests : IDisposable +{ + private readonly MongoDbRunner _runner; + + public SchedulerMongoMigrationTests() + { + _runner = MongoDbRunner.Start(additionalMongodArguments: "--quiet"); + } + + [Fact] + public async Task RunAsync_CreatesCollectionsAndIndexes() + { + var options = new SchedulerMongoOptions + { + ConnectionString = _runner.ConnectionString, + Database = $"scheduler_tests_{Guid.NewGuid():N}" + }; + + var context = new SchedulerMongoContext(Microsoft.Extensions.Options.Options.Create(options), NullLogger.Instance); + var migrations = new ISchedulerMongoMigration[] + { + new EnsureSchedulerCollectionsMigration(NullLogger.Instance), + new EnsureSchedulerIndexesMigration() + }; + + var runner = new SchedulerMongoMigrationRunner(context, migrations, NullLogger.Instance); + await runner.RunAsync(CancellationToken.None); + + var cursor = await context.Database.ListCollectionNamesAsync(cancellationToken: CancellationToken.None); + var collections = await cursor.ToListAsync(); + + Assert.Contains(options.SchedulesCollection, collections); + Assert.Contains(options.RunsCollection, collections); + Assert.Contains(options.ImpactSnapshotsCollection, collections); + Assert.Contains(options.AuditCollection, collections); + Assert.Contains(options.LocksCollection, collections); + Assert.Contains(options.MigrationsCollection, collections); + + await AssertScheduleIndexesAsync(context, options); + await AssertRunIndexesAsync(context, options); + await AssertImpactSnapshotIndexesAsync(context, options); + await AssertAuditIndexesAsync(context, options); + await AssertLockIndexesAsync(context, options); + } + + private static async Task AssertScheduleIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options) + { + var names = await ListIndexNamesAsync(context.Database.GetCollection(options.SchedulesCollection)); + Assert.Contains("tenant_enabled", names); + Assert.Contains("cron_timezone", names); + } + + private static async Task AssertRunIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options) + { + var collection = context.Database.GetCollection(options.RunsCollection); + var indexes = await ListIndexesAsync(collection); + + Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "tenant_createdAt_desc", StringComparison.Ordinal)); + Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "state_lookup", StringComparison.Ordinal)); + Assert.Contains(indexes, doc => string.Equals(doc["name"].AsString, "schedule_createdAt_desc", StringComparison.Ordinal)); + + var ttl = indexes.FirstOrDefault(doc => doc.TryGetValue("name", out var name) && name == "finishedAt_ttl"); + Assert.NotNull(ttl); + Assert.Equal(options.CompletedRunRetention.TotalSeconds, ttl!["expireAfterSeconds"].ToDouble()); + } + + private static async Task AssertImpactSnapshotIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options) + { + var names = await ListIndexNamesAsync(context.Database.GetCollection(options.ImpactSnapshotsCollection)); + Assert.Contains("selector_tenant_scope", names); + Assert.Contains("snapshotId_unique", names); + } + + private static async Task AssertAuditIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options) + { + var names = await ListIndexNamesAsync(context.Database.GetCollection(options.AuditCollection)); + Assert.Contains("tenant_occurredAt_desc", names); + Assert.Contains("correlation_lookup", names); + } + + private static async Task AssertLockIndexesAsync(SchedulerMongoContext context, SchedulerMongoOptions options) + { + var names = await ListIndexNamesAsync(context.Database.GetCollection(options.LocksCollection)); + Assert.Contains("tenant_resource_unique", names); + Assert.Contains("expiresAt_ttl", names); + } + + private static async Task> ListIndexNamesAsync(IMongoCollection collection) + { + var documents = await ListIndexesAsync(collection); + return documents.Select(doc => doc["name"].AsString).ToArray(); + } + + private static async Task> ListIndexesAsync(IMongoCollection collection) + { + using var cursor = await collection.Indexes.ListAsync(); + return await cursor.ToListAsync(); + } + + public void Dispose() + { + _runner.Dispose(); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj b/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj new file mode 100644 index 00000000..65345131 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo.Tests/StellaOps.Scheduler.Storage.Mongo.Tests.csproj @@ -0,0 +1,23 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + + Always + + + diff --git a/src/StellaOps.Scheduler.Storage.Mongo/AGENTS.md b/src/StellaOps.Scheduler.Storage.Mongo/AGENTS.md new file mode 100644 index 00000000..a07b837e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.Storage.Mongo — Agent Charter + +## Mission +Implement Mongo persistence (schedules, runs, impact cursors, locks, audit) per `docs/ARCHITECTURE_SCHEDULER.md`. diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoContext.cs b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoContext.cs new file mode 100644 index 00000000..c493f99b --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoContext.cs @@ -0,0 +1,46 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Options; + +namespace StellaOps.Scheduler.Storage.Mongo.Internal; + +internal sealed class SchedulerMongoContext +{ + public SchedulerMongoContext(IOptions options, ILogger logger) + { + ArgumentNullException.ThrowIfNull(logger); + var value = options?.Value ?? throw new ArgumentNullException(nameof(options)); + + if (string.IsNullOrWhiteSpace(value.ConnectionString)) + { + throw new InvalidOperationException("Scheduler Mongo connection string is not configured."); + } + + if (string.IsNullOrWhiteSpace(value.Database)) + { + throw new InvalidOperationException("Scheduler Mongo database name is not configured."); + } + + Client = new MongoClient(value.ConnectionString); + var settings = new MongoDatabaseSettings(); + if (value.UseMajorityReadConcern) + { + settings.ReadConcern = ReadConcern.Majority; + } + + if (value.UseMajorityWriteConcern) + { + settings.WriteConcern = WriteConcern.WMajority; + } + + Database = Client.GetDatabase(value.Database, settings); + Options = value; + } + + public MongoClient Client { get; } + + public IMongoDatabase Database { get; } + + public SchedulerMongoOptions Options { get; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializer.cs b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializer.cs new file mode 100644 index 00000000..a310974a --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializer.cs @@ -0,0 +1,32 @@ +using Microsoft.Extensions.Logging; +using StellaOps.Scheduler.Storage.Mongo.Migrations; + +namespace StellaOps.Scheduler.Storage.Mongo.Internal; + +internal interface ISchedulerMongoInitializer +{ + Task EnsureMigrationsAsync(CancellationToken cancellationToken = default); +} + +internal sealed class SchedulerMongoInitializer : ISchedulerMongoInitializer +{ + private readonly SchedulerMongoContext _context; + private readonly SchedulerMongoMigrationRunner _migrationRunner; + private readonly ILogger _logger; + + public SchedulerMongoInitializer( + SchedulerMongoContext context, + SchedulerMongoMigrationRunner migrationRunner, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + _migrationRunner = migrationRunner ?? throw new ArgumentNullException(nameof(migrationRunner)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task EnsureMigrationsAsync(CancellationToken cancellationToken = default) + { + _logger.LogInformation("Ensuring Scheduler Mongo migrations are applied for database {Database}.", _context.Options.Database); + await _migrationRunner.RunAsync(cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializerHostedService.cs b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializerHostedService.cs new file mode 100644 index 00000000..8fe61012 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Internal/SchedulerMongoInitializerHostedService.cs @@ -0,0 +1,27 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace StellaOps.Scheduler.Storage.Mongo.Internal; + +internal sealed class SchedulerMongoInitializerHostedService : IHostedService +{ + private readonly ISchedulerMongoInitializer _initializer; + private readonly ILogger _logger; + + public SchedulerMongoInitializerHostedService( + ISchedulerMongoInitializer initializer, + ILogger logger) + { + _initializer = initializer ?? throw new ArgumentNullException(nameof(initializer)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Applying Scheduler Mongo migrations."); + await _initializer.EnsureMigrationsAsync(cancellationToken).ConfigureAwait(false); + } + + public Task StopAsync(CancellationToken cancellationToken) + => Task.CompletedTask; +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs new file mode 100644 index 00000000..fa74d09f --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerCollectionsMigration.cs @@ -0,0 +1,47 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Migrations; + +internal sealed class EnsureSchedulerCollectionsMigration : ISchedulerMongoMigration +{ + private readonly ILogger _logger; + + public EnsureSchedulerCollectionsMigration(ILogger logger) + => _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + public string Id => "20251019_scheduler_collections_v1"; + + public async ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + var requiredCollections = new[] + { + context.Options.SchedulesCollection, + context.Options.RunsCollection, + context.Options.ImpactSnapshotsCollection, + context.Options.AuditCollection, + context.Options.LocksCollection, + context.Options.MigrationsCollection + }; + + var cursor = await context.Database + .ListCollectionNamesAsync(cancellationToken: cancellationToken) + .ConfigureAwait(false); + + var existing = await cursor.ToListAsync(cancellationToken).ConfigureAwait(false); + + foreach (var collection in requiredCollections) + { + if (existing.Contains(collection, StringComparer.Ordinal)) + { + continue; + } + + _logger.LogInformation("Creating Scheduler Mongo collection '{CollectionName}'.", collection); + await context.Database.CreateCollectionAsync(collection, cancellationToken: cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs new file mode 100644 index 00000000..22698899 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/EnsureSchedulerIndexesMigration.cs @@ -0,0 +1,175 @@ +using System; +using MongoDB.Bson; +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Migrations; + +internal sealed class EnsureSchedulerIndexesMigration : ISchedulerMongoMigration +{ + public string Id => "20251019_scheduler_indexes_v1"; + + public async ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(context); + + await EnsureSchedulesIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureRunsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureImpactSnapshotsIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureAuditIndexesAsync(context, cancellationToken).ConfigureAwait(false); + await EnsureLocksIndexesAsync(context, cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureSchedulesIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.SchedulesCollection); + + var tenantEnabled = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Ascending("enabled"), + new CreateIndexOptions + { + Name = "tenant_enabled" + }); + + var cronTimezone = new CreateIndexModel( + Builders.IndexKeys + .Ascending("cronExpression") + .Ascending("timezone"), + new CreateIndexOptions + { + Name = "cron_timezone" + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantEnabled, cronTimezone }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + private static async Task EnsureRunsIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.RunsCollection); + + var tenantCreated = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Descending("createdAt"), + new CreateIndexOptions + { + Name = "tenant_createdAt_desc" + }); + + var stateIndex = new CreateIndexModel( + Builders.IndexKeys + .Ascending("state"), + new CreateIndexOptions + { + Name = "state_lookup" + }); + + var scheduleIndex = new CreateIndexModel( + Builders.IndexKeys + .Ascending("scheduleId") + .Descending("createdAt"), + new CreateIndexOptions + { + Name = "schedule_createdAt_desc" + }); + + var models = new List> { tenantCreated, stateIndex, scheduleIndex }; + + if (context.Options.CompletedRunRetention > TimeSpan.Zero) + { + var ttlModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("finishedAt"), + new CreateIndexOptions + { + Name = "finishedAt_ttl", + ExpireAfter = context.Options.CompletedRunRetention + }); + + models.Add(ttlModel); + } + + await collection.Indexes.CreateManyAsync(models, cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureImpactSnapshotsIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.ImpactSnapshotsCollection); + + var tenantScope = new CreateIndexModel( + Builders.IndexKeys + .Ascending("selector.tenantId") + .Ascending("selector.scope"), + new CreateIndexOptions + { + Name = "selector_tenant_scope" + }); + + var snapshotId = new CreateIndexModel( + Builders.IndexKeys.Ascending("snapshotId"), + new CreateIndexOptions + { + Name = "snapshotId_unique", + Unique = true, + PartialFilterExpression = Builders.Filter.Exists("snapshotId", true) + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantScope, snapshotId }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + private static async Task EnsureAuditIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.AuditCollection); + + var tenantOccurred = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Descending("occurredAt"), + new CreateIndexOptions + { + Name = "tenant_occurredAt_desc" + }); + + var correlationIndex = new CreateIndexModel( + Builders.IndexKeys + .Ascending("correlationId"), + new CreateIndexOptions + { + Name = "correlation_lookup", + PartialFilterExpression = Builders.Filter.Exists("correlationId", true) + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantOccurred, correlationIndex }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + + private static async Task EnsureLocksIndexesAsync(SchedulerMongoContext context, CancellationToken cancellationToken) + { + var collection = context.Database.GetCollection(context.Options.LocksCollection); + + var tenantResource = new CreateIndexModel( + Builders.IndexKeys + .Ascending("tenantId") + .Ascending("resource"), + new CreateIndexOptions + { + Name = "tenant_resource_unique", + Unique = true, + PartialFilterExpression = Builders.Filter.Exists("resource", true) + }); + + var ttlModel = new CreateIndexModel( + Builders.IndexKeys.Ascending("expiresAt"), + new CreateIndexOptions + { + Name = "expiresAt_ttl", + ExpireAfter = TimeSpan.Zero + }); + + await collection.Indexes.CreateManyAsync(new[] { tenantResource, ttlModel }, cancellationToken: cancellationToken) + .ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/ISchedulerMongoMigration.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/ISchedulerMongoMigration.cs new file mode 100644 index 00000000..5ae1f2ca --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/ISchedulerMongoMigration.cs @@ -0,0 +1,10 @@ +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Migrations; + +internal interface ISchedulerMongoMigration +{ + string Id { get; } + + ValueTask ExecuteAsync(SchedulerMongoContext context, CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRecord.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRecord.cs new file mode 100644 index 00000000..7d16049e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRecord.cs @@ -0,0 +1,16 @@ +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace StellaOps.Scheduler.Storage.Mongo.Migrations; + +internal sealed class SchedulerMongoMigrationRecord +{ + [BsonId] + public ObjectId Id { get; set; } + + [BsonElement("migrationId")] + public string MigrationId { get; set; } = string.Empty; + + [BsonElement("appliedAt")] + public DateTimeOffset AppliedAt { get; set; } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRunner.cs b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRunner.cs new file mode 100644 index 00000000..5260935e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Migrations/SchedulerMongoMigrationRunner.cs @@ -0,0 +1,77 @@ +using Microsoft.Extensions.Logging; +using MongoDB.Driver; +using StellaOps.Scheduler.Storage.Mongo.Internal; + +namespace StellaOps.Scheduler.Storage.Mongo.Migrations; + +internal sealed class SchedulerMongoMigrationRunner +{ + private readonly SchedulerMongoContext _context; + private readonly IReadOnlyList _migrations; + private readonly ILogger _logger; + + public SchedulerMongoMigrationRunner( + SchedulerMongoContext context, + IEnumerable migrations, + ILogger logger) + { + _context = context ?? throw new ArgumentNullException(nameof(context)); + ArgumentNullException.ThrowIfNull(migrations); + _migrations = migrations.OrderBy(migration => migration.Id, StringComparer.Ordinal).ToArray(); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public async ValueTask RunAsync(CancellationToken cancellationToken) + { + if (_migrations.Count == 0) + { + return; + } + + var collection = _context.Database.GetCollection(_context.Options.MigrationsCollection); + await EnsureMigrationIndexAsync(collection, cancellationToken).ConfigureAwait(false); + + var applied = await collection + .Find(FilterDefinition.Empty) + .Project(record => record.MigrationId) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var appliedSet = applied.ToHashSet(StringComparer.Ordinal); + + foreach (var migration in _migrations) + { + if (appliedSet.Contains(migration.Id)) + { + continue; + } + + _logger.LogInformation("Applying Scheduler Mongo migration {MigrationId}.", migration.Id); + await migration.ExecuteAsync(_context, cancellationToken).ConfigureAwait(false); + + var record = new SchedulerMongoMigrationRecord + { + Id = MongoDB.Bson.ObjectId.GenerateNewId(), + MigrationId = migration.Id, + AppliedAt = DateTimeOffset.UtcNow + }; + + await collection.InsertOneAsync(record, cancellationToken: cancellationToken).ConfigureAwait(false); + _logger.LogInformation("Completed Scheduler Mongo migration {MigrationId}.", migration.Id); + } + } + + private static async Task EnsureMigrationIndexAsync( + IMongoCollection collection, + CancellationToken cancellationToken) + { + var keys = Builders.IndexKeys.Ascending(record => record.MigrationId); + var model = new CreateIndexModel(keys, new CreateIndexOptions + { + Name = "migrationId_unique", + Unique = true + }); + + await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken).ConfigureAwait(false); + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs b/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs new file mode 100644 index 00000000..72a600b6 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Options/SchedulerMongoOptions.cs @@ -0,0 +1,34 @@ +using System; + +namespace StellaOps.Scheduler.Storage.Mongo.Options; + +/// +/// Configures MongoDB connectivity and collection names for Scheduler storage. +/// +public sealed class SchedulerMongoOptions +{ + public string ConnectionString { get; set; } = "mongodb://localhost:27017"; + + public string Database { get; set; } = "stellaops_scheduler"; + + public string SchedulesCollection { get; set; } = "schedules"; + + public string RunsCollection { get; set; } = "runs"; + + public string ImpactSnapshotsCollection { get; set; } = "impact_snapshots"; + + public string AuditCollection { get; set; } = "audit"; + + public string LocksCollection { get; set; } = "locks"; + + public string MigrationsCollection { get; set; } = "_scheduler_migrations"; + + /// + /// Optional TTL applied to completed runs. When zero or negative no TTL index is created. + /// + public TimeSpan CompletedRunRetention { get; set; } = TimeSpan.FromDays(180); + + public bool UseMajorityReadConcern { get; set; } = true; + + public bool UseMajorityWriteConcern { get; set; } = true; +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Scheduler.Storage.Mongo/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..eb64fb23 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("StellaOps.Scheduler.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Scheduler.Storage.Mongo/README.md b/src/StellaOps.Scheduler.Storage.Mongo/README.md new file mode 100644 index 00000000..241e4298 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/README.md @@ -0,0 +1,25 @@ +# Scheduler Storage Mongo — Sprint 16 Handoff + +This module now consumes the canonical DTOs defined in `StellaOps.Scheduler.Models`. +Samples covering REST shapes live under `samples/api/scheduler/` and are referenced from `docs/11_DATA_SCHEMAS.md#3.1`. + +## Collections & DTO mapping + +| Collection | DTO | Notes | +|-------------------|--------------------------|---------------------------------------------------------------------------------------| +| `schedules` | `Schedule` | Persist `Schedule` as-is. `_id` → `Schedule.Id`. Use compound indexes on `{tenantId, enabled}` and `{whenCron}` per doc. | +| `runs` | `Run` | Store `Run.Stats` inside the document; omit `deltas` array when empty. | +| `impact_snapshots`| `ImpactSet` | Normalise selector filter fields exactly as emitted by the canonical serializer. | +| `audit` | `AuditRecord` | Lower-case metadata keys are already enforced by the model. | + +All timestamps are persisted as UTC (`+00:00`). Empty selector filters remain empty arrays (see `impact-set.json` sample). + +## Implementation guidance + +1. Add a project reference to `StellaOps.Scheduler.Models` and reuse the records directly; avoid duplicate BSON POCOs. +2. When serialising/deserialising to MongoDB, call `CanonicalJsonSerializer` to keep ordering stable for diffable fixtures. +3. Integration tests should load the JSON samples and round-trip through the Mongo persistence layer to guarantee parity. +4. Follow `docs/11_DATA_SCHEMAS.md` for index requirements; update that doc if storage diverges. +5. Register `AddSchedulerMongoStorage` in the host and call `ISchedulerMongoInitializer.EnsureMigrationsAsync` during bootstrap so collections/indexes are created before workers/web APIs start. + +With these artefacts in place the dependency on SCHED-MODELS-16-101/102 is cleared—storage work can move to DOING. diff --git a/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..859da90e --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/ServiceCollectionExtensions.cs @@ -0,0 +1,26 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Scheduler.Storage.Mongo.Internal; +using StellaOps.Scheduler.Storage.Mongo.Migrations; +using StellaOps.Scheduler.Storage.Mongo.Options; + +namespace StellaOps.Scheduler.Storage.Mongo; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddSchedulerMongoStorage(this IServiceCollection services, IConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(services); + ArgumentNullException.ThrowIfNull(configuration); + + services.Configure(configuration); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + + return services; + } +} diff --git a/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj b/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj new file mode 100644 index 00000000..f3f01fcc --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/StellaOps.Scheduler.Storage.Mongo.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + + + + + + + + + + + + + + diff --git a/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md b/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md new file mode 100644 index 00000000..3ace7727 --- /dev/null +++ b/src/StellaOps.Scheduler.Storage.Mongo/TASKS.md @@ -0,0 +1,9 @@ +# Scheduler Storage Task Board (Sprint 16) + +> **Status note (2025-10-19):** Scheduler models/samples delivered in SCHED-MODELS-16-102. Tasks below remain pending for the Storage guild. + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-STORAGE-16-201 | DONE (2025-10-19) | Scheduler Storage Guild | SCHED-MODELS-16-101 | Create Mongo collections (schedules, runs, impact_cursors, locks, audit) with indexes/migrations per architecture. | Migration scripts and indexes implemented; integration tests cover CRUD paths. | +| SCHED-STORAGE-16-202 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Implement repositories/services with tenant scoping, soft delete, TTL for completed runs, and causal consistency options. | Unit tests pass; TTL/soft delete validated; documentation updated. | +| SCHED-STORAGE-16-203 | TODO | Scheduler Storage Guild | SCHED-STORAGE-16-201 | Audit/logging pipeline + run stats materialized views for UI. | Audit entries persisted; stats queries efficient; docs capture usage. | diff --git a/src/StellaOps.Scheduler.WebService/AGENTS.md b/src/StellaOps.Scheduler.WebService/AGENTS.md new file mode 100644 index 00000000..58a39ff2 --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.WebService — Agent Charter + +## Mission +Implement Scheduler control plane per `docs/ARCHITECTURE_SCHEDULER.md`. diff --git a/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj b/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj new file mode 100644 index 00000000..fef56dcc --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/StellaOps.Scheduler.WebService.csproj @@ -0,0 +1,7 @@ + + + net10.0 + enable + enable + + diff --git a/src/StellaOps.Scheduler.WebService/TASKS.md b/src/StellaOps.Scheduler.WebService/TASKS.md new file mode 100644 index 00000000..7bdb4aad --- /dev/null +++ b/src/StellaOps.Scheduler.WebService/TASKS.md @@ -0,0 +1,12 @@ +# Scheduler WebService Task Board (Sprint 16) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-WEB-16-101 | DOING (2025-10-19) | Scheduler WebService Guild | SCHED-MODELS-16-101 | Bootstrap Minimal API host with Authority OpTok + DPoP, health endpoints, plug-in discovery per architecture §§1–2. | Service boots with config validation; `/healthz`/`/readyz` pass; restart-only plug-ins enforced. | +| SCHED-WEB-16-102 | TODO | Scheduler WebService Guild | SCHED-WEB-16-101 | Implement schedules CRUD (tenant-scoped) with cron validation, pause/resume, audit logging. | CRUD operations tested; invalid cron inputs rejected; audit entries persisted. | +| SCHED-WEB-16-103 | TODO | Scheduler WebService Guild | SCHED-WEB-16-102 | Runs API (list/detail/cancel), ad-hoc run POST, and impact preview endpoints. | Integration tests cover run lifecycle; preview returns counts/sample; cancellation honoured. | +| SCHED-WEB-16-104 | TODO | Scheduler WebService Guild | SCHED-QUEUE-16-401, SCHED-STORAGE-16-201 | Webhook endpoints for Feedser/Vexer exports with mTLS/HMAC validation and rate limiting. | Webhooks validated via tests; invalid signatures rejected; rate limits documented. | + +## Notes +- 2025-10-19: SCHED-MODELS-16-101 (schemas/DTOs) is DONE, so API contracts for schedules/runs are ready to consume. +- Next steps for SCHED-WEB-16-101: create Minimal API host project scaffold, wire Authority OpTok + DPoP authentication via existing DI helpers, expose `/healthz` + `/readyz`, and load restart-only plugins per architecture §§1–2. Capture configuration validation and log shape aligned with Scheduler platform guidance before moving to CRUD implementation. diff --git a/src/StellaOps.Scheduler.Worker/AGENTS.md b/src/StellaOps.Scheduler.Worker/AGENTS.md new file mode 100644 index 00000000..7c9a9c4b --- /dev/null +++ b/src/StellaOps.Scheduler.Worker/AGENTS.md @@ -0,0 +1,4 @@ +# StellaOps.Scheduler.Worker — Agent Charter + +## Mission +Implement Scheduler planners/runners per `docs/ARCHITECTURE_SCHEDULER.md`. diff --git a/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj b/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj new file mode 100644 index 00000000..c444aa16 --- /dev/null +++ b/src/StellaOps.Scheduler.Worker/StellaOps.Scheduler.Worker.csproj @@ -0,0 +1,8 @@ + + + net10.0 + enable + enable + Exe + + diff --git a/src/StellaOps.Scheduler.Worker/TASKS.md b/src/StellaOps.Scheduler.Worker/TASKS.md new file mode 100644 index 00000000..6f8a9c33 --- /dev/null +++ b/src/StellaOps.Scheduler.Worker/TASKS.md @@ -0,0 +1,9 @@ +# Scheduler Worker Task Board (Sprint 16) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SCHED-WORKER-16-201 | TODO | Scheduler Worker Guild | SCHED-QUEUE-16-401 | Planner loop (cron + event triggers) with lease management, fairness, and rate limiting (§6). | Planner integration tests cover cron/event triggers; rate limits enforced; logs include run IDs. | +| SCHED-WORKER-16-202 | TODO | Scheduler Worker Guild | SCHED-IMPACT-16-301 | Wire ImpactIndex targeting (ResolveByPurls/vulns), dedupe, shard planning. | Targeting tests confirm correct image selection; dedupe documented; shards evenly distributed. | +| SCHED-WORKER-16-203 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-202 | Runner execution: call Scanner `/reports` (analysis-only) or `/scans` when configured; collect deltas; handle retries. | Runner tests stub Scanner; retries/backoff validated; deltas aggregated deterministically. | +| SCHED-WORKER-16-204 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-203 | Emit events (`scheduler.rescan.delta`, `scanner.report.ready`) for Notify/UI with summaries. | Events published to queue; payload schema documented; integration tests verify consumption. | +| SCHED-WORKER-16-205 | TODO | Scheduler Worker Guild | SCHED-WORKER-16-201 | Metrics/telemetry: run stats, queue depth, planner latency, delta counts. | Metrics exported per spec; dashboards updated; alerts configured. | diff --git a/src/StellaOps.Signer/AGENTS.md b/src/StellaOps.Signer/AGENTS.md new file mode 100644 index 00000000..0f7b37d7 --- /dev/null +++ b/src/StellaOps.Signer/AGENTS.md @@ -0,0 +1,21 @@ +# Signer Guild + +## Mission +Operate the Stella Ops Signer service: authenticate trusted callers, enforce proof‑of‑entitlement and release integrity policy, and mint verifiable DSSE bundles (keyless or KMS-backed) for downstream attestation. + +## Teams On Call +- Team 11 (Signer API) +- Team 12 (Signer Reliability & Quotas) + +## Operating Principles +- Accept requests only with Authority-issued OpToks plus DPoP or mTLS sender binding; reject unsigned/cross-tenant traffic. +- Treat PoE claims as hard gates for quota, version windows, and license validity; cache results deterministically with bounded TTLs. +- Verify scanner image release signatures via OCI Referrers before signing; fail closed on ambiguity. +- Keep the hot path stateless and deterministic; persist audit trails with structured logging, metrics, and correlation IDs. +- Update `TASKS.md`, architecture notes, and tests whenever behaviour or contracts evolve. + +## Key Directories +- `src/StellaOps.Signer/StellaOps.Signer.WebService/` — Minimal API host and HTTP surface (to be scaffolded). +- `src/StellaOps.Signer/StellaOps.Signer.Core/` — Domain contracts, signing pipeline, quota enforcement (to be scaffolded). +- `src/StellaOps.Signer/StellaOps.Signer.Infrastructure/` — External clients (Authority, Licensing, Fulcio/KMS, OCI) and persistence (to be scaffolded). +- `src/StellaOps.Signer/StellaOps.Signer.Tests/` — Unit/integration test suites (to be scaffolded). diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/SignerAbstractions.cs b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerAbstractions.cs new file mode 100644 index 00000000..b61ebfe4 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerAbstractions.cs @@ -0,0 +1,55 @@ +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Signer.Core; + +public interface IProofOfEntitlementIntrospector +{ + ValueTask IntrospectAsync( + ProofOfEntitlement proof, + CallerContext caller, + CancellationToken cancellationToken); +} + +public interface IReleaseIntegrityVerifier +{ + ValueTask VerifyAsync( + string scannerImageDigest, + CancellationToken cancellationToken); +} + +public interface ISignerQuotaService +{ + ValueTask EnsureWithinLimitsAsync( + SigningRequest request, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken); +} + +public interface IDsseSigner +{ + ValueTask SignAsync( + SigningRequest request, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken); +} + +public interface ISignerAuditSink +{ + ValueTask WriteAsync( + SigningRequest request, + SigningBundle bundle, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken); +} + +public interface ISignerPipeline +{ + ValueTask SignAsync( + SigningRequest request, + CallerContext caller, + CancellationToken cancellationToken); +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/SignerContracts.cs b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerContracts.cs new file mode 100644 index 00000000..83408250 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerContracts.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace StellaOps.Signer.Core; + +public enum SignerPoEFormat +{ + Jwt, + Mtls, +} + +public enum SigningMode +{ + Keyless, + Kms, +} + +public sealed record SigningSubject( + string Name, + IReadOnlyDictionary Digest); + +public sealed record ProofOfEntitlement( + SignerPoEFormat Format, + string Value); + +public sealed record SigningOptions( + SigningMode Mode, + int? ExpirySeconds, + string ReturnBundle); + +public sealed record SigningRequest( + IReadOnlyList Subjects, + string PredicateType, + JsonDocument Predicate, + string ScannerImageDigest, + ProofOfEntitlement ProofOfEntitlement, + SigningOptions Options); + +public sealed record CallerContext( + string Subject, + string Tenant, + IReadOnlyList Scopes, + IReadOnlyList Audiences, + string? SenderBinding, + string? ClientCertificateThumbprint); + +public sealed record ProofOfEntitlementResult( + string LicenseId, + string CustomerId, + string Plan, + int MaxArtifactBytes, + int QpsLimit, + int QpsRemaining, + DateTimeOffset ExpiresAtUtc); + +public sealed record ReleaseVerificationResult( + bool Trusted, + string? ReleaseSigner); + +public sealed record SigningIdentity( + string Mode, + string Issuer, + string Subject, + DateTimeOffset? ExpiresAtUtc); + +public sealed record SigningMetadata( + SigningIdentity Identity, + IReadOnlyList CertificateChain, + string ProviderName, + string AlgorithmId); + +public sealed record SigningBundle( + DsseEnvelope Envelope, + SigningMetadata Metadata); + +public sealed record PolicyCounters( + string Plan, + int MaxArtifactBytes, + int QpsRemaining); + +public sealed record SigningOutcome( + SigningBundle Bundle, + PolicyCounters Policy, + string AuditId); + +public sealed record SignerAuditEntry( + string AuditId, + DateTimeOffset TimestampUtc, + string Subject, + string Tenant, + string Plan, + string ScannerImageDigest, + string SigningMode, + string ProviderName, + IReadOnlyList Subjects); + +public sealed record DsseEnvelope( + string Payload, + string PayloadType, + IReadOnlyList Signatures); + +public sealed record DsseSignature( + string Signature, + string? KeyId); diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/SignerExceptions.cs b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerExceptions.cs new file mode 100644 index 00000000..df6077b3 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerExceptions.cs @@ -0,0 +1,46 @@ +using System; + +namespace StellaOps.Signer.Core; + +public abstract class SignerException : Exception +{ + protected SignerException(string code, string message) + : base(message) + { + Code = code; + } + + public string Code { get; } +} + +public sealed class SignerValidationException : SignerException +{ + public SignerValidationException(string code, string message) + : base(code, message) + { + } +} + +public sealed class SignerAuthorizationException : SignerException +{ + public SignerAuthorizationException(string code, string message) + : base(code, message) + { + } +} + +public sealed class SignerReleaseVerificationException : SignerException +{ + public SignerReleaseVerificationException(string code, string message) + : base(code, message) + { + } +} + +public sealed class SignerQuotaException : SignerException +{ + public SignerQuotaException(string code, string message) + : base(code, message) + { + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/SignerPipeline.cs b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerPipeline.cs new file mode 100644 index 00000000..f8b8152c --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerPipeline.cs @@ -0,0 +1,147 @@ +using System; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace StellaOps.Signer.Core; + +public sealed class SignerPipeline : ISignerPipeline +{ + private const string RequiredScope = "signer.sign"; + private const string RequiredAudience = "signer"; + + private readonly IProofOfEntitlementIntrospector _poe; + private readonly IReleaseIntegrityVerifier _releaseVerifier; + private readonly ISignerQuotaService _quotaService; + private readonly IDsseSigner _signer; + private readonly ISignerAuditSink _auditSink; + private readonly TimeProvider _timeProvider; + + public SignerPipeline( + IProofOfEntitlementIntrospector poe, + IReleaseIntegrityVerifier releaseVerifier, + ISignerQuotaService quotaService, + IDsseSigner signer, + ISignerAuditSink auditSink, + TimeProvider timeProvider) + { + _poe = poe ?? throw new ArgumentNullException(nameof(poe)); + _releaseVerifier = releaseVerifier ?? throw new ArgumentNullException(nameof(releaseVerifier)); + _quotaService = quotaService ?? throw new ArgumentNullException(nameof(quotaService)); + _signer = signer ?? throw new ArgumentNullException(nameof(signer)); + _auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public async ValueTask SignAsync( + SigningRequest request, + CallerContext caller, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(caller); + + ValidateCaller(caller); + ValidateRequest(request); + + var entitlement = await _poe + .IntrospectAsync(request.ProofOfEntitlement, caller, cancellationToken) + .ConfigureAwait(false); + + if (entitlement.ExpiresAtUtc <= _timeProvider.GetUtcNow()) + { + throw new SignerAuthorizationException("entitlement_denied", "Proof of entitlement is expired."); + } + + var releaseResult = await _releaseVerifier + .VerifyAsync(request.ScannerImageDigest, cancellationToken) + .ConfigureAwait(false); + if (!releaseResult.Trusted) + { + throw new SignerReleaseVerificationException("release_untrusted", "Scanner image digest failed release verification."); + } + + await _quotaService + .EnsureWithinLimitsAsync(request, entitlement, caller, cancellationToken) + .ConfigureAwait(false); + + var bundle = await _signer + .SignAsync(request, entitlement, caller, cancellationToken) + .ConfigureAwait(false); + + var auditId = await _auditSink + .WriteAsync(request, bundle, entitlement, caller, cancellationToken) + .ConfigureAwait(false); + + var outcome = new SigningOutcome( + bundle, + new PolicyCounters(entitlement.Plan, entitlement.MaxArtifactBytes, entitlement.QpsRemaining), + auditId); + return outcome; + } + + private static void ValidateCaller(CallerContext caller) + { + if (string.IsNullOrWhiteSpace(caller.Subject)) + { + throw new SignerAuthorizationException("invalid_caller", "Caller subject is required."); + } + + if (string.IsNullOrWhiteSpace(caller.Tenant)) + { + throw new SignerAuthorizationException("invalid_caller", "Caller tenant is required."); + } + + if (!caller.Scopes.Contains(RequiredScope, StringComparer.OrdinalIgnoreCase)) + { + throw new SignerAuthorizationException("insufficient_scope", $"Scope '{RequiredScope}' is required."); + } + + if (!caller.Audiences.Contains(RequiredAudience, StringComparer.OrdinalIgnoreCase)) + { + throw new SignerAuthorizationException("invalid_audience", $"Audience '{RequiredAudience}' is required."); + } + } + + private static void ValidateRequest(SigningRequest request) + { + if (request.Subjects.Count == 0) + { + throw new SignerValidationException("subject_missing", "At least one subject must be provided."); + } + + foreach (var subject in request.Subjects) + { + if (string.IsNullOrWhiteSpace(subject.Name)) + { + throw new SignerValidationException("subject_invalid", "Subject name is required."); + } + + if (subject.Digest is null || subject.Digest.Count == 0) + { + throw new SignerValidationException("subject_digest_invalid", "Subject digest is required."); + } + } + + if (string.IsNullOrWhiteSpace(request.PredicateType)) + { + throw new SignerValidationException("predicate_type_missing", "Predicate type is required."); + } + + if (request.Predicate is null || request.Predicate.RootElement.ValueKind == JsonValueKind.Undefined) + { + throw new SignerValidationException("predicate_missing", "Predicate payload is required."); + } + + if (string.IsNullOrWhiteSpace(request.ScannerImageDigest)) + { + throw new SignerValidationException("scanner_digest_missing", "Scanner image digest is required."); + } + + if (request.ProofOfEntitlement is null) + { + throw new SignerValidationException("poe_missing", "Proof of entitlement is required."); + } + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/SignerStatementBuilder.cs b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerStatementBuilder.cs new file mode 100644 index 00000000..9850a993 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/SignerStatementBuilder.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace StellaOps.Signer.Core; + +public static class SignerStatementBuilder +{ + private const string StatementType = "https://in-toto.io/Statement/v0.1"; + + public static byte[] BuildStatementPayload(SigningRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + var subjects = new List(request.Subjects.Count); + foreach (var subject in request.Subjects) + { + var digest = new SortedDictionary(StringComparer.Ordinal); + foreach (var kvp in subject.Digest) + { + digest[kvp.Key.ToLowerInvariant()] = kvp.Value; + } + + subjects.Add(new + { + name = subject.Name, + digest + }); + } + + var statement = new + { + _type = StatementType, + predicateType = request.PredicateType, + subject = subjects, + predicate = request.Predicate.RootElement.Clone() + }; + + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = null, + WriteIndented = false, + }; + options.Converters.Add(new JsonElementConverter()); + return JsonSerializer.SerializeToUtf8Bytes(statement, options); + } + + private sealed class JsonElementConverter : System.Text.Json.Serialization.JsonConverter + { + public override JsonElement Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using var document = JsonDocument.ParseValue(ref reader); + return document.RootElement.Clone(); + } + + public override void Write(Utf8JsonWriter writer, JsonElement value, JsonSerializerOptions options) + { + value.WriteTo(writer); + } + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Core/StellaOps.Signer.Core.csproj b/src/StellaOps.Signer/StellaOps.Signer.Core/StellaOps.Signer.Core.csproj new file mode 100644 index 00000000..2b6aadf4 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Core/StellaOps.Signer.Core.csproj @@ -0,0 +1,9 @@ + + + net10.0 + preview + enable + enable + true + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Auditing/InMemorySignerAuditSink.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Auditing/InMemorySignerAuditSink.cs new file mode 100644 index 00000000..ea017326 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Auditing/InMemorySignerAuditSink.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Signer.Core; + +namespace StellaOps.Signer.Infrastructure.Auditing; + +public sealed class InMemorySignerAuditSink : ISignerAuditSink +{ + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public InMemorySignerAuditSink(TimeProvider timeProvider, ILogger logger) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ValueTask WriteAsync( + SigningRequest request, + SigningBundle bundle, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(bundle); + ArgumentNullException.ThrowIfNull(entitlement); + ArgumentNullException.ThrowIfNull(caller); + + var auditId = Guid.NewGuid().ToString("d"); + var entry = new SignerAuditEntry( + auditId, + _timeProvider.GetUtcNow(), + caller.Subject, + caller.Tenant, + entitlement.Plan, + request.ScannerImageDigest, + bundle.Metadata.Identity.Mode, + bundle.Metadata.ProviderName, + request.Subjects); + + _entries[auditId] = entry; + _logger.LogInformation("Signer audit event {AuditId} recorded for tenant {Tenant}", auditId, caller.Tenant); + return ValueTask.FromResult(auditId); + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerCryptoOptions.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerCryptoOptions.cs new file mode 100644 index 00000000..5e0bd148 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerCryptoOptions.cs @@ -0,0 +1,16 @@ +using System; + +namespace StellaOps.Signer.Infrastructure.Options; + +public sealed class SignerCryptoOptions +{ + public string KeyId { get; set; } = "signer-kms-default"; + + public string AlgorithmId { get; set; } = "HS256"; + + public string Secret { get; set; } = Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes("stellaops-signer-secret")); + + public string ProviderName { get; set; } = "InMemoryHmacProvider"; + + public string Mode { get; set; } = "kms"; +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerEntitlementOptions.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerEntitlementOptions.cs new file mode 100644 index 00000000..ce5854a1 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerEntitlementOptions.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Signer.Infrastructure.Options; + +public sealed class SignerEntitlementOptions +{ + public IDictionary Tokens { get; } = + new Dictionary(StringComparer.Ordinal); +} + +public sealed record SignerEntitlementDefinition( + string LicenseId, + string CustomerId, + string Plan, + int MaxArtifactBytes, + int QpsLimit, + int QpsRemaining, + DateTimeOffset ExpiresAtUtc); diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerReleaseVerificationOptions.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerReleaseVerificationOptions.cs new file mode 100644 index 00000000..ab1108dd --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Options/SignerReleaseVerificationOptions.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace StellaOps.Signer.Infrastructure.Options; + +public sealed class SignerReleaseVerificationOptions +{ + public ISet TrustedScannerDigests { get; } = new HashSet(StringComparer.OrdinalIgnoreCase); + + public string TrustedSigner { get; set; } = "StellaOps Release"; +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs new file mode 100644 index 00000000..feb52d76 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ProofOfEntitlement/InMemoryProofOfEntitlementIntrospector.cs @@ -0,0 +1,54 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using StellaOps.Signer.Core; +using StellaOps.Signer.Infrastructure.Options; + +namespace StellaOps.Signer.Infrastructure.ProofOfEntitlement; + +public sealed class InMemoryProofOfEntitlementIntrospector : IProofOfEntitlementIntrospector +{ + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + + public InMemoryProofOfEntitlementIntrospector( + IOptionsMonitor options, + TimeProvider timeProvider) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public ValueTask IntrospectAsync( + ProofOfEntitlement proof, + CallerContext caller, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(proof); + ArgumentNullException.ThrowIfNull(caller); + + var token = proof.Value ?? string.Empty; + var snapshot = _options.CurrentValue; + if (!snapshot.Tokens.TryGetValue(token, out var definition)) + { + throw new SignerAuthorizationException("entitlement_denied", "Proof of entitlement is invalid or revoked."); + } + + if (definition.ExpiresAtUtc <= _timeProvider.GetUtcNow()) + { + throw new SignerAuthorizationException("entitlement_denied", "Proof of entitlement has expired."); + } + + var result = new ProofOfEntitlementResult( + definition.LicenseId, + definition.CustomerId, + definition.Plan, + definition.MaxArtifactBytes, + definition.QpsLimit, + definition.QpsRemaining, + definition.ExpiresAtUtc); + + return ValueTask.FromResult(result); + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Quotas/InMemoryQuotaService.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Quotas/InMemoryQuotaService.cs new file mode 100644 index 00000000..8dff89d4 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Quotas/InMemoryQuotaService.cs @@ -0,0 +1,100 @@ +using System; +using System.Collections.Concurrent; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using StellaOps.Signer.Core; + +namespace StellaOps.Signer.Infrastructure.Quotas; + +public sealed class InMemoryQuotaService : ISignerQuotaService +{ + private readonly ConcurrentDictionary _windows = new(StringComparer.Ordinal); + private readonly TimeProvider _timeProvider; + private readonly ILogger _logger; + + public InMemoryQuotaService(TimeProvider timeProvider, ILogger logger) + { + _timeProvider = timeProvider ?? TimeProvider.System; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public ValueTask EnsureWithinLimitsAsync( + SigningRequest request, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(entitlement); + ArgumentNullException.ThrowIfNull(caller); + + var payloadSize = EstimatePayloadSize(request); + if (payloadSize > entitlement.MaxArtifactBytes) + { + throw new SignerQuotaException("artifact_too_large", $"Artifact size {payloadSize} exceeds plan cap ({entitlement.MaxArtifactBytes})."); + } + + if (entitlement.QpsLimit <= 0) + { + return ValueTask.CompletedTask; + } + + var window = _windows.GetOrAdd(caller.Tenant, static _ => new QuotaWindow()); + lock (window) + { + var now = _timeProvider.GetUtcNow(); + if (window.ResetAt <= now) + { + window.Reset(now, entitlement.QpsLimit); + } + + if (window.Remaining <= 0) + { + _logger.LogWarning("Quota exceeded for tenant {Tenant}", caller.Tenant); + throw new SignerQuotaException("plan_throttled", "Plan QPS limit exceeded."); + } + + window.Remaining--; + window.LastUpdated = now; + } + + return ValueTask.CompletedTask; + } + + private static int EstimatePayloadSize(SigningRequest request) + { + var predicateBytes = request.Predicate is null + ? Array.Empty() + : Encoding.UTF8.GetBytes(request.Predicate.RootElement.GetRawText()); + + var subjectBytes = 0; + foreach (var subject in request.Subjects) + { + subjectBytes += subject.Name.Length; + foreach (var digest in subject.Digest) + { + subjectBytes += digest.Key.Length + digest.Value.Length; + } + } + + return predicateBytes.Length + subjectBytes; + } + + private sealed class QuotaWindow + { + public DateTimeOffset ResetAt { get; private set; } = DateTimeOffset.MinValue; + + public int Remaining { get; set; } + + public DateTimeOffset LastUpdated { get; set; } + + public void Reset(DateTimeOffset now, int limit) + { + ResetAt = now.AddSeconds(1); + Remaining = limit; + LastUpdated = now; + } + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ReleaseVerification/DefaultReleaseIntegrityVerifier.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ReleaseVerification/DefaultReleaseIntegrityVerifier.cs new file mode 100644 index 00000000..31dea29b --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ReleaseVerification/DefaultReleaseIntegrityVerifier.cs @@ -0,0 +1,38 @@ +using System; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using StellaOps.Signer.Core; +using StellaOps.Signer.Infrastructure.Options; + +namespace StellaOps.Signer.Infrastructure.ReleaseVerification; + +public sealed class DefaultReleaseIntegrityVerifier : IReleaseIntegrityVerifier +{ + private static readonly Regex DigestPattern = new("^sha256:[a-fA-F0-9]{64}$", RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private readonly IOptionsMonitor _options; + + public DefaultReleaseIntegrityVerifier(IOptionsMonitor options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + } + + public ValueTask VerifyAsync(string scannerImageDigest, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(scannerImageDigest) || !DigestPattern.IsMatch(scannerImageDigest)) + { + throw new SignerReleaseVerificationException("release_digest_invalid", "Scanner image digest must be a valid sha256 string."); + } + + var options = _options.CurrentValue; + if (options.TrustedScannerDigests.Count > 0 && + !options.TrustedScannerDigests.Contains(scannerImageDigest)) + { + return ValueTask.FromResult(new ReleaseVerificationResult(false, null)); + } + + return ValueTask.FromResult(new ReleaseVerificationResult(true, options.TrustedSigner)); + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ServiceCollectionExtensions.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..806691ae --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/ServiceCollectionExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.DependencyInjection; +using StellaOps.Signer.Core; +using StellaOps.Signer.Infrastructure.Auditing; +using StellaOps.Signer.Infrastructure.ProofOfEntitlement; +using StellaOps.Signer.Infrastructure.Quotas; +using StellaOps.Signer.Infrastructure.ReleaseVerification; +using StellaOps.Signer.Infrastructure.Signing; + +namespace StellaOps.Signer.Infrastructure; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddSignerPipeline(this IServiceCollection services) + { + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(TimeProvider.System); + return services; + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Signing/HmacDsseSigner.cs b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Signing/HmacDsseSigner.cs new file mode 100644 index 00000000..b199bd9d --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/Signing/HmacDsseSigner.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Options; +using StellaOps.Signer.Core; +using StellaOps.Signer.Infrastructure.Options; + +namespace StellaOps.Signer.Infrastructure.Signing; + +public sealed class HmacDsseSigner : IDsseSigner +{ + private readonly IOptionsMonitor _options; + private readonly TimeProvider _timeProvider; + + public HmacDsseSigner(IOptionsMonitor options, TimeProvider timeProvider) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + _timeProvider = timeProvider ?? TimeProvider.System; + } + + public ValueTask SignAsync( + SigningRequest request, + ProofOfEntitlementResult entitlement, + CallerContext caller, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(request); + ArgumentNullException.ThrowIfNull(entitlement); + ArgumentNullException.ThrowIfNull(caller); + + var options = _options.CurrentValue; + var payloadBytes = SignerStatementBuilder.BuildStatementPayload(request); + + var secretBytes = Convert.FromBase64String(options.Secret); + using var hmac = new HMACSHA256(secretBytes); + var signatureBytes = hmac.ComputeHash(payloadBytes); + var signature = Convert.ToBase64String(signatureBytes); + var payloadBase64 = Convert.ToBase64String(payloadBytes); + + var envelope = new DsseEnvelope( + payloadBase64, + "application/vnd.in-toto+json", + new[] + { + new DsseSignature(signature, options.KeyId), + }); + + var metadata = new SigningMetadata( + new SigningIdentity( + options.Mode, + caller.Subject, + caller.Subject, + _timeProvider.GetUtcNow().AddMinutes(10)), + Array.Empty(), + options.ProviderName, + options.AlgorithmId); + + var bundle = new SigningBundle(envelope, metadata); + return ValueTask.FromResult(bundle); + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj new file mode 100644 index 00000000..a02d954a --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Infrastructure/StellaOps.Signer.Infrastructure.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj b/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj new file mode 100644 index 00000000..d05901a6 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.Tests/StellaOps.Signer.Tests.csproj @@ -0,0 +1,26 @@ + + + net10.0 + preview + enable + enable + true + false + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs new file mode 100644 index 00000000..0c5435cc --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Contracts/SignDsseContracts.cs @@ -0,0 +1,30 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace StellaOps.Signer.WebService.Contracts; + +public sealed record SignDsseSubjectDto(string Name, Dictionary Digest); + +public sealed record SignDssePoeDto(string Format, string Value); + +public sealed record SignDsseOptionsDto(string? SigningMode, int? ExpirySeconds, string? ReturnBundle); + +public sealed record SignDsseRequestDto( + List Subject, + string PredicateType, + JsonElement Predicate, + string ScannerImageDigest, + SignDssePoeDto Poe, + SignDsseOptionsDto? Options); + +public sealed record SignDsseResponseDto(SignDsseBundleDto Bundle, SignDssePolicyDto Policy, string AuditId); + +public sealed record SignDsseBundleDto(SignDsseEnvelopeDto Dsse, IReadOnlyList CertificateChain, string Mode, SignDsseIdentityDto SigningIdentity); + +public sealed record SignDsseEnvelopeDto(string PayloadType, string Payload, IReadOnlyList Signatures); + +public sealed record SignDsseSignatureDto(string Signature, string? KeyId); + +public sealed record SignDsseIdentityDto(string Issuer, string Subject, string? CertExpiry); + +public sealed record SignDssePolicyDto(string Plan, int MaxArtifactBytes, int QpsRemaining); diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs new file mode 100644 index 00000000..680a049b --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Endpoints/SignerEndpoints.cs @@ -0,0 +1,245 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Claims; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using StellaOps.Auth.Abstractions; +using StellaOps.Signer.Core; +using StellaOps.Signer.WebService.Contracts; + +namespace StellaOps.Signer.WebService.Endpoints; + +public static class SignerEndpoints +{ + public static IEndpointRouteBuilder MapSignerEndpoints(this IEndpointRouteBuilder endpoints) + { + var group = endpoints.MapGroup("/api/v1/signer") + .WithTags("Signer") + .RequireAuthorization(); + + group.MapPost("/sign/dsse", SignDsseAsync); + return endpoints; + } + + private static async Task SignDsseAsync( + HttpContext httpContext, + [FromBody] SignDsseRequestDto requestDto, + ISignerPipeline pipeline, + ILoggerFactory loggerFactory, + CancellationToken cancellationToken) + { + if (requestDto is null) + { + return Results.Problem("Request body is required.", statusCode: StatusCodes.Status400BadRequest); + } + + var logger = loggerFactory.CreateLogger("SignerEndpoints.SignDsse"); + try + { + var caller = BuildCallerContext(httpContext); + ValidateSenderBinding(httpContext, requestDto.Poe, caller); + + using var predicateDocument = JsonDocument.Parse(requestDto.Predicate.GetRawText()); + var signingRequest = new SigningRequest( + ConvertSubjects(requestDto.Subject), + requestDto.PredicateType, + predicateDocument, + requestDto.ScannerImageDigest, + new ProofOfEntitlement( + ParsePoeFormat(requestDto.Poe.Format), + requestDto.Poe.Value), + ConvertOptions(requestDto.Options)); + + var outcome = await pipeline.SignAsync(signingRequest, caller, cancellationToken).ConfigureAwait(false); + var response = ConvertOutcome(outcome); + return Results.Ok(response); + } + catch (SignerValidationException ex) + { + logger.LogWarning(ex, "Validation failure while signing DSSE."); + return Results.Problem(ex.Message, statusCode: StatusCodes.Status400BadRequest, type: ex.Code); + } + catch (SignerAuthorizationException ex) + { + logger.LogWarning(ex, "Authorization failure while signing DSSE."); + return Results.Problem(ex.Message, statusCode: StatusCodes.Status403Forbidden, type: ex.Code); + } + catch (SignerReleaseVerificationException ex) + { + logger.LogWarning(ex, "Release verification failed."); + return Results.Problem(ex.Message, statusCode: StatusCodes.Status403Forbidden, type: ex.Code); + } + catch (SignerQuotaException ex) + { + logger.LogWarning(ex, "Quota enforcement rejected request."); + return Results.Problem(ex.Message, statusCode: StatusCodes.Status429TooManyRequests, type: ex.Code); + } + catch (Exception ex) + { + logger.LogError(ex, "Unexpected error while signing DSSE."); + return Results.Problem("Internal server error.", statusCode: StatusCodes.Status500InternalServerError, type: "signing_unavailable"); + } + } + + private static CallerContext BuildCallerContext(HttpContext context) + { + var user = context.User ?? throw new SignerAuthorizationException("invalid_caller", "Caller is not authenticated."); + + string subject = user.FindFirstValue(StellaOpsClaimTypes.Subject) ?? + throw new SignerAuthorizationException("invalid_caller", "Subject claim is required."); + string tenant = user.FindFirstValue(StellaOpsClaimTypes.Tenant) ?? subject; + + var scopes = new HashSet(StringComparer.OrdinalIgnoreCase); + if (user.HasClaim(c => c.Type == StellaOpsClaimTypes.Scope)) + { + foreach (var value in user.FindAll(StellaOpsClaimTypes.Scope)) + { + foreach (var scope in value.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries)) + { + scopes.Add(scope); + } + } + } + + foreach (var scopeClaim in user.FindAll(StellaOpsClaimTypes.ScopeItem)) + { + scopes.Add(scopeClaim.Value); + } + + var audiences = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var audClaim in user.FindAll(StellaOpsClaimTypes.Audience)) + { + if (audClaim.Value.Contains(' ')) + { + foreach (var aud in audClaim.Value.Split(' ', StringSplitOptions.RemoveEmptyEntries)) + { + audiences.Add(aud); + } + } + else + { + audiences.Add(audClaim.Value); + } + } + + if (audiences.Count == 0) + { + throw new SignerAuthorizationException("invalid_audience", "Audience claim is required."); + } + + var sender = context.Request.Headers.TryGetValue("DPoP", out var dpop) + ? dpop.ToString() + : null; + + var clientCert = context.Connection.ClientCertificate?.Thumbprint; + + return new CallerContext( + subject, + tenant, + scopes.ToArray(), + audiences.ToArray(), + sender, + clientCert); + } + + private static void ValidateSenderBinding(HttpContext context, SignDssePoeDto poe, CallerContext caller) + { + if (poe is null) + { + throw new SignerValidationException("poe_missing", "Proof of entitlement is required."); + } + + var format = ParsePoeFormat(poe.Format); + if (format == SignerPoEFormat.Jwt) + { + if (string.IsNullOrWhiteSpace(caller.SenderBinding)) + { + throw new SignerAuthorizationException("invalid_token", "DPoP proof is required for JWT PoE."); + } + } + else if (format == SignerPoEFormat.Mtls) + { + if (string.IsNullOrWhiteSpace(caller.ClientCertificateThumbprint)) + { + throw new SignerAuthorizationException("invalid_token", "Client certificate is required for mTLS PoE."); + } + } + } + + private static IReadOnlyList ConvertSubjects(List subjects) + { + if (subjects is null || subjects.Count == 0) + { + throw new SignerValidationException("subject_missing", "At least one subject is required."); + } + + return subjects.Select(subject => + { + if (subject.Digest is null || subject.Digest.Count == 0) + { + throw new SignerValidationException("subject_digest_invalid", $"Digest for subject '{subject.Name}' is required."); + } + + return new SigningSubject(subject.Name, subject.Digest); + }).ToArray(); + } + + private static SigningOptions ConvertOptions(SignDsseOptionsDto? optionsDto) + { + if (optionsDto is null) + { + return new SigningOptions(SigningMode.Kms, null, "dsse+cert"); + } + + var mode = optionsDto.SigningMode switch + { + null or "" => SigningMode.Kms, + "kms" or "KMS" => SigningMode.Kms, + "keyless" or "KEYLESS" => SigningMode.Keyless, + _ => throw new SignerValidationException("signing_mode_invalid", $"Unsupported signing mode '{optionsDto.SigningMode}'."), + }; + + return new SigningOptions(mode, optionsDto.ExpirySeconds, optionsDto.ReturnBundle ?? "dsse+cert"); + } + + private static SignerPoEFormat ParsePoeFormat(string? format) + { + return format?.ToLowerInvariant() switch + { + "jwt" => SignerPoEFormat.Jwt, + "mtls" => SignerPoEFormat.Mtls, + _ => throw new SignerValidationException("poe_invalid", $"Unsupported PoE format '{format}'."), + }; + } + + private static SignDsseResponseDto ConvertOutcome(SigningOutcome outcome) + { + var signatures = outcome.Bundle.Envelope.Signatures + .Select(signature => new SignDsseSignatureDto(signature.Signature, signature.KeyId)) + .ToArray(); + + var bundle = new SignDsseBundleDto( + new SignDsseEnvelopeDto( + outcome.Bundle.Envelope.PayloadType, + outcome.Bundle.Envelope.Payload, + signatures), + outcome.Bundle.Metadata.CertificateChain, + outcome.Bundle.Metadata.Identity.Mode, + new SignDsseIdentityDto( + outcome.Bundle.Metadata.Identity.Issuer, + outcome.Bundle.Metadata.Identity.Subject, + outcome.Bundle.Metadata.Identity.ExpiresAtUtc?.ToString("O"))); + + var policy = new SignDssePolicyDto( + outcome.Policy.Plan, + outcome.Policy.MaxArtifactBytes, + outcome.Policy.QpsRemaining); + + return new SignDsseResponseDto(bundle, policy, outcome.AuditId); + } +} diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs b/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs new file mode 100644 index 00000000..47ff6f72 --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/Program.cs @@ -0,0 +1,43 @@ +using Microsoft.AspNetCore.Authentication; +using StellaOps.Signer.Infrastructure; +using StellaOps.Signer.Infrastructure.Options; +using StellaOps.Signer.WebService.Endpoints; +using StellaOps.Signer.WebService.Security; + +var builder = WebApplication.CreateBuilder(args); + +builder.Services.AddLogging(); +builder.Services.AddAuthentication(StubBearerAuthenticationDefaults.AuthenticationScheme) + .AddScheme( + StubBearerAuthenticationDefaults.AuthenticationScheme, + _ => { }); + +builder.Services.AddAuthorization(); + +builder.Services.AddSignerPipeline(); +builder.Services.Configure(options => +{ + options.Tokens["valid-poe"] = new SignerEntitlementDefinition( + LicenseId: "LIC-TEST", + CustomerId: "CUST-TEST", + Plan: "pro", + MaxArtifactBytes: 128 * 1024, + QpsLimit: 5, + QpsRemaining: 5, + ExpiresAtUtc: DateTimeOffset.UtcNow.AddHours(1)); +}); +builder.Services.Configure(options => +{ + options.TrustedScannerDigests.Add("sha256:trusted-scanner-digest"); +}); +builder.Services.Configure(_ => { }); + +var app = builder.Build(); + +app.UseAuthentication(); +app.UseAuthorization(); + +app.MapGet("/", () => Results.Ok("StellaOps Signer service ready.")); +app.MapSignerEndpoints(); + +app.Run(); diff --git a/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj b/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj new file mode 100644 index 00000000..63e1750a --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.WebService/StellaOps.Signer.WebService.csproj @@ -0,0 +1,29 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/StellaOps.Signer/StellaOps.Signer.sln b/src/StellaOps.Signer/StellaOps.Signer.sln new file mode 100644 index 00000000..e1cc711e --- /dev/null +++ b/src/StellaOps.Signer/StellaOps.Signer.sln @@ -0,0 +1,174 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Signer.Core", "StellaOps.Signer.Core\StellaOps.Signer.Core.csproj", "{81EB20CC-54DE-4450-9370-92B489B64F19}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Signer.Infrastructure", "StellaOps.Signer.Infrastructure\StellaOps.Signer.Infrastructure.csproj", "{AD28F5E8-CF69-4587-B3D2-C2B42935993D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Signer.WebService", "StellaOps.Signer.WebService\StellaOps.Signer.WebService.csproj", "{104C429B-2122-43B5-BE2A-5FC846FEBDC4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration", "..\StellaOps.Configuration\StellaOps.Configuration.csproj", "{7A261EB8-60DF-4DD7-83E0-43811B0433B3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins.Abstractions", "..\StellaOps.Authority\StellaOps.Authority.Plugins.Abstractions\StellaOps.Authority.Plugins.Abstractions.csproj", "{B0E46302-AAC2-409C-AA2F-526F8328C696}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography", "..\StellaOps.Cryptography\StellaOps.Cryptography.csproj", "{726F764A-EEE9-4910-8149-42F326E37AF0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.DependencyInjection", "..\StellaOps.DependencyInjection\StellaOps.DependencyInjection.csproj", "{D17E135F-57B9-476A-8ECE-BE081F25E917}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Abstractions", "..\StellaOps.Authority\StellaOps.Auth.Abstractions\StellaOps.Auth.Abstractions.csproj", "{526A921C-E020-4B7E-A195-29CC6FD1C634}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client", "..\StellaOps.Authority\StellaOps.Auth.Client\StellaOps.Auth.Client.csproj", "{BA683E2B-350F-4719-ACF7-1C5C35F5B72F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegration", "..\StellaOps.Authority\StellaOps.Auth.ServerIntegration\StellaOps.Auth.ServerIntegration.csproj", "{EA1037DD-3213-4360-87B8-1129936D89CE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Signer.Tests", "StellaOps.Signer.Tests\StellaOps.Signer.Tests.csproj", "{B09322C0-6827-46D6-91AD-D2380BD36F21}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|Any CPU.Build.0 = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|x64.ActiveCfg = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|x64.Build.0 = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|x86.ActiveCfg = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Debug|x86.Build.0 = Debug|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|Any CPU.ActiveCfg = Release|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|Any CPU.Build.0 = Release|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|x64.ActiveCfg = Release|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|x64.Build.0 = Release|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|x86.ActiveCfg = Release|Any CPU + {81EB20CC-54DE-4450-9370-92B489B64F19}.Release|x86.Build.0 = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|x64.ActiveCfg = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|x64.Build.0 = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|x86.ActiveCfg = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Debug|x86.Build.0 = Debug|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|Any CPU.Build.0 = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|x64.ActiveCfg = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|x64.Build.0 = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|x86.ActiveCfg = Release|Any CPU + {AD28F5E8-CF69-4587-B3D2-C2B42935993D}.Release|x86.Build.0 = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|x64.ActiveCfg = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|x64.Build.0 = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|x86.ActiveCfg = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Debug|x86.Build.0 = Debug|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|Any CPU.Build.0 = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|x64.ActiveCfg = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|x64.Build.0 = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|x86.ActiveCfg = Release|Any CPU + {104C429B-2122-43B5-BE2A-5FC846FEBDC4}.Release|x86.Build.0 = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|x64.ActiveCfg = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|x64.Build.0 = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|x86.ActiveCfg = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Debug|x86.Build.0 = Debug|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|Any CPU.Build.0 = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|x64.ActiveCfg = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|x64.Build.0 = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|x86.ActiveCfg = Release|Any CPU + {7A261EB8-60DF-4DD7-83E0-43811B0433B3}.Release|x86.Build.0 = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|x64.ActiveCfg = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|x64.Build.0 = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|x86.ActiveCfg = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Debug|x86.Build.0 = Debug|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|Any CPU.Build.0 = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|x64.ActiveCfg = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|x64.Build.0 = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|x86.ActiveCfg = Release|Any CPU + {B0E46302-AAC2-409C-AA2F-526F8328C696}.Release|x86.Build.0 = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|x64.ActiveCfg = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|x64.Build.0 = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|x86.ActiveCfg = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Debug|x86.Build.0 = Debug|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|Any CPU.Build.0 = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|x64.ActiveCfg = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|x64.Build.0 = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|x86.ActiveCfg = Release|Any CPU + {726F764A-EEE9-4910-8149-42F326E37AF0}.Release|x86.Build.0 = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|x64.ActiveCfg = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|x64.Build.0 = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|x86.ActiveCfg = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Debug|x86.Build.0 = Debug|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|Any CPU.Build.0 = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|x64.ActiveCfg = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|x64.Build.0 = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|x86.ActiveCfg = Release|Any CPU + {D17E135F-57B9-476A-8ECE-BE081F25E917}.Release|x86.Build.0 = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|Any CPU.Build.0 = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|x64.ActiveCfg = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|x64.Build.0 = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|x86.ActiveCfg = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Debug|x86.Build.0 = Debug|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|Any CPU.ActiveCfg = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|Any CPU.Build.0 = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|x64.ActiveCfg = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|x64.Build.0 = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|x86.ActiveCfg = Release|Any CPU + {526A921C-E020-4B7E-A195-29CC6FD1C634}.Release|x86.Build.0 = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|x64.Build.0 = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Debug|x86.Build.0 = Debug|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|Any CPU.Build.0 = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|x64.ActiveCfg = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|x64.Build.0 = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|x86.ActiveCfg = Release|Any CPU + {BA683E2B-350F-4719-ACF7-1C5C35F5B72F}.Release|x86.Build.0 = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|x64.ActiveCfg = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|x64.Build.0 = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|x86.ActiveCfg = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Debug|x86.Build.0 = Debug|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|Any CPU.Build.0 = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|x64.ActiveCfg = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|x64.Build.0 = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|x86.ActiveCfg = Release|Any CPU + {EA1037DD-3213-4360-87B8-1129936D89CE}.Release|x86.Build.0 = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|x64.ActiveCfg = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|x64.Build.0 = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|x86.ActiveCfg = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Debug|x86.Build.0 = Debug|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|Any CPU.Build.0 = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|x64.ActiveCfg = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|x64.Build.0 = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|x86.ActiveCfg = Release|Any CPU + {B09322C0-6827-46D6-91AD-D2380BD36F21}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/StellaOps.Signer/TASKS.md b/src/StellaOps.Signer/TASKS.md new file mode 100644 index 00000000..cec046cd --- /dev/null +++ b/src/StellaOps.Signer/TASKS.md @@ -0,0 +1,11 @@ +# Signer Guild Task Board (UTC 2025-10-19) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| SIGNER-API-11-101 | DOING (2025-10-19) | Signer Guild | — | `/sign/dsse` pipeline with Authority auth, PoE introspection, release verification, DSSE signing. | ✅ `POST /api/v1/signer/sign/dsse` enforces OpTok audience/scope, DPoP/mTLS binding, PoE introspection, and rejects untrusted scanner digests.
          ✅ Signing pipeline supports keyless (Fulcio) plus optional KMS modes, returning DSSE bundles + cert metadata; deterministic audits persisted.
          ✅ Unit/integration tests cover happy path, invalid PoE, untrusted release, Fulcio/KMS failure, and documentation updated in `docs/ARCHITECTURE_SIGNER.md`/API reference. | +| SIGNER-REF-11-102 | DOING (2025-10-19) | Signer Guild | — | `/verify/referrers` endpoint with OCI lookup, caching, and policy enforcement. | ✅ `GET /api/v1/signer/verify/referrers` hits OCI Referrers API, validates cosign signatures against Stella release keys, and hard-fails on ambiguity.
          ✅ Deterministic cache with policy-aware TTLs and invalidation guards repeated registry load; metrics/logs expose hit/miss/error counters.
          ✅ Tests simulate trusted/untrusted digests, cache expiry, and registry failures; docs capture usage and quota interplay. | +| SIGNER-QUOTA-11-103 | DOING (2025-10-19) | Signer Guild | — | Enforce plan quotas, concurrency/QPS limits, artifact size caps with metrics/audit logs. | ✅ Quota middleware derives plan limits from PoE claims, applies per-tenant concurrency/QPS/size caps, and surfaces remaining capacity in responses.
          ✅ Rate limiter + token bucket state stored in Redis (or equivalent) with deterministic keying and backpressure semantics; overruns emit structured audits.
          ✅ Observability dashboards/counters added; failure modes (throttle, oversize, burst) covered by tests and documented operator runbook. | + +> Remark (2025-10-19): Wave 0 prerequisites reviewed—none outstanding. SIGNER-API-11-101, SIGNER-REF-11-102, and SIGNER-QUOTA-11-103 moved to DOING for kickoff per EXECPLAN.md. + +> Update status columns (TODO / DOING / DONE / BLOCKED) in tandem with code changes and associated tests. diff --git a/src/StellaOps.UI/TASKS.md b/src/StellaOps.UI/TASKS.md new file mode 100644 index 00000000..f332bfec --- /dev/null +++ b/src/StellaOps.UI/TASKS.md @@ -0,0 +1,12 @@ +# UI Task Board (Sprints 11 & 13) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| UI-AUTH-13-001 | TODO | UI Guild | AUTH-DPOP-11-001, AUTH-MTLS-11-002 | Integrate Authority OIDC + DPoP flows with session management. | Login/logout flows pass e2e tests; tokens refreshed; DPoP nonce handling validated. | +| UI-SCANS-13-002 | TODO | UI Guild | SCANNER-WEB-09-102, SIGNER-API-11-101 | Build scans module (list/detail/SBOM/diff/attestation) with performance + accessibility targets. | Cypress tests cover SBOM/diff; performance budgets met; accessibility checks pass. | +| UI-VEX-13-003 | TODO | UI Guild | EXCITITOR-CORE-02-001, EXCITITOR-EXPORT-01-005 | Implement VEX explorer + policy editor with preview integration. | VEX views render consensus/conflicts; staged policy preview works; accessibility checks pass. | +| UI-ADMIN-13-004 | TODO | UI Guild | AUTH-MTLS-11-002 | Deliver admin area (tenants/clients/quotas/licensing) with RBAC + audit hooks. | Admin e2e tests pass; unauthorized access blocked; telemetry wired. | +| UI-ATTEST-11-005 | TODO | UI Guild | SIGNER-API-11-101, ATTESTOR-API-11-201 | Attestation visibility (Rekor id, status) on Scan Detail. | UI shows Rekor UUID/status; mock attestation fixtures displayed; tests cover success/failure. | +| UI-SCHED-13-005 | TODO | UI Guild | SCHED-WEB-16-101 | Scheduler panel: schedules CRUD, run history, dry-run preview using API/mocks. | Panel functional with mocked endpoints; UX signoff; integration tests added. | +| UI-NOTIFY-13-006 | DOING (2025-10-19) | UI Guild | NOTIFY-WEB-15-101 | Notify panel: channels/rules CRUD, deliveries view, test send integration. | Panel interacts with mocked Notify API; tests cover rule lifecycle; docs updated. | +| UI-POLICY-13-007 | TODO | UI Guild | POLICY-CORE-09-006, SCANNER-WEB-09-103 | Surface policy confidence metadata (band, age, quiet provenance) on preview and report views. | UI renders new columns/tooltips, accessibility and responsive checks pass, Cypress regression updated with confidence fixtures. | diff --git a/src/StellaOps.Vexer.Attestation/TASKS.md b/src/StellaOps.Vexer.Attestation/TASKS.md deleted file mode 100644 index 7f318efd..00000000 --- a/src/StellaOps.Vexer.Attestation/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-ATTEST-01-001 – In-toto predicate & DSSE builder|Team Vexer Attestation|VEXER-CORE-01-001|**DONE (2025-10-16)** – Added deterministic in-toto predicate/statement models, DSSE envelope builder wired to signer abstraction, and attestation client producing metadata + diagnostics.| -|VEXER-ATTEST-01-002 – Rekor v2 client integration|Team Vexer Attestation|VEXER-ATTEST-01-001|**DONE (2025-10-16)** – Implemented Rekor HTTP client with retry/backoff, transparency log abstraction, DI helpers, and attestation client integration capturing Rekor metadata + diagnostics.| -|VEXER-ATTEST-01-003 – Verification suite & observability|Team Vexer Attestation|VEXER-ATTEST-01-002|TODO – Add verification helpers for Worker/WebService, metrics/logging hooks, and negative-path regression tests.| diff --git a/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md b/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md deleted file mode 100644 index 696fe975..00000000 --- a/src/StellaOps.Vexer.Connectors.Abstractions/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-ABS-01-001 – Connector context & base classes|Team Vexer Connectors|VEXER-CORE-01-003|**DONE (2025-10-17)** – Added `StellaOps.Vexer.Connectors.Abstractions` project with `VexConnectorBase`, deterministic logging scopes, metadata builder helpers, and connector descriptors; docs updated to highlight the shared abstractions.| -|VEXER-CONN-ABS-01-002 – YAML options & validation|Team Vexer Connectors|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Delivered `VexConnectorOptionsBinder` + binder options/validators, environment-variable expansion, data-annotation checks, and custom validation hooks with documentation updates covering the workflow.| -|VEXER-CONN-ABS-01-003 – Plugin packaging & docs|Team Vexer Connectors|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Authored `docs/dev/30_VEXER_CONNECTOR_GUIDE.md`, added quick-start template under `docs/dev/templates/vexer-connector/`, and updated module docs to reference the packaging workflow.| diff --git a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md deleted file mode 100644 index 64bc0381..00000000 --- a/src/StellaOps.Vexer.Connectors.Cisco.CSAF/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-CISCO-01-001 – Endpoint discovery & auth plumbing|Team Vexer Connectors – Cisco|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `CiscoProviderMetadataLoader` with bearer token support, offline snapshot fallback, DI helpers, and tests covering network/offline discovery to unblock subsequent fetch work.| -|VEXER-CONN-CISCO-01-002 – CSAF pull loop & pagination|Team Vexer Connectors – Cisco|VEXER-CONN-CISCO-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Implemented paginated advisory fetch using provider directories, raw document persistence with dedupe/state tracking, offline resiliency, and unit coverage.| -|VEXER-CONN-CISCO-01-003 – Provider trust metadata|Team Vexer Connectors – Cisco|VEXER-CONN-CISCO-01-002, VEXER-POLICY-01-001|TODO – Emit cosign/PGP trust metadata and advisory provenance hints for policy weighting.| diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs deleted file mode 100644 index 7139af96..00000000 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/Configuration/MsrcConnectorOptions.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System; -using System.IO; -using System.IO.Abstractions; - -namespace StellaOps.Vexer.Connectors.MSRC.CSAF.Configuration; - -public sealed class MsrcConnectorOptions -{ - public const string TokenClientName = "vexer.connector.msrc.token"; - public const string DefaultScope = "https://api.msrc.microsoft.com/.default"; - - /// - /// Azure AD tenant identifier (GUID or domain). - /// - public string TenantId { get; set; } = string.Empty; - - /// - /// Azure AD application (client) identifier. - /// - public string ClientId { get; set; } = string.Empty; - - /// - /// Azure AD application secret for client credential flow. - /// - public string? ClientSecret { get; set; } - /// - /// OAuth scope requested for MSRC API access. - /// - public string Scope { get; set; } = DefaultScope; - - /// - /// When true, token acquisition is skipped and the connector expects offline handling. - /// - public bool PreferOfflineToken { get; set; } - /// - /// Optional path to a pre-provisioned bearer token used when is enabled. - /// - public string? OfflineTokenPath { get; set; } - /// - /// Optional fixed bearer token for constrained environments (e.g., short-lived offline bundles). - /// - public string? StaticAccessToken { get; set; } - /// - /// Minimum buffer (seconds) subtracted from token expiry before refresh. - /// - public int ExpiryLeewaySeconds { get; set; } = 60; - - public void Validate(IFileSystem? fileSystem = null) - { - if (PreferOfflineToken) - { - if (string.IsNullOrWhiteSpace(OfflineTokenPath) && string.IsNullOrWhiteSpace(StaticAccessToken)) - { - throw new InvalidOperationException("OfflineTokenPath or StaticAccessToken must be provided when PreferOfflineToken is enabled."); - } - } - else - { - if (string.IsNullOrWhiteSpace(TenantId)) - { - throw new InvalidOperationException("TenantId is required when not operating in offline token mode."); - } - - if (string.IsNullOrWhiteSpace(ClientId)) - { - throw new InvalidOperationException("ClientId is required when not operating in offline token mode."); - } - - if (string.IsNullOrWhiteSpace(ClientSecret)) - { - throw new InvalidOperationException("ClientSecret is required when not operating in offline token mode."); - } - } - - if (string.IsNullOrWhiteSpace(Scope)) - { - Scope = DefaultScope; - } - - if (ExpiryLeewaySeconds < 10) - { - ExpiryLeewaySeconds = 10; - } - - if (!string.IsNullOrWhiteSpace(OfflineTokenPath)) - { - var fs = fileSystem ?? new FileSystem(); - var directory = Path.GetDirectoryName(OfflineTokenPath); - if (!string.IsNullOrWhiteSpace(directory) && !fs.Directory.Exists(directory)) - { - fs.Directory.CreateDirectory(directory); - } - } - } -} diff --git a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md deleted file mode 100644 index 97b7328a..00000000 --- a/src/StellaOps.Vexer.Connectors.MSRC.CSAF/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-MS-01-001 – AAD onboarding & token cache|Team Vexer Connectors – MSRC|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added MSRC connector project with configurable AAD options, token provider (offline/online modes), DI wiring, and unit tests covering caching and fallback scenarios.| -|VEXER-CONN-MS-01-002 – CSAF download pipeline|Team Vexer Connectors – MSRC|VEXER-CONN-MS-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF packages with retry/backoff, checksum verification, and raw document persistence plus quarantine for schema failures.| -|VEXER-CONN-MS-01-003 – Trust metadata & provenance hints|Team Vexer Connectors – MSRC|VEXER-CONN-MS-01-002, VEXER-POLICY-01-001|TODO – Emit cosign/AAD issuer metadata, attach provenance details, and document policy integration.| diff --git a/src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/TASKS.md b/src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/TASKS.md deleted file mode 100644 index a151c406..00000000 --- a/src/StellaOps.Vexer.Connectors.OCI.OpenVEX.Attest/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-OCI-01-001 – OCI discovery & auth plumbing|Team Vexer Connectors – OCI|VEXER-CONN-ABS-01-001|TODO – Resolve OCI references, configure cosign auth (keyless/keyed), and support offline attestation bundles.| -|VEXER-CONN-OCI-01-002 – Attestation fetch & verify loop|Team Vexer Connectors – OCI|VEXER-CONN-OCI-01-001, VEXER-ATTEST-01-002|TODO – Download DSSE attestations, trigger verification, handle retries/backoff, and persist raw statements with metadata.| -|VEXER-CONN-OCI-01-003 – Provenance metadata & policy hooks|Team Vexer Connectors – OCI|VEXER-CONN-OCI-01-002, VEXER-POLICY-01-001|TODO – Emit provenance hints (image, subject digest, issuer) and trust metadata for policy weighting/logging.| diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs deleted file mode 100644 index ec4bb0e3..00000000 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/OracleCsafConnector.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Runtime.CompilerServices; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Oracle.CSAF.Metadata; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Oracle.CSAF; - -public sealed class OracleCsafConnector : VexConnectorBase -{ - private static readonly VexConnectorDescriptor DescriptorInstance = new( - id: "vexer:oracle", - kind: VexProviderKind.Vendor, - displayName: "Oracle CSAF") - { - Tags = ImmutableArray.Create("oracle", "csaf", "cpu"), - }; - - private readonly OracleCatalogLoader _catalogLoader; - private readonly IEnumerable> _validators; - - private OracleConnectorOptions? _options; - private OracleCatalogResult? _catalog; - - public OracleCsafConnector( - OracleCatalogLoader catalogLoader, - IEnumerable> validators, - ILogger logger, - TimeProvider timeProvider) - : base(DescriptorInstance, logger, timeProvider) - { - _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - LogConnectorEvent(LogLevel.Information, "validate", "Oracle CSAF catalogue loaded.", new Dictionary - { - ["catalogEntryCount"] = _catalog.Metadata.Entries.Length, - ["scheduleCount"] = _catalog.Metadata.CpuSchedule.Length, - ["fromOffline"] = _catalog.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_catalog is null) - { - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - } - - LogConnectorEvent(LogLevel.Debug, "fetch", "Oracle CSAF discovery ready; document ingestion handled by follow-up task.", new Dictionary - { - ["since"] = context.Since?.ToString("O"), - }); - - yield break; - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("OracleCsafConnector relies on dedicated CSAF normalizers."); - - public OracleCatalogResult? GetCachedCatalog() => _catalog; -} diff --git a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md deleted file mode 100644 index 69452a71..00000000 --- a/src/StellaOps.Vexer.Connectors.Oracle.CSAF/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-ORACLE-01-001 – Oracle CSAF catalogue discovery|Team Vexer Connectors – Oracle|VEXER-CONN-ABS-01-001|DOING (2025-10-17) – Implement catalogue discovery, CPU calendar awareness, and offline snapshot import for Oracle CSAF feeds.| -|VEXER-CONN-ORACLE-01-002 – CSAF download & dedupe pipeline|Team Vexer Connectors – Oracle|VEXER-CONN-ORACLE-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF documents with retry/backoff, checksum validation, revision deduplication, and raw persistence.| -|VEXER-CONN-ORACLE-01-003 – Trust metadata + provenance|Team Vexer Connectors – Oracle|VEXER-CONN-ORACLE-01-002, VEXER-POLICY-01-001|TODO – Emit Oracle signing metadata (PGP/cosign) and provenance hints for consensus weighting.| diff --git a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md deleted file mode 100644 index 04f43ae1..00000000 --- a/src/StellaOps.Vexer.Connectors.RedHat.CSAF/TASKS.md +++ /dev/null @@ -1,10 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-RH-01-001 – Provider metadata discovery|Team Vexer Connectors – Red Hat|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added `RedHatProviderMetadataLoader` with HTTP/ETag caching, offline snapshot handling, and validation; exposed DI helper + tests covering live, cached, and offline scenarios.| -|VEXER-CONN-RH-01-002 – Incremental CSAF pulls|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Implemented `RedHatCsafConnector` with ROLIE feed parsing, incremental filtering via `context.Since`, CSAF document download + metadata capture, and persistence through `IVexRawDocumentSink`; tests cover live fetch/cache/offline scenarios with ETag handling.| -|VEXER-CONN-RH-01-003 – Trust metadata emission|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-POLICY-01-001|**DONE (2025-10-17)** – Provider metadata loader now emits trust overrides (weight, cosign issuer/pattern, PGP fingerprints) and the connector surfaces provenance hints for policy/consensus layers.| -|VEXER-CONN-RH-01-004 – Resume state persistence|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Connector now loads/saves resume state via `IVexConnectorStateRepository`, tracking last update timestamp and recent document digests to avoid duplicate CSAF ingestion; regression covers state persistence and duplicate skips.| -|VEXER-CONN-RH-01-005 – Worker/WebService integration|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002|**DONE (2025-10-17)** – Worker/WebService now call `AddRedHatCsafConnector`, register the connector + state repo, and default worker scheduling adds the `vexer:redhat` provider so background jobs and orchestration can activate the connector without extra wiring.| -|VEXER-CONN-RH-01-006 – CSAF normalization parity tests|Team Vexer Connectors – Red Hat|VEXER-CONN-RH-01-002, VEXER-FMT-CSAF-01-001|**DONE (2025-10-17)** – Added RHSA fixture-driven regression verifying CSAF normalizer retains Red Hat product metadata, tracking fields, and timestamps (`rhsa-sample.json` + `CsafNormalizerTests.NormalizeAsync_PreservesRedHatSpecificMetadata`).| diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs deleted file mode 100644 index 4b638586..00000000 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/RancherHubConnector.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Runtime.CompilerServices; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Configuration; -using StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Metadata; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.SUSE.RancherVEXHub; - -public sealed class RancherHubConnector : VexConnectorBase -{ - private static readonly VexConnectorDescriptor StaticDescriptor = new( - id: "vexer:suse.rancher", - kind: VexProviderKind.Hub, - displayName: "SUSE Rancher VEX Hub") - { - Tags = ImmutableArray.Create("hub", "suse", "offline"), - }; - - private readonly RancherHubMetadataLoader _metadataLoader; - private readonly IEnumerable> _validators; - - private RancherHubConnectorOptions? _options; - private RancherHubMetadataResult? _metadata; - - public RancherHubConnector( - RancherHubMetadataLoader metadataLoader, - ILogger logger, - TimeProvider timeProvider, - IEnumerable>? validators = null) - : base(StaticDescriptor, logger, timeProvider) - { - _metadataLoader = metadataLoader ?? throw new ArgumentNullException(nameof(metadataLoader)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - - LogConnectorEvent(LogLevel.Information, "validate", "Rancher hub discovery loaded.", new Dictionary - { - ["discoveryUri"] = _options.DiscoveryUri.ToString(), - ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), - ["requiresAuth"] = _metadata.Metadata.Subscription.RequiresAuthentication, - ["fromOffline"] = _metadata.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_metadata is null) - { - _metadata = await _metadataLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - } - - LogConnectorEvent(LogLevel.Debug, "fetch", "Rancher hub connector discovery ready; event ingestion will be implemented in VEXER-CONN-SUSE-01-002.", new Dictionary - { - ["since"] = context.Since?.ToString("O"), - ["subscriptionUri"] = _metadata.Metadata.Subscription.EventsUri.ToString(), - }); - - yield break; - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("RancherHubConnector relies on format-specific normalizers for CSAF/OpenVEX payloads."); - - public RancherHubMetadata? GetCachedMetadata() => _metadata?.Metadata; -} diff --git a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md b/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md deleted file mode 100644 index 9ce451c6..00000000 --- a/src/StellaOps.Vexer.Connectors.SUSE.RancherVEXHub/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-SUSE-01-001 – Rancher hub discovery & auth|Team Vexer Connectors – SUSE|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Rancher hub options/token provider, discovery metadata loader with offline snapshots + caching, connector shell, DI wiring, and unit tests covering network/offline paths.| -|VEXER-CONN-SUSE-01-002 – Checkpointed event ingestion|Team Vexer Connectors – SUSE|VEXER-CONN-SUSE-01-001, VEXER-STORAGE-01-003|TODO – Process hub events with resume checkpoints, deduplication, and quarantine path for malformed payloads.| -|VEXER-CONN-SUSE-01-003 – Trust metadata & policy hints|Team Vexer Connectors – SUSE|VEXER-CONN-SUSE-01-002, VEXER-POLICY-01-001|TODO – Emit provider trust configuration (signers, weight overrides) and attach provenance hints for consensus engine.| diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj deleted file mode 100644 index a99a942f..00000000 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md deleted file mode 100644 index 049b0920..00000000 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CONN-UBUNTU-01-001 – Ubuntu CSAF discovery & channels|Team Vexer Connectors – Ubuntu|VEXER-CONN-ABS-01-001|**DONE (2025-10-17)** – Added Ubuntu connector project with configurable channel options, catalog loader (network/offline), DI wiring, and discovery unit tests.| -|VEXER-CONN-UBUNTU-01-002 – Incremental fetch & deduplication|Team Vexer Connectors – Ubuntu|VEXER-CONN-UBUNTU-01-001, VEXER-STORAGE-01-003|TODO – Fetch CSAF bundles with ETag handling, checksum validation, deduplication, and raw persistence.| -|VEXER-CONN-UBUNTU-01-003 – Trust metadata & provenance|Team Vexer Connectors – Ubuntu|VEXER-CONN-UBUNTU-01-002, VEXER-POLICY-01-001|TODO – Emit Ubuntu signing metadata (GPG fingerprints) plus provenance hints for policy weighting and diagnostics.| diff --git a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs b/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs deleted file mode 100644 index c47702f8..00000000 --- a/src/StellaOps.Vexer.Connectors.Ubuntu.CSAF/UbuntuCsafConnector.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Runtime.CompilerServices; -using Microsoft.Extensions.Logging; -using StellaOps.Vexer.Connectors.Abstractions; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Configuration; -using StellaOps.Vexer.Connectors.Ubuntu.CSAF.Metadata; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Connectors.Ubuntu.CSAF; - -public sealed class UbuntuCsafConnector : VexConnectorBase -{ - private static readonly VexConnectorDescriptor DescriptorInstance = new( - id: "vexer:ubuntu", - kind: VexProviderKind.Distro, - displayName: "Ubuntu CSAF") - { - Tags = ImmutableArray.Create("ubuntu", "csaf", "usn"), - }; - - private readonly UbuntuCatalogLoader _catalogLoader; - private readonly IEnumerable> _validators; - - private UbuntuConnectorOptions? _options; - private UbuntuCatalogResult? _catalog; - - public UbuntuCsafConnector( - UbuntuCatalogLoader catalogLoader, - IEnumerable> validators, - ILogger logger, - TimeProvider timeProvider) - : base(DescriptorInstance, logger, timeProvider) - { - _catalogLoader = catalogLoader ?? throw new ArgumentNullException(nameof(catalogLoader)); - _validators = validators ?? Array.Empty>(); - } - - public override async ValueTask ValidateAsync(VexConnectorSettings settings, CancellationToken cancellationToken) - { - _options = VexConnectorOptionsBinder.Bind( - Descriptor, - settings, - validators: _validators); - - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - LogConnectorEvent(LogLevel.Information, "validate", "Ubuntu CSAF index loaded.", new Dictionary - { - ["channelCount"] = _catalog.Metadata.Channels.Length, - ["fromOffline"] = _catalog.FromOfflineSnapshot, - }); - } - - public override async IAsyncEnumerable FetchAsync(VexConnectorContext context, [EnumeratorCancellation] CancellationToken cancellationToken) - { - ArgumentNullException.ThrowIfNull(context); - - if (_options is null) - { - throw new InvalidOperationException("Connector must be validated before fetch operations."); - } - - if (_catalog is null) - { - _catalog = await _catalogLoader.LoadAsync(_options, cancellationToken).ConfigureAwait(false); - } - - LogConnectorEvent(LogLevel.Debug, "fetch", "Ubuntu CSAF discovery ready; channel catalogs handled in subsequent task.", new Dictionary - { - ["since"] = context.Since?.ToString("O"), - }); - - yield break; - } - - public override ValueTask NormalizeAsync(VexRawDocument document, CancellationToken cancellationToken) - => throw new NotSupportedException("UbuntuCsafConnector relies on CSAF normalizers for document processing."); - - public UbuntuCatalogResult? GetCachedCatalog() => _catalog; -} diff --git a/src/StellaOps.Vexer.Core/TASKS.md b/src/StellaOps.Vexer.Core/TASKS.md deleted file mode 100644 index 4ed131d7..00000000 --- a/src/StellaOps.Vexer.Core/TASKS.md +++ /dev/null @@ -1,9 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-CORE-01-001 – Canonical VEX domain records|Team Vexer Core & Policy|docs/ARCHITECTURE_VEXER.md|DONE (2025-10-15) – Introduced `VexClaim`, `VexConsensus`, provider metadata, export manifest records, and deterministic JSON serialization with tests covering canonical ordering and query signatures.| -|VEXER-CORE-01-002 – Trust-weighted consensus resolver|Team Vexer Core & Policy|VEXER-CORE-01-001|DONE (2025-10-15) – Added consensus resolver, baseline policy (tier weights + justification gate), telemetry output, and tests covering acceptance, conflict ties, and determinism.| -|VEXER-CORE-01-003 – Shared contracts & query signatures|Team Vexer Core & Policy|VEXER-CORE-01-001|DONE (2025-10-15) – Published connector/normalizer/exporter/attestation abstractions and expanded deterministic `VexQuerySignature`/hash utilities with test coverage.| -|VEXER-CORE-02-001 – Context signal schema prep|Team Vexer Core & Policy|VEXER-POLICY-02-001|TODO – Extend `VexClaim`/`VexConsensus` with optional severity/KEV/EPSS payloads, update canonical serializer/hashes, and coordinate migration notes with Storage.| -|VEXER-CORE-02-002 – Deterministic risk scoring engine|Team Vexer Core & Policy|VEXER-CORE-02-001, VEXER-POLICY-02-001|BACKLOG – Introduce the scoring calculator invoked by consensus, persist score envelopes with audit trails, and add regression fixtures covering gate/boost behaviour before enabling exports.| diff --git a/src/StellaOps.Vexer.Core/VexConsensusPolicyOptions.cs b/src/StellaOps.Vexer.Core/VexConsensusPolicyOptions.cs deleted file mode 100644 index 385b9603..00000000 --- a/src/StellaOps.Vexer.Core/VexConsensusPolicyOptions.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Collections.Immutable; - -namespace StellaOps.Vexer.Core; - -public sealed record VexConsensusPolicyOptions -{ - public const string BaselineVersion = "baseline/v1"; - - public VexConsensusPolicyOptions( - string? version = null, - double vendorWeight = 1.0, - double distroWeight = 0.9, - double platformWeight = 0.7, - double hubWeight = 0.5, - double attestationWeight = 0.6, - IEnumerable>? providerOverrides = null) - { - Version = string.IsNullOrWhiteSpace(version) ? BaselineVersion : version.Trim(); - VendorWeight = NormalizeWeight(vendorWeight); - DistroWeight = NormalizeWeight(distroWeight); - PlatformWeight = NormalizeWeight(platformWeight); - HubWeight = NormalizeWeight(hubWeight); - AttestationWeight = NormalizeWeight(attestationWeight); - ProviderOverrides = NormalizeOverrides(providerOverrides); - } - - public string Version { get; } - - public double VendorWeight { get; } - - public double DistroWeight { get; } - - public double PlatformWeight { get; } - - public double HubWeight { get; } - - public double AttestationWeight { get; } - - public ImmutableDictionary ProviderOverrides { get; } - - private static double NormalizeWeight(double weight) - { - if (double.IsNaN(weight) || double.IsInfinity(weight)) - { - throw new ArgumentOutOfRangeException(nameof(weight), "Weight must be a finite number."); - } - - if (weight <= 0) - { - return 0; - } - - if (weight >= 1) - { - return 1; - } - - return weight; - } - - private static ImmutableDictionary NormalizeOverrides( - IEnumerable>? overrides) - { - if (overrides is null) - { - return ImmutableDictionary.Empty; - } - - var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - foreach (var (key, weight) in overrides) - { - if (string.IsNullOrWhiteSpace(key)) - { - continue; - } - - builder[key.Trim()] = NormalizeWeight(weight); - } - - return builder.ToImmutable(); - } -} diff --git a/src/StellaOps.Vexer.Export/Properties/AssemblyInfo.cs b/src/StellaOps.Vexer.Export/Properties/AssemblyInfo.cs deleted file mode 100644 index d7a3cbff..00000000 --- a/src/StellaOps.Vexer.Export/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Vexer.Export.Tests")] diff --git a/src/StellaOps.Vexer.Export/TASKS.md b/src/StellaOps.Vexer.Export/TASKS.md deleted file mode 100644 index 0e737d33..00000000 --- a/src/StellaOps.Vexer.Export/TASKS.md +++ /dev/null @@ -1,9 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-EXPORT-01-001 – Export engine orchestration|Team Vexer Export|VEXER-CORE-01-003|DONE (2025-10-15) – Export engine scaffolding with cache lookup, data source hooks, and deterministic manifest emission.| -|VEXER-EXPORT-01-002 – Cache index & eviction hooks|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-STORAGE-01-003|**DONE (2025-10-16)** – Export engine now invalidates cache entries on force refresh, cache services expose prune/invalidate APIs, and storage maintenance trims expired/dangling records with Mongo2Go coverage.| -|VEXER-EXPORT-01-003 – Artifact store adapters|Team Vexer Export|VEXER-EXPORT-01-001|**DONE (2025-10-16)** – Implemented multi-store pipeline with filesystem, S3-compatible, and offline bundle adapters (hash verification + manifest/zip output) plus unit coverage and DI hooks.| -|VEXER-EXPORT-01-004 – Attestation handoff integration|Team Vexer Export|VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|**DONE (2025-10-17)** – Export engine now invokes attestation client, logs diagnostics, and persists Rekor/envelope metadata on manifests; regression coverage added in `ExportEngineTests.ExportAsync_AttachesAttestationMetadata`.| -|VEXER-EXPORT-01-005 – Score & resolve envelope surfaces|Team Vexer Export|VEXER-EXPORT-01-004, VEXER-CORE-02-001|TODO – Emit consensus+score envelopes in export manifests, include policy/scoring digests, and update offline bundle/ORAS layouts to carry signed VEX responses.| diff --git a/src/StellaOps.Vexer.Formats.CSAF/TASKS.md b/src/StellaOps.Vexer.Formats.CSAF/TASKS.md deleted file mode 100644 index 7b9f41dd..00000000 --- a/src/StellaOps.Vexer.Formats.CSAF/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-FMT-CSAF-01-001 – CSAF normalizer foundation|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – Implemented CSAF normalizer + DI hook, parsing tracking metadata, product tree branches/full names, and mapping product statuses into canonical `VexClaim`s with baseline precedence. Regression added in `CsafNormalizerTests`.| -|VEXER-FMT-CSAF-01-002 – Status/justification mapping|Team Vexer Formats|VEXER-FMT-CSAF-01-001, VEXER-POLICY-01-001|TODO – Normalize CSAF `product_status` + `justification` values into policy-aware enums with audit diagnostics for unsupported codes.| -|VEXER-FMT-CSAF-01-003 – CSAF export adapter|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-CSAF-01-001|TODO – Provide CSAF export writer producing deterministic documents (per vuln/product) and manifest metadata for attestation.| diff --git a/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md b/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md deleted file mode 100644 index cf34f392..00000000 --- a/src/StellaOps.Vexer.Formats.CycloneDX/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-FMT-CYCLONE-01-001 – CycloneDX VEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – CycloneDX normalizer parses `analysis` data, resolves component references, and emits canonical `VexClaim`s; regression lives in `CycloneDxNormalizerTests`.| -|VEXER-FMT-CYCLONE-01-002 – Component reference reconciliation|Team Vexer Formats|VEXER-FMT-CYCLONE-01-001|TODO – Implement helpers to reconcile component/service references against policy expectations and emit diagnostics for missing SBOM links.| -|VEXER-FMT-CYCLONE-01-003 – CycloneDX export serializer|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-CYCLONE-01-001|TODO – Provide exporters producing CycloneDX VEX output with canonical ordering and hash-stable manifests.| diff --git a/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md b/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md deleted file mode 100644 index 75210974..00000000 --- a/src/StellaOps.Vexer.Formats.OpenVEX/TASKS.md +++ /dev/null @@ -1,7 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-FMT-OPENVEX-01-001 – OpenVEX normalizer|Team Vexer Formats|VEXER-CORE-01-001|**DONE (2025-10-17)** – OpenVEX normalizer parses statements/products, maps status/justification, and surfaces provenance metadata; coverage in `OpenVexNormalizerTests`.| -|VEXER-FMT-OPENVEX-01-002 – Statement merge utilities|Team Vexer Formats|VEXER-FMT-OPENVEX-01-001|TODO – Add reducers merging multiple OpenVEX statements, resolving conflicts deterministically, and emitting policy diagnostics.| -|VEXER-FMT-OPENVEX-01-003 – OpenVEX export writer|Team Vexer Formats|VEXER-EXPORT-01-001, VEXER-FMT-OPENVEX-01-001|TODO – Provide export serializer generating canonical OpenVEX documents with optional SBOM references and hash-stable ordering.| diff --git a/src/StellaOps.Vexer.Policy/TASKS.md b/src/StellaOps.Vexer.Policy/TASKS.md deleted file mode 100644 index 6fd59797..00000000 --- a/src/StellaOps.Vexer.Policy/TASKS.md +++ /dev/null @@ -1,11 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-POLICY-01-001 – Policy schema & binding|Team Vexer Policy|VEXER-CORE-01-001|DONE (2025-10-15) – Established `VexPolicyOptions`, options binding, and snapshot provider covering baseline weights/overrides.| -|VEXER-POLICY-01-002 – Policy evaluator service|Team Vexer Policy|VEXER-POLICY-01-001|DONE (2025-10-15) – `VexPolicyEvaluator` exposes immutable snapshots to consensus and normalizes rejection reasons.| -|VEXER-POLICY-01-003 – Operator diagnostics & docs|Team Vexer Policy|VEXER-POLICY-01-001|**DONE (2025-10-16)** – Surface structured diagnostics (CLI/WebService) and author policy upgrade guidance in docs/ARCHITECTURE_VEXER.md appendix.
          2025-10-16: Added `IVexPolicyDiagnostics`/`VexPolicyDiagnosticsReport`, sorted issue ordering, recommendations, and appendix guidance. Tests: `dotnet test src/StellaOps.Vexer.Core.Tests/StellaOps.Vexer.Core.Tests.csproj`.| -|VEXER-POLICY-01-004 – Policy schema validation & YAML binding|Team Vexer Policy|VEXER-POLICY-01-001|**DONE (2025-10-16)** – Added strongly-typed YAML/JSON binding, schema validation, and deterministic diagnostics for operator-supplied policy bundles.| -|VEXER-POLICY-01-005 – Policy change tracking & telemetry|Team Vexer Policy|VEXER-POLICY-01-002|**DONE (2025-10-16)** – Emit revision history, expose snapshot digests via CLI/WebService, and add structured logging/metrics for policy reloads.
          2025-10-16: `VexPolicySnapshot` now carries revision/digest, provider logs reloads, `vex.policy.reloads` metric emitted, binder/diagnostics expose digest metadata. Tests: `dotnet test src/StellaOps.Vexer.Core.Tests/StellaOps.Vexer.Core.Tests.csproj`.| -|VEXER-POLICY-02-001 – Scoring coefficients & weight ceilings|Team Vexer Policy|VEXER-POLICY-01-004|TODO – Extend `VexPolicyOptions` with α/β boosters and optional >1.0 weight ceilings, validate ranges, and document operator guidance in `docs/ARCHITECTURE_VEXER.md`/`docs/VEXER_SCORRING.md`.| -|VEXER-POLICY-02-002 – Diagnostics for scoring signals|Team Vexer Policy|VEXER-POLICY-02-001|BACKLOG – Update diagnostics reports to surface missing severity/KEV/EPSS mappings, coefficient overrides, and provide actionable recommendations for policy tuning.| diff --git a/src/StellaOps.Vexer.Policy/VexPolicyProcessing.cs b/src/StellaOps.Vexer.Policy/VexPolicyProcessing.cs deleted file mode 100644 index 9b1455d3..00000000 --- a/src/StellaOps.Vexer.Policy/VexPolicyProcessing.cs +++ /dev/null @@ -1,166 +0,0 @@ -using System.Collections.Immutable; -using System.Globalization; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Policy; - -internal static class VexPolicyProcessing -{ - public static VexPolicyNormalizationResult Normalize(VexPolicyOptions? options) - { - var issues = ImmutableArray.CreateBuilder(); - - var policyOptions = options ?? new VexPolicyOptions(); - - if (!TryNormalizeWeights( - policyOptions.Weights, - out var normalizedWeights, - issues)) - { - issues.Add(new VexPolicyIssue( - "weights.invalid", - "Weight configuration is invalid; falling back to defaults.", - VexPolicyIssueSeverity.Warning)); - normalizedWeights = new VexConsensusPolicyOptions(); - } - - var overrides = NormalizeOverrides(policyOptions.ProviderOverrides, issues); - - var consensusOptions = new VexConsensusPolicyOptions( - policyOptions.Version ?? VexConsensusPolicyOptions.BaselineVersion, - normalizedWeights.VendorWeight, - normalizedWeights.DistroWeight, - normalizedWeights.PlatformWeight, - normalizedWeights.HubWeight, - normalizedWeights.AttestationWeight, - overrides); - - var orderedIssues = issues.ToImmutable().Sort(IssueComparer); - - return new VexPolicyNormalizationResult(consensusOptions, orderedIssues); - } - - public static ImmutableArray SortIssues(IEnumerable issues) - => issues.ToImmutableArray().Sort(IssueComparer); - - private static bool TryNormalizeWeights( - VexPolicyWeightOptions? options, - out VexConsensusPolicyOptions normalized, - ImmutableArray.Builder issues) - { - if (options is null) - { - normalized = new VexConsensusPolicyOptions(); - return true; - } - - var hasAny = - options.Vendor.HasValue || - options.Distro.HasValue || - options.Platform.HasValue || - options.Hub.HasValue || - options.Attestation.HasValue; - - if (!hasAny) - { - normalized = new VexConsensusPolicyOptions(); - return true; - } - - var vendor = Clamp(options.Vendor, nameof(options.Vendor), issues); - var distro = Clamp(options.Distro, nameof(options.Distro), issues); - var platform = Clamp(options.Platform, nameof(options.Platform), issues); - var hub = Clamp(options.Hub, nameof(options.Hub), issues); - var attestation = Clamp(options.Attestation, nameof(options.Attestation), issues); - - normalized = new VexConsensusPolicyOptions( - VexConsensusPolicyOptions.BaselineVersion, - vendor ?? 1.0, - distro ?? 0.9, - platform ?? 0.7, - hub ?? 0.5, - attestation ?? 0.6); - return true; - } - - private static double? Clamp(double? value, string fieldName, ImmutableArray.Builder issues) - { - if (value is null) - { - return null; - } - - if (double.IsNaN(value.Value) || double.IsInfinity(value.Value)) - { - issues.Add(new VexPolicyIssue( - $"weights.{fieldName}.invalid", - $"{fieldName} must be a finite number.", - VexPolicyIssueSeverity.Warning)); - return null; - } - - if (value.Value < 0 || value.Value > 1) - { - issues.Add(new VexPolicyIssue( - $"weights.{fieldName}.range", - $"{fieldName} must be between 0 and 1; value {value.Value.ToString(CultureInfo.InvariantCulture)} was clamped.", - VexPolicyIssueSeverity.Warning)); - return Math.Clamp(value.Value, 0, 1); - } - - return value.Value; - } - - private static ImmutableDictionary NormalizeOverrides( - IDictionary? overrides, - ImmutableArray.Builder issues) - { - if (overrides is null || overrides.Count == 0) - { - return ImmutableDictionary.Empty; - } - - var builder = ImmutableDictionary.CreateBuilder(StringComparer.Ordinal); - foreach (var kvp in overrides) - { - if (string.IsNullOrWhiteSpace(kvp.Key)) - { - issues.Add(new VexPolicyIssue( - "overrides.key.missing", - "Encountered provider override with empty key; ignoring entry.", - VexPolicyIssueSeverity.Warning)); - continue; - } - - var weight = Clamp(kvp.Value, $"overrides.{kvp.Key}", issues) ?? kvp.Value; - builder[kvp.Key.Trim()] = weight; - } - - return builder.ToImmutable(); - } - - private static int CompareIssues(VexPolicyIssue left, VexPolicyIssue right) - { - var severityCompare = GetSeverityRank(left.Severity).CompareTo(GetSeverityRank(right.Severity)); - if (severityCompare != 0) - { - return severityCompare; - } - - return string.Compare(left.Code, right.Code, StringComparison.Ordinal); - } - - private static int GetSeverityRank(VexPolicyIssueSeverity severity) - => severity switch - { - VexPolicyIssueSeverity.Error => 0, - VexPolicyIssueSeverity.Warning => 1, - _ => 2, - }; - - private static readonly Comparer IssueComparer = Comparer.Create(CompareIssues); - - internal sealed record VexPolicyNormalizationResult( - VexConsensusPolicyOptions ConsensusOptions, - ImmutableArray Issues); -} diff --git a/src/StellaOps.Vexer.Storage.Mongo.Tests/StellaOps.Vexer.Storage.Mongo.Tests.csproj b/src/StellaOps.Vexer.Storage.Mongo.Tests/StellaOps.Vexer.Storage.Mongo.Tests.csproj deleted file mode 100644 index e7bf25c6..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo.Tests/StellaOps.Vexer.Storage.Mongo.Tests.csproj +++ /dev/null @@ -1,15 +0,0 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - diff --git a/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs b/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs deleted file mode 100644 index 7009f034..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo/IVexStorageContracts.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Collections.Immutable; -using System.Threading; -using System.Threading.Tasks; -using StellaOps.Vexer.Core; - -namespace StellaOps.Vexer.Storage.Mongo; - -public interface IVexProviderStore -{ - ValueTask FindAsync(string id, CancellationToken cancellationToken); - - ValueTask> ListAsync(CancellationToken cancellationToken); - - ValueTask SaveAsync(VexProvider provider, CancellationToken cancellationToken); -} - -public interface IVexConsensusStore -{ - ValueTask FindAsync(string vulnerabilityId, string productKey, CancellationToken cancellationToken); - - ValueTask> FindByVulnerabilityAsync(string vulnerabilityId, CancellationToken cancellationToken); - - ValueTask SaveAsync(VexConsensus consensus, CancellationToken cancellationToken); -} - -public sealed record VexConnectorState( - string ConnectorId, - DateTimeOffset? LastUpdated, - ImmutableArray DocumentDigests); - -public interface IVexConnectorStateRepository -{ - ValueTask GetAsync(string connectorId, CancellationToken cancellationToken); - - ValueTask SaveAsync(VexConnectorState state, CancellationToken cancellationToken); -} - -public interface IVexCacheIndex -{ - ValueTask FindAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); - - ValueTask SaveAsync(VexCacheEntry entry, CancellationToken cancellationToken); - - ValueTask RemoveAsync(VexQuerySignature signature, VexExportFormat format, CancellationToken cancellationToken); -} - -public interface IVexCacheMaintenance -{ - ValueTask RemoveExpiredAsync(DateTimeOffset asOf, CancellationToken cancellationToken); - - ValueTask RemoveMissingManifestReferencesAsync(CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Vexer.Storage.Mongo/Properties/AssemblyInfo.cs b/src/StellaOps.Vexer.Storage.Mongo/Properties/AssemblyInfo.cs deleted file mode 100644 index 8f1a1312..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Vexer.Storage.Mongo.Tests")] diff --git a/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs b/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs deleted file mode 100644 index 984bcdc3..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo/ServiceCollectionExtensions.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Storage.Mongo.Migrations; - -namespace StellaOps.Vexer.Storage.Mongo; - -public static class VexMongoServiceCollectionExtensions -{ - public static IServiceCollection AddVexerMongoStorage(this IServiceCollection services) - { - services.AddOptions(); - - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddSingleton(); - services.AddHostedService(); - return services; - } -} diff --git a/src/StellaOps.Vexer.Storage.Mongo/TASKS.md b/src/StellaOps.Vexer.Storage.Mongo/TASKS.md deleted file mode 100644 index d74b3dc3..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo/TASKS.md +++ /dev/null @@ -1,10 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-STORAGE-01-001 – Collection schemas & class maps|Team Vexer Storage|VEXER-CORE-01-001|DONE (2025-10-15) – Added Mongo mapping registry with raw/export entities and service registration groundwork.| -|VEXER-STORAGE-01-002 – Migrations & indices bootstrap|Team Vexer Storage|VEXER-STORAGE-01-001|**DONE (2025-10-16)** – Add bootstrapper creating indices (claims by vulnId/product, exports by querySignature, etc.) and migrations for existing deployments.
          2025-10-16: Introduced migration runner + hosted service, initial index migration covers raw/providers/consensus/exports/cache, and tests use Mongo2Go to verify execution.| -|VEXER-STORAGE-01-003 – Repository layer & transactional flows|Team Vexer Storage|VEXER-STORAGE-01-001|**DONE (2025-10-16)** – Added GridFS-backed raw store with transactional upserts (including fallback for non-replicaset Mongo), export/cache repository coordination, and coverage verifying cache TTL + GridFS round-trips.| -|VEXER-STORAGE-01-004 – Provider/consensus/cache mappings|Team Vexer Storage|VEXER-STORAGE-01-001|**DONE (2025-10-16)** – Registered MongoDB class maps for provider/consensus/cache records with forward-compatible field handling and added coverage ensuring GridFS-linked cache entries round-trip cleanly.| -|VEXER-STORAGE-02-001 – Statement events & scoring signals|Team Vexer Storage|VEXER-CORE-02-001|TODO – Add immutable `vex.statements` collection, extend consensus documents with severity/KEV/EPSS fields, build indices for `policyRevisionId`/`generatedAt`, and script migrations/backfill guidance for Phase 1 rollout.| -|VEXER-STORAGE-MONGO-08-001 – Session + causal consistency hardening|Team Vexer Storage|VEXER-STORAGE-01-003|TODO – Register Mongo client/database with majority read/write concerns, expose scoped session helper enabling causal consistency, thread session handles through raw/export/consensus/cache stores (including GridFS reads), and extend integration tests to verify read-your-write semantics during replica-set failover.| diff --git a/src/StellaOps.Vexer.Storage.Mongo/VexMongoStorageOptions.cs b/src/StellaOps.Vexer.Storage.Mongo/VexMongoStorageOptions.cs deleted file mode 100644 index ae0c7aae..00000000 --- a/src/StellaOps.Vexer.Storage.Mongo/VexMongoStorageOptions.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; - -namespace StellaOps.Vexer.Storage.Mongo; - -/// -/// Configuration controlling Mongo-backed storage for Vexer repositories. -/// -public sealed class VexMongoStorageOptions : IValidatableObject -{ - private const int DefaultInlineThreshold = 256 * 1024; - private static readonly TimeSpan DefaultCacheTtl = TimeSpan.FromHours(12); - - /// - /// Name of the GridFS bucket used for raw VEX payloads that exceed . - /// - public string RawBucketName { get; set; } = "vex.raw"; - - /// - /// Inline raw document payloads smaller than this threshold; larger payloads are stored in GridFS. - /// - public int GridFsInlineThresholdBytes { get; set; } = DefaultInlineThreshold; - - /// - /// Default TTL applied to export cache entries (absolute expiration). - /// - public TimeSpan ExportCacheTtl { get; set; } = DefaultCacheTtl; - - public IEnumerable Validate(ValidationContext validationContext) - { - if (string.IsNullOrWhiteSpace(RawBucketName)) - { - yield return new ValidationResult("Raw bucket name must be provided.", new[] { nameof(RawBucketName) }); - } - - if (GridFsInlineThresholdBytes < 0) - { - yield return new ValidationResult("GridFS inline threshold must be non-negative.", new[] { nameof(GridFsInlineThresholdBytes) }); - } - - if (ExportCacheTtl <= TimeSpan.Zero) - { - yield return new ValidationResult("Export cache TTL must be greater than zero.", new[] { nameof(ExportCacheTtl) }); - } - } -} diff --git a/src/StellaOps.Vexer.WebService/Program.cs b/src/StellaOps.Vexer.WebService/Program.cs deleted file mode 100644 index cef66a25..00000000 --- a/src/StellaOps.Vexer.WebService/Program.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using Microsoft.Extensions.Options; -using StellaOps.Vexer.Attestation.Extensions; -using StellaOps.Vexer.Attestation; -using StellaOps.Vexer.Attestation.Transparency; -using StellaOps.Vexer.ArtifactStores.S3.Extensions; -using StellaOps.Vexer.Export; -using StellaOps.Vexer.Storage.Mongo; -using StellaOps.Vexer.Connectors.RedHat.CSAF.DependencyInjection; - -var builder = WebApplication.CreateBuilder(args); -var configuration = builder.Configuration; -var services = builder.Services; - -services.AddOptions() - .Bind(configuration.GetSection("Vexer:Storage:Mongo")) - .ValidateOnStart(); - -services.AddVexerMongoStorage(); -services.AddVexExportEngine(); -services.AddVexExportCacheServices(); -services.AddVexAttestation(); -services.Configure(configuration.GetSection("Vexer:Attestation:Client")); -services.AddRedHatCsafConnector(); - -var rekorSection = configuration.GetSection("Vexer:Attestation:Rekor"); -if (rekorSection.Exists()) -{ - services.AddVexRekorClient(opts => rekorSection.Bind(opts)); -} - -var fileSystemSection = configuration.GetSection("Vexer:Artifacts:FileSystem"); -if (fileSystemSection.Exists()) -{ - services.AddVexFileSystemArtifactStore(opts => fileSystemSection.Bind(opts)); -} -else -{ - services.AddVexFileSystemArtifactStore(_ => { }); -} - -var s3Section = configuration.GetSection("Vexer:Artifacts:S3"); -if (s3Section.Exists()) -{ - services.AddVexS3ArtifactClient(opts => s3Section.GetSection("Client").Bind(opts)); - services.AddSingleton(provider => - { - var options = new S3ArtifactStoreOptions(); - s3Section.GetSection("Store").Bind(options); - return new S3ArtifactStore( - provider.GetRequiredService(), - Microsoft.Extensions.Options.Options.Create(options), - provider.GetRequiredService>()); - }); -} - -var offlineSection = configuration.GetSection("Vexer:Artifacts:OfflineBundle"); -if (offlineSection.Exists()) -{ - services.AddVexOfflineBundleArtifactStore(opts => offlineSection.Bind(opts)); -} - -services.AddEndpointsApiExplorer(); -services.AddHealthChecks(); -services.AddSingleton(TimeProvider.System); - -var app = builder.Build(); - -app.MapGet("/vexer/status", async (HttpContext context, - IEnumerable artifactStores, - IOptions mongoOptions, - TimeProvider timeProvider) => -{ - var payload = new StatusResponse( - timeProvider.GetUtcNow(), - mongoOptions.Value.RawBucketName, - mongoOptions.Value.GridFsInlineThresholdBytes, - artifactStores.Select(store => store.GetType().Name).ToArray()); - - context.Response.ContentType = "application/json"; - await System.Text.Json.JsonSerializer.SerializeAsync(context.Response.Body, payload); -}); - -app.MapHealthChecks("/vexer/health"); - -app.Run(); - -public partial class Program; - -internal sealed record StatusResponse(DateTimeOffset UtcNow, string MongoBucket, int InlineThreshold, string[] ArtifactStores); diff --git a/src/StellaOps.Vexer.WebService/StellaOps.Vexer.WebService.csproj b/src/StellaOps.Vexer.WebService/StellaOps.Vexer.WebService.csproj deleted file mode 100644 index 0e85d526..00000000 --- a/src/StellaOps.Vexer.WebService/StellaOps.Vexer.WebService.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - diff --git a/src/StellaOps.Vexer.WebService/TASKS.md b/src/StellaOps.Vexer.WebService/TASKS.md deleted file mode 100644 index f29650c5..00000000 --- a/src/StellaOps.Vexer.WebService/TASKS.md +++ /dev/null @@ -1,8 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-WEB-01-001 – Minimal API bootstrap & DI|Team Vexer WebService|VEXER-CORE-01-003, VEXER-STORAGE-01-003|**DONE (2025-10-17)** – Minimal API host composes storage/export/attestation/artifact stores, binds Mongo/attestation options, and exposes `/vexer/status` + health endpoints with regression coverage in `StatusEndpointTests`.| -|VEXER-WEB-01-002 – Ingest & reconcile endpoints|Team Vexer WebService|VEXER-WEB-01-001|TODO – Implement `/vexer/init`, `/vexer/ingest/run`, `/vexer/ingest/resume`, `/vexer/reconcile` with token scope enforcement and structured run telemetry.| -|VEXER-WEB-01-003 – Export & verify endpoints|Team Vexer WebService|VEXER-WEB-01-001, VEXER-EXPORT-01-001, VEXER-ATTEST-01-001|TODO – Add `/vexer/export`, `/vexer/export/{id}`, `/vexer/export/{id}/download`, `/vexer/verify`, returning artifact + attestation metadata with cache awareness.| -|VEXER-WEB-01-004 – Resolve API & signed responses|Team Vexer WebService|VEXER-WEB-01-001, VEXER-ATTEST-01-002|TODO – Deliver `/vexer/resolve` (subject/context), return consensus + score envelopes, attach cosign/Rekor metadata, and document auth + rate guardrails.| diff --git a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs b/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs deleted file mode 100644 index bf53b771..00000000 --- a/src/StellaOps.Vexer.Worker/Options/VexWorkerOptionsValidator.cs +++ /dev/null @@ -1,50 +0,0 @@ -using System.Collections.Generic; -using Microsoft.Extensions.Options; - -namespace StellaOps.Vexer.Worker.Options; - -internal sealed class VexWorkerOptionsValidator : IValidateOptions -{ - public ValidateOptionsResult Validate(string? name, VexWorkerOptions options) - { - var failures = new List(); - - if (options.DefaultInterval <= TimeSpan.Zero) - { - failures.Add("Vexer.Worker.DefaultInterval must be greater than zero."); - } - - if (options.OfflineInterval <= TimeSpan.Zero) - { - failures.Add("Vexer.Worker.OfflineInterval must be greater than zero."); - } - - if (options.DefaultInitialDelay < TimeSpan.Zero) - { - failures.Add("Vexer.Worker.DefaultInitialDelay cannot be negative."); - } - - for (var i = 0; i < options.Providers.Count; i++) - { - var provider = options.Providers[i]; - if (string.IsNullOrWhiteSpace(provider.ProviderId)) - { - failures.Add($"Vexer.Worker.Providers[{i}].ProviderId must be set."); - } - - if (provider.Interval is { } interval && interval <= TimeSpan.Zero) - { - failures.Add($"Vexer.Worker.Providers[{i}].Interval must be greater than zero when specified."); - } - - if (provider.InitialDelay is { } delay && delay < TimeSpan.Zero) - { - failures.Add($"Vexer.Worker.Providers[{i}].InitialDelay cannot be negative."); - } - } - - return failures.Count > 0 - ? ValidateOptionsResult.Fail(failures) - : ValidateOptionsResult.Success; - } -} diff --git a/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs b/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs deleted file mode 100644 index bbbd22fd..00000000 --- a/src/StellaOps.Vexer.Worker/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,3 +0,0 @@ -using System.Runtime.CompilerServices; - -[assembly: InternalsVisibleTo("StellaOps.Vexer.Worker.Tests")] diff --git a/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs b/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs deleted file mode 100644 index 113d1f7c..00000000 --- a/src/StellaOps.Vexer.Worker/Scheduling/DefaultVexProviderRunner.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Linq; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using StellaOps.Plugin; - -namespace StellaOps.Vexer.Worker.Scheduling; - -internal sealed class DefaultVexProviderRunner : IVexProviderRunner -{ - private readonly IServiceProvider _serviceProvider; - private readonly PluginCatalog _pluginCatalog; - private readonly ILogger _logger; - - public DefaultVexProviderRunner( - IServiceProvider serviceProvider, - PluginCatalog pluginCatalog, - ILogger logger) - { - _serviceProvider = serviceProvider ?? throw new ArgumentNullException(nameof(serviceProvider)); - _pluginCatalog = pluginCatalog ?? throw new ArgumentNullException(nameof(pluginCatalog)); - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - } - - public ValueTask RunAsync(string providerId, CancellationToken cancellationToken) - { - ArgumentException.ThrowIfNullOrWhiteSpace(providerId); - - using var scope = _serviceProvider.CreateScope(); - var availablePlugins = _pluginCatalog.GetAvailableConnectorPlugins(scope.ServiceProvider); - var matched = availablePlugins.FirstOrDefault(plugin => - string.Equals(plugin.Name, providerId, StringComparison.OrdinalIgnoreCase)); - - if (matched is null) - { - _logger.LogInformation("No connector plugin registered for provider {ProviderId}; nothing to execute.", providerId); - return ValueTask.CompletedTask; - } - - _logger.LogInformation( - "Connector plugin {PluginName} ({ProviderId}) is available. Execution hooks will be added in subsequent tasks.", - matched.Name, - providerId); - - return ValueTask.CompletedTask; - } -} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs b/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs deleted file mode 100644 index 8d20f39d..00000000 --- a/src/StellaOps.Vexer.Worker/Scheduling/IVexProviderRunner.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace StellaOps.Vexer.Worker.Scheduling; - -internal interface IVexProviderRunner -{ - ValueTask RunAsync(string providerId, CancellationToken cancellationToken); -} diff --git a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs b/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs deleted file mode 100644 index 81b2c68a..00000000 --- a/src/StellaOps.Vexer.Worker/Scheduling/VexWorkerSchedule.cs +++ /dev/null @@ -1,3 +0,0 @@ -namespace StellaOps.Vexer.Worker.Scheduling; - -internal sealed record VexWorkerSchedule(string ProviderId, TimeSpan Interval, TimeSpan InitialDelay); diff --git a/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj b/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj deleted file mode 100644 index 93a50200..00000000 --- a/src/StellaOps.Vexer.Worker/StellaOps.Vexer.Worker.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - net10.0 - preview - enable - enable - true - - - - - - - - - - - - diff --git a/src/StellaOps.Vexer.Worker/TASKS.md b/src/StellaOps.Vexer.Worker/TASKS.md deleted file mode 100644 index 863ebf09..00000000 --- a/src/StellaOps.Vexer.Worker/TASKS.md +++ /dev/null @@ -1,8 +0,0 @@ -If you are working on this file you need to read docs/ARCHITECTURE_VEXER.md and ./AGENTS.md). -# TASKS -| Task | Owner(s) | Depends on | Notes | -|---|---|---|---| -|VEXER-WORKER-01-001 – Worker host & scheduling|Team Vexer Worker|VEXER-STORAGE-01-003, VEXER-WEB-01-001|**DONE (2025-10-17)** – Worker project bootstraps provider schedules from configuration, integrates plugin catalog discovery, and emits structured logs/metrics-ready events via `VexWorkerHostedService`; scheduling logic covered by `VexWorkerOptionsTests`.| -|VEXER-WORKER-01-002 – Resume tokens & retry policy|Team Vexer Worker|VEXER-WORKER-01-001|TODO – Implement durable resume markers, exponential backoff with jitter, and quarantine for failing connectors per architecture spec.| -|VEXER-WORKER-01-003 – Verification & cache GC loops|Team Vexer Worker|VEXER-WORKER-01-001, VEXER-ATTEST-01-003, VEXER-EXPORT-01-002|TODO – Add scheduled attestation re-verification and cache pruning routines, surfacing metrics for export reuse ratios.| -|VEXER-WORKER-01-004 – TTL refresh & stability damper|Team Vexer Worker|VEXER-WORKER-01-001, VEXER-CORE-02-001|TODO – Monitor consensus/VEX TTLs, apply 24–48h dampers before flipping published status/score, and trigger re-resolve when base image or kernel fingerprints change.| diff --git a/src/StellaOps.Web/.editorconfig b/src/StellaOps.Web/.editorconfig new file mode 100644 index 00000000..923742db --- /dev/null +++ b/src/StellaOps.Web/.editorconfig @@ -0,0 +1,16 @@ +# Editor configuration, see https://editorconfig.org +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.ts] +quote_type = single + +[*.md] +max_line_length = off +trim_trailing_whitespace = false diff --git a/src/StellaOps.Web/.gitignore b/src/StellaOps.Web/.gitignore new file mode 100644 index 00000000..70583d78 --- /dev/null +++ b/src/StellaOps.Web/.gitignore @@ -0,0 +1,42 @@ +# See https://docs.github.com/get-started/getting-started-with-git/ignoring-files for more about ignoring files. + +# Compiled output +/dist +/tmp +/out-tsc +/bazel-out + +# Node +/node_modules +npm-debug.log +yarn-error.log + +# IDEs and editors +.idea/ +.project +.classpath +.c9/ +*.launch +.settings/ +*.sublime-workspace + +# Visual Studio Code +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history/* + +# Miscellaneous +/.angular/cache +.sass-cache/ +/connect.lock +/coverage +/libpeerconnection.log +testem.log +/typings + +# System files +.DS_Store +Thumbs.db diff --git a/src/StellaOps.Web/AGENTS.md b/src/StellaOps.Web/AGENTS.md index 03209b69..679d18e0 100644 --- a/src/StellaOps.Web/AGENTS.md +++ b/src/StellaOps.Web/AGENTS.md @@ -1,24 +1,24 @@ -# StellaOps Web Frontend - -## Mission -Design and build the StellaOps web user experience that surfaces backend capabilities (Authority, Feedser, Exporters) through an offline-friendly Angular application. - -## Team Composition -- **UX Specialist** – defines user journeys, interaction patterns, accessibility guidelines, and visual design language. -- **Angular Engineers** – implement the SPA, integrate with backend APIs, and ensure deterministic builds suitable for air-gapped deployments. - -## Operating Principles -- Favor modular Angular architecture (feature modules, shared UI kit) with strong typing via latest TypeScript/Angular releases. -- Align UI flows with backend contracts; coordinate with Authority and Feedser teams for API changes. -- Keep assets and build outputs deterministic and cacheable for Offline Kit packaging. -- Track work using the local `TASKS.md` board; keep statuses (TODO/DOING/REVIEW/BLOCKED/DONE) up to date. - -## Key Paths -- `src/StellaOps.Web` — Angular workspace (to be scaffolded). -- `docs/` — UX specs and mockups (to be added). -- `ops/` — Web deployment manifests for air-gapped environments (future). - -## Coordination -- Sync with DevEx for project scaffolding and build pipelines. -- Partner with Docs Guild to translate UX decisions into operator guides. -- Collaborate with Security Guild to validate authentication flows and session handling. +# StellaOps Web Frontend + +## Mission +Design and build the StellaOps web user experience that surfaces backend capabilities (Authority, Concelier, Exporters) through an offline-friendly Angular application. + +## Team Composition +- **UX Specialist** – defines user journeys, interaction patterns, accessibility guidelines, and visual design language. +- **Angular Engineers** – implement the SPA, integrate with backend APIs, and ensure deterministic builds suitable for air-gapped deployments. + +## Operating Principles +- Favor modular Angular architecture (feature modules, shared UI kit) with strong typing via latest TypeScript/Angular releases. +- Align UI flows with backend contracts; coordinate with Authority and Concelier teams for API changes. +- Keep assets and build outputs deterministic and cacheable for Offline Kit packaging. +- Track work using the local `TASKS.md` board; keep statuses (TODO/DOING/REVIEW/BLOCKED/DONE) up to date. + +## Key Paths +- `src/StellaOps.Web` — Angular workspace (to be scaffolded). +- `docs/` — UX specs and mockups (to be added). +- `ops/` — Web deployment manifests for air-gapped environments (future). + +## Coordination +- Sync with DevEx for project scaffolding and build pipelines. +- Partner with Docs Guild to translate UX decisions into operator guides. +- Collaborate with Security Guild to validate authentication flows and session handling. diff --git a/src/StellaOps.Web/README.md b/src/StellaOps.Web/README.md new file mode 100644 index 00000000..8e3e2bae --- /dev/null +++ b/src/StellaOps.Web/README.md @@ -0,0 +1,27 @@ +# StellaopsWeb + +This project was generated with [Angular CLI](https://github.com/angular/angular-cli) version 17.3.17. + +## Development server + +Run `ng serve` for a dev server. Navigate to `http://localhost:4200/`. The application will automatically reload if you change any of the source files. + +## Code scaffolding + +Run `ng generate component component-name` to generate a new component. You can also use `ng generate directive|pipe|service|class|guard|interface|enum|module`. + +## Build + +Run `ng build` to build the project. The build artifacts will be stored in the `dist/` directory. + +## Running unit tests + +Run `ng test` to execute the unit tests via [Karma](https://karma-runner.github.io). + +## Running end-to-end tests + +Run `ng e2e` to execute the end-to-end tests via a platform of your choice. To use this command, you need to first add a package that implements end-to-end testing capabilities. + +## Further help + +To get more help on the Angular CLI use `ng help` or go check out the [Angular CLI Overview and Command Reference](https://angular.io/cli) page. diff --git a/src/StellaOps.Web/TASKS.md b/src/StellaOps.Web/TASKS.md index 65306a0d..888b0a79 100644 --- a/src/StellaOps.Web/TASKS.md +++ b/src/StellaOps.Web/TASKS.md @@ -1,5 +1,5 @@ -# StellaOps Web Task Board (UTC 2025-10-10) - -| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | -|----|--------|----------|------------|-------------|---------------| -| WEB1.TRIVY-SETTINGS | TODO | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Panel wired to mocked API; ✅ Overrides persisted via settings endpoint; ✅ Manual run button reuses overrides. | +# StellaOps Web Task Board (UTC 2025-10-10) + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| WEB1.TRIVY-SETTINGS | DOING (2025-10-19) | UX Specialist, Angular Eng | Backend `/exporters/trivy-db` contract | Implement Trivy DB exporter settings panel with `publishFull`, `publishDelta`, `includeFull`, `includeDelta` toggles and “Run export now” action using future `/exporters/trivy-db/settings` API. | ✅ Panel wired to mocked API; ✅ Overrides persisted via settings endpoint; ✅ Manual run button reuses overrides. | diff --git a/src/StellaOps.Web/angular.json b/src/StellaOps.Web/angular.json new file mode 100644 index 00000000..302d21cd --- /dev/null +++ b/src/StellaOps.Web/angular.json @@ -0,0 +1,101 @@ +{ + "$schema": "./node_modules/@angular/cli/lib/config/schema.json", + "version": 1, + "newProjectRoot": "projects", + "projects": { + "stellaops-web": { + "projectType": "application", + "schematics": { + "@schematics/angular:component": { + "style": "scss" + } + }, + "root": "", + "sourceRoot": "src", + "prefix": "app", + "architect": { + "build": { + "builder": "@angular-devkit/build-angular:application", + "options": { + "outputPath": "dist/stellaops-web", + "index": "src/index.html", + "browser": "src/main.ts", + "polyfills": [ + "zone.js" + ], + "tsConfig": "tsconfig.app.json", + "inlineStyleLanguage": "scss", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], + "scripts": [] + }, + "configurations": { + "production": { + "budgets": [ + { + "type": "initial", + "maximumWarning": "500kb", + "maximumError": "1mb" + }, + { + "type": "anyComponentStyle", + "maximumWarning": "2kb", + "maximumError": "4kb" + } + ], + "outputHashing": "all" + }, + "development": { + "optimization": false, + "extractLicenses": false, + "sourceMap": true + } + }, + "defaultConfiguration": "production" + }, + "serve": { + "builder": "@angular-devkit/build-angular:dev-server", + "configurations": { + "production": { + "buildTarget": "stellaops-web:build:production" + }, + "development": { + "buildTarget": "stellaops-web:build:development" + } + }, + "defaultConfiguration": "development" + }, + "extract-i18n": { + "builder": "@angular-devkit/build-angular:extract-i18n", + "options": { + "buildTarget": "stellaops-web:build" + } + }, + "test": { + "builder": "@angular-devkit/build-angular:karma", + "options": { + "polyfills": [ + "zone.js", + "zone.js/testing" + ], + "tsConfig": "tsconfig.spec.json", + "inlineStyleLanguage": "scss", + "assets": [ + "src/favicon.ico", + "src/assets" + ], + "styles": [ + "src/styles.scss" + ], + "scripts": [] + } + } + } + } + } +} diff --git a/src/StellaOps.Web/docs/TrivyDbSettings.md b/src/StellaOps.Web/docs/TrivyDbSettings.md new file mode 100644 index 00000000..19c07bbb --- /dev/null +++ b/src/StellaOps.Web/docs/TrivyDbSettings.md @@ -0,0 +1,37 @@ +# WEB1.TRIVY-SETTINGS – Backend Contract & UI Wiring Notes + +## 1. Known backend surfaces + +- `POST /jobs/export:trivy-db` + Payload is wrapped as `{ "trigger": "", "parameters": { ... } }` and accepts the overrides shown in `TrivyDbExportJob` (`publishFull`, `publishDelta`, `includeFull`, `includeDelta`). + Evidence: `src/StellaOps.Cli/Commands/CommandHandlers.cs:263`, `src/StellaOps.Cli/Services/Models/Transport/JobTriggerRequest.cs:5`, `src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportJob.cs:27`. +- Export configuration defaults sit under `TrivyDbExportOptions.Oras` and `.OfflineBundle`. Both booleans default to `true`, so overriding to `false` must be explicit. + Evidence: `src/StellaOps.Concelier.Exporter.TrivyDb/TrivyDbExportOptions.cs:8`. + +## 2. Clarifications needed from Concelier backend + +| Topic | Questions to resolve | Suggested owner | +| --- | --- | --- | +| Settings endpoint surface | `Program.cs` only exposes `/jobs/*` and health endpoints—there is currently **no** `/exporters/trivy-db/settings` route. Confirm the intended path (`/api/v1/concelier/exporters/trivy-db/settings`?), verbs (`GET`/`PUT` or `PATCH`), and DTO schema (flat booleans vs nested `oras`/`offlineBundle`). | Concelier WebService | +| Auth scopes | Verify required roles (likely `concelier.export` or `concelier.admin`) and whether UI needs to request additional scopes beyond existing dashboard access. | Authority & Concelier teams | +| Concurrency control | Determine if settings payload includes an ETag or timestamp we must echo (`If-Match`) to avoid stomping concurrent edits. | Concelier WebService | +| Validation & defaults | Clarify server-side validation rules (e.g., must `publishDelta` be `false` when `publishFull` is `false`?) and shape of Problem+JSON responses. | Concelier WebService | +| Manual run trigger | Confirm whether settings update should immediately kick an export or if UI should call `POST /jobs/export:trivy-db` separately (current CLI behaviour suggests a separate call). | Concelier WebService | + +## 3. Proposed Angular implementation (pending contract lock) + +- **Feature module**: `app/concelier/trivy-db-settings/` with a standalone routed page (`TrivyDbSettingsPage`) and a reusable form component (`TrivyDbSettingsForm`). +- **State & transport**: + - Client wrapper under `core/api/concelier-exporter.client.ts` exposing `getTrivyDbSettings`, `updateTrivyDbSettings`, and `runTrivyDbExport`. + - Store built with `@ngrx/signals` keeping `settings`, `isDirty`, `lastFetchedAt`, and error state; optimistic updates gated on ETag confirmation once the backend specifies the shape. + - Shared DTOs generated from the confirmed schema to keep Concelier/CLI alignment. +- **UX flow**: + - Load settings on navigation; show inline info about current publish/bundle defaults. + - “Run export now” button opens confirmation modal summarising overrides, then calls `runTrivyDbExport` (separate API call) while reusing local state. + - Surface Problem+JSON errors via existing toast/notification pattern and capture correlation IDs for ops visibility. +- **Offline posture**: cache latest successful settings payload in IndexedDB (read-only when offline) and disable the run button when token/scopes are missing. + +## 4. Next steps + +1. Share section 2 with Concelier WebService owners to confirm the REST contract (blocking before scaffolding DTOs). +2. Once confirmed, scaffold the Angular workspace and feature shell, keeping deterministic build outputs per `src/StellaOps.Web/AGENTS.md`. diff --git a/src/StellaOps.Web/package.json b/src/StellaOps.Web/package.json new file mode 100644 index 00000000..0d9a9518 --- /dev/null +++ b/src/StellaOps.Web/package.json @@ -0,0 +1,38 @@ +{ + "name": "stellaops-web", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "@angular/animations": "^17.3.0", + "@angular/common": "^17.3.0", + "@angular/compiler": "^17.3.0", + "@angular/core": "^17.3.0", + "@angular/forms": "^17.3.0", + "@angular/platform-browser": "^17.3.0", + "@angular/platform-browser-dynamic": "^17.3.0", + "@angular/router": "^17.3.0", + "rxjs": "~7.8.0", + "tslib": "^2.3.0", + "zone.js": "~0.14.3" + }, + "devDependencies": { + "@angular-devkit/build-angular": "^17.3.17", + "@angular/cli": "^17.3.17", + "@angular/compiler-cli": "^17.3.0", + "@types/jasmine": "~5.1.0", + "jasmine-core": "~5.1.0", + "karma": "~6.4.0", + "karma-chrome-launcher": "~3.2.0", + "karma-coverage": "~2.2.0", + "karma-jasmine": "~5.1.0", + "karma-jasmine-html-reporter": "~2.1.0", + "typescript": "~5.4.2" + } +} diff --git a/src/StellaOps.Web/src/app/app.component.html b/src/StellaOps.Web/src/app/app.component.html new file mode 100644 index 00000000..ad4f65ac --- /dev/null +++ b/src/StellaOps.Web/src/app/app.component.html @@ -0,0 +1,336 @@ + + + + + + + + + + + +
          +
          +
          + +

          Hello, {{ title }}

          +

          Congratulations! Your app is running. 🎉

          +
          + +
          +
          + @for (item of [ + { title: 'Explore the Docs', link: 'https://angular.dev' }, + { title: 'Learn with Tutorials', link: 'https://angular.dev/tutorials' }, + { title: 'CLI Docs', link: 'https://angular.dev/tools/cli' }, + { title: 'Angular Language Service', link: 'https://angular.dev/tools/language-service' }, + { title: 'Angular DevTools', link: 'https://angular.dev/tools/devtools' }, + ]; track item.title) { + + {{ item.title }} + + + + + } +
          + +
          +
          +
          + + + + + + + + + + + diff --git a/src/StellaOps.Web/src/app/app.component.scss b/src/StellaOps.Web/src/app/app.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/src/StellaOps.Web/src/app/app.component.spec.ts b/src/StellaOps.Web/src/app/app.component.spec.ts new file mode 100644 index 00000000..6aeb253e --- /dev/null +++ b/src/StellaOps.Web/src/app/app.component.spec.ts @@ -0,0 +1,29 @@ +import { TestBed } from '@angular/core/testing'; +import { AppComponent } from './app.component'; + +describe('AppComponent', () => { + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [AppComponent], + }).compileComponents(); + }); + + it('should create the app', () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app).toBeTruthy(); + }); + + it(`should have the 'stellaops-web' title`, () => { + const fixture = TestBed.createComponent(AppComponent); + const app = fixture.componentInstance; + expect(app.title).toEqual('stellaops-web'); + }); + + it('should render title', () => { + const fixture = TestBed.createComponent(AppComponent); + fixture.detectChanges(); + const compiled = fixture.nativeElement as HTMLElement; + expect(compiled.querySelector('h1')?.textContent).toContain('Hello, stellaops-web'); + }); +}); diff --git a/src/StellaOps.Web/src/app/app.component.ts b/src/StellaOps.Web/src/app/app.component.ts new file mode 100644 index 00000000..2c8a178d --- /dev/null +++ b/src/StellaOps.Web/src/app/app.component.ts @@ -0,0 +1,13 @@ +import { Component } from '@angular/core'; +import { RouterOutlet } from '@angular/router'; + +@Component({ + selector: 'app-root', + standalone: true, + imports: [RouterOutlet], + templateUrl: './app.component.html', + styleUrl: './app.component.scss' +}) +export class AppComponent { + title = 'stellaops-web'; +} diff --git a/src/StellaOps.Web/src/app/app.config.ts b/src/StellaOps.Web/src/app/app.config.ts new file mode 100644 index 00000000..22b2bd04 --- /dev/null +++ b/src/StellaOps.Web/src/app/app.config.ts @@ -0,0 +1,8 @@ +import { ApplicationConfig } from '@angular/core'; +import { provideRouter } from '@angular/router'; + +import { routes } from './app.routes'; + +export const appConfig: ApplicationConfig = { + providers: [provideRouter(routes)] +}; diff --git a/src/StellaOps.Web/src/app/app.routes.ts b/src/StellaOps.Web/src/app/app.routes.ts new file mode 100644 index 00000000..4f3af40e --- /dev/null +++ b/src/StellaOps.Web/src/app/app.routes.ts @@ -0,0 +1,3 @@ +import { Routes } from '@angular/router'; + +export const routes: Routes = []; diff --git a/src/StellaOps.Web/src/assets/.gitkeep b/src/StellaOps.Web/src/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/src/StellaOps.Web/src/favicon.ico b/src/StellaOps.Web/src/favicon.ico new file mode 100644 index 00000000..57614f9c Binary files /dev/null and b/src/StellaOps.Web/src/favicon.ico differ diff --git a/src/StellaOps.Web/src/index.html b/src/StellaOps.Web/src/index.html new file mode 100644 index 00000000..b5541629 --- /dev/null +++ b/src/StellaOps.Web/src/index.html @@ -0,0 +1,13 @@ + + + + + StellaopsWeb + + + + + + + + diff --git a/src/StellaOps.Web/src/main.ts b/src/StellaOps.Web/src/main.ts new file mode 100644 index 00000000..89ca5674 --- /dev/null +++ b/src/StellaOps.Web/src/main.ts @@ -0,0 +1,6 @@ +import { bootstrapApplication } from '@angular/platform-browser'; +import { appConfig } from './app/app.config'; +import { AppComponent } from './app/app.component'; + +bootstrapApplication(AppComponent, appConfig) + .catch((err) => console.error(err)); diff --git a/src/StellaOps.Web/src/styles.scss b/src/StellaOps.Web/src/styles.scss new file mode 100644 index 00000000..9907dc13 --- /dev/null +++ b/src/StellaOps.Web/src/styles.scss @@ -0,0 +1 @@ +/* You can add global styles to this file, and also import other style files */ diff --git a/src/StellaOps.Web/tsconfig.app.json b/src/StellaOps.Web/tsconfig.app.json new file mode 100644 index 00000000..2977815c --- /dev/null +++ b/src/StellaOps.Web/tsconfig.app.json @@ -0,0 +1,14 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/app", + "types": [] + }, + "files": [ + "src/main.ts" + ], + "include": [ + "src/**/*.d.ts" + ] +} diff --git a/src/StellaOps.Web/tsconfig.json b/src/StellaOps.Web/tsconfig.json new file mode 100644 index 00000000..16201006 --- /dev/null +++ b/src/StellaOps.Web/tsconfig.json @@ -0,0 +1,32 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "compileOnSave": false, + "compilerOptions": { + "outDir": "./dist/out-tsc", + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "skipLibCheck": true, + "esModuleInterop": true, + "sourceMap": true, + "declaration": false, + "experimentalDecorators": true, + "moduleResolution": "node", + "importHelpers": true, + "target": "ES2022", + "module": "ES2022", + "useDefineForClassFields": false, + "lib": [ + "ES2022", + "dom" + ] + }, + "angularCompilerOptions": { + "enableI18nLegacyMessageIdFormat": false, + "strictInjectionParameters": true, + "strictInputAccessModifiers": true, + "strictTemplates": true + } +} diff --git a/src/StellaOps.Web/tsconfig.spec.json b/src/StellaOps.Web/tsconfig.spec.json new file mode 100644 index 00000000..98d923a2 --- /dev/null +++ b/src/StellaOps.Web/tsconfig.spec.json @@ -0,0 +1,14 @@ +/* To learn more about this file see: https://angular.io/config/tsconfig. */ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/spec", + "types": [ + "jasmine" + ] + }, + "include": [ + "src/**/*.spec.ts", + "src/**/*.d.ts" + ] +} diff --git a/src/StellaOps.Zastava.Core.Tests/Contracts/ZastavaContractVersionsTests.cs b/src/StellaOps.Zastava.Core.Tests/Contracts/ZastavaContractVersionsTests.cs new file mode 100644 index 00000000..22ce7340 --- /dev/null +++ b/src/StellaOps.Zastava.Core.Tests/Contracts/ZastavaContractVersionsTests.cs @@ -0,0 +1,66 @@ +using StellaOps.Zastava.Core.Contracts; + +namespace StellaOps.Zastava.Core.Tests.Contracts; + +public sealed class ZastavaContractVersionsTests +{ + [Theory] + [InlineData("zastava.runtime.event@v1", "zastava.runtime.event", 1, 0)] + [InlineData("zastava.runtime.event@v1.0", "zastava.runtime.event", 1, 0)] + [InlineData("zastava.admission.decision@v1.2", "zastava.admission.decision", 1, 2)] + public void TryParse_ParsesCanonicalForms(string input, string schema, int major, int minor) + { + var success = ZastavaContractVersions.ContractVersion.TryParse(input, out var contract); + + Assert.True(success); + Assert.Equal(schema, contract.Schema); + Assert.Equal(new Version(major, minor), contract.Version); + Assert.Equal($"{schema}@v{major}.{minor}", contract.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("zastava.runtime.event")] + [InlineData("runtime@1.0")] + [InlineData("zastava.runtime.event@vinvalid")] + public void TryParse_InvalidInputs_ReturnsFalse(string input) + { + var success = ZastavaContractVersions.ContractVersion.TryParse(input, out _); + + Assert.False(success); + } + + [Fact] + public void IsRuntimeEventSupported_RespectsMajorCompatibility() + { + Assert.True(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v1")); + Assert.True(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v1.0")); + Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.runtime.event@v2.0")); + Assert.False(ZastavaContractVersions.IsRuntimeEventSupported("zastava.admission.decision@v1")); + } + + [Fact] + public void NegotiateRuntimeEvent_PicksHighestCommonVersion() + { + var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[] + { + "zastava.runtime.event@v1.0", + "zastava.runtime.event@v0.9", + "zastava.admission.decision@v1" + }); + + Assert.Equal("zastava.runtime.event@v1.0", negotiated.ToString()); + } + + [Fact] + public void NegotiateRuntimeEvent_FallsBackToLocalWhenNoMatch() + { + var negotiated = ZastavaContractVersions.NegotiateRuntimeEvent(new[] + { + "zastava.runtime.event@v2.0", + "zastava.admission.decision@v2.0" + }); + + Assert.Equal(ZastavaContractVersions.RuntimeEvent.ToString(), negotiated.ToString()); + } +} diff --git a/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs b/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs new file mode 100644 index 00000000..6c520e6e --- /dev/null +++ b/src/StellaOps.Zastava.Core.Tests/Serialization/ZastavaCanonicalJsonSerializerTests.cs @@ -0,0 +1,204 @@ +using System.Text; +using StellaOps.Zastava.Core.Contracts; +using StellaOps.Zastava.Core.Hashing; +using StellaOps.Zastava.Core.Serialization; + +namespace StellaOps.Zastava.Core.Tests.Serialization; + +public sealed class ZastavaCanonicalJsonSerializerTests +{ + [Fact] + public void Serialize_RuntimeEventEnvelope_ProducesDeterministicOrdering() + { + var runtimeEvent = new RuntimeEvent + { + EventId = "evt-123", + When = DateTimeOffset.Parse("2025-10-19T12:34:56Z"), + Kind = RuntimeEventKind.ContainerStart, + Tenant = "tenant-01", + Node = "node-a", + Runtime = new RuntimeEngine + { + Engine = "containerd", + Version = "1.7.19" + }, + Workload = new RuntimeWorkload + { + Platform = "kubernetes", + Namespace = "payments", + Pod = "api-7c9fbbd8b7-ktd84", + Container = "api", + ContainerId = "containerd://abc", + ImageRef = "ghcr.io/acme/api@sha256:abcd", + Owner = new RuntimeWorkloadOwner + { + Kind = "Deployment", + Name = "api" + } + }, + Process = new RuntimeProcess + { + Pid = 12345, + Entrypoint = new[] { "/entrypoint.sh", "--serve" }, + EntryTrace = new[] + { + new RuntimeEntryTrace + { + File = "/entrypoint.sh", + Line = 3, + Op = "exec", + Target = "/usr/bin/python3" + } + } + }, + LoadedLibraries = new[] + { + new RuntimeLoadedLibrary + { + Path = "/lib/x86_64-linux-gnu/libssl.so.3", + Inode = 123456, + Sha256 = "abc123" + } + }, + Posture = new RuntimePosture + { + ImageSigned = true, + SbomReferrer = "present", + Attestation = new RuntimeAttestation + { + Uuid = "rekor-uuid", + Verified = true + } + }, + Delta = new RuntimeDelta + { + BaselineImageDigest = "sha256:abcd", + ChangedFiles = new[] { "/opt/app/server.py" }, + NewBinaries = new[] + { + new RuntimeNewBinary + { + Path = "/usr/local/bin/helper", + Sha256 = "def456" + } + } + }, + Evidence = new[] + { + new RuntimeEvidence + { + Signal = "procfs.maps", + Value = "/lib/.../libssl.so.3@0x7f..." + } + }, + Annotations = new Dictionary + { + ["source"] = "unit-test" + } + }; + + var envelope = RuntimeEventEnvelope.Create(runtimeEvent, ZastavaContractVersions.RuntimeEvent); + var json = ZastavaCanonicalJsonSerializer.Serialize(envelope); + + var expectedOrder = new[] + { + "\"schemaVersion\"", + "\"event\"", + "\"eventId\"", + "\"when\"", + "\"kind\"", + "\"tenant\"", + "\"node\"", + "\"runtime\"", + "\"engine\"", + "\"version\"", + "\"workload\"", + "\"platform\"", + "\"namespace\"", + "\"pod\"", + "\"container\"", + "\"containerId\"", + "\"imageRef\"", + "\"owner\"", + "\"kind\"", + "\"name\"", + "\"process\"", + "\"pid\"", + "\"entrypoint\"", + "\"entryTrace\"", + "\"loadedLibs\"", + "\"posture\"", + "\"imageSigned\"", + "\"sbomReferrer\"", + "\"attestation\"", + "\"uuid\"", + "\"verified\"", + "\"delta\"", + "\"baselineImageDigest\"", + "\"changedFiles\"", + "\"newBinaries\"", + "\"path\"", + "\"sha256\"", + "\"evidence\"", + "\"signal\"", + "\"value\"", + "\"annotations\"", + "\"source\"" + }; + + var cursor = -1; + foreach (var token in expectedOrder) + { + var position = json.IndexOf(token, cursor + 1, StringComparison.Ordinal); + Assert.True(position > cursor, $"Property token {token} not found in the expected order."); + cursor = position; + } + + Assert.DoesNotContain(" ", json, StringComparison.Ordinal); + Assert.StartsWith("{\"schemaVersion\"", json, StringComparison.Ordinal); + Assert.EndsWith("}}", json, StringComparison.Ordinal); + } + + [Fact] + public void ComputeMultihash_ProducesStableBase64UrlDigest() + { + var decision = AdmissionDecisionEnvelope.Create( + new AdmissionDecision + { + AdmissionId = "admission-123", + Namespace = "payments", + PodSpecDigest = "sha256:deadbeef", + Images = new[] + { + new AdmissionImageVerdict + { + Name = "ghcr.io/acme/api:1.2.3", + Resolved = "ghcr.io/acme/api@sha256:abcd", + Signed = true, + HasSbomReferrers = true, + PolicyVerdict = PolicyVerdict.Pass, + Reasons = Array.Empty(), + Rekor = new AdmissionRekorEvidence + { + Uuid = "xyz", + Verified = true + } + } + }, + Decision = AdmissionDecisionOutcome.Allow, + TtlSeconds = 300 + }, + ZastavaContractVersions.AdmissionDecision); + + var canonicalJson = ZastavaCanonicalJsonSerializer.Serialize(decision); + var expectedDigestBytes = SHA256.HashData(Encoding.UTF8.GetBytes(canonicalJson)); + var expected = $"sha256-{Convert.ToBase64String(expectedDigestBytes).TrimEnd('=').Replace('+', '-').Replace('/', '_')}"; + + var hash = ZastavaHashing.ComputeMultihash(decision); + + Assert.Equal(expected, hash); + + var sha512 = ZastavaHashing.ComputeMultihash(Encoding.UTF8.GetBytes(canonicalJson), "sha512"); + Assert.StartsWith("sha512-", sha512, StringComparison.Ordinal); + } +} diff --git a/src/StellaOps.Zastava.Core.Tests/StellaOps.Zastava.Core.Tests.csproj b/src/StellaOps.Zastava.Core.Tests/StellaOps.Zastava.Core.Tests.csproj new file mode 100644 index 00000000..6d31fc79 --- /dev/null +++ b/src/StellaOps.Zastava.Core.Tests/StellaOps.Zastava.Core.Tests.csproj @@ -0,0 +1,14 @@ + + + net10.0 + preview + enable + enable + + + + + + + + diff --git a/src/StellaOps.Zastava.Core/Contracts/AdmissionDecision.cs b/src/StellaOps.Zastava.Core/Contracts/AdmissionDecision.cs new file mode 100644 index 00000000..97c1e000 --- /dev/null +++ b/src/StellaOps.Zastava.Core/Contracts/AdmissionDecision.cs @@ -0,0 +1,86 @@ +namespace StellaOps.Zastava.Core.Contracts; + +/// +/// Envelope returned by the admission webhook to the Kubernetes API server. +/// +public sealed record class AdmissionDecisionEnvelope +{ + public required string SchemaVersion { get; init; } + + public required AdmissionDecision Decision { get; init; } + + public static AdmissionDecisionEnvelope Create(AdmissionDecision decision, ZastavaContractVersions.ContractVersion contract) + { + ArgumentNullException.ThrowIfNull(decision); + return new AdmissionDecisionEnvelope + { + SchemaVersion = contract.ToString(), + Decision = decision + }; + } + + public bool IsSupported() + => ZastavaContractVersions.IsAdmissionDecisionSupported(SchemaVersion); +} + +/// +/// Canonical admission decision payload. +/// +public sealed record class AdmissionDecision +{ + public required string AdmissionId { get; init; } + + [JsonPropertyName("namespace")] + public required string Namespace { get; init; } + + public required string PodSpecDigest { get; init; } + + public IReadOnlyList Images { get; init; } = Array.Empty(); + + public required AdmissionDecisionOutcome Decision { get; init; } + + public int TtlSeconds { get; init; } + + public IReadOnlyDictionary? Annotations { get; init; } +} + +public enum AdmissionDecisionOutcome +{ + Allow, + Deny +} + +public sealed record class AdmissionImageVerdict +{ + public required string Name { get; init; } + + public required string Resolved { get; init; } + + public bool Signed { get; init; } + + [JsonPropertyName("hasSbomReferrers")] + public bool HasSbomReferrers { get; init; } + + public PolicyVerdict PolicyVerdict { get; init; } + + public IReadOnlyList Reasons { get; init; } = Array.Empty(); + + public AdmissionRekorEvidence? Rekor { get; init; } + + public IReadOnlyDictionary? Metadata { get; init; } +} + +public enum PolicyVerdict +{ + Pass, + Warn, + Fail, + Error +} + +public sealed record class AdmissionRekorEvidence +{ + public string? Uuid { get; init; } + + public bool? Verified { get; init; } +} diff --git a/src/StellaOps.Zastava.Core/Contracts/RuntimeEvent.cs b/src/StellaOps.Zastava.Core/Contracts/RuntimeEvent.cs new file mode 100644 index 00000000..39d65683 --- /dev/null +++ b/src/StellaOps.Zastava.Core/Contracts/RuntimeEvent.cs @@ -0,0 +1,179 @@ +namespace StellaOps.Zastava.Core.Contracts; + +/// +/// Envelope published by the observer towards Scanner runtime ingestion. +/// +public sealed record class RuntimeEventEnvelope +{ + /// + /// Contract identifier consumed by negotiation logic (zastava.runtime.event@v1). + /// + public required string SchemaVersion { get; init; } + + /// + /// Runtime event payload. + /// + public required RuntimeEvent Event { get; init; } + + /// + /// Creates an envelope using the provided runtime contract version. + /// + public static RuntimeEventEnvelope Create(RuntimeEvent runtimeEvent, ZastavaContractVersions.ContractVersion contract) + { + ArgumentNullException.ThrowIfNull(runtimeEvent); + return new RuntimeEventEnvelope + { + SchemaVersion = contract.ToString(), + Event = runtimeEvent + }; + } + + /// + /// Checks whether the envelope schema is supported by the current runtime. + /// + public bool IsSupported() + => ZastavaContractVersions.IsRuntimeEventSupported(SchemaVersion); +} + +/// +/// Canonical runtime event emitted by the observer. +/// +public sealed record class RuntimeEvent +{ + public required string EventId { get; init; } + + public required DateTimeOffset When { get; init; } + + public required RuntimeEventKind Kind { get; init; } + + public required string Tenant { get; init; } + + public required string Node { get; init; } + + public required RuntimeEngine Runtime { get; init; } + + public required RuntimeWorkload Workload { get; init; } + + public RuntimeProcess? Process { get; init; } + + [JsonPropertyName("loadedLibs")] + public IReadOnlyList LoadedLibraries { get; init; } = Array.Empty(); + + public RuntimePosture? Posture { get; init; } + + public RuntimeDelta? Delta { get; init; } + + public IReadOnlyList Evidence { get; init; } = Array.Empty(); + + public IReadOnlyDictionary? Annotations { get; init; } +} + +public enum RuntimeEventKind +{ + ContainerStart, + ContainerStop, + Drift, + PolicyViolation, + AttestationStatus +} + +public sealed record class RuntimeEngine +{ + public required string Engine { get; init; } + + public string? Version { get; init; } +} + +public sealed record class RuntimeWorkload +{ + public required string Platform { get; init; } + + [JsonPropertyName("namespace")] + public string? Namespace { get; init; } + + public string? Pod { get; init; } + + public string? Container { get; init; } + + public string? ContainerId { get; init; } + + public string? ImageRef { get; init; } + + public RuntimeWorkloadOwner? Owner { get; init; } +} + +public sealed record class RuntimeWorkloadOwner +{ + public string? Kind { get; init; } + + public string? Name { get; init; } +} + +public sealed record class RuntimeProcess +{ + public int Pid { get; init; } + + public IReadOnlyList Entrypoint { get; init; } = Array.Empty(); + + [JsonPropertyName("entryTrace")] + public IReadOnlyList EntryTrace { get; init; } = Array.Empty(); +} + +public sealed record class RuntimeEntryTrace +{ + public string? File { get; init; } + + public int? Line { get; init; } + + public string? Op { get; init; } + + public string? Target { get; init; } +} + +public sealed record class RuntimeLoadedLibrary +{ + public required string Path { get; init; } + + public long? Inode { get; init; } + + public string? Sha256 { get; init; } +} + +public sealed record class RuntimePosture +{ + public bool? ImageSigned { get; init; } + + public string? SbomReferrer { get; init; } + + public RuntimeAttestation? Attestation { get; init; } +} + +public sealed record class RuntimeAttestation +{ + public string? Uuid { get; init; } + + public bool? Verified { get; init; } +} + +public sealed record class RuntimeDelta +{ + public string? BaselineImageDigest { get; init; } + + public IReadOnlyList ChangedFiles { get; init; } = Array.Empty(); + + public IReadOnlyList NewBinaries { get; init; } = Array.Empty(); +} + +public sealed record class RuntimeNewBinary +{ + public required string Path { get; init; } + + public string? Sha256 { get; init; } +} + +public sealed record class RuntimeEvidence +{ + public required string Signal { get; init; } + + public string? Value { get; init; } +} diff --git a/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs b/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs new file mode 100644 index 00000000..1751d2e0 --- /dev/null +++ b/src/StellaOps.Zastava.Core/Contracts/ZastavaContractVersions.cs @@ -0,0 +1,173 @@ +namespace StellaOps.Zastava.Core.Contracts; + +/// +/// Centralises schema identifiers and version negotiation rules for Zastava contracts. +/// +public static class ZastavaContractVersions +{ + /// + /// Current local runtime event contract (major version 1). + /// + public static ContractVersion RuntimeEvent { get; } = new("zastava.runtime.event", new Version(1, 0)); + + /// + /// Current local admission decision contract (major version 1). + /// + public static ContractVersion AdmissionDecision { get; } = new("zastava.admission.decision", new Version(1, 0)); + + /// + /// Determines whether the provided schema string is supported for runtime events. + /// + public static bool IsRuntimeEventSupported(string schemaVersion) + => ContractVersion.TryParse(schemaVersion, out var candidate) && candidate.IsCompatibleWith(RuntimeEvent); + + /// + /// Determines whether the provided schema string is supported for admission decisions. + /// + public static bool IsAdmissionDecisionSupported(string schemaVersion) + => ContractVersion.TryParse(schemaVersion, out var candidate) && candidate.IsCompatibleWith(AdmissionDecision); + + /// + /// Selects the newest runtime event contract shared between the local implementation and a remote peer. + /// + public static ContractVersion NegotiateRuntimeEvent(IEnumerable offeredSchemaVersions) + => Negotiate(RuntimeEvent, offeredSchemaVersions); + + /// + /// Selects the newest admission decision contract shared between the local implementation and a remote peer. + /// + public static ContractVersion NegotiateAdmissionDecision(IEnumerable offeredSchemaVersions) + => Negotiate(AdmissionDecision, offeredSchemaVersions); + + private static ContractVersion Negotiate(ContractVersion local, IEnumerable offered) + { + ArgumentNullException.ThrowIfNull(offered); + + ContractVersion? best = null; + foreach (var entry in offered) + { + if (!ContractVersion.TryParse(entry, out var candidate)) + { + continue; + } + + if (!candidate.Schema.Equals(local.Schema, StringComparison.Ordinal)) + { + continue; + } + + if (candidate.Version.Major != local.Version.Major) + { + continue; + } + + if (candidate.Version > local.Version) + { + continue; + } + + if (best is null || candidate.Version > best.Value.Version) + { + best = candidate; + } + } + + return best ?? local; + } + + /// + /// Represents a schema + semantic version pairing in canonical form. + /// + public readonly record struct ContractVersion + { + public ContractVersion(string schema, Version version) + { + if (string.IsNullOrWhiteSpace(schema)) + { + throw new ArgumentException("Schema cannot be null or whitespace.", nameof(schema)); + } + + Schema = schema.Trim(); + Version = new Version(Math.Max(version.Major, 0), Math.Max(version.Minor, 0)); + } + + /// + /// Schema identifier (e.g. zastava.runtime.event). + /// + public string Schema { get; } + + /// + /// Major/minor version recognised by the implementation. + /// + public Version Version { get; } + + /// + /// Canonical string representation (schema@vMajor.Minor). + /// + public override string ToString() + => $"{Schema}@v{Version.ToString(2, CultureInfo.InvariantCulture)}"; + + /// + /// Determines whether a remote contract is compatible with the local definition. + /// + public bool IsCompatibleWith(ContractVersion local) + { + if (!Schema.Equals(local.Schema, StringComparison.Ordinal)) + { + return false; + } + + if (Version.Major != local.Version.Major) + { + return false; + } + + return Version <= local.Version; + } + + /// + /// Attempts to parse a schema string in canonical format. + /// + public static bool TryParse(string? value, out ContractVersion contract) + { + contract = default; + if (string.IsNullOrWhiteSpace(value)) + { + return false; + } + + var trimmed = value.Trim(); + var separator = trimmed.IndexOf('@'); + if (separator < 0) + { + return false; + } + + var schema = trimmed[..separator]; + if (!schema.Contains('.', StringComparison.Ordinal)) + { + return false; + } + + var versionToken = trimmed[(separator + 1)..]; + if (versionToken.Length == 0) + { + return false; + } + + if (versionToken[0] is 'v' or 'V') + { + versionToken = versionToken[1..]; + } + + if (!Version.TryParse(versionToken, out var parsed)) + { + return false; + } + + var canonical = new Version(Math.Max(parsed.Major, 0), Math.Max(parsed.Minor, 0)); + contract = new ContractVersion(schema, canonical); + return true; + } + } +} diff --git a/src/StellaOps.Zastava.Core/GlobalUsings.cs b/src/StellaOps.Zastava.Core/GlobalUsings.cs new file mode 100644 index 00000000..845a5bda --- /dev/null +++ b/src/StellaOps.Zastava.Core/GlobalUsings.cs @@ -0,0 +1,10 @@ +global using System.Collections.Generic; +global using System.Collections.Immutable; +global using System.Diagnostics; +global using System.Diagnostics.Metrics; +global using System.Security.Cryptography; +global using System.Text; +global using System.Text.Json; +global using System.Text.Json.Serialization; +global using System.Text.Json.Serialization.Metadata; +global using System.Globalization; diff --git a/src/StellaOps.Zastava.Core/Hashing/ZastavaHashing.cs b/src/StellaOps.Zastava.Core/Hashing/ZastavaHashing.cs new file mode 100644 index 00000000..7fff0904 --- /dev/null +++ b/src/StellaOps.Zastava.Core/Hashing/ZastavaHashing.cs @@ -0,0 +1,59 @@ +using StellaOps.Zastava.Core.Serialization; + +namespace StellaOps.Zastava.Core.Hashing; + +/// +/// Produces deterministic multihashes for runtime and admission payloads. +/// +public static class ZastavaHashing +{ + public const string DefaultAlgorithm = "sha256"; + + /// + /// Serialises the payload using canonical options and computes a multihash string. + /// + public static string ComputeMultihash(T value, string? algorithm = null) + { + ArgumentNullException.ThrowIfNull(value); + var bytes = ZastavaCanonicalJsonSerializer.SerializeToUtf8Bytes(value); + return ComputeMultihash(bytes, algorithm); + } + + /// + /// Computes a multihash string from the provided payload. + /// + public static string ComputeMultihash(ReadOnlySpan payload, string? algorithm = null) + { + var normalized = NormalizeAlgorithm(algorithm); + var digest = normalized switch + { + "sha256" => SHA256.HashData(payload), + "sha512" => SHA512.HashData(payload), + _ => throw new NotSupportedException($"Hash algorithm '{normalized}' is not supported.") + }; + + return $"{normalized}-{ToBase64Url(digest)}"; + } + + private static string NormalizeAlgorithm(string? algorithm) + { + if (string.IsNullOrWhiteSpace(algorithm)) + { + return DefaultAlgorithm; + } + + var normalized = algorithm.Trim().ToLowerInvariant(); + return normalized switch + { + "sha-256" or "sha256" => "sha256", + "sha-512" or "sha512" => "sha512", + _ => normalized + }; + } + + private static string ToBase64Url(ReadOnlySpan bytes) + { + var base64 = Convert.ToBase64String(bytes); + return base64.TrimEnd('=').Replace('+', '-').Replace('/', '_'); + } +} diff --git a/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs b/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs new file mode 100644 index 00000000..e4219303 --- /dev/null +++ b/src/StellaOps.Zastava.Core/Serialization/ZastavaCanonicalJsonSerializer.cs @@ -0,0 +1,110 @@ +namespace StellaOps.Zastava.Core.Serialization; + +/// +/// Deterministic serializer used for runtime/admission contracts. +/// +public static class ZastavaCanonicalJsonSerializer +{ + private static readonly JsonSerializerOptions CompactOptions = CreateOptions(writeIndented: false); + private static readonly JsonSerializerOptions PrettyOptions = CreateOptions(writeIndented: true); + + private static readonly IReadOnlyDictionary PropertyOrderOverrides = new Dictionary + { + { typeof(RuntimeEventEnvelope), new[] { "schemaVersion", "event" } }, + { typeof(RuntimeEvent), new[] { "eventId", "when", "kind", "tenant", "node", "runtime", "workload", "process", "loadedLibs", "posture", "delta", "evidence", "annotations" } }, + { typeof(RuntimeEngine), new[] { "engine", "version" } }, + { typeof(RuntimeWorkload), new[] { "platform", "namespace", "pod", "container", "containerId", "imageRef", "owner" } }, + { typeof(RuntimeWorkloadOwner), new[] { "kind", "name" } }, + { typeof(RuntimeProcess), new[] { "pid", "entrypoint", "entryTrace" } }, + { typeof(RuntimeEntryTrace), new[] { "file", "line", "op", "target" } }, + { typeof(RuntimeLoadedLibrary), new[] { "path", "inode", "sha256" } }, + { typeof(RuntimePosture), new[] { "imageSigned", "sbomReferrer", "attestation" } }, + { typeof(RuntimeAttestation), new[] { "uuid", "verified" } }, + { typeof(RuntimeDelta), new[] { "baselineImageDigest", "changedFiles", "newBinaries" } }, + { typeof(RuntimeNewBinary), new[] { "path", "sha256" } }, + { typeof(RuntimeEvidence), new[] { "signal", "value" } }, + { typeof(AdmissionDecisionEnvelope), new[] { "schemaVersion", "decision" } }, + { typeof(AdmissionDecision), new[] { "admissionId", "namespace", "podSpecDigest", "images", "decision", "ttlSeconds", "annotations" } }, + { typeof(AdmissionImageVerdict), new[] { "name", "resolved", "signed", "hasSbomReferrers", "policyVerdict", "reasons", "rekor", "metadata" } }, + { typeof(AdmissionRekorEvidence), new[] { "uuid", "verified" } }, + { typeof(ZastavaContractVersions.ContractVersion), new[] { "schema", "version" } } + }; + + public static string Serialize(T value) + => JsonSerializer.Serialize(value, CompactOptions); + + public static string SerializeIndented(T value) + => JsonSerializer.Serialize(value, PrettyOptions); + + public static byte[] SerializeToUtf8Bytes(T value) + => JsonSerializer.SerializeToUtf8Bytes(value, CompactOptions); + + public static T Deserialize(string json) + => JsonSerializer.Deserialize(json, CompactOptions)!; + + private static JsonSerializerOptions CreateOptions(bool writeIndented) + { + var options = new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DictionaryKeyPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = writeIndented, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + var baselineResolver = options.TypeInfoResolver ?? new DefaultJsonTypeInfoResolver(); + options.TypeInfoResolver = new DeterministicTypeInfoResolver(baselineResolver); + options.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, allowIntegerValues: false)); + return options; + } + + private sealed class DeterministicTypeInfoResolver : IJsonTypeInfoResolver + { + private readonly IJsonTypeInfoResolver inner; + + public DeterministicTypeInfoResolver(IJsonTypeInfoResolver inner) + { + this.inner = inner ?? throw new ArgumentNullException(nameof(inner)); + } + + public JsonTypeInfo GetTypeInfo(Type type, JsonSerializerOptions options) + { + var info = inner.GetTypeInfo(type, options); + if (info is null) + { + throw new InvalidOperationException($"Unable to resolve JsonTypeInfo for '{type}'."); + } + + if (info.Kind is JsonTypeInfoKind.Object && info.Properties is { Count: > 1 }) + { + var ordered = info.Properties + .OrderBy(property => GetPropertyOrder(type, property.Name)) + .ThenBy(property => property.Name, StringComparer.Ordinal) + .ToArray(); + + info.Properties.Clear(); + foreach (var property in ordered) + { + info.Properties.Add(property); + } + } + + return info; + } + + private static int GetPropertyOrder(Type type, string propertyName) + { + if (PropertyOrderOverrides.TryGetValue(type, out var order)) + { + var index = Array.IndexOf(order, propertyName); + if (index >= 0) + { + return index; + } + } + + return int.MaxValue; + } + } +} diff --git a/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj b/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj new file mode 100644 index 00000000..783ae2d3 --- /dev/null +++ b/src/StellaOps.Zastava.Core/StellaOps.Zastava.Core.csproj @@ -0,0 +1,20 @@ + + + net10.0 + preview + enable + enable + true + + + + + + + + + + + + + diff --git a/src/StellaOps.Zastava.Core/TASKS.md b/src/StellaOps.Zastava.Core/TASKS.md new file mode 100644 index 00000000..431e7aab --- /dev/null +++ b/src/StellaOps.Zastava.Core/TASKS.md @@ -0,0 +1,10 @@ +# Zastava Core Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| ZASTAVA-CORE-12-201 | DOING (2025-10-19) | Zastava Core Guild | — | Define runtime event/admission DTOs, hashing helpers, and versioning strategy. | DTOs cover runtime events and admission verdict envelopes with canonical JSON schema; hashing helpers accept payloads and yield deterministic multihash outputs; version negotiation rules documented and exercised by serialization tests. | +| ZASTAVA-CORE-12-202 | DOING (2025-10-19) | Zastava Core Guild | — | Provide configuration/logging/metrics utilities shared by Observer/Webhook. | Shared options bind from configuration with validation; logging scopes/metrics exporters registered via reusable DI extension; integration test host demonstrates Observer/Webhook consumption with deterministic instrumentation. | +| ZASTAVA-CORE-12-203 | DOING (2025-10-19) | Zastava Core Guild | — | Authority client helpers, OpTok caching, and security guardrails for runtime services. | Typed Authority client surfaces OpTok retrieval + renewal with configurable cache; guardrails enforce DPoP/mTLS expectations and emit structured audit logs; negative-path tests cover expired/invalid tokens and configuration toggles. | +| ZASTAVA-OPS-12-204 | DOING (2025-10-19) | Zastava Core Guild | — | Operational runbooks, alert rules, and dashboard exports for runtime plane. | Runbooks capture install/upgrade/rollback + incident handling; alert rules and dashboard JSON exported for Prometheus/Grafana bundle; docs reference Offline Kit packaging and verification checklist. | + +> Remark (2025-10-19): Prerequisites reviewed—none outstanding. ZASTAVA-CORE-12-201, ZASTAVA-CORE-12-202, ZASTAVA-CORE-12-203, and ZASTAVA-OPS-12-204 moved to DOING for Wave 0 kickoff. diff --git a/src/StellaOps.Zastava.Observer/TASKS.md b/src/StellaOps.Zastava.Observer/TASKS.md new file mode 100644 index 00000000..99c80e7c --- /dev/null +++ b/src/StellaOps.Zastava.Observer/TASKS.md @@ -0,0 +1,9 @@ +# Zastava Observer Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| ZASTAVA-OBS-12-001 | TODO | Zastava Observer Guild | ZASTAVA-CORE-12-201 | Build container lifecycle watcher that tails CRI (containerd/cri-o/docker) events and emits deterministic runtime records with buffering + backoff. | Fixture cluster produces start/stop events with stable ordering, jitter/backoff tested, metrics/logging wired. | +| ZASTAVA-OBS-12-002 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-001 | Capture entrypoint traces and loaded libraries, hashing binaries and correlating to SBOM baseline per architecture sections 2.1 and 10. | EntryTrace parser covers shell/python/node launchers, loaded library hashes recorded, fixtures assert linkage to SBOM usage view. | +| ZASTAVA-OBS-12-003 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Implement runtime posture checks (signature/SBOM/attestation presence) with offline caching and warning surfaces. | Observer marks posture status, caches refresh across restarts, integration tests prove offline tolerance. | +| ZASTAVA-OBS-12-004 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Batch `/runtime/events` submissions with disk-backed buffer, rate limits, and deterministic envelopes. | Buffered submissions survive restart, rate-limits enforced in tests, JSON envelopes match schema in docs/events. | +| ZASTAVA-OBS-17-005 | TODO | Zastava Observer Guild | ZASTAVA-OBS-12-002 | Collect GNU build-id for ELF processes and attach it to emitted runtime events to enable symbol lookup + debug-store correlation. | Observer reads build-id via `/proc//exe`/notes without pausing workloads, runtime events include `buildId` field, fixtures cover glibc/musl images, docs updated with retrieval notes. | diff --git a/src/StellaOps.Zastava.Webhook.Tests/Certificates/SecretFileCertificateSourceTests.cs b/src/StellaOps.Zastava.Webhook.Tests/Certificates/SecretFileCertificateSourceTests.cs new file mode 100644 index 00000000..8380491a --- /dev/null +++ b/src/StellaOps.Zastava.Webhook.Tests/Certificates/SecretFileCertificateSourceTests.cs @@ -0,0 +1,80 @@ +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging.Abstractions; +using StellaOps.Zastava.Webhook.Certificates; +using StellaOps.Zastava.Webhook.Configuration; +using Xunit; + +namespace StellaOps.Zastava.Webhook.Tests.Certificates; + +public sealed class SecretFileCertificateSourceTests +{ + [Fact] + public void LoadCertificate_FromPemPair_Succeeds() + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=zastava-webhook", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + using var certificateWithKey = certificate.CopyWithPrivateKey(rsa); + + var certificatePath = Path.GetTempFileName(); + var privateKeyPath = Path.GetTempFileName(); + + try + { + File.WriteAllText(certificatePath, certificateWithKey.ExportCertificatePem()); + using var exportRsa = certificateWithKey.GetRSAPrivateKey() ?? throw new InvalidOperationException("Missing RSA private key"); + var privateKeyPem = PemEncoding.Write("PRIVATE KEY", exportRsa.ExportPkcs8PrivateKey()); + File.WriteAllText(privateKeyPath, privateKeyPem); + + var source = new SecretFileCertificateSource(NullLogger.Instance); + var options = new ZastavaWebhookTlsOptions + { + Mode = ZastavaWebhookTlsMode.Secret, + CertificatePath = certificatePath, + PrivateKeyPath = privateKeyPath + }; + + using var loaded = source.LoadCertificate(options); + + Assert.Equal(certificateWithKey.Thumbprint, loaded.Thumbprint); + Assert.NotNull(loaded.GetRSAPrivateKey()); + } + finally + { + File.Delete(certificatePath); + File.Delete(privateKeyPath); + } + } + + [Fact] + public void LoadCertificate_FromPfx_Succeeds() + { + using var rsa = RSA.Create(2048); + var request = new CertificateRequest("CN=zastava-webhook", rsa, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + using var certificate = request.CreateSelfSigned(DateTimeOffset.UtcNow.AddMinutes(-5), DateTimeOffset.UtcNow.AddHours(1)); + using var certificateWithKey = certificate.CopyWithPrivateKey(rsa); + + var pfxPath = Path.GetTempFileName(); + try + { + var pfxBytes = certificateWithKey.Export(X509ContentType.Pfx, "test"); + File.WriteAllBytes(pfxPath, pfxBytes); + + var source = new SecretFileCertificateSource(NullLogger.Instance); + var options = new ZastavaWebhookTlsOptions + { + Mode = ZastavaWebhookTlsMode.Secret, + PfxPath = pfxPath, + PfxPassword = "test" + }; + + using var loaded = source.LoadCertificate(options); + Assert.Equal(certificateWithKey.Thumbprint, loaded.Thumbprint); + } + finally + { + File.Delete(pfxPath); + } + } +} diff --git a/src/StellaOps.Zastava.Webhook.Tests/Certificates/WebhookCertificateProviderTests.cs b/src/StellaOps.Zastava.Webhook.Tests/Certificates/WebhookCertificateProviderTests.cs new file mode 100644 index 00000000..b707e4e6 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook.Tests/Certificates/WebhookCertificateProviderTests.cs @@ -0,0 +1,43 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StellaOps.Zastava.Webhook.Certificates; +using StellaOps.Zastava.Webhook.Configuration; +using Xunit; + +namespace StellaOps.Zastava.Webhook.Tests.Certificates; + +public sealed class WebhookCertificateProviderTests +{ + [Fact] + public void Provider_UsesMatchingSource() + { + var options = Options.Create(new ZastavaWebhookOptions + { + Tls = new ZastavaWebhookTlsOptions + { + Mode = ZastavaWebhookTlsMode.Secret, + CertificatePath = "/tmp/cert.pem", + PrivateKeyPath = "/tmp/key.pem" + } + }); + + var source = new ThrowingCertificateSource(); + var provider = new WebhookCertificateProvider(options, new[] { source }, NullLogger.Instance); + + Assert.Throws(() => provider.GetCertificate()); + Assert.True(source.Requested); + } + + private sealed class ThrowingCertificateSource : IWebhookCertificateSource + { + public bool Requested { get; private set; } + + public bool CanHandle(ZastavaWebhookTlsMode mode) => true; + + public System.Security.Cryptography.X509Certificates.X509Certificate2 LoadCertificate(ZastavaWebhookTlsOptions options) + { + Requested = true; + throw new InvalidOperationException("test"); + } + } +} diff --git a/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj b/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj new file mode 100644 index 00000000..2294fcc2 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook.Tests/StellaOps.Zastava.Webhook.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + preview + enable + enable + false + true + false + + + + + + + + + + diff --git a/src/StellaOps.Zastava.Webhook/Authority/AuthorityTokenProvider.cs b/src/StellaOps.Zastava.Webhook/Authority/AuthorityTokenProvider.cs new file mode 100644 index 00000000..585e8f48 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Authority/AuthorityTokenProvider.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Options; +using StellaOps.Zastava.Webhook.Configuration; + +namespace StellaOps.Zastava.Webhook.Authority; + +public interface IAuthorityTokenProvider +{ + ValueTask GetTokenAsync(CancellationToken cancellationToken = default); +} + +public sealed record AuthorityToken(string Value, DateTimeOffset? ExpiresAtUtc); + +public sealed class StaticAuthorityTokenProvider : IAuthorityTokenProvider +{ + private readonly ZastavaWebhookAuthorityOptions _options; + private readonly ILogger _logger; + private AuthorityToken? _cachedToken; + + public StaticAuthorityTokenProvider( + IOptionsMonitor options, + ILogger logger) + { + _options = options.CurrentValue.Authority; + _logger = logger; + } + + public ValueTask GetTokenAsync(CancellationToken cancellationToken = default) + { + if (_cachedToken is { } token) + { + return ValueTask.FromResult(token); + } + + var value = !string.IsNullOrWhiteSpace(_options.StaticTokenValue) + ? _options.StaticTokenValue + : LoadTokenFromFile(_options.StaticTokenPath); + + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException("No Authority token configured. Provide either 'StaticTokenValue' or 'StaticTokenPath'."); + } + + token = new AuthorityToken(value.Trim(), ExpiresAtUtc: null); + _cachedToken = token; + _logger.LogInformation("Loaded static Authority token (length {Length}).", token.Value.Length); + return ValueTask.FromResult(token); + } + + private string LoadTokenFromFile(string? path) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new InvalidOperationException("Authority static token path not set."); + } + + if (!File.Exists(path)) + { + throw new FileNotFoundException("Authority static token file not found.", path); + } + + return File.ReadAllText(path); + } +} + +public sealed class AuthorityTokenHealthCheck : IHealthCheck +{ + private readonly IAuthorityTokenProvider _tokenProvider; + private readonly ILogger _logger; + + public AuthorityTokenHealthCheck(IAuthorityTokenProvider tokenProvider, ILogger logger) + { + _tokenProvider = tokenProvider; + _logger = logger; + } + + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + var token = await _tokenProvider.GetTokenAsync(cancellationToken); + return HealthCheckResult.Healthy("Authority token acquired.", data: new Dictionary + { + ["expiresAtUtc"] = token.ExpiresAtUtc?.ToString("O") ?? "static" + }); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to obtain Authority token."); + return HealthCheckResult.Unhealthy("Failed to obtain Authority token.", ex); + } + } +} diff --git a/src/StellaOps.Zastava.Webhook/Certificates/CsrCertificateSource.cs b/src/StellaOps.Zastava.Webhook/Certificates/CsrCertificateSource.cs new file mode 100644 index 00000000..02495566 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Certificates/CsrCertificateSource.cs @@ -0,0 +1,25 @@ +using System.Security.Cryptography.X509Certificates; +using StellaOps.Zastava.Webhook.Configuration; + +namespace StellaOps.Zastava.Webhook.Certificates; + +/// +/// Placeholder implementation for CSR-based certificate provisioning. +/// +public sealed class CsrCertificateSource : IWebhookCertificateSource +{ + private readonly ILogger _logger; + + public CsrCertificateSource(ILogger logger) + { + _logger = logger; + } + + public bool CanHandle(ZastavaWebhookTlsMode mode) => mode == ZastavaWebhookTlsMode.CertificateSigningRequest; + + public X509Certificate2 LoadCertificate(ZastavaWebhookTlsOptions options) + { + _logger.LogError("CSR certificate mode is not implemented yet. Configuration requested CSR mode."); + throw new NotSupportedException("CSR certificate provisioning is not implemented (tracked by ZASTAVA-WEBHOOK-12-101)."); + } +} diff --git a/src/StellaOps.Zastava.Webhook/Certificates/IWebhookCertificateProvider.cs b/src/StellaOps.Zastava.Webhook/Certificates/IWebhookCertificateProvider.cs new file mode 100644 index 00000000..6346a80d --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Certificates/IWebhookCertificateProvider.cs @@ -0,0 +1,49 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Options; +using StellaOps.Zastava.Webhook.Configuration; + +namespace StellaOps.Zastava.Webhook.Certificates; + +public interface IWebhookCertificateProvider +{ + X509Certificate2 GetCertificate(); +} + +public sealed class WebhookCertificateProvider : IWebhookCertificateProvider +{ + private readonly ILogger _logger; + private readonly ZastavaWebhookTlsOptions _options; + private readonly Lazy _certificate; + private readonly IWebhookCertificateSource _certificateSource; + + public WebhookCertificateProvider( + IOptions options, + IEnumerable certificateSources, + ILogger logger) + { + _logger = logger; + _options = options.Value.Tls; + _certificateSource = certificateSources.FirstOrDefault(source => source.CanHandle(_options.Mode)) + ?? throw new InvalidOperationException($"No certificate source registered for mode {_options.Mode}."); + + _certificate = new Lazy(LoadCertificate, LazyThreadSafetyMode.ExecutionAndPublication); + } + + public X509Certificate2 GetCertificate() => _certificate.Value; + + private X509Certificate2 LoadCertificate() + { + _logger.LogInformation("Loading webhook TLS certificate using {Mode} mode.", _options.Mode); + var certificate = _certificateSource.LoadCertificate(_options); + _logger.LogInformation("Loaded webhook TLS certificate with subject {Subject} and thumbprint {Thumbprint}.", + certificate.Subject, certificate.Thumbprint); + return certificate; + } +} + +public interface IWebhookCertificateSource +{ + bool CanHandle(ZastavaWebhookTlsMode mode); + + X509Certificate2 LoadCertificate(ZastavaWebhookTlsOptions options); +} diff --git a/src/StellaOps.Zastava.Webhook/Certificates/SecretFileCertificateSource.cs b/src/StellaOps.Zastava.Webhook/Certificates/SecretFileCertificateSource.cs new file mode 100644 index 00000000..2de3a3a4 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Certificates/SecretFileCertificateSource.cs @@ -0,0 +1,98 @@ +using System.Security.Cryptography.X509Certificates; +using Microsoft.Extensions.Logging; +using StellaOps.Zastava.Webhook.Configuration; + +namespace StellaOps.Zastava.Webhook.Certificates; + +public sealed class SecretFileCertificateSource : IWebhookCertificateSource +{ + private readonly ILogger _logger; + + public SecretFileCertificateSource(ILogger logger) + { + _logger = logger; + } + + public bool CanHandle(ZastavaWebhookTlsMode mode) => mode == ZastavaWebhookTlsMode.Secret; + + public X509Certificate2 LoadCertificate(ZastavaWebhookTlsOptions options) + { + if (options is null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (!string.IsNullOrWhiteSpace(options.PfxPath)) + { + return LoadFromPfx(options.PfxPath, options.PfxPassword); + } + + if (string.IsNullOrWhiteSpace(options.CertificatePath) || string.IsNullOrWhiteSpace(options.PrivateKeyPath)) + { + throw new InvalidOperationException("TLS mode 'Secret' requires either a PFX bundle or both PEM certificate and private key paths."); + } + + if (!File.Exists(options.CertificatePath)) + { + throw new FileNotFoundException("Webhook certificate file not found.", options.CertificatePath); + } + + if (!File.Exists(options.PrivateKeyPath)) + { + throw new FileNotFoundException("Webhook certificate private key file not found.", options.PrivateKeyPath); + } + + try + { + var certificate = X509Certificate2.CreateFromPemFile(options.CertificatePath, options.PrivateKeyPath) + .WithExportablePrivateKey(); + + _logger.LogDebug("Loaded certificate {Subject} from PEM secret files.", certificate.Subject); + return certificate; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load webhook certificate from PEM files {CertPath} / {KeyPath}.", + options.CertificatePath, options.PrivateKeyPath); + throw; + } + } + + private X509Certificate2 LoadFromPfx(string pfxPath, string? password) + { + if (!File.Exists(pfxPath)) + { + throw new FileNotFoundException("Webhook certificate PFX bundle not found.", pfxPath); + } + + try + { + var storageFlags = X509KeyStorageFlags.MachineKeySet | X509KeyStorageFlags.EphemeralKeySet; + var certificate = X509CertificateLoader.LoadPkcs12FromFile(pfxPath, password, storageFlags); + _logger.LogDebug("Loaded certificate {Subject} from PFX bundle.", certificate.Subject); + return certificate; + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load webhook certificate from PFX bundle {PfxPath}.", pfxPath); + throw; + } + } +} + +internal static class X509Certificate2Extensions +{ + public static X509Certificate2 WithExportablePrivateKey(this X509Certificate2 certificate) + { + // Ensure the private key is exportable for Kestrel; CreateFromPemFile returns a temporary key material otherwise. + using var rsa = certificate.GetRSAPrivateKey(); + if (rsa is null) + { + return certificate; + } + + var certificateWithKey = certificate.CopyWithPrivateKey(rsa); + certificate.Dispose(); + return certificateWithKey; + } +} diff --git a/src/StellaOps.Zastava.Webhook/Certificates/WebhookCertificateHealthCheck.cs b/src/StellaOps.Zastava.Webhook/Certificates/WebhookCertificateHealthCheck.cs new file mode 100644 index 00000000..3a3a3a0d --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Certificates/WebhookCertificateHealthCheck.cs @@ -0,0 +1,56 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace StellaOps.Zastava.Webhook.Certificates; + +public sealed class WebhookCertificateHealthCheck : IHealthCheck +{ + private readonly IWebhookCertificateProvider _certificateProvider; + private readonly ILogger _logger; + private readonly TimeSpan _expiryThreshold = TimeSpan.FromDays(7); + + public WebhookCertificateHealthCheck( + IWebhookCertificateProvider certificateProvider, + ILogger logger) + { + _certificateProvider = certificateProvider; + _logger = logger; + } + + public Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + var certificate = _certificateProvider.GetCertificate(); + var expires = certificate.NotAfter.ToUniversalTime(); + var remaining = expires - DateTimeOffset.UtcNow; + + if (remaining <= TimeSpan.Zero) + { + return Task.FromResult(HealthCheckResult.Unhealthy("Webhook certificate expired.", data: new Dictionary + { + ["expiresAtUtc"] = expires.ToString("O") + })); + } + + if (remaining <= _expiryThreshold) + { + return Task.FromResult(HealthCheckResult.Degraded("Webhook certificate nearing expiry.", data: new Dictionary + { + ["expiresAtUtc"] = expires.ToString("O"), + ["daysRemaining"] = remaining.TotalDays + })); + } + + return Task.FromResult(HealthCheckResult.Healthy("Webhook certificate valid.", data: new Dictionary + { + ["expiresAtUtc"] = expires.ToString("O"), + ["daysRemaining"] = remaining.TotalDays + })); + } + catch (Exception ex) + { + _logger.LogError(ex, "Failed to load webhook certificate."); + return Task.FromResult(HealthCheckResult.Unhealthy("Failed to load webhook certificate.", ex)); + } + } +} diff --git a/src/StellaOps.Zastava.Webhook/Configuration/ZastavaWebhookOptions.cs b/src/StellaOps.Zastava.Webhook/Configuration/ZastavaWebhookOptions.cs new file mode 100644 index 00000000..2d152854 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Configuration/ZastavaWebhookOptions.cs @@ -0,0 +1,146 @@ +using System.ComponentModel.DataAnnotations; + +namespace StellaOps.Zastava.Webhook.Configuration; + +public sealed class ZastavaWebhookOptions +{ + public const string SectionName = "zastava:webhook"; + + [Required] + public ZastavaWebhookTlsOptions Tls { get; init; } = new(); + + [Required] + public ZastavaWebhookAuthorityOptions Authority { get; init; } = new(); + + [Required] + public ZastavaWebhookAdmissionOptions Admission { get; init; } = new(); +} + +public sealed class ZastavaWebhookAdmissionOptions +{ + /// + /// Namespaces that default to fail-open when backend calls fail. + /// + public HashSet FailOpenNamespaces { get; init; } = new(StringComparer.Ordinal); + + /// + /// Namespaces that must fail-closed even if the global default is fail-open. + /// + public HashSet FailClosedNamespaces { get; init; } = new(StringComparer.Ordinal); + + /// + /// Global fail-open toggle. When true, namespaces not in will allow requests on backend failures. + /// + public bool FailOpenByDefault { get; init; } + + /// + /// Enables tag resolution to immutable digests when set. + /// + public bool ResolveTags { get; init; } = true; + + /// + /// Optional cache seed path for pre-computed runtime verdicts. + /// + public string? CacheSeedPath { get; init; } +} + +public enum ZastavaWebhookTlsMode +{ + Secret = 0, + CertificateSigningRequest = 1 +} + +public sealed class ZastavaWebhookTlsOptions +{ + [Required] + public ZastavaWebhookTlsMode Mode { get; init; } = ZastavaWebhookTlsMode.Secret; + + /// + /// PEM certificate path when using . + /// + public string? CertificatePath { get; init; } + + /// + /// PEM private key path when using . + /// + public string? PrivateKeyPath { get; init; } + + /// + /// Optional PFX bundle path; takes precedence over PEM values when provided. + /// + public string? PfxPath { get; init; } + + /// + /// Optional password for the PFX bundle. + /// + public string? PfxPassword { get; init; } + + /// + /// Optional CA bundle path to present to Kubernetes when configuring webhook registration. + /// + public string? CaBundlePath { get; init; } + + /// + /// CSR related settings when equals . + /// + public ZastavaWebhookTlsCsrOptions Csr { get; init; } = new(); +} + +public sealed class ZastavaWebhookTlsCsrOptions +{ + /// + /// Kubernetes namespace that owns the CertificateSigningRequest object. + /// + [Required(AllowEmptyStrings = false)] + public string Namespace { get; init; } = "stellaops"; + + /// + /// CSR object name; defaults to zastava-webhook. + /// + [Required(AllowEmptyStrings = false)] + [MaxLength(253)] + public string Name { get; init; } = "zastava-webhook"; + + /// + /// DNS names placed in the CSR subjectAltName. + /// + [MinLength(1)] + public string[] DnsNames { get; init; } = Array.Empty(); + + /// + /// Where the signed certificate is persisted after approval (mounted emptyDir). + /// + [Required(AllowEmptyStrings = false)] + public string PersistPath { get; init; } = "/var/run/zastava-webhook/certs"; +} + +public sealed class ZastavaWebhookAuthorityOptions +{ + /// + /// Authority issuer URL for token acquisition. + /// + [Required(AllowEmptyStrings = false)] + public Uri Issuer { get; init; } = new("https://authority.internal"); + + /// + /// Audience that tokens must target. + /// + [MinLength(1)] + public string[] Audience { get; init; } = new[] { "scanner", "zastava" }; + + /// + /// Optional path to static OpTok for bootstrap environments. + /// + public string? StaticTokenPath { get; init; } + + /// + /// Optional literal token value (test only). Takes precedence over . + /// + public string? StaticTokenValue { get; init; } + + /// + /// Interval for refreshing cached tokens before expiry. + /// + [Range(typeof(double), "1", "3600")] + public double RefreshSkewSeconds { get; init; } = TimeSpan.FromMinutes(5).TotalSeconds; +} diff --git a/src/StellaOps.Zastava.Webhook/DependencyInjection/ServiceCollectionExtensions.cs b/src/StellaOps.Zastava.Webhook/DependencyInjection/ServiceCollectionExtensions.cs new file mode 100644 index 00000000..5bf3436c --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/DependencyInjection/ServiceCollectionExtensions.cs @@ -0,0 +1,33 @@ +using Microsoft.Extensions.DependencyInjection.Extensions; +using StellaOps.Zastava.Webhook.Authority; +using StellaOps.Zastava.Webhook.Certificates; +using StellaOps.Zastava.Webhook.Configuration; +using StellaOps.Zastava.Webhook.Hosting; + +namespace Microsoft.Extensions.DependencyInjection; + +public static class ServiceCollectionExtensions +{ + public static IServiceCollection AddZastavaWebhook(this IServiceCollection services, IConfiguration configuration) + { + services.AddOptions() + .Bind(configuration.GetSection(ZastavaWebhookOptions.SectionName)) + .ValidateDataAnnotations() + .ValidateOnStart(); + + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddSingleton(); + services.TryAddSingleton(); + + services.TryAddSingleton(); + services.TryAddSingleton(); + services.AddHostedService(); + + services.AddHealthChecks() + .AddCheck("webhook_tls") + .AddCheck("authority_token"); + + return services; + } +} diff --git a/src/StellaOps.Zastava.Webhook/Hosting/StartupValidationHostedService.cs b/src/StellaOps.Zastava.Webhook/Hosting/StartupValidationHostedService.cs new file mode 100644 index 00000000..f1c9473d --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Hosting/StartupValidationHostedService.cs @@ -0,0 +1,31 @@ +using StellaOps.Zastava.Webhook.Authority; +using StellaOps.Zastava.Webhook.Certificates; + +namespace StellaOps.Zastava.Webhook.Hosting; + +public sealed class StartupValidationHostedService : IHostedService +{ + private readonly IWebhookCertificateProvider _certificateProvider; + private readonly IAuthorityTokenProvider _authorityTokenProvider; + private readonly ILogger _logger; + + public StartupValidationHostedService( + IWebhookCertificateProvider certificateProvider, + IAuthorityTokenProvider authorityTokenProvider, + ILogger logger) + { + _certificateProvider = certificateProvider; + _authorityTokenProvider = authorityTokenProvider; + _logger = logger; + } + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Running webhook startup validation."); + _certificateProvider.GetCertificate(); + await _authorityTokenProvider.GetTokenAsync(cancellationToken); + _logger.LogInformation("Webhook startup validation complete."); + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} diff --git a/src/StellaOps.Zastava.Webhook/IMPLEMENTATION_PLAN.md b/src/StellaOps.Zastava.Webhook/IMPLEMENTATION_PLAN.md new file mode 100644 index 00000000..cb520d71 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/IMPLEMENTATION_PLAN.md @@ -0,0 +1,105 @@ +# Zastava Webhook · Wave 0 Implementation Notes + +> Authored 2025-10-19 by Zastava Webhook Guild. + +## ZASTAVA-WEBHOOK-12-101 — Admission Controller Host (TLS bootstrap + Authority auth) + +**Objectives** +- Provide a deterministic, restart-safe .NET 10 host that exposes a Kubernetes ValidatingAdmissionWebhook endpoint. +- Load serving certificates at start-up only (per restart-time plug-in rule) and surface reload guidance via documentation rather than hot-reload. +- Authenticate outbound calls to Authority/Scanner using OpTok + DPoP as defined in `docs/ARCHITECTURE_ZASTAVA.md`. + +**Plan** +1. **Project scaffolding** + - Create `StellaOps.Zastava.Webhook` project with minimal API pipeline (`Program.cs`, `Startup` equivalent via extension methods). + - Reference shared helpers once `ZASTAVA-CORE-12-201/202` land; temporarily stub interfaces behind `IZastavaAdmissionRequest`/`IZastavaAdmissionResult`. +2. **TLS bootstrap** + - Support two certificate sources: + 1. Mounted secret path (`/var/run/secrets/zastava-webhook/tls.{crt,key}`) with optional CA bundle. + 2. CSR workflow: generate CSR + private key, submit to Kubernetes Certificates API when `admission.tls.autoApprove` enabled; persist signed cert/key to mounted emptyDir for reuse across replicas. + - Validate cert/key pair on boot; abort start-up if invalid to preserve deterministic behavior. + - Configure Kestrel for mutual TLS off (API Server already provides client auth) but enforce minimum TLS 1.3, strong cipher suite list, HTTP/2 disabled (K8s uses HTTP/1.1). +3. **Authority auth** + - Bootstrap Authority client via shared DI extension (`AuthorityClientBuilder` once exposed); until then, placeholder `IAuthorityTokenSource` reading static OpTok from secret for smoke testing. + - Implement DPoP proof generator bound to webhook host keypair (prefer Ed25519) with configurable rotation period (default 24h, triggered at restart). + - Add background health check verifying token freshness and surfacing metrics (`zastava.authority_token_renew_failures_total`). +4. **Hosting concerns** + - Configure structured logging with correlation id from AdmissionReview UID. + - Expose `/healthz` (reads cert expiry, Authority token status) and `/metrics` (Prometheus). + - Add readiness gate that requires initial TLS and Authority bootstrap to succeed. + +**Deliverables** +- Compilable host project with integration tests covering TLS load (mounted files + CSR mock) and Authority token acquisition. +- Documentation snippet for deploy charts describing secret/CSR wiring. + +**Open Questions** +- Need confirmation from Core guild on DTO naming (`AdmissionReviewEnvelope`, `AdmissionDecision`) to avoid rework. +- Determine whether CSR auto-approval is acceptable for air-gapped clusters without Kubernetes cert-manager; may require fallback manual cert import path. + +## ZASTAVA-WEBHOOK-12-102 — Backend policy query & digest resolution + +**Objectives** +- Resolve all images within AdmissionReview to immutable digests before policy evaluation. +- Call Scanner WebService `/api/v1/scanner/policy/runtime` with namespace/labels/images payload, enforce verdicts with deterministic error messaging. + +**Plan** +1. **Image resolution** + - Implement resolver service with pluggable strategies: + - Use existing digest if present. + - Resolve tags via registry HEAD (respecting `admission.resolveTags` flag); fallback to Observer-provided digest once core DTOs available. + - Cache per-registry auth to minimise latency; adhere to allow/deny lists from configuration. +2. **Scanner client** + - Define typed request/response models mirroring `docs/ARCHITECTURE_ZASTAVA.md` structure (`ttlSeconds`, `results[digest] -> { signed, hasSbom, policyVerdict, reasons, rekor }`). + - Implement retry policy (3 attempts, exponential backoff) and map HTTP errors to webhook fail-open/closed depending on namespace configuration. + - Instrument latency (`zastava.backend_latency_seconds`) and failure counts. +3. **Verdict enforcement** + - Evaluate per-image results: if any `policyVerdict != pass` (or `warn` when `enforceWarnings=false`), deny with aggregated reasons. + - Attach `ttlSeconds` to admission response annotations for auditing. + - Record structured logs with namespace, pod, image digest, decision, reasons, backend latency. +4. **Contract coordination** + - Schedule joint review with Scanner WebService guild once SCANNER-RUNTIME-12-302 schema stabilises; track in TASKS sub-items. + - Provide sample payload fixtures for CLI team (`CLI-RUNTIME-13-005`) to validate table output; ensure field names stay aligned. + +**Deliverables** +- Registry resolver unit tests (tag->digest) with deterministic fixtures. +- HTTP client integration tests using Scanner stub returning varied verdict combinations. +- Documentation update summarising contract and failure handling. + +**Open Questions** +- Confirm expected policy verdict enumeration (`pass|warn|fail|error`?) and textual reason codes. +- Need TTL behaviour: should webhook reduce TTL when backend returns > configured max? + +## ZASTAVA-WEBHOOK-12-103 — Caching, fail-open/closed toggles, metrics/logging + +**Objectives** +- Provide deterministic caching layer respecting backend TTL while ensuring eviction on policy mutation. +- Allow namespace-scoped fail-open behaviour with explicit metrics and alerts. +- Surface actionable metrics/logging aligned with Architecture doc. + +**Plan** +1. **Cache design** + - In-memory LRU keyed by image digest; value carries verdict payload + expiry timestamp. + - Support optional persistent seed (read-only) to prime hot digests for offline clusters (config: `admission.cache.seedPath`). + - On startup, load seed file and emit metric `zastava.cache_seed_entries_total`. + - Evict entries on TTL or when `policyRevision` annotation in AdmissionReview changes (requires hook from Core DTO). +2. **Fail-open/closed toggles** + - Configuration: global default + namespace overrides through `admission.failOpenNamespaces`, `admission.failClosedNamespaces`. + - Decision matrix: + - Backend success + verdict PASS → allow. + - Backend success + non-pass → deny unless namespace override says warn allowed. + - Backend failure → allow if namespace fail-open, deny otherwise; annotate response with `zastava.ops/fail-open=true`. + - Implement policy change event hook (future) to clear cache if observer signals revocation. +3. **Metrics & logging** + - Counters: `zastava.admission_requests_total{decision}`, `zastava.cache_hits_total{result=hit|miss}`, `zastava.fail_open_total`, `zastava.backend_failures_total{stage}`. + - Histograms: `zastava.admission_latency_seconds` (overall), `zastava.resolve_latency_seconds`. + - Logs: structured JSON with `decision`, `namespace`, `pod`, `imageDigest`, `reasons`, `cacheStatus`, `failMode`. + - Optionally emit OpenTelemetry span for admission path with attributes capturing backend latency + cache path. +4. **Testing & ops hooks** + - Unit tests for cache TTL, namespace override logic, fail-open metric increments. + - Integration test simulating backend outage ensuring fail-open/closed behaviour matches config. + - Document runbook snippet describing interpreting metrics and toggling namespaces. + +**Open Questions** +- Confirm whether cache entries should include `policyRevision` to detect backend policy updates; requires coordination with Policy guild. +- Need guidance on maximum cache size (default suggestions: 5k entries per replica?) to avoid memory blow-up. + diff --git a/src/StellaOps.Zastava.Webhook/Program.cs b/src/StellaOps.Zastava.Webhook/Program.cs new file mode 100644 index 00000000..81b064bb --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/Program.cs @@ -0,0 +1,68 @@ +using System.Security.Authentication; +using Microsoft.AspNetCore.Diagnostics.HealthChecks; +using Serilog; +using Serilog.Events; +using StellaOps.Zastava.Webhook.Authority; +using StellaOps.Zastava.Webhook.Certificates; +using StellaOps.Zastava.Webhook.Configuration; + +var builder = WebApplication.CreateBuilder(args); + +builder.Host.UseSerilog((context, services, loggerConfiguration) => +{ + loggerConfiguration + .MinimumLevel.Information() + .MinimumLevel.Override("Microsoft.Hosting.Lifetime", LogEventLevel.Information) + .MinimumLevel.Override("Microsoft.AspNetCore", LogEventLevel.Warning) + .Enrich.FromLogContext() + .WriteTo.Console(); +}); + +builder.Services.AddRouting(); +builder.Services.AddProblemDetails(); +builder.Services.AddEndpointsApiExplorer(); +builder.Services.AddHttpClient(); +builder.Services.AddZastavaWebhook(builder.Configuration); + +builder.WebHost.ConfigureKestrel((context, options) => +{ + options.AddServerHeader = false; + options.Limits.MinRequestBodyDataRate = null; // Admission payloads are small; relax defaults for determinism. + + options.ConfigureHttpsDefaults(httpsOptions => + { + var certificateProvider = options.ApplicationServices?.GetRequiredService() + ?? throw new InvalidOperationException("Webhook certificate provider unavailable."); + + httpsOptions.SslProtocols = SslProtocols.Tls13; + httpsOptions.ClientCertificateMode = Microsoft.AspNetCore.Server.Kestrel.Https.ClientCertificateMode.NoCertificate; + httpsOptions.CheckCertificateRevocation = false; // Kubernetes API server terminates client auth; revocation handled upstream. + httpsOptions.ServerCertificate = certificateProvider.GetCertificate(); + }); +}); + +var app = builder.Build(); + +app.UseSerilogRequestLogging(); +app.UseRouting(); + +app.UseStatusCodePages(); + +// Health endpoints. +app.MapHealthChecks("/healthz/ready", new HealthCheckOptions +{ + AllowCachingResponses = false +}); +app.MapHealthChecks("/healthz/live", new HealthCheckOptions +{ + AllowCachingResponses = false, + Predicate = _ => false +}); + +// Placeholder admission endpoint; will be replaced as tasks 12-102/12-103 land. +app.MapPost("/admission", () => Results.StatusCode(StatusCodes.Status501NotImplemented)) + .WithName("AdmissionReview"); + +app.MapGet("/", () => Results.Ok(new { status = "ok", service = "zastava-webhook" })); + +app.Run(); diff --git a/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj b/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj new file mode 100644 index 00000000..730d0896 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/StellaOps.Zastava.Webhook.csproj @@ -0,0 +1,16 @@ + + + net10.0 + preview + enable + enable + true + StellaOps.Zastava.Webhook + $(NoWarn);CA2254 + + + + + + + diff --git a/src/StellaOps.Zastava.Webhook/TASKS.md b/src/StellaOps.Zastava.Webhook/TASKS.md new file mode 100644 index 00000000..5314fd18 --- /dev/null +++ b/src/StellaOps.Zastava.Webhook/TASKS.md @@ -0,0 +1,9 @@ +# Zastava Webhook Task Board + +| ID | Status | Owner(s) | Depends on | Description | Exit Criteria | +|----|--------|----------|------------|-------------|---------------| +| ZASTAVA-WEBHOOK-12-101 | DOING | Zastava Webhook Guild | — | Admission controller host with TLS bootstrap and Authority auth. | Webhook host boots with deterministic TLS bootstrap, enforces Authority-issued credentials, e2e smoke proves admission callback lifecycle, structured logs + metrics emit on each decision. | +| ZASTAVA-WEBHOOK-12-102 | DOING | Zastava Webhook Guild | — | Query Scanner `/policy/runtime`, resolve digests, enforce verdicts. | Scanner client resolves image digests + policy verdicts, unit tests cover allow/deny, integration harness rejects/admits workloads per policy with deterministic payloads. | +| ZASTAVA-WEBHOOK-12-103 | DOING | Zastava Webhook Guild | — | Caching, fail-open/closed toggles, metrics/logging for admission decisions. | Configurable cache TTL + seeds survive restart, fail-open/closed toggles verified via tests, metrics/logging exported per decision path, docs note operational knobs. | + +> Status update · 2025-10-19: Confirmed no prerequisites for ZASTAVA-WEBHOOK-12-101/102/103; tasks moved to DOING for kickoff. Implementation plan covering TLS bootstrap, backend contract, caching/metrics recorded in `IMPLEMENTATION_PLAN.md`. diff --git a/src/StellaOps.sln b/src/StellaOps.sln index 2bdce374..4323b558 100644 --- a/src/StellaOps.sln +++ b/src/StellaOps.sln @@ -23,135 +23,137 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugins EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Authority.Plugin.Standard", "StellaOps.Authority\StellaOps.Authority.Plugin.Standard\StellaOps.Authority.Plugin.Standard.csproj", "{93DB06DC-B254-48A9-8F2C-6130A5658F27}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{40094279-250C-42AE-992A-856718FEFBAC}" -EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin", "StellaOps.Plugin\StellaOps.Plugin.csproj", "{03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Plugin.Tests", "StellaOps.Plugin.Tests\StellaOps.Plugin.Tests.csproj", "{C6DC3C29-C2AD-4015-8872-42E95A0FE63F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli", "StellaOps.Cli\StellaOps.Cli.csproj", "{40094279-250C-42AE-992A-856718FEFBAC}" +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cli.Tests", "StellaOps.Cli.Tests\StellaOps.Cli.Tests.csproj", "{B2967228-F8F7-4931-B257-1C63CB58CE1D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Testing", "StellaOps.Feedser.Testing\StellaOps.Feedser.Testing.csproj", "{6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Testing", "StellaOps.Concelier.Testing\StellaOps.Concelier.Testing.csproj", "{6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common", "StellaOps.Feedser.Source.Common\StellaOps.Feedser.Source.Common.csproj", "{37F203A3-624E-4794-9C99-16CAC22C17DF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common", "StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj", "{37F203A3-624E-4794-9C99-16CAC22C17DF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo", "StellaOps.Feedser.Storage.Mongo\StellaOps.Feedser.Storage.Mongo.csproj", "{3FF93987-A30A-4D50-8815-7CF3BB7CAE05}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo", "StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj", "{3FF93987-A30A-4D50-8815-7CF3BB7CAE05}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core", "StellaOps.Feedser.Core\StellaOps.Feedser.Core.csproj", "{AACE8717-0760-42F2-A225-8FCCE876FB65}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core", "StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj", "{AACE8717-0760-42F2-A225-8FCCE876FB65}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models", "StellaOps.Feedser.Models\StellaOps.Feedser.Models.csproj", "{4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models", "StellaOps.Concelier.Models\StellaOps.Concelier.Models.csproj", "{4AAD6965-E879-44AD-A8ED-E1D713A3CD6D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization", "StellaOps.Feedser.Normalization\StellaOps.Feedser.Normalization.csproj", "{85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Normalization", "StellaOps.Concelier.Normalization\StellaOps.Concelier.Normalization.csproj", "{85D82A87-1F4A-4B1B-8422-5B7A7B7704E3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Core.Tests", "StellaOps.Feedser.Core.Tests\StellaOps.Feedser.Core.Tests.csproj", "{FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Core.Tests", "StellaOps.Concelier.Core.Tests\StellaOps.Concelier.Core.Tests.csproj", "{FE227DF2-875D-4BEA-A4E0-14EA7F3EC1D0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json", "StellaOps.Feedser.Exporter.Json\StellaOps.Feedser.Exporter.Json.csproj", "{D0FB54BA-4D14-4A32-B09F-7EC94F369460}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.Json", "StellaOps.Concelier.Exporter.Json\StellaOps.Concelier.Exporter.Json.csproj", "{D0FB54BA-4D14-4A32-B09F-7EC94F369460}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.Json.Tests", "StellaOps.Feedser.Exporter.Json.Tests\StellaOps.Feedser.Exporter.Json.Tests.csproj", "{69C9E010-CBDD-4B89-84CF-7AB56D6A078A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.Json.Tests", "StellaOps.Concelier.Exporter.Json.Tests\StellaOps.Concelier.Exporter.Json.Tests.csproj", "{69C9E010-CBDD-4B89-84CF-7AB56D6A078A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb", "StellaOps.Feedser.Exporter.TrivyDb\StellaOps.Feedser.Exporter.TrivyDb.csproj", "{E471176A-E1F3-4DE5-8D30-0865903A217A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.TrivyDb", "StellaOps.Concelier.Exporter.TrivyDb\StellaOps.Concelier.Exporter.TrivyDb.csproj", "{E471176A-E1F3-4DE5-8D30-0865903A217A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Exporter.TrivyDb.Tests", "StellaOps.Feedser.Exporter.TrivyDb.Tests\StellaOps.Feedser.Exporter.TrivyDb.Tests.csproj", "{FA013511-DF20-45F7-8077-EBA2D6224D64}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Exporter.TrivyDb.Tests", "StellaOps.Concelier.Exporter.TrivyDb.Tests\StellaOps.Concelier.Exporter.TrivyDb.Tests.csproj", "{FA013511-DF20-45F7-8077-EBA2D6224D64}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge", "StellaOps.Feedser.Merge\StellaOps.Feedser.Merge.csproj", "{B9F84697-54FE-4648-B173-EE3D904FFA4D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Merge", "StellaOps.Concelier.Merge\StellaOps.Concelier.Merge.csproj", "{B9F84697-54FE-4648-B173-EE3D904FFA4D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Merge.Tests", "StellaOps.Feedser.Merge.Tests\StellaOps.Feedser.Merge.Tests.csproj", "{6751A76C-8ED8-40F4-AE2B-069DB31395FE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Merge.Tests", "StellaOps.Concelier.Merge.Tests\StellaOps.Concelier.Merge.Tests.csproj", "{6751A76C-8ED8-40F4-AE2B-069DB31395FE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Models.Tests", "StellaOps.Feedser.Models.Tests\StellaOps.Feedser.Models.Tests.csproj", "{DDBFA2EF-9CAE-473F-A438-369CAC25C66A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Models.Tests", "StellaOps.Concelier.Models.Tests\StellaOps.Concelier.Models.Tests.csproj", "{DDBFA2EF-9CAE-473F-A438-369CAC25C66A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Normalization.Tests", "StellaOps.Feedser.Normalization.Tests\StellaOps.Feedser.Normalization.Tests.csproj", "{063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Normalization.Tests", "StellaOps.Concelier.Normalization.Tests\StellaOps.Concelier.Normalization.Tests.csproj", "{063DE5E1-C8FE-47D0-A12A-22A25CDF2C22}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Acsc", "StellaOps.Feedser.Source.Acsc\StellaOps.Feedser.Source.Acsc.csproj", "{35350FAB-FC51-4FE8-81FB-011003134C37}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Acsc", "StellaOps.Concelier.Connector.Acsc\StellaOps.Concelier.Connector.Acsc.csproj", "{35350FAB-FC51-4FE8-81FB-011003134C37}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cccs", "StellaOps.Feedser.Source.Cccs\StellaOps.Feedser.Source.Cccs.csproj", "{1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Cccs", "StellaOps.Concelier.Connector.Cccs\StellaOps.Concelier.Connector.Cccs.csproj", "{1BFC95B4-4C8A-44B2-903A-11FBCAAB9519}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertBund", "StellaOps.Feedser.Source.CertBund\StellaOps.Feedser.Source.CertBund.csproj", "{C4A65377-22F7-4D15-92A3-4F05847D167E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertBund", "StellaOps.Concelier.Connector.CertBund\StellaOps.Concelier.Connector.CertBund.csproj", "{C4A65377-22F7-4D15-92A3-4F05847D167E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertCc", "StellaOps.Feedser.Source.CertCc\StellaOps.Feedser.Source.CertCc.csproj", "{BDDE59E1-C643-4C87-8608-0F9A7A54DE09}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertCc", "StellaOps.Concelier.Connector.CertCc\StellaOps.Concelier.Connector.CertCc.csproj", "{BDDE59E1-C643-4C87-8608-0F9A7A54DE09}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr", "StellaOps.Feedser.Source.CertFr\StellaOps.Feedser.Source.CertFr.csproj", "{0CC116C8-A7E5-4B94-9688-32920177FF97}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertFr", "StellaOps.Concelier.Connector.CertFr\StellaOps.Concelier.Connector.CertFr.csproj", "{0CC116C8-A7E5-4B94-9688-32920177FF97}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertFr.Tests", "StellaOps.Feedser.Source.CertFr.Tests\StellaOps.Feedser.Source.CertFr.Tests.csproj", "{E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertFr.Tests", "StellaOps.Concelier.Connector.CertFr.Tests\StellaOps.Concelier.Connector.CertFr.Tests.csproj", "{E8862F6E-85C1-4FDB-AA92-0BB489B7EA1E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn", "StellaOps.Feedser.Source.CertIn\StellaOps.Feedser.Source.CertIn.csproj", "{84DEDF05-A5BD-4644-86B9-6B7918FE3F31}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertIn", "StellaOps.Concelier.Connector.CertIn\StellaOps.Concelier.Connector.CertIn.csproj", "{84DEDF05-A5BD-4644-86B9-6B7918FE3F31}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.CertIn.Tests", "StellaOps.Feedser.Source.CertIn.Tests\StellaOps.Feedser.Source.CertIn.Tests.csproj", "{9DEB1F54-94B5-40C4-AC44-220E680B016D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.CertIn.Tests", "StellaOps.Concelier.Connector.CertIn.Tests\StellaOps.Concelier.Connector.CertIn.Tests.csproj", "{9DEB1F54-94B5-40C4-AC44-220E680B016D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Common.Tests", "StellaOps.Feedser.Source.Common.Tests\StellaOps.Feedser.Source.Common.Tests.csproj", "{7C3E87F2-93D8-4968-95E3-52C46947D46C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Common.Tests", "StellaOps.Concelier.Connector.Common.Tests\StellaOps.Concelier.Connector.Common.Tests.csproj", "{7C3E87F2-93D8-4968-95E3-52C46947D46C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve", "StellaOps.Feedser.Source.Cve\StellaOps.Feedser.Source.Cve.csproj", "{C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Cve", "StellaOps.Concelier.Connector.Cve\StellaOps.Concelier.Connector.Cve.csproj", "{C0504D97-9BCD-4AE4-B0DC-B31C17B150F2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian", "StellaOps.Feedser.Source.Distro.Debian\StellaOps.Feedser.Source.Distro.Debian.csproj", "{31B05493-104F-437F-9FA7-CA5286CE697C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Debian", "StellaOps.Concelier.Connector.Distro.Debian\StellaOps.Concelier.Connector.Distro.Debian.csproj", "{31B05493-104F-437F-9FA7-CA5286CE697C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Debian.Tests", "StellaOps.Feedser.Source.Distro.Debian.Tests\StellaOps.Feedser.Source.Distro.Debian.Tests.csproj", "{937AF12E-D770-4534-8FF8-C59042609C2A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Debian.Tests", "StellaOps.Concelier.Connector.Distro.Debian.Tests\StellaOps.Concelier.Connector.Distro.Debian.Tests.csproj", "{937AF12E-D770-4534-8FF8-C59042609C2A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat", "StellaOps.Feedser.Source.Distro.RedHat\StellaOps.Feedser.Source.Distro.RedHat.csproj", "{5A028B04-9D76-470B-B5B3-766CE4CE860C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.RedHat", "StellaOps.Concelier.Connector.Distro.RedHat\StellaOps.Concelier.Connector.Distro.RedHat.csproj", "{5A028B04-9D76-470B-B5B3-766CE4CE860C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.RedHat.Tests", "StellaOps.Feedser.Source.Distro.RedHat.Tests\StellaOps.Feedser.Source.Distro.RedHat.Tests.csproj", "{749DE4C8-F733-43F8-B2A8-6649E71C7570}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.RedHat.Tests", "StellaOps.Concelier.Connector.Distro.RedHat.Tests\StellaOps.Concelier.Connector.Distro.RedHat.Tests.csproj", "{749DE4C8-F733-43F8-B2A8-6649E71C7570}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse", "StellaOps.Feedser.Source.Distro.Suse\StellaOps.Feedser.Source.Distro.Suse.csproj", "{56D2C79E-2737-4FF9-9D19-150065F568D5}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Suse", "StellaOps.Concelier.Connector.Distro.Suse\StellaOps.Concelier.Connector.Distro.Suse.csproj", "{56D2C79E-2737-4FF9-9D19-150065F568D5}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Suse.Tests", "StellaOps.Feedser.Source.Distro.Suse.Tests\StellaOps.Feedser.Source.Distro.Suse.Tests.csproj", "{E41F6DC4-68B5-4EE3-97AE-801D725A2C13}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Suse.Tests", "StellaOps.Concelier.Connector.Distro.Suse.Tests\StellaOps.Concelier.Connector.Distro.Suse.Tests.csproj", "{E41F6DC4-68B5-4EE3-97AE-801D725A2C13}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu", "StellaOps.Feedser.Source.Distro.Ubuntu\StellaOps.Feedser.Source.Distro.Ubuntu.csproj", "{285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Ubuntu", "StellaOps.Concelier.Connector.Distro.Ubuntu\StellaOps.Concelier.Connector.Distro.Ubuntu.csproj", "{285F1D0F-501F-4E2E-8FA0-F2CF28AE3798}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Distro.Ubuntu.Tests", "StellaOps.Feedser.Source.Distro.Ubuntu.Tests\StellaOps.Feedser.Source.Distro.Ubuntu.Tests.csproj", "{26055403-C7F5-4709-8813-0F7387102791}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Distro.Ubuntu.Tests", "StellaOps.Concelier.Connector.Distro.Ubuntu.Tests\StellaOps.Concelier.Connector.Distro.Ubuntu.Tests.csproj", "{26055403-C7F5-4709-8813-0F7387102791}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa", "StellaOps.Feedser.Source.Ghsa\StellaOps.Feedser.Source.Ghsa.csproj", "{0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ghsa", "StellaOps.Concelier.Connector.Ghsa\StellaOps.Concelier.Connector.Ghsa.csproj", "{0C00D0DA-C4C3-4B23-941F-A3DB2DBF33AF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Cisa", "StellaOps.Feedser.Source.Ics.Cisa\StellaOps.Feedser.Source.Ics.Cisa.csproj", "{258327E9-431E-475C-933B-50893676E452}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Cisa", "StellaOps.Concelier.Connector.Ics.Cisa\StellaOps.Concelier.Connector.Ics.Cisa.csproj", "{258327E9-431E-475C-933B-50893676E452}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky", "StellaOps.Feedser.Source.Ics.Kaspersky\StellaOps.Feedser.Source.Ics.Kaspersky.csproj", "{42AF60C8-A5E1-40E0-86F8-98256364AF6F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Kaspersky", "StellaOps.Concelier.Connector.Ics.Kaspersky\StellaOps.Concelier.Connector.Ics.Kaspersky.csproj", "{42AF60C8-A5E1-40E0-86F8-98256364AF6F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ics.Kaspersky.Tests", "StellaOps.Feedser.Source.Ics.Kaspersky.Tests\StellaOps.Feedser.Source.Ics.Kaspersky.Tests.csproj", "{88C6A9C3-B433-4C36-8767-429C8C2396F8}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ics.Kaspersky.Tests", "StellaOps.Concelier.Connector.Ics.Kaspersky.Tests\StellaOps.Concelier.Connector.Ics.Kaspersky.Tests.csproj", "{88C6A9C3-B433-4C36-8767-429C8C2396F8}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn", "StellaOps.Feedser.Source.Jvn\StellaOps.Feedser.Source.Jvn.csproj", "{6B7099AB-01BF-4EC4-87D0-5C9C032266DE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Jvn", "StellaOps.Concelier.Connector.Jvn\StellaOps.Concelier.Connector.Jvn.csproj", "{6B7099AB-01BF-4EC4-87D0-5C9C032266DE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Jvn.Tests", "StellaOps.Feedser.Source.Jvn.Tests\StellaOps.Feedser.Source.Jvn.Tests.csproj", "{14C918EA-693E-41FE-ACAE-2E82DF077BEA}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Jvn.Tests", "StellaOps.Concelier.Connector.Jvn.Tests\StellaOps.Concelier.Connector.Jvn.Tests.csproj", "{14C918EA-693E-41FE-ACAE-2E82DF077BEA}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev", "StellaOps.Feedser.Source.Kev\StellaOps.Feedser.Source.Kev.csproj", "{81111B26-74F6-4912-9084-7115FD119945}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Kev", "StellaOps.Concelier.Connector.Kev\StellaOps.Concelier.Connector.Kev.csproj", "{81111B26-74F6-4912-9084-7115FD119945}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kisa", "StellaOps.Feedser.Source.Kisa\StellaOps.Feedser.Source.Kisa.csproj", "{80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Kisa", "StellaOps.Concelier.Connector.Kisa\StellaOps.Concelier.Connector.Kisa.csproj", "{80E2D661-FF3E-4A10-A2DF-AFD4F3D433FE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd", "StellaOps.Feedser.Source.Nvd\StellaOps.Feedser.Source.Nvd.csproj", "{8D0F501D-01B1-4E24-958B-FAF35B267705}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Nvd", "StellaOps.Concelier.Connector.Nvd\StellaOps.Concelier.Connector.Nvd.csproj", "{8D0F501D-01B1-4E24-958B-FAF35B267705}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Nvd.Tests", "StellaOps.Feedser.Source.Nvd.Tests\StellaOps.Feedser.Source.Nvd.Tests.csproj", "{5BA91095-7F10-4717-B296-49DFBFC1C9C2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Nvd.Tests", "StellaOps.Concelier.Connector.Nvd.Tests\StellaOps.Concelier.Connector.Nvd.Tests.csproj", "{5BA91095-7F10-4717-B296-49DFBFC1C9C2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv", "StellaOps.Feedser.Source.Osv\StellaOps.Feedser.Source.Osv.csproj", "{99616566-4EF1-4DC7-B655-825FE43D203D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Osv", "StellaOps.Concelier.Connector.Osv\StellaOps.Concelier.Connector.Osv.csproj", "{99616566-4EF1-4DC7-B655-825FE43D203D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Osv.Tests", "StellaOps.Feedser.Source.Osv.Tests\StellaOps.Feedser.Source.Osv.Tests.csproj", "{EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Osv.Tests", "StellaOps.Concelier.Connector.Osv.Tests\StellaOps.Concelier.Connector.Osv.Tests.csproj", "{EE3C03AD-E604-4C57-9B78-CF7F49FBFCB0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Bdu", "StellaOps.Feedser.Source.Ru.Bdu\StellaOps.Feedser.Source.Ru.Bdu.csproj", "{A3B19095-2D95-4B09-B07E-2C082C72394B}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Bdu", "StellaOps.Concelier.Connector.Ru.Bdu\StellaOps.Concelier.Connector.Ru.Bdu.csproj", "{A3B19095-2D95-4B09-B07E-2C082C72394B}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ru.Nkcki", "StellaOps.Feedser.Source.Ru.Nkcki\StellaOps.Feedser.Source.Ru.Nkcki.csproj", "{807837AF-B392-4589-ADF1-3FDB34D6C5BF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ru.Nkcki", "StellaOps.Concelier.Connector.Ru.Nkcki\StellaOps.Concelier.Connector.Ru.Nkcki.csproj", "{807837AF-B392-4589-ADF1-3FDB34D6C5BF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe", "StellaOps.Feedser.Source.Vndr.Adobe\StellaOps.Feedser.Source.Vndr.Adobe.csproj", "{64EAFDCF-8283-4D5C-AC78-7969D5FE926A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Adobe", "StellaOps.Concelier.Connector.Vndr.Adobe\StellaOps.Concelier.Connector.Vndr.Adobe.csproj", "{64EAFDCF-8283-4D5C-AC78-7969D5FE926A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Adobe.Tests", "StellaOps.Feedser.Source.Vndr.Adobe.Tests\StellaOps.Feedser.Source.Vndr.Adobe.Tests.csproj", "{68F4D8A1-E32F-487A-B460-325F36989BE3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Adobe.Tests", "StellaOps.Concelier.Connector.Vndr.Adobe.Tests\StellaOps.Concelier.Connector.Vndr.Adobe.Tests.csproj", "{68F4D8A1-E32F-487A-B460-325F36989BE3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Apple", "StellaOps.Feedser.Source.Vndr.Apple\StellaOps.Feedser.Source.Vndr.Apple.csproj", "{4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Apple", "StellaOps.Concelier.Connector.Vndr.Apple\StellaOps.Concelier.Connector.Vndr.Apple.csproj", "{4A3DA4AE-7B88-4674-A7E2-F5D42B8256F2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium", "StellaOps.Feedser.Source.Vndr.Chromium\StellaOps.Feedser.Source.Vndr.Chromium.csproj", "{606C751B-7CF1-47CF-A25C-9248A55C814F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Chromium", "StellaOps.Concelier.Connector.Vndr.Chromium\StellaOps.Concelier.Connector.Vndr.Chromium.csproj", "{606C751B-7CF1-47CF-A25C-9248A55C814F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Chromium.Tests", "StellaOps.Feedser.Source.Vndr.Chromium.Tests\StellaOps.Feedser.Source.Vndr.Chromium.Tests.csproj", "{0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Chromium.Tests", "StellaOps.Concelier.Connector.Vndr.Chromium.Tests\StellaOps.Concelier.Connector.Vndr.Chromium.Tests.csproj", "{0BE44D0A-CC4B-4E84-8AF3-D8D99551C431}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco", "StellaOps.Feedser.Source.Vndr.Cisco\StellaOps.Feedser.Source.Vndr.Cisco.csproj", "{CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Cisco", "StellaOps.Concelier.Connector.Vndr.Cisco\StellaOps.Concelier.Connector.Vndr.Cisco.csproj", "{CC4CCE5F-55BC-4745-A204-4FA92BC1BADC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Cisco.Tests", "StellaOps.Feedser.Source.Vndr.Cisco.Tests\StellaOps.Feedser.Source.Vndr.Cisco.Tests.csproj", "{99BAE717-9A2E-41F5-9ECC-5FB97E4A6066}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Cisco.Tests", "StellaOps.Concelier.Connector.Vndr.Cisco.Tests\StellaOps.Concelier.Connector.Vndr.Cisco.Tests.csproj", "{99BAE717-9A2E-41F5-9ECC-5FB97E4A6066}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Msrc", "StellaOps.Feedser.Source.Vndr.Msrc\StellaOps.Feedser.Source.Vndr.Msrc.csproj", "{5CCE0DB7-C115-4B21-A7AE-C8488C22A853}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Msrc", "StellaOps.Concelier.Connector.Vndr.Msrc\StellaOps.Concelier.Connector.Vndr.Msrc.csproj", "{5CCE0DB7-C115-4B21-A7AE-C8488C22A853}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle", "StellaOps.Feedser.Source.Vndr.Oracle\StellaOps.Feedser.Source.Vndr.Oracle.csproj", "{A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Oracle", "StellaOps.Concelier.Connector.Vndr.Oracle\StellaOps.Concelier.Connector.Vndr.Oracle.csproj", "{A09C9E66-5496-47EC-8B23-9EEB7CBDC75E}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Oracle.Tests", "StellaOps.Feedser.Source.Vndr.Oracle.Tests\StellaOps.Feedser.Source.Vndr.Oracle.Tests.csproj", "{06DC817F-A936-4F83-8929-E00622B32245}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Oracle.Tests", "StellaOps.Concelier.Connector.Vndr.Oracle.Tests\StellaOps.Concelier.Connector.Vndr.Oracle.Tests.csproj", "{06DC817F-A936-4F83-8929-E00622B32245}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware", "StellaOps.Feedser.Source.Vndr.Vmware\StellaOps.Feedser.Source.Vndr.Vmware.csproj", "{2C999476-0291-4161-B3E9-1AA99A3B1139}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Vmware", "StellaOps.Concelier.Connector.Vndr.Vmware\StellaOps.Concelier.Connector.Vndr.Vmware.csproj", "{2C999476-0291-4161-B3E9-1AA99A3B1139}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Vndr.Vmware.Tests", "StellaOps.Feedser.Source.Vndr.Vmware.Tests\StellaOps.Feedser.Source.Vndr.Vmware.Tests.csproj", "{476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Vndr.Vmware.Tests", "StellaOps.Concelier.Connector.Vndr.Vmware.Tests\StellaOps.Concelier.Connector.Vndr.Vmware.Tests.csproj", "{476EAADA-1B39-4049-ABE4-CCAC21FFE9E2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Storage.Mongo.Tests", "StellaOps.Feedser.Storage.Mongo.Tests\StellaOps.Feedser.Storage.Mongo.Tests.csproj", "{0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Storage.Mongo.Tests", "StellaOps.Concelier.Storage.Mongo.Tests\StellaOps.Concelier.Storage.Mongo.Tests.csproj", "{0EF56124-E6E8-4E89-95DD-5A5D5FF05A98}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService", "StellaOps.Feedser.WebService\StellaOps.Feedser.WebService.csproj", "{0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.WebService", "StellaOps.Concelier.WebService\StellaOps.Concelier.WebService.csproj", "{0DBB9FC4-2E46-4C3E-BE88-2A8DCB59DB7D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.WebService.Tests", "StellaOps.Feedser.WebService.Tests\StellaOps.Feedser.WebService.Tests.csproj", "{8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.WebService.Tests", "StellaOps.Concelier.WebService.Tests\StellaOps.Concelier.WebService.Tests.csproj", "{8A40142F-E8C8-4E86-BE70-7DD4AB1FFDEE}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Configuration.Tests", "StellaOps.Configuration.Tests\StellaOps.Configuration.Tests.csproj", "{C9D20F74-EE5F-4C9E-9AB1-C03E90B34F92}" EndProject @@ -169,11 +171,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.ServerIntegr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Client.Tests", "StellaOps.Authority\StellaOps.Auth.Client.Tests\StellaOps.Auth.Client.Tests.csproj", "{7DBE31A6-D2FD-499E-B675-4092723175AD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Kev.Tests", "StellaOps.Feedser.Source.Kev.Tests\StellaOps.Feedser.Source.Kev.Tests.csproj", "{D99E6EAE-D278-4480-AA67-85F025383E47}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Kev.Tests", "StellaOps.Concelier.Connector.Kev.Tests\StellaOps.Concelier.Connector.Kev.Tests.csproj", "{D99E6EAE-D278-4480-AA67-85F025383E47}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Cve.Tests", "StellaOps.Feedser.Source.Cve.Tests\StellaOps.Feedser.Source.Cve.Tests.csproj", "{D3825714-3DDA-44B7-A99C-5F3E65716691}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Cve.Tests", "StellaOps.Concelier.Connector.Cve.Tests\StellaOps.Concelier.Connector.Cve.Tests.csproj", "{D3825714-3DDA-44B7-A99C-5F3E65716691}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Feedser.Source.Ghsa.Tests", "StellaOps.Feedser.Source.Ghsa.Tests\StellaOps.Feedser.Source.Ghsa.Tests.csproj", "{FAB78D21-7372-48FE-B2C3-DE1807F1157D}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Concelier.Connector.Ghsa.Tests", "StellaOps.Concelier.Connector.Ghsa.Tests\StellaOps.Concelier.Connector.Ghsa.Tests.csproj", "{FAB78D21-7372-48FE-B2C3-DE1807F1157D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography", "StellaOps.Cryptography\StellaOps.Cryptography.csproj", "{EADFA337-B0FA-4712-A24A-7C08235BDF98}" EndProject @@ -181,61 +183,147 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.Test EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Cryptography.DependencyInjection", "StellaOps.Cryptography.DependencyInjection\StellaOps.Cryptography.DependencyInjection.csproj", "{B84FE2DD-A1AD-437C-95CF-89C1DCCFDF6F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Core", "StellaOps.Vexer.Core\StellaOps.Vexer.Core.csproj", "{3288F0F8-FF86-4DB3-A1FD-8EB51893E8C2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Core", "StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj", "{3288F0F8-FF86-4DB3-A1FD-8EB51893E8C2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Core.Tests", "StellaOps.Vexer.Core.Tests\StellaOps.Vexer.Core.Tests.csproj", "{680CA103-DCE8-4D02-8979-72DEA5BE8C00}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Core.Tests", "StellaOps.Excititor.Core.Tests\StellaOps.Excititor.Core.Tests.csproj", "{680CA103-DCE8-4D02-8979-72DEA5BE8C00}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Policy", "StellaOps.Vexer.Policy\StellaOps.Vexer.Policy.csproj", "{7F4B19D4-569A-4CCF-B481-EBE04860451A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy", "StellaOps.Excititor.Policy\StellaOps.Excititor.Policy.csproj", "{7F4B19D4-569A-4CCF-B481-EBE04860451A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Policy.Tests", "StellaOps.Vexer.Policy.Tests\StellaOps.Vexer.Policy.Tests.csproj", "{DE9863B5-E6D6-4C5F-B52A-ED9E964008A3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Policy.Tests", "StellaOps.Excititor.Policy.Tests\StellaOps.Excititor.Policy.Tests.csproj", "{DE9863B5-E6D6-4C5F-B52A-ED9E964008A3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Storage.Mongo", "StellaOps.Vexer.Storage.Mongo\StellaOps.Vexer.Storage.Mongo.csproj", "{E380F242-031E-483E-8570-0EF7EA525C4F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Storage.Mongo", "StellaOps.Excititor.Storage.Mongo\StellaOps.Excititor.Storage.Mongo.csproj", "{E380F242-031E-483E-8570-0EF7EA525C4F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Export", "StellaOps.Vexer.Export\StellaOps.Vexer.Export.csproj", "{42582C16-F5A9-417F-9D33-BC489925324F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Export", "StellaOps.Excititor.Export\StellaOps.Excititor.Export.csproj", "{42582C16-F5A9-417F-9D33-BC489925324F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Export.Tests", "StellaOps.Vexer.Export.Tests\StellaOps.Vexer.Export.Tests.csproj", "{06F40DA8-FEFA-4C2B-907B-155BD92BB859}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Export.Tests", "StellaOps.Excititor.Export.Tests\StellaOps.Excititor.Export.Tests.csproj", "{06F40DA8-FEFA-4C2B-907B-155BD92BB859}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.RedHat.CSAF", "StellaOps.Vexer.Connectors.RedHat.CSAF\StellaOps.Vexer.Connectors.RedHat.CSAF.csproj", "{A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.RedHat.CSAF", "StellaOps.Excititor.Connectors.RedHat.CSAF\StellaOps.Excititor.Connectors.RedHat.CSAF.csproj", "{A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.RedHat.CSAF.Tests", "StellaOps.Vexer.Connectors.RedHat.CSAF.Tests\StellaOps.Vexer.Connectors.RedHat.CSAF.Tests.csproj", "{3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.RedHat.CSAF.Tests", "StellaOps.Excititor.Connectors.RedHat.CSAF.Tests\StellaOps.Excititor.Connectors.RedHat.CSAF.Tests.csproj", "{3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Abstractions", "StellaOps.Vexer.Connectors.Abstractions\StellaOps.Vexer.Connectors.Abstractions.csproj", "{F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Abstractions", "StellaOps.Excititor.Connectors.Abstractions\StellaOps.Excititor.Connectors.Abstractions.csproj", "{F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Worker", "StellaOps.Vexer.Worker\StellaOps.Vexer.Worker.csproj", "{781EC793-1DB0-4E31-95BC-12A2B373045F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Worker", "StellaOps.Excititor.Worker\StellaOps.Excititor.Worker.csproj", "{781EC793-1DB0-4E31-95BC-12A2B373045F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Worker.Tests", "StellaOps.Vexer.Worker.Tests\StellaOps.Vexer.Worker.Tests.csproj", "{BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Worker.Tests", "StellaOps.Excititor.Worker.Tests\StellaOps.Excititor.Worker.Tests.csproj", "{BB863E0C-50FF-41AE-9C13-4E8A1BABC62C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CSAF", "StellaOps.Vexer.Formats.CSAF\StellaOps.Vexer.Formats.CSAF.csproj", "{14E9D043-F0EF-4F68-AE83-D6F579119D9A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.CSAF", "StellaOps.Excititor.Formats.CSAF\StellaOps.Excititor.Formats.CSAF.csproj", "{14E9D043-F0EF-4F68-AE83-D6F579119D9A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CSAF.Tests", "StellaOps.Vexer.Formats.CSAF.Tests\StellaOps.Vexer.Formats.CSAF.Tests.csproj", "{27E94B6E-DEF8-4B89-97CB-424703790ECE}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.CSAF.Tests", "StellaOps.Excititor.Formats.CSAF.Tests\StellaOps.Excititor.Formats.CSAF.Tests.csproj", "{27E94B6E-DEF8-4B89-97CB-424703790ECE}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CycloneDX", "StellaOps.Vexer.Formats.CycloneDX\StellaOps.Vexer.Formats.CycloneDX.csproj", "{361E3E23-B215-423D-9906-A84171E20AD3}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.CycloneDX", "StellaOps.Excititor.Formats.CycloneDX\StellaOps.Excititor.Formats.CycloneDX.csproj", "{361E3E23-B215-423D-9906-A84171E20AD3}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.CycloneDX.Tests", "StellaOps.Vexer.Formats.CycloneDX.Tests\StellaOps.Vexer.Formats.CycloneDX.Tests.csproj", "{7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.CycloneDX.Tests", "StellaOps.Excititor.Formats.CycloneDX.Tests\StellaOps.Excititor.Formats.CycloneDX.Tests.csproj", "{7A7A3480-C6C3-4A9F-AF46-1889424B9AC2}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.OpenVEX", "StellaOps.Vexer.Formats.OpenVEX\StellaOps.Vexer.Formats.OpenVEX.csproj", "{C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.OpenVEX", "StellaOps.Excititor.Formats.OpenVEX\StellaOps.Excititor.Formats.OpenVEX.csproj", "{C3EAFCB8-0394-4B74-B9A6-3DBA4509201F}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Formats.OpenVEX.Tests", "StellaOps.Vexer.Formats.OpenVEX.Tests\StellaOps.Vexer.Formats.OpenVEX.Tests.csproj", "{E86CF4A6-2463-4589-A9D8-9DF557C48367}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Formats.OpenVEX.Tests", "StellaOps.Excititor.Formats.OpenVEX.Tests\StellaOps.Excititor.Formats.OpenVEX.Tests.csproj", "{E86CF4A6-2463-4589-A9D8-9DF557C48367}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Cisco.CSAF", "StellaOps.Vexer.Connectors.Cisco.CSAF\StellaOps.Vexer.Connectors.Cisco.CSAF.csproj", "{B308B94C-E01F-4449-A5A6-CD7A48E52D15}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Cisco.CSAF", "StellaOps.Excititor.Connectors.Cisco.CSAF\StellaOps.Excititor.Connectors.Cisco.CSAF.csproj", "{B308B94C-E01F-4449-A5A6-CD7A48E52D15}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Cisco.CSAF.Tests", "StellaOps.Vexer.Connectors.Cisco.CSAF.Tests\StellaOps.Vexer.Connectors.Cisco.CSAF.Tests.csproj", "{9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Cisco.CSAF.Tests", "StellaOps.Excititor.Connectors.Cisco.CSAF.Tests\StellaOps.Excititor.Connectors.Cisco.CSAF.Tests.csproj", "{9FBA3EC4-D794-48BD-82FA-0289E5A2A5FF}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub", "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub\StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.csproj", "{E076DC9C-B436-44BF-B02E-FA565086F805}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub", "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub\StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.csproj", "{E076DC9C-B436-44BF-B02E-FA565086F805}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests", "StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests\StellaOps.Vexer.Connectors.SUSE.RancherVEXHub.Tests.csproj", "{55500025-FE82-4F97-A261-9BAEA4B10845}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests", "StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests\StellaOps.Excititor.Connectors.SUSE.RancherVEXHub.Tests.csproj", "{55500025-FE82-4F97-A261-9BAEA4B10845}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.MSRC.CSAF", "StellaOps.Vexer.Connectors.MSRC.CSAF\StellaOps.Vexer.Connectors.MSRC.CSAF.csproj", "{CD12875F-9367-41BD-810C-7FBE76314F17}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.MSRC.CSAF", "StellaOps.Excititor.Connectors.MSRC.CSAF\StellaOps.Excititor.Connectors.MSRC.CSAF.csproj", "{CD12875F-9367-41BD-810C-7FBE76314F17}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.MSRC.CSAF.Tests", "StellaOps.Vexer.Connectors.MSRC.CSAF.Tests\StellaOps.Vexer.Connectors.MSRC.CSAF.Tests.csproj", "{063D3280-9918-465A-AF2D-3650A2A50D03}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.MSRC.CSAF.Tests", "StellaOps.Excititor.Connectors.MSRC.CSAF.Tests\StellaOps.Excititor.Connectors.MSRC.CSAF.Tests.csproj", "{063D3280-9918-465A-AF2D-3650A2A50D03}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Oracle.CSAF", "StellaOps.Vexer.Connectors.Oracle.CSAF\StellaOps.Vexer.Connectors.Oracle.CSAF.csproj", "{A3EEE400-3655-4B34-915A-598E60CD55FB}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Oracle.CSAF", "StellaOps.Excititor.Connectors.Oracle.CSAF\StellaOps.Excititor.Connectors.Oracle.CSAF.csproj", "{A3EEE400-3655-4B34-915A-598E60CD55FB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Oracle.CSAF.Tests", "StellaOps.Vexer.Connectors.Oracle.CSAF.Tests\StellaOps.Vexer.Connectors.Oracle.CSAF.Tests.csproj", "{577025AD-2FDD-42DF-BFA2-3FC095B50539}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Oracle.CSAF.Tests", "StellaOps.Excititor.Connectors.Oracle.CSAF.Tests\StellaOps.Excititor.Connectors.Oracle.CSAF.Tests.csproj", "{577025AD-2FDD-42DF-BFA2-3FC095B50539}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Ubuntu.CSAF", "StellaOps.Vexer.Connectors.Ubuntu.CSAF\StellaOps.Vexer.Connectors.Ubuntu.CSAF.csproj", "{DD3B2076-E5E0-4533-8D27-7724225D7758}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Ubuntu.CSAF", "StellaOps.Excititor.Connectors.Ubuntu.CSAF\StellaOps.Excititor.Connectors.Ubuntu.CSAF.csproj", "{DD3B2076-E5E0-4533-8D27-7724225D7758}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests", "StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests\StellaOps.Vexer.Connectors.Ubuntu.CSAF.Tests.csproj", "{CADA1364-8EB1-479E-AB6F-4105C26335C8}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests", "StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests\StellaOps.Excititor.Connectors.Ubuntu.CSAF.Tests.csproj", "{CADA1364-8EB1-479E-AB6F-4105C26335C8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Core", "StellaOps.Scanner.Core\StellaOps.Scanner.Core.csproj", "{8CC4441E-9D1A-4E00-831B-34828A3F9446}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Core.Tests", "StellaOps.Scanner.Core.Tests\StellaOps.Scanner.Core.Tests.csproj", "{01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StellaOps.Authority", "StellaOps.Authority", "{BDB24B64-FE4E-C4BD-9F80-9428F98EDF6F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Policy", "StellaOps.Policy\StellaOps.Policy.csproj", "{37BB9502-CCD1-425A-BF45-D56968B0C2F9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Policy.Tests", "StellaOps.Policy.Tests\StellaOps.Policy.Tests.csproj", "{015A7A95-2C07-4C7F-8048-DB591AAC5FE5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.WebService", "StellaOps.Scanner.WebService\StellaOps.Scanner.WebService.csproj", "{EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.WebService.Tests", "StellaOps.Scanner.WebService.Tests\StellaOps.Scanner.WebService.Tests.csproj", "{27D951AD-696D-4330-B4F5-F8F81344C191}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Storage", "StellaOps.Scanner.Storage\StellaOps.Scanner.Storage.csproj", "{31277AFF-9BFF-4C17-8593-B562A385058E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Storage.Tests", "StellaOps.Scanner.Storage.Tests\StellaOps.Scanner.Storage.Tests.csproj", "{3A8F090F-678D-46E2-8899-67402129749C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Worker", "StellaOps.Scanner.Worker\StellaOps.Scanner.Worker.csproj", "{19FACEC7-D6D4-40F5-84AD-14E2983F18F7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Worker.Tests", "StellaOps.Scanner.Worker.Tests\StellaOps.Scanner.Worker.Tests.csproj", "{8342286A-BE36-4ACA-87FF-EBEB4E268498}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.EntryTrace", "StellaOps.Scanner.EntryTrace\StellaOps.Scanner.EntryTrace.csproj", "{05D844B6-51C1-4926-919C-D99E24FB3BC9}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.EntryTrace.Tests", "StellaOps.Scanner.EntryTrace.Tests\StellaOps.Scanner.EntryTrace.Tests.csproj", "{03E15545-D6A0-4287-A88C-6EDE77C0DCBE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang", "StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj", "{A072C46F-BA45-419E-B1B6-416919F78440}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Tests", "StellaOps.Scanner.Analyzers.Lang.Tests\StellaOps.Scanner.Analyzers.Lang.Tests.csproj", "{6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Diff", "StellaOps.Scanner.Diff\StellaOps.Scanner.Diff.csproj", "{10088067-7B8F-4D2E-A8E1-ED546DC17369}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Diff.Tests", "StellaOps.Scanner.Diff.Tests\StellaOps.Scanner.Diff.Tests.csproj", "{E014565C-2456-4BD0-9481-557F939C1E36}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Emit", "StellaOps.Scanner.Emit\StellaOps.Scanner.Emit.csproj", "{44825FDA-68D2-4675-8B1D-6D5303DC38CF}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Emit.Tests", "StellaOps.Scanner.Emit.Tests\StellaOps.Scanner.Emit.Tests.csproj", "{6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Cache", "StellaOps.Scanner.Cache\StellaOps.Scanner.Cache.csproj", "{5E5EB0A7-7A19-4144-81FE-13C31DB678B2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Cache.Tests", "StellaOps.Scanner.Cache.Tests\StellaOps.Scanner.Cache.Tests.csproj", "{7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Java", "StellaOps.Scanner.Analyzers.Lang.Java\StellaOps.Scanner.Analyzers.Lang.Java.csproj", "{B86C287A-734E-4527-A03E-6B970F22E27E}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.OS", "StellaOps.Scanner.Analyzers.OS\StellaOps.Scanner.Analyzers.OS.csproj", "{E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.OS.Apk", "StellaOps.Scanner.Analyzers.OS.Apk\StellaOps.Scanner.Analyzers.OS.Apk.csproj", "{50D014B5-99A6-46FC-B745-26687595B293}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.OS.Dpkg", "StellaOps.Scanner.Analyzers.OS.Dpkg\StellaOps.Scanner.Analyzers.OS.Dpkg.csproj", "{D99C1F78-67EA-40E7-BD4C-985592F5265A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.OS.Rpm", "StellaOps.Scanner.Analyzers.OS.Rpm\StellaOps.Scanner.Analyzers.OS.Rpm.csproj", "{1CBC0B9C-A96B-4143-B70F-37C69229FFF2}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.OS.Tests", "StellaOps.Scanner.Analyzers.OS.Tests\StellaOps.Scanner.Analyzers.OS.Tests.csproj", "{760E2855-31B3-4CCB-BACB-34B7196A59B8}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Node", "StellaOps.Scanner.Analyzers.Lang.Node\StellaOps.Scanner.Analyzers.Lang.Node.csproj", "{3F688F21-7E31-4781-8995-9DD34276773F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Python", "StellaOps.Scanner.Analyzers.Lang.Python\StellaOps.Scanner.Analyzers.Lang.Python.csproj", "{80AD7C4D-E4C6-4700-87AD-77B5698B338F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Go", "StellaOps.Scanner.Analyzers.Lang.Go\StellaOps.Scanner.Analyzers.Lang.Go.csproj", "{60ABAB54-2EE9-4A16-A109-67F7B6F29184}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.DotNet", "StellaOps.Scanner.Analyzers.Lang.DotNet\StellaOps.Scanner.Analyzers.Lang.DotNet.csproj", "{D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Scanner.Analyzers.Lang.Rust", "StellaOps.Scanner.Analyzers.Lang.Rust\StellaOps.Scanner.Analyzers.Lang.Rust.csproj", "{5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Auth.Security", "StellaOps.Auth.Security\StellaOps.Auth.Security.csproj", "{05475C0A-C225-4F07-A3C7-9E17E660042E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StellaOps.Attestor", "StellaOps.Attestor", "{78C966F5-2242-D8EC-ADCA-A1A9C7F723A6}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Core", "StellaOps.Attestor\StellaOps.Attestor.Core\StellaOps.Attestor.Core.csproj", "{BA47D456-4657-4C86-A665-21293E3AC47F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Infrastructure", "StellaOps.Attestor\StellaOps.Attestor.Infrastructure\StellaOps.Attestor.Infrastructure.csproj", "{49EF86AC-1CC2-4A24-8637-C5151E23DF9D}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.WebService", "StellaOps.Attestor\StellaOps.Attestor.WebService\StellaOps.Attestor.WebService.csproj", "{C22333B3-D132-4960-A490-6BEF1EB1C917}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Attestor.Tests", "StellaOps.Attestor\StellaOps.Attestor.Tests\StellaOps.Attestor.Tests.csproj", "{B8B15A8D-F647-41AE-A55F-A283A47E97C4}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "StellaOps.Zastava", "StellaOps.Zastava", "{F1F029E6-2E4B-4A42-8D8F-AB325EE3B608}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Core", "StellaOps.Zastava.Core\StellaOps.Zastava.Core.csproj", "{CBE6E3D8-230C-4513-B98F-99D82B83B9F7}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Core.Tests", "StellaOps.Zastava.Core.Tests\StellaOps.Zastava.Core.Tests.csproj", "{821C7F88-B775-4D3C-8D89-850B6C34E818}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Webhook", "StellaOps.Zastava.Webhook\StellaOps.Zastava.Webhook.csproj", "{3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StellaOps.Zastava.Webhook.Tests", "StellaOps.Zastava.Webhook.Tests\StellaOps.Zastava.Webhook.Tests.csproj", "{3C500ECB-5422-4FFB-BD3D-48A850763D31}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -352,9 +440,21 @@ Global {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|Any CPU.ActiveCfg = Release|Any CPU {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|Any CPU.Build.0 = Release|Any CPU {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x64.ActiveCfg = Release|Any CPU - {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x64.Build.0 = Release|Any CPU - {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.ActiveCfg = Release|Any CPU - {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.Build.0 = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x64.Build.0 = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.ActiveCfg = Release|Any CPU + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A}.Release|x86.Build.0 = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|x64.ActiveCfg = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|x64.Build.0 = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|x86.ActiveCfg = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Debug|x86.Build.0 = Debug|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|Any CPU.Build.0 = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|x64.ActiveCfg = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|x64.Build.0 = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|x86.ActiveCfg = Release|Any CPU + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F}.Release|x86.Build.0 = Release|Any CPU {40094279-250C-42AE-992A-856718FEFBAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {40094279-250C-42AE-992A-856718FEFBAC}.Debug|Any CPU.Build.0 = Debug|Any CPU {40094279-250C-42AE-992A-856718FEFBAC}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -1627,6 +1727,486 @@ Global {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x64.Build.0 = Release|Any CPU {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x86.ActiveCfg = Release|Any CPU {CADA1364-8EB1-479E-AB6F-4105C26335C8}.Release|x86.Build.0 = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|x64.ActiveCfg = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|x64.Build.0 = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|x86.ActiveCfg = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Debug|x86.Build.0 = Debug|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|Any CPU.Build.0 = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|x64.ActiveCfg = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|x64.Build.0 = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|x86.ActiveCfg = Release|Any CPU + {8CC4441E-9D1A-4E00-831B-34828A3F9446}.Release|x86.Build.0 = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|Any CPU.Build.0 = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|x64.ActiveCfg = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|x64.Build.0 = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|x86.ActiveCfg = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Debug|x86.Build.0 = Debug|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|Any CPU.ActiveCfg = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|Any CPU.Build.0 = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|x64.ActiveCfg = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|x64.Build.0 = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|x86.ActiveCfg = Release|Any CPU + {01B8AC3F-1B97-4F79-93C6-BE1CBA26FE17}.Release|x86.Build.0 = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|x64.ActiveCfg = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|x64.Build.0 = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|x86.ActiveCfg = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Debug|x86.Build.0 = Debug|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|Any CPU.Build.0 = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|x64.ActiveCfg = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|x64.Build.0 = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|x86.ActiveCfg = Release|Any CPU + {37BB9502-CCD1-425A-BF45-D56968B0C2F9}.Release|x86.Build.0 = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|x64.ActiveCfg = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|x64.Build.0 = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|x86.ActiveCfg = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Debug|x86.Build.0 = Debug|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|Any CPU.Build.0 = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|x64.ActiveCfg = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|x64.Build.0 = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|x86.ActiveCfg = Release|Any CPU + {015A7A95-2C07-4C7F-8048-DB591AAC5FE5}.Release|x86.Build.0 = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|x64.ActiveCfg = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|x64.Build.0 = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|x86.ActiveCfg = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Debug|x86.Build.0 = Debug|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|Any CPU.Build.0 = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|x64.ActiveCfg = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|x64.Build.0 = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|x86.ActiveCfg = Release|Any CPU + {EF59DAD6-30CE-47CB-862A-DD79F31BFDE4}.Release|x86.Build.0 = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|Any CPU.Build.0 = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|x64.ActiveCfg = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|x64.Build.0 = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|x86.ActiveCfg = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Debug|x86.Build.0 = Debug|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|Any CPU.ActiveCfg = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|Any CPU.Build.0 = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|x64.ActiveCfg = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|x64.Build.0 = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|x86.ActiveCfg = Release|Any CPU + {27D951AD-696D-4330-B4F5-F8F81344C191}.Release|x86.Build.0 = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|x64.ActiveCfg = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|x64.Build.0 = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|x86.ActiveCfg = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Debug|x86.Build.0 = Debug|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|Any CPU.Build.0 = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|x64.ActiveCfg = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|x64.Build.0 = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|x86.ActiveCfg = Release|Any CPU + {31277AFF-9BFF-4C17-8593-B562A385058E}.Release|x86.Build.0 = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|x64.ActiveCfg = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|x64.Build.0 = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|x86.ActiveCfg = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Debug|x86.Build.0 = Debug|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|Any CPU.Build.0 = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|x64.ActiveCfg = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|x64.Build.0 = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|x86.ActiveCfg = Release|Any CPU + {3A8F090F-678D-46E2-8899-67402129749C}.Release|x86.Build.0 = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|x64.ActiveCfg = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|x64.Build.0 = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|x86.ActiveCfg = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Debug|x86.Build.0 = Debug|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|Any CPU.Build.0 = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|x64.ActiveCfg = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|x64.Build.0 = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|x86.ActiveCfg = Release|Any CPU + {19FACEC7-D6D4-40F5-84AD-14E2983F18F7}.Release|x86.Build.0 = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|x64.ActiveCfg = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|x64.Build.0 = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|x86.ActiveCfg = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Debug|x86.Build.0 = Debug|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|Any CPU.Build.0 = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|x64.ActiveCfg = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|x64.Build.0 = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|x86.ActiveCfg = Release|Any CPU + {8342286A-BE36-4ACA-87FF-EBEB4E268498}.Release|x86.Build.0 = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|x64.ActiveCfg = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|x64.Build.0 = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|x86.ActiveCfg = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Debug|x86.Build.0 = Debug|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|Any CPU.Build.0 = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|x64.ActiveCfg = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|x64.Build.0 = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|x86.ActiveCfg = Release|Any CPU + {05D844B6-51C1-4926-919C-D99E24FB3BC9}.Release|x86.Build.0 = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|x64.ActiveCfg = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|x64.Build.0 = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|x86.ActiveCfg = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Debug|x86.Build.0 = Debug|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|Any CPU.Build.0 = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|x64.ActiveCfg = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|x64.Build.0 = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|x86.ActiveCfg = Release|Any CPU + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE}.Release|x86.Build.0 = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|x64.ActiveCfg = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|x64.Build.0 = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|x86.ActiveCfg = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Debug|x86.Build.0 = Debug|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|Any CPU.Build.0 = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|x64.ActiveCfg = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|x64.Build.0 = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|x86.ActiveCfg = Release|Any CPU + {A072C46F-BA45-419E-B1B6-416919F78440}.Release|x86.Build.0 = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|x64.ActiveCfg = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|x64.Build.0 = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|x86.ActiveCfg = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Debug|x86.Build.0 = Debug|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|Any CPU.Build.0 = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|x64.ActiveCfg = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|x64.Build.0 = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|x86.ActiveCfg = Release|Any CPU + {6DE0F48D-8CEA-44C1-82FF-0DC891B33FE3}.Release|x86.Build.0 = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|Any CPU.Build.0 = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|x64.ActiveCfg = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|x64.Build.0 = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|x86.ActiveCfg = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Debug|x86.Build.0 = Debug|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|Any CPU.ActiveCfg = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|Any CPU.Build.0 = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|x64.ActiveCfg = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|x64.Build.0 = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|x86.ActiveCfg = Release|Any CPU + {10088067-7B8F-4D2E-A8E1-ED546DC17369}.Release|x86.Build.0 = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|x64.ActiveCfg = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|x64.Build.0 = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|x86.ActiveCfg = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Debug|x86.Build.0 = Debug|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|Any CPU.Build.0 = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|x64.ActiveCfg = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|x64.Build.0 = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|x86.ActiveCfg = Release|Any CPU + {E014565C-2456-4BD0-9481-557F939C1E36}.Release|x86.Build.0 = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|x64.ActiveCfg = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|x64.Build.0 = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|x86.ActiveCfg = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Debug|x86.Build.0 = Debug|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|Any CPU.Build.0 = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|x64.ActiveCfg = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|x64.Build.0 = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|x86.ActiveCfg = Release|Any CPU + {44825FDA-68D2-4675-8B1D-6D5303DC38CF}.Release|x86.Build.0 = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|x64.ActiveCfg = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|x64.Build.0 = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|x86.ActiveCfg = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Debug|x86.Build.0 = Debug|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|Any CPU.Build.0 = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|x64.ActiveCfg = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|x64.Build.0 = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|x86.ActiveCfg = Release|Any CPU + {6D46DB08-C8D1-4F67-A6D0-D50FE84F19E0}.Release|x86.Build.0 = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|x64.ActiveCfg = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|x64.Build.0 = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|x86.ActiveCfg = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Debug|x86.Build.0 = Debug|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|Any CPU.Build.0 = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|x64.ActiveCfg = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|x64.Build.0 = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|x86.ActiveCfg = Release|Any CPU + {5E5EB0A7-7A19-4144-81FE-13C31DB678B2}.Release|x86.Build.0 = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|x64.ActiveCfg = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|x64.Build.0 = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|x86.ActiveCfg = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Debug|x86.Build.0 = Debug|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|Any CPU.Build.0 = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|x64.ActiveCfg = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|x64.Build.0 = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|x86.ActiveCfg = Release|Any CPU + {7F3D4F33-341A-44A1-96EA-A1729BC2E5D8}.Release|x86.Build.0 = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|x64.ActiveCfg = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|x64.Build.0 = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|x86.ActiveCfg = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Debug|x86.Build.0 = Debug|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|Any CPU.Build.0 = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|x64.ActiveCfg = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|x64.Build.0 = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|x86.ActiveCfg = Release|Any CPU + {B86C287A-734E-4527-A03E-6B970F22E27E}.Release|x86.Build.0 = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|x64.ActiveCfg = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|x64.Build.0 = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|x86.ActiveCfg = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Debug|x86.Build.0 = Debug|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|Any CPU.Build.0 = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|x64.ActiveCfg = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|x64.Build.0 = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|x86.ActiveCfg = Release|Any CPU + {E23FBF14-EE5B-49D4-8938-E8368CF4A4B5}.Release|x86.Build.0 = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|x64.ActiveCfg = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|x64.Build.0 = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|x86.ActiveCfg = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Debug|x86.Build.0 = Debug|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|Any CPU.Build.0 = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|x64.ActiveCfg = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|x64.Build.0 = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|x86.ActiveCfg = Release|Any CPU + {50D014B5-99A6-46FC-B745-26687595B293}.Release|x86.Build.0 = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|x64.ActiveCfg = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|x64.Build.0 = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|x86.ActiveCfg = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Debug|x86.Build.0 = Debug|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|Any CPU.Build.0 = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|x64.ActiveCfg = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|x64.Build.0 = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|x86.ActiveCfg = Release|Any CPU + {D99C1F78-67EA-40E7-BD4C-985592F5265A}.Release|x86.Build.0 = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|x64.ActiveCfg = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|x64.Build.0 = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|x86.ActiveCfg = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Debug|x86.Build.0 = Debug|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|Any CPU.Build.0 = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|x64.ActiveCfg = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|x64.Build.0 = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|x86.ActiveCfg = Release|Any CPU + {1CBC0B9C-A96B-4143-B70F-37C69229FFF2}.Release|x86.Build.0 = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|x64.ActiveCfg = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|x64.Build.0 = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|x86.ActiveCfg = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Debug|x86.Build.0 = Debug|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|Any CPU.Build.0 = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|x64.ActiveCfg = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|x64.Build.0 = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|x86.ActiveCfg = Release|Any CPU + {760E2855-31B3-4CCB-BACB-34B7196A59B8}.Release|x86.Build.0 = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|x64.ActiveCfg = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|x64.Build.0 = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|x86.ActiveCfg = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Debug|x86.Build.0 = Debug|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|Any CPU.Build.0 = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|x64.ActiveCfg = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|x64.Build.0 = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|x86.ActiveCfg = Release|Any CPU + {3F688F21-7E31-4781-8995-9DD34276773F}.Release|x86.Build.0 = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|x64.ActiveCfg = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|x64.Build.0 = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|x86.ActiveCfg = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Debug|x86.Build.0 = Debug|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|Any CPU.Build.0 = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|x64.ActiveCfg = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|x64.Build.0 = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|x86.ActiveCfg = Release|Any CPU + {80AD7C4D-E4C6-4700-87AD-77B5698B338F}.Release|x86.Build.0 = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|Any CPU.Build.0 = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|x64.ActiveCfg = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|x64.Build.0 = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|x86.ActiveCfg = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Debug|x86.Build.0 = Debug|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|Any CPU.ActiveCfg = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|Any CPU.Build.0 = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|x64.ActiveCfg = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|x64.Build.0 = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|x86.ActiveCfg = Release|Any CPU + {60ABAB54-2EE9-4A16-A109-67F7B6F29184}.Release|x86.Build.0 = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|x64.ActiveCfg = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|x64.Build.0 = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|x86.ActiveCfg = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Debug|x86.Build.0 = Debug|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|Any CPU.Build.0 = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|x64.ActiveCfg = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|x64.Build.0 = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|x86.ActiveCfg = Release|Any CPU + {D32C1D26-C9A1-4F2A-9DBA-DBF0353E3972}.Release|x86.Build.0 = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|x64.ActiveCfg = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|x64.Build.0 = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|x86.ActiveCfg = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Debug|x86.Build.0 = Debug|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|Any CPU.Build.0 = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|x64.ActiveCfg = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|x64.Build.0 = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|x86.ActiveCfg = Release|Any CPU + {5CA4E28E-6305-4B21-AD2E-0DF24D47A65B}.Release|x86.Build.0 = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|x64.ActiveCfg = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|x64.Build.0 = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|x86.ActiveCfg = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Debug|x86.Build.0 = Debug|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|Any CPU.Build.0 = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|x64.ActiveCfg = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|x64.Build.0 = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|x86.ActiveCfg = Release|Any CPU + {05475C0A-C225-4F07-A3C7-9E17E660042E}.Release|x86.Build.0 = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|x64.ActiveCfg = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|x64.Build.0 = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|x86.ActiveCfg = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Debug|x86.Build.0 = Debug|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|Any CPU.Build.0 = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|x64.ActiveCfg = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|x64.Build.0 = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|x86.ActiveCfg = Release|Any CPU + {BA47D456-4657-4C86-A665-21293E3AC47F}.Release|x86.Build.0 = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|x64.ActiveCfg = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|x64.Build.0 = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|x86.ActiveCfg = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Debug|x86.Build.0 = Debug|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|Any CPU.Build.0 = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|x64.ActiveCfg = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|x64.Build.0 = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|x86.ActiveCfg = Release|Any CPU + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D}.Release|x86.Build.0 = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|x64.ActiveCfg = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|x64.Build.0 = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|x86.ActiveCfg = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Debug|x86.Build.0 = Debug|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|Any CPU.Build.0 = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|x64.ActiveCfg = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|x64.Build.0 = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|x86.ActiveCfg = Release|Any CPU + {C22333B3-D132-4960-A490-6BEF1EB1C917}.Release|x86.Build.0 = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|x64.ActiveCfg = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|x64.Build.0 = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|x86.ActiveCfg = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Debug|x86.Build.0 = Debug|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|Any CPU.Build.0 = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|x64.ActiveCfg = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|x64.Build.0 = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|x86.ActiveCfg = Release|Any CPU + {B8B15A8D-F647-41AE-A55F-A283A47E97C4}.Release|x86.Build.0 = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|x64.ActiveCfg = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|x64.Build.0 = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|x86.ActiveCfg = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Debug|x86.Build.0 = Debug|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|Any CPU.Build.0 = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|x64.ActiveCfg = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|x64.Build.0 = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|x86.ActiveCfg = Release|Any CPU + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7}.Release|x86.Build.0 = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|Any CPU.Build.0 = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|x64.ActiveCfg = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|x64.Build.0 = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|x86.ActiveCfg = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Debug|x86.Build.0 = Debug|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|Any CPU.ActiveCfg = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|Any CPU.Build.0 = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|x64.ActiveCfg = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|x64.Build.0 = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|x86.ActiveCfg = Release|Any CPU + {821C7F88-B775-4D3C-8D89-850B6C34E818}.Release|x86.Build.0 = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|x64.ActiveCfg = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|x64.Build.0 = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|x86.ActiveCfg = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Debug|x86.Build.0 = Debug|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|Any CPU.Build.0 = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|x64.ActiveCfg = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|x64.Build.0 = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|x86.ActiveCfg = Release|Any CPU + {3ABEAD26-B056-45CC-8F72-F40C8B8DBCBC}.Release|x86.Build.0 = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|x64.ActiveCfg = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|x64.Build.0 = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|x86.ActiveCfg = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Debug|x86.Build.0 = Debug|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|Any CPU.Build.0 = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|x64.ActiveCfg = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|x64.Build.0 = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|x86.ActiveCfg = Release|Any CPU + {3C500ECB-5422-4FFB-BD3D-48A850763D31}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1641,8 +2221,9 @@ Global {85AB3BB7-C493-4387-B39A-EB299AC37312} = {361838C4-72E2-1C48-5D76-CA6D1A861242} {5C5E91CA-3F98-4E9A-922B-F6415EABD1A3} = {361838C4-72E2-1C48-5D76-CA6D1A861242} {93DB06DC-B254-48A9-8F2C-6130A5658F27} = {361838C4-72E2-1C48-5D76-CA6D1A861242} - {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} - {40094279-250C-42AE-992A-856718FEFBAC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {03CA315C-8AA1-4CEA-A28B-5EB35C586F4A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {C6DC3C29-C2AD-4015-8872-42E95A0FE63F} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {40094279-250C-42AE-992A-856718FEFBAC} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {B2967228-F8F7-4931-B257-1C63CB58CE1D} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {6D52EC2B-0A1A-4693-A8EE-5AB32A4A3ED9} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {37F203A3-624E-4794-9C99-16CAC22C17DF} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} @@ -1722,5 +2303,16 @@ Global {A2E3F03A-0CAD-4E2A-8C71-DDEBB1B7E4F7} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {3A1AF0AD-4DAE-4D82-9CCF-2DCB83CC3679} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} {F1DF0F07-1BCB-4B55-8353-07BF8A4B2A67} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {31277AFF-9BFF-4C17-8593-B562A385058E} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {3A8F090F-678D-46E2-8899-67402129749C} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {05D844B6-51C1-4926-919C-D99E24FB3BC9} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {03E15545-D6A0-4287-A88C-6EDE77C0DCBE} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {BA47D456-4657-4C86-A665-21293E3AC47F} = {78C966F5-2242-D8EC-ADCA-A1A9C7F723A6} + {49EF86AC-1CC2-4A24-8637-C5151E23DF9D} = {78C966F5-2242-D8EC-ADCA-A1A9C7F723A6} + {C22333B3-D132-4960-A490-6BEF1EB1C917} = {78C966F5-2242-D8EC-ADCA-A1A9C7F723A6} + {B8B15A8D-F647-41AE-A55F-A283A47E97C4} = {78C966F5-2242-D8EC-ADCA-A1A9C7F723A6} + {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {CBE6E3D8-230C-4513-B98F-99D82B83B9F7} = {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} + {821C7F88-B775-4D3C-8D89-850B6C34E818} = {F1F029E6-2E4B-4A42-8D8F-AB325EE3B608} EndGlobalSection EndGlobal diff --git a/tools/FixtureUpdater/FixtureUpdater.csproj b/tools/FixtureUpdater/FixtureUpdater.csproj index ee29c5d9..d0436e1b 100644 --- a/tools/FixtureUpdater/FixtureUpdater.csproj +++ b/tools/FixtureUpdater/FixtureUpdater.csproj @@ -1,20 +1,20 @@ - - - - Exe - net10.0 - enable - enable - - - - - - - - - - - - - + + + + Exe + net10.0 + enable + enable + + + + + + + + + + + + + diff --git a/tools/FixtureUpdater/Program.cs b/tools/FixtureUpdater/Program.cs index 6387c5dd..86626780 100644 --- a/tools/FixtureUpdater/Program.cs +++ b/tools/FixtureUpdater/Program.cs @@ -1,378 +1,378 @@ -using System.Linq; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using MongoDB.Bson; -using StellaOps.Feedser.Models; -using StellaOps.Feedser.Source.Ghsa; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Ghsa.Internal; -using StellaOps.Feedser.Source.Osv.Internal; -using StellaOps.Feedser.Source.Osv; -using StellaOps.Feedser.Source.Nvd; -using StellaOps.Feedser.Storage.Mongo.Documents; -using StellaOps.Feedser.Storage.Mongo.Dtos; - -var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) -{ - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, -}; - -var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); - -var osvFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Osv.Tests", "Fixtures"); -var ghsaFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Ghsa.Tests", "Fixtures"); -var nvdFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Feedser.Source.Nvd.Tests", "Nvd", "Fixtures"); - -RewriteOsvFixtures(osvFixturesPath); -RewriteSnapshotFixtures(osvFixturesPath); -RewriteGhsaFixtures(osvFixturesPath); -RewriteCreditParityFixtures(ghsaFixturesPath, nvdFixturesPath); -return; - -void RewriteOsvFixtures(string fixturesPath) -{ - var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-osv.json"); - if (!File.Exists(rawPath)) - { - Console.WriteLine($"[FixtureUpdater] OSV raw fixture missing: {rawPath}"); - return; - } - - using var document = JsonDocument.Parse(File.ReadAllText(rawPath)); - var advisories = new List(); - foreach (var element in document.RootElement.EnumerateArray()) - { - var dto = JsonSerializer.Deserialize(element.GetRawText(), serializerOptions); - if (dto is null) - { - continue; - } - - var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "unknown"; - var uri = new Uri($"https://osv.dev/vulnerability/{dto.Id}"); - var documentRecord = new DocumentRecord( - Guid.NewGuid(), - OsvConnectorPlugin.SourceName, - uri.ToString(), - DateTimeOffset.UtcNow, - "fixture-sha", - DocumentStatuses.PendingMap, - "application/json", - null, - new Dictionary(StringComparer.Ordinal) - { - ["osv.ecosystem"] = ecosystem, - }, - null, - DateTimeOffset.UtcNow, - null, - null); - - var payload = BsonDocument.Parse(element.GetRawText()); - var dtoRecord = new DtoRecord( - Guid.NewGuid(), - documentRecord.Id, - OsvConnectorPlugin.SourceName, - "osv.v1", - payload, - DateTimeOffset.UtcNow); - - var advisory = OsvMapper.Map(dto, documentRecord, dtoRecord, ecosystem); - advisories.Add(advisory); - } - - advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); - var snapshot = SnapshotSerializer.ToSnapshot(advisories); - File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.osv.json"), snapshot); - Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.osv.json")}"); -} - -void RewriteSnapshotFixtures(string fixturesPath) -{ - var baselinePublished = new DateTimeOffset(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); - var baselineModified = new DateTimeOffset(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); - var baselineFetched = new DateTimeOffset(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); - - var cases = new (string Ecosystem, string Purl, string PackageName, string SnapshotFile)[] - { - ("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json"), - ("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json"), - }; - - foreach (var (ecosystem, purl, packageName, snapshotFile) in cases) - { - var dto = new OsvVulnerabilityDto - { - Id = $"OSV-2025-{ecosystem}-0001", - Summary = $"{ecosystem} package vulnerability", - Details = $"Detailed description for {ecosystem} package {packageName}.", - Published = baselinePublished, - Modified = baselineModified, - Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, - Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, - References = new[] - { - new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, - new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, - }, - Severity = new[] - { - new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, - }, - Affected = new[] - { - new OsvAffectedPackageDto - { - Package = new OsvPackageDto - { - Ecosystem = ecosystem, - Name = packageName, - Purl = purl, - }, - Ranges = new[] - { - new OsvRangeDto - { - Type = "SEMVER", - Events = new[] - { - new OsvEventDto { Introduced = "0" }, - new OsvEventDto { Fixed = "2.0.0" }, - }, - }, - }, - Versions = new[] { "1.0.0", "1.5.0" }, - EcosystemSpecific = JsonDocument.Parse("{\"severity\":\"high\"}").RootElement.Clone(), - }, - }, - DatabaseSpecific = JsonDocument.Parse("{\"source\":\"osv.dev\"}").RootElement.Clone(), - }; - - var document = new DocumentRecord( - Guid.NewGuid(), - OsvConnectorPlugin.SourceName, - $"https://osv.dev/vulnerability/{dto.Id}", - baselineFetched, - "fixture-sha", - DocumentStatuses.PendingParse, - "application/json", - null, - new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, - null, - baselineModified, - null); - - var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, serializerOptions)); - var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, baselineModified); - - var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); - var snapshot = SnapshotSerializer.ToSnapshot(advisory); - File.WriteAllText(Path.Combine(fixturesPath, snapshotFile), snapshot); - Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, snapshotFile)}"); - } -} - -void RewriteGhsaFixtures(string fixturesPath) -{ - var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-ghsa.json"); - if (!File.Exists(rawPath)) - { - Console.WriteLine($"[FixtureUpdater] GHSA raw fixture missing: {rawPath}"); - return; - } - - JsonDocument document; - try - { - document = JsonDocument.Parse(File.ReadAllText(rawPath)); - } - catch (JsonException ex) - { - Console.WriteLine($"[FixtureUpdater] Failed to parse GHSA raw fixture '{rawPath}': {ex.Message}"); - return; - } - using (document) - { - var advisories = new List(); - foreach (var element in document.RootElement.EnumerateArray()) - { - GhsaRecordDto dto; - try - { - dto = GhsaRecordParser.Parse(Encoding.UTF8.GetBytes(element.GetRawText())); - } - catch (JsonException) - { - continue; - } - - var uri = new Uri($"https://github.com/advisories/{dto.GhsaId}"); - var documentRecord = new DocumentRecord( - Guid.NewGuid(), - GhsaConnectorPlugin.SourceName, - uri.ToString(), - DateTimeOffset.UtcNow, - "fixture-sha", - DocumentStatuses.PendingMap, - "application/json", - null, - new Dictionary(StringComparer.Ordinal), - null, - DateTimeOffset.UtcNow, - null, - null); - - var advisory = GhsaMapper.Map(dto, documentRecord, DateTimeOffset.UtcNow); - advisories.Add(advisory); - } - - advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); - var snapshot = SnapshotSerializer.ToSnapshot(advisories); - File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.ghsa.json"), snapshot); - Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.ghsa.json")}"); - } -} - -void RewriteCreditParityFixtures(string ghsaFixturesPath, string nvdFixturesPath) -{ - Directory.CreateDirectory(ghsaFixturesPath); - Directory.CreateDirectory(nvdFixturesPath); - - var advisoryKeyGhsa = "GHSA-credit-parity"; - var advisoryKeyNvd = "CVE-2025-5555"; - var recordedAt = new DateTimeOffset(2025, 10, 10, 15, 0, 0, TimeSpan.Zero); - var published = new DateTimeOffset(2025, 10, 9, 18, 30, 0, TimeSpan.Zero); - var modified = new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero); - - AdvisoryCredit[] CreateCredits(string source) => - [ - CreateCredit("Alice Researcher", "reporter", new[] { "mailto:alice.researcher@example.com" }, source), - CreateCredit("Bob Maintainer", "remediation_developer", new[] { "https://github.com/acme/bob-maintainer" }, source) - ]; - - AdvisoryCredit CreateCredit(string displayName, string role, IReadOnlyList contacts, string source) - { - var provenance = new AdvisoryProvenance( - source, - "credit", - $"{source}:{displayName.ToLowerInvariant().Replace(' ', '-')}", - recordedAt, - new[] { ProvenanceFieldMasks.Credits }); - - return new AdvisoryCredit(displayName, role, contacts, provenance); - } - - AdvisoryReference[] CreateReferences(string sourceName, params (string Url, string Kind)[] entries) - { - if (entries is null || entries.Length == 0) - { - return Array.Empty(); - } - - var references = new List(entries.Length); - foreach (var entry in entries) - { - var provenance = new AdvisoryProvenance( - sourceName, - "reference", - entry.Url, - recordedAt, - new[] { ProvenanceFieldMasks.References }); - - references.Add(new AdvisoryReference( - entry.Url, - entry.Kind, - sourceTag: null, - summary: null, - provenance)); - } - - return references.ToArray(); - } - - Advisory CreateAdvisory( - string sourceName, - string advisoryKey, - IEnumerable aliases, - AdvisoryCredit[] credits, - AdvisoryReference[] references, - string documentValue) - { - var documentProvenance = new AdvisoryProvenance( - sourceName, - "document", - documentValue, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - var mappingProvenance = new AdvisoryProvenance( - sourceName, - "mapping", - advisoryKey, - recordedAt, - new[] { ProvenanceFieldMasks.Advisory }); - - return new Advisory( - advisoryKey, - "Credit parity regression fixture", - "Credit parity regression fixture", - "en", - published, - modified, - "moderate", - exploitKnown: false, - aliases, - credits, - references, - Array.Empty(), - Array.Empty(), - new[] { documentProvenance, mappingProvenance }); - } - - var ghsa = CreateAdvisory( - "ghsa", - advisoryKeyGhsa, - new[] { advisoryKeyGhsa, advisoryKeyNvd }, - CreateCredits("ghsa"), - CreateReferences( - "ghsa", - ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), - ( "https://example.com/ghsa/patch", "patch")), - $"security/advisories/{advisoryKeyGhsa}"); - - var osv = CreateAdvisory( - OsvConnectorPlugin.SourceName, - advisoryKeyGhsa, - new[] { advisoryKeyGhsa, advisoryKeyNvd }, - CreateCredits(OsvConnectorPlugin.SourceName), - CreateReferences( - OsvConnectorPlugin.SourceName, - ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), - ( $"https://osv.dev/vulnerability/{advisoryKeyGhsa}", "advisory")), - $"https://osv.dev/vulnerability/{advisoryKeyGhsa}"); - - var nvd = CreateAdvisory( - NvdConnectorPlugin.SourceName, - advisoryKeyNvd, - new[] { advisoryKeyNvd, advisoryKeyGhsa }, - CreateCredits(NvdConnectorPlugin.SourceName), - CreateReferences( - NvdConnectorPlugin.SourceName, - ( $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}", "advisory"), - ( "https://example.com/nvd/reference", "report")), - $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}"); - - var ghsaSnapshot = SnapshotSerializer.ToSnapshot(ghsa); - var osvSnapshot = SnapshotSerializer.ToSnapshot(osv); - var nvdSnapshot = SnapshotSerializer.ToSnapshot(nvd); - - File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); - File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.osv.json"), osvSnapshot); - File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); - - File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); - File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.osv.json"), osvSnapshot); - File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); - - Console.WriteLine($"[FixtureUpdater] Updated credit parity fixtures under {ghsaFixturesPath} and {nvdFixturesPath}"); -} +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using MongoDB.Bson; +using StellaOps.Concelier.Models; +using StellaOps.Concelier.Connector.Ghsa; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Ghsa.Internal; +using StellaOps.Concelier.Connector.Osv.Internal; +using StellaOps.Concelier.Connector.Osv; +using StellaOps.Concelier.Connector.Nvd; +using StellaOps.Concelier.Storage.Mongo.Documents; +using StellaOps.Concelier.Storage.Mongo.Dtos; + +var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web) +{ + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, +}; + +var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..")); + +var osvFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Osv.Tests", "Fixtures"); +var ghsaFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Ghsa.Tests", "Fixtures"); +var nvdFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Nvd.Tests", "Nvd", "Fixtures"); + +RewriteOsvFixtures(osvFixturesPath); +RewriteSnapshotFixtures(osvFixturesPath); +RewriteGhsaFixtures(osvFixturesPath); +RewriteCreditParityFixtures(ghsaFixturesPath, nvdFixturesPath); +return; + +void RewriteOsvFixtures(string fixturesPath) +{ + var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-osv.json"); + if (!File.Exists(rawPath)) + { + Console.WriteLine($"[FixtureUpdater] OSV raw fixture missing: {rawPath}"); + return; + } + + using var document = JsonDocument.Parse(File.ReadAllText(rawPath)); + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + var dto = JsonSerializer.Deserialize(element.GetRawText(), serializerOptions); + if (dto is null) + { + continue; + } + + var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "unknown"; + var uri = new Uri($"https://osv.dev/vulnerability/{dto.Id}"); + var documentRecord = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + uri.ToString(), + DateTimeOffset.UtcNow, + "fixture-sha", + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) + { + ["osv.ecosystem"] = ecosystem, + }, + null, + DateTimeOffset.UtcNow, + null, + null); + + var payload = BsonDocument.Parse(element.GetRawText()); + var dtoRecord = new DtoRecord( + Guid.NewGuid(), + documentRecord.Id, + OsvConnectorPlugin.SourceName, + "osv.v1", + payload, + DateTimeOffset.UtcNow); + + var advisory = OsvMapper.Map(dto, documentRecord, dtoRecord, ecosystem); + advisories.Add(advisory); + } + + advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.osv.json"), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.osv.json")}"); +} + +void RewriteSnapshotFixtures(string fixturesPath) +{ + var baselinePublished = new DateTimeOffset(2025, 1, 5, 12, 0, 0, TimeSpan.Zero); + var baselineModified = new DateTimeOffset(2025, 1, 8, 6, 30, 0, TimeSpan.Zero); + var baselineFetched = new DateTimeOffset(2025, 1, 8, 7, 0, 0, TimeSpan.Zero); + + var cases = new (string Ecosystem, string Purl, string PackageName, string SnapshotFile)[] + { + ("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json"), + ("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json"), + }; + + foreach (var (ecosystem, purl, packageName, snapshotFile) in cases) + { + var dto = new OsvVulnerabilityDto + { + Id = $"OSV-2025-{ecosystem}-0001", + Summary = $"{ecosystem} package vulnerability", + Details = $"Detailed description for {ecosystem} package {packageName}.", + Published = baselinePublished, + Modified = baselineModified, + Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" }, + Related = new[] { $"OSV-RELATED-{ecosystem}-42" }, + References = new[] + { + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" }, + new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" }, + }, + Severity = new[] + { + new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }, + }, + Affected = new[] + { + new OsvAffectedPackageDto + { + Package = new OsvPackageDto + { + Ecosystem = ecosystem, + Name = packageName, + Purl = purl, + }, + Ranges = new[] + { + new OsvRangeDto + { + Type = "SEMVER", + Events = new[] + { + new OsvEventDto { Introduced = "0" }, + new OsvEventDto { Fixed = "2.0.0" }, + }, + }, + }, + Versions = new[] { "1.0.0", "1.5.0" }, + EcosystemSpecific = JsonDocument.Parse("{\"severity\":\"high\"}").RootElement.Clone(), + }, + }, + DatabaseSpecific = JsonDocument.Parse("{\"source\":\"osv.dev\"}").RootElement.Clone(), + }; + + var document = new DocumentRecord( + Guid.NewGuid(), + OsvConnectorPlugin.SourceName, + $"https://osv.dev/vulnerability/{dto.Id}", + baselineFetched, + "fixture-sha", + DocumentStatuses.PendingParse, + "application/json", + null, + new Dictionary(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem }, + null, + baselineModified, + null); + + var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, serializerOptions)); + var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, baselineModified); + + var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem); + var snapshot = SnapshotSerializer.ToSnapshot(advisory); + File.WriteAllText(Path.Combine(fixturesPath, snapshotFile), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, snapshotFile)}"); + } +} + +void RewriteGhsaFixtures(string fixturesPath) +{ + var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-ghsa.json"); + if (!File.Exists(rawPath)) + { + Console.WriteLine($"[FixtureUpdater] GHSA raw fixture missing: {rawPath}"); + return; + } + + JsonDocument document; + try + { + document = JsonDocument.Parse(File.ReadAllText(rawPath)); + } + catch (JsonException ex) + { + Console.WriteLine($"[FixtureUpdater] Failed to parse GHSA raw fixture '{rawPath}': {ex.Message}"); + return; + } + using (document) + { + var advisories = new List(); + foreach (var element in document.RootElement.EnumerateArray()) + { + GhsaRecordDto dto; + try + { + dto = GhsaRecordParser.Parse(Encoding.UTF8.GetBytes(element.GetRawText())); + } + catch (JsonException) + { + continue; + } + + var uri = new Uri($"https://github.com/advisories/{dto.GhsaId}"); + var documentRecord = new DocumentRecord( + Guid.NewGuid(), + GhsaConnectorPlugin.SourceName, + uri.ToString(), + DateTimeOffset.UtcNow, + "fixture-sha", + DocumentStatuses.PendingMap, + "application/json", + null, + new Dictionary(StringComparer.Ordinal), + null, + DateTimeOffset.UtcNow, + null, + null); + + var advisory = GhsaMapper.Map(dto, documentRecord, DateTimeOffset.UtcNow); + advisories.Add(advisory); + } + + advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal)); + var snapshot = SnapshotSerializer.ToSnapshot(advisories); + File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.ghsa.json"), snapshot); + Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.ghsa.json")}"); + } +} + +void RewriteCreditParityFixtures(string ghsaFixturesPath, string nvdFixturesPath) +{ + Directory.CreateDirectory(ghsaFixturesPath); + Directory.CreateDirectory(nvdFixturesPath); + + var advisoryKeyGhsa = "GHSA-credit-parity"; + var advisoryKeyNvd = "CVE-2025-5555"; + var recordedAt = new DateTimeOffset(2025, 10, 10, 15, 0, 0, TimeSpan.Zero); + var published = new DateTimeOffset(2025, 10, 9, 18, 30, 0, TimeSpan.Zero); + var modified = new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero); + + AdvisoryCredit[] CreateCredits(string source) => + [ + CreateCredit("Alice Researcher", "reporter", new[] { "mailto:alice.researcher@example.com" }, source), + CreateCredit("Bob Maintainer", "remediation_developer", new[] { "https://github.com/acme/bob-maintainer" }, source) + ]; + + AdvisoryCredit CreateCredit(string displayName, string role, IReadOnlyList contacts, string source) + { + var provenance = new AdvisoryProvenance( + source, + "credit", + $"{source}:{displayName.ToLowerInvariant().Replace(' ', '-')}", + recordedAt, + new[] { ProvenanceFieldMasks.Credits }); + + return new AdvisoryCredit(displayName, role, contacts, provenance); + } + + AdvisoryReference[] CreateReferences(string sourceName, params (string Url, string Kind)[] entries) + { + if (entries is null || entries.Length == 0) + { + return Array.Empty(); + } + + var references = new List(entries.Length); + foreach (var entry in entries) + { + var provenance = new AdvisoryProvenance( + sourceName, + "reference", + entry.Url, + recordedAt, + new[] { ProvenanceFieldMasks.References }); + + references.Add(new AdvisoryReference( + entry.Url, + entry.Kind, + sourceTag: null, + summary: null, + provenance)); + } + + return references.ToArray(); + } + + Advisory CreateAdvisory( + string sourceName, + string advisoryKey, + IEnumerable aliases, + AdvisoryCredit[] credits, + AdvisoryReference[] references, + string documentValue) + { + var documentProvenance = new AdvisoryProvenance( + sourceName, + "document", + documentValue, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + var mappingProvenance = new AdvisoryProvenance( + sourceName, + "mapping", + advisoryKey, + recordedAt, + new[] { ProvenanceFieldMasks.Advisory }); + + return new Advisory( + advisoryKey, + "Credit parity regression fixture", + "Credit parity regression fixture", + "en", + published, + modified, + "moderate", + exploitKnown: false, + aliases, + credits, + references, + Array.Empty(), + Array.Empty(), + new[] { documentProvenance, mappingProvenance }); + } + + var ghsa = CreateAdvisory( + "ghsa", + advisoryKeyGhsa, + new[] { advisoryKeyGhsa, advisoryKeyNvd }, + CreateCredits("ghsa"), + CreateReferences( + "ghsa", + ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), + ( "https://example.com/ghsa/patch", "patch")), + $"security/advisories/{advisoryKeyGhsa}"); + + var osv = CreateAdvisory( + OsvConnectorPlugin.SourceName, + advisoryKeyGhsa, + new[] { advisoryKeyGhsa, advisoryKeyNvd }, + CreateCredits(OsvConnectorPlugin.SourceName), + CreateReferences( + OsvConnectorPlugin.SourceName, + ( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"), + ( $"https://osv.dev/vulnerability/{advisoryKeyGhsa}", "advisory")), + $"https://osv.dev/vulnerability/{advisoryKeyGhsa}"); + + var nvd = CreateAdvisory( + NvdConnectorPlugin.SourceName, + advisoryKeyNvd, + new[] { advisoryKeyNvd, advisoryKeyGhsa }, + CreateCredits(NvdConnectorPlugin.SourceName), + CreateReferences( + NvdConnectorPlugin.SourceName, + ( $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}", "advisory"), + ( "https://example.com/nvd/reference", "report")), + $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}"); + + var ghsaSnapshot = SnapshotSerializer.ToSnapshot(ghsa); + var osvSnapshot = SnapshotSerializer.ToSnapshot(osv); + var nvdSnapshot = SnapshotSerializer.ToSnapshot(nvd); + + File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); + File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.osv.json"), osvSnapshot); + File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); + + File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot); + File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.osv.json"), osvSnapshot); + File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.nvd.json"), nvdSnapshot); + + Console.WriteLine($"[FixtureUpdater] Updated credit parity fixtures under {ghsaFixturesPath} and {nvdFixturesPath}"); +} diff --git a/tools/SourceStateSeeder/Program.cs b/tools/SourceStateSeeder/Program.cs index 183c5166..863655fd 100644 --- a/tools/SourceStateSeeder/Program.cs +++ b/tools/SourceStateSeeder/Program.cs @@ -1,14 +1,13 @@ using System.Globalization; -using System.Security.Cryptography; using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -using MongoDB.Bson; using MongoDB.Driver; -using StellaOps.Feedser.Source.Common; -using StellaOps.Feedser.Source.Common.Fetch; -using StellaOps.Feedser.Storage.Mongo; -using StellaOps.Feedser.Storage.Mongo.Documents; +using StellaOps.Concelier.Connector.Common; +using StellaOps.Concelier.Connector.Common.Fetch; +using StellaOps.Concelier.Connector.Common.State; +using StellaOps.Concelier.Storage.Mongo; +using StellaOps.Concelier.Storage.Mongo.Documents; namespace SourceStateSeeder; @@ -40,58 +39,28 @@ internal static class Program return 1; } + var specification = await BuildSpecificationAsync(seed, sourceName, options.InputPath, CancellationToken.None).ConfigureAwait(false); + var client = new MongoClient(options.ConnectionString); var database = client.GetDatabase(options.DatabaseName); - var loggerFactory = NullLoggerFactory.Instance; + var documentStore = new DocumentStore(database, loggerFactory.CreateLogger()); var rawStorage = new RawDocumentStorage(database); var stateRepository = new MongoSourceStateRepository(database, loggerFactory.CreateLogger()); - var pendingDocumentIds = new List(); - var pendingMappingIds = new List(); - var knownAdvisories = new List(); - - var now = DateTimeOffset.UtcNow; - var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(options.InputPath)) ?? Directory.GetCurrentDirectory(); - - foreach (var document in seed.Documents) - { - var (record, addedToPendingDocs, addedToPendingMaps, known) = await UpsertDocumentAsync( - documentStore, - rawStorage, - sourceName, - baseDirectory, - now, - document, - cancellationToken: default).ConfigureAwait(false); - - if (addedToPendingDocs) - { - pendingDocumentIds.Add(record.Id); - } - - if (addedToPendingMaps) - { - pendingMappingIds.Add(record.Id); - } - - if (known is not null) - { - knownAdvisories.AddRange(known); - } - } - - await UpdateCursorAsync( + var processor = new SourceStateSeedProcessor( + documentStore, + rawStorage, stateRepository, - sourceName, - seed.Cursor, - pendingDocumentIds, - pendingMappingIds, - knownAdvisories, - now).ConfigureAwait(false); + TimeProvider.System, + loggerFactory.CreateLogger()); - Console.WriteLine($"Seeded {pendingDocumentIds.Count + pendingMappingIds.Count} documents for {sourceName}."); + var result = await processor.ProcessAsync(specification, CancellationToken.None).ConfigureAwait(false); + + Console.WriteLine( + $"Seeded {result.DocumentsProcessed} document(s) for {sourceName} " + + $"(pendingDocuments+= {result.PendingDocumentsAdded}, pendingMappings+= {result.PendingMappingsAdded}, knownAdvisories+= {result.KnownAdvisoriesAdded.Count})."); return 0; } catch (Exception ex) @@ -109,13 +78,33 @@ internal static class Program return seed; } - private static async Task<(DocumentRecord Record, bool PendingDoc, bool PendingMap, IReadOnlyCollection? Known)> UpsertDocumentAsync( - DocumentStore documentStore, - RawDocumentStorage rawStorage, + private static async Task BuildSpecificationAsync( + StateSeed seed, string sourceName, - string baseDirectory, - DateTimeOffset fetchedAt, + string inputPath, + CancellationToken cancellationToken) + { + var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(inputPath)) ?? Directory.GetCurrentDirectory(); + var documents = new List(seed.Documents.Count); + + foreach (var documentSeed in seed.Documents) + { + documents.Add(await BuildDocumentAsync(documentSeed, baseDirectory, cancellationToken).ConfigureAwait(false)); + } + + return new SourceStateSeedSpecification + { + Source = sourceName, + Documents = documents.AsReadOnly(), + Cursor = BuildCursor(seed.Cursor), + KnownAdvisories = NormalizeStrings(seed.KnownAdvisories), + CompletedAt = seed.CompletedAt, + }; + } + + private static async Task BuildDocumentAsync( DocumentSeed seed, + string baseDirectory, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(seed.Uri)) @@ -128,152 +117,120 @@ internal static class Program throw new InvalidOperationException($"Seed entry for '{seed.Uri}' missing 'contentFile'."); } - var contentPath = Path.IsPathRooted(seed.ContentFile) - ? seed.ContentFile - : Path.GetFullPath(Path.Combine(baseDirectory, seed.ContentFile)); - + var contentPath = ResolvePath(seed.ContentFile, baseDirectory); if (!File.Exists(contentPath)) { throw new FileNotFoundException($"Content file not found for '{seed.Uri}'.", contentPath); } var contentBytes = await File.ReadAllBytesAsync(contentPath, cancellationToken).ConfigureAwait(false); - var sha256 = Convert.ToHexString(SHA256.HashData(contentBytes)).ToLowerInvariant(); - var gridId = await rawStorage.UploadAsync( - sourceName, - seed.Uri, - contentBytes, - seed.ContentType, - seed.ExpiresAt, - cancellationToken).ConfigureAwait(false); var metadata = seed.Metadata is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) + ? null : new Dictionary(seed.Metadata, StringComparer.OrdinalIgnoreCase); var headers = seed.Headers is null - ? new Dictionary(StringComparer.OrdinalIgnoreCase) + ? null : new Dictionary(seed.Headers, StringComparer.OrdinalIgnoreCase); - if (!headers.ContainsKey("content-type") && !string.IsNullOrWhiteSpace(seed.ContentType)) + if (!string.IsNullOrWhiteSpace(seed.ContentType)) { - headers["content-type"] = seed.ContentType!; + headers ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + if (!headers.ContainsKey("content-type")) + { + headers["content-type"] = seed.ContentType!; + } } - var lastModified = seed.LastModified is null - ? (DateTimeOffset?)null - : DateTimeOffset.Parse(seed.LastModified, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); - - var record = new DocumentRecord( - Guid.NewGuid(), - sourceName, - seed.Uri, - fetchedAt, - sha256, - string.IsNullOrWhiteSpace(seed.Status) ? DocumentStatuses.PendingParse : seed.Status, - seed.ContentType, - headers, - metadata, - seed.Etag, - lastModified, - gridId, - seed.ExpiresAt); - - var upserted = await documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false); - - return (upserted, seed.AddToPendingDocuments, seed.AddToPendingMappings, seed.KnownIdentifiers); + return new SourceStateSeedDocument + { + Uri = seed.Uri, + DocumentId = seed.DocumentId, + Content = contentBytes, + ContentType = seed.ContentType, + Status = string.IsNullOrWhiteSpace(seed.Status) ? DocumentStatuses.PendingParse : seed.Status, + Headers = headers, + Metadata = metadata, + Etag = seed.Etag, + LastModified = ParseOptionalDate(seed.LastModified), + ExpiresAt = seed.ExpiresAt, + FetchedAt = ParseOptionalDate(seed.FetchedAt), + AddToPendingDocuments = seed.AddToPendingDocuments, + AddToPendingMappings = seed.AddToPendingMappings, + KnownIdentifiers = NormalizeStrings(seed.KnownIdentifiers), + }; } - private static async Task UpdateCursorAsync( - ISourceStateRepository repository, - string sourceName, - CursorSeed? cursorSeed, - IReadOnlyCollection pendingDocuments, - IReadOnlyCollection pendingMappings, - IReadOnlyCollection knownAdvisories, - DateTimeOffset completedAt) + private static SourceStateSeedCursor? BuildCursor(CursorSeed? cursorSeed) { - var state = await repository.TryGetAsync(sourceName, CancellationToken.None).ConfigureAwait(false); - var cursor = state?.Cursor ?? new BsonDocument(); - - MergeGuidArray(cursor, "pendingDocuments", pendingDocuments); - MergeGuidArray(cursor, "pendingMappings", pendingMappings); - - if (knownAdvisories.Count > 0) + if (cursorSeed is null) { - MergeStringArray(cursor, "knownAdvisories", knownAdvisories); + return null; } - if (cursorSeed is not null) + return new SourceStateSeedCursor { - if (cursorSeed.LastModifiedCursor.HasValue) - { - cursor["lastModifiedCursor"] = cursorSeed.LastModifiedCursor.Value.UtcDateTime; - } - - if (cursorSeed.LastFetchAt.HasValue) - { - cursor["lastFetchAt"] = cursorSeed.LastFetchAt.Value.UtcDateTime; - } - - if (cursorSeed.Additional is not null) - { - foreach (var kvp in cursorSeed.Additional) - { - cursor[kvp.Key] = kvp.Value; - } - } - } - - cursor["lastSeededAt"] = completedAt.UtcDateTime; - - await repository.UpdateCursorAsync(sourceName, cursor, completedAt, CancellationToken.None).ConfigureAwait(false); + PendingDocuments = NormalizeGuids(cursorSeed.PendingDocuments), + PendingMappings = NormalizeGuids(cursorSeed.PendingMappings), + KnownAdvisories = NormalizeStrings(cursorSeed.KnownAdvisories), + LastModifiedCursor = cursorSeed.LastModifiedCursor, + LastFetchAt = cursorSeed.LastFetchAt, + Additional = cursorSeed.Additional is null + ? null + : new Dictionary(cursorSeed.Additional, StringComparer.OrdinalIgnoreCase), + }; } - private static void MergeGuidArray(BsonDocument cursor, string field, IReadOnlyCollection values) + private static IReadOnlyCollection? NormalizeGuids(IEnumerable? values) { - if (values.Count == 0) + if (values is null) { - return; + return null; } - var existing = cursor.TryGetValue(field, out var value) && value is BsonArray array - ? array.Select(v => Guid.TryParse(v?.AsString, out var parsed) ? parsed : Guid.Empty) - .Where(g => g != Guid.Empty) - .ToHashSet() - : new HashSet(); - + var set = new HashSet(); foreach (var guid in values) { - existing.Add(guid); - } - - cursor[field] = new BsonArray(existing.Select(g => g.ToString())); - } - - private static void MergeStringArray(BsonDocument cursor, string field, IReadOnlyCollection values) - { - if (values.Count == 0) - { - return; - } - - var existing = cursor.TryGetValue(field, out var value) && value is BsonArray array - ? array.Select(v => v?.AsString ?? string.Empty) - .Where(s => !string.IsNullOrWhiteSpace(s)) - .ToHashSet(StringComparer.OrdinalIgnoreCase) - : new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var entry in values) - { - if (!string.IsNullOrWhiteSpace(entry)) + if (guid != Guid.Empty) { - existing.Add(entry.Trim()); + set.Add(guid); } } - cursor[field] = new BsonArray(existing.OrderBy(s => s, StringComparer.OrdinalIgnoreCase)); + return set.Count == 0 ? null : set.ToList(); } + + private static IReadOnlyCollection? NormalizeStrings(IEnumerable? values) + { + if (values is null) + { + return null; + } + + var set = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var value in values) + { + if (!string.IsNullOrWhiteSpace(value)) + { + set.Add(value.Trim()); + } + } + + return set.Count == 0 ? null : set.ToList(); + } + + private static DateTimeOffset? ParseOptionalDate(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + return DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal); + } + + private static string ResolvePath(string path, string baseDirectory) + => Path.IsPathRooted(path) ? path : Path.GetFullPath(Path.Combine(baseDirectory, path)); } internal sealed record SeedOptions @@ -356,12 +313,15 @@ internal sealed record StateSeed public string? Source { get; init; } public List Documents { get; init; } = new(); public CursorSeed? Cursor { get; init; } + public List? KnownAdvisories { get; init; } + public DateTimeOffset? CompletedAt { get; init; } } internal sealed record DocumentSeed { public string Uri { get; init; } = string.Empty; public string ContentFile { get; init; } = string.Empty; + public Guid? DocumentId { get; init; } public string? ContentType { get; init; } public Dictionary? Metadata { get; init; } public Dictionary? Headers { get; init; } @@ -369,13 +329,17 @@ internal sealed record DocumentSeed public bool AddToPendingDocuments { get; init; } = true; public bool AddToPendingMappings { get; init; } public string? LastModified { get; init; } + public string? FetchedAt { get; init; } public string? Etag { get; init; } public DateTimeOffset? ExpiresAt { get; init; } - public IReadOnlyCollection? KnownIdentifiers { get; init; } + public List? KnownIdentifiers { get; init; } } internal sealed record CursorSeed { + public List? PendingDocuments { get; init; } + public List? PendingMappings { get; init; } + public List? KnownAdvisories { get; init; } public DateTimeOffset? LastModifiedCursor { get; init; } public DateTimeOffset? LastFetchAt { get; init; } public Dictionary? Additional { get; init; } diff --git a/tools/SourceStateSeeder/SourceStateSeeder.csproj b/tools/SourceStateSeeder/SourceStateSeeder.csproj index 0a1cbb03..d80cd1a0 100644 --- a/tools/SourceStateSeeder/SourceStateSeeder.csproj +++ b/tools/SourceStateSeeder/SourceStateSeeder.csproj @@ -6,7 +6,7 @@ enable - - + + diff --git a/tools/certbund_offline_snapshot.py b/tools/certbund_offline_snapshot.py index 8d7aafc4..ba33adb6 100644 --- a/tools/certbund_offline_snapshot.py +++ b/tools/certbund_offline_snapshot.py @@ -67,7 +67,7 @@ class CertBundClient: raise RuntimeError( "CERT-Bund XSRF token not available. Provide --xsrf-token or a cookie file " - "containing XSRF-TOKEN (see docs/ops/feedser-certbund-operations.md)." + "containing XSRF-TOKEN (see docs/ops/concelier-certbund-operations.md)." ) def fetch_search_pages( @@ -281,7 +281,7 @@ def _build_search_record(path: Path) -> Dict[str, Any]: return { "type": "search", "path": path, - "source": "feedser.cert-bund.search", + "source": "concelier.cert-bund.search", "itemCount": len(content), "from": range_from, "to": range_to, @@ -301,7 +301,7 @@ def _build_export_record(path: Path) -> Dict[str, Any]: return { "type": "export", "path": path, - "source": "feedser.cert-bund.export", + "source": "concelier.cert-bund.export", "itemCount": None, "from": from_value, "to": to_value, @@ -358,7 +358,7 @@ def build_manifest(root: Path, records: Iterable[Dict[str, Any]], manifest_path: manifest_path.parent.mkdir(parents=True, exist_ok=True) manifest_document = { - "source": "feedser.cert-bund", + "source": "concelier.cert-bund", "generatedAt": dt.datetime.now(tz=UTC).isoformat(), "artifacts": manifest_entries, }